From d0630968516f31a9c6ebc9a5643b7cd19f6cb184 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 22 Jan 2024 13:22:21 +0100 Subject: improved errors from jupiter upstream (#636) * get more detailed error messages from jupiter upstream --- src/zenhttp/httpserver.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'src/zenhttp/httpserver.cpp') diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 3270855ad..dcbeac907 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -94,6 +94,7 @@ MapContentTypeToString(HttpContentType ContentType) static constinit uint32_t HashBinary = HashStringDjb2("application/octet-stream"sv); static constinit uint32_t HashJson = HashStringDjb2("json"sv); static constinit uint32_t HashApplicationJson = HashStringDjb2("application/json"sv); +static constinit uint32_t HashApplicationProblemJson = HashStringDjb2("application/problem+json"sv); static constinit uint32_t HashYaml = HashStringDjb2("yaml"sv); static constinit uint32_t HashTextYaml = HashStringDjb2("text/yaml"sv); static constinit uint32_t HashText = HashStringDjb2("text/plain"sv); @@ -132,6 +133,7 @@ struct HashedTypeEntry {HashCompactBinaryPackageOffer, HttpContentType::kCbPackageOffer}, {HashJson, HttpContentType::kJSON}, {HashApplicationJson, HttpContentType::kJSON}, + {HashApplicationProblemJson, HttpContentType::kJSON}, {HashYaml, HttpContentType::kYAML}, {HashTextYaml, HttpContentType::kYAML}, {HashText, HttpContentType::kText}, @@ -156,7 +158,14 @@ ParseContentTypeImpl(const std::string_view& ContentTypeString) { if (!ContentTypeString.empty()) { - const uint32_t CtHash = HashStringDjb2(ContentTypeString); + size_t ContentEnd = ContentTypeString.find(';'); + if (ContentEnd == std::string_view::npos) + { + ContentEnd = ContentTypeString.length(); + } + std::string_view ContentString(ContentTypeString.substr(0, ContentEnd)); + + const uint32_t CtHash = HashStringDjb2(ContentString); if (auto It = std::lower_bound(std::begin(TypeHashTable), std::end(TypeHashTable), -- cgit v1.2.3 From 66cf5342b2fe8bd3f2961dc70b9c302bf4de12bf Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 26 Mar 2024 13:28:07 +0100 Subject: add filter to projectstore entries request (#25) * Add HttpServerRequest::Decode to decode http request parameters * Add support to filter projectstore `entries` request using the `fieldfilter` where the wanted fields are comma (,) delimited * Add support for responding with compressed payloads for projectstore `entries` requests by adding AcceptType `compressed-binary` to the request header * Add support for responding with compressed payloads for projectstore `files` requests by adding AcceptType `compressed-binary` to the request header * Add support for responding with compressed payloads for projectstore `chunkinfo` requests by adding AcceptType `compressed-binary` to the request header --- src/zenhttp/httpserver.cpp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'src/zenhttp/httpserver.cpp') diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index dcbeac907..adb3128bf 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -530,6 +530,40 @@ HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType WriteResponse(ResponseCode, ContentType, Buffers); } +std::string +HttpServerRequest::Decode(std::string_view PercentEncodedString) +{ + size_t Length = PercentEncodedString.length(); + std::string Decoded; + Decoded.reserve(Length); + size_t Offset = 0; + while (Offset < Length) + { + char C = PercentEncodedString[Offset]; + if (C == '%' && (Offset <= (Length - 3))) + { + std::string_view CharHash(&PercentEncodedString[Offset + 1], 2); + uint8_t DecodedChar = 0; + if (ParseHexBytes(CharHash, &DecodedChar)) + { + Decoded.push_back((char)DecodedChar); + Offset += 3; + } + else + { + Decoded.push_back(C); + Offset++; + } + } + else + { + Decoded.push_back(C); + Offset++; + } + } + return Decoded; +} + HttpServerRequest::QueryParams HttpServerRequest::GetQueryParams() { -- cgit v1.2.3 From 59e51bf811a5907983b84f3e2ed14e47a7210e75 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 26 Mar 2024 13:31:10 +0100 Subject: add yaml serialization support (#3) this change adds serialization of payloads as YAML, but not parsing. The implementation is somewhat based on the JSON path, and may be collapsed eventually as it is possible to serialize JSON format using the same code it also separates out the JSON serialization into a separate file for ease of maintenance any HTTP request response may be formatted as yaml by using a `.yaml` suffix or an `Accept: text/yaml` header --- src/zenhttp/httpserver.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src/zenhttp/httpserver.cpp') diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index adb3128bf..a0d4ef3f3 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -477,6 +477,11 @@ HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, CbObject Data) ExtendableStringBuilder<1024> Sb; WriteResponse(ResponseCode, HttpContentType::kJSON, Data.ToJson(Sb).ToView()); } + else if (m_AcceptType == HttpContentType::kYAML) + { + ExtendableStringBuilder<1024> Sb; + WriteResponse(ResponseCode, HttpContentType::kYAML, Data.ToYaml(Sb).ToView()); + } else { SharedBuffer Buf = Data.GetBuffer(); @@ -493,6 +498,11 @@ HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, CbArray Array) ExtendableStringBuilder<1024> Sb; WriteResponse(ResponseCode, HttpContentType::kJSON, Array.ToJson(Sb).ToView()); } + else if (m_AcceptType == HttpContentType::kYAML) + { + ExtendableStringBuilder<1024> Sb; + WriteResponse(ResponseCode, HttpContentType::kYAML, Array.ToYaml(Sb).ToView()); + } else { SharedBuffer Buf = Array.GetBuffer(); -- cgit v1.2.3 From a5158f9fc806d506590dd9bf0e3282cb76c3ac4e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 16 Jan 2025 09:19:08 +0100 Subject: move basicfile.h/cpp -> zencore (#273) move jupiter.h/cpp -> zenutil move packageformat.h/.cpp -> zenhttp zenutil now depends on zenhttp instead of the inverse --- src/zenhttp/httpserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/zenhttp/httpserver.cpp') diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index a0d4ef3f3..1fbe22628 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include -- cgit v1.2.3 From 920120bbcec9f91df3336f62970b3e010a4fa6c2 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 6 Mar 2025 16:18:32 +0100 Subject: reduced memory churn using fixed_xxx containers (#236) * Added EASTL to help with eliminating memory allocations * Applied EASTL to eliminate memory allocations, primarily by using `fixed_vector` et al to use stack allocations / inline struct allocations Reduces memory events in traces by close to a factor of 10 in test scenario (starting editor for project F) --- src/zenhttp/httpserver.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/zenhttp/httpserver.cpp') diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 1fbe22628..27a09f339 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -31,6 +31,8 @@ #include #include +#include + namespace zen { using namespace std::literals; @@ -529,7 +531,7 @@ HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType { std::span Segments = Payload.GetSegments(); - std::vector Buffers; + eastl::fixed_vector Buffers; Buffers.reserve(Segments.size()); for (auto& Segment : Segments) @@ -537,7 +539,7 @@ HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType Buffers.push_back(Segment.AsIoBuffer()); } - WriteResponse(ResponseCode, ContentType, Buffers); + WriteResponse(ResponseCode, ContentType, std::span(begin(Buffers), end(Buffers))); } std::string -- cgit v1.2.3 From ea2918767afa3c00c8dfa7dd7d9960a80a460eb9 Mon Sep 17 00:00:00 2001 From: Dmytro Ivanov Date: Wed, 9 Apr 2025 15:35:37 +0200 Subject: Added config, versioning and logging for plugins --- src/zenhttp/httpserver.cpp | 177 ++++++++++++++++++++++----------------------- 1 file changed, 86 insertions(+), 91 deletions(-) (limited to 'src/zenhttp/httpserver.cpp') diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 27a09f339..d12f89a92 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -787,120 +787,115 @@ HttpRpcHandler::AddRpc(std::string_view RpcId, std::function -CreateHttpServerClass(HttpServerClass Class, const HttpServerConfig& Config) +CreateHttpServerClass(const std::string_view ServerClass, const HttpServerConfig& Config) { - switch (Class) + if (ServerClass == "asio"sv) { - default: - case HttpServerClass::kHttpAsio: - ZEN_INFO("using asio HTTP server implementation"); - return CreateHttpAsioServer(Config.ForceLoopback, Config.ThreadCount); - - case HttpServerClass::kHttpMulti: - { - ZEN_INFO("using multi HTTP server implementation"); - Ref Server{new HttpMultiServer()}; - - // This is hardcoded for now, but should be configurable in the future - Server->AddServer(CreateHttpServerClass(HttpServerClass::kHttpSys, Config)); - Server->AddServer(CreateHttpServerClass(HttpServerClass::kHttpPlugin, Config)); + ZEN_INFO("using asio HTTP server implementation") + return CreateHttpAsioServer(Config.ForceLoopback, Config.ThreadCount); + } +#if ZEN_WITH_HTTPSYS + else if (ServerClass == "httpsys"sv) + { + ZEN_INFO("using http.sys server implementation") + return Ref(CreateHttpSysServer({.ThreadCount = Config.ThreadCount, + .AsyncWorkThreadCount = Config.HttpSys.AsyncWorkThreadCount, + .IsAsyncResponseEnabled = Config.HttpSys.IsAsyncResponseEnabled, + .IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled, + .IsDedicatedServer = Config.IsDedicatedServer, + .ForceLoopback = Config.ForceLoopback})); + } +#endif + else if (ServerClass == "null"sv) + { + ZEN_INFO("using null HTTP server implementation") + return Ref(new HttpNullServer); + } + else + { + ZEN_WARN("unknown HTTP server implementation '{}', falling back to default", ServerClass) - return Server; - } +#if ZEN_WITH_HTTPSYS + return CreateHttpServerClass("httpsys"sv, Config); +#else + return CreateHttpServerClass("asio"sv, Config); +#endif + } +} #if ZEN_WITH_PLUGINS - case HttpServerClass::kHttpPlugin: - { - ZEN_INFO("using plugin HTTP server implementation"); - Ref Server{CreateHttpPluginServer()}; - - // This is hardcoded for now, but should be configurable in the future +Ref +CreateHttpServerPlugin(const HttpServerPluginConfig& PluginConfig) +{ + const std::string& PluginName = PluginConfig.PluginName; -# if 0 - Ref WinsockPlugin{CreateSocketTransportPlugin()}; - WinsockPlugin->Configure("port", "8558"); - Server->AddPlugin(WinsockPlugin); -# endif + ZEN_INFO("using '{}' plugin HTTP server implementation", PluginName) + Ref Server{CreateHttpPluginServer()}; + if (PluginName.starts_with("builtin:"sv)) + { # if 0 - Ref AsioPlugin{CreateAsioTransportPlugin()}; - AsioPlugin->Configure("port", "8558"); - Server->AddPlugin(AsioPlugin); -# endif - -# if 1 - Ref DllPlugin{CreateDllTransportPlugin()}; - DllPlugin->LoadDll("winsock"); - DllPlugin->ConfigureDll("winsock", "port", "8558"); - Server->AddPlugin(DllPlugin); -# endif + Ref Plugin = {}; + if (PluginName == "builtin:winsock"sv) + { + Plugin = CreateSocketTransportPlugin(); + } + else if (PluginName == "builtin:asio"sv) + { + Plugin = CreateAsioTransportPlugin(); + } - return Server; + if (!Plugin.IsNull()) + { + for (const std::pair& Option : PluginConfig.PluginOptions) + { + Plugin->Configure(Option.first.c_str(), Option.second.c_str()); } -#endif - -#if ZEN_WITH_HTTPSYS - case HttpServerClass::kHttpSys: - ZEN_INFO("using http.sys server implementation"); - return Ref(CreateHttpSysServer({.ThreadCount = Config.ThreadCount, - .AsyncWorkThreadCount = Config.HttpSys.AsyncWorkThreadCount, - .IsAsyncResponseEnabled = Config.HttpSys.IsAsyncResponseEnabled, - .IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled, - .IsDedicatedServer = Config.IsDedicatedServer, - .ForceLoopback = Config.ForceLoopback})); -#endif - - case HttpServerClass::kHttpNull: - ZEN_INFO("using null HTTP server implementation"); - return Ref(new HttpNullServer); + Server->AddPlugin(Plugin); + } +# endif } + else + { + Ref DllPlugin{CreateDllTransportPlugin()}; + DllPlugin->LoadDll(PluginName); + for (const std::pair& Option : PluginConfig.PluginOptions) + { + DllPlugin->ConfigureDll(PluginName, Option.first.c_str(), Option.second.c_str()); + } + Server->AddPlugin(DllPlugin); + } + + return Server; } +#endif Ref CreateHttpServer(const HttpServerConfig& Config) { using namespace std::literals; - HttpServerClass Class = HttpServerClass::kHttpNull; - -#if ZEN_WITH_HTTPSYS - Class = HttpServerClass::kHttpSys; -#else - Class = HttpServerClass::kHttpAsio; -#endif - - if (Config.ServerClass == "asio"sv) - { - Class = HttpServerClass::kHttpAsio; - } - else if (Config.ServerClass == "httpsys"sv) - { - Class = HttpServerClass::kHttpSys; - } - else if (Config.ServerClass == "plugin"sv) - { - Class = HttpServerClass::kHttpPlugin; - } - else if (Config.ServerClass == "null"sv) +#if ZEN_WITH_PLUGINS + if (Config.PluginConfigs.empty()) { - Class = HttpServerClass::kHttpNull; + return CreateHttpServerClass(Config.ServerClass, Config); } - else if (Config.ServerClass == "multi"sv) + else { - Class = HttpServerClass::kHttpMulti; - } + Ref Server{new HttpMultiServer()}; + Server->AddServer(CreateHttpServerClass(Config.ServerClass, Config)); - return CreateHttpServerClass(Class, Config); + for (const HttpServerPluginConfig& PluginConfig : Config.PluginConfigs) + { + Server->AddServer(CreateHttpServerPlugin(PluginConfig)); + } + + return Server; + } +#else + return CreateHttpServerClass(Config.ServerClass, Config); +#endif } ////////////////////////////////////////////////////////////////////////// -- cgit v1.2.3 From 199553c86444dcd9c4cff91165938dbd9a5b4bbb Mon Sep 17 00:00:00 2001 From: Dmytro Ivanov Date: Wed, 23 Apr 2025 10:54:22 +0200 Subject: Make plugin loading errors non fatal (#364) make plugin loading errors non fatal --- src/zenhttp/httpserver.cpp | 48 ++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 16 deletions(-) (limited to 'src/zenhttp/httpserver.cpp') diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index d12f89a92..764f2a2a7 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -831,7 +831,6 @@ CreateHttpServerPlugin(const HttpServerPluginConfig& PluginConfig) const std::string& PluginName = PluginConfig.PluginName; ZEN_INFO("using '{}' plugin HTTP server implementation", PluginName) - Ref Server{CreateHttpPluginServer()}; if (PluginName.starts_with("builtin:"sv)) { @@ -845,28 +844,41 @@ CreateHttpServerPlugin(const HttpServerPluginConfig& PluginConfig) { Plugin = CreateAsioTransportPlugin(); } + else + { + ZEN_WARN("Unknown builtin plugin '{}'", PluginName) + return {}; + } + + ZEN_ASSERT(!Plugin.IsNull()); - if (!Plugin.IsNull()) + for (const std::pair& Option : PluginConfig.PluginOptions) { - for (const std::pair& Option : PluginConfig.PluginOptions) - { - Plugin->Configure(Option.first.c_str(), Option.second.c_str()); - } - Server->AddPlugin(Plugin); + Plugin->Configure(Option.first.c_str(), Option.second.c_str()); } + + Ref Server{CreateHttpPluginServer()}; + Server->AddPlugin(Plugin); + return Server; +# else + ZEN_WARN("Builtin plugin '{}' is not supported", PluginName) + return {}; # endif } - else + + Ref DllPlugin{CreateDllTransportPlugin()}; + if (!DllPlugin->LoadDll(PluginName)) { - Ref DllPlugin{CreateDllTransportPlugin()}; - DllPlugin->LoadDll(PluginName); - for (const std::pair& Option : PluginConfig.PluginOptions) - { - DllPlugin->ConfigureDll(PluginName, Option.first.c_str(), Option.second.c_str()); - } - Server->AddPlugin(DllPlugin); + return {}; + } + + for (const std::pair& Option : PluginConfig.PluginOptions) + { + DllPlugin->ConfigureDll(PluginName, Option.first.c_str(), Option.second.c_str()); } + Ref Server{CreateHttpPluginServer()}; + Server->AddPlugin(DllPlugin); return Server; } #endif @@ -888,7 +900,11 @@ CreateHttpServer(const HttpServerConfig& Config) for (const HttpServerPluginConfig& PluginConfig : Config.PluginConfigs) { - Server->AddServer(CreateHttpServerPlugin(PluginConfig)); + Ref PluginServer = CreateHttpServerPlugin(PluginConfig); + if (!PluginServer.IsNull()) + { + Server->AddServer(PluginServer); + } } return Server; -- cgit v1.2.3 From 9f575bd416e1f7afbd11d4b221074f34bb89605c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 4 Sep 2025 13:17:25 +0200 Subject: add validation of compact binary payloads before reading them (#483) * add validation of compact binary payloads before reading them --- src/zenhttp/httpserver.cpp | 64 ++++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 25 deletions(-) (limited to 'src/zenhttp/httpserver.cpp') diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 764f2a2a7..2c063d646 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -665,9 +666,13 @@ HttpServerRequest::ReadPayloadObject() } return CbObject(); } - return LoadCompactBinaryObject(std::move(Payload)); + CbValidateError ValidationError = CbValidateError::None; + if (CbObject ResponseObject = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidationError); + ValidationError == CbValidateError::None) + { + return ResponseObject; + } } - return {}; } @@ -931,42 +936,51 @@ HandlePackageOffers(HttpService& Service, HttpServerRequest& Request, Ref OfferCids; - - for (auto& CidEntry : OfferMessage["offer"]) + CbValidateError ValidationError = CbValidateError::None; + if (CbObject OfferMessage = ValidateAndReadCompactBinaryObject(IoBuffer(Request.ReadPayload()), ValidationError); + ValidationError == CbValidateError::None) { - if (!CidEntry.IsHash()) + std::vector OfferCids; + + for (auto& CidEntry : OfferMessage["offer"]) { - // Should yield bad request response? + if (!CidEntry.IsHash()) + { + // Should yield bad request response? + + ZEN_WARN("found invalid entry in offer"); - ZEN_WARN("found invalid entry in offer"); + continue; + } - continue; + OfferCids.push_back(CidEntry.AsHash()); } - OfferCids.push_back(CidEntry.AsHash()); - } + ZEN_TRACE("request #{} -> filtering offer of {} entries", Request.RequestId(), OfferCids.size()); + + PackageHandlerRef->FilterOffer(OfferCids); - ZEN_TRACE("request #{} -> filtering offer of {} entries", Request.RequestId(), OfferCids.size()); + ZEN_TRACE("request #{} -> filtered to {} entries", Request.RequestId(), OfferCids.size()); - PackageHandlerRef->FilterOffer(OfferCids); + CbObjectWriter ResponseWriter; + ResponseWriter.BeginArray("need"); - ZEN_TRACE("request #{} -> filtered to {} entries", Request.RequestId(), OfferCids.size()); + for (const IoHash& Cid : OfferCids) + { + ResponseWriter.AddHash(Cid); + } - CbObjectWriter ResponseWriter; - ResponseWriter.BeginArray("need"); + ResponseWriter.EndArray(); - for (const IoHash& Cid : OfferCids) + // Emit filter response + Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); + } + else { - ResponseWriter.AddHash(Cid); + Request.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid request payload: '{}'", ToString(ValidationError))); } - - ResponseWriter.EndArray(); - - // Emit filter response - Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); return true; } } -- cgit v1.2.3