Merge remote-tracking branch 'origin/main' into sb/rpc-analysis

1 files changed, 197 insertions, 117 deletions

diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp
index 3270855ad..2c063d646 100644
--- a/src/zenhttp/httpserver.cpp
+++ b/src/zenhttp/httpserver.cpp
@@ -18,19 +18,22 @@
 #include <zencore/compactbinary.h>
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/compactbinarypackage.h>
+#include <zencore/compactbinaryutil.h>
 #include <zencore/iobuffer.h>
 #include <zencore/logging.h>
 #include <zencore/stream.h>
 #include <zencore/string.h>
 #include <zencore/testing.h>
 #include <zencore/thread.h>
-#include <zenutil/packageformat.h>
+#include <zenhttp/packageformat.h>
 
 #include <charconv>
 #include <mutex>
 #include <span>
 #include <string_view>
 
+#include <EASTL/fixed_vector.h>
+
 namespace zen {
 
 using namespace std::literals;
@@ -94,6 +97,7 @@ MapContentTypeToString(HttpContentType ContentType)
 static constinit uint32_t HashBinary					= HashStringDjb2("application/octet-stream"sv);
 static constinit uint32_t HashJson						= HashStringDjb2("json"sv);
 static constinit uint32_t HashApplicationJson			= HashStringDjb2("application/json"sv);
+static constinit uint32_t HashApplicationProblemJson	= HashStringDjb2("application/problem+json"sv);
 static constinit uint32_t HashYaml						= HashStringDjb2("yaml"sv);
 static constinit uint32_t HashTextYaml					= HashStringDjb2("text/yaml"sv);
 static constinit uint32_t HashText						= HashStringDjb2("text/plain"sv);
@@ -132,6 +136,7 @@ struct HashedTypeEntry
 	{HashCompactBinaryPackageOffer, HttpContentType::kCbPackageOffer},
 	{HashJson,						HttpContentType::kJSON},
 	{HashApplicationJson,			HttpContentType::kJSON},
+	{HashApplicationProblemJson,	HttpContentType::kJSON},
 	{HashYaml,						HttpContentType::kYAML},
 	{HashTextYaml,					HttpContentType::kYAML},
 	{HashText,						HttpContentType::kText},
@@ -156,7 +161,14 @@ ParseContentTypeImpl(const std::string_view& ContentTypeString)
 {
 	if (!ContentTypeString.empty())
 	{
-		const uint32_t CtHash = HashStringDjb2(ContentTypeString);
+		size_t ContentEnd = ContentTypeString.find(';');
+		if (ContentEnd == std::string_view::npos)
+		{
+			ContentEnd = ContentTypeString.length();
+		}
+		std::string_view ContentString(ContentTypeString.substr(0, ContentEnd));
+
+		const uint32_t CtHash = HashStringDjb2(ContentString);
 
 		if (auto It = std::lower_bound(std::begin(TypeHashTable),
 									   std::end(TypeHashTable),
@@ -468,6 +480,11 @@ HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, CbObject Data)
 		ExtendableStringBuilder<1024> Sb;
 		WriteResponse(ResponseCode, HttpContentType::kJSON, Data.ToJson(Sb).ToView());
 	}
+	else if (m_AcceptType == HttpContentType::kYAML)
+	{
+		ExtendableStringBuilder<1024> Sb;
+		WriteResponse(ResponseCode, HttpContentType::kYAML, Data.ToYaml(Sb).ToView());
+	}
 	else
 	{
 		SharedBuffer			Buf = Data.GetBuffer();
@@ -484,6 +501,11 @@ HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, CbArray Array)
 		ExtendableStringBuilder<1024> Sb;
 		WriteResponse(ResponseCode, HttpContentType::kJSON, Array.ToJson(Sb).ToView());
 	}
+	else if (m_AcceptType == HttpContentType::kYAML)
+	{
+		ExtendableStringBuilder<1024> Sb;
+		WriteResponse(ResponseCode, HttpContentType::kYAML, Array.ToYaml(Sb).ToView());
+	}
 	else
 	{
 		SharedBuffer			Buf = Array.GetBuffer();
@@ -510,7 +532,7 @@ HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType
 {
 	std::span<const SharedBuffer> Segments = Payload.GetSegments();
 
-	std::vector<IoBuffer> Buffers;
+	eastl::fixed_vector<IoBuffer, 64> Buffers;
 	Buffers.reserve(Segments.size());
 
 	for (auto& Segment : Segments)
@@ -518,7 +540,41 @@ HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType
 		Buffers.push_back(Segment.AsIoBuffer());
 	}
 
-	WriteResponse(ResponseCode, ContentType, Buffers);
+	WriteResponse(ResponseCode, ContentType, std::span<IoBuffer>(begin(Buffers), end(Buffers)));
+}
+
+std::string
+HttpServerRequest::Decode(std::string_view PercentEncodedString)
+{
+	size_t		Length = PercentEncodedString.length();
+	std::string Decoded;
+	Decoded.reserve(Length);
+	size_t Offset = 0;
+	while (Offset < Length)
+	{
+		char C = PercentEncodedString[Offset];
+		if (C == '%' && (Offset <= (Length - 3)))
+		{
+			std::string_view CharHash(&PercentEncodedString[Offset + 1], 2);
+			uint8_t			 DecodedChar = 0;
+			if (ParseHexBytes(CharHash, &DecodedChar))
+			{
+				Decoded.push_back((char)DecodedChar);
+				Offset += 3;
+			}
+			else
+			{
+				Decoded.push_back(C);
+				Offset++;
+			}
+		}
+		else
+		{
+			Decoded.push_back(C);
+			Offset++;
+		}
+	}
+	return Decoded;
 }
 
 HttpServerRequest::QueryParams
@@ -610,9 +666,13 @@ HttpServerRequest::ReadPayloadObject()
 			}
 			return CbObject();
 		}
-		return LoadCompactBinaryObject(std::move(Payload));
+		CbValidateError ValidationError = CbValidateError::None;
+		if (CbObject ResponseObject = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidationError);
+			ValidationError == CbValidateError::None)
+		{
+			return ResponseObject;
+		}
 	}
-
 	return {};
 }
 
@@ -732,120 +792,131 @@ HttpRpcHandler::AddRpc(std::string_view RpcId, std::function<void(CbObject& RpcA
 
 //////////////////////////////////////////////////////////////////////////
 
-enum class HttpServerClass
-{
-	kHttpAsio,
-	kHttpSys,
-	kHttpPlugin,
-	kHttpMulti,
-	kHttpNull
-};
-
 Ref<HttpServer>
-CreateHttpServerClass(HttpServerClass Class, const HttpServerConfig& Config)
+CreateHttpServerClass(const std::string_view ServerClass, const HttpServerConfig& Config)
 {
-	switch (Class)
+	if (ServerClass == "asio"sv)
 	{
-		default:
-		case HttpServerClass::kHttpAsio:
-			ZEN_INFO("using asio HTTP server implementation");
-			return CreateHttpAsioServer(Config.ForceLoopback, Config.ThreadCount);
-
-		case HttpServerClass::kHttpMulti:
-			{
-				ZEN_INFO("using multi HTTP server implementation");
-				Ref<HttpMultiServer> Server{new HttpMultiServer()};
-
-				// This is hardcoded for now, but should be configurable in the future
-				Server->AddServer(CreateHttpServerClass(HttpServerClass::kHttpSys, Config));
-				Server->AddServer(CreateHttpServerClass(HttpServerClass::kHttpPlugin, Config));
+		ZEN_INFO("using asio HTTP server implementation")
+		return CreateHttpAsioServer(Config.ForceLoopback, Config.ThreadCount);
+	}
+#if ZEN_WITH_HTTPSYS
+	else if (ServerClass == "httpsys"sv)
+	{
+		ZEN_INFO("using http.sys server implementation")
+		return Ref<HttpServer>(CreateHttpSysServer({.ThreadCount			 = Config.ThreadCount,
+													.AsyncWorkThreadCount	 = Config.HttpSys.AsyncWorkThreadCount,
+													.IsAsyncResponseEnabled	 = Config.HttpSys.IsAsyncResponseEnabled,
+													.IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled,
+													.IsDedicatedServer		 = Config.IsDedicatedServer,
+													.ForceLoopback			 = Config.ForceLoopback}));
+	}
+#endif
+	else if (ServerClass == "null"sv)
+	{
+		ZEN_INFO("using null HTTP server implementation")
+		return Ref<HttpServer>(new HttpNullServer);
+	}
+	else
+	{
+		ZEN_WARN("unknown HTTP server implementation '{}', falling back to default", ServerClass)
 
-				return Server;
-			}
+#if ZEN_WITH_HTTPSYS
+		return CreateHttpServerClass("httpsys"sv, Config);
+#else
+		return CreateHttpServerClass("asio"sv, Config);
+#endif
+	}
+}
 
 #if ZEN_WITH_PLUGINS
-		case HttpServerClass::kHttpPlugin:
-			{
-				ZEN_INFO("using plugin HTTP server implementation");
-				Ref<HttpPluginServer> Server{CreateHttpPluginServer()};
+Ref<HttpServer>
+CreateHttpServerPlugin(const HttpServerPluginConfig& PluginConfig)
+{
+	const std::string& PluginName = PluginConfig.PluginName;
 
-				// This is hardcoded for now, but should be configurable in the future
+	ZEN_INFO("using '{}' plugin HTTP server implementation", PluginName)
 
+	if (PluginName.starts_with("builtin:"sv))
+	{
 #	if 0
-				Ref<TransportPlugin> WinsockPlugin{CreateSocketTransportPlugin()};
-				WinsockPlugin->Configure("port", "8558");
-				Server->AddPlugin(WinsockPlugin);
-#	endif
+		Ref<TransportPlugin> Plugin = {};
+		if (PluginName == "builtin:winsock"sv)
+		{
+			Plugin = CreateSocketTransportPlugin();
+		}
+		else if (PluginName == "builtin:asio"sv)
+		{
+			Plugin = CreateAsioTransportPlugin();
+		}
+		else
+		{
+			ZEN_WARN("Unknown builtin plugin '{}'", PluginName)
+			return {};
+		}
 
-#	if 0
-				Ref<TransportPlugin> AsioPlugin{CreateAsioTransportPlugin()};
-				AsioPlugin->Configure("port", "8558");
-				Server->AddPlugin(AsioPlugin);
-#	endif
+		ZEN_ASSERT(!Plugin.IsNull());
 
-#	if 1
-				Ref<DllTransportPlugin> DllPlugin{CreateDllTransportPlugin()};
-				DllPlugin->LoadDll("winsock");
-				DllPlugin->ConfigureDll("winsock", "port", "8558");
-				Server->AddPlugin(DllPlugin);
-#	endif
+		for (const std::pair<std::string, std::string>& Option : PluginConfig.PluginOptions)
+		{
+			Plugin->Configure(Option.first.c_str(), Option.second.c_str());
+		}
 
-				return Server;
-			}
-#endif
+		Ref<HttpPluginServer> Server{CreateHttpPluginServer()};
+		Server->AddPlugin(Plugin);
+		return Server;
+#	else
+		ZEN_WARN("Builtin plugin '{}' is not supported", PluginName)
+		return {};
+#	endif
+	}
 
-#if ZEN_WITH_HTTPSYS
-		case HttpServerClass::kHttpSys:
-			ZEN_INFO("using http.sys server implementation");
-			return Ref<HttpServer>(CreateHttpSysServer({.ThreadCount			 = Config.ThreadCount,
-														.AsyncWorkThreadCount	 = Config.HttpSys.AsyncWorkThreadCount,
-														.IsAsyncResponseEnabled	 = Config.HttpSys.IsAsyncResponseEnabled,
-														.IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled,
-														.IsDedicatedServer		 = Config.IsDedicatedServer,
-														.ForceLoopback			 = Config.ForceLoopback}));
-#endif
+	Ref<DllTransportPlugin> DllPlugin{CreateDllTransportPlugin()};
+	if (!DllPlugin->LoadDll(PluginName))
+	{
+		return {};
+	}
 
-		case HttpServerClass::kHttpNull:
-			ZEN_INFO("using null HTTP server implementation");
-			return Ref<HttpServer>(new HttpNullServer);
+	for (const std::pair<std::string, std::string>& Option : PluginConfig.PluginOptions)
+	{
+		DllPlugin->ConfigureDll(PluginName, Option.first.c_str(), Option.second.c_str());
 	}
+
+	Ref<HttpPluginServer> Server{CreateHttpPluginServer()};
+	Server->AddPlugin(DllPlugin);
+	return Server;
 }
+#endif
 
 Ref<HttpServer>
 CreateHttpServer(const HttpServerConfig& Config)
 {
 	using namespace std::literals;
 
-	HttpServerClass Class = HttpServerClass::kHttpNull;
-
-#if ZEN_WITH_HTTPSYS
-	Class = HttpServerClass::kHttpSys;
-#else
-	Class = HttpServerClass::kHttpAsio;
-#endif
-
-	if (Config.ServerClass == "asio"sv)
-	{
-		Class = HttpServerClass::kHttpAsio;
-	}
-	else if (Config.ServerClass == "httpsys"sv)
-	{
-		Class = HttpServerClass::kHttpSys;
-	}
-	else if (Config.ServerClass == "plugin"sv)
-	{
-		Class = HttpServerClass::kHttpPlugin;
-	}
-	else if (Config.ServerClass == "null"sv)
+#if ZEN_WITH_PLUGINS
+	if (Config.PluginConfigs.empty())
 	{
-		Class = HttpServerClass::kHttpNull;
+		return CreateHttpServerClass(Config.ServerClass, Config);
 	}
-	else if (Config.ServerClass == "multi"sv)
+	else
 	{
-		Class = HttpServerClass::kHttpMulti;
-	}
+		Ref<HttpMultiServer> Server{new HttpMultiServer()};
+		Server->AddServer(CreateHttpServerClass(Config.ServerClass, Config));
 
-	return CreateHttpServerClass(Class, Config);
+		for (const HttpServerPluginConfig& PluginConfig : Config.PluginConfigs)
+		{
+			Ref<HttpServer> PluginServer = CreateHttpServerPlugin(PluginConfig);
+			if (!PluginServer.IsNull())
+			{
+				Server->AddServer(PluginServer);
+			}
+		}
+
+		return Server;
+	}
+#else
+	return CreateHttpServerClass(Config.ServerClass, Config);
+#endif
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -865,42 +936,51 @@ HandlePackageOffers(HttpService& Service, HttpServerRequest& Request, Ref<IHttpP
 
 			if (PackageHandlerRef)
 			{
-				CbObject OfferMessage = LoadCompactBinaryObject(Request.ReadPayload());
-
-				std::vector<IoHash> OfferCids;
-
-				for (auto& CidEntry : OfferMessage["offer"])
+				CbValidateError ValidationError = CbValidateError::None;
+				if (CbObject OfferMessage = ValidateAndReadCompactBinaryObject(IoBuffer(Request.ReadPayload()), ValidationError);
+					ValidationError == CbValidateError::None)
 				{
-					if (!CidEntry.IsHash())
+					std::vector<IoHash> OfferCids;
+
+					for (auto& CidEntry : OfferMessage["offer"])
 					{
-						// Should yield bad request response?
+						if (!CidEntry.IsHash())
+						{
+							// Should yield bad request response?
+
+							ZEN_WARN("found invalid entry in offer");
 
-						ZEN_WARN("found invalid entry in offer");
+							continue;
+						}
 
-						continue;
+						OfferCids.push_back(CidEntry.AsHash());
 					}
 
-					OfferCids.push_back(CidEntry.AsHash());
-				}
+					ZEN_TRACE("request #{} -> filtering offer of {} entries", Request.RequestId(), OfferCids.size());
 
-				ZEN_TRACE("request #{} -> filtering offer of {} entries", Request.RequestId(), OfferCids.size());
+					PackageHandlerRef->FilterOffer(OfferCids);
 
-				PackageHandlerRef->FilterOffer(OfferCids);
+					ZEN_TRACE("request #{} -> filtered to {} entries", Request.RequestId(), OfferCids.size());
 
-				ZEN_TRACE("request #{} -> filtered to {} entries", Request.RequestId(), OfferCids.size());
+					CbObjectWriter ResponseWriter;
+					ResponseWriter.BeginArray("need");
 
-				CbObjectWriter ResponseWriter;
-				ResponseWriter.BeginArray("need");
+					for (const IoHash& Cid : OfferCids)
+					{
+						ResponseWriter.AddHash(Cid);
+					}
+
+					ResponseWriter.EndArray();
 
-				for (const IoHash& Cid : OfferCids)
+					// Emit filter response
+					Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save());
+				}
+				else
 				{
-					ResponseWriter.AddHash(Cid);
+					Request.WriteResponse(HttpResponseCode::BadRequest,
+										  HttpContentType::kText,
+										  fmt::format("Invalid request payload: '{}'", ToString(ValidationError)));
 				}
-
-				ResponseWriter.EndArray();
-
-				// Emit filter response
-				Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save());
 				return true;
 			}
 		}