moved source directories into `/src` (#264)

* moved source directories into `/src`
* updated bundle.lua for new `src` path
* moved some docs, icon
* removed old test trees

6 files changed, 983 insertions, 0 deletions

diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h
new file mode 100644
index 000000000..8316a9b9f
--- /dev/null
+++ b/src/zenhttp/include/zenhttp/httpclient.h
@@ -0,0 +1,47 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zenhttp.h"
+
+#include <zencore/iobuffer.h>
+#include <zencore/uid.h>
+#include <zenhttp/httpcommon.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <cpr/cpr.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+class CbPackage;
+
+/** HTTP client implementation for Zen use cases
+
+	Currently simple and synchronous, should become lean and asynchronous
+ */
+class HttpClient
+{
+public:
+	HttpClient(std::string_view BaseUri);
+	~HttpClient();
+
+	struct Response
+	{
+		int		 StatusCode = 0;
+		IoBuffer ResponsePayload;  // Note: this also includes the content type
+	};
+
+	[[nodiscard]] Response Put(std::string_view Url, IoBuffer Payload);
+	[[nodiscard]] Response Get(std::string_view Url);
+	[[nodiscard]] Response TransactPackage(std::string_view Url, CbPackage Package);
+	[[nodiscard]] Response Delete(std::string_view Url);
+
+private:
+	std::string m_BaseUri;
+	std::string m_SessionId;
+};
+
+}  // namespace zen
+
+void httpclient_forcelink();  // internal
diff --git a/src/zenhttp/include/zenhttp/httpcommon.h b/src/zenhttp/include/zenhttp/httpcommon.h
new file mode 100644
index 000000000..19fda8db4
--- /dev/null
+++ b/src/zenhttp/include/zenhttp/httpcommon.h
@@ -0,0 +1,181 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iobuffer.h>
+
+#include <string_view>
+
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+using HttpContentType = ZenContentType;
+
+class IoBuffer;
+class CbObject;
+class CbPackage;
+class StringBuilderBase;
+
+struct HttpRange
+{
+	uint32_t Start = ~uint32_t(0);
+	uint32_t End   = ~uint32_t(0);
+};
+
+using HttpRanges = std::vector<HttpRange>;
+
+std::string_view MapContentTypeToString(HttpContentType ContentType);
+extern HttpContentType (*ParseContentType)(const std::string_view& ContentTypeString);
+std::string_view ReasonStringForHttpResultCode(int HttpCode);
+bool			 TryParseHttpRangeHeader(std::string_view RangeHeader, HttpRanges& Ranges);
+
+[[nodiscard]] inline bool
+IsHttpSuccessCode(int HttpCode)
+{
+	return (HttpCode >= 200) && (HttpCode < 300);
+}
+
+enum class HttpVerb : uint8_t
+{
+	kGet	 = 1 << 0,
+	kPut	 = 1 << 1,
+	kPost	 = 1 << 2,
+	kDelete	 = 1 << 3,
+	kHead	 = 1 << 4,
+	kCopy	 = 1 << 5,
+	kOptions = 1 << 6
+};
+
+gsl_DEFINE_ENUM_BITMASK_OPERATORS(HttpVerb);
+
+const std::string_view ToString(HttpVerb Verb);
+
+enum class HttpResponseCode
+{
+	// 1xx - Informational
+
+	Continue = 100,	 //!< Indicates that the initial part of a request has been received and has not yet been rejected by the server.
+	SwitchingProtocols = 101,  //!< Indicates that the server understands and is willing to comply with the client's request, via the
+							   //!< Upgrade header field, for a change in the application protocol being used on this connection.
+	Processing = 102,  //!< Is an interim response used to inform the client that the server has accepted the complete request, but has not
+					   //!< yet completed it.
+	EarlyHints = 103,  //!< Indicates to the client that the server is likely to send a final response with the header fields included in
+					   //!< the informational response.
+
+	// 2xx - Successful
+
+	OK		 = 200,	 //!< Indicates that the request has succeeded.
+	Created	 = 201,	 //!< Indicates that the request has been fulfilled and has resulted in one or more new resources being created.
+	Accepted = 202,	 //!< Indicates that the request has been accepted for processing, but the processing has not been completed.
+	NonAuthoritativeInformation = 203,	//!< Indicates that the request was successful but the enclosed payload has been modified from that
+										//!< of the origin server's 200 (OK) response by a transforming proxy.
+	NoContent = 204,  //!< Indicates that the server has successfully fulfilled the request and that there is no additional content to send
+					  //!< in the response payload body.
+	ResetContent = 205,	   //!< Indicates that the server has fulfilled the request and desires that the user agent reset the \"document
+						   //!< view\", which caused the request to be sent, to its original state as received from the origin server.
+	PartialContent = 206,  //!< Indicates that the server is successfully fulfilling a range request for the target resource by transferring
+						   //!< one or more parts of the selected representation that correspond to the satisfiable ranges found in the
+						   //!< requests's Range header field.
+	MultiStatus		= 207,	//!< Provides status for multiple independent operations.
+	AlreadyReported = 208,	//!< Used inside a DAV:propstat response element to avoid enumerating the internal members of multiple bindings
+							//!< to the same collection repeatedly. [RFC 5842]
+	IMUsed = 226,  //!< The server has fulfilled a GET request for the resource, and the response is a representation of the result of one
+				   //!< or more instance-manipulations applied to the current instance.
+
+	// 3xx - Redirection
+
+	MultipleChoices = 300,	 //!< Indicates that the target resource has more than one representation, each with its own more specific
+							 //!< identifier, and information about the alternatives is being provided so that the user (or user agent) can
+							 //!< select a preferred representation by redirecting its request to one or more of those identifiers.
+	MovedPermanently = 301,	 //!< Indicates that the target resource has been assigned a new permanent URI and any future references to this
+							 //!< resource ought to use one of the enclosed URIs.
+	Found	 = 302,			 //!< Indicates that the target resource resides temporarily under a different URI.
+	SeeOther = 303,		//!< Indicates that the server is redirecting the user agent to a different resource, as indicated by a URI in the
+						//!< Location header field, that is intended to provide an indirect response to the original request.
+	NotModified = 304,	//!< Indicates that a conditional GET request has been received and would have resulted in a 200 (OK) response if it
+						//!< were not for the fact that the condition has evaluated to false.
+	UseProxy = 305,		//!< \deprecated \parblock Due to security concerns regarding in-band configuration of a proxy. \endparblock
+						//!< The requested resource MUST be accessed through the proxy given by the Location field.
+	TemporaryRedirect = 307,  //!< Indicates that the target resource resides temporarily under a different URI and the user agent MUST NOT
+							  //!< change the request method if it performs an automatic redirection to that URI.
+	PermanentRedirect = 308,  //!< The target resource has been assigned a new permanent URI and any future references to this resource
+							  //!< ought to use one of the enclosed URIs. [...] This status code is similar to 301 Moved Permanently
+							  //!< (Section 7.3.2 of rfc7231), except that it does not allow rewriting the request method from POST to GET.
+
+	// 4xx - Client Error
+	BadRequest = 400,		//!< Indicates that the server cannot or will not process the request because the received syntax is invalid,
+							//!< nonsensical, or exceeds some limitation on what the server is willing to process.
+	Unauthorized = 401,		//!< Indicates that the request has not been applied because it lacks valid authentication credentials for the
+							//!< target resource.
+	PaymentRequired = 402,	//!< *Reserved*
+	Forbidden		= 403,	//!< Indicates that the server understood the request but refuses to authorize it.
+	NotFound = 404,	 //!< Indicates that the origin server did not find a current representation for the target resource or is not willing
+					 //!< to disclose that one exists.
+	MethodNotAllowed = 405,	 //!< Indicates that the method specified in the request-line is known by the origin server but not supported by
+							 //!< the target resource.
+	NotAcceptable = 406,	 //!< Indicates that the target resource does not have a current representation that would be acceptable to the
+						  //!< user agent, according to the proactive negotiation header fields received in the request, and the server is
+						  //!< unwilling to supply a default representation.
+	ProxyAuthenticationRequired =
+		407,  //!< Is similar to 401 (Unauthorized), but indicates that the client needs to authenticate itself in order to use a proxy.
+	RequestTimeout =
+		408,		 //!< Indicates that the server did not receive a complete request message within the time that it was prepared to wait.
+	Conflict = 409,	 //!< Indicates that the request could not be completed due to a conflict with the current state of the resource.
+	Gone = 410,	 //!< Indicates that access to the target resource is no longer available at the origin server and that this condition is
+				 //!< likely to be permanent.
+	LengthRequired = 411,  //!< Indicates that the server refuses to accept the request without a defined Content-Length.
+	PreconditionFailed =
+		412,  //!< Indicates that one or more preconditions given in the request header fields evaluated to false when tested on the server.
+	PayloadTooLarge = 413,	//!< Indicates that the server is refusing to process a request because the request payload is larger than the
+							//!< server is willing or able to process.
+	URITooLong = 414,		//!< Indicates that the server is refusing to service the request because the request-target is longer than the
+							//!< server is willing to interpret.
+	UnsupportedMediaType = 415,	 //!< Indicates that the origin server is refusing to service the request because the payload is in a format
+								 //!< not supported by the target resource for this method.
+	RangeNotSatisfiable = 416,	//!< Indicates that none of the ranges in the request's Range header field overlap the current extent of the
+								//!< selected resource or that the set of ranges requested has been rejected due to invalid ranges or an
+								//!< excessive request of small or overlapping ranges.
+	ExpectationFailed = 417,	//!< Indicates that the expectation given in the request's Expect header field could not be met by at least
+								//!< one of the inbound servers.
+	ImATeapot			= 418,	//!< Any attempt to brew coffee with a teapot should result in the error code 418 I'm a teapot.
+	UnprocessableEntity = 422,	//!< Means the server understands the content type of the request entity (hence a 415(Unsupported Media
+								//!< Type) status code is inappropriate), and the syntax of the request entity is correct (thus a 400 (Bad
+								//!< Request) status code is inappropriate) but was unable to process the contained instructions.
+	Locked			 = 423,		//!< Means the source or destination resource of a method is locked.
+	FailedDependency = 424,		//!< Means that the method could not be performed on the resource because the requested action depended on
+								//!< another action and that action failed.
+	UpgradeRequired = 426,	//!< Indicates that the server refuses to perform the request using the current protocol but might be willing to
+							//!< do so after the client upgrades to a different protocol.
+	PreconditionRequired = 428,	 //!< Indicates that the origin server requires the request to be conditional.
+	TooManyRequests		 = 429,	 //!< Indicates that the user has sent too many requests in a given amount of time (\"rate limiting\").
+	RequestHeaderFieldsTooLarge =
+		431,  //!< Indicates that the server is unwilling to process the request because its header fields are too large.
+	UnavailableForLegalReasons =
+		451,  //!< This status code indicates that the server is denying access to the resource in response to a legal demand.
+
+	// 5xx - Server Error
+
+	InternalServerError =
+		500,			   //!< Indicates that the server encountered an unexpected condition that prevented it from fulfilling the request.
+	NotImplemented = 501,  //!< Indicates that the server does not support the functionality required to fulfill the request.
+	BadGateway	   = 502,  //!< Indicates that the server, while acting as a gateway or proxy, received an invalid response from an inbound
+						   //!< server it accessed while attempting to fulfill the request.
+	ServiceUnavailable = 503,  //!< Indicates that the server is currently unable to handle the request due to a temporary overload or
+							   //!< scheduled maintenance, which will likely be alleviated after some delay.
+	GatewayTimeout = 504,	   //!< Indicates that the server, while acting as a gateway or proxy, did not receive a timely response from an
+							   //!< upstream server it needed to access in order to complete the request.
+	HTTPVersionNotSupported = 505,	//!< Indicates that the server does not support, or refuses to support, the protocol version that was
+									//!< used in the request message.
+	VariantAlsoNegotiates =
+		506,  //!< Indicates that the server has an internal configuration error: the chosen variant resource is configured to engage in
+			  //!< transparent content negotiation itself, and is therefore not a proper end point in the negotiation process.
+	InsufficientStorage = 507,	//!< Means the method could not be performed on the resource because the server is unable to store the
+								//!< representation needed to successfully complete the request.
+	LoopDetected = 508,	 //!< Indicates that the server terminated an operation because it encountered an infinite loop while processing a
+						 //!< request with "Depth: infinity". [RFC 5842]
+	NotExtended					  = 510,  //!< The policy for accessing the resource has not been met in the request. [RFC 2774]
+	NetworkAuthenticationRequired = 511,  //!< Indicates that the client needs to authenticate to gain network access.
+};
+
+}  // namespace zen
diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h
new file mode 100644
index 000000000..3b9fa50b4
--- /dev/null
+++ b/src/zenhttp/include/zenhttp/httpserver.h
@@ -0,0 +1,315 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "zenhttp.h"
+
+#include <zencore/compactbinary.h>
+#include <zencore/enumflags.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/refcount.h>
+#include <zencore/string.h>
+#include <zencore/uid.h>
+#include <zenhttp/httpcommon.h>
+
+#include <functional>
+#include <gsl/gsl-lite.hpp>
+#include <list>
+#include <map>
+#include <regex>
+#include <span>
+#include <unordered_map>
+
+namespace zen {
+
+/** HTTP Server Request
+ */
+class HttpServerRequest
+{
+public:
+	HttpServerRequest();
+	~HttpServerRequest();
+
+	// Synchronous operations
+
+	[[nodiscard]] inline std::string_view RelativeUri() const { return m_Uri; }	 // Returns URI without service prefix
+	[[nodiscard]] std::string_view		  RelativeUriWithExtension() const { return m_UriWithExtension; }
+	[[nodiscard]] inline std::string_view QueryString() const { return m_QueryString; }
+
+	struct QueryParams
+	{
+		std::vector<std::pair<std::string_view, std::string_view>> KvPairs;
+
+		std::string_view GetValue(std::string_view ParamName) const
+		{
+			for (const auto& Kv : KvPairs)
+			{
+				const std::string_view& Key = Kv.first;
+
+				if (Key.size() == ParamName.size())
+				{
+					if (0 == StrCaseCompare(Key.data(), ParamName.data(), Key.size()))
+					{
+						return Kv.second;
+					}
+				}
+			}
+
+			return std::string_view();
+		}
+	};
+
+	virtual bool TryGetRanges(HttpRanges&) { return false; }
+
+	QueryParams GetQueryParams();
+
+	inline HttpVerb		   RequestVerb() const { return m_Verb; }
+	inline HttpContentType RequestContentType() { return m_ContentType; }
+	inline HttpContentType AcceptContentType() { return m_AcceptType; }
+
+	inline uint64_t ContentLength() const { return m_ContentLength; }
+	Oid				SessionId() const;
+	uint32_t		RequestId() const;
+
+	inline bool IsHandled() const { return !!(m_Flags & kIsHandled); }
+	inline bool SuppressBody() const { return !!(m_Flags & kSuppressBody); }
+	inline void SetSuppressResponseBody() { m_Flags |= kSuppressBody; }
+
+	/** Read POST/PUT payload for request body, which is always available without delay
+	 */
+	virtual IoBuffer ReadPayload() = 0;
+
+	ZENCORE_API CbObject  ReadPayloadObject();
+	ZENCORE_API CbPackage ReadPayloadPackage();
+
+	/** Respond with payload
+
+		No data will have been sent when any of these functions return. Instead, the response will be transmitted
+		asynchronously, after returning from a request handler function.
+
+		Note that this is destructive in the sense that the IoBuffer instances referred to by Blobs will be
+		moved into our response handler array where they are kept alive, in order to reduce ref-counting storms
+	  */
+	virtual void WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs)		  = 0;
+	virtual void WriteResponse(HttpResponseCode ResponseCode)																  = 0;
+	virtual void WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::u8string_view ResponseString) = 0;
+	virtual void WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, CompositeBuffer& Payload);
+
+	void WriteResponse(HttpResponseCode ResponseCode, CbObject Data);
+	void WriteResponse(HttpResponseCode ResponseCode, CbArray Array);
+	void WriteResponse(HttpResponseCode ResponseCode, CbPackage Package);
+	void WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::string_view ResponseString);
+	void WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, IoBuffer Blob);
+
+	virtual void WriteResponseAsync(std::function<void(HttpServerRequest&)>&& ContinuationHandler) = 0;
+
+protected:
+	enum
+	{
+		kIsHandled	   = 1 << 0,
+		kSuppressBody  = 1 << 1,
+		kHaveRequestId = 1 << 2,
+		kHaveSessionId = 1 << 3,
+	};
+
+	mutable uint32_t m_Flags		 = 0;
+	HttpVerb		 m_Verb			 = HttpVerb::kGet;
+	HttpContentType	 m_ContentType	 = HttpContentType::kBinary;
+	HttpContentType	 m_AcceptType	 = HttpContentType::kUnknownContentType;
+	uint64_t		 m_ContentLength = ~0ull;
+	std::string_view m_Uri;
+	std::string_view m_UriWithExtension;
+	std::string_view m_QueryString;
+	mutable uint32_t m_RequestId = ~uint32_t(0);
+	mutable Oid		 m_SessionId = Oid::Zero;
+
+	inline void SetIsHandled() { m_Flags |= kIsHandled; }
+
+	virtual Oid		 ParseSessionId() const = 0;
+	virtual uint32_t ParseRequestId() const = 0;
+};
+
+class IHttpPackageHandler : public RefCounted
+{
+public:
+	virtual void	 FilterOffer(std::vector<IoHash>& OfferCids)		   = 0;
+	virtual void	 OnRequestBegin()									   = 0;
+	virtual IoBuffer CreateTarget(const IoHash& Cid, uint64_t StorageSize) = 0;
+	virtual void	 OnRequestComplete()								   = 0;
+};
+
+/**
+ * Base class for implementing an HTTP "service"
+ *
+ * A service exposes one or more endpoints with a certain URI prefix
+ *
+ */
+
+class HttpService
+{
+public:
+	HttpService()		   = default;
+	virtual ~HttpService() = default;
+
+	virtual const char*				 BaseUri() const									  = 0;
+	virtual void					 HandleRequest(HttpServerRequest& HttpServiceRequest) = 0;
+	virtual Ref<IHttpPackageHandler> HandlePackageRequest(HttpServerRequest& HttpServiceRequest);
+
+	// Internals
+
+	inline void SetUriPrefixLength(size_t PrefixLength) { m_UriPrefixLength = (int)PrefixLength; }
+	inline int	UriPrefixLength() const { return m_UriPrefixLength; }
+
+private:
+	int m_UriPrefixLength = 0;
+};
+
+/** HTTP server
+ *
+ *	Implements the main event loop to service HTTP requests, and handles routing
+ *  requests to the appropriate handler as registered via RegisterService
+ */
+class HttpServer : public RefCounted
+{
+public:
+	virtual void RegisterService(HttpService& Service) = 0;
+	virtual int	 Initialize(int BasePort)			   = 0;
+	virtual void Run(bool IsInteractiveSession)		   = 0;
+	virtual void RequestExit()						   = 0;
+};
+
+Ref<HttpServer> CreateHttpServer(std::string_view ServerClass);
+
+//////////////////////////////////////////////////////////////////////////
+
+class HttpRouterRequest
+{
+public:
+	HttpRouterRequest(HttpServerRequest& Request) : m_HttpRequest(Request) {}
+
+	ZENCORE_API std::string	  GetCapture(uint32_t Index) const;
+	inline HttpServerRequest& ServerRequest() { return m_HttpRequest; }
+
+private:
+	using MatchResults_t = std::match_results<std::string_view::const_iterator>;
+
+	HttpServerRequest& m_HttpRequest;
+	MatchResults_t	   m_Match;
+
+	friend class HttpRequestRouter;
+};
+
+inline std::string
+HttpRouterRequest::GetCapture(uint32_t Index) const
+{
+	ZEN_ASSERT(Index < m_Match.size());
+
+	return m_Match[Index];
+}
+
+/** HTTP request router helper
+ *
+ * This helper class allows a service implementer to register one or more
+ * endpoints using pattern matching (currently using regex matching)
+ *
+ * This is intended to be initialized once only, there is no thread
+ * safety so you can absolutely not add or remove endpoints once the handler
+ * goes live
+ */
+
+class HttpRequestRouter
+{
+public:
+	typedef std::function<void(HttpRouterRequest&)> HandlerFunc_t;
+
+	/**
+	 * @brief Add pattern which can be referenced by name, commonly used for URL components
+	 * @param Id String used to identify patterns for replacement
+	 * @param Regex String which will replace the Id string in any registered URL paths
+	 */
+	void AddPattern(const char* Id, const char* Regex);
+
+	/**
+	 * @brief Register a an endpoint handler for the given route
+	 * @param Regex Regular expression used to match the handler to a request. This may
+	 *		  contain pattern aliases registered via AddPattern
+	 * @param HandlerFunc Handler function to call for any matching request
+	 * @param SupportedVerbs Supported HTTP verbs for this handler
+	 */
+	void RegisterRoute(const char* Regex, HandlerFunc_t&& HandlerFunc, HttpVerb SupportedVerbs);
+
+	void ProcessRegexSubstitutions(const char* Regex, StringBuilderBase& ExpandedRegex);
+
+	/**
+	 * @brief HTTP request handling function - this should be called to route the
+	 *		  request to a registered handler
+	 * @param Request Request to route to a handler
+	 * @return Function returns true if the request was routed successfully
+	 */
+	bool HandleRequest(zen::HttpServerRequest& Request);
+
+private:
+	struct HandlerEntry
+	{
+		HandlerEntry(const char* Regex, HttpVerb SupportedVerbs, HandlerFunc_t&& Handler, const char* Pattern)
+		: RegEx(Regex, std::regex::icase | std::regex::ECMAScript)
+		, Verbs(SupportedVerbs)
+		, Handler(std::move(Handler))
+		, Pattern(Pattern)
+		{
+		}
+
+		~HandlerEntry() = default;
+
+		std::regex	  RegEx;
+		HttpVerb	  Verbs;
+		HandlerFunc_t Handler;
+		const char*	  Pattern;
+
+	private:
+		HandlerEntry& operator=(const HandlerEntry&) = delete;
+		HandlerEntry(const HandlerEntry&)			 = delete;
+	};
+
+	std::list<HandlerEntry>						 m_Handlers;
+	std::unordered_map<std::string, std::string> m_PatternMap;
+};
+
+/** HTTP RPC request helper
+ */
+
+class RpcResult
+{
+	RpcResult(CbObject Result) : m_Result(std::move(Result)) {}
+
+private:
+	CbObject m_Result;
+};
+
+class HttpRpcHandler
+{
+public:
+	HttpRpcHandler();
+	~HttpRpcHandler();
+
+	HttpRpcHandler(const HttpRpcHandler&) = delete;
+	HttpRpcHandler operator=(const HttpRpcHandler&) = delete;
+
+	void AddRpc(std::string_view RpcId, std::function<void(CbObject& RpcArgs)> HandlerFunction);
+
+private:
+	struct RpcFunction
+	{
+		std::function<void(CbObject& RpcArgs)> Function;
+		std::string							   Identifier;
+	};
+
+	std::map<std::string, RpcFunction> m_Functions;
+};
+
+bool HandlePackageOffers(HttpService& Service, HttpServerRequest& Request, Ref<IHttpPackageHandler>& PackageHandlerRef);
+
+void http_forcelink();	// internal
+
+}  // namespace zen
diff --git a/src/zenhttp/include/zenhttp/httpshared.h b/src/zenhttp/include/zenhttp/httpshared.h
new file mode 100644
index 000000000..d335572c5
--- /dev/null
+++ b/src/zenhttp/include/zenhttp/httpshared.h
@@ -0,0 +1,163 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compactbinarypackage.h>
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+
+#include <functional>
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+class IoBuffer;
+class CbPackage;
+class CompositeBuffer;
+
+/**	   _____ _     _____           _
+	  / ____| |   |  __ \         | |
+	 | |    | |__ | |__) |_ _  ___| | ____ _  __ _  ___
+	 | |    | '_ \|  ___/ _` |/ __| |/ / _` |/ _` |/ _ \
+	 | |____| |_) | |  | (_| | (__|   < (_| | (_| |  __/
+	  \_____|_.__/|_|   \__,_|\___|_|\_\__,_|\__, |\___|
+											  __/ |
+											 |___/
+
+	Structures and code related to handling CbPackage transactions
+
+	CbPackage instances are marshaled across the wire using a distinct message
+	format. We don't use the CbPackage serialization format provided by the
+	CbPackage implementation itself since that does not provide much flexibility
+	in how the attachment payloads are transmitted. The scheme below separates
+	metadata cleanly from payloads and this enables us to more efficiently
+	transmit them either via sendfile/TransmitFile like mechanisms, or by
+	reference/memory mapping in the local case.
+ */
+
+struct CbPackageHeader
+{
+	uint32_t HeaderMagic;
+	uint32_t AttachmentCount;  // TODO: should add ability to opt out of implicit root document?
+	uint32_t Reserved1;
+	uint32_t Reserved2;
+};
+
+static_assert(sizeof(CbPackageHeader) == 16);
+
+enum : uint32_t
+{
+	kCbPkgMagic = 0xaa77aacc
+};
+
+struct CbAttachmentEntry
+{
+	uint64_t PayloadSize;	  // Size of the associated payload data in the message
+	uint32_t Flags;			  // See flags below
+	IoHash	 AttachmentHash;  // Content Id for the attachment
+
+	enum
+	{
+		kIsCompressed = (1u << 0),	// Is marshaled using compressed buffer storage format
+		kIsObject	  = (1u << 1),	// Is compact binary object
+		kIsError	  = (1u << 2),	// Is error (compact binary formatted) object
+		kIsLocalRef	  = (1u << 3),	// Is "local reference"
+	};
+};
+
+struct CbAttachmentReferenceHeader
+{
+	uint64_t PayloadByteOffset	= 0;
+	uint64_t PayloadByteSize	= ~0u;
+	uint16_t AbsolutePathLength = 0;
+
+	// This header will be followed by UTF8 encoded absolute path to backing file
+};
+
+static_assert(sizeof(CbAttachmentEntry) == 32);
+
+enum class FormatFlags
+{
+	kDefault					= 0,
+	kAllowLocalReferences		= (1u << 0),
+	kDenyPartialLocalReferences = (1u << 1)
+};
+
+gsl_DEFINE_ENUM_BITMASK_OPERATORS(FormatFlags);
+
+enum class RpcAcceptOptions : uint16_t
+{
+	kNone						 = 0,
+	kAllowLocalReferences		 = (1u << 0),
+	kAllowPartialLocalReferences = (1u << 1)
+};
+
+gsl_DEFINE_ENUM_BITMASK_OPERATORS(RpcAcceptOptions);
+
+std::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data, FormatFlags Flags, int TargetProcessPid = 0);
+CompositeBuffer		  FormatPackageMessageBuffer(const CbPackage& Data, FormatFlags Flags, int TargetProcessPid = 0);
+CbPackage			  ParsePackageMessage(
+				IoBuffer												  Payload,
+				std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> CreateBuffer = [](const IoHash&, uint64_t Size) -> IoBuffer {
+		return IoBuffer{Size};
+				});
+bool IsPackageMessage(IoBuffer Payload);
+
+bool ParsePackageMessageWithLegacyFallback(const IoBuffer& Response, CbPackage& OutPackage);
+
+std::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data, int TargetProcessPid = 0);
+CompositeBuffer		  FormatPackageMessageBuffer(const CbPackage& Data, int TargetProcessPid = 0);
+
+/** Streaming reader for compact binary packages
+
+	The goal is to ultimately support zero-copy I/O, but for now there'll be some
+	copying involved on some platforms at least.
+
+	This approach to deserializing CbPackage data is more efficient than
+	`ParsePackageMessage` since it does not require the entire message to
+	be resident in a memory buffer
+
+  */
+class CbPackageReader
+{
+public:
+	CbPackageReader();
+	~CbPackageReader();
+
+	void SetPayloadBufferCreator(std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> CreateBuffer);
+
+	/** Process compact binary package data stream
+
+		The data stream must be in the serialization format produced by FormatPackageMessage
+
+		\return How many bytes must be fed to this function in the next call
+	 */
+	uint64_t ProcessPackageHeaderData(const void* Data, uint64_t DataBytes);
+
+	void							 Finalize();
+	const std::vector<CbAttachment>& GetAttachments() { return m_Attachments; }
+	CbObject						 GetRootObject() { return m_RootObject; }
+	std::span<IoBuffer>				 GetPayloadBuffers() { return m_PayloadBuffers; }
+
+private:
+	enum class State
+	{
+		kInitialState,
+		kReadingHeader,
+		kReadingAttachmentEntries,
+		kReadingBuffers
+	} m_CurrentState = State::kInitialState;
+
+	std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> m_CreateBuffer;
+	std::vector<IoBuffer>									  m_PayloadBuffers;
+	std::vector<CbAttachmentEntry>							  m_AttachmentEntries;
+	std::vector<CbAttachment>								  m_Attachments;
+	CbObject												  m_RootObject;
+	CbPackageHeader											  m_PackageHeader;
+
+	IoBuffer MarshalLocalChunkReference(IoBuffer AttachmentBuffer);
+};
+
+void forcelink_httpshared();
+
+}  // namespace zen
diff --git a/src/zenhttp/include/zenhttp/websocket.h b/src/zenhttp/include/zenhttp/websocket.h
new file mode 100644
index 000000000..adca7e988
--- /dev/null
+++ b/src/zenhttp/include/zenhttp/websocket.h
@@ -0,0 +1,256 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/compactbinarypackage.h>
+#include <zencore/memory.h>
+
+#include <compare>
+#include <functional>
+#include <future>
+#include <memory>
+#include <optional>
+
+#pragma once
+
+namespace asio {
+class io_context;
+}
+
+namespace zen {
+
+class BinaryWriter;
+
+/**
+ * A unique socket ID.
+ */
+class WebSocketId
+{
+	static std::atomic_uint32_t NextId;
+
+public:
+	WebSocketId() = default;
+
+	uint32_t Value() const { return m_Value; }
+
+	auto operator<=>(const WebSocketId&) const = default;
+
+	static WebSocketId New() { return WebSocketId(NextId.fetch_add(1)); }
+
+private:
+	WebSocketId(uint32_t Value) : m_Value(Value) {}
+
+	uint32_t m_Value{};
+};
+
+/**
+ * Type of web socket message.
+ */
+enum class WebSocketMessageType : uint8_t
+{
+	kInvalid,
+	kNotification,
+	kRequest,
+	kStreamRequest,
+	kResponse,
+	kStreamResponse,
+	kStreamCompleteResponse,
+	kCount
+};
+
+inline std::string_view
+ToString(WebSocketMessageType Type)
+{
+	switch (Type)
+	{
+		case WebSocketMessageType::kInvalid:
+			return std::string_view("Invalid");
+		case WebSocketMessageType::kNotification:
+			return std::string_view("Notification");
+		case WebSocketMessageType::kRequest:
+			return std::string_view("Request");
+		case WebSocketMessageType::kStreamRequest:
+			return std::string_view("StreamRequest");
+		case WebSocketMessageType::kResponse:
+			return std::string_view("Response");
+		case WebSocketMessageType::kStreamResponse:
+			return std::string_view("StreamResponse");
+		case WebSocketMessageType::kStreamCompleteResponse:
+			return std::string_view("StreamCompleteResponse");
+		default:
+			return std::string_view("Unknown");
+	};
+}
+
+/**
+ * Web socket message.
+ */
+class WebSocketMessage
+{
+	struct Header
+	{
+		static constexpr uint32_t ExpectedMagic = 0x7a776d68;  // zwmh
+
+		uint64_t			 MessageSize{};
+		uint32_t			 Magic{ExpectedMagic};
+		uint32_t			 CorrelationId{};
+		uint32_t			 StatusCode{200u};
+		WebSocketMessageType MessageType{};
+		uint8_t				 Reserved[3] = {0};
+
+		bool IsValid() const;
+	};
+
+	static_assert(sizeof(Header) == 24);
+
+	static std::atomic_uint32_t NextCorrelationId;
+
+public:
+	static constexpr size_t HeaderSize = sizeof(Header);
+
+	WebSocketMessage() = default;
+
+	WebSocketId			 SocketId() const { return m_SocketId; }
+	void				 SetSocketId(WebSocketId Id) { m_SocketId = Id; }
+	uint64_t			 MessageSize() const { return m_Header.MessageSize; }
+	void				 SetMessageType(WebSocketMessageType MessageType);
+	void				 SetCorrelationId(uint32_t Id) { m_Header.CorrelationId = Id; }
+	uint32_t			 CorrelationId() const { return m_Header.CorrelationId; }
+	uint32_t			 StatusCode() const { return m_Header.StatusCode; }
+	void				 SetStatusCode(uint32_t StatusCode) { m_Header.StatusCode = StatusCode; }
+	WebSocketMessageType MessageType() const { return m_Header.MessageType; }
+
+	const CbPackage& Body() const { return m_Body.value(); }
+	void			 SetBody(CbPackage&& Body);
+	void			 SetBody(CbObject&& Body);
+	bool			 HasBody() const { return m_Body.has_value(); }
+
+	void Save(BinaryWriter& Writer);
+	bool TryLoadHeader(MemoryView Memory);
+
+	bool IsValid() const { return m_Header.MessageType != WebSocketMessageType::kInvalid; }
+
+private:
+	Header					 m_Header{};
+	WebSocketId				 m_SocketId{};
+	std::optional<CbPackage> m_Body;
+};
+
+class WebSocketServer;
+
+/**
+ * Base class for handling web socket requests and notifications from connected client(s).
+ */
+class WebSocketService
+{
+public:
+	virtual ~WebSocketService() = default;
+
+	void Configure(WebSocketServer& Server);
+
+	virtual bool HandleRequest(const WebSocketMessage&) { ZEN_ASSERT(false); }
+	virtual void HandleNotification(const WebSocketMessage&) { ZEN_ASSERT(false); }
+
+protected:
+	WebSocketService() = default;
+
+	virtual void RegisterHandlers(WebSocketServer& Server) = 0;
+	void		 SendStreamResponse(WebSocketId SocketId, uint32_t CorrelationId, CbPackage&& StreamResponse, bool IsStreamComplete);
+	void		 SendStreamResponse(WebSocketId SocketId, uint32_t CorrelationId, CbObject&& StreamResponse, bool IsStreamComplete);
+
+	WebSocketServer& SocketServer()
+	{
+		ZEN_ASSERT(m_SocketServer);
+		return *m_SocketServer;
+	}
+
+private:
+	WebSocketServer* m_SocketServer{};
+};
+
+/**
+ * Server options.
+ */
+struct WebSocketServerOptions
+{
+	uint16_t Port		 = 2337;
+	uint32_t ThreadCount = 1;
+};
+
+/**
+ * The web socket server manages client connections and routing of requests and notifications.
+ */
+class WebSocketServer
+{
+public:
+	virtual ~WebSocketServer() = default;
+
+	virtual bool Run()		= 0;
+	virtual void Shutdown() = 0;
+
+	virtual void RegisterService(WebSocketService& Service)									  = 0;
+	virtual void RegisterNotificationHandler(std::string_view Key, WebSocketService& Service) = 0;
+	virtual void RegisterRequestHandler(std::string_view Key, WebSocketService& Service)	  = 0;
+
+	virtual void SendNotification(WebSocketMessage&& Notification) = 0;
+	virtual void SendResponse(WebSocketMessage&& Response)		   = 0;
+
+	static std::unique_ptr<WebSocketServer> Create(const WebSocketServerOptions& Options);
+};
+
+/**
+ * The state of the web socket.
+ */
+enum class WebSocketState : uint32_t
+{
+	kNone,
+	kHandshaking,
+	kConnected,
+	kDisconnected,
+	kError
+};
+
+/**
+ * Type of web socket client event.
+ */
+enum class WebSocketEvent : uint32_t
+{
+	kConnected,
+	kDisconnected,
+	kError
+};
+
+/**
+ * Web socket client connection info.
+ */
+struct WebSocketConnectInfo
+{
+	std::string				 Host;
+	int16_t					 Port{8848};
+	std::string				 Endpoint;
+	std::vector<std::string> Protocols;
+	uint16_t				 Version{13};
+};
+
+/**
+ * A connection to a web socket server for sending requests and listening for notifications.
+ */
+class WebSocketClient
+{
+public:
+	using EventCallback		   = std::function<void()>;
+	using NotificationCallback = std::function<void(WebSocketMessage&&)>;
+
+	virtual ~WebSocketClient() = default;
+
+	virtual std::future<bool> Connect(const WebSocketConnectInfo& Info) = 0;
+	virtual void			  Disconnect()								= 0;
+	virtual bool			  IsConnected() const						= 0;
+	virtual WebSocketState	  State() const								= 0;
+
+	virtual std::future<WebSocketMessage> SendRequest(WebSocketMessage&& Request)		  = 0;
+	virtual void						  OnNotification(NotificationCallback&& Cb)		  = 0;
+	virtual void						  OnEvent(WebSocketEvent Evt, EventCallback&& Cb) = 0;
+
+	static std::shared_ptr<WebSocketClient> Create(asio::io_context& IoCtx);
+};
+
+}  // namespace zen
diff --git a/src/zenhttp/include/zenhttp/zenhttp.h b/src/zenhttp/include/zenhttp/zenhttp.h
new file mode 100644
index 000000000..59c64b31f
--- /dev/null
+++ b/src/zenhttp/include/zenhttp/zenhttp.h
@@ -0,0 +1,21 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#ifndef ZEN_WITH_HTTPSYS
+#	if ZEN_PLATFORM_WINDOWS
+#		define ZEN_WITH_HTTPSYS 1
+#	else
+#		define ZEN_WITH_HTTPSYS 0
+#	endif
+#endif
+
+#define ZENHTTP_API	 // Placeholder to allow DLL configs in the future
+
+namespace zen {
+
+ZENHTTP_API void zenhttp_forcelinktests();
+
+}