// Copyright Epic Games, Inc. All Rights Reserved. #include #include "servers/httpasio.h" #include "servers/httpmulti.h" #include "servers/httpnull.h" #include "servers/httpsys.h" #include "zenhttp/httpplugin.h" #if ZEN_WITH_PLUGINS # include "transports/asiotransport.h" # include "transports/dlltransport.h" # include "transports/winsocktransport.h" #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace zen { using namespace std::literals; std::string_view MapContentTypeToString(HttpContentType ContentType) { switch (ContentType) { default: case HttpContentType::kUnknownContentType: case HttpContentType::kBinary: return "application/octet-stream"sv; case HttpContentType::kText: return "text/plain"sv; case HttpContentType::kJSON: return "application/json"sv; case HttpContentType::kCbObject: return "application/x-ue-cb"sv; case HttpContentType::kCbPackage: return "application/x-ue-cbpkg"sv; case HttpContentType::kCbPackageOffer: return "application/x-ue-offer"sv; case HttpContentType::kCompressedBinary: return "application/x-ue-comp"sv; case HttpContentType::kYAML: return "text/yaml"sv; case HttpContentType::kHTML: return "text/html"sv; case HttpContentType::kJavaScript: return "application/javascript"sv; case HttpContentType::kCSS: return "text/css"sv; case HttpContentType::kPNG: return "image/png"sv; case HttpContentType::kIcon: return "image/x-icon"sv; case HttpContentType::kXML: return "application/xml"sv; } } ////////////////////////////////////////////////////////////////////////// // // Note that in addition to MIME types we accept abbreviated versions, for // use in suffix parsing as well as for convenience when using curl static constinit uint32_t HashBinary = HashStringDjb2("application/octet-stream"sv); static constinit uint32_t HashJson = HashStringDjb2("json"sv); static constinit uint32_t HashApplicationJson = HashStringDjb2("application/json"sv); static constinit uint32_t HashApplicationProblemJson = HashStringDjb2("application/problem+json"sv); static constinit uint32_t HashYaml = HashStringDjb2("yaml"sv); static constinit uint32_t HashTextYaml = HashStringDjb2("text/yaml"sv); static constinit uint32_t HashText = HashStringDjb2("text/plain"sv); static constinit uint32_t HashApplicationCompactBinary = HashStringDjb2("application/x-ue-cb"sv); static constinit uint32_t HashCompactBinary = HashStringDjb2("ucb"sv); static constinit uint32_t HashCompactBinaryPackage = HashStringDjb2("application/x-ue-cbpkg"sv); static constinit uint32_t HashCompactBinaryPackageShort = HashStringDjb2("cbpkg"sv); static constinit uint32_t HashCompactBinaryPackageOffer = HashStringDjb2("application/x-ue-offer"sv); static constinit uint32_t HashCompressedBinary = HashStringDjb2("application/x-ue-comp"sv); static constinit uint32_t HashHtml = HashStringDjb2("html"sv); static constinit uint32_t HashTextHtml = HashStringDjb2("text/html"sv); static constinit uint32_t HashJavaScript = HashStringDjb2("js"sv); static constinit uint32_t HashJavaScriptSourceMap = HashStringDjb2("map"sv); // actually .js.map static constinit uint32_t HashApplicationJavaScript = HashStringDjb2("application/javascript"sv); static constinit uint32_t HashCss = HashStringDjb2("css"sv); static constinit uint32_t HashTextCss = HashStringDjb2("text/css"sv); static constinit uint32_t HashPng = HashStringDjb2("png"sv); static constinit uint32_t HashImagePng = HashStringDjb2("image/png"sv); static constinit uint32_t HashIcon = HashStringDjb2("ico"sv); static constinit uint32_t HashImageIcon = HashStringDjb2("image/x-icon"sv); static constinit uint32_t HashXml = HashStringDjb2("application/xml"sv); std::once_flag InitContentTypeLookup; struct HashedTypeEntry { uint32_t Hash; HttpContentType Type; } TypeHashTable[] = { // clang-format off {HashBinary, HttpContentType::kBinary}, {HashApplicationCompactBinary, HttpContentType::kCbObject}, {HashCompactBinary, HttpContentType::kCbObject}, {HashCompactBinaryPackage, HttpContentType::kCbPackage}, {HashCompactBinaryPackageShort, HttpContentType::kCbPackage}, {HashCompactBinaryPackageOffer, HttpContentType::kCbPackageOffer}, {HashJson, HttpContentType::kJSON}, {HashApplicationJson, HttpContentType::kJSON}, {HashApplicationProblemJson, HttpContentType::kJSON}, {HashYaml, HttpContentType::kYAML}, {HashTextYaml, HttpContentType::kYAML}, {HashText, HttpContentType::kText}, {HashCompressedBinary, HttpContentType::kCompressedBinary}, {HashHtml, HttpContentType::kHTML}, {HashTextHtml, HttpContentType::kHTML}, {HashJavaScript, HttpContentType::kJavaScript}, {HashApplicationJavaScript, HttpContentType::kJavaScript}, {HashJavaScriptSourceMap, HttpContentType::kJavaScript}, {HashCss, HttpContentType::kCSS}, {HashTextCss, HttpContentType::kCSS}, {HashPng, HttpContentType::kPNG}, {HashImagePng, HttpContentType::kPNG}, {HashIcon, HttpContentType::kIcon}, {HashImageIcon, HttpContentType::kIcon}, {HashXml, HttpContentType::kXML}, // clang-format on }; HttpContentType ParseContentTypeImpl(const std::string_view& ContentTypeString) { if (!ContentTypeString.empty()) { size_t ContentEnd = ContentTypeString.find(';'); if (ContentEnd == std::string_view::npos) { ContentEnd = ContentTypeString.length(); } std::string_view ContentString(ContentTypeString.substr(0, ContentEnd)); const uint32_t CtHash = HashStringDjb2(ContentString); if (auto It = std::lower_bound(std::begin(TypeHashTable), std::end(TypeHashTable), CtHash, [](const HashedTypeEntry& Lhs, const uint32_t Rhs) { return Lhs.Hash < Rhs; }); It != std::end(TypeHashTable)) { if (It->Hash == CtHash) { return It->Type; } } } return HttpContentType::kUnknownContentType; } HttpContentType ParseContentTypeInit(const std::string_view& ContentTypeString) { std::call_once(InitContentTypeLookup, [] { std::sort(std::begin(TypeHashTable), std::end(TypeHashTable), [](const HashedTypeEntry& Lhs, const HashedTypeEntry& Rhs) { return Lhs.Hash < Rhs.Hash; }); // validate that there are no hash collisions uint32_t LastHash = 0; for (const auto& Item : TypeHashTable) { ZEN_ASSERT(LastHash != Item.Hash); LastHash = Item.Hash; } }); ParseContentType = ParseContentTypeImpl; return ParseContentTypeImpl(ContentTypeString); } HttpContentType (*ParseContentType)(const std::string_view& ContentTypeString) = &ParseContentTypeInit; bool TryParseHttpRangeHeader(std::string_view RangeHeader, HttpRanges& Ranges) { if (RangeHeader.empty()) { return false; } const size_t Count = Ranges.size(); std::size_t UnitDelim = RangeHeader.find_first_of('='); if (UnitDelim == std::string_view::npos) { return false; } // only bytes for now std::string_view Unit = RangeHeader.substr(0, UnitDelim); if (Unit != "bytes"sv) { return false; } std::string_view Tokens = RangeHeader.substr(UnitDelim); while (!Tokens.empty()) { // Skip =, Tokens = Tokens.substr(1); size_t Delim = Tokens.find_first_of(','); if (Delim == std::string_view::npos) { Delim = Tokens.length(); } std::string_view Token = Tokens.substr(0, Delim); Tokens = Tokens.substr(Delim); Delim = Token.find_first_of('-'); if (Delim == std::string_view::npos) { return false; } const auto Start = ParseInt(Token.substr(0, Delim)); const auto End = ParseInt(Token.substr(Delim + 1)); if (Start.has_value() && End.has_value() && End.value() > Start.value()) { Ranges.push_back({.Start = Start.value(), .End = End.value()}); } else if (Start) { Ranges.push_back({.Start = Start.value()}); } else if (End) { Ranges.push_back({.End = End.value()}); } } return Count != Ranges.size(); } ////////////////////////////////////////////////////////////////////////// const std::string_view ToString(HttpVerb Verb) { switch (Verb) { case HttpVerb::kGet: return "GET"sv; case HttpVerb::kPut: return "PUT"sv; case HttpVerb::kPost: return "POST"sv; case HttpVerb::kDelete: return "DELETE"sv; case HttpVerb::kHead: return "HEAD"sv; case HttpVerb::kCopy: return "COPY"sv; case HttpVerb::kOptions: return "OPTIONS"sv; default: return "???"sv; } } std::string_view ToString(HttpResponseCode HttpCode) { return ReasonStringForHttpResultCode(int(HttpCode)); } std::string_view ReasonStringForHttpResultCode(int HttpCode) { switch (HttpCode) { // 1xx Informational case 100: return "Continue"sv; case 101: return "Switching Protocols"sv; // 2xx Success case 200: return "OK"sv; case 201: return "Created"sv; case 202: return "Accepted"sv; case 204: return "No Content"sv; case 205: return "Reset Content"sv; case 206: return "Partial Content"sv; // 3xx Redirection case 300: return "Multiple Choices"sv; case 301: return "Moved Permanently"sv; case 302: return "Found"sv; case 303: return "See Other"sv; case 304: return "Not Modified"sv; case 305: return "Use Proxy"sv; case 306: return "Switch Proxy"sv; case 307: return "Temporary Redirect"sv; case 308: return "Permanent Redirect"sv; // 4xx Client errors case 400: return "Bad Request"sv; case 401: return "Unauthorized"sv; case 402: return "Payment Required"sv; case 403: return "Forbidden"sv; case 404: return "Not Found"sv; case 405: return "Method Not Allowed"sv; case 406: return "Not Acceptable"sv; case 407: return "Proxy Authentication Required"sv; case 408: return "Request Timeout"sv; case 409: return "Conflict"sv; case 410: return "Gone"sv; case 411: return "Length Required"sv; case 412: return "Precondition Failed"sv; case 413: return "Payload Too Large"sv; case 414: return "URI Too Long"sv; case 415: return "Unsupported Media Type"sv; case 416: return "Range Not Satisifiable"sv; case 417: return "Expectation Failed"sv; case 418: return "I'm a teapot"sv; case 421: return "Misdirected Request"sv; case 422: return "Unprocessable Entity"sv; case 423: return "Locked"sv; case 424: return "Failed Dependency"sv; case 425: return "Too Early"sv; case 426: return "Upgrade Required"sv; case 428: return "Precondition Required"sv; case 429: return "Too Many Requests"sv; case 431: return "Request Header Fields Too Large"sv; // 5xx Server errors case 500: return "Internal Server Error"sv; case 501: return "Not Implemented"sv; case 502: return "Bad Gateway"sv; case 503: return "Service Unavailable"sv; case 504: return "Gateway Timeout"sv; case 505: return "HTTP Version Not Supported"sv; case 506: return "Variant Also Negotiates"sv; case 507: return "Insufficient Storage"sv; case 508: return "Loop Detected"sv; case 510: return "Not Extended"sv; case 511: return "Network Authentication Required"sv; default: return "Unknown Result"sv; } } ////////////////////////////////////////////////////////////////////////// Ref HttpService::HandlePackageRequest(HttpServerRequest& HttpServiceRequest) { ZEN_UNUSED(HttpServiceRequest); return Ref(); } ////////////////////////////////////////////////////////////////////////// HttpServerRequest::HttpServerRequest() { } HttpServerRequest::~HttpServerRequest() { } void HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, CbPackage Data) { std::vector ResponseBuffers = FormatPackageMessage(Data); return WriteResponse(ResponseCode, HttpContentType::kCbPackage, ResponseBuffers); } void HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, CbObject Data) { if (m_AcceptType == HttpContentType::kJSON) { ExtendableStringBuilder<1024> Sb; WriteResponse(ResponseCode, HttpContentType::kJSON, Data.ToJson(Sb).ToView()); } else { SharedBuffer Buf = Data.GetBuffer(); std::array Buffers{IoBufferBuilder::MakeCloneFromMemory(Buf.GetData(), Buf.GetSize())}; return WriteResponse(ResponseCode, HttpContentType::kCbObject, Buffers); } } void HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, CbArray Array) { if (m_AcceptType == HttpContentType::kJSON) { ExtendableStringBuilder<1024> Sb; WriteResponse(ResponseCode, HttpContentType::kJSON, Array.ToJson(Sb).ToView()); } else { SharedBuffer Buf = Array.GetBuffer(); std::array Buffers{IoBufferBuilder::MakeCloneFromMemory(Buf.GetData(), Buf.GetSize())}; return WriteResponse(ResponseCode, HttpContentType::kCbObject, Buffers); } } void HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::string_view ResponseString) { return WriteResponse(ResponseCode, ContentType, std::u8string_view{(char8_t*)ResponseString.data(), ResponseString.size()}); } void HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, IoBuffer Blob) { std::array Buffers{Blob}; return WriteResponse(ResponseCode, ContentType, Buffers); } void HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, CompositeBuffer& Payload) { std::span Segments = Payload.GetSegments(); std::vector Buffers; Buffers.reserve(Segments.size()); for (auto& Segment : Segments) { Buffers.push_back(Segment.AsIoBuffer()); } WriteResponse(ResponseCode, ContentType, Buffers); } HttpServerRequest::QueryParams HttpServerRequest::GetQueryParams() { QueryParams Params; const std::string_view QStr = QueryString(); const char* QueryIt = QStr.data(); const char* QueryEnd = QueryIt + QStr.size(); while (QueryIt != QueryEnd) { if (*QueryIt == '&') { ++QueryIt; continue; } size_t QueryLen = ptrdiff_t(QueryEnd - QueryIt); const std::string_view Query{QueryIt, QueryLen}; size_t DelimIndex = Query.find('&', 0); if (DelimIndex == std::string_view::npos) { DelimIndex = Query.size(); } std::string_view ThisQuery{QueryIt, DelimIndex}; size_t EqIndex = ThisQuery.find('=', 0); if (EqIndex != std::string_view::npos) { std::string_view Param{ThisQuery.data(), EqIndex}; ThisQuery.remove_prefix(EqIndex + 1); Params.KvPairs.emplace_back(Param, ThisQuery); } QueryIt += DelimIndex; } return Params; } Oid HttpServerRequest::SessionId() const { if (m_Flags & kHaveSessionId) { return m_SessionId; } m_SessionId = ParseSessionId(); m_Flags |= kHaveSessionId; return m_SessionId; } uint32_t HttpServerRequest::RequestId() const { if (m_Flags & kHaveRequestId) { return m_RequestId; } m_RequestId = ParseRequestId(); m_Flags |= kHaveRequestId; return m_RequestId; } CbObject HttpServerRequest::ReadPayloadObject() { if (IoBuffer Payload = ReadPayload()) { if (m_ContentType == HttpContentType::kJSON) { std::string Json(reinterpret_cast(Payload.GetData()), Payload.GetSize()); std::string Err; CbFieldIterator It = LoadCompactBinaryFromJson(Json, Err); if (Err.empty()) { return It.AsObject(); } return CbObject(); } return LoadCompactBinaryObject(std::move(Payload)); } return {}; } CbPackage HttpServerRequest::ReadPayloadPackage() { if (IoBuffer Payload = ReadPayload()) { return ParsePackageMessage(std::move(Payload)); } return {}; } ////////////////////////////////////////////////////////////////////////// void HttpRequestRouter::AddPattern(const char* Id, const char* Regex) { ZEN_ASSERT(m_PatternMap.find(Id) == m_PatternMap.end()); m_PatternMap.insert({Id, Regex}); } void HttpRequestRouter::RegisterRoute(const char* Regex, HttpRequestRouter::HandlerFunc_t&& HandlerFunc, HttpVerb SupportedVerbs) { ExtendableStringBuilder<128> ExpandedRegex; ProcessRegexSubstitutions(Regex, ExpandedRegex); m_Handlers.emplace_back(ExpandedRegex.c_str(), SupportedVerbs, std::move(HandlerFunc), Regex); } void HttpRequestRouter::ProcessRegexSubstitutions(const char* Regex, StringBuilderBase& OutExpandedRegex) { size_t RegexLen = strlen(Regex); for (size_t i = 0; i < RegexLen;) { bool matched = false; if (Regex[i] == '{' && ((i == 0) || (Regex[i - 1] != '\\'))) { // Might have a pattern reference - find closing brace for (size_t j = i + 1; j < RegexLen; ++j) { if (Regex[j] == '}') { std::string Pattern(&Regex[i + 1], j - i - 1); if (auto it = m_PatternMap.find(Pattern); it != m_PatternMap.end()) { OutExpandedRegex.Append(it->second.c_str()); } else { // Default to anything goes (or should this just be an error?) OutExpandedRegex.Append("(.+?)"); } // skip ahead i = j + 1; matched = true; break; } } } if (!matched) { OutExpandedRegex.Append(Regex[i++]); } } } bool HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request) { const HttpVerb Verb = Request.RequestVerb(); std::string_view Uri = Request.RelativeUri(); HttpRouterRequest RouterRequest(Request); for (const auto& Handler : m_Handlers) { if ((Handler.Verbs & Verb) == Verb && regex_match(begin(Uri), end(Uri), RouterRequest.m_Match, Handler.RegEx)) { Handler.Handler(RouterRequest); return true; // Route matched } } return false; // No route matched } ////////////////////////////////////////////////////////////////////////// HttpRpcHandler::HttpRpcHandler() { } HttpRpcHandler::~HttpRpcHandler() { } void HttpRpcHandler::AddRpc(std::string_view RpcId, std::function HandlerFunction) { ZEN_UNUSED(RpcId, HandlerFunction); } ////////////////////////////////////////////////////////////////////////// enum class HttpServerClass { kHttpAsio, kHttpSys, kHttpPlugin, kHttpMulti, kHttpNull }; Ref CreateHttpServerClass(HttpServerClass Class, const HttpServerConfig& Config) { switch (Class) { default: case HttpServerClass::kHttpAsio: ZEN_INFO("using asio HTTP server implementation"); return CreateHttpAsioServer(Config.ForceLoopback, Config.ThreadCount); case HttpServerClass::kHttpMulti: { ZEN_INFO("using multi HTTP server implementation"); Ref Server{new HttpMultiServer()}; // This is hardcoded for now, but should be configurable in the future Server->AddServer(CreateHttpServerClass(HttpServerClass::kHttpSys, Config)); Server->AddServer(CreateHttpServerClass(HttpServerClass::kHttpPlugin, Config)); return Server; } #if ZEN_WITH_PLUGINS case HttpServerClass::kHttpPlugin: { ZEN_INFO("using plugin HTTP server implementation"); Ref Server{CreateHttpPluginServer()}; // This is hardcoded for now, but should be configurable in the future # if 0 Ref WinsockPlugin{CreateSocketTransportPlugin()}; WinsockPlugin->Configure("port", "8558"); Server->AddPlugin(WinsockPlugin); # endif # if 0 Ref AsioPlugin{CreateAsioTransportPlugin()}; AsioPlugin->Configure("port", "8558"); Server->AddPlugin(AsioPlugin); # endif # if 1 Ref DllPlugin{CreateDllTransportPlugin()}; DllPlugin->LoadDll("winsock"); DllPlugin->ConfigureDll("winsock", "port", "8558"); Server->AddPlugin(DllPlugin); # endif return Server; } #endif #if ZEN_WITH_HTTPSYS case HttpServerClass::kHttpSys: ZEN_INFO("using http.sys server implementation"); return Ref(CreateHttpSysServer({.ThreadCount = Config.ThreadCount, .AsyncWorkThreadCount = Config.HttpSys.AsyncWorkThreadCount, .IsAsyncResponseEnabled = Config.HttpSys.IsAsyncResponseEnabled, .IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled, .IsDedicatedServer = Config.IsDedicatedServer, .ForceLoopback = Config.ForceLoopback})); #endif case HttpServerClass::kHttpNull: ZEN_INFO("using null HTTP server implementation"); return Ref(new HttpNullServer); } } Ref CreateHttpServer(const HttpServerConfig& Config) { using namespace std::literals; HttpServerClass Class = HttpServerClass::kHttpNull; #if ZEN_WITH_HTTPSYS Class = HttpServerClass::kHttpSys; #else Class = HttpServerClass::kHttpAsio; #endif if (Config.ServerClass == "asio"sv) { Class = HttpServerClass::kHttpAsio; } else if (Config.ServerClass == "httpsys"sv) { Class = HttpServerClass::kHttpSys; } else if (Config.ServerClass == "plugin"sv) { Class = HttpServerClass::kHttpPlugin; } else if (Config.ServerClass == "null"sv) { Class = HttpServerClass::kHttpNull; } else if (Config.ServerClass == "multi"sv) { Class = HttpServerClass::kHttpMulti; } return CreateHttpServerClass(Class, Config); } ////////////////////////////////////////////////////////////////////////// bool HandlePackageOffers(HttpService& Service, HttpServerRequest& Request, Ref& PackageHandlerRef) { if (Request.RequestVerb() == HttpVerb::kPost) { if (Request.RequestContentType() == HttpContentType::kCbPackageOffer) { // The client is presenting us with a package attachments offer, we need // to filter it down to the list of attachments we need them to send in // the follow-up request PackageHandlerRef = Service.HandlePackageRequest(Request); if (PackageHandlerRef) { CbObject OfferMessage = LoadCompactBinaryObject(Request.ReadPayload()); std::vector OfferCids; for (auto& CidEntry : OfferMessage["offer"]) { if (!CidEntry.IsHash()) { // Should yield bad request response? ZEN_WARN("found invalid entry in offer"); continue; } OfferCids.push_back(CidEntry.AsHash()); } ZEN_TRACE("request #{} -> filtering offer of {} entries", Request.RequestId(), OfferCids.size()); PackageHandlerRef->FilterOffer(OfferCids); ZEN_TRACE("request #{} -> filtered to {} entries", Request.RequestId(), OfferCids.size()); CbObjectWriter ResponseWriter; ResponseWriter.BeginArray("need"); for (const IoHash& Cid : OfferCids) { ResponseWriter.AddHash(Cid); } ResponseWriter.EndArray(); // Emit filter response Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); return true; } } else if (Request.RequestContentType() == HttpContentType::kCbPackage) { // Process chunks in package request PackageHandlerRef = Service.HandlePackageRequest(Request); // TODO: this should really be done in a streaming fashion, currently this emulates // the intended flow from an API perspective if (PackageHandlerRef) { PackageHandlerRef->OnRequestBegin(); auto CreateBuffer = [&](const IoHash& Cid, uint64_t Size) -> IoBuffer { return PackageHandlerRef->CreateTarget(Cid, Size); }; CbPackage Package = ParsePackageMessage(Request.ReadPayload(), CreateBuffer); PackageHandlerRef->OnRequestComplete(); } } } return false; } ////////////////////////////////////////////////////////////////////////// #if ZEN_WITH_TESTS TEST_CASE("http.common") { using namespace std::literals; SUBCASE("router") { HttpRequestRouter r; r.AddPattern("a", "[[:alpha:]]+"); r.RegisterRoute( "{a}", [&](auto) {}, HttpVerb::kGet); // struct TestHttpServerRequest : public HttpServerRequest //{ // TestHttpServerRequest(std::string_view Uri) : m_uri{Uri} {} //}; // TestHttpServerRequest req{}; // r.HandleRequest(req); } SUBCASE("content-type") { for (uint8_t i = 0; i < uint8_t(HttpContentType::kCOUNT); ++i) { HttpContentType Ct{i}; if (Ct != HttpContentType::kUnknownContentType) { CHECK_EQ(Ct, ParseContentType(MapContentTypeToString(Ct))); } } } } void http_forcelink() { } #endif } // namespace zen