asio large header support (#484)

- Improvement: Allow large headers support in asio server implementation

2 files changed, 128 insertions, 123 deletions

diff --git a/src/zenhttp/servers/httpparser.cpp b/src/zenhttp/servers/httpparser.cpp
index 9bb354a5e..93094e21b 100644
--- a/src/zenhttp/servers/httpparser.cpp
+++ b/src/zenhttp/servers/httpparser.cpp
@@ -6,6 +6,8 @@
 #include <zencore/logging.h>
 #include <zencore/string.h>
 
+#include <limits>
+
 namespace zen {
 
 using namespace std::literals;
@@ -69,23 +71,21 @@ HttpRequestParser::ConsumeData(const char* InputData, size_t DataSize)
 int
 HttpRequestParser::OnUrl(const char* Data, size_t Bytes)
 {
-	if (!m_Url)
+	const size_t RemainingBufferSpace = std::numeric_limits<std::uint32_t>::max() - m_HeaderData.size();
+	if (RemainingBufferSpace < Bytes)
 	{
-		ZEN_ASSERT_SLOW(m_UrlLength == 0);
-		m_Url = m_HeaderCursor;
+		ZEN_WARN("HTTP parser does not have enough space for incoming request headers, need {} more bytes", Bytes - RemainingBufferSpace);
+		return 1;
 	}
 
-	const size_t RemainingBufferSpace = sizeof m_HeaderBuffer + m_HeaderBuffer - m_HeaderCursor;
-
-	if (RemainingBufferSpace < Bytes)
+	if (m_UrlRange.Length == 0)
 	{
-		ZEN_WARN("HTTP parser does not have enough space for incoming request, need {} more bytes", Bytes - RemainingBufferSpace);
-		return 1;
+		ZEN_ASSERT_SLOW(m_UrlRange.Offset == 0);
+		m_UrlRange.Offset = (uint32_t)m_HeaderData.size();
 	}
 
-	memcpy(m_HeaderCursor, Data, Bytes);
-	m_HeaderCursor += Bytes;
-	m_UrlLength += Bytes;
+	m_HeaderData.insert(m_HeaderData.end(), Data, &Data[Bytes]);
+	m_UrlRange.Length += (uint32_t)Bytes;
 
 	return 0;
 }
@@ -93,59 +93,66 @@ HttpRequestParser::OnUrl(const char* Data, size_t Bytes)
 int
 HttpRequestParser::OnHeader(const char* Data, size_t Bytes)
 {
-	if (m_CurrentHeaderValueLength)
+	const size_t RemainingBufferSpace = std::numeric_limits<std::uint32_t>::max() - m_HeaderData.size();
+	if (RemainingBufferSpace < Bytes)
 	{
-		AppendCurrentHeader();
+		ZEN_WARN("HTTP parser does not have enough space for incoming request headers, need {} more bytes", Bytes - RemainingBufferSpace);
+		return 1;
+	}
 
-		m_CurrentHeaderNameLength  = 0;
-		m_CurrentHeaderValueLength = 0;
-		m_CurrentHeaderName		   = m_HeaderCursor;
+	if (m_HeaderEntries.empty())
+	{
+		m_HeaderEntries.resize(1);
 	}
-	else if (m_CurrentHeaderName == nullptr)
+	HeaderEntry* CurrentHeaderEntry = &m_HeaderEntries.back();
+	if (CurrentHeaderEntry->ValueRange.Length)
 	{
-		m_CurrentHeaderName = m_HeaderCursor;
+		ParseCurrentHeader();
+		m_HeaderEntries.emplace_back(HeaderEntry{.NameRange = {.Offset = (uint32_t)m_HeaderData.size()}});
+		CurrentHeaderEntry = &m_HeaderEntries.back();
 	}
-
-	const size_t RemainingBufferSpace = sizeof m_HeaderBuffer + m_HeaderBuffer - m_HeaderCursor;
-	if (RemainingBufferSpace < Bytes)
+	else if (CurrentHeaderEntry->NameRange.Length == 0)
 	{
-		ZEN_WARN("HTTP parser does not have enough space for incoming header name, need {} more bytes", Bytes - RemainingBufferSpace);
-		return 1;
+		m_HeaderEntries.emplace_back(HeaderEntry{.NameRange = {.Offset = (uint32_t)m_HeaderData.size()}});
+		CurrentHeaderEntry = &m_HeaderEntries.back();
 	}
 
-	memcpy(m_HeaderCursor, Data, Bytes);
-	m_HeaderCursor += Bytes;
-	m_CurrentHeaderNameLength += Bytes;
+	m_HeaderData.insert(m_HeaderData.end(), Data, &Data[Bytes]);
+	CurrentHeaderEntry->NameRange.Length += (uint32_t)Bytes;
 
 	return 0;
 }
 
 void
-HttpRequestParser::AppendCurrentHeader()
+HttpRequestParser::ParseCurrentHeader()
 {
-	std::string_view HeaderName(m_CurrentHeaderName, m_CurrentHeaderNameLength);
-	if (m_Headers.size() == std::numeric_limits<int8_t>::max())
+	ZEN_ASSERT_SLOW(!m_HeaderEntries.empty());
+	const HeaderEntry&	   CurrentHeaderEntry = m_HeaderEntries.back();
+	const size_t		   CurrentHeaderCount = m_HeaderEntries.size();
+	const std::string_view HeaderName(GetHeaderSubString(CurrentHeaderEntry.NameRange));
+	if (CurrentHeaderCount > std::numeric_limits<int8_t>::max())
 	{
 		ZEN_WARN("HttpRequestParser parser only supports up to {} headers, can't store header '{}'. Dropping it.",
 				 std::numeric_limits<int8_t>::max(),
 				 HeaderName);
 		return;
 	}
-	std::string_view HeaderValue(m_CurrentHeaderValue, m_CurrentHeaderValueLength);
+	const std::string_view HeaderValue(GetHeaderSubString(CurrentHeaderEntry.ValueRange));
 
-	const uint32_t HeaderHash = HashStringAsLowerDjb2(HeaderName);
+	const uint32_t HeaderHash		  = HashStringAsLowerDjb2(HeaderName);
+	const int8_t   CurrentHeaderIndex = int8_t(CurrentHeaderCount - 1);
 
 	if (HeaderHash == HashContentLength)
 	{
-		m_ContentLengthHeaderIndex = (int8_t)m_Headers.size();
+		m_ContentLengthHeaderIndex = CurrentHeaderIndex;
 	}
 	else if (HeaderHash == HashAccept)
 	{
-		m_AcceptHeaderIndex = (int8_t)m_Headers.size();
+		m_AcceptHeaderIndex = CurrentHeaderIndex;
 	}
 	else if (HeaderHash == HashContentType)
 	{
-		m_ContentTypeHeaderIndex = (int8_t)m_Headers.size();
+		m_ContentTypeHeaderIndex = CurrentHeaderIndex;
 	}
 	else if (HeaderHash == HashSession)
 	{
@@ -169,38 +176,38 @@ HttpRequestParser::AppendCurrentHeader()
 	}
 	else if (HeaderHash == HashRange)
 	{
-		m_RangeHeaderIndex = (int8_t)m_Headers.size();
+		m_RangeHeaderIndex = CurrentHeaderIndex;
 	}
-
-	m_Headers.emplace_back(HeaderName, HeaderValue);
 }
 
 int
 HttpRequestParser::OnHeaderValue(const char* Data, size_t Bytes)
 {
-	if (m_CurrentHeaderValueLength == 0)
-	{
-		m_CurrentHeaderValue = m_HeaderCursor;
-	}
-
-	const size_t RemainingBufferSpace = sizeof m_HeaderBuffer + m_HeaderBuffer - m_HeaderCursor;
+	const size_t RemainingBufferSpace = std::numeric_limits<std::uint32_t>::max() - m_HeaderData.size();
 	if (RemainingBufferSpace < Bytes)
 	{
-		ZEN_WARN("HTTP parser does not have enough space for incoming header value, need {} more bytes", Bytes - RemainingBufferSpace);
+		ZEN_WARN("HTTP parser does not have enough space for incoming request headers, need {} more bytes", Bytes - RemainingBufferSpace);
 		return 1;
 	}
 
-	memcpy(m_HeaderCursor, Data, Bytes);
-	m_HeaderCursor += Bytes;
-	m_CurrentHeaderValueLength += Bytes;
+	ZEN_ASSERT_SLOW(!m_HeaderEntries.empty());
+	HeaderEntry& CurrentHeaderEntry = m_HeaderEntries.back();
+	if (CurrentHeaderEntry.ValueRange.Length == 0)
+	{
+		CurrentHeaderEntry.ValueRange.Offset = (uint32_t)m_HeaderData.size();
+	}
 
+	m_HeaderData.insert(m_HeaderData.end(), Data, &Data[Bytes]);
+	CurrentHeaderEntry.ValueRange.Length += (uint32_t)Bytes;
 	return 0;
 }
 
 static void
-NormalizeUrlPath(const char* Url, size_t UrlLength, std::string& NormalizedUrl)
+NormalizeUrlPath(std::string_view InUrl, std::string& NormalizedUrl)
 {
-	bool LastCharWasSeparator = false;
+	bool		 LastCharWasSeparator = false;
+	const char*	 Url				  = InUrl.data();
+	const size_t UrlLength			  = InUrl.length();
 	for (std::string_view::size_type UrlIndex = 0; UrlIndex < UrlLength; ++UrlIndex)
 	{
 		const char UrlChar	   = Url[UrlIndex];
@@ -233,9 +240,13 @@ HttpRequestParser::OnHeadersComplete()
 {
 	try
 	{
-		if (m_CurrentHeaderValueLength)
+		if (!m_HeaderEntries.empty())
 		{
-			AppendCurrentHeader();
+			HeaderEntry& CurrentHeaderEntry = m_HeaderEntries.back();
+			if (CurrentHeaderEntry.NameRange.Length)
+			{
+				ParseCurrentHeader();
+			}
 		}
 
 		m_KeepAlive = !!http_should_keep_alive(&m_Parser);
@@ -275,21 +286,21 @@ HttpRequestParser::OnHeadersComplete()
 				break;
 		}
 
-		std::string_view Url(m_Url, m_UrlLength);
+		std::string_view FullUrl(GetHeaderSubString(m_UrlRange));
 
-		if (auto QuerySplit = Url.find_first_of('?'); QuerySplit != std::string_view::npos)
+		if (auto QuerySplit = FullUrl.find_first_of('?'); QuerySplit != std::string_view::npos)
 		{
-			m_UrlLength	  = QuerySplit;
-			m_QueryString = m_Url + QuerySplit + 1;
-			m_QueryLength = Url.size() - QuerySplit - 1;
+			m_UrlRange.Length  = uint32_t(QuerySplit);
+			m_QueryStringRange = {.Offset = uint32_t(m_UrlRange.Offset + QuerySplit + 1),
+								  .Length = uint32_t(FullUrl.size() - QuerySplit - 1)};
 		}
 
-		NormalizeUrlPath(m_Url, m_UrlLength, m_NormalizedUrl);
+		NormalizeUrlPath(FullUrl, m_NormalizedUrl);
 
-		if (m_ContentLengthHeaderIndex >= 0)
+		std::string_view Value = GetHeaderValue(m_ContentLengthHeaderIndex);
+		if (!Value.empty())
 		{
-			std::string_view& Value			= m_Headers[m_ContentLengthHeaderIndex].Value;
-			uint64_t		  ContentLength = 0;
+			uint64_t ContentLength = 0;
 			std::from_chars(Value.data(), Value.data() + Value.size(), ContentLength);
 
 			if (ContentLength)
@@ -337,15 +348,11 @@ HttpRequestParser::OnBody(const char* Data, size_t Bytes)
 void
 HttpRequestParser::ResetState()
 {
-	m_HeaderCursor			   = m_HeaderBuffer;
-	m_CurrentHeaderName		   = nullptr;
-	m_CurrentHeaderNameLength  = 0;
-	m_CurrentHeaderValue	   = nullptr;
-	m_CurrentHeaderValueLength = 0;
-	m_Url					   = nullptr;
-	m_UrlLength				   = 0;
-	m_QueryString			   = nullptr;
-	m_QueryLength			   = 0;
+	m_UrlRange		   = {};
+	m_QueryStringRange = {};
+
+	m_HeaderEntries.clear();
+
 	m_ContentLengthHeaderIndex = -1;
 	m_AcceptHeaderIndex		   = -1;
 	m_ContentTypeHeaderIndex   = -1;
@@ -353,7 +360,8 @@ HttpRequestParser::ResetState()
 	m_Expect100Continue		   = false;
 	m_BodyBuffer			   = {};
 	m_BodyPosition			   = 0;
-	m_Headers.clear();
+
+	m_HeaderData.clear();
 	m_NormalizedUrl.clear();
 }
 
diff --git a/src/zenhttp/servers/httpparser.h b/src/zenhttp/servers/httpparser.h
index bdbcab4d9..0d2664ec5 100644
--- a/src/zenhttp/servers/httpparser.h
+++ b/src/zenhttp/servers/httpparser.h
@@ -5,6 +5,8 @@
 #include <zencore/uid.h>
 #include <zenhttp/httpcommon.h>
 
+#include <EASTL/fixed_vector.h>
+
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <http_parser.h>
 ZEN_THIRD_PARTY_INCLUDES_END
@@ -31,73 +33,68 @@ struct HttpRequestParser
 
 	HttpVerb		 RequestVerb() const { return m_RequestVerb; }
 	bool			 IsKeepAlive() const { return m_KeepAlive; }
-	std::string_view Url() const { return m_NormalizedUrl.empty() ? std::string_view(m_Url, m_UrlLength) : m_NormalizedUrl; }
-	std::string_view QueryString() const { return std::string_view(m_QueryString, m_QueryLength); }
+	std::string_view Url() const { return m_NormalizedUrl.empty() ? GetHeaderSubString(m_UrlRange) : m_NormalizedUrl; }
+	std::string_view QueryString() const { return GetHeaderSubString(m_QueryStringRange); }
 	IoBuffer		 Body() { return m_BodyBuffer; }
 
-	inline HttpContentType ContentType()
-	{
-		if (m_ContentTypeHeaderIndex < 0)
-		{
-			return HttpContentType::kUnknownContentType;
-		}
-
-		return ParseContentType(m_Headers[m_ContentTypeHeaderIndex].Value);
-	}
+	inline HttpContentType ContentType() { return ParseContentType(GetHeaderValue(m_ContentTypeHeaderIndex)); }
 
-	inline HttpContentType AcceptType()
-	{
-		if (m_AcceptHeaderIndex < 0)
-		{
-			return HttpContentType::kUnknownContentType;
-		}
-
-		return ParseContentType(m_Headers[m_AcceptHeaderIndex].Value);
-	}
+	inline HttpContentType AcceptType() { return ParseContentType(GetHeaderValue(m_AcceptHeaderIndex)); }
 
 	Oid SessionId() const { return m_SessionId; }
 	int RequestId() const { return m_RequestId; }
 
-	std::string_view RangeHeader() const { return m_RangeHeaderIndex != -1 ? m_Headers[m_RangeHeaderIndex].Value : std::string_view(); }
+	std::string_view RangeHeader() const { return GetHeaderValue(m_RangeHeaderIndex); }
 
 private:
+	struct HeaderRange
+	{
+		uint32_t Offset = 0;
+		uint32_t Length = 0;
+	};
+
 	struct HeaderEntry
 	{
-		HeaderEntry() = default;
+		HeaderRange NameRange;
+		HeaderRange ValueRange;
+	};
 
-		HeaderEntry(std::string_view InName, std::string_view InValue) : Name(InName), Value(InValue) {}
+	inline std::string_view GetHeaderValue(int8_t HeaderIndex) const
+	{
+		if (HeaderIndex == -1)
+		{
+			return {};
+		}
+		ZEN_ASSERT(size_t(HeaderIndex) < m_HeaderEntries.size());
+		return GetHeaderSubString(m_HeaderEntries[HeaderIndex].ValueRange);
+	}
 
-		std::string_view Name;
-		std::string_view Value;
-	};
+	std::string_view GetHeaderSubString(const HeaderRange& Range) const
+	{
+		ZEN_ASSERT_SLOW(Range.Offset + Range.Length <= m_HeaderData.size());
+		return std::string_view(m_HeaderData.begin(), m_HeaderData.size()).substr(Range.Offset, Range.Length);
+	}
 
-	HttpRequestParserCallbacks& m_Connection;
-	char*						m_HeaderCursor			   = m_HeaderBuffer;
-	char*						m_Url					   = nullptr;
-	size_t						m_UrlLength				   = 0;
-	char*						m_QueryString			   = nullptr;
-	size_t						m_QueryLength			   = 0;
-	char*						m_CurrentHeaderName		   = nullptr;  // Used while parsing headers
-	size_t						m_CurrentHeaderNameLength  = 0;
-	char*						m_CurrentHeaderValue	   = nullptr;  // Used while parsing headers
-	size_t						m_CurrentHeaderValueLength = 0;
-	std::vector<HeaderEntry>	m_Headers;
-	int8_t						m_ContentLengthHeaderIndex;
-	int8_t						m_AcceptHeaderIndex;
-	int8_t						m_ContentTypeHeaderIndex;
-	int8_t						m_RangeHeaderIndex;
-	HttpVerb					m_RequestVerb;
-	std::atomic_bool			m_KeepAlive{false};
-	bool						m_Expect100Continue = false;
-	int							m_RequestId			= -1;
-	Oid							m_SessionId{};
-	IoBuffer					m_BodyBuffer;
-	uint64_t					m_BodyPosition = 0;
-	http_parser					m_Parser;
-	char						m_HeaderBuffer[1024];
-	std::string					m_NormalizedUrl;
-
-	void AppendCurrentHeader();
+	HttpRequestParserCallbacks&			 m_Connection;
+	HeaderRange							 m_UrlRange;
+	HeaderRange							 m_QueryStringRange;
+	eastl::fixed_vector<HeaderEntry, 16> m_HeaderEntries;
+	int8_t								 m_ContentLengthHeaderIndex;
+	int8_t								 m_AcceptHeaderIndex;
+	int8_t								 m_ContentTypeHeaderIndex;
+	int8_t								 m_RangeHeaderIndex;
+	HttpVerb							 m_RequestVerb;
+	std::atomic_bool					 m_KeepAlive{false};
+	bool								 m_Expect100Continue = false;
+	int									 m_RequestId		 = -1;
+	Oid									 m_SessionId{};
+	IoBuffer							 m_BodyBuffer;
+	uint64_t							 m_BodyPosition = 0;
+	http_parser							 m_Parser;
+	eastl::fixed_vector<char, 512>		 m_HeaderData;
+	std::string							 m_NormalizedUrl;
+
+	void ParseCurrentHeader();
 
 	int OnMessageBegin();
 	int OnUrl(const char* Data, size_t Bytes);