Merge branch 'gc' of https://github.com/EpicGames/zen into gc

10 files changed, 228 insertions, 72 deletions

diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp
index c4694733c..aafb365f3 100644
--- a/zencore/compactbinary.cpp
+++ b/zencore/compactbinary.cpp
@@ -16,6 +16,10 @@
 #include <fmt/format.h>
 #include <string_view>
 
+#if ZEN_PLATFORM_WINDOWS
+#	include <zencore/windows.h>
+#endif
+
 #if ZEN_WITH_TESTS
 #	include <json11.hpp>
 #	include <zencore/compactbinarybuilder.h>
@@ -42,6 +46,18 @@ IsLeapYear(int Year)
 	return false;
 }
 
+DateTime
+DateTime::Now()
+{
+#if ZEN_PLATFORM_WINDOWS
+	FILETIME SysTime;
+	GetSystemTimeAsFileTime(&SysTime);
+	return DateTime{504911232000000000ull + (uint64_t(SysTime.dwHighDateTime) << 32) | SysTime.dwLowDateTime};
+#else
+#	error Needs implementation
+#endif
+}
+
 void
 DateTime::Set(int Year, int Month, int Day, int Hour, int Minute, int Second, int MilliSecond)
 {
@@ -96,31 +112,31 @@ TimeSpan::ToString(const char* Format) const
 			switch (*Format)
 			{
 				case 'd':
-					Result.Append("%i"_format(GetDays()));
+					Result.Append("{}"_format(GetDays()));
 					break;
 				case 'D':
-					Result.Append("%08i"_format(GetDays()));
+					Result.Append("{:08}"_format(GetDays()));
 					break;
 				case 'h':
-					Result.Append("%02i"_format(GetHours()));
+					Result.Append("{:02}"_format(GetHours()));
 					break;
 				case 'm':
-					Result.Append("%02i"_format(GetMinutes()));
+					Result.Append("{:02}"_format(GetMinutes()));
 					break;
 				case 's':
-					Result.Append("%02i"_format(GetSeconds()));
+					Result.Append("{:02}"_format(GetSeconds()));
 					break;
 				case 'f':
-					Result.Append("%03i"_format(GetFractionMilli()));
+					Result.Append("{:03}"_format(GetFractionMilli()));
 					break;
 				case 'u':
-					Result.Append("%06i"_format(GetFractionMicro()));
+					Result.Append("{:06}"_format(GetFractionMicro()));
 					break;
 				case 't':
-					Result.Append("%07i"_format(GetFractionTicks()));
+					Result.Append("{:07}"_format(GetFractionTicks()));
 					break;
 				case 'n':
-					Result.Append("%09i"_format(GetFractionNano()));
+					Result.Append("{:09}"_format(GetFractionNano()));
 					break;
 				default:
 					Result.Append(*Format);
@@ -247,8 +263,8 @@ DateTime::ToString(const char* Format) const
 {
 	using namespace fmt::literals;
 
-	StringBuilder<32> Result;
-	int				  Year, Month, Day;
+	ExtendableStringBuilder<32> Result;
+	int							Year, Month, Day;
 
 	GetDate(Year, Month, Day);
 
@@ -263,32 +279,32 @@ DateTime::ToString(const char* Format) const
 					// case 'a': Result.Append(IsMorning() ? TEXT("am") : TEXT("pm")); break;
 					// case 'A': Result.Append(IsMorning() ? TEXT("AM") : TEXT("PM")); break;
 					case 'd':
-						Result.Append("%02i"_format(Day));
+						Result.Append("{:02}"_format(Day));
 						break;
 						// case 'D': Result.Appendf(TEXT("%03i"), GetDayOfYear()); break;
 					case 'm':
-						Result.Append("%02i"_format(Month));
+						Result.Append("{:02}"_format(Month));
 						break;
 					case 'y':
-						Result.Append("%02i"_format(Year % 100));
+						Result.Append("{:02}"_format(Year % 100));
 						break;
 					case 'Y':
-						Result.Append("%04i"_format(Year % 100));
+						Result.Append("{:04}"_format(Year));
 						break;
 					case 'h':
-						Result.Append("%02i"_format(GetHour12()));
+						Result.Append("{:02}"_format(GetHour12()));
 						break;
 					case 'H':
-						Result.Append("%02i"_format(GetHour()));
+						Result.Append("{:02}"_format(GetHour()));
 						break;
 					case 'M':
-						Result.Append("%02i"_format(GetMinute()));
+						Result.Append("{:02}"_format(GetMinute()));
 						break;
 					case 'S':
-						Result.Append("%02i"_format(GetSecond()));
+						Result.Append("{:02}"_format(GetSecond()));
 						break;
 					case 's':
-						Result.Append("%03i"_format(GetMillisecond()));
+						Result.Append("{:03}"_format(GetMillisecond()));
 						break;
 					default:
 						Result.Append(*Format);
@@ -1659,10 +1675,10 @@ private:
 	}
 
 private:
-	StringBuilderBase& Builder;
-	StringBuilder<32>  NewLineAndIndent;
-	bool			   NeedsComma{false};
-	bool			   NeedsNewLine{false};
+	StringBuilderBase&			Builder;
+	ExtendableStringBuilder<32> NewLineAndIndent;
+	bool						NeedsComma{false};
+	bool						NeedsNewLine{false};
 };
 
 void
@@ -1902,6 +1918,33 @@ TEST_CASE("uson.json")
 		CHECK(DoubleValue == 0);
 	}
 }
+
+TEST_CASE("uson.datetime")
+{
+	using namespace std::literals;
+
+	{
+		DateTime D1600(1601, 1, 1);
+		CHECK_EQ(D1600.GetYear(), 1601);
+		CHECK_EQ(D1600.GetMonth(), 1);
+		CHECK_EQ(D1600.GetDay(), 1);
+		CHECK_EQ(D1600.GetHour(), 0);
+		CHECK_EQ(D1600.GetMinute(), 0);
+		CHECK_EQ(D1600.GetSecond(), 0);
+
+		CHECK_EQ(D1600.ToIso8601(), "1601-01-01T00:00:00.000Z"sv);
+	}
+
+	{
+		DateTime D72(1972, 2, 23, 17, 30, 10);
+		CHECK_EQ(D72.GetYear(), 1972);
+		CHECK_EQ(D72.GetMonth(), 2);
+		CHECK_EQ(D72.GetDay(), 23);
+		CHECK_EQ(D72.GetHour(), 17);
+		CHECK_EQ(D72.GetMinute(), 30);
+		CHECK_EQ(D72.GetSecond(), 10);
+	}
+}
 #endif
 
 }  // namespace zen
diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h
index ab01402f8..06331c510 100644
--- a/zencore/include/zencore/compactbinary.h
+++ b/zencore/include/zencore/compactbinary.h
@@ -43,6 +43,8 @@ public:
 
 	inline uint64_t GetTicks() const { return Ticks; }
 
+	static DateTime Now();
+
 	int	 GetYear() const;
 	int	 GetMonth() const;
 	int	 GetDay() const;
diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
index db462e238..8679511c9 100644
--- a/zencore/include/zencore/iobuffer.h
+++ b/zencore/include/zencore/iobuffer.h
@@ -357,6 +357,9 @@ public:
 	[[nodiscard]] inline ZenContentType GetContentType() const { return m_Core->GetContentType(); }
 	[[nodiscard]] ZENCORE_API bool		GetFileReference(IoBufferFileReference& OutRef) const;
 
+	template<typename T>
+	[[nodiscard]] const T* Data() const { return reinterpret_cast<const T*>(m_Core->DataPointer()); }
+
 private:
 	RefPtr<IoBufferCore> m_Core = new IoBufferCore;
 
diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h
index a94e063a4..630341056 100644
--- a/zencore/include/zencore/string.h
+++ b/zencore/include/zencore/string.h
@@ -632,7 +632,7 @@ HashStringAsLowerDjb2(const std::string_view& InString)
 {
 	uint32_t HashValue = 5381;
 
-	for (int CurChar : InString)
+	for (uint8_t CurChar : InString)
 	{
 		CurChar -= ((CurChar - 'A') <= ('Z' - 'A')) * ('A' - 'a');	// this should be compiled into branchless logic
 		HashValue = HashValue * 33 + CurChar;
@@ -650,7 +650,7 @@ ToLower(const std::string_view& InString)
 
 	for (char& CurChar : Out)
 	{
-		CurChar -= ((CurChar - 'A') <= ('Z' - 'A')) * ('A' - 'a');	// this should be compiled into branchless logic
+		CurChar -= (uint8_t(CurChar - 'A') <= ('Z' - 'A')) * ('A' - 'a');	// this should be compiled into branchless logic
 	}
 
 	return Out;
diff --git a/zencore/string.cpp b/zencore/string.cpp
index 8e7921bb6..43381aa5d 100644
--- a/zencore/string.cpp
+++ b/zencore/string.cpp
@@ -936,6 +936,12 @@ TEST_CASE("string")
 		CHECK(HashStringAsLowerDjb2("aBCd"sv) == HashStringDjb2(ToLower("aBCd"sv)));
 	}
 
+	SUBCASE("tolower") 
+	{
+		CHECK_EQ(ToLower("te!st"sv), "te!st"sv); 
+		CHECK_EQ(ToLower("TE%St"sv), "te%st"sv);
+	}
+
 	SUBCASE("ForEachStrTok")
 	{
 		const auto		   Tokens			  = "here,is,my,different,tokens"sv;
diff --git a/zenhttp/httpasio.cpp b/zenhttp/httpasio.cpp
index 12fbdb61e..ce6503a96 100644
--- a/zenhttp/httpasio.cpp
+++ b/zenhttp/httpasio.cpp
@@ -2,14 +2,17 @@
 
 #include "httpasio.h"
 
+#include <zencore/logging.h>
 #include <zenhttp/httpserver.h>
 
-#include <conio.h>
-#include <zencore/logging.h>
+#include <deque>
+#include <memory_resource>
 
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <conio.h>
 #include <http_parser.h>
 #include <asio.hpp>
-#include <deque>
+ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen::asio_http {
 
@@ -24,6 +27,7 @@ struct HttpServerConnection;
 static constinit uint32_t HashContentLength = HashStringAsLowerDjb2("Content-Length"sv);
 static constinit uint32_t HashContentType	= HashStringAsLowerDjb2("Content-Type"sv);
 static constinit uint32_t HashAccept		= HashStringAsLowerDjb2("Accept"sv);
+static constinit uint32_t HashExpect		= HashStringAsLowerDjb2("Expect"sv);
 static constinit uint32_t HashSession		= HashStringAsLowerDjb2("UE-Session"sv);
 static constinit uint32_t HashRequest		= HashStringAsLowerDjb2("UE-Request"sv);
 
@@ -31,7 +35,7 @@ inline spdlog::logger&
 InitLogger()
 {
 	spdlog::logger& Logger = logging::Get("asio");
-	//	Logger.set_level(spdlog::level::trace);
+	//Logger.set_level(spdlog::level::trace);
 	return Logger;
 }
 
@@ -144,7 +148,6 @@ private:
 	};
 
 	HttpServerConnection&	 m_Connection;
-	char					 m_HeaderBuffer[512];
 	char*					 m_HeaderCursor				= m_HeaderBuffer;
 	char*					 m_Url						= nullptr;
 	size_t					 m_UrlLength				= 0;
@@ -158,13 +161,15 @@ private:
 	int8_t					 m_ContentLengthHeaderIndex;
 	int8_t					 m_AcceptHeaderIndex;
 	int8_t					 m_ContentTypeHeaderIndex;
-	int						 m_RequestId = -1;
+	HttpVerb				 m_RequestVerb;
+	bool					 m_KeepAlive		 = false;
+	bool					 m_Expect100Continue = false;
+	int						 m_RequestId		 = -1;
 	Oid						 m_SessionId{};
 	IoBuffer				 m_BodyBuffer;
-	uint64_t				 m_BodyPosition;
+	uint64_t				 m_BodyPosition = 0;
 	http_parser				 m_Parser;
-	HttpVerb				 m_RequestVerb;
-	bool					 m_KeepAlive = false;
+	char					 m_HeaderBuffer[512];
 
 	void AppendInputBytes(const char* Data, size_t Bytes);
 	void AppendCurrentHeader();
@@ -327,6 +332,7 @@ HttpServerConnection::HttpServerConnection(HttpAsioServerImpl& Server, std::uniq
 
 HttpServerConnection::~HttpServerConnection()
 {
+	ZEN_TRACE("destroying connection #{}", m_ConnectionId);
 }
 
 void
@@ -701,6 +707,18 @@ HttpRequest::AppendCurrentHeader()
 	{
 		std::from_chars(HeaderValue.data(), HeaderValue.data() + HeaderValue.size(), m_RequestId);
 	}
+	else if (HeaderHash == HashExpect)
+	{
+		if (HeaderValue == "100-continue"sv)
+		{
+			// We don't currently do anything with this
+			m_Expect100Continue = true;
+		}
+		else
+		{
+			ZEN_INFO("Unexpected expect - Expect: {}", HeaderValue);
+		}
+	}
 
 	m_Headers.emplace_back(HeaderName, HeaderValue);
 }
@@ -744,10 +762,6 @@ HttpRequest::OnHeadersComplete()
 		{
 			m_BodyBuffer = IoBuffer(ContentLength);
 		}
-		else
-		{
-			m_BodyBuffer = {};
-		}
 
 		m_BodyBuffer.SetContentType(ContentType());
 
@@ -827,13 +841,14 @@ HttpRequest::ResetState()
 	m_CurrentHeaderName		   = nullptr;
 	m_Url					   = nullptr;
 	m_UrlLength				   = 0;
+	m_QueryString			   = nullptr;
+	m_QueryLength			   = 0;
 	m_ContentLengthHeaderIndex = -1;
 	m_AcceptHeaderIndex		   = -1;
 	m_ContentTypeHeaderIndex   = -1;
-
-	m_BodyBuffer   = {};
-	m_BodyPosition = 0;
-
+	m_Expect100Continue		   = false;
+	m_BodyBuffer			   = {};
+	m_BodyPosition			   = 0;
 	m_Headers.clear();
 }
 
@@ -1115,6 +1130,7 @@ HttpAsioServerImpl::RouteRequest(std::string_view Url)
 namespace zen {
 HttpAsioServer::HttpAsioServer() : m_Impl(std::make_unique<asio_http::HttpAsioServerImpl>())
 {
+	ZEN_DEBUG("Request object size: {} ({:#x})", sizeof(asio_http::HttpRequest), sizeof(asio_http::HttpRequest));
 }
 
 HttpAsioServer::~HttpAsioServer()
diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp
index f3568bbd1..cdf9e0a39 100644
--- a/zenhttp/httpsys.cpp
+++ b/zenhttp/httpsys.cpp
@@ -341,8 +341,6 @@ HttpMessageResponseRequest::~HttpMessageResponseRequest()
 void
 HttpMessageResponseRequest::InitializeForPayload(uint16_t ResponseCode, std::span<IoBuffer> BlobList)
 {
-	m_ResponseCode = ResponseCode;
-
 	const uint32_t ChunkCount = gsl::narrow<uint32_t>(BlobList.size());
 
 	m_HttpDataChunks.reserve(ChunkCount);
@@ -407,6 +405,16 @@ HttpMessageResponseRequest::InitializeForPayload(uint16_t ResponseCode, std::spa
 
 	m_RemainingChunkCount = gsl::narrow<uint32_t>(m_HttpDataChunks.size());
 	m_TotalDataSize		  = LocalDataSize;
+
+	if (m_TotalDataSize == 0 && ResponseCode == 200)
+	{
+		// Some HTTP clients really don't like empty responses unless a 204 response is sent
+		m_ResponseCode = uint16_t(HttpResponseCode::NoContent);
+	}
+	else
+	{
+		m_ResponseCode = ResponseCode;
+	}
 }
 
 void
diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp
index 9c9b73042..fec6ca17c 100644
--- a/zenserver-test/zenserver-test.cpp
+++ b/zenserver-test/zenserver-test.cpp
@@ -1036,10 +1036,11 @@ TEST_CASE("project.basic")
 
 			zen::CbObject Op = OpWriter.Save();
 
-			zen::BinaryWriter MemOut;
 			zen::CbPackage	  OpPackage(Op);
 			OpPackage.AddAttachment(Attach);
-			OpPackage.Save(MemOut);
+
+			zen::BinaryWriter MemOut;
+			legacy::SaveCbPackage(OpPackage, MemOut);
 
 			{
 				zen::StringBuilder<64> PostUri;
@@ -1829,7 +1830,7 @@ TEST_CASE("zcache.policy")
 		{
 			cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key)},
 											cpr::Header{{"Accept", "application/x-ue-cbpkg"}});
-			CHECK(Result.status_code == 200);
+			CHECK(IsHttpSuccessCode(Result.status_code));
 			CHECK(Result.text.size() == 0);
 		}
 
@@ -1837,7 +1838,7 @@ TEST_CASE("zcache.policy")
 		{
 			cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key)},
 											cpr::Header{{"Accept", "application/x-ue-cbobject"}});
-			CHECK(Result.status_code == 200);
+			CHECK(IsHttpSuccessCode(Result.status_code));
 			CHECK(Result.text.size() == 0);
 		}
 
@@ -1845,7 +1846,7 @@ TEST_CASE("zcache.policy")
 		{
 			cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key, PayloadId)},
 											cpr::Header{{"Accept", "application/x-ue-comp"}});
-			CHECK(Result.status_code == 200);
+			CHECK(IsHttpSuccessCode(Result.status_code));
 			CHECK(Result.text.size() == 0);
 		}
 	}
@@ -1873,7 +1874,7 @@ TEST_CASE("zcache.policy")
 		{
 			cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key)},
 											cpr::Header{{"Accept", "application/octet-stream"}});
-			CHECK(Result.status_code == 200);
+			CHECK(IsHttpSuccessCode(Result.status_code));
 			CHECK(Result.text.size() == 0);
 		}
 	}
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 39348b436..d54d03450 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -1613,6 +1613,8 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
 						}
 
 						m_ProjectStore->DeleteProject(ProjectId);
+
+						return Req.ServerRequest().WriteResponse(HttpResponseCode::NoContent);
 					}
 					break;
 			}
diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp
index 807bba3b3..7ec91cfa5 100644
--- a/zenstore/CAS.cpp
+++ b/zenstore/CAS.cpp
@@ -5,6 +5,9 @@
 #include "compactcas.h"
 #include "filecas.h"
 
+#include <zencore/compactbinary.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryvalidation.h>
 #include <zencore/except.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
@@ -105,6 +108,18 @@ private:
 	CasContainerStrategy m_TinyStrategy;
 	CasContainerStrategy m_SmallStrategy;
 	FileCasStrategy		 m_LargeStrategy;
+	CbObject			 m_ManifestObject;
+
+	enum class StorageScheme
+	{
+		Legacy		   = 0,
+		WithCbManifest = 1
+	};
+
+	StorageScheme m_StorageScheme = StorageScheme::Legacy;
+
+	bool OpenOrCreateManifest();
+	void UpdateManifest(bool IsNewStore);
 };
 
 CasImpl::CasImpl() : m_TinyStrategy(m_Config), m_SmallStrategy(m_Config), m_LargeStrategy(m_Config)
@@ -127,39 +142,99 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig)
 	std::filesystem::create_directories(m_Config.RootDirectory);
 
 	// Open or create manifest
-	//
-	//  The manifest is not currently fully implemented. The goal is to
-	//  use it for recovery and configuration
 
+	const bool IsNewStore = OpenOrCreateManifest();
+
+	// Initialize payload storage
+
+	m_TinyStrategy.Initialize("tobs", 16, IsNewStore);
+	m_SmallStrategy.Initialize("sobs", 4096, IsNewStore);
+}
+
+bool
+CasImpl::OpenOrCreateManifest()
+{
 	bool IsNewStore = false;
 
-	{
-		std::filesystem::path ManifestPath = m_Config.RootDirectory;
-		ManifestPath /= ".ucas_root";
+	std::filesystem::path ManifestPath = m_Config.RootDirectory;
+	ManifestPath /= ".ucas_root";
 
-		std::error_code Ec;
-		BasicFile		Marker;
-		Marker.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec);
+	std::error_code Ec;
+	BasicFile		ManifestFile;
+	ManifestFile.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec);
 
-		if (Ec)
+	bool ManifestIsOk = false;
+
+	if (Ec)
+	{
+		if (Ec == std::errc::no_such_file_or_directory)
 		{
 			IsNewStore = true;
+		}
+	}
+	else
+	{
+		IoBuffer ManifestBuffer = ManifestFile.ReadAll();
+		ManifestFile.Close();
 
-			ExtendableStringBuilder<128> manifest;
-			manifest.Append("#CAS_ROOT\n");
-			manifest.Append("ID=");
-			zen::Oid id = zen::Oid::NewOid();
-			id.ToString(manifest);
-
-			Marker.Open(ManifestPath.c_str(), /* IsCreate */ true);
-			Marker.Write(manifest.c_str(), (DWORD)manifest.Size(), 0);
+		if (ManifestBuffer.Size() > 0 && ManifestBuffer.Data<uint8_t>()[0] == '#')
+		{
+			// Old-style manifest, does not contain any useful information, so we may as well update it
+		}
+		else
+		{
+			CbObject		Manifest{SharedBuffer(ManifestBuffer)};
+			CbValidateError ValidationResult = ValidateCompactBinary(ManifestBuffer, CbValidateMode::All);
+
+			if (ValidationResult == CbValidateError::None)
+			{
+				if (Manifest["id"])
+				{
+					ManifestIsOk = true;
+				}
+			}
+			else
+			{
+				ZEN_ERROR("Store manifest validation failed: {:#x}, will generate new manifest to recover", ValidationResult);
+			}
+
+			if (ManifestIsOk)
+			{
+				m_ManifestObject = std::move(Manifest);
+			}
 		}
 	}
 
-	// Initialize payload storage
+	if (!ManifestIsOk)
+	{
+		UpdateManifest(true);
+	}
 
-	m_TinyStrategy.Initialize("tobs", 16, IsNewStore);
-	m_SmallStrategy.Initialize("sobs", 4096, IsNewStore);
+	return IsNewStore;
+}
+
+void
+CasImpl::UpdateManifest(bool IsNewStore)
+{
+	if (!m_ManifestObject)
+	{
+		CbObjectWriter Cbo;
+		Cbo << "id" << zen::Oid::NewOid() << "created" << DateTime::Now();
+		m_ManifestObject = Cbo.Save();
+	}
+
+	// Write manifest to file
+
+	std::filesystem::path ManifestPath = m_Config.RootDirectory;
+	ManifestPath /= ".ucas_root";
+
+	// This will throw on failure
+
+	ZEN_TRACE("Writing new manifest to '{}'", ManifestPath);
+
+	BasicFile Marker;
+	Marker.Open(ManifestPath.c_str(), /* IsCreate */ true);
+	Marker.Write(m_ManifestObject.GetBuffer(), 0);
 }
 
 CasStore::InsertResult