Merge branch 'main' of https://github.com/EpicGames/zen

8 files changed, 148 insertions, 133 deletions

diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 07866d4f0..45bbe062b 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -77,7 +77,7 @@ struct PutRequestData
 };
 
 namespace {
-	static constexpr std::string_view HttpZCacheRPCPrefix = "$rpc"sv;
+	static constinit std::string_view HttpZCacheRPCPrefix = "$rpc"sv;
 
 	struct HttpRequestData
 	{
@@ -87,8 +87,8 @@ namespace {
 		std::optional<IoHash>	   ValueContentId;
 	};
 
-	const char* ValidNamespaceNameCharacters = "abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ";
-	const char* ValidBucketNameCharacters	 = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+	constinit AsciiSet ValidNamespaceNameCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
+	constinit AsciiSet ValidBucketNameCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
 
 	std::optional<std::string> GetValidNamespaceName(std::string_view Name)
 	{
@@ -104,7 +104,7 @@ namespace {
 			return {};
 		}
 
-		if (Name.find_first_not_of(ValidNamespaceNameCharacters) != std::string::npos)
+		if (!AsciiSet::HasOnly(Name, ValidNamespaceNameCharactersSet))
 		{
 			ZEN_WARN("Namespace '{}' is invalid, invalid characters detected", Name);
 			return {};
@@ -120,11 +120,13 @@ namespace {
 			ZEN_WARN("Bucket name is invalid, empty bucket name is not allowed");
 			return {};
 		}
-		if (Name.find_first_not_of(ValidBucketNameCharacters) != std::string::npos)
+
+		if (!AsciiSet::HasOnly(Name, ValidBucketNameCharactersSet))
 		{
 			ZEN_WARN("Bucket name '{}' is invalid, invalid characters detected", Name);
 			return {};
 		}
+
 		return ToLower(Name);
 	}
 
diff --git a/zenserver/frontend/frontend.cpp b/zenserver/frontend/frontend.cpp
index 6d576876f..842587708 100644
--- a/zenserver/frontend/frontend.cpp
+++ b/zenserver/frontend/frontend.cpp
@@ -203,51 +203,40 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
 	// Dismiss if the URI contains .. anywhere to prevent arbitrary file reads
 	if (Uri.find("..") != Uri.npos)
 	{
-		Request.WriteResponse(HttpResponseCode::Forbidden);
-		return;
+		return Request.WriteResponse(HttpResponseCode::Forbidden);
 	}
 
 	// Map the file extension to a MIME type. To keep things constrained, only a
-	// small subset of file extensions is allowed.
-	HttpContentType ContentType = HttpContentType::kCOUNT;
-	size_t			DotIndex	= Uri.rfind(".");
-	if (DotIndex != Uri.npos)
+	// small subset of file extensions is allowed
+
+	HttpContentType ContentType = HttpContentType::kUnknownContentType;
+
+	if (const size_t DotIndex = Uri.rfind("."); DotIndex != Uri.npos)
 	{
-		const std::string_view DotExt = Uri.substr(DotIndex);
-		if (DotExt == ".html")
-			ContentType = HttpContentType::kHTML;
-		else if (DotExt == ".js")
-			ContentType = HttpContentType::kJSON;
-		else if (DotExt == ".css")
-			ContentType = HttpContentType::kCSS;
-		else if (DotExt == ".png")
-			ContentType = HttpContentType::kPNG;
-		else if (DotExt == ".ico")
-			ContentType = HttpContentType::kIcon;
+		const std::string_view DotExt = Uri.substr(DotIndex + 1);
+
+		ContentType = ParseContentType(DotExt);
 	}
 
-	if (ContentType == HttpContentType::kCOUNT)
+	if (ContentType == HttpContentType::kUnknownContentType)
 	{
-		Request.WriteResponse(HttpResponseCode::Forbidden);
-		return;
+		return Request.WriteResponse(HttpResponseCode::Forbidden);
 	}
 
 	// The given content directory overrides any zip-fs discovered in the binary
 	if (!m_Directory.empty())
 	{
 		FileContents File = ReadFile(m_Directory / Uri);
+
 		if (!File.ErrorCode)
 		{
-			Request.WriteResponse(HttpResponseCode::OK, ContentType, File.Data[0]);
-			return;
+			return Request.WriteResponse(HttpResponseCode::OK, ContentType, File.Data[0]);
 		}
 	}
 
-	IoBuffer FileBuffer = m_ZipFs.GetFile(Uri);
-	if (FileBuffer)
+	if (IoBuffer FileBuffer = m_ZipFs.GetFile(Uri))
 	{
-		Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer);
-		return;
+		return Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer);
 	}
 
 	Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv);
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index d18ae9e1a..7a55911da 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -450,7 +450,7 @@ ProjectStore::Oplog::IterateOplog(std::function<void(CbObject)>&& Handler)
 }
 
 std::optional<CbObject>
-ProjectStore::Oplog::GetOplog(const Oid& Key)
+ProjectStore::Oplog::GetOpByKey(const Oid& Key)
 {
 	RwLock::SharedLockScope _(m_OplogLock);
 
@@ -465,6 +465,19 @@ ProjectStore::Oplog::GetOplog(const Oid& Key)
 	return {};
 }
 
+std::optional<CbObject>
+ProjectStore::Oplog::GetOpByIndex(int Index)
+{
+	RwLock::SharedLockScope _(m_OplogLock);
+
+	if (const auto AddressEntryIt = m_OpAddressMap.find(Index); AddressEntryIt != m_OpAddressMap.end())
+	{
+		return m_Storage->GetOp(AddressEntryIt->second);
+	}
+
+	return {};
+}
+
 bool
 ProjectStore::Oplog::AddFileMapping(Oid FileId, IoHash Hash, std::string_view ServerPath, std::string_view ClientPath)
 {
@@ -809,7 +822,7 @@ ProjectStore::Project::DiscoverOplogs()
 
 	for (const std::filesystem::path& DirPath : DirContent.Directories)
 	{
-		OpenOplog(PathToUtf8(DirPath.stem()));
+		OpenOplog(PathToUtf8(DirPath.filename()));
 	}
 }
 
@@ -893,7 +906,7 @@ ProjectStore::DiscoverProjects()
 
 	for (const std::filesystem::path& DirPath : DirContent.Directories)
 	{
-		std::string DirName = PathToUtf8(DirPath.stem());
+		std::string DirName = PathToUtf8(DirPath.filename());
 		Project*	Project = OpenProject(DirName);
 
 		if (Project)
@@ -917,8 +930,6 @@ ProjectStore::IterateProjects(std::function<void(Project& Prj)>&& Fn)
 void
 ProjectStore::Flush()
 {
-	// TODO
-
 	RwLock::SharedLockScope _(m_ProjectsLock);
 
 	for (auto& Kv : m_Projects)
@@ -1057,6 +1068,8 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 	m_Router.AddPattern("chunk", "([[:xdigit:]]{24})");
 	m_Router.AddPattern("hash", "([[:xdigit:]]{40})");
 
+	// This would ideally just be the response for the root /prj endpoint but this is
+	// currently not possible for (arbitrary, external) technical reasons
 	m_Router.RegisterRoute(
 		"list",
 		[this](HttpRouterRequest& Req) {
@@ -1064,6 +1077,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 
 			CbWriter Response;
 			Response.BeginArray();
+
 			m_ProjectStore->IterateProjects([&Response](ProjectStore::Project& Prj) {
 				Response.BeginObject();
 				Response << "Id"sv << Prj.Identifier;
@@ -1265,7 +1279,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 
 			ProjectStore::Oplog& Log = *FoundLog;
 
-			Oid Obj = Oid::FromHexString(ChunkId);
+			const Oid Obj = Oid::FromHexString(ChunkId);
 
 			IoBuffer Chunk = Log.FindChunk(Obj);
 			if (!Chunk)
@@ -1606,12 +1620,88 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 
 	m_Router.RegisterRoute(
 		"{project}/oplog/{log}/{op}",
-		[](HttpRouterRequest& Req) {
+		[this](HttpRouterRequest& Req) {
 			HttpServerRequest& HttpReq = Req.ServerRequest();
 
-			// TODO: look up op and respond with the payload!
+			const std::string& ProjectId  = Req.GetCapture(1);
+			const std::string& OplogId	  = Req.GetCapture(2);
+			const std::string& OpIdString = Req.GetCapture(3);
+
+			ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId);
+
+			if (FoundLog == nullptr)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+
+			ProjectStore::Oplog& Oplog = *FoundLog;
+
+			if (const std::optional<int32_t> OpId = zen::ParseInt<uint32_t>(OpIdString))
+			{
+				if (std::optional<CbObject> MaybeOp = Oplog.GetOpByIndex(OpId.value()))
+				{
+					CbObject& Op = MaybeOp.value();
+					if (Req.ServerRequest().AcceptContentType() == ZenContentType::kCbPackage)
+					{
+						CbPackage Package;
+						Package.SetObject(Op);
+
+						Op.IterateAttachments([&](CbFieldView FieldView) {
+							const IoHash AttachmentHash = FieldView.AsAttachment();
+							IoBuffer	 Payload		= m_CidStore.FindChunkByCid(AttachmentHash);
+
+							// We force this for now as content type is not consistently tracked (will
+							// be fixed in CidStore refactor)
+							Payload.SetContentType(ZenContentType::kCompressedBinary);
+
+							if (Payload)
+							{
+								switch (Payload.GetContentType())
+								{
+									case ZenContentType::kCbObject:
+										if (CbObject Object = LoadCompactBinaryObject(Payload))
+										{
+											Package.AddAttachment(CbAttachment(Object));
+										}
+										else
+										{
+											// Error - malformed object
+
+											ZEN_ERROR("malformed object returned for {}", AttachmentHash);
+										}
+										break;
+
+									case ZenContentType::kCompressedBinary:
+										if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload)))
+										{
+											Package.AddAttachment(CbAttachment(Compressed));
+										}
+										else
+										{
+											// Error - not compressed!
+
+											ZEN_ERROR("invalid compressed binary returned for {}", AttachmentHash);
+										}
+										break;
+
+									default:
+										Package.AddAttachment(CbAttachment(SharedBuffer(Payload)));
+										break;
+								}
+							}
+						});
+
+						return HttpReq.WriteResponse(HttpResponseCode::Accepted, Package);
+					}
+					else
+					{
+						// Client cannot accept a package, so we only send the core object
+						return HttpReq.WriteResponse(HttpResponseCode::Accepted, Op);
+					}
+				}
+			}
 
-			HttpReq.WriteResponse(HttpResponseCode::Accepted, HttpContentType::kText, u8"yeee"sv);
+			return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 		},
 		HttpVerb::kGet);
 
@@ -1718,7 +1808,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 				if (auto OpKey = Params.GetValue("opkey"); !OpKey.empty())
 				{
 					Oid						OpKeyId = OpKeyStringAsOId(OpKey);
-					std::optional<CbObject> Op		= FoundLog->GetOplog(OpKeyId);
+					std::optional<CbObject> Op		= FoundLog->GetOpByKey(OpKeyId);
 
 					if (Op.has_value())
 					{
diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h
index f71434783..bf6afa592 100644
--- a/zenserver/projectstore.h
+++ b/zenserver/projectstore.h
@@ -69,7 +69,8 @@ public:
 
 		void IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn);
 		void IterateOplog(std::function<void(CbObject)>&& Fn);
-		std::optional<CbObject> GetOplog(const Oid& Key);
+		std::optional<CbObject> GetOpByKey(const Oid& Key);
+		std::optional<CbObject> GetOpByIndex(int Index);
 
 		IoBuffer FindChunk(Oid ChunkId);
 
diff --git a/zenserver/sos/sos.cpp b/zenserver/sos/sos.cpp
deleted file mode 100644
index 5fa6ffaae..000000000
--- a/zenserver/sos/sos.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "sos.h"
-
-#include <zencore/logging.h>
-
-namespace zen {
-
-HttpCommonStructuredObjectStore::HttpCommonStructuredObjectStore() : m_Log(logging::Get("sos"))
-{
-	m_Router.AddPattern("ns", "([[:alnum:]_-.]+)");
-	m_Router.AddPattern("bucket", "([[:alnum:]_-.]+)");
-	m_Router.AddPattern("hash", "([[:xdigit:]]{40})");
-}
-
-HttpCommonStructuredObjectStore::~HttpCommonStructuredObjectStore()
-{
-}
-
-const char*
-HttpCommonStructuredObjectStore::BaseUri() const
-{
-	return "/sos/";
-}
-
-void
-HttpCommonStructuredObjectStore::HandleRequest(zen::HttpServerRequest& HttpServiceRequest)
-{
-	ZEN_UNUSED(HttpServiceRequest);
-}
-
-}  // namespace zen
diff --git a/zenserver/sos/sos.h b/zenserver/sos/sos.h
deleted file mode 100644
index e602df8c4..000000000
--- a/zenserver/sos/sos.h
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zenhttp/httpserver.h>
-
-#include <zencore/logging.h>
-
-namespace zen {
-
-/** Simple Object Store API
- *
- * Implements an API shared with Jupiter
- *
- * - Objects (compact binary), named and private
- * - Blobs (unstructured binary), named and private
- *
- */
-
-class HttpCommonStructuredObjectStore : public zen::HttpService
-{
-public:
-	HttpCommonStructuredObjectStore();
-	virtual ~HttpCommonStructuredObjectStore();
-
-	virtual const char* BaseUri() const override;
-	virtual void		HandleRequest(zen::HttpServerRequest& HttpServiceRequest) override;
-
-private:
-	spdlog::logger&		   m_Log;
-	zen::HttpRequestRouter m_Router;
-};
-
-}  // namespace zen
diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp
index 0237ec346..cd6a531ca 100644
--- a/zenserver/upstream/zen.cpp
+++ b/zenserver/upstream/zen.cpp
@@ -408,15 +408,14 @@ ZenStructuredCacheSession::CheckHealth()
 }
 
 ZenCacheResult
-ZenStructuredCacheSession::GetCacheRecord(std::string_view, std::string_view BucketId, const IoHash& Key, ZenContentType Type)
+ZenStructuredCacheSession::GetCacheRecord(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType Type)
 {
 	ExtendableStringBuilder<256> Uri;
 	Uri << m_Client.ServiceUrl() << "/z$/";
-	// TODO: DE20220530: Disable adding namespace into URL until we have updated the shared instances with namespace support
-	//	if (Namespace != ZenCacheStore::DefaultNamespace)
-	//	{
-	//		Uri << Namespace << "/";
-	//	}
+	if (Namespace != ZenCacheStore::DefaultNamespace)
+	{
+		Uri << Namespace << "/";
+	}
 	Uri << BucketId << "/" << Key.ToHexString();
 
 	cpr::Session& Session = m_SessionState->GetSession();
@@ -438,15 +437,17 @@ ZenStructuredCacheSession::GetCacheRecord(std::string_view, std::string_view Buc
 }
 
 ZenCacheResult
-ZenStructuredCacheSession::GetCacheValue(std::string_view, std::string_view BucketId, const IoHash& Key, const IoHash& ValueContentId)
+ZenStructuredCacheSession::GetCacheValue(std::string_view Namespace,
+										 std::string_view BucketId,
+										 const IoHash&	  Key,
+										 const IoHash&	  ValueContentId)
 {
 	ExtendableStringBuilder<256> Uri;
 	Uri << m_Client.ServiceUrl() << "/z$/";
-	// TODO: DE20220530: Disable adding namespace into URL until we have updated the shared instances with namespace support
-	//	if (Namespace != ZenCacheStore::DefaultNamespace)
-	//	{
-	//		Uri << Namespace << "/";
-	//	}
+	if (Namespace != ZenCacheStore::DefaultNamespace)
+	{
+		Uri << Namespace << "/";
+	}
 	Uri << BucketId << "/" << Key.ToHexString() << "/" << ValueContentId.ToHexString();
 
 	cpr::Session& Session = m_SessionState->GetSession();
@@ -473,7 +474,7 @@ ZenStructuredCacheSession::GetCacheValue(std::string_view, std::string_view Buck
 }
 
 ZenCacheResult
-ZenStructuredCacheSession::PutCacheRecord(std::string_view,
+ZenStructuredCacheSession::PutCacheRecord(std::string_view Namespace,
 										  std::string_view BucketId,
 										  const IoHash&	   Key,
 										  IoBuffer		   Value,
@@ -481,11 +482,10 @@ ZenStructuredCacheSession::PutCacheRecord(std::string_view,
 {
 	ExtendableStringBuilder<256> Uri;
 	Uri << m_Client.ServiceUrl() << "/z$/";
-	// TODO: DE20220530: Disable adding namespace into URL until we have updated the shared instances with namespace support
-	//	if (Namespace != ZenCacheStore::DefaultNamespace)
-	//	{
-	//		Uri << Namespace << "/";
-	//	}
+	if (Namespace != ZenCacheStore::DefaultNamespace)
+	{
+		Uri << Namespace << "/";
+	}
 	Uri << BucketId << "/" << Key.ToHexString();
 
 	cpr::Session& Session = m_SessionState->GetSession();
@@ -510,7 +510,7 @@ ZenStructuredCacheSession::PutCacheRecord(std::string_view,
 }
 
 ZenCacheResult
-ZenStructuredCacheSession::PutCacheValue(std::string_view,
+ZenStructuredCacheSession::PutCacheValue(std::string_view Namespace,
 										 std::string_view BucketId,
 										 const IoHash&	  Key,
 										 const IoHash&	  ValueContentId,
@@ -518,11 +518,10 @@ ZenStructuredCacheSession::PutCacheValue(std::string_view,
 {
 	ExtendableStringBuilder<256> Uri;
 	Uri << m_Client.ServiceUrl() << "/z$/";
-	// TODO: DE20220530: Disable adding namespace into URL until we have updated the shared instances with namespace support
-	//	if (Namespace != ZenCacheStore::DefaultNamespace)
-	//	{
-	//		Uri << Namespace << "/";
-	//	}
+	if (Namespace != ZenCacheStore::DefaultNamespace)
+	{
+		Uri << Namespace << "/";
+	}
 	Uri << BucketId << "/" << Key.ToHexString() << "/" << ValueContentId.ToHexString();
 
 	cpr::Session& Session = m_SessionState->GetSession();
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 4db69c265..ff7d256f9 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -982,7 +982,7 @@ ZenEntryPoint::Run()
 		}
 		else
 		{
-			printf("sentry_init returned failure!");
+			printf("sentry_init returned failure! (error code: %d)", ErrorCode);
 		}
 	}