Merge branch 'main' into zs/put-overwrite-policy

1 files changed, 93 insertions, 19 deletions

diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 0b8e5f13b..317a419eb 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -8,6 +8,7 @@
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/compactbinarypackage.h>
 #include <zencore/compactbinaryutil.h>
+#include <zencore/compactbinaryvalidation.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
@@ -234,10 +235,15 @@ namespace {
 
 //////////////////////////////////////////////////////////////////////////
 
-HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr)
+HttpProjectService::HttpProjectService(CidStore&		  Store,
+									   ProjectStore*	  Projects,
+									   HttpStatusService& StatusService,
+									   HttpStatsService&  StatsService,
+									   AuthMgr&			  AuthMgr)
 : m_Log(logging::Get("project"))
 , m_CidStore(Store)
 , m_ProjectStore(Projects)
+, m_StatusService(StatusService)
 , m_StatsService(StatsService)
 , m_AuthMgr(AuthMgr)
 {
@@ -245,8 +251,6 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 
 	using namespace std::literals;
 
-	m_StatsService.RegisterHandler("prj", *this);
-
 	m_Router.AddPattern("project", "([[:alnum:]_.]+)");
 	m_Router.AddPattern("log", "([[:alnum:]_.]+)");
 	m_Router.AddPattern("op", "([[:digit:]]+?)");
@@ -365,11 +369,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 		"details\\$/{project}/{log}/{chunk}",
 		[this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); },
 		HttpVerb::kGet);
+
+	m_StatusService.RegisterHandler("prj", *this);
+	m_StatsService.RegisterHandler("prj", *this);
 }
 
 HttpProjectService::~HttpProjectService()
 {
 	m_StatsService.UnregisterHandler("prj", *this);
+	m_StatusService.UnregisterHandler("prj", *this);
 }
 
 const char*
@@ -465,6 +473,15 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
 }
 
 void
+HttpProjectService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	ZEN_TRACE_CPU("HttpProjectService::Status");
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
 HttpProjectService::HandleProjectListRequest(HttpRouterRequest& Req)
 {
 	ZEN_TRACE_CPU("ProjectService::ProjectList");
@@ -885,10 +902,63 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req)
 		case HttpVerb::kGet:
 			{
 				IoBuffer								 Value;
-				std::pair<HttpResponseCode, std::string> Result =
-					m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value, nullptr);
+				std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, Value, nullptr);
 				if (Result.first == HttpResponseCode::OK)
 				{
+					if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary ||
+						AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML ||
+						AcceptType == ZenContentType::kCbObject)
+					{
+						CompressedBuffer Compressed			= CompressedBuffer::FromCompressedNoValidate(std::move(Value));
+						IoBuffer		 DecompressedBuffer = Compressed.Decompress().AsIoBuffer();
+
+						if (DecompressedBuffer)
+						{
+							if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML ||
+								AcceptType == ZenContentType::kCbObject)
+							{
+								CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default);
+								if (!!CbErr)
+								{
+									m_ProjectStats.BadRequestCount++;
+									ZEN_DEBUG(
+										"chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not convert to object`",
+										ProjectId,
+										OplogId,
+										Cid,
+										ToString(AcceptType));
+									return HttpReq.WriteResponse(
+										HttpResponseCode::NotAcceptable,
+										HttpContentType::kText,
+										fmt::format("Content format not supported, requested {} format, but could not convert to object",
+													ToString(AcceptType)));
+								}
+
+								m_ProjectStats.ChunkHitCount++;
+								CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer);
+								return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject);
+							}
+							else
+							{
+								Value = DecompressedBuffer;
+								Value.SetContentType(ZenContentType::kBinary);
+							}
+						}
+						else
+						{
+							m_ProjectStats.BadRequestCount++;
+							ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not decompress stored data`",
+									  ProjectId,
+									  OplogId,
+									  Cid,
+									  ToString(AcceptType));
+							return HttpReq.WriteResponse(
+								HttpResponseCode::NotAcceptable,
+								HttpContentType::kText,
+								fmt::format("Content format not supported, requested {} format, but could not decompress stored data",
+											ToString(AcceptType)));
+						}
+					}
 					m_ProjectStats.ChunkHitCount++;
 					return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value);
 				}
@@ -983,15 +1053,19 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req)
 	IoBuffer Payload	   = HttpReq.ReadPayload();
 	CbObject RequestObject = LoadCompactBinaryObject(Payload);
 
-	std::vector<IoHash> ChunkList;
-	CbArrayView			HaveList = RequestObject["have"sv].AsArrayView();
-	ChunkList.reserve(HaveList.Num());
-	for (auto& Entry : HaveList)
+	std::vector<IoHash> NeedList;
+
 	{
-		ChunkList.push_back(Entry.AsHash());
-	}
+		eastl::fixed_vector<IoHash, 16> ChunkList;
+		CbArrayView						HaveList = RequestObject["have"sv].AsArrayView();
+		ChunkList.reserve(HaveList.Num());
+		for (auto& Entry : HaveList)
+		{
+			ChunkList.push_back(Entry.AsHash());
+		}
 
-	std::vector<IoHash> NeedList = FoundLog->CheckPendingChunkReferences(ChunkList, std::chrono::minutes(2));
+		NeedList = FoundLog->CheckPendingChunkReferences(std::span(begin(ChunkList), end(ChunkList)), std::chrono::minutes(2));
+	}
 
 	CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1);
 	Cbo.BeginArray("need");
@@ -1151,7 +1225,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 		return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified");
 	}
 
-	std::vector<IoHash> ReferencedChunks;
+	eastl::fixed_vector<IoHash, 16> ReferencedChunks;
 	Core.IterateAttachments([&ReferencedChunks](CbFieldView View) { ReferencedChunks.push_back(View.AsAttachment()); });
 
 	// Write core to oplog
@@ -1169,7 +1243,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 	// Once we stored the op, we no longer need to retain any chunks this op references
 	if (!ReferencedChunks.empty())
 	{
-		FoundLog->RemovePendingChunkReferences(ReferencedChunks);
+		FoundLog->RemovePendingChunkReferences(std::span(begin(ReferencedChunks), end(ReferencedChunks)));
 	}
 
 	m_ProjectStats.OpWriteCount++;
@@ -1301,9 +1375,9 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
 
 	HttpServerRequest& HttpReq = Req.ServerRequest();
 
-	const std::string& ProjectId  = Req.GetCapture(1);
-	const std::string& OplogId	  = Req.GetCapture(2);
-	const std::string& OpIdString = Req.GetCapture(3);
+	const std::string_view ProjectId  = Req.GetCapture(1);
+	const std::string_view OplogId	  = Req.GetCapture(2);
+	const std::string_view OpIdString = Req.GetCapture(3);
 
 	Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
 	if (!Project)
@@ -1690,8 +1764,8 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req)
 
 	using namespace std::literals;
 
-	HttpServerRequest& HttpReq	 = Req.ServerRequest();
-	const std::string  ProjectId = Req.GetCapture(1);
+	HttpServerRequest&	   HttpReq	 = Req.ServerRequest();
+	const std::string_view ProjectId = Req.GetCapture(1);
 
 	switch (HttpReq.RequestVerb())
 	{