Merge pull request #593 from EpicGames/zs/get-all-chunk-infos

Zs/get all chunk infos

4 files changed, 74 insertions, 16 deletions

diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 261485834..0ba49cf8a 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -276,6 +276,11 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 		HttpVerb::kGet);
 
 	m_Router.RegisterRoute(
+		"{project}/oplog/{log}/chunkinfos",
+		[this](HttpRouterRequest& Req) { HandleChunkInfosRequest(Req); },
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
 		"{project}/oplog/{log}/{chunk}/info",
 		[this](HttpRouterRequest& Req) { HandleChunkInfoRequest(Req); },
 		HttpVerb::kGet);
@@ -643,6 +648,41 @@ HttpProjectService::HandleFilesRequest(HttpRouterRequest& Req)
 }
 
 void
+HttpProjectService::HandleChunkInfosRequest(HttpRouterRequest& Req)
+{
+	ZEN_TRACE_CPU("ProjectService::ChunkInfos");
+
+	HttpServerRequest& HttpReq = Req.ServerRequest();
+
+	const auto& ProjectId = Req.GetCapture(1);
+	const auto& OplogId	  = Req.GetCapture(2);
+
+	CbObject								 ResponsePayload;
+	std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetProjectChunkInfos(ProjectId, OplogId, ResponsePayload);
+	if (Result.first == HttpResponseCode::OK)
+	{
+		return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload);
+	}
+	else
+	{
+		if (Result.first == HttpResponseCode::BadRequest)
+		{
+			m_ProjectStats.BadRequestCount++;
+		}
+		ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
+				  ToString(HttpReq.RequestVerb()),
+				  HttpReq.QueryString(),
+				  static_cast<int>(Result.first),
+				  Result.second);
+	}
+	if (Result.second.empty())
+	{
+		return HttpReq.WriteResponse(Result.first);
+	}
+	return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
+}
+
+void
 HttpProjectService::HandleChunkInfoRequest(HttpRouterRequest& Req)
 {
 	ZEN_TRACE_CPU("ProjectService::ChunkInfo");
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
index 9998ae83e..9990ee264 100644
--- a/src/zenserver/projectstore/httpprojectstore.h
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -64,6 +64,7 @@ private:
 	void HandleProjectListRequest(HttpRouterRequest& Req);
 	void HandleChunkBatchRequest(HttpRouterRequest& Req);
 	void HandleFilesRequest(HttpRouterRequest& Req);
+	void HandleChunkInfosRequest(HttpRouterRequest& Req);
 	void HandleChunkInfoRequest(HttpRouterRequest& Req);
 	void HandleChunkByIdRequest(HttpRouterRequest& Req);
 	void HandleChunkByCidRequest(HttpRouterRequest& Req);
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index c6097dea2..9ba8e3a19 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -752,6 +752,21 @@ ProjectStore::Oplog::GetAllChunksInfo()
 }
 
 void
+ProjectStore::Oplog::IterateChunkMap(std::function<void(const Oid&, const IoHash&)>&& Fn)
+{
+	RwLock::SharedLockScope _(m_OplogLock);
+	if (!m_Storage)
+	{
+		return;
+	}
+
+	for (const auto& Kv : m_ChunkMap)
+	{
+		Fn(Kv.first, Kv.second);
+	}
+}
+
+void
 ProjectStore::Oplog::IterateFileMap(
 	std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn)
 {
@@ -2215,9 +2230,9 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId, const std::strin
 }
 
 std::pair<HttpResponseCode, std::string>
-ProjectStore::GetProjectChunks(const std::string_view ProjectId, const std::string_view OplogId, CbObject& OutPayload)
+ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId, const std::string_view OplogId, CbObject& OutPayload)
 {
-	ZEN_TRACE_CPU("ProjectStore::GetProjectChunks");
+	ZEN_TRACE_CPU("ProjectStore::GetProjectChunkInfos");
 
 	using namespace std::literals;
 
@@ -2235,21 +2250,22 @@ ProjectStore::GetProjectChunks(const std::string_view ProjectId, const std::stri
 	}
 	Project->TouchOplog(OplogId);
 
-	std::vector<ProjectStore::Oplog::ChunkInfo> ChunkInfo = FoundLog->GetAllChunksInfo();
+	std::vector<std::pair<Oid, IoHash>> ChunkInfos;
+	FoundLog->IterateChunkMap([&ChunkInfos](const Oid& Id, const IoHash& Hash) { ChunkInfos.push_back({Id, Hash}); });
 
 	CbObjectWriter Response;
+	Response.BeginArray("chunkinfos"sv);
 
-	Response.BeginArray("chunks"sv);
-	for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo)
+	for (const auto& ChunkInfo : ChunkInfos)
 	{
-		Response << Info.ChunkId;
-	}
-	Response.EndArray();
-
-	Response.BeginArray("sizes"sv);
-	for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo)
-	{
-		Response << Info.ChunkSize;
+		if (IoBuffer Chunk = FoundLog->FindChunk(ChunkInfo.first))
+		{
+			Response.BeginObject();
+			Response << "id"sv << ChunkInfo.first;
+			Response << "rawhash"sv << ChunkInfo.second;
+			Response << "rawsize"sv << Chunk.GetSize();
+			Response.EndObject();
+		}
 	}
 	Response.EndArray();
 
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 555f8bdf2..57cda8ae7 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -93,6 +93,7 @@ public:
 		};
 
 		std::vector<ChunkInfo> GetAllChunksInfo();
+		void				   IterateChunkMap(std::function<void(const Oid&, const IoHash& Hash)>&& Fn);
 		void IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn);
 		void IterateOplog(std::function<void(CbObjectView)>&& Fn);
 		void IterateOplogWithKey(std::function<void(int, const Oid&, CbObjectView)>&& Fn);
@@ -306,9 +307,9 @@ public:
 															 const std::string_view OplogId,
 															 bool					FilterClient,
 															 CbObject&				OutPayload);
-	std::pair<HttpResponseCode, std::string> GetProjectChunks(const std::string_view ProjectId,
-															  const std::string_view OplogId,
-															  CbObject&				 OutPayload);
+	std::pair<HttpResponseCode, std::string> GetProjectChunkInfos(const std::string_view ProjectId,
+																  const std::string_view OplogId,
+																  CbObject&				 OutPayload);
 	std::pair<HttpResponseCode, std::string> GetChunkInfo(const std::string_view ProjectId,
 														  const std::string_view OplogId,
 														  const std::string_view ChunkId,