Added oplog prep step to avoid redundant chunk transfers

1 files changed, 74 insertions, 0 deletions

diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index dcad483a6..0d7ad0f16 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -1039,6 +1039,59 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
 		HttpVerb::kGet | HttpVerb::kHead);
 
 	m_Router.RegisterRoute(
+		"{project}/oplog/{log}/prep",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest& HttpReq = Req.ServerRequest();
+
+			const auto& ProjectId = Req.GetCapture(1);
+			const auto& OplogId	  = Req.GetCapture(2);
+
+			ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId);
+
+			if (FoundLog == nullptr)
+			{
+				return HttpReq.WriteResponse(HttpResponse::NotFound);
+			}
+
+			ProjectStore::Oplog& Log = *FoundLog;
+
+			// This operation takes a list of referenced hashes and decides which
+			// chunks are not present on this server. This list is then returned in
+			// the "need" list in the response
+
+			IoBuffer Payload	   = HttpReq.ReadPayload();
+			CbObject RequestObject = LoadCompactBinaryObject(Payload);
+
+			std::vector<IoHash> NeedList;
+
+			for (auto Entry : RequestObject["have"sv])
+			{
+				const IoHash FileHash = Entry.AsHash();
+
+				if (!m_CasStore.FindChunk(FileHash))
+				{
+					spdlog::debug("NEED: {}", FileHash);
+
+					NeedList.push_back(FileHash);
+				}
+			}
+
+			CbObjectWriter Cbo;
+			Cbo.BeginArray("need");
+
+			for (const IoHash& Hash : NeedList)
+			{
+				Cbo << Hash;
+			}
+
+			Cbo.EndArray();
+			CbObject Response = Cbo.Save();
+
+			return HttpReq.WriteResponse(HttpResponse::OK, Response);
+		},
+		HttpVerb::kPost);
+
+	m_Router.RegisterRoute(
 		"{project}/oplog/{log}/new",
 		[this](HttpRouterRequest& Req) {
 			HttpServerRequest& HttpReq = Req.ServerRequest();
@@ -1057,6 +1110,13 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
 
 			IoBuffer Payload = HttpReq.ReadPayload();
 
+			// This will attempt to open files which may not exist for the case where
+			// the prep step rejected the chunk. This should be fixed since there's
+			// a performance cost associated with any file system activity
+
+			bool IsValid = true;
+			std::vector<IoHash> MissingChunks;
+
 			CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
 				std::filesystem::path AttachmentPath = Log.TempPath() / Hash.ToHexString();
 
@@ -1064,8 +1124,15 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
 				{
 					return SharedBuffer(std::move(Data));
 				}
+				else if (Data = m_CasStore.FindChunk(Hash))
+				{
+					return SharedBuffer(std::move(Data));
+				}
 				else
 				{
+					IsValid = false;
+					MissingChunks.push_back(Hash);
+
 					return {};
 				}
 			};
@@ -1073,6 +1140,13 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
 			CbPackage Package;
 			Package.Load(Payload, &UniqueBuffer::Alloc, &Resolver);
 
+			if (!IsValid)
+			{
+				// TODO: emit diagnostics identifying missing chunks
+
+				return HttpReq.WriteResponse(HttpResponse::NotFound, HttpContentType::kText, "Missing chunk reference");
+			}
+
 			CbObject Core = Package.GetObject();
 
 			if (!Core["key"sv])