Make sure we always write "data" attachment hash for snapshotted oplog entries (#355)

* Make sure we always write "data" attachment hash for snapshotted oplog entries
* Make sure to add chunk mappings for files moved to attatchment in snapshot operation
* fix inverted timoute for expiration (we don't want time expiry in these cases)
* increase timeout for jupiter oplog in project to 3 min
* changelog

5 files changed, 38 insertions, 14 deletions

diff --git a/src/zencore/include/zencore/compactbinaryutil.h b/src/zencore/include/zencore/compactbinaryutil.h
index 9d69266bf..9524d1fc4 100644
--- a/src/zencore/include/zencore/compactbinaryutil.h
+++ b/src/zencore/include/zencore/compactbinaryutil.h
@@ -23,11 +23,9 @@ namespace zen {
 	return `false` to have RewriteCbObject copy the field into the new object.
 */
 
-CbObject
-RewriteCbObject(CbObjectView InObj, Invocable<CbObjectWriter&, CbFieldView&> auto Rewriter)
+void
+RewriteCbObject(CbObjectWriter& CboWriter, CbObjectView InObj, Invocable<CbObjectWriter&, CbFieldView&> auto Rewriter)
 {
-	CbObjectWriter CboWriter;
-
 	for (CbFieldView InnerField : InObj)
 	{
 		if (!Rewriter(CboWriter, InnerField))
@@ -35,8 +33,14 @@ RewriteCbObject(CbObjectView InObj, Invocable<CbObjectWriter&, CbFieldView&> aut
 			CboWriter.AddField(InnerField.GetName(), InnerField);
 		}
 	}
+}
 
-	return CboWriter.Save();
+CbObject
+RewriteCbObject(CbObjectView InObj, Invocable<CbObjectWriter&, CbFieldView&> auto Rewriter)
+{
+	CbObjectWriter Writer;
+	RewriteCbObject(Writer, InObj, Rewriter);
+	return Writer.Save();
 }
 
 }  // namespace zen
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 185f2783d..7e7517c33 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -936,7 +936,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 						CbObjectWriter Cb;
 						Cb << "id"sv << Log.OplogId() << "project"sv << Project->Identifier << "tempdir"sv << Log.TempPath().c_str()
 						   << "markerpath"sv << Log.MarkerPath().c_str() << "totalsize"sv << Log.TotalSize() << "opcount"
-						   << Log.OplogCount() << "expired"sv << Project->IsExpired(GcClock::TimePoint::max(), Log);
+						   << Log.OplogCount() << "expired"sv << Project->IsExpired(GcClock::TimePoint::min(), Log);
 
 						HttpReq.WriteResponse(HttpResponseCode::OK, Cb.Save());
 					}
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index a7bf25ad6..4be58256c 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -252,7 +252,7 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options)
 	CloudCacheClientOptions ClientOptions{.Name			  = "Remote store"sv,
 										  .ServiceUrl	  = Url,
 										  .ConnectTimeout = std::chrono::milliseconds(2000),
-										  .Timeout		  = std::chrono::milliseconds(120000),
+										  .Timeout		  = std::chrono::milliseconds(180000),
 										  .AssumeHttp2	  = Options.AssumeHttp2};
 	// 1) Access token as parameter in request
 	// 2) Environment variable (different win vs linux/mac)
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 64f0244d5..4865f048b 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -769,6 +769,16 @@ ProjectStore::Oplog::GetOpByIndex(int Index)
 }
 
 void
+ProjectStore::Oplog::AddChunkMappings(const std::unordered_map<Oid, IoHash, Oid::Hasher>& ChunkMappings)
+{
+	RwLock::ExclusiveLockScope OplogLock(m_OplogLock);
+	for (const auto& It : ChunkMappings)
+	{
+		AddChunkMapping(OplogLock, It.first, It.second);
+	}
+}
+
+void
 ProjectStore::Oplog::AddFileMapping(const RwLock::ExclusiveLockScope&,
 									Oid				 FileId,
 									IoHash			 Hash,
@@ -1337,7 +1347,7 @@ ProjectStore::Project::ScrubStorage(ScrubContext& Ctx)
 		OpenOplog(OpLogId);
 	}
 	IterateOplogs([&](const Oplog& Ops) {
-		if (!IsExpired(GcClock::TimePoint::max(), Ops))
+		if (!IsExpired(GcClock::TimePoint::min(), Ops))
 		{
 			Ops.ScrubStorage(Ctx);
 		}
@@ -1559,7 +1569,7 @@ ProjectStore::ScrubStorage(ScrubContext& Ctx)
 
 		for (auto& Kv : m_Projects)
 		{
-			if (Kv.second->IsExpired(GcClock::TimePoint::max()))
+			if (Kv.second->IsExpired(GcClock::TimePoint::min()))
 			{
 				continue;
 			}
@@ -2462,7 +2472,8 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 		uint64_t TotalBytes	  = 0;
 		uint64_t TotalFiles	  = 0;
 
-		std::vector<CbObject> NewOps;
+		std::vector<CbObject>						 NewOps;
+		std::unordered_map<Oid, IoHash, Oid::Hasher> NewChunkMappings;
 
 		Oplog->IterateOplog([&](CbObject Op) {
 			bool OpRewritten = false;
@@ -2499,6 +2510,7 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 						{
 							// Read file contents into memory, compress and store in CidStore
 
+							Oid					   ChunkId			= View["id"sv].AsObjectId();
 							IoBuffer			   FileIoBuffer		= DataFile.ReadAll();
 							CompressedBuffer	   Compressed		= CompressedBuffer::Compress(SharedBuffer(FileIoBuffer));
 							const IoHash		   RawHash			= Compressed.DecodeRawHash();
@@ -2516,12 +2528,11 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 							}
 
 							// Rewrite file array entry with new data reference
-
-							CbObject RewrittenOp = RewriteCbObject(View, [&](CbObjectWriter& Writer, CbFieldView Field) -> bool {
+							CbObjectWriter Writer;
+							RewriteCbObject(Writer, View, [&](CbObjectWriter&, CbFieldView Field) -> bool {
 								if (Field.GetName() == "data"sv)
 								{
-									Writer.AddBinaryAttachment("data"sv, RawHash);
-
+									// omit this field as we will write it explicitly ourselves
 									return true;
 								}
 								else if (Field.GetName() == "serverpath"sv)
@@ -2532,9 +2543,13 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 
 								return false;
 							});
+							Writer.AddBinaryAttachment("data"sv, RawHash);
 
+							CbObject RewrittenOp = Writer.Save();
 							Cbo.AddObject(std::move(RewrittenOp));
 							CopyField = false;
+
+							NewChunkMappings.insert_or_assign(ChunkId, RawHash);
 						}
 					}
 				}
@@ -2571,6 +2586,9 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 			OpCount++;
 		});
 
+		// Make sure we have references to our attachments
+		Oplog->AddChunkMappings(NewChunkMappings);
+
 		CbObjectWriter ResponseObj;
 
 		// Persist rewritten oplog entries
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index c9816ec55..0d0d73263 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -128,6 +128,8 @@ public:
 
 		std::filesystem::path PrepareForDelete(bool MoveFolder);
 
+		void AddChunkMappings(const std::unordered_map<Oid, IoHash, Oid::Hasher>& ChunkMappings);
+
 	private:
 		struct FileMapEntry
 		{