oplog capture new attachments for gc (#210)

* make sure we track added attachments properly in oplogs

3 files changed, 12 insertions, 7 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 26a112829..bd1ff27c6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@
 - Improvement: Added progress bar for `zen oplog-mirror` command
 - Improvement: Verify that oplog has not been deleted from disk behind our back
 - Bugfix: If zenserver fails to pick up a request for a sponsor process, make sure to clear the slot
+- Bugfix: Fix potential hole where an chunk could be lost during GC if the package referenced a chunk that already existed in store
 
 ## 5.5.9
 - Feature: Added command `zen cache-get` to fetch a cache value/record or an attachment from a cache record
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 8dbd94d39..1b45e66f3 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -1045,6 +1045,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
 	std::vector<IoHash> MissingChunks;
 
 	CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
+		Oplog.CaptureAddedAttachments(std::vector<IoHash>{Hash});
 		if (m_CidStore.ContainsChunk(Hash))
 		{
 			// Return null attachment as we already have it, no point in reading it and storing it again
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 25be159b9..7e03432d6 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -4242,6 +4242,7 @@ ProjectStore::PutChunk(const std::string_view ProjectId,
 		return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid payload format for chunk '{}'", Cid)};
 	}
 
+	FoundLog->CaptureAddedAttachments(std::vector<IoHash>{Hash});
 	CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk, Hash);
 	return {Result.New ? HttpResponseCode::Created : HttpResponseCode::OK, {}};
 }
@@ -4566,6 +4567,7 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 				WriteRawHashes.push_back(RawHash);
 			}
 
+			Oplog->CaptureAddedAttachments(WriteRawHashes);
 			m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly);
 		}
 		HttpReq.WriteResponse(HttpResponseCode::OK);
@@ -4628,13 +4630,14 @@ ProjectStore::Rpc(HttpServerRequest&	 HttpReq,
 							{
 								// Read file contents into memory, compress and store in CidStore
 
-								Oid					   ChunkId			= View["id"sv].AsObjectId();
-								IoBuffer			   FileIoBuffer		= DataFile.ReadAll();
-								CompressedBuffer	   Compressed		= CompressedBuffer::Compress(SharedBuffer(std::move(FileIoBuffer)));
-								const IoHash		   RawHash			= Compressed.DecodeRawHash();
-								const uint64_t		   RawSize			= Compressed.DecodeRawSize();
-								IoBuffer			   CompressedBuffer = Compressed.GetCompressed().Flatten().AsIoBuffer();
-								CidStore::InsertResult Result			= m_CidStore.AddChunk(CompressedBuffer, RawHash);
+								Oid				 ChunkId		  = View["id"sv].AsObjectId();
+								IoBuffer		 FileIoBuffer	  = DataFile.ReadAll();
+								CompressedBuffer Compressed		  = CompressedBuffer::Compress(SharedBuffer(std::move(FileIoBuffer)));
+								const IoHash	 RawHash		  = Compressed.DecodeRawHash();
+								const uint64_t	 RawSize		  = Compressed.DecodeRawSize();
+								IoBuffer		 CompressedBuffer = Compressed.GetCompressed().Flatten().AsIoBuffer();
+								Oplog->CaptureAddedAttachments(std::vector<IoHash>{RawHash});
+								CidStore::InsertResult Result = m_CidStore.AddChunk(CompressedBuffer, RawHash);
 
 								TotalBytes += RawSize;
 								++TotalFiles;