check existance of reused blocks (#18)

* Add HasAttachments to RemoteProjectStore so we can query if attachment blocks actually exist
* use individual requests for compressed blob check in Jupiter
* remove weird 1000.5 to 1000.0 when converting seconds to milliseconds

5 files changed, 167 insertions, 43 deletions

diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp
index defa7bf80..4248bbf2a 100644
--- a/src/zenserver/projectstore/fileremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp
@@ -84,7 +84,7 @@ public:
 			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
 			Result.Reason	 = fmt::format("Failed saving oplog container to '{}'. Reason: {}", ContainerPath, Ex.what());
 		}
-		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 		return Result;
 	}
 
@@ -114,7 +114,7 @@ public:
 				Result.Reason	 = fmt::format("Failed saving oplog attachment to '{}'. Reason: {}", ChunkPath, Ex.what());
 			}
 		}
-		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 		return Result;
 	}
 
@@ -128,12 +128,12 @@ public:
 			SaveAttachmentResult ChunkResult = SaveAttachment(Compressed.GetCompressed(), Compressed.DecodeRawHash());
 			if (ChunkResult.ErrorCode)
 			{
-				ChunkResult.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+				ChunkResult.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 				return SaveAttachmentsResult{ChunkResult};
 			}
 		}
 		SaveAttachmentsResult Result;
-		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 		return Result;
 	}
 
@@ -148,6 +148,23 @@ public:
 		}
 		return LoadContainer(m_OptionalBaseName);
 	}
+
+	virtual HasAttachmentsResult HasAttachments(const std::span<IoHash> RawHashes) override
+	{
+		Stopwatch			 Timer;
+		HasAttachmentsResult Result;
+		for (const IoHash& RawHash : RawHashes)
+		{
+			std::filesystem::path ChunkPath = GetAttachmentPath(RawHash);
+			if (!std::filesystem::is_regular_file(ChunkPath))
+			{
+				Result.Needs.insert(RawHash);
+			}
+		}
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
+		return Result;
+	}
+
 	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
 	{
 		Stopwatch			  Timer;
@@ -157,7 +174,7 @@ public:
 		{
 			Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound);
 			Result.Reason = fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string());
-			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 			return Result;
 		}
 		{
@@ -165,7 +182,7 @@ public:
 			ChunkFile.Open(ChunkPath, BasicFile::Mode::kRead);
 			Result.Bytes = ChunkFile.ReadAll();
 		}
-		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 		return Result;
 	}
 
@@ -178,7 +195,7 @@ public:
 			LoadAttachmentResult ChunkResult = LoadAttachment(Hash);
 			if (ChunkResult.ErrorCode)
 			{
-				ChunkResult.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+				ChunkResult.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 				return LoadAttachmentsResult{ChunkResult};
 			}
 			ZEN_DEBUG("Loaded attachment in {}", NiceTimeSpanMs(static_cast<uint64_t>(ChunkResult.ElapsedSeconds * 1000)));
@@ -199,7 +216,7 @@ private:
 		{
 			Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound);
 			Result.Reason = fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", SourcePath.string());
-			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 			return Result;
 		}
 		IoBuffer ContainerPayload;
@@ -213,10 +230,10 @@ private:
 		{
 			Result.ErrorCode	  = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
 			Result.Reason		  = fmt::format("The file {} is not formatted as a compact binary object", SourcePath.string());
-			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 			return Result;
 		}
-		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 		return Result;
 	}
 
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index 418c2aa84..b288a667c 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -132,6 +132,53 @@ public:
 		return LoadContainer(m_OptionalBaseKey);
 	}
 
+	virtual HasAttachmentsResult HasAttachments(const std::span<IoHash> RawHashes) override
+	{
+		CloudCacheSession Session(m_CloudClient.Get());
+		const bool		  UseBatchRequest =
+			false;	// The batch request API in Jupiter is currently broken so we have to resort to individual cheks
+
+		if (UseBatchRequest)
+		{
+			CloudCacheExistsResult ExistsResult = Session.BlobExists(m_Namespace, std::set<IoHash>(RawHashes.begin(), RawHashes.end()));
+			HasAttachmentsResult   Result{ConvertResult(ExistsResult),
+										  std::unordered_set<IoHash, IoHash::Hasher>(ExistsResult.Needs.begin(), ExistsResult.Needs.end())};
+			if (ExistsResult.ErrorCode)
+			{
+				Result.Reason = fmt::format("Failed checking attachment existance in {}/{}. Reason: '{}'",
+											m_CloudClient->ServiceUrl(),
+											m_Namespace,
+											Result.Reason);
+			}
+			return Result;
+		}
+		else
+		{
+			Stopwatch			 Timer;
+			HasAttachmentsResult Result;
+			for (const IoHash& RawHash : RawHashes)
+			{
+				CloudCacheResult ExistsResult = Session.CompressedBlobExists(m_Namespace, RawHash);
+				if (ExistsResult.ErrorCode)
+				{
+					Result		  = {ConvertResult(ExistsResult)};
+					Result.Reason = fmt::format("Failed checking oplog attachment existance {}/{}/{}. Reason: '{}'",
+												m_CloudClient->ServiceUrl(),
+												m_Namespace,
+												RawHash,
+												Result.Reason);
+					return Result;
+				}
+				if (!ExistsResult.Success)
+				{
+					Result.Needs.insert(RawHash);
+				}
+			}
+			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
+			return Result;
+		}
+	}
+
 	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
 	{
 		CloudCacheSession	 Session(m_CloudClient.Get());
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index ab207930f..d8dfb215d 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -1424,12 +1424,15 @@ UploadAttachments(WorkerThreadPool&														  WorkerPool,
 	size_t BulkAttachmentCountToUpload	= 0;
 	AttachmentsToUpload.reserve(ForceAll ? CreatedBlocks.size() + LargeAttachments.size() : Needs.size());
 
+	std::unordered_set<IoHash, IoHash::Hasher> UnknownAttachments(Needs);
+
 	for (const auto& CreatedBlock : CreatedBlocks)
 	{
 		if (ForceAll || Needs.contains(CreatedBlock.first))
 		{
 			AttachmentsToUpload.insert(CreatedBlock.first);
 			BlockAttachmentCountToUpload++;
+			UnknownAttachments.erase(CreatedBlock.first);
 		}
 	}
 	for (const IoHash& LargeAttachment : LargeAttachments)
@@ -1438,6 +1441,7 @@ UploadAttachments(WorkerThreadPool&														  WorkerPool,
 		{
 			AttachmentsToUpload.insert(LargeAttachment);
 			LargeAttachmentCountToUpload++;
+			UnknownAttachments.erase(LargeAttachment);
 		}
 	}
 	for (const std::vector<std::pair<IoHash, FetchChunkFunc>>& BlockHashes : BlockChunks)
@@ -1448,6 +1452,7 @@ UploadAttachments(WorkerThreadPool&														  WorkerPool,
 			{
 				BulkBlockAttachmentsToUpload.insert(std::make_pair(Chunk.first, Chunk.second));
 				BulkAttachmentCountToUpload++;
+				UnknownAttachments.erase(Chunk.first);
 			}
 		}
 	}
@@ -1458,6 +1463,17 @@ UploadAttachments(WorkerThreadPool&														  WorkerPool,
 		return;
 	}
 
+	if (!UnknownAttachments.empty())
+	{
+		RemoteResult.SetError(
+			gsl::narrow<int>(HttpResponseCode::NotFound),
+			fmt::format("Upload requested of {} missing attachments, the base container referenced blocks that are no longer available",
+						UnknownAttachments.size()),
+			"");
+		ReportMessage(OptionalContext, fmt::format("Aborting ({}): {}", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
+		return;
+	}
+
 	if (IsCancelled(OptionalContext))
 	{
 		if (!RemoteResult.IsError())
@@ -1793,34 +1809,65 @@ SaveOplog(CidStore&				 ChunkStore,
 				ReportMessage(OptionalContext, fmt::format("Loaded oplog base container in {:.3} s", BaseContainerResult.ElapsedSeconds));
 
 				CbArrayView BlocksArray = BaseContainerResult.ContainerObject["blocks"sv].AsArrayView();
-				KnownBlocks.reserve(BlocksArray.Num());
+
+				std::vector<IoHash> BlockHashes;
+				BlockHashes.reserve(BlocksArray.Num());
 				for (CbFieldView BlockField : BlocksArray)
 				{
 					CbObjectView BlockView = BlockField.AsObjectView();
 					IoHash		 BlockHash = BlockView["rawhash"sv].AsBinaryAttachment();
+					BlockHashes.push_back(BlockHash);
+				}
 
-					std::vector<IoHash> ChunksInBlock;
-					CbArrayView			ChunksArray = BlockView["chunks"sv].AsArrayView();
-					if (BlockHash == IoHash::Zero)
+				RemoteProjectStore::HasAttachmentsResult HasResult = RemoteStore.HasAttachments(BlockHashes);
+				if (HasResult.ErrorCode == 0)
+				{
+					ReportMessage(OptionalContext,
+								  fmt::format("Checked the existance of {} block{} in remote store in {}",
+											  BlockHashes.size(),
+											  BlockHashes.size() > 1 ? "S"sv : ""sv,
+											  NiceTimeSpanMs(static_cast<uint64_t>(HasResult.ElapsedSeconds * 1000))));
+					if (HasResult.Needs.size() < BlocksArray.Num())
 					{
-						continue;
-					}
+						KnownBlocks.reserve(BlocksArray.Num() - HasResult.Needs.size());
 
-					ChunksInBlock.reserve(ChunksArray.Num());
-					for (CbFieldView ChunkField : ChunksArray)
-					{
-						ChunksInBlock.push_back(ChunkField.AsHash());
+						const std::unordered_set<IoHash, IoHash::Hasher> MissingBlocks(HasResult.Needs);
+
+						for (CbFieldView BlockField : BlocksArray)
+						{
+							CbObjectView BlockView = BlockField.AsObjectView();
+							IoHash		 BlockHash = BlockView["rawhash"sv].AsBinaryAttachment();
+							if (!MissingBlocks.contains(BlockHash))
+							{
+								std::vector<IoHash> ChunksInBlock;
+								CbArrayView			ChunksArray = BlockView["chunks"sv].AsArrayView();
+								if (BlockHash == IoHash::Zero)
+								{
+									continue;
+								}
+
+								ChunksInBlock.reserve(ChunksArray.Num());
+								for (CbFieldView ChunkField : ChunksArray)
+								{
+									ChunksInBlock.push_back(ChunkField.AsHash());
+								}
+								KnownBlocks.push_back({.BlockHash = BlockHash, .ChunksInBlock = std::move(ChunksInBlock)});
+							}
+						}
 					}
-					KnownBlocks.push_back({.BlockHash = BlockHash, .ChunksInBlock = std::move(ChunksInBlock)});
-				};
+				}
+				else
+				{
+					ReportMessage(OptionalContext,
+								  fmt::format("Unable to determine which blocks in base container exist in remote store, assuming none "
+											  "does: '{}', error code : {}",
+											  HasResult.Reason,
+											  HasResult.ErrorCode));
+				}
 			}
 		}
 	}
 
-	// TODO: We need to check if remote store actually *has* all KnownBlocks
-	// We can't reconstruct known blocks on demand as they may contain chunks that we don't have
-	// and we don't care about :(
-
 	CbObject OplogContainerObject = BuildContainer(ChunkStore,
 												   Project,
 												   Oplog,
@@ -1844,7 +1891,7 @@ SaveOplog(CidStore&				 ChunkStore,
 		if (IsCancelled(OptionalContext))
 		{
 			RemoteProjectStore::Result Result = {.ErrorCode		 = 0,
-												 .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500,
+												 .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0,
 												 .Text			 = "Operation cancelled"};
 			ReportMessage(OptionalContext, fmt::format("Aborting ({}): {}", RemoteResult.GetError(), RemoteResult.GetErrorReason()));
 			return Result;
@@ -1891,7 +1938,7 @@ SaveOplog(CidStore&				 ChunkStore,
 			if (IsCancelled(OptionalContext))
 			{
 				RemoteProjectStore::Result Result = {.ErrorCode		 = 0,
-													 .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500,
+													 .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0,
 													 .Text			 = "Operation cancelled"};
 				ReportMessage(OptionalContext, fmt::format("Aborting ({}): {}", Result.ErrorCode, Result.Text));
 				return Result;
@@ -1923,7 +1970,7 @@ SaveOplog(CidStore&				 ChunkStore,
 			if (IsCancelled(OptionalContext))
 			{
 				RemoteProjectStore::Result Result = {.ErrorCode		 = 0,
-													 .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500,
+													 .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0,
 													 .Text			 = "Operation cancelled"};
 				return Result;
 			}
@@ -1951,7 +1998,7 @@ SaveOplog(CidStore&				 ChunkStore,
 		CreatedBlocks.clear();
 	}
 	RemoteProjectStore::Result Result = RemoteResult.ConvertResult();
-	Result.ElapsedSeconds			  = Timer.GetElapsedTimeMs() / 1000.500;
+	Result.ElapsedSeconds			  = Timer.GetElapsedTimeMs() / 1000.0;
 
 	ReportMessage(OptionalContext,
 				  fmt::format("Saved oplog '{}' {} in {} ({}), Blocks: {} ({}), Attachments: {} ({})",
@@ -2077,7 +2124,7 @@ SaveOplogContainer(ProjectStore::Oplog&																 Oplog,
 		{
 			ReportMessage(OptionalContext, fmt::format("Failed to save oplog container: '{}'", "Section has unexpected data type"));
 			return RemoteProjectStore::Result{gsl::narrow<int>(HttpResponseCode::BadRequest),
-											  Timer.GetElapsedTimeMs() / 1000.500,
+											  Timer.GetElapsedTimeMs() / 1000.0,
 											  "Section has unexpected data type",
 											  "Failed to save oplog container"};
 		}
@@ -2130,7 +2177,7 @@ SaveOplogContainer(ProjectStore::Oplog&																 Oplog,
 			AppendBatch();
 		}
 	}
-	return RemoteProjectStore::Result{.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500};
+	return RemoteProjectStore::Result{.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0};
 }
 
 RemoteProjectStore::Result
@@ -2173,7 +2220,7 @@ LoadOplog(CidStore&			   ChunkStore,
 			OptionalContext,
 			fmt::format("Failed to load oplog container: '{}', error code: {}", LoadContainerResult.Reason, LoadContainerResult.ErrorCode));
 		return RemoteProjectStore::Result{.ErrorCode	  = LoadContainerResult.ErrorCode,
-										  .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500,
+										  .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0,
 										  .Reason		  = LoadContainerResult.Reason,
 										  .Text			  = LoadContainerResult.Text};
 	}
@@ -2554,7 +2601,7 @@ LoadOplog(CidStore&			   ChunkStore,
 		Result = RemoteResult.ConvertResult();
 	}
 
-	Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+	Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 
 	ReportMessage(OptionalContext,
 				  fmt::format("Loaded oplog '{}' {} in {} ({}), Blocks: {} ({}), Attachments: {} ({}), Stored: {} ({}), Missing: {}",
diff --git a/src/zenserver/projectstore/remoteprojectstore.h b/src/zenserver/projectstore/remoteprojectstore.h
index da93f0a27..020069441 100644
--- a/src/zenserver/projectstore/remoteprojectstore.h
+++ b/src/zenserver/projectstore/remoteprojectstore.h
@@ -43,6 +43,11 @@ public:
 	{
 	};
 
+	struct HasAttachmentsResult : public Result
+	{
+		std::unordered_set<IoHash, IoHash::Hasher> Needs;
+	};
+
 	struct LoadAttachmentResult : public Result
 	{
 		IoBuffer Bytes;
@@ -80,6 +85,7 @@ public:
 	virtual LoadContainerResult	  LoadContainer()										= 0;
 	virtual LoadContainerResult	  LoadBaseContainer()									= 0;
 	virtual LoadAttachmentResult  LoadAttachment(const IoHash& RawHash)					= 0;
+	virtual HasAttachmentsResult  HasAttachments(const std::span<IoHash> RawHashes)		= 0;
 	virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) = 0;
 };
 
diff --git a/src/zenserver/projectstore/zenremoteprojectstore.cpp b/src/zenserver/projectstore/zenremoteprojectstore.cpp
index 95fcc9e21..cfb558040 100644
--- a/src/zenserver/projectstore/zenremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/zenremoteprojectstore.cpp
@@ -67,7 +67,7 @@ public:
 										m_Project,
 										m_Oplog,
 										Result.Reason);
-			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 			return Result;
 		}
 		IoBuffer ResponsePayload(IoBuffer::Wrap, Response.text.data(), Response.text.size());
@@ -79,7 +79,7 @@ public:
 										m_Project,
 										m_Oplog);
 			Result.ErrorCode	  = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
-			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+			Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 			return Result;
 		}
 		CbArrayView NeedsArray = ResponseObject["need"sv].AsArrayView();
@@ -90,7 +90,7 @@ public:
 		}
 
 		Result.RawHash		  = IoHash::HashBuffer(Payload);
-		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 		return Result;
 	}
 
@@ -125,7 +125,7 @@ public:
 										RawHash,
 										Result.Reason);
 		}
-		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 		return Result;
 	}
 
@@ -180,7 +180,7 @@ public:
 										m_Oplog,
 										Result.Reason);
 		}
-		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 		return Result;
 	}
 
@@ -234,7 +234,7 @@ public:
 										m_Oplog,
 										Result.Reason);
 		}
-		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 		return Result;
 	};
 
@@ -244,7 +244,7 @@ public:
 
 		RwLock::ExclusiveLockScope _(SessionsLock);
 		Sessions.clear();
-		return FinalizeResult{Result{.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500}};
+		return FinalizeResult{Result{.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0}};
 	}
 
 	virtual LoadContainerResult LoadContainer() override
@@ -281,7 +281,7 @@ public:
 				Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
 			}
 		}
-		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 		return Result;
 	}
 
@@ -290,6 +290,13 @@ public:
 		return LoadContainerResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent)}};
 	}
 
+	virtual HasAttachmentsResult HasAttachments(const std::span<IoHash>) override
+	{
+		// For zen as remote store we never store blocks so we should never get here
+		ZEN_ASSERT(false);
+		return HasAttachmentsResult{};
+	}
+
 	virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
 	{
 		Stopwatch Timer;
@@ -315,7 +322,7 @@ public:
 										RawHash,
 										Result.Reason);
 		}
-		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500;
+		Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
 		return Result;
 	}