improvements and infrastructure for upcoming builds api command line (#284)

* add modification tick to filesystem traversal
* add ShowDetails option to ProgressBar
* log callstack if we terminate process
* handle chunking if MaxSize > 1MB
* BasicFile write helpers and WriteToTempFile simplifications
* bugfix for CompositeBuffer::IterateRange when using DecompressToComposite for actually comrpessed data
revert of earlier optimization
* faster compress/decompress for large disk-based files
* enable progress feedback in IoHash::HashBuffer
* add payload validation in HttpClient::Get
* fix range requests (range is including end byte)
* remove BuildPartId for blob/block related operations in builds api

3 files changed, 20 insertions, 68 deletions

diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp
index b0212ab07..5d96de225 100644
--- a/src/zenserver/objectstore/objectstore.cpp
+++ b/src/zenserver/objectstore/objectstore.cpp
@@ -376,7 +376,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
 			Writer.BeginArray("Contents"sv);
 		}
 
-		void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override
+		void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override
 		{
 			const fs::path FullPath		= Parent / fs::path(File);
 			fs::path	   RelativePath = fs::relative(FullPath, BucketPath);
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
index 412769174..e4e91104c 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
@@ -123,18 +123,17 @@ public:
 		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
 
 		JupiterResult PutResult =
-			Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, ZenContentType::kCompressedBinary, Payload);
+			Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
 		AddStats(PutResult);
 
 		SaveAttachmentResult Result{ConvertResult(PutResult)};
 		if (Result.ErrorCode)
 		{
-			Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}/{}/{}. Reason: '{}'",
+			Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}/{}. Reason: '{}'",
 										m_JupiterClient->ServiceUrl(),
 										m_Namespace,
 										m_Bucket,
 										m_BuildId,
-										m_OplogBuildPartId,
 										RawHash,
 										Result.Reason);
 			return Result;
@@ -147,18 +146,16 @@ public:
 
 			IoBuffer MetaPayload = BuildChunkBlockDescription(Block, BlockMetaData.Save()).GetBuffer().AsIoBuffer();
 			MetaPayload.SetContentType(ZenContentType::kCbObject);
-			JupiterResult PutMetaResult =
-				Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, MetaPayload);
+			JupiterResult PutMetaResult = Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, RawHash, MetaPayload);
 			AddStats(PutMetaResult);
 			RemoteProjectStore::Result MetaDataResult = ConvertResult(PutMetaResult);
 			if (MetaDataResult.ErrorCode)
 			{
-				ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}/{}. Reason: '{}'",
+				ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}. Reason: '{}'",
 						 m_JupiterClient->ServiceUrl(),
 						 m_Namespace,
 						 m_Bucket,
 						 m_BuildId,
-						 m_OplogBuildPartId,
 						 RawHash,
 						 MetaDataResult.Reason);
 			}
@@ -311,30 +308,28 @@ public:
 	{
 		ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
 		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
-		JupiterResult  FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId);
+		JupiterResult  FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId);
 		AddStats(FindResult);
 		GetKnownBlocksResult Result{ConvertResult(FindResult)};
 		if (Result.ErrorCode)
 		{
 			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
-			Result.Reason	 = fmt::format("Failed listing know blocks for {}/{}/{}/{}/{}. Reason: '{}'",
+			Result.Reason	 = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'",
 										   m_JupiterClient->ServiceUrl(),
 										   m_Namespace,
 										   m_Bucket,
 										   m_BuildId,
-										   m_OplogBuildPartId,
 										   Result.Reason);
 			return Result;
 		}
 		if (ValidateCompactBinary(FindResult.Response.GetView(), CbValidateMode::Default) != CbValidateError::None)
 		{
 			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
-			Result.Reason	 = fmt::format("The block list {}/{}/{}/{} is not formatted as a compact binary object"sv,
+			Result.Reason	 = fmt::format("The block list {}/{}/{} is not formatted as a compact binary object"sv,
 										   m_JupiterClient->ServiceUrl(),
 										   m_Namespace,
 										   m_Bucket,
-										   m_BuildId,
-										   m_OplogBuildPartId);
+										   m_BuildId);
 			return Result;
 		}
 		std::optional<std::vector<ChunkBlockDescription>> Blocks =
@@ -342,12 +337,11 @@ public:
 		if (!Blocks)
 		{
 			Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
-			Result.Reason	 = fmt::format("The block list {}/{}/{}/{} is not formatted as a list of blocks"sv,
+			Result.Reason	 = fmt::format("The block list {}/{}/{} is not formatted as a list of blocks"sv,
 										   m_JupiterClient->ServiceUrl(),
 										   m_Namespace,
 										   m_Bucket,
-										   m_BuildId,
-										   m_OplogBuildPartId);
+										   m_BuildId);
 			return Result;
 		}
 		Result.Blocks = std::move(Blocks.value());
@@ -358,18 +352,17 @@ public:
 	{
 		ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
 		JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
-		JupiterResult  GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, m_TempFilePath);
+		JupiterResult  GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, m_TempFilePath);
 		AddStats(GetResult);
 
 		LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)};
 		if (GetResult.ErrorCode)
 		{
-			Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}&{}/{}/{}. Reason: '{}'",
+			Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}/{}. Reason: '{}'",
 										m_JupiterClient->ServiceUrl(),
 										m_Namespace,
 										m_Bucket,
 										m_BuildId,
-										m_OplogBuildPartId,
 										RawHash,
 										Result.Reason);
 		}
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index 0285cc22f..b4b2c6fc4 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -1466,10 +1466,7 @@ BuildContainer(CidStore&																	ChunkStore,
 		};
 		std::vector<ChunkedFile> ChunkedFiles;
 
-		auto ChunkFile = [AttachmentTempPath](const IoHash&				   RawHash,
-											  IoBuffer&					   RawData,
-											  const IoBufferFileReference& FileRef,
-											  JobContext*) -> ChunkedFile {
+		auto ChunkFile = [](const IoHash& RawHash, IoBuffer& RawData, const IoBufferFileReference& FileRef, JobContext*) -> ChunkedFile {
 			ChunkedFile Chunked;
 			Stopwatch	Timer;
 
@@ -1578,22 +1575,7 @@ BuildContainer(CidStore&																	ChunkStore,
 
 									std::filesystem::path AttachmentPath = AttachmentTempPath;
 									AttachmentPath.append(RawHash.ToHexString());
-									uint32_t RetriesLeft = 3;
-
-									IoBuffer TempAttachmentBuffer =
-										WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath, [&](std::error_code& Ec) {
-											if (RetriesLeft == 0)
-											{
-												return false;
-											}
-											ZEN_WARN("Failed to create temporary attachment '{}': '{}', retries left: {}.",
-													 AttachmentPath,
-													 Ec.message(),
-													 RetriesLeft);
-											Sleep(100 - (3 - RetriesLeft) * 100);  // Total 600 ms
-											RetriesLeft--;
-											return true;
-										});
+									IoBuffer TempAttachmentBuffer = WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath);
 									ZEN_INFO("Saved temp attachment to '{}', {} ({})",
 											 AttachmentPath,
 											 NiceBytes(RawSize),
@@ -1612,34 +1594,21 @@ BuildContainer(CidStore&																	ChunkStore,
 								std::filesystem::path AttachmentPath = AttachmentTempPath;
 								AttachmentPath.append(RawHash.ToHexString());
 
-								uint32_t RetriesLeft = 3;
-								IoBuffer TempAttachmentBuffer =
-									WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath, [&](std::error_code& Ec) {
-										if (RetriesLeft == 0)
-										{
-											return false;
-										}
-										ZEN_WARN("Failed to create temporary attachment '{}': '{}', retries left: {}.",
-												 AttachmentPath,
-												 Ec.message(),
-												 RetriesLeft);
-										Sleep(100 - (3 - RetriesLeft) * 100);  // Total 600 ms
-										RetriesLeft--;
-										return true;
-									});
+								uint64_t CompressedSize		  = Compressed.GetCompressedSize();
+								IoBuffer TempAttachmentBuffer = WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath);
 								ZEN_INFO("Saved temp attachment to '{}', {} ({})",
 										 AttachmentPath,
 										 NiceBytes(RawSize),
 										 NiceBytes(TempAttachmentBuffer.GetSize()));
 
-								if (Compressed.GetCompressedSize() > MaxChunkEmbedSize)
+								if (CompressedSize > MaxChunkEmbedSize)
 								{
 									OnLargeAttachment(RawHash, [Data = std::move(TempAttachmentBuffer)](const IoHash&) { return Data; });
 									ResolveLock.WithExclusiveLock([RawHash, &LargeChunkHashes]() { LargeChunkHashes.insert(RawHash); });
 								}
 								else
 								{
-									UploadAttachment->Size = Compressed.GetCompressedSize();
+									UploadAttachment->Size = CompressedSize;
 									ResolveLock.WithExclusiveLock(
 										[RawHash, RawSize, &LooseUploadAttachments, Data = std::move(TempAttachmentBuffer)]() {
 											LooseUploadAttachments.insert_or_assign(RawHash, std::make_pair(RawSize, std::move(Data)));
@@ -2362,17 +2331,7 @@ SaveOplog(CidStore&				 ChunkStore,
 		BlockPath.append(Block.BlockHash.ToHexString());
 		try
 		{
-			uint32_t RetriesLeft = 3;
-			IoBuffer BlockBuffer = WriteToTempFile(std::move(CompressedBlock).GetCompressed(), BlockPath, [&](std::error_code& Ec) {
-				if (RetriesLeft == 0)
-				{
-					return false;
-				}
-				ZEN_WARN("Failed to create temporary oplog block '{}': '{}', retries left: {}.", BlockPath, Ec.message(), RetriesLeft);
-				Sleep(100 - (3 - RetriesLeft) * 100);  // Total 600 ms
-				RetriesLeft--;
-				return true;
-			});
+			IoBuffer				   BlockBuffer = WriteToTempFile(std::move(CompressedBlock).GetCompressed(), BlockPath);
 			RwLock::ExclusiveLockScope __(AttachmentsLock);
 			CreatedBlocks.insert({Block.BlockHash, {.Payload = std::move(BlockBuffer), .Block = std::move(Block)}});
 			ZEN_DEBUG("Saved temp block to '{}', {}", AttachmentTempPath, NiceBytes(BlockBuffer.GetSize()));