improvements and infrastructure for upcoming builds api command line (#284)

* add modification tick to filesystem traversal
* add ShowDetails option to ProgressBar
* log callstack if we terminate process
* handle chunking if MaxSize > 1MB
* BasicFile write helpers and WriteToTempFile simplifications
* bugfix for CompositeBuffer::IterateRange when using DecompressToComposite for actually comrpessed data
revert of earlier optimization
* faster compress/decompress for large disk-based files
* enable progress feedback in IoHash::HashBuffer
* add payload validation in HttpClient::Get
* fix range requests (range is including end byte)
* remove BuildPartId for blob/block related operations in builds api

1 files changed, 29 insertions, 56 deletions

diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp
index d56927b44..06ac6ae36 100644
--- a/src/zenutil/jupiter/jupitersession.cpp
+++ b/src/zenutil/jupiter/jupitersession.cpp
@@ -436,15 +436,14 @@ JupiterResult
 JupiterSession::PutBuildBlob(std::string_view		Namespace,
 							 std::string_view		BucketId,
 							 const Oid&				BuildId,
-							 const Oid&				PartId,
 							 const IoHash&			Hash,
 							 ZenContentType			ContentType,
 							 const CompositeBuffer& Payload)
 {
-	HttpClient::Response Response = m_HttpClient.Upload(
-		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
-		Payload,
-		ContentType);
+	HttpClient::Response Response =
+		m_HttpClient.Upload(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()),
+							Payload,
+							ContentType);
 	return detail::ConvertResponse(Response, "JupiterSession::PutBuildBlob"sv);
 }
 
@@ -452,7 +451,6 @@ JupiterResult
 JupiterSession::PutMultipartBuildBlob(std::string_view												  Namespace,
 									  std::string_view												  BucketId,
 									  const Oid&													  BuildId,
-									  const Oid&													  PartId,
 									  const IoHash&													  Hash,
 									  ZenContentType												  ContentType,
 									  uint64_t														  PayloadSize,
@@ -498,12 +496,8 @@ JupiterSession::PutMultipartBuildBlob(std::string_view												  Namespace,
 	StartMultipartPayloadWriter.AddInteger("blobLength"sv, PayloadSize);
 	CbObject StartMultipartPayload = StartMultipartPayloadWriter.Save();
 
-	std::string StartMultipartResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/startMultipartUpload",
-																  Namespace,
-																  BucketId,
-																  BuildId,
-																  PartId,
-																  Hash.ToHexString());
+	std::string StartMultipartResponseRequestString =
+		fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/startMultipartUpload", Namespace, BucketId, BuildId, Hash.ToHexString());
 	// ZEN_INFO("POST: {}", StartMultipartResponseRequestString);
 	HttpClient::Response StartMultipartResponse =
 		m_HttpClient.Post(StartMultipartResponseRequestString, StartMultipartPayload, HttpClient::Accept(ZenContentType::kCbObject));
@@ -529,15 +523,14 @@ JupiterSession::PutMultipartBuildBlob(std::string_view												  Namespace,
 
 	for (size_t PartIndex = 0; PartIndex < Workload->PartDescription.Parts.size(); PartIndex++)
 	{
-		OutWorkItems.emplace_back([this, Namespace, BucketId, BuildId, PartId, Hash, ContentType, Workload, PartIndex](
+		OutWorkItems.emplace_back([this, Namespace, BucketId, BuildId, Hash, ContentType, Workload, PartIndex](
 									  bool& OutIsComplete) -> JupiterResult {
 			const MultipartUploadResponse::Part& Part		 = Workload->PartDescription.Parts[PartIndex];
 			IoBuffer							 PartPayload = Workload->Transmitter(Part.FirstByte, Part.LastByte - Part.FirstByte);
-			std::string MultipartUploadResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/uploadMultipart{}",
+			std::string MultipartUploadResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}",
 																		   Namespace,
 																		   BucketId,
 																		   BuildId,
-																		   PartId,
 																		   Hash.ToHexString(),
 																		   Part.QueryString);
 			// ZEN_INFO("PUT: {}", MultipartUploadResponseRequestString);
@@ -571,12 +564,7 @@ JupiterSession::PutMultipartBuildBlob(std::string_view												  Namespace,
 					CbObject CompletePayload = CompletePayloadWriter.Save();
 
 					std::string MultipartEndResponseRequestString =
-						fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/completeMultipart",
-									Namespace,
-									BucketId,
-									BuildId,
-									PartId,
-									Hash.ToHexString());
+						fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/completeMultipart", Namespace, BucketId, BuildId, Hash.ToHexString());
 
 					MultipartEndResponse = m_HttpClient.Post(MultipartEndResponseRequestString,
 															 CompletePayload,
@@ -600,13 +588,12 @@ JupiterSession::PutMultipartBuildBlob(std::string_view												  Namespace,
 								size_t								 RetryPartIndex = PartNameToIndex.at(RetryPartId);
 								const MultipartUploadResponse::Part& RetryPart		= Workload->PartDescription.Parts[RetryPartIndex];
 								IoBuffer							 RetryPartPayload =
-									Workload->Transmitter(RetryPart.FirstByte, RetryPart.LastByte - RetryPart.FirstByte);
+									Workload->Transmitter(RetryPart.FirstByte, RetryPart.LastByte - RetryPart.FirstByte - 1);
 								std::string RetryMultipartUploadResponseRequestString =
-									fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/uploadMultipart{}",
+									fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}",
 												Namespace,
 												BucketId,
 												BuildId,
-												RetryPartId,
 												Hash.ToHexString(),
 												RetryPart.QueryString);
 
@@ -642,14 +629,12 @@ JupiterResult
 JupiterSession::GetMultipartBuildBlob(std::string_view																		 Namespace,
 									  std::string_view																		 BucketId,
 									  const Oid&																			 BuildId,
-									  const Oid&																			 PartId,
 									  const IoHash&																			 Hash,
 									  uint64_t																				 ChunkSize,
 									  std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver,
 									  std::vector<std::function<JupiterResult()>>&											 OutWorkItems)
 {
-	std::string RequestUrl =
-		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString());
+	std::string			 RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString());
 	HttpClient::Response Response =
 		m_HttpClient.Get(RequestUrl, HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", 0, ChunkSize - 1)}}));
 	if (Response.IsSuccess())
@@ -683,17 +668,12 @@ JupiterSession::GetMultipartBuildBlob(std::string_view																		 Namespa
 						{
 							uint64_t PartSize = Min(ChunkSize, TotalSize - Offset);
 							OutWorkItems.emplace_back(
-								[this, Namespace, BucketId, BuildId, PartId, Hash, TotalSize, Workload, Offset, PartSize]()
-									-> JupiterResult {
-									std::string			 RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}",
-																		  Namespace,
-																		  BucketId,
-																		  BuildId,
-																		  PartId,
-																		  Hash.ToHexString());
-									HttpClient::Response Response	= m_HttpClient.Get(
-										  RequestUrl,
-										  HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}}));
+								[this, Namespace, BucketId, BuildId, Hash, TotalSize, Workload, Offset, PartSize]() -> JupiterResult {
+									std::string RequestUrl =
+										fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString());
+									HttpClient::Response Response = m_HttpClient.Get(
+										RequestUrl,
+										HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}}));
 									if (Response.IsSuccess())
 									{
 										uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Response.ResponsePayload.GetSize());
@@ -717,13 +697,12 @@ JupiterResult
 JupiterSession::GetBuildBlob(std::string_view	   Namespace,
 							 std::string_view	   BucketId,
 							 const Oid&			   BuildId,
-							 const Oid&			   PartId,
 							 const IoHash&		   Hash,
 							 std::filesystem::path TempFolderPath)
 {
-	HttpClient::Response Response = m_HttpClient.Download(
-		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
-		TempFolderPath);
+	HttpClient::Response Response =
+		m_HttpClient.Download(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()),
+							  TempFolderPath);
 
 	return detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv);
 }
@@ -732,14 +711,13 @@ JupiterResult
 JupiterSession::PutBlockMetadata(std::string_view Namespace,
 								 std::string_view BucketId,
 								 const Oid&		  BuildId,
-								 const Oid&		  PartId,
 								 const IoHash&	  Hash,
 								 const IoBuffer&  Payload)
 {
 	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
-	HttpClient::Response Response = m_HttpClient.Put(
-		fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/{}/metadata", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
-		Payload);
+	HttpClient::Response Response =
+		m_HttpClient.Put(fmt::format("/api/v2/builds/{}/{}/{}/blocks/{}/metadata", Namespace, BucketId, BuildId, Hash.ToHexString()),
+						 Payload);
 	return detail::ConvertResponse(Response, "JupiterSession::PutBlockMetadata"sv);
 }
 
@@ -772,24 +750,19 @@ JupiterSession::FinalizeBuildPart(std::string_view Namespace,
 }
 
 JupiterResult
-JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId)
+JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId)
 {
-	HttpClient::Response Response =
-		m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/listBlocks", Namespace, BucketId, BuildId, PartId),
-						 HttpClient::Accept(ZenContentType::kCbObject));
+	HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/blocks/listBlocks", Namespace, BucketId, BuildId),
+													 HttpClient::Accept(ZenContentType::kCbObject));
 	return detail::ConvertResponse(Response, "JupiterSession::FindBlocks"sv);
 }
 
 JupiterResult
-JupiterSession::GetBlockMetadata(std::string_view Namespace,
-								 std::string_view BucketId,
-								 const Oid&		  BuildId,
-								 const Oid&		  PartId,
-								 IoBuffer		  Payload)
+JupiterSession::GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload)
 {
 	ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
 	HttpClient::Response Response =
-		m_HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/getBlockMetadata", Namespace, BucketId, BuildId, PartId),
+		m_HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/blocks/getBlockMetadata", Namespace, BucketId, BuildId),
 						  Payload,
 						  HttpClient::Accept(ZenContentType::kCbObject));
 	return detail::ConvertResponse(Response, "JupiterSession::GetBlockMetadata"sv);