Add retry with optional resume logic to HttpClient::Download (#639)

- Improvement: Refactored Jupiter upstream to use HttpClient
- Improvement: Added retry and resume logic to HttpClient
- Improvement: Added authentication support to HttpClient
- Improvement: Clearer logging in GCV2 compact of FileCas/BlockStore
- Improvement: Size details in oplog import logging

1 files changed, 84 insertions, 80 deletions

diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index 73a8ad538..5bcb7f5b4 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -1158,88 +1158,73 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 		}
 		ZEN_ASSERT(OldBlockFile);
 
-		ZEN_INFO("Moving {} chunks from '{}' to new block", KeepChunkIndexes.size(), GetBlockPath(m_BlocksBasePath, BlockIndex));
-
 		uint64_t OldBlockSize = OldBlockFile->FileSize();
 
-		std::vector<uint8_t> Chunk;
-		for (const size_t& ChunkIndex : KeepChunkIndexes)
+		if (KeepChunkIndexes.empty())
+		{
+			ZEN_INFO("Dropping all chunks from '{}'", GetBlockPath(m_BlocksBasePath, BlockIndex));
+		}
+		else
 		{
-			const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex];
-			if (ChunkLocation.Offset + ChunkLocation.Size > OldBlockSize)
+			std::vector<uint8_t> Chunk;
+			for (const size_t& ChunkIndex : KeepChunkIndexes)
 			{
-				ZEN_WARN(
-					"Compact Block skipping chunk outside of block range in '{}', Chunk start {}, Chunk size {} in Block {}, Block "
-					"size {}",
-					m_BlocksBasePath,
-					ChunkLocation.Offset,
-					ChunkLocation.Size,
-					OldBlockFile->GetPath(),
-					OldBlockSize);
-				continue;
-			}
+				const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex];
+				if (ChunkLocation.Offset + ChunkLocation.Size > OldBlockSize)
+				{
+					ZEN_WARN(
+						"Compact Block skipping chunk outside of block range in '{}', Chunk start {}, Chunk size {} in Block {}, Block "
+						"size {}",
+						m_BlocksBasePath,
+						ChunkLocation.Offset,
+						ChunkLocation.Size,
+						OldBlockFile->GetPath(),
+						OldBlockSize);
+					continue;
+				}
 
-			Chunk.resize(ChunkLocation.Size);
-			OldBlockFile->Read(Chunk.data(), Chunk.size(), ChunkLocation.Offset);
+				Chunk.resize(ChunkLocation.Size);
+				OldBlockFile->Read(Chunk.data(), Chunk.size(), ChunkLocation.Offset);
 
-			if ((WriteOffset + Chunk.size()) > m_MaxBlockSize)
-			{
-				if (NewBlockFile)
+				if ((WriteOffset + Chunk.size()) > m_MaxBlockSize)
 				{
-					ZEN_ASSERT_SLOW(NewBlockFile->IsOpen());
-					NewBlockFile->Flush();
-					MovedSize += NewBlockFile->FileSize();
-					NewBlockFile = nullptr;
+					if (NewBlockFile)
+					{
+						ZEN_ASSERT_SLOW(NewBlockFile->IsOpen());
+						NewBlockFile->Flush();
+						MovedSize += NewBlockFile->FileSize();
+						NewBlockFile = nullptr;
 
-					ZEN_ASSERT(!MovedChunks.empty() || RemovedSize > 0);  // We should not have a new block if we haven't moved anything
+						ZEN_ASSERT(!MovedChunks.empty() || RemovedSize > 0);  // We should not have a new block if we haven't moved anything
 
-					if (!ReportChanges())
-					{
-						return false;
+						if (!ReportChanges())
+						{
+							return false;
+						}
 					}
-				}
 
-				uint32_t NextBlockIndex = m_WriteBlockIndex.load(std::memory_order_relaxed);
-				{
-					RwLock::ExclusiveLockScope InsertLock(m_InsertLock);
-					std::filesystem::path	   NewBlockPath;
-					NextBlockIndex = GetFreeBlockIndex(NextBlockIndex, InsertLock, NewBlockPath);
-					if (NextBlockIndex == (uint32_t)m_MaxBlockCount)
+					uint32_t NextBlockIndex = m_WriteBlockIndex.load(std::memory_order_relaxed);
 					{
-						ZEN_ERROR("unable to allocate a new block in '{}', count limit {} exeeded",
-								  m_BlocksBasePath,
-								  static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1);
-						return false;
-					}
-
-					NewBlockFile				  = new BlockStoreFile(NewBlockPath);
-					m_ChunkBlocks[NextBlockIndex] = NewBlockFile;
-				}
+						RwLock::ExclusiveLockScope InsertLock(m_InsertLock);
+						std::filesystem::path	   NewBlockPath;
+						NextBlockIndex = GetFreeBlockIndex(NextBlockIndex, InsertLock, NewBlockPath);
+						if (NextBlockIndex == (uint32_t)m_MaxBlockCount)
+						{
+							ZEN_ERROR("unable to allocate a new block in '{}', count limit {} exeeded",
+									  m_BlocksBasePath,
+									  static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1);
+							return false;
+						}
 
-				std::error_code Error;
-				DiskSpace		Space = DiskSpaceInfo(m_BlocksBasePath, Error);
-				if (Error)
-				{
-					ZEN_ERROR("get disk space in '{}' FAILED, reason: '{}'", m_BlocksBasePath, Error.message());
-					{
-						RwLock::ExclusiveLockScope _l(m_InsertLock);
-						ZEN_ASSERT(m_ChunkBlocks[NextBlockIndex] == NewBlockFile);
-						m_ChunkBlocks.erase(NextBlockIndex);
+						NewBlockFile				  = new BlockStoreFile(NewBlockPath);
+						m_ChunkBlocks[NextBlockIndex] = NewBlockFile;
 					}
-					ZEN_ASSERT_SLOW(!NewBlockFile->IsOpen());
-					NewBlockFile = nullptr;
-					return false;
-				}
 
-				if (Space.Free < m_MaxBlockSize)
-				{
-					uint64_t ReclaimedSpace = DiskReserveCallback();
-					if (Space.Free + ReclaimedSpace < m_MaxBlockSize)
+					std::error_code Error;
+					DiskSpace		Space = DiskSpaceInfo(m_BlocksBasePath, Error);
+					if (Error)
 					{
-						ZEN_WARN("garbage collect for '{}' FAILED, required disk space {}, free {}",
-								 m_BlocksBasePath,
-								 m_MaxBlockSize,
-								 NiceBytes(Space.Free + ReclaimedSpace));
+						ZEN_ERROR("get disk space in '{}' FAILED, reason: '{}'", m_BlocksBasePath, Error.message());
 						{
 							RwLock::ExclusiveLockScope _l(m_InsertLock);
 							ZEN_ASSERT(m_ChunkBlocks[NextBlockIndex] == NewBlockFile);
@@ -1250,23 +1235,42 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 						return false;
 					}
 
-					ZEN_INFO("using gc reserve for '{}', reclaimed {}, disk free {}",
-							 m_BlocksBasePath,
-							 ReclaimedSpace,
-							 NiceBytes(Space.Free + ReclaimedSpace));
+					if (Space.Free < m_MaxBlockSize)
+					{
+						uint64_t ReclaimedSpace = DiskReserveCallback();
+						if (Space.Free + ReclaimedSpace < m_MaxBlockSize)
+						{
+							ZEN_WARN("garbage collect for '{}' FAILED, required disk space {}, free {}",
+									 m_BlocksBasePath,
+									 m_MaxBlockSize,
+									 NiceBytes(Space.Free + ReclaimedSpace));
+							{
+								RwLock::ExclusiveLockScope _l(m_InsertLock);
+								ZEN_ASSERT(m_ChunkBlocks[NextBlockIndex] == NewBlockFile);
+								m_ChunkBlocks.erase(NextBlockIndex);
+							}
+							ZEN_ASSERT_SLOW(!NewBlockFile->IsOpen());
+							NewBlockFile = nullptr;
+							return false;
+						}
+
+						ZEN_INFO("using gc reserve for '{}', reclaimed {}, disk free {}",
+								 m_BlocksBasePath,
+								 ReclaimedSpace,
+								 NiceBytes(Space.Free + ReclaimedSpace));
+					}
+					NewBlockFile->Create(m_MaxBlockSize);
+					NewBlockIndex = NextBlockIndex;
+					WriteOffset	  = 0;
 				}
-				NewBlockFile->Create(m_MaxBlockSize);
-				NewBlockIndex = NextBlockIndex;
-				WriteOffset	  = 0;
-			}
 
-			NewBlockFile->Write(Chunk.data(), ChunkLocation.Size, WriteOffset);
-			MovedChunks.push_back(
-				{ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = gsl::narrow<uint32_t>(WriteOffset), .Size = ChunkLocation.Size}});
-			WriteOffset = RoundUp(WriteOffset + ChunkLocation.Size, PayloadAlignment);
-			AddedSize += Chunk.size();
+				NewBlockFile->Write(Chunk.data(), ChunkLocation.Size, WriteOffset);
+				MovedChunks.push_back(
+					{ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = gsl::narrow<uint32_t>(WriteOffset), .Size = ChunkLocation.Size}});
+				WriteOffset = RoundUp(WriteOffset + ChunkLocation.Size, PayloadAlignment);
+				AddedSize += Chunk.size();
+			}
 		}
-		Chunk.clear();
 
 		if (!ReportChanges())
 		{