pick up existing cache (#299)

- Improvement: Scavenge .zen temp folders for existing data (downloaded, decompressed or written) from previous failed run
- Improvement: Faster abort during stream compression
- Improvement: Try to move downloaded blobs with rename if possible avoiding an extra disk write
- Improvement: Only clean temp folders on successful or cancelled build - keep it if download fails

2 files changed, 42 insertions, 18 deletions

diff --git a/src/zenutil/chunkblock.cpp b/src/zenutil/chunkblock.cpp
index a19cf5c1b..f3c14edc4 100644
--- a/src/zenutil/chunkblock.cpp
+++ b/src/zenutil/chunkblock.cpp
@@ -187,31 +187,54 @@ GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks,
 	return CompressedBlock;
 }
 
-bool
-IterateChunkBlock(const SharedBuffer&														  BlockPayload,
-				  std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor,
-				  uint64_t&																	  OutHeaderSize)
+std::vector<uint32_t>
+ReadChunkBlockHeader(const MemoryView BlockView, uint64_t& OutHeaderSize)
 {
-	ZEN_ASSERT(BlockPayload);
-	if (BlockPayload.GetSize() < 1)
-	{
-		return false;
-	}
-
-	MemoryView	   BlockView = BlockPayload.GetView();
-	const uint8_t* ReadPtr	 = reinterpret_cast<const uint8_t*>(BlockView.GetData());
+	const uint8_t* ReadPtr = reinterpret_cast<const uint8_t*>(BlockView.GetData());
 	uint32_t	   NumberSize;
 	uint64_t	   ChunkCount = ReadVarUInt(ReadPtr, NumberSize);
 	ReadPtr += NumberSize;
-	std::vector<uint64_t> ChunkSizes;
+	std::vector<uint32_t> ChunkSizes;
 	ChunkSizes.reserve(ChunkCount);
 	while (ChunkCount--)
 	{
-		ChunkSizes.push_back(ReadVarUInt(ReadPtr, NumberSize));
+		if (ReadPtr >= BlockView.GetDataEnd())
+		{
+			throw std::runtime_error("Invalid block header, block data ended unexpectedly");
+		}
+		uint64_t ChunkSize = ReadVarUInt(ReadPtr, NumberSize);
+		if (ChunkSize > std::numeric_limits<uint32_t>::max())
+		{
+			throw std::runtime_error("Invalid block header, header data is corrupt");
+		}
+		if (ChunkSize < 1)
+		{
+			throw std::runtime_error("Invalid block header, header data is corrupt");
+		}
+		ChunkSizes.push_back(gsl::narrow<uint32_t>(ChunkSize));
 		ReadPtr += NumberSize;
 	}
 	uint64_t Offset = std::distance((const uint8_t*)BlockView.GetData(), ReadPtr);
 	OutHeaderSize	= Offset;
+	return ChunkSizes;
+}
+
+bool
+IterateChunkBlock(const SharedBuffer&														  BlockPayload,
+				  std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor,
+				  uint64_t&																	  OutHeaderSize)
+{
+	ZEN_ASSERT(BlockPayload);
+	if (BlockPayload.GetSize() < 1)
+	{
+		return false;
+	}
+
+	MemoryView BlockView = BlockPayload.GetView();
+
+	std::vector<uint32_t> ChunkSizes = ReadChunkBlockHeader(BlockView, OutHeaderSize);
+	uint64_t			  Offset	 = OutHeaderSize;
+	OutHeaderSize					 = Offset;
 	for (uint64_t ChunkSize : ChunkSizes)
 	{
 		IoBuffer		 Chunk(BlockPayload.AsIoBuffer(), Offset, ChunkSize);
diff --git a/src/zenutil/include/zenutil/chunkblock.h b/src/zenutil/include/zenutil/chunkblock.h
index 21107fb7c..277580c74 100644
--- a/src/zenutil/include/zenutil/chunkblock.h
+++ b/src/zenutil/include/zenutil/chunkblock.h
@@ -31,9 +31,10 @@ CbObject						   BuildChunkBlockDescription(const ChunkBlockDescription& Block,
 ChunkBlockDescription			   GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash);
 typedef std::function<std::pair<uint64_t, CompressedBuffer>(const IoHash& RawHash)> FetchChunkFunc;
 
-CompressedBuffer GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, ChunkBlockDescription& OutBlock);
-bool			 IterateChunkBlock(const SharedBuffer&														   BlockPayload,
-								   std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor,
-								   uint64_t&																   OutHeaderSize);
+CompressedBuffer	  GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, ChunkBlockDescription& OutBlock);
+bool				  IterateChunkBlock(const SharedBuffer&															BlockPayload,
+										std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor,
+										uint64_t&																	OutHeaderSize);
+std::vector<uint32_t> ReadChunkBlockHeader(const MemoryView BlockView, uint64_t& OutHeaderSize);
 
 }  // namespace zen