refactor use chunk sequence download (#291)

* work on chunk sequences on download, not paths

* write chunksequences to .tmp file and move when complete

* cleanup

* Added on the fly validation `zen builds download` of files built from smaller chunks as each file is completed
Added `--verify` option to `zen builds upload` to verify all uploaded data once entire upload is complete
Added `--verify` option to `zen builds download` to verify all files in target folder once entire download is complete
Fixed/improved progress updated
Multithreaded part validation

* added rates to Write Chunks task

* b/s -> bits/s

* dont validate partial content as complete payload

* handle legacy c# builds

1 files changed, 40 insertions, 13 deletions

diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h
index 15c687462..309341550 100644
--- a/src/zenutil/include/zenutil/chunkedcontent.h
+++ b/src/zenutil/include/zenutil/chunkedcontent.h
@@ -122,32 +122,59 @@ ChunkedFolderContent ChunkFolderContent(ChunkingStatistics&												  Stats,
 
 struct ChunkedContentLookup
 {
-	struct ChunkLocation
+	struct ChunkSequenceLocation
 	{
-		uint32_t PathIndex;
+		uint32_t SequenceIndex;
 		uint64_t Offset;
 	};
 	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
-	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceRawHashIndex;
-	std::vector<uint32_t>							 SequenceRawHashIndexChunkOrderOffset;
-	std::vector<ChunkLocation>						 ChunkLocations;
-	std::vector<size_t>	  ChunkLocationOffset;	// ChunkLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex
-	std::vector<uint32_t> ChunkLocationCounts;	// ChunkLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex
+	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceIndex;
+	std::vector<uint32_t>							 SequenceIndexChunkOrderOffset;
+	std::vector<ChunkSequenceLocation>				 ChunkSequenceLocations;
+	std::vector<size_t>
+		ChunkSequenceLocationOffset;  // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex
+	std::vector<uint32_t> ChunkSequenceLocationCounts;	// ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex
+	std::vector<uint32_t> SequenceIndexFirstPathIndex;	// SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash
 };
 
 ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content);
 
 inline std::pair<size_t, uint32_t>
-GetChunkLocationRange(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex)
+GetChunkSequenceLocationRange(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex)
 {
-	return std::make_pair(Lookup.ChunkLocationOffset[ChunkIndex], Lookup.ChunkLocationCounts[ChunkIndex]);
+	return std::make_pair(Lookup.ChunkSequenceLocationOffset[ChunkIndex], Lookup.ChunkSequenceLocationCounts[ChunkIndex]);
 }
 
-inline std::span<const ChunkedContentLookup::ChunkLocation>
-GetChunkLocations(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex)
+inline std::span<const ChunkedContentLookup::ChunkSequenceLocation>
+GetChunkSequenceLocations(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex)
 {
-	std::pair<size_t, uint32_t> Range = GetChunkLocationRange(Lookup, ChunkIndex);
-	return std::span<const ChunkedContentLookup::ChunkLocation>(Lookup.ChunkLocations).subspan(Range.first, Range.second);
+	std::pair<size_t, uint32_t> Range = GetChunkSequenceLocationRange(Lookup, ChunkIndex);
+	return std::span<const ChunkedContentLookup::ChunkSequenceLocation>(Lookup.ChunkSequenceLocations).subspan(Range.first, Range.second);
+}
+
+inline uint32_t
+GetSequenceIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash)
+{
+	return Lookup.RawHashToSequenceIndex.at(RawHash);
+}
+
+inline uint32_t
+GetChunkIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash)
+{
+	return Lookup.RawHashToSequenceIndex.at(RawHash);
+}
+
+inline uint32_t
+GetFirstPathIndexForSeqeuenceIndex(const ChunkedContentLookup& Lookup, const uint32_t SequenceIndex)
+{
+	return Lookup.SequenceIndexFirstPathIndex[SequenceIndex];
+}
+
+inline uint32_t
+GetFirstPathIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash)
+{
+	const uint32_t SequenceIndex = GetSequenceIndexForRawHash(Lookup, RawHash);
+	return GetFirstPathIndexForSeqeuenceIndex(Lookup, SequenceIndex);
 }
 
 namespace compactbinary_helpers {