moving and small refactor of chunk blocks to prepare for builds api (#282)

2 files changed, 90 insertions, 0 deletions

diff --git a/src/zenutil/include/zenutil/chunkblock.h b/src/zenutil/include/zenutil/chunkblock.h
new file mode 100644
index 000000000..9b7414629
--- /dev/null
+++ b/src/zenutil/include/zenutil/chunkblock.h
@@ -0,0 +1,32 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iohash.h>
+
+#include <zencore/compactbinary.h>
+#include <zencore/compress.h>
+
+#include <optional>
+#include <vector>
+
+namespace zen {
+
+struct ChunkBlockDescription
+{
+	IoHash				  BlockHash;
+	std::vector<IoHash>	  ChunkHashes;
+	std::vector<uint32_t> ChunkRawLengths;
+};
+
+std::vector<ChunkBlockDescription> ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject);
+ChunkBlockDescription			   ParseChunkBlockDescription(const CbObjectView& BlockObject);
+CbObject						   BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData);
+
+typedef std::function<std::pair<uint64_t, CompressedBuffer>(const IoHash& RawHash)> FetchChunkFunc;
+
+CompressedBuffer GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, ChunkBlockDescription& OutBlock);
+bool			 IterateChunkBlock(const SharedBuffer&														   BlockPayload,
+								   std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor);
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/chunkedfile.h b/src/zenutil/include/zenutil/chunkedfile.h
new file mode 100644
index 000000000..7110ad317
--- /dev/null
+++ b/src/zenutil/include/zenutil/chunkedfile.h
@@ -0,0 +1,58 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/iobuffer.h>
+#include <zencore/iohash.h>
+#include <zencore/zencore.h>
+
+#include <functional>
+#include <vector>
+
+namespace zen {
+
+class BasicFile;
+
+struct ChunkedInfo
+{
+	uint64_t			  RawSize = 0;
+	IoHash				  RawHash;
+	std::vector<uint32_t> ChunkSequence;
+	std::vector<IoHash>	  ChunkHashes;
+};
+
+struct ChunkSource
+{
+	uint64_t Offset;  // 8
+	uint32_t Size;	  // 4
+};
+
+struct ChunkedInfoWithSource
+{
+	ChunkedInfo				 Info;
+	std::vector<ChunkSource> ChunkSources;
+};
+
+struct ChunkedParams
+{
+	bool   UseThreshold = true;
+	size_t MinSize		= (2u * 1024u) - 128u;
+	size_t MaxSize		= (16u * 1024u);
+	size_t AvgSize		= (3u * 1024u);
+};
+
+static const ChunkedParams UShaderByteCodeParams = {.UseThreshold = true, .MinSize = 17280, .MaxSize = 139264, .AvgSize = 36340};
+
+ChunkedInfoWithSource ChunkData(BasicFile&			   RawData,
+								uint64_t			   Offset,
+								uint64_t			   Size,
+								ChunkedParams		   Params		  = {},
+								std::atomic<uint64_t>* BytesProcessed = nullptr);
+void				  Reconstruct(const ChunkedInfo&							   Info,
+								  const std::filesystem::path&					   TargetPath,
+								  std::function<IoBuffer(const IoHash& ChunkHash)> GetChunk);
+IoBuffer			  SerializeChunkedInfo(const ChunkedInfo& Info);
+ChunkedInfo			  DeserializeChunkedInfo(IoBuffer& Buffer);
+
+void chunkedfile_forcelink();
+}  // namespace zen