move chunking code to zenremotestore lib (#545)

4 files changed, 0 insertions, 462 deletions

diff --git a/src/zenutil/include/zenutil/chunkblock.h b/src/zenutil/include/zenutil/chunkblock.h
deleted file mode 100644
index 277580c74..000000000
--- a/src/zenutil/include/zenutil/chunkblock.h
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zencore/iohash.h>
-
-#include <zencore/compactbinary.h>
-#include <zencore/compress.h>
-
-#include <optional>
-#include <vector>
-
-namespace zen {
-
-struct ThinChunkBlockDescription
-{
-	IoHash				BlockHash;
-	std::vector<IoHash> ChunkRawHashes;
-};
-
-struct ChunkBlockDescription : public ThinChunkBlockDescription
-{
-	uint64_t			  HeaderSize;
-	std::vector<uint32_t> ChunkRawLengths;
-	std::vector<uint32_t> ChunkCompressedLengths;
-};
-
-std::vector<ChunkBlockDescription> ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject);
-ChunkBlockDescription			   ParseChunkBlockDescription(const CbObjectView& BlockObject);
-CbObject						   BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData);
-ChunkBlockDescription			   GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash);
-typedef std::function<std::pair<uint64_t, CompressedBuffer>(const IoHash& RawHash)> FetchChunkFunc;
-
-CompressedBuffer	  GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, ChunkBlockDescription& OutBlock);
-bool				  IterateChunkBlock(const SharedBuffer&															BlockPayload,
-										std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor,
-										uint64_t&																	OutHeaderSize);
-std::vector<uint32_t> ReadChunkBlockHeader(const MemoryView BlockView, uint64_t& OutHeaderSize);
-
-}  // namespace zen
diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h
deleted file mode 100644
index 306a5d990..000000000
--- a/src/zenutil/include/zenutil/chunkedcontent.h
+++ /dev/null
@@ -1,288 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zencore/compactbinary.h>
-#include <zencore/compactbinarybuilder.h>
-#include <zencore/iohash.h>
-
-#include <filesystem>
-#include <vector>
-
-ZEN_THIRD_PARTY_INCLUDES_START
-#include <tsl/robin_map.h>
-ZEN_THIRD_PARTY_INCLUDES_END
-
-namespace zen {
-
-class CbWriter;
-class ChunkingController;
-class WorkerThreadPool;
-
-enum class SourcePlatform
-{
-	Windows = 0,
-	Linux	= 1,
-	MacOS	= 2,
-	_Count
-};
-
-std::string_view ToString(SourcePlatform Platform);
-SourcePlatform	 FromString(std::string_view Platform, SourcePlatform Default);
-SourcePlatform	 GetSourceCurrentPlatform();
-
-struct FolderContent
-{
-	SourcePlatform					   Platform = GetSourceCurrentPlatform();
-	std::vector<std::filesystem::path> Paths;
-	std::vector<uint64_t>			   RawSizes;
-	std::vector<uint32_t>			   Attributes;
-	std::vector<uint64_t>			   ModificationTicks;
-
-	bool operator==(const FolderContent& Rhs) const;
-
-	bool		AreKnownFilesEqual(const FolderContent& Rhs) const;
-	void		UpdateState(const FolderContent& Rhs, std::vector<uint32_t>& PathIndexesOufOfDate);
-	static bool AreFileAttributesEqual(const uint32_t Lhs, const uint32_t Rhs);
-};
-
-FolderContent GetUpdatedContent(const FolderContent&				Old,
-								const FolderContent&				New,
-								std::vector<std::filesystem::path>& OutDeletedPathIndexes);
-
-void		  SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output);
-FolderContent LoadFolderContentToCompactBinary(CbObjectView Input);
-
-struct GetFolderContentStatistics
-{
-	std::atomic<uint64_t> FoundFileCount		= 0;
-	std::atomic<uint64_t> FoundFileByteCount	= 0;
-	std::atomic<uint64_t> AcceptedFileCount		= 0;
-	std::atomic<uint64_t> AcceptedFileByteCount = 0;
-	uint64_t			  ElapsedWallTimeUS		= 0;
-};
-
-FolderContent GetFolderContent(GetFolderContentStatistics&																Stats,
-							   const std::filesystem::path&																RootPath,
-							   std::function<bool(const std::string_view& RelativePath)>&&								AcceptDirectory,
-							   std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& AcceptFile,
-							   WorkerThreadPool&																		WorkerPool,
-							   int32_t																					UpdateIntervalMS,
-							   std::function<void(bool IsAborted, std::ptrdiff_t PendingWork)>&&						UpdateCallback,
-							   std::atomic<bool>&																		AbortFlag);
-
-struct ChunkedContentData
-{
-	// To describe one asset with a particular RawHash, find the index of the hash in SequenceRawHashes
-	// ChunkCounts for that index will be the number of indexes in ChunkOrders that describe
-	// the sequence of chunks required to reconstruct the asset.
-	// Offset into ChunkOrders is based on how many entries in ChunkOrders the previous [n - 1] SequenceRawHashes uses
-	std::vector<IoHash>	  SequenceRawHashes;  // Raw hash for Chunk sequence
-	std::vector<uint32_t> ChunkCounts;		  // Chunk count of ChunkOrder for SequenceRawHashes[n]
-	std::vector<uint32_t> ChunkOrders;		  // Chunk sequence indexed into ChunkHashes, ChunkCounts[n] indexes per SequenceRawHashes[n]
-	std::vector<IoHash>	  ChunkHashes;		  // Unique chunk hashes
-	std::vector<uint64_t> ChunkRawSizes;	  // Unique chunk raw size for ChunkHash[n]
-};
-
-struct ChunkedFolderContent
-{
-	SourcePlatform					   Platform = GetSourceCurrentPlatform();
-	std::vector<std::filesystem::path> Paths;
-	std::vector<uint64_t>			   RawSizes;
-	std::vector<uint32_t>			   Attributes;
-	std::vector<IoHash>				   RawHashes;
-	ChunkedContentData				   ChunkedContent;
-};
-
-struct ChunkedContentLookup
-{
-	struct ChunkSequenceLocation
-	{
-		uint32_t SequenceIndex = (uint32_t)-1;
-		uint64_t Offset		   = (uint64_t)-1;
-	};
-	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
-	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceIndex;
-	std::vector<uint32_t>							 SequenceIndexChunkOrderOffset;
-	std::vector<ChunkSequenceLocation>				 ChunkSequenceLocations;
-	std::vector<size_t>
-		ChunkSequenceLocationOffset;  // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex
-	std::vector<uint32_t> ChunkSequenceLocationCounts;	// ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex
-	std::vector<uint32_t> SequenceIndexFirstPathIndex;	// SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash
-	std::vector<uint32_t> PathExtensionHash;
-};
-
-void				 SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output);
-ChunkedFolderContent LoadChunkedFolderContentToCompactBinary(CbObjectView Input);
-
-ChunkedFolderContent MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span<const ChunkedFolderContent> Overlays);
-ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent&			  Base,
-												   const ChunkedContentLookup&			  BaseContentLookup,
-												   std::span<const std::filesystem::path> DeletedPaths);
-ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent& Base, std::span<const std::filesystem::path> DeletedPaths);
-
-struct ChunkingStatistics
-{
-	std::atomic<uint64_t> FilesProcessed	   = 0;
-	std::atomic<uint64_t> FilesChunked		   = 0;
-	std::atomic<uint64_t> BytesHashed		   = 0;
-	std::atomic<uint64_t> UniqueChunksFound	   = 0;
-	std::atomic<uint64_t> UniqueSequencesFound = 0;
-	std::atomic<uint64_t> UniqueBytesFound	   = 0;
-	uint64_t			  ElapsedWallTimeUS	   = 0;
-};
-
-ChunkedFolderContent ChunkFolderContent(ChunkingStatistics&			 Stats,
-										WorkerThreadPool&			 WorkerPool,
-										const std::filesystem::path& RootPath,
-										const FolderContent&		 Content,
-										const ChunkingController&	 InChunkingController,
-										int32_t						 UpdateIntervalMS,
-										std::function<void(bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork)>&& UpdateCallback,
-										std::atomic<bool>&																 AbortFlag,
-										std::atomic<bool>&																 PauseFlag);
-
-ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content);
-
-inline std::pair<size_t, uint32_t>
-GetChunkSequenceLocationRange(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex)
-{
-	return std::make_pair(Lookup.ChunkSequenceLocationOffset[ChunkIndex], Lookup.ChunkSequenceLocationCounts[ChunkIndex]);
-}
-
-inline std::span<const ChunkedContentLookup::ChunkSequenceLocation>
-GetChunkSequenceLocations(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex)
-{
-	std::pair<size_t, uint32_t> Range = GetChunkSequenceLocationRange(Lookup, ChunkIndex);
-	return std::span<const ChunkedContentLookup::ChunkSequenceLocation>(Lookup.ChunkSequenceLocations).subspan(Range.first, Range.second);
-}
-
-inline uint32_t
-GetSequenceIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash)
-{
-	return Lookup.RawHashToSequenceIndex.at(RawHash);
-}
-
-inline uint32_t
-GetChunkIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash)
-{
-	return Lookup.RawHashToSequenceIndex.at(RawHash);
-}
-
-inline uint32_t
-GetFirstPathIndexForSeqeuenceIndex(const ChunkedContentLookup& Lookup, const uint32_t SequenceIndex)
-{
-	return Lookup.SequenceIndexFirstPathIndex[SequenceIndex];
-}
-
-inline uint32_t
-GetFirstPathIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash)
-{
-	const uint32_t SequenceIndex = GetSequenceIndexForRawHash(Lookup, RawHash);
-	return GetFirstPathIndexForSeqeuenceIndex(Lookup, SequenceIndex);
-}
-
-namespace compactbinary_helpers {
-	template<typename Type>
-	void WriteArray(std::span<const Type> Values, std::string_view ArrayName, CbWriter& Output)
-	{
-		Output.BeginArray(ArrayName);
-		for (const Type Value : Values)
-		{
-			Output << Value;
-		}
-		Output.EndArray();
-	}
-
-	template<typename Type>
-	void WriteArray(const std::vector<Type>& Values, std::string_view ArrayName, CbWriter& Output)
-	{
-		WriteArray(std::span<const Type>(Values), ArrayName, Output);
-	}
-
-	template<>
-	inline void WriteArray(std::span<const std::filesystem::path> Values, std::string_view ArrayName, CbWriter& Output)
-	{
-		Output.BeginArray(ArrayName);
-		for (const std::filesystem::path& Path : Values)
-		{
-			Output.AddString((const char*)Path.generic_u8string().c_str());
-		}
-		Output.EndArray();
-	}
-
-	template<>
-	inline void WriteArray(const std::vector<std::filesystem::path>& Values, std::string_view ArrayName, CbWriter& Output)
-	{
-		WriteArray(std::span<const std::filesystem::path>(Values), ArrayName, Output);
-	}
-
-	inline void WriteBinaryAttachmentArray(std::span<const IoHash> Values, std::string_view ArrayName, CbWriter& Output)
-	{
-		Output.BeginArray(ArrayName);
-		for (const IoHash& Hash : Values)
-		{
-			Output.AddBinaryAttachment(Hash);
-		}
-		Output.EndArray();
-	}
-
-	inline void WriteBinaryAttachmentArray(const std::vector<IoHash>& Values, std::string_view ArrayName, CbWriter& Output)
-	{
-		WriteArray(std::span<const IoHash>(Values), ArrayName, Output);
-	}
-
-	inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<uint32_t>& Result)
-	{
-		CbArrayView Array = Input[ArrayName].AsArrayView();
-		Result.reserve(Array.Num());
-		for (CbFieldView ItemView : Array)
-		{
-			Result.push_back(ItemView.AsUInt32());
-		}
-	}
-
-	inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<uint64_t>& Result)
-	{
-		CbArrayView Array = Input[ArrayName].AsArrayView();
-		Result.reserve(Array.Num());
-		for (CbFieldView ItemView : Array)
-		{
-			Result.push_back(ItemView.AsUInt64());
-		}
-	}
-
-	inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<std::filesystem::path>& Result)
-	{
-		CbArrayView Array = Input[ArrayName].AsArrayView();
-		Result.reserve(Array.Num());
-		for (CbFieldView ItemView : Array)
-		{
-			std::u8string_view U8Path = ItemView.AsU8String();
-			Result.push_back(std::filesystem::path(U8Path));
-		}
-	}
-
-	inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<IoHash>& Result)
-	{
-		CbArrayView Array = Input[ArrayName].AsArrayView();
-		Result.reserve(Array.Num());
-		for (CbFieldView ItemView : Array)
-		{
-			Result.push_back(ItemView.AsHash());
-		}
-	}
-
-	inline void ReadBinaryAttachmentArray(std::string_view ArrayName, CbObjectView Input, std::vector<IoHash>& Result)
-	{
-		CbArrayView Array = Input[ArrayName].AsArrayView();
-		Result.reserve(Array.Num());
-		for (CbFieldView ItemView : Array)
-		{
-			Result.push_back(ItemView.AsBinaryAttachment());
-		}
-	}
-
-}  // namespace compactbinary_helpers
-
-}  // namespace zen
diff --git a/src/zenutil/include/zenutil/chunkedfile.h b/src/zenutil/include/zenutil/chunkedfile.h
deleted file mode 100644
index 4cec80fdb..000000000
--- a/src/zenutil/include/zenutil/chunkedfile.h
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zencore/iobuffer.h>
-#include <zencore/iohash.h>
-#include <zencore/zencore.h>
-
-#include <functional>
-#include <vector>
-
-namespace zen {
-
-class BasicFile;
-
-struct ChunkedInfo
-{
-	uint64_t			  RawSize = 0;
-	IoHash				  RawHash;
-	std::vector<uint32_t> ChunkSequence;
-	std::vector<IoHash>	  ChunkHashes;
-};
-
-struct ChunkSource
-{
-	uint64_t Offset;  // 8
-	uint32_t Size;	  // 4
-};
-
-struct ChunkedInfoWithSource
-{
-	ChunkedInfo				 Info;
-	std::vector<ChunkSource> ChunkSources;
-};
-
-struct ChunkedParams
-{
-	bool   UseThreshold = true;
-	size_t MinSize		= (2u * 1024u) - 128u;
-	size_t MaxSize		= (16u * 1024u);
-	size_t AvgSize		= (3u * 1024u);
-};
-
-static const ChunkedParams UShaderByteCodeParams = {.UseThreshold = true, .MinSize = 17280, .MaxSize = 139264, .AvgSize = 36340};
-
-ChunkedInfoWithSource ChunkData(BasicFile&			   RawData,
-								uint64_t			   Offset,
-								uint64_t			   Size,
-								ChunkedParams		   Params		  = {},
-								std::atomic<uint64_t>* BytesProcessed = nullptr,
-								std::atomic<bool>*	   AbortFlag	  = nullptr);
-void				  Reconstruct(const ChunkedInfo&							   Info,
-								  const std::filesystem::path&					   TargetPath,
-								  std::function<IoBuffer(const IoHash& ChunkHash)> GetChunk);
-IoBuffer			  SerializeChunkedInfo(const ChunkedInfo& Info);
-ChunkedInfo			  DeserializeChunkedInfo(IoBuffer& Buffer);
-
-void chunkedfile_forcelink();
-}  // namespace zen
diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h
deleted file mode 100644
index 315502265..000000000
--- a/src/zenutil/include/zenutil/chunkingcontroller.h
+++ /dev/null
@@ -1,75 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zencore/compactbinary.h>
-
-#include <zenutil/chunkedfile.h>
-
-#include <atomic>
-#include <filesystem>
-
-namespace zen {
-
-const std::vector<std::string> DefaultChunkingExcludeExtensions =
-	{".exe", ".dll", ".pdb", ".self", ".mp4", ".zip", ".7z", ".bzip", ".rar", ".gzip"};
-const std::vector<std::string> DefaultFixedChunkingExtensions	= {".apk", ".nsp", ".xvc", ".pkg", ".dmg", ".ipa"};
-const bool					   DefaultChunkingExcludeElfFiles	= true;
-const bool					   DefaultChunkingExcludeMachOFiles = true;
-
-const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u,
-											.MaxSize = 128u * 1024u,
-											.AvgSize = ((8u * 4u) * 1024u) + 128u};
-
-const size_t DefaultChunkingFileSizeLimit = DefaultChunkedParams.MaxSize;
-
-const uint64_t DefaultFixedChunkingChunkSize  = 32u * 1024u * 1024u;
-const uint64_t DefaultMinSizeForFixedChunking = DefaultFixedChunkingChunkSize * 8u;
-
-struct ChunkedInfoWithSource;
-
-class ChunkingController
-{
-public:
-	virtual ~ChunkingController() {}
-
-	// Return true if the input file was processed. If true is returned OutChunked will contain the chunked info
-	virtual bool			 ProcessFile(const std::filesystem::path& InputPath,
-										 uint64_t					  RawSize,
-										 ChunkedInfoWithSource&		  OutChunked,
-										 std::atomic<uint64_t>&		  BytesProcessed,
-										 std::atomic<bool>&			  AbortFlag) const = 0;
-	virtual std::string_view GetName() const								 = 0;
-	virtual CbObject		 GetParameters() const							 = 0;
-};
-
-struct BasicChunkingControllerSettings
-{
-	std::vector<std::string> ExcludeExtensions	= DefaultChunkingExcludeExtensions;
-	bool					 ExcludeElfFiles	= DefaultChunkingExcludeElfFiles;
-	bool					 ExcludeMachOFiles	= DefaultChunkingExcludeMachOFiles;
-	uint64_t				 ChunkFileSizeLimit = DefaultChunkingFileSizeLimit;
-	ChunkedParams			 ChunkingParams		= DefaultChunkedParams;
-};
-
-std::unique_ptr<ChunkingController> CreateBasicChunkingController(const BasicChunkingControllerSettings& Settings);
-std::unique_ptr<ChunkingController> CreateBasicChunkingController(CbObjectView Parameters);
-
-struct ChunkingControllerWithFixedChunkingSettings
-{
-	std::vector<std::string> FixedChunkingExtensions = DefaultFixedChunkingExtensions;
-	std::vector<std::string> ExcludeExtensions		 = DefaultChunkingExcludeExtensions;
-	bool					 ExcludeElfFiles		 = DefaultChunkingExcludeElfFiles;
-	bool					 ExcludeMachOFiles		 = DefaultChunkingExcludeMachOFiles;
-	uint64_t				 ChunkFileSizeLimit		 = DefaultChunkingFileSizeLimit;
-	ChunkedParams			 ChunkingParams			 = DefaultChunkedParams;
-	uint64_t				 FixedChunkingChunkSize	 = DefaultFixedChunkingChunkSize;
-	uint64_t				 MinSizeForFixedChunking = DefaultMinSizeForFixedChunking;
-};
-
-std::unique_ptr<ChunkingController> CreateChunkingControllerWithFixedChunking(const ChunkingControllerWithFixedChunkingSettings& Setting);
-std::unique_ptr<ChunkingController> CreateChunkingControllerWithFixedChunking(CbObjectView Parameters);
-
-std::unique_ptr<ChunkingController> CreateChunkingController(std::string_view Name, CbObjectView Parameters);
-
-}  // namespace zen