builds scanning cache (#727)v5.7.19-pre0

- Feature: Added `--chunking-cache-path` option to `zen builds upload` and `zen builds diff`
  - Path to cache for chunking information of scanned files. Default is empty resulting in no caching

4 files changed, 65 insertions, 1 deletions

diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
index 057ad1a10..9f7d93398 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h
@@ -551,7 +551,9 @@ public:
 
 	std::vector<std::pair<Oid, std::string>> Execute(const Oid&					  BuildPartId,
 													 const std::string_view		  BuildPartName,
-													 const std::filesystem::path& ManifestPath);
+													 const std::filesystem::path& ManifestPath,
+													 ChunkingController&		  ChunkController,
+													 ChunkingCache&				  ChunkCache);
 
 	DiskStatistics			   m_DiskStats;
 	GetFolderContentStatistics m_LocalFolderScanStats;
@@ -659,6 +661,7 @@ private:
 	};
 
 	void UploadBuildPart(ChunkingController& ChunkController,
+						 ChunkingCache&		 ChunkCache,
 						 uint32_t			 PartIndex,
 						 const UploadPart&	 Part,
 						 uint32_t			 PartStepOffset,
diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h
index 1e8e878df..d402bd3f0 100644
--- a/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h
+++ b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h
@@ -17,6 +17,7 @@ namespace zen {
 
 class CbWriter;
 class ChunkingController;
+class ChunkingCache;
 class WorkerThreadPool;
 
 enum class SourcePlatform
@@ -165,6 +166,12 @@ struct ChunkingStatistics
 	std::atomic<uint64_t> UniqueChunksFound	   = 0;
 	std::atomic<uint64_t> UniqueSequencesFound = 0;
 	std::atomic<uint64_t> UniqueBytesFound	   = 0;
+	std::atomic<uint64_t> FilesFoundInCache	   = 0;
+	std::atomic<uint64_t> ChunksFoundInCache   = 0;
+	std::atomic<uint64_t> BytesFoundInCache	   = 0;
+	std::atomic<uint64_t> FilesStoredInCache   = 0;
+	std::atomic<uint64_t> ChunksStoredInCache  = 0;
+	std::atomic<uint64_t> BytesStoredInCache   = 0;
 	uint64_t			  ElapsedWallTimeUS	   = 0;
 
 	inline ChunkingStatistics& operator+=(const ChunkingStatistics& Rhs)
@@ -176,6 +183,12 @@ struct ChunkingStatistics
 		UniqueSequencesFound += Rhs.UniqueSequencesFound;
 		UniqueBytesFound += Rhs.UniqueBytesFound;
 		ElapsedWallTimeUS += Rhs.ElapsedWallTimeUS;
+		FilesFoundInCache += Rhs.FilesFoundInCache;
+		ChunksFoundInCache += Rhs.ChunksFoundInCache;
+		BytesFoundInCache += Rhs.BytesFoundInCache;
+		FilesStoredInCache += Rhs.FilesStoredInCache;
+		ChunksStoredInCache += Rhs.ChunksStoredInCache;
+		BytesStoredInCache += Rhs.BytesStoredInCache;
 		return *this;
 	}
 };
@@ -185,6 +198,7 @@ ChunkedFolderContent ChunkFolderContent(ChunkingStatistics&			 Stats,
 										const std::filesystem::path& RootPath,
 										const FolderContent&		 Content,
 										const ChunkingController&	 InChunkingController,
+										ChunkingCache&				 InChunkingCache,
 										int32_t						 UpdateIntervalMS,
 										std::function<void(bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork)>&& UpdateCallback,
 										std::atomic<bool>&																 AbortFlag,
diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkedfile.h b/src/zenremotestore/include/zenremotestore/chunking/chunkedfile.h
index 4cec80fdb..64e2c9c29 100644
--- a/src/zenremotestore/include/zenremotestore/chunking/chunkedfile.h
+++ b/src/zenremotestore/include/zenremotestore/chunking/chunkedfile.h
@@ -21,11 +21,14 @@ struct ChunkedInfo
 	std::vector<IoHash>	  ChunkHashes;
 };
 
+#pragma pack(push)
+#pragma pack(4)
 struct ChunkSource
 {
 	uint64_t Offset;  // 8
 	uint32_t Size;	  // 4
 };
+#pragma pack(pop)
 
 struct ChunkedInfoWithSource
 {
diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkingcache.h b/src/zenremotestore/include/zenremotestore/chunking/chunkingcache.h
new file mode 100644
index 000000000..e213bc41b
--- /dev/null
+++ b/src/zenremotestore/include/zenremotestore/chunking/chunkingcache.h
@@ -0,0 +1,44 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <filesystem>
+
+namespace zen {
+
+struct ChunkedInfoWithSource;
+class ChunkingController;
+
+class ChunkingCache
+{
+public:
+	virtual ~ChunkingCache() {}
+
+	/*
+	 * Attempting to fetch a cached file with mismatching RawSize of ModificationTick will delete any existing cached data for that
+	 * InputPath
+	 *
+	 * If GetCachedFile returns false, OutChunked is untouched
+	 */
+	virtual bool GetCachedFile(const std::filesystem::path& InputPath,
+							   uint64_t						RawSize,
+							   uint64_t						ModificationTick,
+							   ChunkedInfoWithSource&		OutChunked) = 0;
+
+	/*
+	 * Putting a cached entry with an existing InputPath will overwrite it with the new ModificationTick and Chunked data
+	 */
+	virtual bool PutCachedFile(const std::filesystem::path& InputPath, uint64_t ModificationTick, const ChunkedInfoWithSource& Chunked) = 0;
+};
+
+std::unique_ptr<ChunkingCache> CreateNullChunkingCache();
+std::unique_ptr<ChunkingCache> CreateMemoryChunkingCache();
+std::unique_ptr<ChunkingCache> CreateDiskChunkingCache(const std::filesystem::path& RootPath,
+													   ChunkingController&			ChunkController,
+													   uint64_t						MinimumRawSizeForCaching);
+
+#if ZEN_WITH_TESTS
+void chunkingcache_forcelink();
+#endif	// ZEN_WITH_TESTS
+
+}  // namespace zen