chunking moved to zenstore (#490)

* moved chunking into zenstore
* removed vestiges of experimental chunking command

1 files changed, 56 insertions, 0 deletions

diff --git a/src/zenstore/chunking.h b/src/zenstore/chunking.h
new file mode 100644
index 000000000..09c56454f
--- /dev/null
+++ b/src/zenstore/chunking.h
@@ -0,0 +1,56 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+#include <zencore/zencore.h>
+
+namespace zen {
+
+/** Content-defined chunking helper
+ */
+class ZenChunkHelper
+{
+public:
+	void   SetChunkSize(size_t MinSize, size_t MaxSize, size_t AvgSize);
+	size_t ScanChunk(const void* DataBytes, size_t ByteCount);
+	void   Reset();
+
+	// This controls which chunking approach is used - threshold or
+	// modulo based. Threshold is faster and generates similarly sized
+	// chunks
+	void SetUseThreshold(bool NewState) { m_UseThreshold = NewState; }
+
+	inline size_t	ChunkSizeMin() const { return m_ChunkSizeMin; }
+	inline size_t	ChunkSizeMax() const { return m_ChunkSizeMax; }
+	inline size_t	ChunkSizeAvg() const { return m_ChunkSizeAvg; }
+	inline uint64_t BytesScanned() const { return m_BytesScanned; }
+
+	static constexpr size_t kNoBoundaryFound = size_t(~0ull);
+
+private:
+	size_t m_ChunkSizeMin = 0;
+	size_t m_ChunkSizeMax = 0;
+	size_t m_ChunkSizeAvg = 0;
+
+	uint32_t m_Discriminator = 0;  // Computed in SetChunkSize()
+	uint32_t m_Threshold	 = 0;  // Computed in SetChunkSize()
+
+	bool m_UseThreshold = true;
+
+	static constexpr size_t kChunkSizeLimitMax		 = 64 * 1024 * 1024;
+	static constexpr size_t kChunkSizeLimitMin		 = 1024;
+	static constexpr size_t kDefaultAverageChunkSize = 64 * 1024;
+
+	static constexpr int kWindowSize = 48;
+	uint8_t				 m_Window[kWindowSize];
+	uint32_t			 m_WindowSize = 0;
+
+	uint32_t m_CurrentHash		= 0;
+	uint32_t m_CurrentChunkSize = 0;
+
+	uint64_t m_BytesScanned = 0;
+
+	size_t InternalScanChunk(const void* DataBytes, size_t ByteCount);
+	void   InternalReset();
+};
+
+}  // namespace zen