1 files changed, 126 insertions, 0 deletions
diff --git a/zenstore/chunkbundler.h b/zenstore/chunkbundler.h
new file mode 100644
index 000000000..498320a6a
--- /dev/null
+++ b/zenstore/chunkbundler.h
@@ -0,0 +1,126 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <zenstore/basicfile.h>
+#include <zenstore/cas.h>
+#include <zenstore/caslog.h>
+#include <zenstore/gc.h>
+
+#include <atomic>
+#include <unordered_map>
+
+namespace spdlog {
+class logger;
+}
+
+namespace zen {
+
+class ChunkBundlerValidator
+{
+public:
+	virtual bool ValidateChunk(IoBuffer Buffer, IoHash Key) = 0;
+};
+
+#pragma pack(push)
+#pragma pack(1)
+
+struct CompactDiskLocation
+{
+	CompactDiskLocation(uint64_t InOffset, uint64_t InSize)
+	{
+		ZEN_ASSERT(InOffset <= 0xff'ffff'ffff);
+		ZEN_ASSERT(InSize <= 0xff'ffff'ffff);
+
+		memcpy(&m_Offset[0], &InOffset, sizeof m_Offset);
+		memcpy(&m_Size[0], &InSize, sizeof m_Size);
+	}
+
+	CompactDiskLocation() = default;
+
+	inline uint64_t GetOffset() const
+	{
+		uint64_t Offset = 0;
+		memcpy(&Offset, &m_Offset, sizeof m_Offset);
+		return Offset;
+	}
+
+	inline uint64_t GetSize() const
+	{
+		uint64_t Size = 0;
+		memcpy(&Size, &m_Size, sizeof m_Size);
+		return Size;
+	}
+
+private:
+	uint8_t m_Offset[5];
+	uint8_t m_Size[5];
+};
+
+struct CompactDiskIndexEntry
+{
+	static const uint8_t kTombstone = 0x01;
+
+	IoHash				Key;
+	CompactDiskLocation Location;
+	ZenContentType		ContentType = ZenContentType::kUnknownContentType;
+	uint8_t				Flags		= 0;
+};
+
+#pragma pack(pop)
+
+static_assert(sizeof(CompactDiskIndexEntry) == 32);
+
+class ChunkBundler final
+{
+public:
+	ChunkBundler(std::filesystem::path RootDirectory, ChunkBundlerValidator* Validator);
+	~ChunkBundler();
+
+	struct InsertResult
+	{
+		bool New = false;
+	};
+
+	void		  Initialize(const std::string_view ContainerBaseName, uint64_t Alignment, bool IsNewStore);
+	InsertResult  InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash);
+	InsertResult  InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash);
+	IoBuffer	  FindChunk(const IoHash& ChunkHash);
+	bool		  HaveChunk(const IoHash& ChunkHash);
+	void		  FilterChunks(CasChunkSet& InOutChunks);
+	void		  Flush();
+	void		  Scrub(ScrubContext& Ctx);
+	void		  CollectGarbage(GcContext& GcCtx);
+	GcStorageSize StorageSize() const;
+
+private:
+	spdlog::logger& Log() { return m_Log; }
+
+	ChunkBundlerValidator*			   m_Validator;
+	spdlog::logger&					   m_Log;
+	uint64_t						   m_PayloadAlignment = 1 << 4;
+	bool							   m_IsInitialized	  = false;
+	BasicFile						   m_SmallObjectFile;
+	BasicFile						   m_SmallObjectIndex;
+	std::filesystem::path			   RootDirectory;
+	TCasLogFile<CompactDiskIndexEntry> m_OpLog;
+	std::filesystem::path			   m_RootDirectory;
+	std::string						   m_ContainerBaseName;
+
+	RwLock															m_LocationMapLock;
+	RwLock															m_InsertLock;  // used to serialize inserts
+	std::unordered_map<IoHash, CompactDiskLocation, IoHash::Hasher> m_LocationMap;
+	std::atomic_uint64_t											m_CurrentInsertOffset{};
+	std::atomic_uint64_t											m_CurrentIndexOffset{};
+	std::atomic_uint64_t											m_TotalSize{};
+
+	void MakeIndexSnapshot();
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+void chunkbundler_forcelink();
+
+}  // namespace zen