1 files changed, 28 insertions, 62 deletions

diff --git a/zenstore/compactcas.h b/zenstore/compactcas.h
index c039feec9..11da37202 100644
--- a/zenstore/compactcas.h
+++ b/zenstore/compactcas.h
@@ -3,22 +3,13 @@
 #pragma once
 
 #include <zencore/zencore.h>
-
-#include <zencore/iobuffer.h>
-#include <zencore/iohash.h>
-#include <zencore/string.h>
-#include <zencore/thread.h>
-#include <zencore/uid.h>
-#include <zenstore/basicfile.h>
+#include <zenstore/blockstore.h>
 #include <zenstore/cas.h>
 #include <zenstore/caslog.h>
 #include <zenstore/gc.h>
 
-#if ZEN_PLATFORM_WINDOWS
-#	include <zencore/windows.h>
-#endif
-
 #include <atomic>
+#include <limits>
 #include <unordered_map>
 
 namespace spdlog {
@@ -32,46 +23,14 @@ namespace zen {
 #pragma pack(push)
 #pragma pack(1)
 
-struct CasDiskLocation
-{
-	CasDiskLocation(uint64_t InOffset, uint64_t InSize)
-	{
-		ZEN_ASSERT(InOffset <= 0xff'ffff'ffff);
-		ZEN_ASSERT(InSize <= 0xff'ffff'ffff);
-
-		memcpy(&m_Offset[0], &InOffset, sizeof m_Offset);
-		memcpy(&m_Size[0], &InSize, sizeof m_Size);
-	}
-
-	CasDiskLocation() = default;
-
-	inline uint64_t GetOffset() const
-	{
-		uint64_t Offset = 0;
-		memcpy(&Offset, &m_Offset, sizeof m_Offset);
-		return Offset;
-	}
-
-	inline uint64_t GetSize() const
-	{
-		uint64_t Size = 0;
-		memcpy(&Size, &m_Size, sizeof m_Size);
-		return Size;
-	}
-
-private:
-	uint8_t m_Offset[5];
-	uint8_t m_Size[5];
-};
-
 struct CasDiskIndexEntry
 {
 	static const uint8_t kTombstone = 0x01;
 
-	IoHash			Key;
-	CasDiskLocation Location;
-	ZenContentType	ContentType = ZenContentType::kUnknownContentType;
-	uint8_t			Flags		= 0;
+	IoHash				   Key;
+	BlockStoreDiskLocation Location;
+	ZenContentType		   ContentType = ZenContentType::kUnknownContentType;
+	uint8_t				   Flags	   = 0;
 };
 
 #pragma pack(pop)
@@ -91,39 +50,46 @@ struct CasContainerStrategy final : public GcStorage
 	CasContainerStrategy(const CasStoreConfiguration& Config, CasGc& Gc);
 	~CasContainerStrategy();
 
-	CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash);
 	CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash);
 	IoBuffer			   FindChunk(const IoHash& ChunkHash);
 	bool				   HaveChunk(const IoHash& ChunkHash);
 	void				   FilterChunks(CasChunkSet& InOutChunks);
-	void				   Initialize(const std::string_view ContainerBaseName, uint64_t Alignment, bool IsNewStore);
+	void				   Initialize(const std::string_view ContainerBaseName, uint32_t MaxBlockSize, uint64_t Alignment, bool IsNewStore);
 	void				   Flush();
 	void				   Scrub(ScrubContext& Ctx);
 	virtual void		   CollectGarbage(GcContext& GcCtx) override;
-	virtual GcStorageSize  StorageSize() const override { return {.DiskSize = m_TotalSize.load(std::memory_order::relaxed)}; }
+	virtual GcStorageSize  StorageSize() const override { return {.DiskSize = m_TotalSize.load(std::memory_order::acquire)}; }
 
 private:
-	void			OpenContainer(bool IsNewStore);
-	void			CloseContainer();
+	CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash);
+	void				   MakeIndexSnapshot();
+	uint64_t			   ReadIndexFile();
+	uint64_t			   ReadLog(uint64_t SkipEntryCount);
+	uint64_t			   MigrateLegacyData(bool CleanSource);
+	void				   OpenContainer(bool IsNewStore);
+
 	spdlog::logger& Log() { return m_Log; }
 
 	const CasStoreConfiguration&   m_Config;
 	spdlog::logger&				   m_Log;
-	uint64_t					   m_PayloadAlignment = 1 << 4;
+	uint64_t					   m_PayloadAlignment = 1u << 4;
+	uint64_t					   m_MaxBlockSize	  = 1u << 28;
 	bool						   m_IsInitialized	  = false;
-	BasicFile					   m_SmallObjectFile;
-	BasicFile					   m_SmallObjectIndex;
 	TCasLogFile<CasDiskIndexEntry> m_CasLog;
 	std::string					   m_ContainerBaseName;
+	std::filesystem::path		   m_BlocksBasePath;
+
+	RwLock																	   m_LocationMapLock;
+	typedef std::unordered_map<IoHash, BlockStoreDiskLocation, IoHash::Hasher> LocationMap_t;
+	LocationMap_t															   m_LocationMap;
+	std::unordered_map<uint32_t, Ref<BlockStoreFile>>						   m_ChunkBlocks;
 
-	RwLock														m_LocationMapLock;
-	std::unordered_map<IoHash, CasDiskLocation, IoHash::Hasher> m_LocationMap;
-	RwLock														m_InsertLock;  // used to serialize inserts
-	std::atomic_uint64_t										m_CurrentInsertOffset{};
-	std::atomic_uint64_t										m_CurrentIndexOffset{};
-	std::atomic_uint64_t										m_TotalSize{};
+	RwLock				m_InsertLock;  // used to serialize inserts
+	Ref<BlockStoreFile> m_WriteBlock;
+	std::uint64_t		m_CurrentInsertOffset = 0;
 
-	void MakeSnapshot();
+	std::atomic_uint32_t m_WriteBlockIndex{};
+	std::atomic_uint64_t m_TotalSize{};
 };
 
 void compactcas_forcelink();