Migration now works in larger disk IO chunks

BasicFile and CasLogFile now has new explicit modes instead of create true/false

12 files changed, 485 insertions, 296 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 769167433..10a19f949 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -476,8 +476,8 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const fs::path& BucketDir, const bool Is
 	std::filesystem::path SobsPath{BucketDir / "zen.sobs"};
 	std::filesystem::path SlogPath{BucketDir / "zen.slog"};
 
-	m_SobsFile.Open(SobsPath, IsNew);
-	m_SlogFile.Open(SlogPath, IsNew);
+	m_SobsFile.Open(SobsPath, IsNew ? BasicFile::EMode::kTruncate : BasicFile::EMode::kWrite);
+	m_SlogFile.Open(SlogPath, IsNew ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite);
 
 	m_SlogFile.Replay([&](const DiskIndexEntry& Entry) {
 		if (Entry.Key == IoHash::Zero)
@@ -905,8 +905,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 			m_SlogFile.Close();
 
 			const bool IsNew = true;
-			m_SobsFile.Open(m_BucketDir / "zen.sobs", IsNew);
-			m_SlogFile.Open(m_BucketDir / "zen.slog", IsNew);
+			m_SobsFile.Open(m_BucketDir / "zen.sobs", IsNew ? BasicFile::EMode::kTruncate : BasicFile::EMode::kWrite);
+			m_SlogFile.Open(m_BucketDir / "zen.slog", IsNew ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite);
 
 			m_SobsCursor = 0;
 			m_TotalSize	 = 0;
@@ -967,8 +967,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 			uint64_t					TmpCursor{};
 			std::vector<uint8_t>		Chunk;
 
-			TmpSobs.Open(TmpSobsPath, true);
-			TmpLog.Open(TmpSlogPath, true);
+			TmpSobs.Open(TmpSobsPath, BasicFile::EMode::kTruncate);
+			TmpLog.Open(TmpSlogPath, CasLogFile::EMode::kTruncate);
 
 			for (const auto& Entry : ValidEntries)
 			{
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 58b806989..f916b2250 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -114,10 +114,10 @@ struct ProjectStore::OplogStorage : public RefCounted
 			CreateDirectories(m_OplogStoragePath);
 		}
 
-		m_Oplog.Open(m_OplogStoragePath / "ops.zlog", IsCreate);
+		m_Oplog.Open(m_OplogStoragePath / "ops.zlog", IsCreate ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite);
 		m_Oplog.Initialize();
 
-		m_OpBlobs.Open(m_OplogStoragePath / "ops.zops", IsCreate);
+		m_OpBlobs.Open(m_OplogStoragePath / "ops.zops", IsCreate ? BasicFile::EMode::kTruncate : BasicFile::EMode::kWrite);
 
 		ZEN_ASSERT(IsPow2(m_OpsAlign));
 		ZEN_ASSERT(!(m_NextOpsOffset & (m_OpsAlign - 1)));
@@ -653,7 +653,7 @@ ProjectStore::Project::Read()
 	ZEN_INFO("reading config for project '{}' from {}", Identifier, ProjectStateFilePath);
 
 	BasicFile Blob;
-	Blob.Open(ProjectStateFilePath, false);
+	Blob.Open(ProjectStateFilePath, BasicFile::EMode::kRead);
 
 	IoBuffer		Obj				= Blob.ReadAll();
 	CbValidateError ValidationError = ValidateCompactBinary(MemoryView(Obj.Data(), Obj.Size()), CbValidateMode::All);
@@ -693,7 +693,7 @@ ProjectStore::Project::Write()
 	ZEN_INFO("persisting config for project '{}' to {}", Identifier, ProjectStateFilePath);
 
 	BasicFile Blob;
-	Blob.Open(ProjectStateFilePath, true);
+	Blob.Open(ProjectStateFilePath, BasicFile::EMode::kTruncate);
 	Blob.Write(Mem.Data(), Mem.Size(), 0);
 	Blob.Flush();
 }
diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp
index 77bada95f..fc7282941 100644
--- a/zenstore/basicfile.cpp
+++ b/zenstore/basicfile.cpp
@@ -29,10 +29,10 @@ BasicFile::~BasicFile()
 }
 
 void
-BasicFile::Open(const std::filesystem::path& FileName, bool IsCreate)
+BasicFile::Open(const std::filesystem::path& FileName, EMode Mode)
 {
 	std::error_code Ec;
-	Open(FileName, IsCreate, Ec);
+	Open(FileName, Mode, Ec);
 
 	if (Ec)
 	{
@@ -41,16 +41,20 @@ BasicFile::Open(const std::filesystem::path& FileName, bool IsCreate)
 }
 
 void
-BasicFile::Open(const std::filesystem::path& FileName, bool IsCreate, std::error_code& Ec)
+BasicFile::Open(const std::filesystem::path& FileName, EMode Mode, std::error_code& Ec)
 {
 	Ec.clear();
+	uint32_t ModeFlags = static_cast<uint32_t>(Mode);
 
 #if ZEN_PLATFORM_WINDOWS
-	const DWORD dwCreationDisposition = IsCreate ? CREATE_ALWAYS : OPEN_EXISTING;
-	DWORD		dwDesiredAccess		  = GENERIC_READ | GENERIC_WRITE | DELETE;
-	const DWORD dwShareMode			  = FILE_SHARE_READ;
-	const DWORD dwFlagsAndAttributes  = FILE_ATTRIBUTE_NORMAL;
-	HANDLE		hTemplateFile		  = nullptr;
+	const DWORD dwCreationDisposition = (ModeFlags & static_cast<uint32_t>(kAccessTruncate)) ? CREATE_ALWAYS : OPEN_EXISTING;
+	DWORD		dwDesiredAccess		  = GENERIC_READ;
+	dwDesiredAccess |= (ModeFlags & static_cast<uint32_t>(kAccessWrite)) ? GENERIC_WRITE : 0;
+	dwDesiredAccess |= (ModeFlags & static_cast<uint32_t>(kAccessDelete)) ? DELETE : 0;
+
+	const DWORD dwShareMode			 = FILE_SHARE_READ;
+	const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL;
+	HANDLE		hTemplateFile		 = nullptr;
 
 	HANDLE FileHandle = CreateFile(FileName.c_str(),
 								   dwDesiredAccess,
@@ -67,8 +71,9 @@ BasicFile::Open(const std::filesystem::path& FileName, bool IsCreate, std::error
 		return;
 	}
 #else
-	int OpenFlags = O_RDWR | O_CLOEXEC;
-	OpenFlags |= IsCreate ? O_CREAT | O_TRUNC : 0;
+	int OpenFlags = O_CLOEXEC;
+	OpenFlags |= (ModeFlags & static_cast<uint32_t>(kAccessWrite)) ? O_RDWR : 0;
+	OpenFlags |= (ModeFlags & static_cast<uint32_t>(kAccessTruncate)) ? (O_CREAT | O_TRUNC) : 0;
 
 	int Fd = open(FileName.c_str(), OpenFlags, 0666);
 	if (Fd < 0)
@@ -105,27 +110,33 @@ BasicFile::Close()
 void
 BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset)
 {
-	const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
+	const uint64_t MaxChunkSize	 = 2u * 1024 * 1024 * 1024;
+	uint64_t	   TotalFileSize = FileSize();
+	ZEN_ASSERT((FileOffset + BytesToRead) <= TotalFileSize);
 
 	while (BytesToRead)
 	{
 		const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize);
+		uint64_t	   NumberOfBytesRead;
 
 #if ZEN_PLATFORM_WINDOWS
 		OVERLAPPED Ovl{};
 
 		Ovl.Offset	   = DWORD(FileOffset & 0xffff'ffffu);
 		Ovl.OffsetHigh = DWORD(FileOffset >> 32);
+		ZEN_ASSERT((FileOffset + NumberOfBytesToRead) <= TotalFileSize);
 
 		DWORD dwNumberOfBytesRead = 0;
 		BOOL  Success			  = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl);
 
-		ZEN_ASSERT(dwNumberOfBytesRead == NumberOfBytesToRead);
+		ZEN_ASSERT((dwNumberOfBytesRead <= NumberOfBytesToRead) && (dwNumberOfBytesRead > 0));
+		NumberOfBytesRead = dwNumberOfBytesRead;
 #else
 		static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files");
-		int	 Fd		   = int(uintptr_t(m_FileHandle));
-		int	 BytesRead = pread(Fd, Data, NumberOfBytesToRead, FileOffset);
-		bool Success   = (BytesRead > 0);
+		int	 Fd			  = int(uintptr_t(m_FileHandle));
+		int	 BytesRead	  = pread(Fd, Data, NumberOfBytesToRead, FileOffset);
+		bool Success	  = (BytesRead > 0);
+		NumberOfBytesRead = static_cast<uint64_t>(BytesRead);
 #endif
 
 		if (!Success)
@@ -133,9 +144,9 @@ BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset)
 			ThrowLastError(fmt::format("Failed to read from file '{}'", zen::PathFromHandle(m_FileHandle)));
 		}
 
-		BytesToRead -= NumberOfBytesToRead;
-		FileOffset += NumberOfBytesToRead;
-		Data = reinterpret_cast<uint8_t*>(Data) + NumberOfBytesToRead;
+		BytesToRead -= NumberOfBytesRead;
+		FileOffset += NumberOfBytesRead;
+		Data = reinterpret_cast<uint8_t*>(Data) + NumberOfBytesRead;
 	}
 }
 
@@ -263,7 +274,14 @@ BasicFile::FileSize()
 #if ZEN_PLATFORM_WINDOWS
 	ULARGE_INTEGER liFileSize;
 	liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart);
-
+	if (liFileSize.LowPart == INVALID_FILE_SIZE)
+	{
+		int Error = GetLastError();
+		if (Error)
+		{
+			ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", zen::PathFromHandle(m_FileHandle)));
+		}
+	}
 	return uint64_t(liFileSize.QuadPart);
 #else
 	int Fd = int(uintptr_t(m_FileHandle));
@@ -351,9 +369,7 @@ TemporaryFile::CreateTemporary(std::filesystem::path TempDirName, std::error_cod
 
 	m_TempPath = TempDirName / TempName.c_str();
 
-	const bool IsCreate = true;
-
-	Open(m_TempPath, IsCreate, Ec);
+	Open(m_TempPath, BasicFile::EMode::kTruncateDelete, Ec);
 }
 
 void
@@ -453,8 +469,8 @@ TEST_CASE("BasicFile")
 	ScopedCurrentDirectoryChange _;
 
 	BasicFile File1;
-	CHECK_THROWS(File1.Open("zonk", false));
-	CHECK_NOTHROW(File1.Open("zonk", true));
+	CHECK_THROWS(File1.Open("zonk", BasicFile::EMode::kRead));
+	CHECK_NOTHROW(File1.Open("zonk", BasicFile::EMode::kTruncate));
 	CHECK_NOTHROW(File1.Write("abcd", 4, 0));
 	CHECK(File1.FileSize() == 4);
 	{
diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp
index 441b5e926..e43c3a00f 100644
--- a/zenstore/blockstore.cpp
+++ b/zenstore/blockstore.cpp
@@ -42,7 +42,7 @@ BlockStoreFile::InternalOpen()
 	{
 		return;
 	}
-	m_File.Open(m_Path, false);
+	m_File.Open(m_Path, BasicFile::EMode::kDelete);
 	void* FileHandle = m_File.Handle();
 	m_IoBuffer		 = IoBuffer(IoBuffer::File, FileHandle, 0, m_File.FileSize());
 }
@@ -65,7 +65,7 @@ BlockStoreFile::Create(uint64_t InitialSize)
 		CreateDirectories(ParentPath);
 	}
 
-	m_File.Open(m_Path, true);
+	m_File.Open(m_Path, BasicFile::EMode::kTruncateDelete);
 	if (InitialSize > 0)
 	{
 		m_File.SetFileSize(InitialSize);
diff --git a/zenstore/cas.cpp b/zenstore/cas.cpp
index b569a7dae..f30439d39 100644
--- a/zenstore/cas.cpp
+++ b/zenstore/cas.cpp
@@ -164,7 +164,7 @@ CasImpl::OpenOrCreateManifest()
 
 	std::error_code Ec;
 	BasicFile		ManifestFile;
-	ManifestFile.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec);
+	ManifestFile.Open(ManifestPath.c_str(), BasicFile::EMode::kWrite, Ec);
 
 	bool ManifestIsOk = false;
 
@@ -236,7 +236,7 @@ CasImpl::UpdateManifest()
 	ZEN_TRACE("Writing new manifest to '{}'", ManifestPath);
 
 	BasicFile Marker;
-	Marker.Open(ManifestPath.c_str(), /* IsCreate */ true);
+	Marker.Open(ManifestPath.c_str(), BasicFile::EMode::kTruncate);
 	Marker.Write(m_ManifestObject.GetBuffer(), 0);
 }
 
diff --git a/zenstore/caslog.cpp b/zenstore/caslog.cpp
index 909605af9..906c97348 100644
--- a/zenstore/caslog.cpp
+++ b/zenstore/caslog.cpp
@@ -39,13 +39,23 @@ CasLogFile::~CasLogFile()
 }
 
 void
-CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, bool IsCreate)
+CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, EMode Mode)
 {
 	m_RecordSize = RecordSize;
 
-	std::error_code Ec;
-	m_File.Open(FileName, IsCreate, Ec);
+	std::error_code	 Ec;
+	BasicFile::EMode FileMode = BasicFile::EMode::kRead;
+	switch (Mode)
+	{
+		case EMode::kWrite:
+			FileMode = BasicFile::EMode::kWrite;
+			break;
+		case EMode::kTruncate:
+			FileMode = BasicFile::EMode::kTruncate;
+			break;
+	}
 
+	m_File.Open(FileName, FileMode, Ec);
 	if (Ec)
 	{
 		throw std::system_error(Ec, fmt::format("Failed to open log file '{}'", FileName));
@@ -53,8 +63,12 @@ CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, bool IsCreat
 
 	uint64_t AppendOffset = 0;
 
-	if (IsCreate || (m_File.FileSize() < sizeof(FileHeader)))
+	if ((Mode == EMode::kTruncate) || (m_File.FileSize() < sizeof(FileHeader)))
 	{
+		if (Mode == EMode::kRead)
+		{
+			throw std::runtime_error(fmt::format("Mangled log header (file to small) in '{}'", FileName));
+		}
 		// Initialize log by writing header
 		FileHeader Header = {.RecordSize = gsl::narrow<uint32_t>(RecordSize), .LogId = Oid::NewOid(), .ValidatedTail = 0};
 		memcpy(Header.Magic, FileHeader::MagicSequence, sizeof Header.Magic);
diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp
index 8b53a8304..5ddf32e73 100644
--- a/zenstore/cidstore.cpp
+++ b/zenstore/cidstore.cpp
@@ -127,7 +127,7 @@ struct CidStore::Impl
 
 		bool IsNew = !std::filesystem::exists(SlogPath);
 
-		m_LogFile.Open(SlogPath, IsNew);
+		m_LogFile.Open(SlogPath, IsNew ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite);
 
 		ZEN_DEBUG("Initializing index from '{}' ({})", SlogPath, NiceBytes(m_LogFile.GetLogSize()));
 
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index ec0a5a7ba..9d0f72442 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -27,15 +27,15 @@ namespace zen {
 
 struct CasDiskIndexHeader
 {
-	static constexpr uint32_t ExpectedMagic		= 0x75696478;  // 'uidx';
-	static constexpr uint32_t CurrentVersion	= 1;
-	uint32_t				  Magic				= ExpectedMagic;
-	uint32_t				  Version			= CurrentVersion;
-	uint32_t				  PayloadAlignement = 0;
-	uint32_t				  Reserved0			= 0;
-	uint64_t				  EntryCount		= 0;
-	uint32_t				  Reserved1			= 0;
-	uint32_t				  Reserved2			= 0;
+	static constexpr uint32_t ExpectedMagic	   = 0x75696478;  // 'uidx';
+	static constexpr uint32_t CurrentVersion   = 1;
+	uint32_t				  Magic			   = ExpectedMagic;
+	uint32_t				  Version		   = CurrentVersion;
+	uint32_t				  PayloadAlignment = 0;
+	uint32_t				  Reserved0		   = 0;
+	uint64_t				  EntryCount	   = 0;
+	uint32_t				  Reserved1		   = 0;
+	uint32_t				  Reserved2		   = 0;
 };
 
 static_assert(sizeof(CasDiskIndexHeader) == 32);
@@ -122,10 +122,351 @@ namespace {
 		return RootPath / (ContainerBaseName + DataExtension);
 	}
 
+	struct LegacyCasDiskLocation
+	{
+		LegacyCasDiskLocation(uint64_t InOffset, uint64_t InSize)
+		{
+			ZEN_ASSERT(InOffset <= 0xff'ffff'ffff);
+			ZEN_ASSERT(InSize <= 0xff'ffff'ffff);
+
+			memcpy(&m_Offset[0], &InOffset, sizeof m_Offset);
+			memcpy(&m_Size[0], &InSize, sizeof m_Size);
+		}
+
+		LegacyCasDiskLocation() = default;
+
+		inline uint64_t GetOffset() const
+		{
+			uint64_t Offset = 0;
+			memcpy(&Offset, &m_Offset, sizeof m_Offset);
+			return Offset;
+		}
+
+		inline uint64_t GetSize() const
+		{
+			uint64_t Size = 0;
+			memcpy(&Size, &m_Size, sizeof m_Size);
+			return Size;
+		}
+
+	private:
+		uint8_t m_Offset[5];
+		uint8_t m_Size[5];
+	};
+
+	struct LegacyCasDiskIndexEntry
+	{
+		static const uint8_t kTombstone = 0x01;
+
+		IoHash				  Key;
+		LegacyCasDiskLocation Location;
+		ZenContentType		  ContentType = ZenContentType::kUnknownContentType;
+		uint8_t				  Flags		  = 0;
+	};
+
+	void Migrate(const std::filesystem::path& RootPath,
+				 const std::string&			  ContainerBaseName,
+				 uint64_t					  MaxBlockSize,
+				 uint64_t					  PayloadAlignment,
+				 bool						  Destructive,
+				 bool						  Overwrite)
+	{
+		std::filesystem::path BlocksBasePath = GetBlocksBasePath(RootPath, ContainerBaseName);
+		std::filesystem::path LegacyLogPath	 = GetLegacyLogPath(RootPath, ContainerBaseName);
+		std::filesystem::path LegacySobsPath = GetLegacyUcasPath(RootPath, ContainerBaseName);
+		if (!std::filesystem::is_regular_file(LegacyLogPath) || !std::filesystem::is_regular_file(LegacySobsPath))
+		{
+			ZEN_DEBUG("migrate of {} SKIPPED, no legacy data found", RootPath / ContainerBaseName);
+			return;
+		}
+
+		std::filesystem::path SlogPath = GetLogPath(RootPath, ContainerBaseName);
+		if (std::filesystem::is_directory(SlogPath.parent_path()))
+		{
+			if (!Overwrite)
+			{
+				ZEN_WARN("migrate of {} SKIPPED, new content already exists", RootPath / ContainerBaseName);
+				return;
+			}
+		}
+
+		uint32_t   NewBlockIndex = 0;
+		Stopwatch  MigrationTimer;
+		uint64_t   TotalSize = 0;
+		const auto Guard	 = MakeGuard([RootPath, ContainerBaseName, &MigrationTimer, &NewBlockIndex, &TotalSize] {
+			ZEN_INFO("migrated store {} to {} blocks in {} ({})",
+					 RootPath / ContainerBaseName,
+					 NewBlockIndex + 1,
+					 NiceTimeSpanMs(MigrationTimer.GetElapsedTimeMs()),
+					 NiceBytes(TotalSize));
+		});
 
-//	void Migrate(const std::filesystem::path& RootPath, const std::string_view ContainerBaseName, uint32_t MaxBlockSize, uint64_t Alignment)
-//	{
-//	}
+		std::error_code Error;
+		DiskSpace		Space = DiskSpaceInfo(RootPath, Error);
+		if (Error)
+		{
+			ZEN_ERROR("get disk space in {} FAILED, reason '{}'", ContainerBaseName, Error.message());
+			return;
+		}
+
+		if (Space.Free < MaxBlockSize)
+		{
+			ZEN_ERROR("legacy store migration from '{}' FAILED, required disk space {}, free {}",
+					  RootPath / ContainerBaseName,
+					  MaxBlockSize,
+					  NiceBytes(Space.Free));
+			return;
+		}
+
+		BasicFile BlockFile;
+		BlockFile.Open(LegacySobsPath, BasicFile::EMode::kRead);
+		uint64_t FileSize = BlockFile.FileSize();
+
+		std::unordered_map<IoHash, LegacyCasDiskIndexEntry, IoHash::Hasher> LegacyDiskIndex;
+
+		TCasLogFile<LegacyCasDiskIndexEntry> LegacyCasLog;
+		LegacyCasLog.Open(LegacyLogPath, CasLogFile::EMode::kRead);
+		LegacyCasLog.Replay([&](const LegacyCasDiskIndexEntry& Record) {
+			if (Record.Flags & LegacyCasDiskIndexEntry::kTombstone)
+			{
+				LegacyDiskIndex.erase(Record.Key);
+				return;
+			}
+			uint64_t EntryEnd = Record.Location.GetOffset() + Record.Location.GetSize();
+			if (EntryEnd > FileSize)
+			{
+				return;
+			}
+			LegacyDiskIndex[Record.Key] = Record;
+		});
+
+		uint64_t MaxUsedSize = 0;
+		for (const auto& Entry : LegacyDiskIndex)
+		{
+			const LegacyCasDiskIndexEntry& Record(Entry.second);
+			uint64_t					   EntryEnd = Record.Location.GetOffset() + Record.Location.GetSize();
+			if (EntryEnd > MaxUsedSize)
+			{
+				MaxUsedSize = EntryEnd;
+			}
+			TotalSize += Record.Location.GetSize();
+		}
+
+		uint64_t RequiredDiskSpace	   = TotalSize + ((PayloadAlignment - 1) * LegacyDiskIndex.size());
+		uint64_t MaxRequiredBlockCount = RoundUp(RequiredDiskSpace, MaxBlockSize) / MaxBlockSize;
+		if (MaxRequiredBlockCount > BlockStoreDiskLocation::MaxBlockIndex)
+		{
+			ZEN_ERROR("legacy store migration from '{}' FAILED, required block count {}, possible {}",
+					  RootPath / ContainerBaseName,
+					  MaxRequiredBlockCount,
+					  BlockStoreDiskLocation::MaxBlockIndex);
+			return;
+		}
+		if (Destructive)
+		{
+			if (Space.Free < (MaxBlockSize + (1 << 28)))
+			{
+				ZEN_INFO("legacy store migration from {} aborted, not enough disk space available {} ({})",
+						 RootPath / ContainerBaseName,
+						 NewBlockIndex + 1,
+						 NiceBytes(MaxBlockSize + (1 << 28)),
+						 NiceBytes(Space.Free));
+				return;
+			}
+		}
+		else
+		{
+			if (Space.Free < (RequiredDiskSpace + (1 << 28)))
+			{
+				ZEN_INFO("legacy store migration from {} aborted, not enough disk space available {} ({})",
+						 RootPath / ContainerBaseName,
+						 NewBlockIndex + 1,
+						 NiceBytes(RequiredDiskSpace + (1 << 28)),
+						 NiceBytes(Space.Free));
+				return;
+			}
+		}
+
+		CreateDirectories(SlogPath.parent_path());
+		TCasLogFile<CasDiskIndexEntry> CasLog;
+		CasLog.Open(SlogPath, CasLogFile::EMode::kTruncate);
+
+		if (Destructive && (MaxRequiredBlockCount < 2))
+		{
+			std::vector<CasDiskIndexEntry> LogEntries;
+			LogEntries.reserve(LegacyDiskIndex.size());
+
+			// We can use the block as is, just move it and add the blocks to our new log
+			for (auto& Entry : LegacyDiskIndex)
+			{
+				const LegacyCasDiskIndexEntry& Record(Entry.second);
+
+				BlockStoreLocation NewChunkLocation(0, Record.Location.GetOffset(), Record.Location.GetSize());
+				LogEntries.push_back({.Key		   = Entry.second.Key,
+									  .Location	   = BlockStoreDiskLocation(NewChunkLocation, PayloadAlignment),
+									  .ContentType = Record.ContentType,
+									  .Flags	   = Record.Flags});
+			}
+			auto BlockPath = GetBlockPath(BlocksBasePath, 0);
+			CreateDirectories(BlockPath.parent_path());
+			BlockFile.Close();
+			std::filesystem::rename(LegacySobsPath, BlockPath);
+		}
+		else
+		{
+			std::vector<IoHash> ChunkHashes;
+			ChunkHashes.reserve(LegacyDiskIndex.size());
+			for (const auto& Entry : LegacyDiskIndex)
+			{
+				ChunkHashes.push_back(Entry.first);
+			}
+
+			std::sort(begin(ChunkHashes), end(ChunkHashes), [&](IoHash Lhs, IoHash Rhs) {
+				auto LhsKeyIt = LegacyDiskIndex.find(Lhs);
+				auto RhsKeyIt = LegacyDiskIndex.find(Rhs);
+				return LhsKeyIt->second.Location.GetOffset() < RhsKeyIt->second.Location.GetOffset();
+			});
+
+			uint64_t						BlockSize	= 0;
+			uint64_t						BlockOffset = 0;
+			uint32_t						BlockIndex	= 0;
+			std::vector<BlockStoreLocation> NewLocations;
+			struct BlockData
+			{
+				std::vector<std::pair<IoHash, BlockStoreLocation>> Chunks;
+				uint64_t										   BlockOffset;
+				uint64_t										   BlockSize;
+				uint32_t										   BlockIndex;
+			};
+
+			std::vector<BlockData>							   BlockRanges;
+			std::vector<std::pair<IoHash, BlockStoreLocation>> Chunks;
+			BlockRanges.reserve(MaxRequiredBlockCount);
+			for (const IoHash& ChunkHash : ChunkHashes)
+			{
+				const auto&					 LegacyEntry		 = LegacyDiskIndex[ChunkHash];
+				const LegacyCasDiskLocation& LegacyChunkLocation = LegacyEntry.Location;
+
+				uint64_t ChunkOffset = LegacyChunkLocation.GetOffset();
+				uint64_t ChunkSize	 = LegacyChunkLocation.GetSize();
+
+				#if 0
+				{
+					std::vector<uint8_t> Data(ChunkSize);
+					BlockFile.Read(Data.data(), ChunkSize, ChunkOffset);
+					const IoHash ComputedHash = IoHash::HashBuffer(Data.data(), ChunkSize);
+					if (ComputedHash != ChunkHash)
+					{
+						ZEN_ERROR("migrating store {}, invalid hash for chunk {}. Got {}",
+								  RootPath / ContainerBaseName,
+								  ChunkHash,
+								  ComputedHash);
+					}
+				}
+				#endif // 0
+
+				if (BlockSize == 0)
+				{
+					BlockOffset = ChunkOffset;
+				}
+				if ((BlockSize + ChunkSize) > MaxBlockSize)
+				{
+					BlockData BlockRange{.BlockOffset = BlockOffset, .BlockSize = BlockSize, .BlockIndex = BlockIndex};
+					BlockRange.Chunks.swap(Chunks);
+					BlockRanges.push_back(BlockRange);
+
+					BlockIndex++;
+					BlockOffset = ChunkOffset;
+					BlockSize	= 0;
+				}
+				BlockSize						 = RoundUp(BlockSize, PayloadAlignment);
+				BlockStoreLocation ChunkLocation = {.BlockIndex = BlockIndex, .Offset = BlockSize, .Size = ChunkSize};
+				Chunks.push_back({ChunkHash, ChunkLocation});
+				BlockSize += ChunkSize;
+			}
+			if (BlockSize > 0)
+			{
+				BlockRanges.push_back(
+					{.Chunks = std::move(Chunks), .BlockOffset = BlockOffset, .BlockSize = BlockSize, .BlockIndex = BlockIndex});
+			}
+
+			std::reverse(BlockRanges.begin(), BlockRanges.end());
+			std::vector<std::uint8_t> Buffer(1 << 28);
+			for (size_t Idx = 0; Idx < BlockRanges.size(); ++Idx)
+			{
+				const BlockData& BlockRange = BlockRanges[Idx];
+
+				ZEN_INFO("migrating store {} {}/{} blocks, remaining {} ({})",
+						 RootPath / ContainerBaseName,
+						 Idx,
+						 BlockRanges.size(),
+						 NiceBytes(BlockRange.BlockOffset),
+						 NiceBytes(TotalSize));
+
+				auto		   BlockPath = GetBlockPath(BlocksBasePath, BlockRange.BlockIndex);
+				BlockStoreFile ChunkBlock(BlockPath);
+				ChunkBlock.Create(BlockRange.BlockSize);
+				uint64_t Offset = 0;
+				while (Offset < BlockRange.BlockSize)
+				{
+					uint64_t Size = BlockRange.BlockSize - Offset;
+					if (Size > Buffer.size())
+					{
+						Size = Buffer.size();
+					}
+					BlockFile.Read(Buffer.data(), Size, BlockRange.BlockOffset + Offset);
+					ChunkBlock.Write(Buffer.data(), Size, Offset);
+					Offset += Size;
+				}
+				ChunkBlock.Flush();
+
+				std::vector<CasDiskIndexEntry> LogEntries;
+				LogEntries.reserve(BlockRange.Chunks.size());
+				for (const auto& Entry : BlockRange.Chunks)
+				{
+					const LegacyCasDiskIndexEntry& LegacyEntry = LegacyDiskIndex[Entry.first];
+					BlockStoreDiskLocation		   Location(Entry.second, PayloadAlignment);
+					LogEntries.push_back(
+						{.Key = Entry.first, .Location = Location, .ContentType = LegacyEntry.ContentType, .Flags = LegacyEntry.Flags});
+				}
+				CasLog.Append(LogEntries);
+				#if 0
+				for (const CasDiskIndexEntry& Entry : LogEntries)
+				{
+					std::vector<uint8_t> Data(Entry.Location.GetSize());
+					ChunkBlock.Read(Data.data(), Entry.Location.GetSize(), Entry.Location.GetOffset(PayloadAlignment));
+					const IoHash ComputedHash = IoHash::HashBuffer(Data.data(), Entry.Location.GetSize());
+					if (ComputedHash != Entry.Key)
+					{
+						ZEN_ERROR("migrating store {}, invalid hash for chunk {}. Got {}",
+								  RootPath / ContainerBaseName,
+								  Entry.Key,
+								  ComputedHash);
+					}
+				}
+				#endif // 0
+
+				if (Destructive)
+				{
+					std::vector<LegacyCasDiskIndexEntry> LegacyLogEntries;
+					LegacyLogEntries.reserve(BlockRange.Chunks.size());
+					for (const auto& Entry : BlockRange.Chunks)
+					{
+						LegacyLogEntries.push_back({.Key = Entry.first, .Flags = LegacyCasDiskIndexEntry::kTombstone});
+					}
+					LegacyCasLog.Append(LegacyLogEntries);
+					BlockFile.SetFileSize(BlockRange.BlockOffset);
+				}
+			}
+		}
+		LegacyCasLog.Close();
+		CasLog.Close();
+
+		if (Destructive)
+		{
+			std::filesystem::remove(LegacyLogPath);
+		}
+	}
 
 }  // namespace
 
@@ -219,7 +560,7 @@ CasContainerStrategy::InsertChunk(const void* ChunkData, size_t ChunkSize, const
 	BlockStoreDiskLocation	Location({.BlockIndex = WriteBlockIndex, .Offset = InsertOffset, .Size = ChunkSize}, m_PayloadAlignment);
 	const CasDiskIndexEntry IndexEntry{.Key = ChunkHash, .Location = Location};
 
-	m_TotalSize.fetch_add(static_cast<uint64_t>(ChunkSize));
+	m_TotalSize.fetch_add(static_cast<uint64_t>(ChunkSize), std::memory_order_release);
 	{
 		RwLock::ExclusiveLockScope __(m_LocationMapLock);
 		m_LocationMap.emplace(ChunkHash, Location);
@@ -508,8 +849,8 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 	TotalChunkCount = LocationMap.size();
 
 	std::unordered_map<uint32_t, size_t> BlockIndexToChunkMapIndex;
-	std::vector<std::vector<IoHash> >	 KeepChunks;
-	std::vector<std::vector<IoHash> >	 DeleteChunks;
+	std::vector<std::vector<IoHash>>	 KeepChunks;
+	std::vector<std::vector<IoHash>>	 DeleteChunks;
 
 	BlockIndexToChunkMapIndex.reserve(BlockCount);
 	KeepChunks.reserve(BlockCount);
@@ -774,7 +1115,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 	}
 	BasicFile GCReserveFile;
 	CreateDirectories(GCReservePath.parent_path());
-	GCReserveFile.Open(GCReservePath, true);
+	GCReserveFile.Open(GCReservePath, BasicFile::EMode::kTruncate);
 	GCReserveFile.SetFileSize(m_MaxBlockSize);
 
 	ZEN_DEBUG("recreated garbage collect reserve '{}', {} bytes", m_Config.RootDirectory / m_ContainerBaseName, NiceBytes(Space.Free));
@@ -817,7 +1158,7 @@ CasContainerStrategy::MakeIndexSnapshot()
 		fs::rename(SlogPath, STmplogPath);
 
 		// Open an new log
-		m_CasLog.Open(SlogPath, true);
+		m_CasLog.Open(SlogPath, CasLogFile::EMode::kTruncate);
 	}
 
 	try
@@ -839,8 +1180,8 @@ CasContainerStrategy::MakeIndexSnapshot()
 		}
 
 		BasicFile ObjectIndexFile;
-		ObjectIndexFile.Open(SidxPath, true);
-		CasDiskIndexHeader Header = {.PayloadAlignement = gsl::narrow<uint32_t>(m_PayloadAlignment), .EntryCount = Entries.size()};
+		ObjectIndexFile.Open(SidxPath, BasicFile::EMode::kTruncate);
+		CasDiskIndexHeader Header = {.PayloadAlignment = gsl::narrow<uint32_t>(m_PayloadAlignment), .EntryCount = Entries.size()};
 		ObjectIndexFile.Write(&Header, sizeof(CasDiskIndexEntry), 0);
 		ObjectIndexFile.Write(Entries.data(), Entries.size() * sizeof(CasDiskIndexEntry), sizeof(CasDiskIndexEntry));
 		ObjectIndexFile.Close();
@@ -857,7 +1198,7 @@ CasContainerStrategy::MakeIndexSnapshot()
 			Records.reserve(m_LocationMap.size());
 			{
 				TCasLogFile<CasDiskIndexEntry> OldCasLog;
-				OldCasLog.Open(STmplogPath, false);
+				OldCasLog.Open(STmplogPath, CasLogFile::EMode::kRead);
 				OldCasLog.Replay([&](const CasDiskIndexEntry& Record) { Records.push_back(Record); });
 			}
 			{
@@ -865,7 +1206,7 @@ CasContainerStrategy::MakeIndexSnapshot()
 			}
 
 			TCasLogFile<CasDiskIndexEntry> RecoveredCasLog;
-			RecoveredCasLog.Open(SRecoveredlogPath, true);
+			RecoveredCasLog.Open(SRecoveredlogPath, CasLogFile::EMode::kWrite);
 			RecoveredCasLog.Append(Records);
 			RecoveredCasLog.Close();
 
@@ -896,52 +1237,6 @@ CasContainerStrategy::MakeIndexSnapshot()
 	}
 }
 
-namespace {
-	struct LegacyCasDiskLocation
-	{
-		LegacyCasDiskLocation(uint64_t InOffset, uint64_t InSize)
-		{
-			ZEN_ASSERT(InOffset <= 0xff'ffff'ffff);
-			ZEN_ASSERT(InSize <= 0xff'ffff'ffff);
-
-			memcpy(&m_Offset[0], &InOffset, sizeof m_Offset);
-			memcpy(&m_Size[0], &InSize, sizeof m_Size);
-		}
-
-		LegacyCasDiskLocation() = default;
-
-		inline uint64_t GetOffset() const
-		{
-			uint64_t Offset = 0;
-			memcpy(&Offset, &m_Offset, sizeof m_Offset);
-			return Offset;
-		}
-
-		inline uint64_t GetSize() const
-		{
-			uint64_t Size = 0;
-			memcpy(&Size, &m_Size, sizeof m_Size);
-			return Size;
-		}
-
-	private:
-		uint8_t m_Offset[5];
-		uint8_t m_Size[5];
-	};
-
-	struct LegacyCasDiskIndexEntry
-	{
-		static const uint8_t kTombstone = 0x01;
-
-		IoHash				  Key;
-		LegacyCasDiskLocation Location;
-		ZenContentType		  ContentType = ZenContentType::kUnknownContentType;
-		uint8_t				  Flags		  = 0;
-	};
-
-}  // namespace
-
-
 void
 CasContainerStrategy::OpenContainer(bool IsNewStore)
 {
@@ -975,179 +1270,17 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 			std::filesystem::remove(SidxPath);
 		}
 		CreateDirectories(SlogPath.parent_path());
-		m_CasLog.Open(SlogPath, true);
+		m_CasLog.Open(SlogPath, CasLogFile::EMode::kTruncate);
 	}
 	else
 	{
-		if (std::filesystem::is_regular_file(LegacyLogPath) && std::filesystem::is_regular_file(LegacySobsPath))
-		{
-			uint32_t   NewBlockIndex = 0;
-			Stopwatch  MigrationTimer;
-			uint64_t   TotalSize = 0;
-			const auto Guard	 = MakeGuard([this, &MigrationTimer, &NewBlockIndex, &TotalSize] {
-				ZEN_INFO("migrated store {} to {} blocks in {} ({})",
-						 m_Config.RootDirectory / m_ContainerBaseName,
-						 NewBlockIndex + 1,
-						 NiceTimeSpanMs(MigrationTimer.GetElapsedTimeMs()),
-						 NiceBytes(TotalSize));
-			});
-
-			std::error_code Error;
-			DiskSpace		Space = DiskSpaceInfo(m_Config.RootDirectory, Error);
-			if (Error)
-			{
-				ZEN_ERROR("get disk space in {} FAILED, reason '{}'", m_ContainerBaseName, Error.message());
-				return;
-			}
-
-			if (Space.Free < m_MaxBlockSize)  // Never let GC steal the last block space
-			{
-				ZEN_ERROR("legacy store migration from '{}' FAILED, required disk space {}, free {}",
-						  m_Config.RootDirectory / m_ContainerBaseName,
-						  m_MaxBlockSize,
-						  NiceBytes(Space.Free));
-				return;
-			}
-
-			BasicFile BlockFile;
-			BlockFile.Open(LegacySobsPath, false);
-
-			std::unordered_map<IoHash, LegacyCasDiskIndexEntry, IoHash::Hasher> LegacyDiskIndex;
-
-			TCasLogFile<LegacyCasDiskIndexEntry> LegacyCasLog;
-			LegacyCasLog.Open(LegacyLogPath, false);
-			LegacyCasLog.Replay([&](const LegacyCasDiskIndexEntry& Record) {
-				if (Record.Flags & LegacyCasDiskIndexEntry::kTombstone)
-				{
-					LegacyDiskIndex.erase(Record.Key);
-				}
-				else
-				{
-					LegacyDiskIndex[Record.Key] = Record;
-				}
-			});
-
-			uint64_t MaxUsedSize = 0;
-			for (const auto& Entry : LegacyDiskIndex)
-			{
-				const LegacyCasDiskIndexEntry& Record(Entry.second);
-				uint64_t					   EntryEnd = Record.Location.GetOffset() + Record.Location.GetSize();
-				if (EntryEnd > MaxUsedSize)
-				{
-					MaxUsedSize = EntryEnd;
-				}
-				TotalSize += Record.Location.GetSize();
-			}
-			LegacyCasLog.Close();
-
-			BlockFile.SetFileSize(MaxUsedSize);
-			uint64_t MaxRequiredBlockCount = RoundUp(MaxUsedSize, m_MaxBlockSize) / m_MaxBlockSize;
-			if (MaxRequiredBlockCount > BlockStoreDiskLocation::MaxBlockIndex)
-			{
-				ZEN_ERROR("legacy store migration from '{}' FAILED, required block count {}, possible {}",
-						  m_Config.RootDirectory / m_ContainerBaseName,
-						  MaxRequiredBlockCount,
-						  BlockStoreDiskLocation::MaxBlockIndex);
-				return;
-			}
-
-			CreateDirectories(SlogPath.parent_path());
-			m_CasLog.Open(SlogPath, true);
-
-			std::vector<CasDiskIndexEntry> LogEntries;
-			LogEntries.reserve(LegacyDiskIndex.size());
-			if (MaxRequiredBlockCount < 2)
-			{
-				for (const auto& Entry : LegacyDiskIndex)
-				{
-					const LegacyCasDiskIndexEntry& Record(Entry.second);
-					BlockStoreLocation			   NewChunkLocation(0, Record.Location.GetOffset(), Record.Location.GetSize());
-					LogEntries.push_back({.Key		   = Entry.second.Key,
-										  .Location	   = BlockStoreDiskLocation(NewChunkLocation, m_PayloadAlignment),
-										  .ContentType = Record.ContentType,
-										  .Flags	   = Record.Flags});
-				}
-				auto BlockPath = GetBlockPath(m_BlocksBasePath, 0);
-				CreateDirectories(BlockPath.parent_path());
-				BlockFile.Close();
-				std::filesystem::rename(LegacySobsPath, BlockPath);
-			}
-			else
-			{
-				std::vector<IoHash> ChunkHashes;
-				ChunkHashes.reserve(LegacyDiskIndex.size());
-				for (const auto& Entry : LegacyDiskIndex)
-				{
-					ChunkHashes.push_back(Entry.first);
-				}
-
-				// Sort from biggest position to smallest
-				std::sort(begin(ChunkHashes), end(ChunkHashes), [&](IoHash Lhs, IoHash Rhs) {
-					auto LhsKeyIt = LegacyDiskIndex.find(Lhs);
-					auto RhsKeyIt = LegacyDiskIndex.find(Rhs);
-					return RhsKeyIt->second.Location.GetOffset() < LhsKeyIt->second.Location.GetOffset();
-				});
-
-				std::unique_ptr<BlockStoreFile> NewBlockFile;
-				uint64_t						WriteOffset = 0;
-
-				std::vector<uint8_t> Chunk;
-				for (const IoHash& ChunkHash : ChunkHashes)
-				{
-					const auto&					 Entry		   = LegacyDiskIndex[ChunkHash];
-					const LegacyCasDiskLocation& ChunkLocation = Entry.Location;
-					Chunk.resize(ChunkLocation.GetSize());
-					BlockFile.Read(Chunk.data(), Chunk.size(), ChunkLocation.GetOffset());
-					if (!NewBlockFile)
-					{
-						auto BlockPath = GetBlockPath(m_BlocksBasePath, NewBlockIndex);
-						NewBlockFile   = std::make_unique<BlockStoreFile>(BlockPath);
-						NewBlockFile->Create(m_MaxBlockSize);
-					}
-					else if (WriteOffset + Chunk.size() > m_MaxBlockSize)
-					{
-						m_CasLog.Append(LogEntries);
-						LogEntries.clear();
-						NewBlockFile.reset();
-						uint64_t ChunkEnd = ChunkLocation.GetOffset() + Chunk.size();
-						BlockFile.SetFileSize(ChunkEnd);
-						NewBlockIndex = NewBlockIndex + 1;
-						ZEN_INFO("migrating store {} {}/{} blocks, remaining {} ({})",
-								 m_Config.RootDirectory / m_ContainerBaseName,
-								 NewBlockIndex,
-								 MaxRequiredBlockCount,
-								 NiceBytes(ChunkEnd),
-								 NiceBytes(TotalSize));
-						auto BlockPath = GetBlockPath(m_BlocksBasePath, NewBlockIndex);
-						NewBlockFile   = std::make_unique<BlockStoreFile>(BlockPath);
-						NewBlockFile->Create(m_MaxBlockSize);
-						WriteOffset = 0;
-					}
-					NewBlockFile->Write(Chunk.data(), Chunk.size(), WriteOffset);
-					BlockStoreLocation NewChunkLocation(NewBlockIndex, WriteOffset, Chunk.size());
-					LogEntries.push_back({.Key		   = ChunkHash,
-										  .Location	   = BlockStoreDiskLocation(NewChunkLocation, m_PayloadAlignment),
-										  .ContentType = Entry.ContentType,
-										  .Flags	   = Entry.Flags});
-					WriteOffset = RoundUp(WriteOffset + Chunk.size(), m_PayloadAlignment);
-				}
-				NewBlockFile.reset();
-				BlockFile.Close();
-			}
-			if (!LogEntries.empty())
-			{
-				m_CasLog.Append(LogEntries);
-			}
-			m_CasLog.Close();
-
-			std::filesystem::remove(LegacyLogPath);
-			CasLogEmpty = false;
-		}
+		// Keep the old cache intact for now
+		Migrate(m_Config.RootDirectory, m_ContainerBaseName, m_MaxBlockSize, m_PayloadAlignment, false, true);
 
 		if (std::filesystem::is_regular_file(SidxPath))
 		{
 			BasicFile ObjectIndexFile;
-			ObjectIndexFile.Open(SidxPath, false);
+			ObjectIndexFile.Open(SidxPath, BasicFile::EMode::kRead);
 			uint64_t Size = ObjectIndexFile.FileSize();
 			if (Size >= sizeof(CasDiskIndexHeader))
 			{
@@ -1155,7 +1288,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 				CasDiskIndexHeader Header;
 				ObjectIndexFile.Read(&Header, sizeof(Header), 0);
 				if (Header.Magic == CasDiskIndexHeader::ExpectedMagic && Header.Version == CasDiskIndexHeader::CurrentVersion &&
-					Header.PayloadAlignement > 0 && Header.EntryCount == ExpectedEntryCount)
+					Header.PayloadAlignment > 0 && Header.EntryCount == ExpectedEntryCount)
 				{
 					std::vector<CasDiskIndexEntry> Entries{Header.EntryCount};
 					ObjectIndexFile.Read(Entries.data(), Header.EntryCount * sizeof(CasDiskIndexEntry), sizeof(CasDiskIndexHeader));
@@ -1164,12 +1297,12 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 					{
 						m_LocationMap[Entry.Key] = Entry.Location;
 					}
-					m_PayloadAlignment = Header.PayloadAlignement;
+					m_PayloadAlignment = Header.PayloadAlignment;
 				}
 			}
 		}
 
-		m_CasLog.Open(SlogPath, false);
+		m_CasLog.Open(SlogPath, CasLogFile::EMode::kWrite);
 		m_CasLog.Replay([&](const CasDiskIndexEntry& Record) {
 			if (Record.Flags & CasDiskIndexEntry::kTombstone)
 			{
@@ -1187,7 +1320,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 	for (const auto& Entry : m_LocationMap)
 	{
 		const BlockStoreDiskLocation& Location = Entry.second;
-		m_TotalSize.fetch_add(Location.GetSize());
+		m_TotalSize.fetch_add(Location.GetSize(), std::memory_order_release);
 		BlockUsage.insert(Location.GetBlockIndex());
 	}
 
@@ -1253,7 +1386,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 	BasicFile GCReserveFile;
 	if (std::filesystem::is_regular_file(GCReservePath))
 	{
-		GCReserveFile.Open(GCReservePath, false);
+		GCReserveFile.Open(GCReservePath, BasicFile::EMode::kWrite);
 		std::uint64_t CurrentSize = GCReserveFile.FileSize();
 		if (CurrentSize != m_MaxBlockSize)
 		{
@@ -1281,7 +1414,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 		if (Space.Free > m_MaxBlockSize)
 		{
 			CreateDirectories(GCReservePath.parent_path());
-			GCReserveFile.Open(GCReservePath, true);
+			GCReserveFile.Open(GCReservePath, BasicFile::EMode::kTruncate);
 			GCReserveFile.SetFileSize(m_MaxBlockSize);
 		}
 	}
@@ -1938,7 +2071,7 @@ TEST_CASE("compactcas.legacyconversion")
 	if (std::filesystem::is_regular_file(SidxPath))
 	{
 		BasicFile ObjectIndexFile;
-		ObjectIndexFile.Open(SidxPath, false);
+		ObjectIndexFile.Open(SidxPath, BasicFile::EMode::kRead);
 		uint64_t Size = ObjectIndexFile.FileSize();
 		if (Size >= sizeof(CasDiskIndexHeader))
 		{
@@ -1946,7 +2079,7 @@ TEST_CASE("compactcas.legacyconversion")
 			CasDiskIndexHeader Header;
 			ObjectIndexFile.Read(&Header, sizeof(Header), 0);
 			if (Header.Magic == CasDiskIndexHeader::ExpectedMagic && Header.Version == CasDiskIndexHeader::CurrentVersion &&
-				Header.PayloadAlignement > 0 && Header.EntryCount == ExpectedEntryCount)
+				Header.PayloadAlignment > 0 && Header.EntryCount == ExpectedEntryCount)
 			{
 				LogEntries.resize(Header.EntryCount);
 				ObjectIndexFile.Read(LogEntries.data(), Header.EntryCount * sizeof(CasDiskIndexEntry), sizeof(CasDiskIndexHeader));
@@ -1959,12 +2092,12 @@ TEST_CASE("compactcas.legacyconversion")
 	std::filesystem::path SlogPath = GetLogPath(CasConfig.RootDirectory, "test");
 	{
 		TCasLogFile<CasDiskIndexEntry> CasLog;
-		CasLog.Open(SlogPath, false);
+		CasLog.Open(SlogPath, CasLogFile::EMode::kRead);
 		CasLog.Replay([&](const CasDiskIndexEntry& Record) { LogEntries.push_back(Record); });
 	}
 	TCasLogFile<LegacyCasDiskIndexEntry> LegacyCasLog;
 	std::filesystem::path				 SLegacylogPath = GetLegacyLogPath(CasConfig.RootDirectory, "test");
-	LegacyCasLog.Open(SLegacylogPath, true);
+	LegacyCasLog.Open(SLegacylogPath, CasLogFile::EMode::kTruncate);
 	for (const CasDiskIndexEntry& Entry : LogEntries)
 	{
 		BlockStoreLocation		Location = Entry.Location.Get(16);
@@ -2186,7 +2319,10 @@ TEST_CASE("compactcas.threadedinsert")	// * doctest::skip(true))
 
 TEST_CASE("compactcas.migrate.large.data" * doctest::skip(true))
 {
-	const char*			  BigDataPath = "D:\\zen-data\\dc4-zen-cache-t\\cas";
+	const char* BigDataPath = "D:\\zen-data\\dc4-zen-cache-t\\cas";
+	Migrate(BigDataPath, "tobs", 1u << 28, 16, false, true);
+	Migrate(BigDataPath, "sobs", 1u << 30, 4096, false, true);
+
 	CasStoreConfiguration CasConfig;
 	CasConfig.RootDirectory = BigDataPath;
 
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 758c0665b..247f0806a 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -88,7 +88,7 @@ FileCasStrategy::Initialize(bool IsNewStore)
 
 	CreateDirectories(m_Config.RootDirectory);
 
-	m_CasLog.Open(m_Config.RootDirectory / "cas.ulog", IsNewStore);
+	m_CasLog.Open(m_Config.RootDirectory / "cas.ulog", IsNewStore ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite);
 
 	m_CasLog.Replay([&](const FileCasIndexEntry& Entry) {
 		if (Entry.IsFlagSet(FileCasIndexEntry::kTombStone))
@@ -565,7 +565,7 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, BasicFile&
 
 				BasicFile		PayloadFile;
 				std::error_code Ec;
-				PayloadFile.Open(Parent / File, false, Ec);
+				PayloadFile.Open(Parent / File, BasicFile::EMode::kWrite, Ec);
 
 				if (!Ec)
 				{
@@ -747,7 +747,7 @@ TEST_CASE("cas.file.move")
 		IoHash	 ZeroHash = IoHash::HashBuffer(ZeroBytes);
 
 		BasicFile PayloadFile;
-		PayloadFile.Open(Payload1Path, true);
+		PayloadFile.Open(Payload1Path, BasicFile::EMode::kTruncate);
 		PayloadFile.Write(ZeroBytes, 0);
 		PayloadFile.Close();
 
diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h
index 30bb4ee8f..0be9e34f1 100644
--- a/zenstore/include/zenstore/basicfile.h
+++ b/zenstore/include/zenstore/basicfile.h
@@ -31,8 +31,21 @@ public:
 	BasicFile(const BasicFile&) = delete;
 	BasicFile& operator=(const BasicFile&) = delete;
 
-	void	 Open(const std::filesystem::path& FileName, bool IsCreate);
-	void	 Open(const std::filesystem::path& FileName, bool IsCreate, std::error_code& Ec);
+	static constexpr uint32_t kAccessTruncate = 1 << 0;
+	static constexpr uint32_t kAccessWrite	  = 1 << 1;
+	static constexpr uint32_t kAccessDelete	  = 1 << 2;
+
+	enum class EMode : uint32_t
+	{
+		kRead			= 0,
+		kWrite			= kAccessWrite,
+		kTruncate		= kAccessWrite | kAccessTruncate,
+		kDelete			= kAccessWrite | kAccessDelete,
+		kTruncateDelete = kAccessWrite | kAccessTruncate | kAccessDelete
+	};
+
+	void	 Open(const std::filesystem::path& FileName, EMode Mode);
+	void	 Open(const std::filesystem::path& FileName, EMode Mode, std::error_code& Ec);
 	void	 Close();
 	void	 Read(void* Data, uint64_t Size, uint64_t FileOffset);
 	void	 StreamFile(std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
@@ -58,6 +71,7 @@ public:
 
 protected:
 	void* m_FileHandle = nullptr;  // This is either null or valid
+private:
 };
 
 /**
diff --git a/zenstore/include/zenstore/blockstore.h b/zenstore/include/zenstore/blockstore.h
index 306282665..5222ee50e 100644
--- a/zenstore/include/zenstore/blockstore.h
+++ b/zenstore/include/zenstore/blockstore.h
@@ -50,11 +50,11 @@ struct BlockStoreDiskLocation
 		return static_cast<std::uint32_t>(PackedOffset >> MaxOffsetBits);
 	}
 
-	inline uint64_t GetOffset() const
+	inline uint64_t GetOffset(uint64_t OffsetAlignment) const
 	{
 		uint64_t PackedOffset = 0;
 		memcpy(&PackedOffset, &m_Offset, sizeof m_Offset);
-		return PackedOffset & MaxOffset;
+		return (PackedOffset & MaxOffset) * OffsetAlignment;
 	}
 
 	inline uint64_t GetSize() const { return m_Size; }
diff --git a/zenstore/include/zenstore/caslog.h b/zenstore/include/zenstore/caslog.h
index 4c1bf8196..5b6cc24a0 100644
--- a/zenstore/include/zenstore/caslog.h
+++ b/zenstore/include/zenstore/caslog.h
@@ -15,7 +15,14 @@ public:
 	CasLogFile();
 	~CasLogFile();
 
-	void	 Open(std::filesystem::path FileName, size_t RecordSize, bool isCreate);
+	enum class EMode
+	{
+		kRead,
+		kWrite,
+		kTruncate
+	};
+
+	void	 Open(std::filesystem::path FileName, size_t RecordSize, EMode Mode);
 	void	 Append(const void* DataPointer, uint64_t DataSize);
 	void	 Replay(std::function<void(const void*)>&& Handler);
 	void	 Flush();
@@ -41,6 +48,8 @@ private:
 	static_assert(sizeof(FileHeader) == 64);
 
 private:
+	void Open(std::filesystem::path FileName, size_t RecordSize, BasicFile::EMode Mode);
+
 	BasicFile			  m_File;
 	FileHeader			  m_Header;
 	size_t				  m_RecordSize	 = 1;
@@ -51,7 +60,7 @@ template<typename T>
 class TCasLogFile : public CasLogFile
 {
 public:
-	void Open(std::filesystem::path FileName, bool IsCreate) { CasLogFile::Open(FileName, sizeof(T), IsCreate); }
+	void Open(std::filesystem::path FileName, EMode Mode) { CasLogFile::Open(FileName, sizeof(T), Mode); }
 
 	// This should be called before the Replay() is called to do some basic sanity checking
 	bool Initialize() { return true; }