faster cas write during migration

better logging during migration

1 files changed, 26 insertions, 14 deletions

diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index cf49a10f9..f9cc34302 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -923,7 +923,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 			Stopwatch  MigrationTimer;
 			uint64_t   TotalSize = 0;
 			const auto Guard	 = MakeGuard([this, &MigrationTimer, &NewBlockIndex, &TotalSize] {
-				ZEN_INFO("migrated store {} to {} chunks in {} ({})",
+				ZEN_INFO("migrated store {} to {} blocks in {} ({})",
 						 m_Config.RootDirectory / m_ContainerBaseName,
 						 NewBlockIndex + 1,
 						 NiceTimeSpanMs(MigrationTimer.GetElapsedTimeMs()),
@@ -979,21 +979,22 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 			LegacyCasLog.Close();
 
 			BlockFile.SetFileSize(MaxUsedSize);
-			uint64_t MaxRequiredChunkCount = RoundUp(MaxUsedSize, m_MaxBlockSize) / m_MaxBlockSize;
-			if (MaxRequiredChunkCount > BlockStoreDiskLocation::MaxBlockIndex)
+			uint64_t MaxRequiredBlockCount = RoundUp(MaxUsedSize, m_MaxBlockSize) / m_MaxBlockSize;
+			if (MaxRequiredBlockCount > BlockStoreDiskLocation::MaxBlockIndex)
 			{
 				ZEN_ERROR("legacy store migration from '{}' FAILED, required block count {}, possible {}",
 						  m_Config.RootDirectory / m_ContainerBaseName,
-						  MaxRequiredChunkCount,
+						  MaxRequiredBlockCount,
 						  BlockStoreDiskLocation::MaxBlockIndex);
 				return;
 			}
 
 			m_CasLog.Open(SlogPath, true);
-			if (MaxRequiredChunkCount < 2)
+
+			std::vector<CasDiskIndexEntry> LogEntries;
+			LogEntries.reserve(LegacyDiskIndex.size());
+			if (MaxRequiredBlockCount < 2)
 			{
-				std::vector<CasDiskIndexEntry> LogEntries;
-				LogEntries.reserve(LegacyDiskIndex.size());
 				for (const auto& Entry : LegacyDiskIndex)
 				{
 					const LegacyCasDiskIndexEntry& Record(Entry.second);
@@ -1003,7 +1004,6 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 										  .ContentType = Record.ContentType,
 										  .Flags	   = Record.Flags});
 				}
-				m_CasLog.Append(LogEntries);
 				auto BlockPath = BuildUcasPath(m_BlocksBasePath, 0);
 				CreateDirectories(BlockPath.parent_path());
 				BlockFile.Close();
@@ -1043,10 +1043,18 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 					}
 					else if (WriteOffset + Chunk.size() > m_MaxBlockSize)
 					{
+						m_CasLog.Append(LogEntries);
+						LogEntries.clear();
 						NewBlockFile.reset();
 						uint64_t ChunkEnd = ChunkLocation.GetOffset() + Chunk.size();
 						BlockFile.SetFileSize(ChunkEnd);
-						NewBlockIndex  = NewBlockIndex + 1;
+						NewBlockIndex = NewBlockIndex + 1;
+						ZEN_INFO("migrating store {} {}/{} blocks, remaining {} ({})",
+								 m_Config.RootDirectory / m_ContainerBaseName,
+								 NewBlockIndex,
+								 MaxRequiredBlockCount,
+								 NiceBytes(ChunkEnd),
+								 NiceBytes(TotalSize));
 						auto BlockPath = BuildUcasPath(m_BlocksBasePath, NewBlockIndex);
 						NewBlockFile   = std::make_unique<BlockStoreFile>(BlockPath);
 						NewBlockFile->Create(m_MaxBlockSize);
@@ -1054,15 +1062,19 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 					}
 					NewBlockFile->Write(Chunk.data(), Chunk.size(), WriteOffset);
 					BlockStoreLocation NewChunkLocation(NewBlockIndex, WriteOffset, Chunk.size());
-					m_CasLog.Append({.Key		  = ChunkHash,
-									 .Location	  = BlockStoreDiskLocation(NewChunkLocation, m_PayloadAlignment),
-									 .ContentType = Entry.ContentType,
-									 .Flags		  = Entry.Flags});
+					LogEntries.push_back({.Key		   = ChunkHash,
+										  .Location	   = BlockStoreDiskLocation(NewChunkLocation, m_PayloadAlignment),
+										  .ContentType = Entry.ContentType,
+										  .Flags	   = Entry.Flags});
 					WriteOffset = RoundUp(WriteOffset + Chunk.size(), m_PayloadAlignment);
 				}
 				NewBlockFile.reset();
 				BlockFile.Close();
 			}
+			if (!LogEntries.empty())
+			{
+				m_CasLog.Append(LogEntries);
+			}
 			m_CasLog.Close();
 
 			std::filesystem::remove(LegacySobsPath);
@@ -2121,7 +2133,7 @@ TEST_CASE("compactcas.migrate.large.data" * doctest::skip(false))
 	CasContainerStrategy SobsCas(CasConfig, SobsCasGc);
 	SobsCas.Initialize("sobs", 1u << 30, 4096, false);
 	GcContext SobsGcCtx;
-	TobsCas.CollectGarbage(SobsGcCtx);
+	SobsCas.CollectGarbage(SobsGcCtx);
 }
 
 #endif