Don't rewrite object file if it is smaller than max block size

1 files changed, 71 insertions, 48 deletions

diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index 887197c2d..cf49a10f9 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -974,11 +974,12 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 				{
 					MaxUsedSize = EntryEnd;
 				}
+				TotalSize += Record.Location.GetSize();
 			}
+			LegacyCasLog.Close();
 
 			BlockFile.SetFileSize(MaxUsedSize);
-
-			uint64_t MaxRequiredChunkCount = MaxUsedSize / m_MaxBlockSize;
+			uint64_t MaxRequiredChunkCount = RoundUp(MaxUsedSize, m_MaxBlockSize) / m_MaxBlockSize;
 			if (MaxRequiredChunkCount > BlockStoreDiskLocation::MaxBlockIndex)
 			{
 				ZEN_ERROR("legacy store migration from '{}' FAILED, required block count {}, possible {}",
@@ -988,60 +989,82 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 				return;
 			}
 
-			std::vector<IoHash> ChunkHashes;
-			ChunkHashes.reserve(LegacyDiskIndex.size());
-			for (const auto& Entry : LegacyDiskIndex)
+			m_CasLog.Open(SlogPath, true);
+			if (MaxRequiredChunkCount < 2)
 			{
-				ChunkHashes.push_back(Entry.first);
-				TotalSize += Entry.second.Location.GetSize();
+				std::vector<CasDiskIndexEntry> LogEntries;
+				LogEntries.reserve(LegacyDiskIndex.size());
+				for (const auto& Entry : LegacyDiskIndex)
+				{
+					const LegacyCasDiskIndexEntry& Record(Entry.second);
+					BlockStoreLocation			   NewChunkLocation(0, Record.Location.GetOffset(), Record.Location.GetSize());
+					LogEntries.push_back({.Key		   = Entry.second.Key,
+										  .Location	   = BlockStoreDiskLocation(NewChunkLocation, m_PayloadAlignment),
+										  .ContentType = Record.ContentType,
+										  .Flags	   = Record.Flags});
+				}
+				m_CasLog.Append(LogEntries);
+				auto BlockPath = BuildUcasPath(m_BlocksBasePath, 0);
+				CreateDirectories(BlockPath.parent_path());
+				BlockFile.Close();
+				std::filesystem::rename(LegacySobsPath, BlockPath);
 			}
-			LegacyCasLog.Close();
-
-			// Sort from biggest position to smallest
-			std::sort(begin(ChunkHashes), end(ChunkHashes), [&](IoHash Lhs, IoHash Rhs) {
-				auto LhsKeyIt = LegacyDiskIndex.find(Lhs);
-				auto RhsKeyIt = LegacyDiskIndex.find(Rhs);
-				return RhsKeyIt->second.Location.GetOffset() < LhsKeyIt->second.Location.GetOffset();
-			});
-
-			m_CasLog.Open(SlogPath, true);
-
-			std::unique_ptr<BlockStoreFile> NewBlockFile;
-			uint64_t						WriteOffset = 0;
-
-			std::vector<uint8_t> Chunk;
-			for (const IoHash& ChunkHash : ChunkHashes)
+			else
 			{
-				const auto&					 Entry		   = LegacyDiskIndex[ChunkHash];
-				const LegacyCasDiskLocation& ChunkLocation = Entry.Location;
-				Chunk.resize(ChunkLocation.GetSize());
-				BlockFile.Read(Chunk.data(), Chunk.size(), ChunkLocation.GetOffset());
-				if (!NewBlockFile)
+				std::vector<IoHash> ChunkHashes;
+				ChunkHashes.reserve(LegacyDiskIndex.size());
+				for (const auto& Entry : LegacyDiskIndex)
 				{
-					auto BlockPath = BuildUcasPath(m_BlocksBasePath, NewBlockIndex);
-					NewBlockFile   = std::make_unique<BlockStoreFile>(BlockPath);
-					NewBlockFile->Create(m_MaxBlockSize);
+					ChunkHashes.push_back(Entry.first);
 				}
-				else if (WriteOffset + Chunk.size() > m_MaxBlockSize)
+
+				// Sort from biggest position to smallest
+				std::sort(begin(ChunkHashes), end(ChunkHashes), [&](IoHash Lhs, IoHash Rhs) {
+					auto LhsKeyIt = LegacyDiskIndex.find(Lhs);
+					auto RhsKeyIt = LegacyDiskIndex.find(Rhs);
+					return RhsKeyIt->second.Location.GetOffset() < LhsKeyIt->second.Location.GetOffset();
+				});
+
+				std::unique_ptr<BlockStoreFile> NewBlockFile;
+				uint64_t						WriteOffset = 0;
+
+				std::vector<uint8_t> Chunk;
+				for (const IoHash& ChunkHash : ChunkHashes)
 				{
-					NewBlockFile.reset();
-					uint64_t ChunkEnd = ChunkLocation.GetOffset() + Chunk.size();
-					BlockFile.SetFileSize(ChunkEnd);
-					NewBlockIndex  = NewBlockIndex + 1;
-					auto BlockPath = BuildUcasPath(m_BlocksBasePath, NewBlockIndex);
-					NewBlockFile   = std::make_unique<BlockStoreFile>(BlockPath);
-					NewBlockFile->Create(m_MaxBlockSize);
-					WriteOffset = 0;
+					const auto&					 Entry		   = LegacyDiskIndex[ChunkHash];
+					const LegacyCasDiskLocation& ChunkLocation = Entry.Location;
+					Chunk.resize(ChunkLocation.GetSize());
+					BlockFile.Read(Chunk.data(), Chunk.size(), ChunkLocation.GetOffset());
+					if (!NewBlockFile)
+					{
+						auto BlockPath = BuildUcasPath(m_BlocksBasePath, NewBlockIndex);
+						NewBlockFile   = std::make_unique<BlockStoreFile>(BlockPath);
+						NewBlockFile->Create(m_MaxBlockSize);
+					}
+					else if (WriteOffset + Chunk.size() > m_MaxBlockSize)
+					{
+						NewBlockFile.reset();
+						uint64_t ChunkEnd = ChunkLocation.GetOffset() + Chunk.size();
+						BlockFile.SetFileSize(ChunkEnd);
+						NewBlockIndex  = NewBlockIndex + 1;
+						auto BlockPath = BuildUcasPath(m_BlocksBasePath, NewBlockIndex);
+						NewBlockFile   = std::make_unique<BlockStoreFile>(BlockPath);
+						NewBlockFile->Create(m_MaxBlockSize);
+						WriteOffset = 0;
+					}
+					NewBlockFile->Write(Chunk.data(), Chunk.size(), WriteOffset);
+					BlockStoreLocation NewChunkLocation(NewBlockIndex, WriteOffset, Chunk.size());
+					m_CasLog.Append({.Key		  = ChunkHash,
+									 .Location	  = BlockStoreDiskLocation(NewChunkLocation, m_PayloadAlignment),
+									 .ContentType = Entry.ContentType,
+									 .Flags		  = Entry.Flags});
+					WriteOffset = RoundUp(WriteOffset + Chunk.size(), m_PayloadAlignment);
 				}
-				NewBlockFile->Write(Chunk.data(), Chunk.size(), WriteOffset);
-				BlockStoreLocation NewChunkLocation(NewBlockIndex, WriteOffset, Chunk.size());
-				m_CasLog.Append({.Key = ChunkHash, .Location = BlockStoreDiskLocation(NewChunkLocation, m_PayloadAlignment)});
-				WriteOffset = RoundUp(WriteOffset + Chunk.size(), m_PayloadAlignment);
+				NewBlockFile.reset();
+				BlockFile.Close();
 			}
-			NewBlockFile.reset();
 			m_CasLog.Close();
 
-			BlockFile.Close();
 			std::filesystem::remove(LegacySobsPath);
 			CasLogEmpty = false;
 		}
@@ -2082,9 +2105,9 @@ TEST_CASE("compactcas.threadedinsert")	// * doctest::skip(true))
 	}
 }
 
-TEST_CASE("compactcas.migrate.large.data" * doctest::skip(true))
+TEST_CASE("compactcas.migrate.large.data" * doctest::skip(false))
 {
-	const char* BigDataPath = "D:\\zen-data\\dc4-zen-cache-t\\cas";
+	const char*			  BigDataPath = "D:\\zen-data\\dc4-zen-cache-t\\cas";
 	CasStoreConfiguration CasConfig;
 	CasConfig.RootDirectory = BigDataPath;