add test for legacy store conversion

1 files changed, 129 insertions, 12 deletions

diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index 0e6910f25..933a7c712 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -978,17 +978,6 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 			BasicFile SmallObjectFile;
 			SmallObjectFile.Open(LegacySobsPath, false);
 
-			uint64_t MaxRequiredChunkCount = SmallObjectFile.FileSize() / m_MaxBlockSize;
-			uint64_t MaxPossibleChunkCount = static_cast<uint64_t>(std::numeric_limits<std::uint32_t>::max()) + 1;
-			if (MaxRequiredChunkCount > MaxPossibleChunkCount)
-			{
-				ZEN_ERROR("legacy store migration from '{}' FAILED, required block count {}, possible {}",
-						  m_Config.RootDirectory / m_ContainerBaseName,
-						  MaxRequiredChunkCount,
-						  MaxPossibleChunkCount);
-				return;
-			}
-
 			std::unordered_map<IoHash, LegacyCasDiskIndexEntry, IoHash::Hasher> LegacyDiskIndex;
 
 			TCasLogFile<LegacyCasDiskIndexEntry> LegacyCasLog;
@@ -996,7 +985,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 			LegacyCasLog.Replay([&](const LegacyCasDiskIndexEntry& Record) {
 				if (Record.Flags & LegacyCasDiskIndexEntry::kTombstone)
 				{
-					m_LocationMap.erase(Record.Key);
+					LegacyDiskIndex.erase(Record.Key);
 				}
 				else
 				{
@@ -1004,6 +993,29 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 				}
 			});
 
+			uint64_t MaxUsedSize = 0;
+			for (const auto& Entry : LegacyDiskIndex)
+			{
+				const LegacyCasDiskIndexEntry& Record(Entry.second);
+				uint64_t					   EntryEnd = Record.Location.GetOffset() + Record.Location.GetSize();
+				if (EntryEnd > MaxUsedSize)
+				{
+					MaxUsedSize = EntryEnd;
+				}
+			}
+
+			SmallObjectFile.SetFileSize(MaxUsedSize);
+
+			uint64_t MaxRequiredChunkCount = MaxUsedSize / m_MaxBlockSize;
+			if (MaxRequiredChunkCount > CasDiskLocation::MaxBlockIndex)
+			{
+				ZEN_ERROR("legacy store migration from '{}' FAILED, required block count {}, possible {}",
+						  m_Config.RootDirectory / m_ContainerBaseName,
+						  MaxRequiredChunkCount,
+						  CasDiskLocation::MaxBlockIndex);
+				return;
+			}
+
 			std::vector<IoHash> ChunkHashes;
 			ChunkHashes.reserve(LegacyDiskIndex.size());
 			for (const auto& Entry : LegacyDiskIndex)
@@ -1842,6 +1854,111 @@ TEST_CASE("compactcas.gc.handleopeniobuffer")
 
 	CHECK(ChunkHashes[5] == IoHash::HashBuffer(RetainChunk));
 }
+
+TEST_CASE("compactcas.legacyconversion")
+{
+	ScopedTemporaryDirectory TempDir;
+
+	uint64_t ChunkSizes[]	 = {2041, 1123, 1223, 1239, 341, 1412, 912, 774, 341, 431, 554, 1098, 2048, 339, 561, 16, 16, 2048, 2048};
+	size_t	 ChunkCount		 = sizeof(ChunkSizes) / sizeof(uint64_t);
+	size_t	 SingleBlockSize = 0;
+	std::vector<IoBuffer> Chunks;
+	Chunks.reserve(ChunkCount);
+	for (const auto& Size : ChunkSizes)
+	{
+		Chunks.push_back(CreateChunk(Size));
+		SingleBlockSize += Size;
+	}
+
+	std::vector<IoHash> ChunkHashes;
+	ChunkHashes.reserve(ChunkCount);
+	for (const auto& Chunk : Chunks)
+	{
+		ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size()));
+	}
+
+	CasStoreConfiguration CasConfig;
+	CasConfig.RootDirectory = TempDir.Path();
+	CreateDirectories(CasConfig.RootDirectory);
+
+	{
+		CasGc				 Gc;
+		CasContainerStrategy Cas(CasConfig, Gc);
+		Cas.Initialize("test", gsl::narrow<uint32_t>(SingleBlockSize * 2), 16, true);
+
+		for (size_t i = 0; i < ChunkCount; i++)
+		{
+			CHECK(Cas.InsertChunk(Chunks[i], ChunkHashes[i]).New);
+		}
+
+		std::vector<IoHash> KeepChunks;
+		for (size_t i = 0; i < ChunkCount; i += 2)
+		{
+			KeepChunks.push_back(ChunkHashes[i]);
+		}
+		GcContext GcCtx;
+		GcCtx.CollectSmallObjects(true);
+		GcCtx.ContributeCas(KeepChunks);
+		Gc.CollectGarbage(GcCtx);
+	}
+
+	auto CasPath	   = BuildUcasPath(CasConfig.RootDirectory / "test" / "blocks", 1);
+	auto LegacyCasPath = CasConfig.RootDirectory / "test.ucas";
+	std::filesystem::rename(CasPath, LegacyCasPath);
+
+	std::vector<CasDiskIndexEntry> LogEntries;
+	std::filesystem::path		   SidxPath = CasConfig.RootDirectory / ("test.uidx");
+	if (std::filesystem::is_regular_file(SidxPath))
+	{
+		BasicFile SmallObjectIndex;
+		SmallObjectIndex.Open(SidxPath, false);
+		uint64_t Size = SmallObjectIndex.FileSize();
+		if (Size >= sizeof(CasDiskIndexHeader))
+		{
+			uint64_t		   ExpectedEntryCount = (Size - sizeof(sizeof(CasDiskIndexHeader))) / sizeof(CasDiskIndexEntry);
+			CasDiskIndexHeader Header;
+			SmallObjectIndex.Read(&Header, sizeof(Header), 0);
+			if (Header.Magic == CasDiskIndexHeader::ExpectedMagic && Header.Version == CasDiskIndexHeader::CurrentVersion &&
+				Header.PayloadAlignement > 0 && Header.EntryCount == ExpectedEntryCount)
+			{
+				LogEntries.resize(Header.EntryCount);
+				SmallObjectIndex.Read(LogEntries.data(), Header.EntryCount * sizeof(CasDiskIndexEntry), sizeof(CasDiskIndexHeader));
+			}
+		}
+		SmallObjectIndex.Close();
+		std::filesystem::remove(SidxPath);
+	}
+
+	std::filesystem::path				 SlogPath = CasConfig.RootDirectory / "test.ulog";
+	TCasLogFile<LegacyCasDiskIndexEntry> LegacyCasLog;
+	LegacyCasLog.Open(SlogPath, true);
+	for (const auto& Entry : LogEntries)
+	{
+		CasLocation Location = Entry.Location.Get(16);
+		CHECK(Location.BlockIndex == 1);
+		LegacyCasDiskLocation	LegacyLocation(Location.Offset, Location.Size);
+		LegacyCasDiskIndexEntry LegacyEntry = {.Key			= Entry.Key,
+											   .Location	= LegacyLocation,
+											   .ContentType = Entry.ContentType,
+											   .Flags		= Entry.Flags};
+		LegacyCasLog.Append(LegacyEntry);
+	}
+	LegacyCasLog.Close();
+
+	{
+		CasGc				 Gc;
+		CasContainerStrategy Cas(CasConfig, Gc);
+		Cas.Initialize("test", 2048, 16, false);
+
+		for (size_t i = 0; i < ChunkCount; i += 2)
+		{
+			CHECK(Cas.HaveChunk(ChunkHashes[i]));
+			CHECK(!Cas.HaveChunk(ChunkHashes[i + 1]));
+			CHECK(ChunkHashes[i] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[i])));
+		}
+	}
+}
+
 #endif
 
 void