collectgarbage for compactcas and structured cache uses shared implementation

3 files changed, 37 insertions, 68 deletions

diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp
index 4cf3c6486..309c99d1e 100644
--- a/zenstore/blockstore.cpp
+++ b/zenstore/blockstore.cpp
@@ -1,11 +1,11 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
-#include "compactcas.h"
+#include <zenstore/blockstore.h>
 
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/scopeguard.h>
-#include <zenstore/blockstore.h>
+#include <zencore/timer.h>
 
 #if ZEN_WITH_TESTS
 #	include <zencore/compactbinarybuilder.h>
@@ -111,26 +111,6 @@ BlockStoreFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::functio
 	m_File.StreamByteRange(FileOffset, Size, std::move(ChunkFun));
 }
 
-namespace {
-	const char* DataExtension = ".ucas";
-
-	std::filesystem::path GetBlockPath(const std::filesystem::path& BlocksBasePath, const uint32_t BlockIndex)
-	{
-		ExtendablePathBuilder<256> Path;
-
-		char BlockHexString[9];
-		ToHexNumber(BlockIndex, BlockHexString);
-
-		Path.Append(BlocksBasePath);
-		Path.AppendSeparator();
-		Path.AppendAsciiRange(BlockHexString, BlockHexString + 4);
-		Path.AppendSeparator();
-		Path.Append(BlockHexString);
-		Path.Append(DataExtension);
-		return Path.ToPath();
-	}
-}  // namespace
-
 void
 BlockStore::Initialize(const std::filesystem::path&			  BlocksBasePath,
 					   uint64_t								  MaxBlockSize,
@@ -171,7 +151,7 @@ BlockStore::Initialize(const std::filesystem::path&			  BlocksBasePath,
 				if (Entry.is_regular_file())
 				{
 					const std::filesystem::path Path = Entry.path();
-					if (Path.extension() != DataExtension)
+					if (Path.extension() != GetBlockFileExtension())
 					{
 						continue;
 					}
@@ -595,6 +575,29 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 	return;
 }
 
+const char*
+BlockStore::GetBlockFileExtension()
+{
+	return ".ucas";
+}
+
+std::filesystem::path
+BlockStore::GetBlockPath(const std::filesystem::path& BlocksBasePath, const uint32_t BlockIndex)
+{
+	ExtendablePathBuilder<256> Path;
+
+	char BlockHexString[9];
+	ToHexNumber(BlockIndex, BlockHexString);
+
+	Path.Append(BlocksBasePath);
+	Path.AppendSeparator();
+	Path.AppendAsciiRange(BlockHexString, BlockHexString + 4);
+	Path.AppendSeparator();
+	Path.Append(BlockHexString);
+	Path.Append(GetBlockFileExtension());
+	return Path.ToPath();
+}
+
 #if ZEN_WITH_TESTS
 
 static bool
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index 84019d7aa..30747f554 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -66,7 +66,6 @@ namespace {
 
 	const char* IndexExtension = ".uidx";
 	const char* LogExtension   = ".ulog";
-	const char* DataExtension  = ".ucas";
 
 	std::filesystem::path GetBasePath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
 	{
@@ -93,22 +92,6 @@ namespace {
 		return GetBasePath(RootPath, ContainerBaseName) / "blocks";
 	}
 
-	std::filesystem::path GetBlockPath(const std::filesystem::path& BlocksBasePath, const uint32_t BlockIndex)
-	{
-		ExtendablePathBuilder<256> Path;
-
-		char BlockHexString[9];
-		ToHexNumber(BlockIndex, BlockHexString);
-
-		Path.Append(BlocksBasePath);
-		Path.AppendSeparator();
-		Path.AppendAsciiRange(BlockHexString, BlockHexString + 4);
-		Path.AppendSeparator();
-		Path.Append(BlockHexString);
-		Path.Append(DataExtension);
-		return Path.ToPath();
-	}
-
 	std::filesystem::path GetLegacyLogPath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
 	{
 		return RootPath / (ContainerBaseName + LogExtension);
@@ -116,7 +99,7 @@ namespace {
 
 	std::filesystem::path GetLegacyDataPath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
 	{
-		return RootPath / (ContainerBaseName + DataExtension);
+		return RootPath / (ContainerBaseName + ".ucas");
 	}
 
 	std::filesystem::path GetLegacyIndexPath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
@@ -315,7 +298,7 @@ CasContainerStrategy::FindChunk(const IoHash& ChunkHash)
 	BlockStoreLocation Location = KeyIt->second.Get(m_PayloadAlignment);
 	_.ReleaseNow();
 
-	Ref<BlockStoreFile> ChunkBlock = m_BlockStore.GetChunkBlock(Location);	// m_ChunkBlocks[Location.BlockIndex];
+	Ref<BlockStoreFile> ChunkBlock = m_BlockStore.GetChunkBlock(Location);
 	if (!ChunkBlock)
 	{
 		return IoBuffer();
@@ -348,17 +331,6 @@ void
 CasContainerStrategy::Flush()
 {
 	m_BlockStore.Flush();
-	/* {
-		RwLock::ExclusiveLockScope _(m_InsertLock);
-		if (m_CurrentInsertOffset > 0)
-		{
-			uint32_t WriteBlockIndex = m_WriteBlockIndex.load(std::memory_order_acquire);
-			WriteBlockIndex			 = (WriteBlockIndex + 1) & BlockStoreDiskLocation::MaxBlockIndex;
-			m_WriteBlock			 = nullptr;
-			m_WriteBlockIndex.store(WriteBlockIndex, std::memory_order_release);
-			m_CurrentInsertOffset = 0;
-		}
-	}*/
 	MakeIndexSnapshot();
 }
 
@@ -533,8 +505,6 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 		return;
 	}
 
-	auto GetChunkLocations = [] {};
-
 	std::vector<IoHash> DeletedChunks;
 	m_BlockStore.ReclaimSpace(
 		BlockStoreState,
@@ -543,7 +513,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 		m_PayloadAlignment,
 		false,
 		[this, &DeletedChunks, &ChunkIndexToChunkHash, &LocationMap, &ReadBlockTimeUs, &ReadBlockLongestTimeUs](
-			uint32_t											  BlockIndex,
+			uint32_t,
 			const std::unordered_map<size_t, BlockStoreLocation>& MovedChunks,
 			const std::vector<size_t>&							  RemovedChunks) {
 			std::vector<CasDiskIndexEntry> LogEntries;
@@ -578,17 +548,12 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 					if (Entry.Flags & CasDiskIndexEntry::kTombstone)
 					{
 						m_LocationMap.erase(Entry.Key);
-						auto	 KeyIt	   = m_LocationMap.find(Entry.Key);
 						uint64_t ChunkSize = Entry.Location.GetSize();
 						m_TotalSize.fetch_sub(ChunkSize);
 						continue;
 					}
 					m_LocationMap[Entry.Key] = Entry.Location;
 				}
-				for (const auto& Entry : m_LocationMap)
-				{
-					ZEN_ASSERT(Entry.second.GetBlockIndex() != BlockIndex);
-				}
 			}
 		});
 
@@ -809,7 +774,7 @@ CasContainerStrategy::MigrateLegacyData(bool CleanSource)
 	});
 
 	uint32_t WriteBlockIndex = 0;
-	while (std::filesystem::exists(GetBlockPath(m_BlocksBasePath, WriteBlockIndex)))
+	while (std::filesystem::exists(BlockStore::GetBlockPath(m_BlocksBasePath, WriteBlockIndex)))
 	{
 		++WriteBlockIndex;
 	}
@@ -976,7 +941,7 @@ CasContainerStrategy::MigrateLegacyData(bool CleanSource)
 			LogEntries.push_back(
 				{.Key = Entry.second.Key, .Location = NewLocation, .ContentType = Record.ContentType, .Flags = Record.Flags});
 		}
-		std::filesystem::path BlockPath = GetBlockPath(m_BlocksBasePath, WriteBlockIndex);
+		std::filesystem::path BlockPath = BlockStore::GetBlockPath(m_BlocksBasePath, WriteBlockIndex);
 		CreateDirectories(BlockPath.parent_path());
 		BlockFile.Close();
 		std::filesystem::rename(LegacyDataPath, BlockPath);
@@ -1038,7 +1003,7 @@ CasContainerStrategy::MigrateLegacyData(bool CleanSource)
 				BlockRanges.push_back(BlockRange);
 
 				WriteBlockIndex++;
-				while (std::filesystem::exists(GetBlockPath(m_BlocksBasePath, WriteBlockIndex)))
+				while (std::filesystem::exists(BlockStore::GetBlockPath(m_BlocksBasePath, WriteBlockIndex)))
 				{
 					++WriteBlockIndex;
 				}
@@ -1077,7 +1042,7 @@ CasContainerStrategy::MigrateLegacyData(bool CleanSource)
 						 NiceTimeSpanMs(ETA));
 			}
 
-			std::filesystem::path BlockPath = GetBlockPath(m_BlocksBasePath, BlockRange.BlockIndex);
+			std::filesystem::path BlockPath = BlockStore::GetBlockPath(m_BlocksBasePath, BlockRange.BlockIndex);
 			BlockStoreFile		  ChunkBlock(BlockPath);
 			ChunkBlock.Create(BlockRange.BlockSize);
 			uint64_t Offset = 0;
@@ -1176,14 +1141,12 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 	std::filesystem::path LogPath = GetLogPath(m_Config.RootDirectory, m_ContainerBaseName);
 	m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite);
 
-	std::unordered_set<uint32_t>	KnownBlocks;
 	std::vector<BlockStoreLocation> KnownLocations;
 	KnownLocations.reserve(m_LocationMap.size());
 	for (const auto& Entry : m_LocationMap)
 	{
 		const BlockStoreDiskLocation& Location = Entry.second;
 		m_TotalSize.fetch_add(Location.GetSize(), std::memory_order_seq_cst);
-		KnownBlocks.insert(Location.GetBlockIndex());
 		KnownLocations.push_back(Location.Get(m_PayloadAlignment));
 	}
 
@@ -1842,7 +1805,7 @@ TEST_CASE("compactcas.legacyconversion")
 		Gc.CollectGarbage(GcCtx);
 	}
 
-	std::filesystem::path BlockPath		 = GetBlockPath(GetBlocksBasePath(CasConfig.RootDirectory, "test"), 1);
+	std::filesystem::path BlockPath		 = BlockStore::GetBlockPath(GetBlocksBasePath(CasConfig.RootDirectory, "test"), 1);
 	std::filesystem::path LegacyDataPath = GetLegacyDataPath(CasConfig.RootDirectory, "test");
 	std::filesystem::rename(BlockPath, LegacyDataPath);
 
diff --git a/zenstore/include/zenstore/blockstore.h b/zenstore/include/zenstore/blockstore.h
index 084142636..1eff46367 100644
--- a/zenstore/include/zenstore/blockstore.h
+++ b/zenstore/include/zenstore/blockstore.h
@@ -143,6 +143,9 @@ public:
 						const ReclaimCallback& Callback = [](uint32_t, const std::unordered_map<size_t, BlockStoreLocation>&, const std::vector<size_t>&) {
 		});
 
+	static const char*			 GetBlockFileExtension();
+	static std::filesystem::path GetBlockPath(const std::filesystem::path& BlocksBasePath, const uint32_t BlockIndex);
+
 private:
 	std::unordered_map<uint32_t, Ref<BlockStoreFile>> m_ChunkBlocks;