clean up paths

1 files changed, 110 insertions, 43 deletions

diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index f9cc34302..ec0a5a7ba 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -41,7 +41,57 @@ struct CasDiskIndexHeader
 static_assert(sizeof(CasDiskIndexHeader) == 32);
 
 namespace {
-	std::filesystem::path BuildUcasPath(const std::filesystem::path& BlocksBasePath, const uint32_t BlockIndex)
+	std::vector<CasDiskIndexEntry> MakeCasDiskEntries(const std::unordered_map<IoHash, BlockStoreDiskLocation>& MovedChunks,
+													  const std::vector<IoHash>&								DeletedChunks)
+	{
+		std::vector<CasDiskIndexEntry> result;
+		result.reserve(MovedChunks.size());
+		for (const auto& MovedEntry : MovedChunks)
+		{
+			result.push_back({.Key = MovedEntry.first, .Location = MovedEntry.second});
+		}
+		for (const IoHash& ChunkHash : DeletedChunks)
+		{
+			result.push_back({.Key = ChunkHash, .Flags = CasDiskIndexEntry::kTombstone});
+		}
+		return result;
+	}
+
+	const char* IndexExtension = ".uidx";
+	const char* LogExtension   = ".ulog";
+	const char* DataExtension  = ".ucas";
+
+	std::filesystem::path GetIndexPath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
+	{
+		return RootPath / ContainerBaseName / (ContainerBaseName + IndexExtension);
+	}
+
+	std::filesystem::path GetTempIndexPath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
+	{
+		return RootPath / ContainerBaseName / (ContainerBaseName + ".tmp" + LogExtension);
+	}
+
+	std::filesystem::path GetLogPath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
+	{
+		return RootPath / ContainerBaseName / (ContainerBaseName + LogExtension);
+	}
+
+	std::filesystem::path GetTempLogPath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
+	{
+		return RootPath / ContainerBaseName / (ContainerBaseName + ".tmp" + LogExtension);
+	}
+
+	std::filesystem::path GetRecoverLogPath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
+	{
+		return RootPath / ContainerBaseName / (ContainerBaseName + ".recover" + LogExtension);
+	}
+
+	std::filesystem::path GetBlocksBasePath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
+	{
+		return RootPath / ContainerBaseName / "blocks";
+	}
+
+	std::filesystem::path GetBlockPath(const std::filesystem::path& BlocksBasePath, const uint32_t BlockIndex)
 	{
 		ExtendablePathBuilder<256> Path;
 
@@ -57,22 +107,26 @@ namespace {
 		return Path.ToPath();
 	}
 
-	std::vector<CasDiskIndexEntry> MakeCasDiskEntries(const std::unordered_map<IoHash, BlockStoreDiskLocation>& MovedChunks,
-													  const std::vector<IoHash>&								DeletedChunks)
+	std::filesystem::path GetGCReservePath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
 	{
-		std::vector<CasDiskIndexEntry> result;
-		result.reserve(MovedChunks.size());
-		for (const auto& MovedEntry : MovedChunks)
-		{
-			result.push_back({.Key = MovedEntry.first, .Location = MovedEntry.second});
-		}
-		for (const IoHash& ChunkHash : DeletedChunks)
-		{
-			result.push_back({.Key = ChunkHash, .Flags = CasDiskIndexEntry::kTombstone});
-		}
-		return result;
+		return RootPath / ContainerBaseName / (ContainerBaseName + ".gc.reserve" + DataExtension);
+	}
+
+	std::filesystem::path GetLegacyLogPath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
+	{
+		return RootPath / (ContainerBaseName + LogExtension);
 	}
 
+	std::filesystem::path GetLegacyUcasPath(const std::filesystem::path& RootPath, const std::string& ContainerBaseName)
+	{
+		return RootPath / (ContainerBaseName + DataExtension);
+	}
+
+
+//	void Migrate(const std::filesystem::path& RootPath, const std::string_view ContainerBaseName, uint32_t MaxBlockSize, uint64_t Alignment)
+//	{
+//	}
+
 }  // namespace
 
 //////////////////////////////////////////////////////////////////////////
@@ -98,7 +152,7 @@ CasContainerStrategy::Initialize(const std::string_view ContainerBaseName, uint3
 	m_ContainerBaseName = ContainerBaseName;
 	m_PayloadAlignment	= Alignment;
 	m_MaxBlockSize		= MaxBlockSize;
-	m_BlocksBasePath	= m_Config.RootDirectory / m_ContainerBaseName / "blocks";
+	m_BlocksBasePath	= GetBlocksBasePath(m_Config.RootDirectory, m_ContainerBaseName);
 
 	OpenContainer(IsNewStore);
 
@@ -143,7 +197,7 @@ CasContainerStrategy::InsertChunk(const void* ChunkData, size_t ChunkSize, const
 				{
 					WriteBlockIndex = (WriteBlockIndex + 1) & BlockStoreDiskLocation::MaxBlockIndex;
 				}
-				auto BlockPath				   = BuildUcasPath(m_BlocksBasePath, WriteBlockIndex);
+				auto BlockPath				   = GetBlockPath(m_BlocksBasePath, WriteBlockIndex);
 				m_WriteBlock				   = std::make_shared<BlockStoreFile>(BlockPath);
 				m_ChunkBlocks[WriteBlockIndex] = m_WriteBlock;
 				m_WriteBlockIndex.store(WriteBlockIndex, std::memory_order_release);
@@ -611,7 +665,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 					{
 						NextBlockIndex = (NextBlockIndex + 1) & BlockStoreDiskLocation::MaxBlockIndex;
 					}
-					auto NewBlockPath			  = BuildUcasPath(m_BlocksBasePath, NextBlockIndex);
+					auto NewBlockPath			  = GetBlockPath(m_BlocksBasePath, NextBlockIndex);
 					NewBlockFile				  = std::make_shared<BlockStoreFile>(NewBlockPath);
 					m_ChunkBlocks[NextBlockIndex] = NewBlockFile;
 				}
@@ -629,7 +683,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 
 				if (Space.Free < m_MaxBlockSize)
 				{
-					std::filesystem::path GCReservePath = m_Config.RootDirectory / (m_ContainerBaseName + ".gc.reserve.ucas");
+					std::filesystem::path GCReservePath = GetGCReservePath(m_Config.RootDirectory, m_ContainerBaseName);
 					if (!std::filesystem::is_regular_file(GCReservePath))
 					{
 						ZEN_INFO("garbage collect from '{}' FAILED, required disk space {}, free {}",
@@ -646,7 +700,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 					ZEN_INFO("using gc reserve for '{}', disk free {}",
 							 m_Config.RootDirectory / m_ContainerBaseName,
 							 NiceBytes(Space.Free));
-					auto NewBlockPath = BuildUcasPath(m_BlocksBasePath, NextBlockIndex);
+					auto NewBlockPath = GetBlockPath(m_BlocksBasePath, NextBlockIndex);
 					std::filesystem::rename(GCReservePath, NewBlockPath);
 					NewBlockFile->Open();
 				}
@@ -697,7 +751,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 
 	GcCtx.DeletedCas(DeletedChunks);
 
-	std::filesystem::path GCReservePath = m_Config.RootDirectory / (m_ContainerBaseName + ".gc.reserve.ucas");
+	std::filesystem::path GCReservePath = GetGCReservePath(m_Config.RootDirectory, m_ContainerBaseName);
 	if (std::filesystem::is_regular_file(GCReservePath))
 	{
 		return;
@@ -719,6 +773,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 		return;
 	}
 	BasicFile GCReserveFile;
+	CreateDirectories(GCReservePath.parent_path());
 	GCReserveFile.Open(GCReservePath, true);
 	GCReserveFile.SetFileSize(m_MaxBlockSize);
 
@@ -732,11 +787,11 @@ CasContainerStrategy::MakeIndexSnapshot()
 
 	namespace fs = std::filesystem;
 
-	fs::path SlogPath		   = m_Config.RootDirectory / (m_ContainerBaseName + ".ulog");
-	fs::path SidxPath		   = m_Config.RootDirectory / (m_ContainerBaseName + ".uidx");
-	fs::path STmplogPath	   = m_Config.RootDirectory / (m_ContainerBaseName + ".tmp.ulog");
-	fs::path STmpSidxPath	   = m_Config.RootDirectory / (m_ContainerBaseName + ".tmp.uidx");
-	fs::path SRecoveredlogPath = m_Config.RootDirectory / (m_ContainerBaseName + ".recover.ulog");
+	fs::path SlogPath		   = GetLogPath(m_Config.RootDirectory, m_ContainerBaseName);
+	fs::path SidxPath		   = GetIndexPath(m_Config.RootDirectory, m_ContainerBaseName);
+	fs::path STmplogPath	   = GetTempLogPath(m_Config.RootDirectory, m_ContainerBaseName);
+	fs::path STmpSidxPath	   = GetTempIndexPath(m_Config.RootDirectory, m_ContainerBaseName);
+	fs::path SRecoveredlogPath = GetRecoverLogPath(m_Config.RootDirectory, m_ContainerBaseName);
 
 	// Index away, we keep it if something goes wrong
 	if (fs::is_regular_file(STmpSidxPath))
@@ -886,6 +941,7 @@ namespace {
 
 }  // namespace
 
+
 void
 CasContainerStrategy::OpenContainer(bool IsNewStore)
 {
@@ -893,14 +949,19 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 
 	m_LocationMap.clear();
 
-	std::filesystem::path SidxPath		 = m_Config.RootDirectory / (m_ContainerBaseName + ".uidx");
-	std::filesystem::path SlogPath		 = m_Config.RootDirectory / (m_ContainerBaseName + ".ulog");
-	std::filesystem::path LegacySobsPath = m_Config.RootDirectory / (m_ContainerBaseName + ".ucas");
+	std::filesystem::path SidxPath		 = GetIndexPath(m_Config.RootDirectory, m_ContainerBaseName);
+	std::filesystem::path SlogPath		 = GetLogPath(m_Config.RootDirectory, m_ContainerBaseName);
+	std::filesystem::path LegacyLogPath	 = GetLegacyLogPath(m_Config.RootDirectory, m_ContainerBaseName);
+	std::filesystem::path LegacySobsPath = GetLegacyUcasPath(m_Config.RootDirectory, m_ContainerBaseName);
 
 	bool CasLogEmpty = true;
 
 	if (IsNewStore)
 	{
+		if (std::filesystem::is_regular_file(LegacyLogPath))
+		{
+			std::filesystem::remove(LegacyLogPath);
+		}
 		if (std::filesystem::is_regular_file(LegacySobsPath))
 		{
 			std::filesystem::remove(LegacySobsPath);
@@ -913,11 +974,12 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 		{
 			std::filesystem::remove(SidxPath);
 		}
+		CreateDirectories(SlogPath.parent_path());
 		m_CasLog.Open(SlogPath, true);
 	}
 	else
 	{
-		if (std::filesystem::is_regular_file(LegacySobsPath))
+		if (std::filesystem::is_regular_file(LegacyLogPath) && std::filesystem::is_regular_file(LegacySobsPath))
 		{
 			uint32_t   NewBlockIndex = 0;
 			Stopwatch  MigrationTimer;
@@ -953,7 +1015,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 			std::unordered_map<IoHash, LegacyCasDiskIndexEntry, IoHash::Hasher> LegacyDiskIndex;
 
 			TCasLogFile<LegacyCasDiskIndexEntry> LegacyCasLog;
-			LegacyCasLog.Open(SlogPath, false);
+			LegacyCasLog.Open(LegacyLogPath, false);
 			LegacyCasLog.Replay([&](const LegacyCasDiskIndexEntry& Record) {
 				if (Record.Flags & LegacyCasDiskIndexEntry::kTombstone)
 				{
@@ -989,6 +1051,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 				return;
 			}
 
+			CreateDirectories(SlogPath.parent_path());
 			m_CasLog.Open(SlogPath, true);
 
 			std::vector<CasDiskIndexEntry> LogEntries;
@@ -1004,7 +1067,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 										  .ContentType = Record.ContentType,
 										  .Flags	   = Record.Flags});
 				}
-				auto BlockPath = BuildUcasPath(m_BlocksBasePath, 0);
+				auto BlockPath = GetBlockPath(m_BlocksBasePath, 0);
 				CreateDirectories(BlockPath.parent_path());
 				BlockFile.Close();
 				std::filesystem::rename(LegacySobsPath, BlockPath);
@@ -1037,7 +1100,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 					BlockFile.Read(Chunk.data(), Chunk.size(), ChunkLocation.GetOffset());
 					if (!NewBlockFile)
 					{
-						auto BlockPath = BuildUcasPath(m_BlocksBasePath, NewBlockIndex);
+						auto BlockPath = GetBlockPath(m_BlocksBasePath, NewBlockIndex);
 						NewBlockFile   = std::make_unique<BlockStoreFile>(BlockPath);
 						NewBlockFile->Create(m_MaxBlockSize);
 					}
@@ -1055,7 +1118,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 								 MaxRequiredBlockCount,
 								 NiceBytes(ChunkEnd),
 								 NiceBytes(TotalSize));
-						auto BlockPath = BuildUcasPath(m_BlocksBasePath, NewBlockIndex);
+						auto BlockPath = GetBlockPath(m_BlocksBasePath, NewBlockIndex);
 						NewBlockFile   = std::make_unique<BlockStoreFile>(BlockPath);
 						NewBlockFile->Create(m_MaxBlockSize);
 						WriteOffset = 0;
@@ -1077,7 +1140,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 			}
 			m_CasLog.Close();
 
-			std::filesystem::remove(LegacySobsPath);
+			std::filesystem::remove(LegacyLogPath);
 			CasLogEmpty = false;
 		}
 
@@ -1145,7 +1208,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 				if (Entry.is_regular_file())
 				{
 					const std::filesystem::path Path = Entry.path();
-					if (Path.extension() != ".ucas")
+					if (Path.extension() != DataExtension)
 					{
 						continue;
 					}
@@ -1167,7 +1230,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 						std::filesystem::remove(Path);
 						continue;
 					}
-					auto BlockPath			  = BuildUcasPath(m_BlocksBasePath, BlockIndex);
+					auto BlockPath			  = GetBlockPath(m_BlocksBasePath, BlockIndex);
 					auto BlockFile			  = std::make_shared<BlockStoreFile>(BlockPath);
 					m_ChunkBlocks[BlockIndex] = BlockFile;
 				}
@@ -1177,7 +1240,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 	}
 
 	// Create GC reserve file if possible
-	std::filesystem::path GCReservePath = m_Config.RootDirectory / (m_ContainerBaseName + ".gc.reserve.ucas");
+	std::filesystem::path GCReservePath = GetGCReservePath(m_Config.RootDirectory, m_ContainerBaseName);
 
 	std::error_code Error;
 	DiskSpace		Space = DiskSpaceInfo(m_Config.RootDirectory, Error);
@@ -1217,6 +1280,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 	{
 		if (Space.Free > m_MaxBlockSize)
 		{
+			CreateDirectories(GCReservePath.parent_path());
 			GCReserveFile.Open(GCReservePath, true);
 			GCReserveFile.SetFileSize(m_MaxBlockSize);
 		}
@@ -1865,12 +1929,12 @@ TEST_CASE("compactcas.legacyconversion")
 		Gc.CollectGarbage(GcCtx);
 	}
 
-	auto CasPath	   = BuildUcasPath(CasConfig.RootDirectory / "test" / "blocks", 1);
-	auto LegacyCasPath = CasConfig.RootDirectory / "test.ucas";
+	auto CasPath	   = GetBlockPath(GetBlocksBasePath(CasConfig.RootDirectory, "test"), 1);
+	auto LegacyCasPath = GetLegacyUcasPath(CasConfig.RootDirectory, "test");
 	std::filesystem::rename(CasPath, LegacyCasPath);
 
 	std::vector<CasDiskIndexEntry> LogEntries;
-	std::filesystem::path		   SidxPath = CasConfig.RootDirectory / ("test.uidx");
+	std::filesystem::path		   SidxPath = GetIndexPath(CasConfig.RootDirectory, "test");
 	if (std::filesystem::is_regular_file(SidxPath))
 	{
 		BasicFile ObjectIndexFile;
@@ -1892,14 +1956,15 @@ TEST_CASE("compactcas.legacyconversion")
 		std::filesystem::remove(SidxPath);
 	}
 
-	std::filesystem::path SlogPath = CasConfig.RootDirectory / "test.ulog";
+	std::filesystem::path SlogPath = GetLogPath(CasConfig.RootDirectory, "test");
 	{
 		TCasLogFile<CasDiskIndexEntry> CasLog;
 		CasLog.Open(SlogPath, false);
 		CasLog.Replay([&](const CasDiskIndexEntry& Record) { LogEntries.push_back(Record); });
 	}
 	TCasLogFile<LegacyCasDiskIndexEntry> LegacyCasLog;
-	LegacyCasLog.Open(SlogPath, true);
+	std::filesystem::path				 SLegacylogPath = GetLegacyLogPath(CasConfig.RootDirectory, "test");
+	LegacyCasLog.Open(SLegacylogPath, true);
 	for (const CasDiskIndexEntry& Entry : LogEntries)
 	{
 		BlockStoreLocation		Location = Entry.Location.Get(16);
@@ -1912,6 +1977,8 @@ TEST_CASE("compactcas.legacyconversion")
 	}
 	LegacyCasLog.Close();
 
+	std::filesystem::remove_all(CasConfig.RootDirectory / "test");
+
 	{
 		CasGc				 Gc;
 		CasContainerStrategy Cas(CasConfig, Gc);
@@ -2117,7 +2184,7 @@ TEST_CASE("compactcas.threadedinsert")	// * doctest::skip(true))
 	}
 }
 
-TEST_CASE("compactcas.migrate.large.data" * doctest::skip(false))
+TEST_CASE("compactcas.migrate.large.data" * doctest::skip(true))
 {
 	const char*			  BigDataPath = "D:\\zen-data\\dc4-zen-cache-t\\cas";
 	CasStoreConfiguration CasConfig;