Wired up scrubbing to more higher level services

Also moved sharding logic for filecas into a function to redduce cut/pasta

1 files changed, 53 insertions, 30 deletions

diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 968c9f3a0..1fcae6d02 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -31,7 +31,16 @@ namespace zen {
 
 using namespace fmt::literals;
 
-FileCasStrategy::FileCasStrategy(const CasStoreConfiguration& Config) : m_Config(Config)
+FileCasStrategy::ShardingHelper::ShardingHelper(const std::filesystem::path& RootPath, const IoHash& ChunkHash)
+{
+	ShardedPath.Append(RootPath.c_str());
+	ShardedPath.Append(std::filesystem::path::preferred_separator);
+	MakeShardedPath(ShardedPath, ChunkHash, /* out */ Shard2len);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+FileCasStrategy::FileCasStrategy(const CasStoreConfiguration& Config) : m_Config(Config), m_Log(logging::Get("filecas"))
 {
 }
 
@@ -78,11 +87,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
 	IoBufferFileReference FileRef;
 	if (Chunk.IsWholeFile() && Chunk.GetFileReference(/* out */ FileRef))
 	{
-		size_t							 Shard2len = 0;
-		ExtendableWideStringBuilder<128> ShardedPath;
-		ShardedPath.Append(m_Config.RootDirectory.c_str());
-		ShardedPath.Append(std::filesystem::path::preferred_separator);
-		MakeShardedPath(ShardedPath, ChunkHash, /* out */ Shard2len);
+		ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash);
 
 		auto DeletePayloadFileOnClose = [&] {
 			// This will cause the file to be deleted when the last handle to it is closed
@@ -105,7 +110,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
 		{
 			CAtlFile PayloadFile;
 
-			if (HRESULT hRes = PayloadFile.Create(ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); SUCCEEDED(hRes))
+			if (HRESULT hRes = PayloadFile.Create(Name.ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); SUCCEEDED(hRes))
 			{
 				// If we succeeded in opening the target file then we don't need to do anything else because it already exists
 				// and should contain the content we were about to insert
@@ -118,7 +123,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
 			}
 		}
 
-		std::filesystem::path FullPath(ShardedPath.c_str());
+		std::filesystem::path FullPath(Name.ShardedPath.c_str());
 
 		std::filesystem::path FilePath = FullPath.parent_path();
 		std::wstring		  FileName = FullPath.native();
@@ -194,11 +199,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash)
 CasStore::InsertResult
 FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize, const IoHash& ChunkHash)
 {
-	size_t							 Shard2len = 0;
-	ExtendableWideStringBuilder<128> ShardedPath;
-	ShardedPath.Append(m_Config.RootDirectory.c_str());
-	ShardedPath.Append(std::filesystem::path::preferred_separator);
-	MakeShardedPath(ShardedPath, ChunkHash, /* out */ Shard2len);
+	ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash);
 
 	// See if file already exists
 	//
@@ -206,7 +207,7 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize
 
 	CAtlFile PayloadFile;
 
-	HRESULT hRes = PayloadFile.Create(ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);
+	HRESULT hRes = PayloadFile.Create(Name.ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);
 
 	if (SUCCEEDED(hRes))
 	{
@@ -221,7 +222,7 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize
 
 	// For now, use double-checked locking to see if someone else was first
 
-	hRes = PayloadFile.Create(ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);
+	hRes = PayloadFile.Create(Name.ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);
 
 	if (SUCCEEDED(hRes))
 	{
@@ -235,7 +236,7 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize
 		ZEN_WARN("Unexpected error code when opening shard file for read: {:#x}", uint32_t(hRes));
 	}
 
-	auto InternalCreateFile = [&] { return PayloadFile.Create(ShardedPath.c_str(), GENERIC_WRITE, FILE_SHARE_DELETE, CREATE_ALWAYS); };
+	auto InternalCreateFile = [&] { return PayloadFile.Create(Name.ShardedPath.c_str(), GENERIC_WRITE, FILE_SHARE_DELETE, CREATE_ALWAYS); };
 
 	hRes = InternalCreateFile();
 
@@ -243,14 +244,14 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize
 	{
 		// Ensure parent directories exist and retry file creation
 
-		std::filesystem::create_directories(std::wstring_view(ShardedPath.c_str(), Shard2len));
+		std::filesystem::create_directories(std::wstring_view(Name.ShardedPath.c_str(), Name.Shard2len));
 
 		hRes = InternalCreateFile();
 	}
 
 	if (FAILED(hRes))
 	{
-		ThrowSystemException(hRes, "Failed to open shard file '{}'"_format(WideToUtf8(ShardedPath)));
+		ThrowSystemException(hRes, "Failed to open shard file '{}'"_format(WideToUtf8(Name.ShardedPath)));
 	}
 
 	size_t ChunkRemain = ChunkSize;
@@ -276,36 +277,37 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize
 IoBuffer
 FileCasStrategy::FindChunk(const IoHash& ChunkHash)
 {
-	size_t							 Shard2len = 0;
-	ExtendableWideStringBuilder<128> ShardedPath;
-	ShardedPath.Append(m_Config.RootDirectory.c_str());
-	ShardedPath.Append(std::filesystem::path::preferred_separator);
-	MakeShardedPath(ShardedPath, ChunkHash, /* out */ Shard2len);
+	ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash);
 
 	RwLock::SharedLockScope _(LockForHash(ChunkHash));
 
-	return IoBufferBuilder::MakeFromFile(ShardedPath.c_str());
+	return IoBufferBuilder::MakeFromFile(Name.ShardedPath.c_str());
 }
 
 bool
 FileCasStrategy::HaveChunk(const IoHash& ChunkHash)
 {
-	size_t							 Shard2len = 0;
-	ExtendableWideStringBuilder<128> ShardedPath;
-	ShardedPath.Append(m_Config.RootDirectory.c_str());
-	ShardedPath.Append(std::filesystem::path::preferred_separator);
-	MakeShardedPath(ShardedPath, ChunkHash, /* out */ Shard2len);
+	ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash);
 
 	RwLock::SharedLockScope _(LockForHash(ChunkHash));
 
 	std::error_code Ec;
-	if (std::filesystem::exists(ShardedPath.c_str(), Ec))
+	if (std::filesystem::exists(Name.ShardedPath.c_str(), Ec))
 	{
 		return true;
 	}
 
 	return false;
 }
+void
+FileCasStrategy::DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec)
+{
+	ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash);
+
+	ZEN_DEBUG("deleting CAS payload file '{}'", WideToUtf8(Name.ShardedPath));
+
+	std::filesystem::remove(Name.ShardedPath.c_str(), Ec);
+}
 
 void
 FileCasStrategy::FilterChunks(CasChunkSet& InOutChunks)
@@ -421,6 +423,27 @@ FileCasStrategy::Scrub(ScrubContext& Ctx)
 		}
 	});
 
+	if (!BadHashes.empty())
+	{
+		ZEN_ERROR("file CAS scrubbing: {} bad chunks found", BadHashes.size());
+
+		if (Ctx.RunRecovery())
+		{
+			ZEN_WARN("recovery: deleting backing files for {} bad chunks which were found to be bad", BadHashes.size());
+
+			for (const IoHash& Hash : BadHashes)
+			{
+				std::error_code Ec;
+				DeleteChunk(Hash, Ec);
+
+				if (Ec)
+				{
+					ZEN_WARN("failed to delete file for chunk {}", Hash);
+				}
+			}
+		}
+	}
+
 	Ctx.ReportBadChunks(BadHashes);
 }