Encapsulated containers in CasChunkSet to allow for more efficient udpates/queries

1 files changed, 21 insertions, 38 deletions

diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 1fcae6d02..c036efd35 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -35,23 +35,8 @@ FileCasStrategy::ShardingHelper::ShardingHelper(const std::filesystem::path& Roo
 {
 	ShardedPath.Append(RootPath.c_str());
 	ShardedPath.Append(std::filesystem::path::preferred_separator);
-	MakeShardedPath(ShardedPath, ChunkHash, /* out */ Shard2len);
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-FileCasStrategy::FileCasStrategy(const CasStoreConfiguration& Config) : m_Config(Config), m_Log(logging::Get("filecas"))
-{
-}
-
-FileCasStrategy::~FileCasStrategy()
-{
-}
 
-WideStringBuilderBase&
-FileCasStrategy::MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHash& ChunkHash, size_t& OutShard2len)
-{
-	ExtendableStringBuilder<96> HashString;
+	ExtendableStringBuilder<64> HashString;
 	ChunkHash.ToHexString(HashString);
 
 	const char* str = HashString.c_str();
@@ -62,20 +47,31 @@ FileCasStrategy::MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHas
 	// This results in a maximum of 4096 * 256 directories
 	//
 	// The numbers have been chosen somewhat arbitrarily but are large to scale
-	// to very large chunk repositories. It may or may not make sense to make
-	// this a configurable policy, and it would probably be a good idea to
-	// measure performance for different policies and chunk counts
+	// to very large chunk repositories without creating too many directories
+	// on a single level since NTFS does not deal very well with this.
+	//
+	// It may or may not make sense to make this a configurable policy, and it
+	// would probably be a good idea to measure performance for different
+	// policies and chunk counts
 
 	ShardedPath.AppendAsciiRange(str, str + 3);
 
-	ShardedPath.Append('\\');
+	ShardedPath.Append(std::filesystem::path::preferred_separator);
 	ShardedPath.AppendAsciiRange(str + 3, str + 5);
-	OutShard2len = ShardedPath.Size();
+	Shard2len = ShardedPath.Size();
 
-	ShardedPath.Append('\\');
+	ShardedPath.Append(std::filesystem::path::preferred_separator);
 	ShardedPath.AppendAsciiRange(str + 5, str + 40);
+}
 
-	return ShardedPath;
+//////////////////////////////////////////////////////////////////////////
+
+FileCasStrategy::FileCasStrategy(const CasStoreConfiguration& Config) : m_Config(Config), m_Log(logging::Get("filecas"))
+{
+}
+
+FileCasStrategy::~FileCasStrategy()
+{
 }
 
 CasStore::InsertResult
@@ -320,20 +316,7 @@ FileCasStrategy::FilterChunks(CasChunkSet& InOutChunks)
 	// a caller, this is something which needs to be taken into account by anyone consuming
 	// this functionality in any case
 
-	std::unordered_set<IoHash> HaveSet;
-
-	for (const IoHash& Hash : InOutChunks.GetChunkSet())
-	{
-		if (HaveChunk(Hash))
-		{
-			HaveSet.insert(Hash);
-		}
-	}
-
-	for (const IoHash& Hash : HaveSet)
-	{
-		InOutChunks.RemoveIfPresent(Hash);
-	}
+	InOutChunks.RemoveChunksIf([&](const IoHash& Hash) { return HaveChunk(Hash); });
 }
 
 void
@@ -429,7 +412,7 @@ FileCasStrategy::Scrub(ScrubContext& Ctx)
 
 		if (Ctx.RunRecovery())
 		{
-			ZEN_WARN("recovery: deleting backing files for {} bad chunks which were found to be bad", BadHashes.size());
+			ZEN_WARN("recovery: deleting backing files for {} bad chunks which were identified as bad", BadHashes.size());
 
 			for (const IoHash& Hash : BadHashes)
 			{