Merge branch 'main' of https://github.com/EpicGames/zen

8 files changed, 80 insertions, 71 deletions

diff --git a/README.md b/README.md
index 7d51d3686..055ecca3a 100644
--- a/README.md
+++ b/README.md
@@ -3,6 +3,9 @@
 This is the implementation of the local storage service for UE5. It is intended to be deployed on
 user machines either as a daemon or launched ad hoc as required during  of editor/cooker/game startup
 
+Zen can also be deployed as a shared instance for use as a shared cache. It also supports upstream 
+connectivity to cloud storage services as well as other Zen server instances.
+
 We currently only support building and running the server on Windows. Linux and Mac support is in progress
 
 ## Building on Windows
diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp
index 3197eaee4..15d9e0141 100644
--- a/zenserver/compute/apply.cpp
+++ b/zenserver/compute/apply.cpp
@@ -375,7 +375,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
 
 									FunctionSpec.IterateAttachments([&](CbFieldView Field) {
 										const IoHash Hash = Field.AsHash();
-										ChunkSet.AddChunk(Hash);
+										ChunkSet.AddChunkToSet(Hash);
 									});
 
 									// Note that we store executables uncompressed to make it
@@ -399,16 +399,15 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
 										CbObjectWriter ResponseWriter;
 										ResponseWriter.BeginArray("need");
 
-										for (const IoHash& Hash : ChunkSet.GetChunkSet())
-										{
+										ChunkSet.IterateChunks([&](const IoHash& Hash) {
 											ZEN_DEBUG("worker {}: need chunk {}", WorkerId, Hash);
 
 											ResponseWriter.AddHash(Hash);
-										}
+										});
 
 										ResponseWriter.EndArray();
 
-										ZEN_DEBUG("worker {}: need {} attachments", WorkerId, ChunkSet.GetChunkSet().size());
+										ZEN_DEBUG("worker {}: need {} attachments", WorkerId, ChunkSet.GetSize());
 
 										return HttpReq.WriteResponse(HttpResponseCode::NotFound, ResponseWriter.Save());
 									}
diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp
index a3837d159..eaf72cb41 100644
--- a/zenstore/CAS.cpp
+++ b/zenstore/CAS.cpp
@@ -26,6 +26,39 @@
 namespace zen {
 
 void
+CasChunkSet::AddChunkToSet(const IoHash& HashToAdd)
+{
+	m_ChunkSet.insert(HashToAdd);
+}
+
+void
+CasChunkSet::RemoveChunksIf(std::function<bool(const IoHash& CandidateHash)>&& Predicate)
+{
+	for (auto It = begin(m_ChunkSet), ItEnd = end(m_ChunkSet); It != ItEnd;)
+	{
+		if (Predicate(*It))
+		{
+			It = m_ChunkSet.erase(It);
+		}
+		else
+		{
+			++It;
+		}
+	}
+}
+
+void
+CasChunkSet::IterateChunks(std::function<void(const IoHash& ChunkHash)>&& Callback)
+{
+	for (auto It = begin(m_ChunkSet), ItEnd = end(m_ChunkSet); It != ItEnd;)
+	{
+		Callback(*It);
+	}
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+void
 ScrubContext::ReportBadChunks(std::span<IoHash> BadChunks)
 {
 	ZEN_UNUSED(BadChunks);
@@ -226,11 +259,11 @@ TEST_CASE("CasStore")
 	CHECK(Result2.New);
 
 	CasChunkSet ChunkSet;
-	ChunkSet.AddChunk(Hash1);
-	ChunkSet.AddChunk(Hash2);
+	ChunkSet.AddChunkToSet(Hash1);
+	ChunkSet.AddChunkToSet(Hash2);
 
 	Store->FilterChunks(ChunkSet);
-	CHECK(ChunkSet.GetChunkSet().size() == 0);
+	CHECK(ChunkSet.IsEmpty());
 
 	IoBuffer Lookup1 = Store->FindChunk(Hash1);
 	CHECK(Lookup1);
diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp
index 9c8d4742c..08a3192ff 100644
--- a/zenstore/cidstore.cpp
+++ b/zenstore/cidstore.cpp
@@ -139,7 +139,7 @@ struct CidStore::Impl
 
 			for (auto& Kv : m_CidMap)
 			{
-				ChunkSet.AddChunk(Kv.second);
+				ChunkSet.AddChunkToSet(Kv.second);
 			}
 		}
 
@@ -152,21 +152,21 @@ struct CidStore::Impl
 		}
 
 		ZEN_ERROR("Scrubbing found that {} cid mappings (out of {}) mapped to non-existent CAS chunks. These mappings will be removed",
-				  ChunkSet.GetChunkSet().size(),
+				  ChunkSet.GetSize(),
 				  m_CidMap.size());
 
 		// Erase all mappings to chunks which are not present in the underlying CAS store
 		// we do this by removing mappings from the in-memory lookup structure and also
 		// by emitting tombstone records to the commit log
 
-		const auto&			MissingChunks = ChunkSet.GetChunkSet();
 		std::vector<IoHash> BadChunks;
+
 		{
 			RwLock::SharedLockScope _(m_Lock);
 
 			for (auto It = begin(m_CidMap), ItEnd = end(m_CidMap); It != ItEnd;)
 			{
-				if (auto MissingIt = MissingChunks.find(It->second); MissingIt != MissingChunks.end())
+				if (ChunkSet.ContainsChunk(It->second))
 				{
 					const IoHash& BadHash = It->first;
 
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index f17b8d376..5fc3ac356 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -147,20 +147,7 @@ CasContainerStrategy::FilterChunks(CasChunkSet& InOutChunks)
 	// we're likely to already have a large proportion of the
 	// chunks in the set
 
-	std::unordered_set<IoHash> HaveSet;
-
-	for (const IoHash& Hash : InOutChunks.GetChunkSet())
-	{
-		if (HaveChunk(Hash))
-		{
-			HaveSet.insert(Hash);
-		}
-	}
-
-	for (const IoHash& Hash : HaveSet)
-	{
-		InOutChunks.RemoveIfPresent(Hash);
-	}
+	InOutChunks.RemoveChunksIf([&](const IoHash& Hash) { return HaveChunk(Hash); });
 }
 
 void
@@ -213,7 +200,8 @@ CasContainerStrategy::Scrub(ScrubContext& Ctx)
 						continue;
 					}
 
-					const IoHash ComputedHash = IoHash::HashBuffer(reinterpret_cast<uint8_t*>(BufferBase) + Entry.second.Offset - WindowStart, Entry.second.Size);
+					const IoHash ComputedHash =
+						IoHash::HashBuffer(reinterpret_cast<uint8_t*>(BufferBase) + Entry.second.Offset - WindowStart, Entry.second.Size);
 
 					if (Entry.first != ComputedHash)
 					{
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 1fcae6d02..c036efd35 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -35,23 +35,8 @@ FileCasStrategy::ShardingHelper::ShardingHelper(const std::filesystem::path& Roo
 {
 	ShardedPath.Append(RootPath.c_str());
 	ShardedPath.Append(std::filesystem::path::preferred_separator);
-	MakeShardedPath(ShardedPath, ChunkHash, /* out */ Shard2len);
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-FileCasStrategy::FileCasStrategy(const CasStoreConfiguration& Config) : m_Config(Config), m_Log(logging::Get("filecas"))
-{
-}
-
-FileCasStrategy::~FileCasStrategy()
-{
-}
 
-WideStringBuilderBase&
-FileCasStrategy::MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHash& ChunkHash, size_t& OutShard2len)
-{
-	ExtendableStringBuilder<96> HashString;
+	ExtendableStringBuilder<64> HashString;
 	ChunkHash.ToHexString(HashString);
 
 	const char* str = HashString.c_str();
@@ -62,20 +47,31 @@ FileCasStrategy::MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHas
 	// This results in a maximum of 4096 * 256 directories
 	//
 	// The numbers have been chosen somewhat arbitrarily but are large to scale
-	// to very large chunk repositories. It may or may not make sense to make
-	// this a configurable policy, and it would probably be a good idea to
-	// measure performance for different policies and chunk counts
+	// to very large chunk repositories without creating too many directories
+	// on a single level since NTFS does not deal very well with this.
+	//
+	// It may or may not make sense to make this a configurable policy, and it
+	// would probably be a good idea to measure performance for different
+	// policies and chunk counts
 
 	ShardedPath.AppendAsciiRange(str, str + 3);
 
-	ShardedPath.Append('\\');
+	ShardedPath.Append(std::filesystem::path::preferred_separator);
 	ShardedPath.AppendAsciiRange(str + 3, str + 5);
-	OutShard2len = ShardedPath.Size();
+	Shard2len = ShardedPath.Size();
 
-	ShardedPath.Append('\\');
+	ShardedPath.Append(std::filesystem::path::preferred_separator);
 	ShardedPath.AppendAsciiRange(str + 5, str + 40);
+}
 
-	return ShardedPath;
+//////////////////////////////////////////////////////////////////////////
+
+FileCasStrategy::FileCasStrategy(const CasStoreConfiguration& Config) : m_Config(Config), m_Log(logging::Get("filecas"))
+{
+}
+
+FileCasStrategy::~FileCasStrategy()
+{
 }
 
 CasStore::InsertResult
@@ -320,20 +316,7 @@ FileCasStrategy::FilterChunks(CasChunkSet& InOutChunks)
 	// a caller, this is something which needs to be taken into account by anyone consuming
 	// this functionality in any case
 
-	std::unordered_set<IoHash> HaveSet;
-
-	for (const IoHash& Hash : InOutChunks.GetChunkSet())
-	{
-		if (HaveChunk(Hash))
-		{
-			HaveSet.insert(Hash);
-		}
-	}
-
-	for (const IoHash& Hash : HaveSet)
-	{
-		InOutChunks.RemoveIfPresent(Hash);
-	}
+	InOutChunks.RemoveChunksIf([&](const IoHash& Hash) { return HaveChunk(Hash); });
 }
 
 void
@@ -429,7 +412,7 @@ FileCasStrategy::Scrub(ScrubContext& Ctx)
 
 		if (Ctx.RunRecovery())
 		{
-			ZEN_WARN("recovery: deleting backing files for {} bad chunks which were found to be bad", BadHashes.size());
+			ZEN_WARN("recovery: deleting backing files for {} bad chunks which were identified as bad", BadHashes.size());
 
 			for (const IoHash& Hash : BadHashes)
 			{
diff --git a/zenstore/filecas.h b/zenstore/filecas.h
index 2e09367df..51e5fbc98 100644
--- a/zenstore/filecas.h
+++ b/zenstore/filecas.h
@@ -45,7 +45,6 @@ private:
 	spdlog::logger&				 Log() { return m_Log; }
 
 	inline RwLock&				  LockForHash(const IoHash& Hash) { return m_ShardLocks[Hash.Hash[19]]; }
-	static WideStringBuilderBase& MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHash& ChunkHash, size_t& OutShard2len);
 	void						  IterateChunks(std::function<void(const IoHash& Hash, BasicFile& PayloadFile)>&& Callback);
 	void						  DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec);
 
diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/CAS.h
index ed235bb4b..93454ca6f 100644
--- a/zenstore/include/zenstore/CAS.h
+++ b/zenstore/include/zenstore/CAS.h
@@ -9,8 +9,10 @@
 #include <zencore/iohash.h>
 #include <zencore/refcount.h>
 #include <zencore/timer.h>
+
 #include <atomic>
 #include <filesystem>
+#include <functional>
 #include <memory>
 #include <string>
 #include <unordered_set>
@@ -39,7 +41,7 @@ private:
 };
 
 /** Context object for data scrubbing
- * 
+ *
  * Data scrubbing is when we traverse stored data to validate it and
  * optionally correct/recover
  */
@@ -59,10 +61,12 @@ private:
 class CasChunkSet
 {
 public:
-	void							  AddChunk(const IoHash& HashToAdd) { m_ChunkSet.insert(HashToAdd); }
-	bool							  RemoveIfPresent(const IoHash& HashToRemove) { return 0 != m_ChunkSet.erase(HashToRemove); }
-	const std::unordered_set<IoHash>& GetChunkSet() const { return m_ChunkSet; }
-	bool							  IsEmpty() const { return m_ChunkSet.empty(); }
+	void						AddChunkToSet(const IoHash& HashToAdd);
+	void						RemoveChunksIf(std::function<bool(const IoHash& CandidateHash)>&& Predicate);
+	void						IterateChunks(std::function<void(const IoHash& ChunkHash)>&& Callback);
+	inline [[nodiscard]] bool	ContainsChunk(const IoHash& Hash) const { return m_ChunkSet.find(Hash) != m_ChunkSet.end(); }
+	inline [[nodiscard]] bool	IsEmpty() const { return m_ChunkSet.empty(); }
+	inline [[nodiscard]] size_t GetSize() const { return m_ChunkSet.size(); }
 
 private:
 	std::unordered_set<IoHash> m_ChunkSet;