From d020b6522b2d962db67f8a66410e74d61cf3da24 Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan.engelbrecht@epicgames.com>
Date: Tue, 17 Sep 2024 15:05:40 +0200
Subject: gc performance improvements (#160)

* optimized ValidateCbUInt
* optimized iohash comparision
* replace unordered set/map with tsl/robin set/map in blockstore
* increase max buffer size when writing cache bucket sidecar
* only store meta data for files < 4Gb
* faster ReadAttachmentsFromMetaData
* remove memcpy call in BlockStoreDiskLocation
* only write cache bucket state to disk if GC deleted anything
---
 src/zenstore/compactcas.cpp | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

(limited to 'src/zenstore/compactcas.cpp')

diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 1e647b284..15f80d4cf 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -28,7 +28,7 @@
 #	include <algorithm>
 #	include <random>
 ZEN_THIRD_PARTY_INCLUDES_START
-#	include <tsl/robin_map.h>
+#	include <tsl/robin_set.h>
 ZEN_THIRD_PARTY_INCLUDES_END
 #endif
 
@@ -728,10 +728,11 @@ public:
 
 					uint32_t BlockIndex = Loc.GetBlockIndex();
 					uint64_t ChunkSize	= RoundUp(Loc.GetSize(), m_CasContainerStrategy.m_PayloadAlignment);
-					if (auto It = BlockUsage.find(BlockIndex); It != BlockUsage.end())
+					if (BlockStore::BlockUsageMap::iterator It = BlockUsage.find(BlockIndex); It != BlockUsage.end())
 					{
-						It->second.EntryCount++;
-						It->second.DiskUsage += ChunkSize;
+						BlockStore::BlockUsageInfo& Info = It.value();
+						Info.EntryCount++;
+						Info.DiskUsage += ChunkSize;
 					}
 					else
 					{
@@ -1869,7 +1870,7 @@ TEST_CASE_TEMPLATE("compactcas.threadedinsert", GCV2, FalseType, TrueType)
 		const int32_t  kChunkCount	= 4096;
 		uint64_t	   ExpectedSize = 0;
 
-		std::unordered_map<IoHash, IoBuffer, IoHash::Hasher> Chunks;
+		tsl::robin_map<IoHash, IoBuffer, IoHash::Hasher> Chunks;
 		Chunks.reserve(kChunkCount);
 
 		for (int32_t Idx = 0; Idx < kChunkCount; ++Idx)
@@ -1932,7 +1933,7 @@ TEST_CASE_TEMPLATE("compactcas.threadedinsert", GCV2, FalseType, TrueType)
 			}
 		}
 
-		std::unordered_set<IoHash, IoHash::Hasher> GcChunkHashes;
+		tsl::robin_set<IoHash, IoHash::Hasher> GcChunkHashes;
 		GcChunkHashes.reserve(Chunks.size());
 		for (const auto& Chunk : Chunks)
 		{
@@ -1940,7 +1941,7 @@ TEST_CASE_TEMPLATE("compactcas.threadedinsert", GCV2, FalseType, TrueType)
 		}
 		{
 			WorkCompleted = 0;
-			std::unordered_map<IoHash, IoBuffer, IoHash::Hasher> NewChunks;
+			tsl::robin_map<IoHash, IoBuffer, IoHash::Hasher> NewChunks;
 			NewChunks.reserve(kChunkCount);
 
 			for (int32_t Idx = 0; Idx < kChunkCount; ++Idx)
@@ -1974,8 +1975,8 @@ TEST_CASE_TEMPLATE("compactcas.threadedinsert", GCV2, FalseType, TrueType)
 				});
 			}
 
-			std::unordered_set<IoHash, IoHash::Hasher> ChunksToDelete;
-			std::vector<IoHash>						   KeepHashes(GcChunkHashes.begin(), GcChunkHashes.end());
+			tsl::robin_set<IoHash, IoHash::Hasher> ChunksToDelete;
+			std::vector<IoHash>					   KeepHashes(GcChunkHashes.begin(), GcChunkHashes.end());
 
 			size_t C = 0;
 			while (C < KeepHashes.size())
@@ -1998,10 +1999,10 @@ TEST_CASE_TEMPLATE("compactcas.threadedinsert", GCV2, FalseType, TrueType)
 				C++;
 			}
 
-			auto DoGC = [](CasContainerStrategy&							 Cas,
-						   const std::unordered_set<IoHash, IoHash::Hasher>& ChunksToDelete,
-						   const std::vector<IoHash>&						 KeepHashes,
-						   std::unordered_set<IoHash, IoHash::Hasher>&		 GcChunkHashes) {
+			auto DoGC = [](CasContainerStrategy&						 Cas,
+						   const tsl::robin_set<IoHash, IoHash::Hasher>& ChunksToDelete,
+						   const std::vector<IoHash>&					 KeepHashes,
+						   tsl::robin_set<IoHash, IoHash::Hasher>&		 GcChunkHashes) {
 				if (GCV2::Enabled)
 				{
 					std::atomic_bool	  IsCancelledFlag = false;
-- 
cgit v1.2.3