From 8269e0616cf4333fd1007ccd8a7b1dac09743e11 Mon Sep 17 00:00:00 2001
From: Dan Engelbrecht <dan@engelbrecht.com>
Date: Mon, 4 Dec 2023 08:37:05 -0500
Subject: reserve vectors in gcv2 upfront / load factor for robin_map (#582)

* reserve vectors in gcv2 upfront
* set max load factor for robin_map indexes to reduce memory usage
* set min load factor for robin_map indexes to allow them to shrink
---
 src/zenstore/compactcas.cpp | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

(limited to 'src/zenstore/compactcas.cpp')

diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 5de82f219..42302c4a9 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -25,6 +25,9 @@
 #	include <zenstore/cidstore.h>
 #	include <algorithm>
 #	include <random>
+ZEN_THIRD_PARTY_INCLUDES_START
+#	include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
 #endif
 
 //////////////////////////////////////////////////////////////////////////
@@ -114,8 +117,14 @@ namespace {
 
 //////////////////////////////////////////////////////////////////////////
 
+static const float IndexMinLoadFactor = 0.2f;
+static const float IndexMaxLoadFactor = 0.7f;
+
 CasContainerStrategy::CasContainerStrategy(GcManager& Gc) : m_Log(logging::Get("containercas")), m_Gc(Gc)
 {
+	m_LocationMap.min_load_factor(IndexMinLoadFactor);
+	m_LocationMap.max_load_factor(IndexMaxLoadFactor);
+
 	m_Gc.AddGcStorage(this);
 	m_Gc.AddGcReferenceStore(*this);
 }
@@ -576,7 +585,7 @@ public:
 
 		if (Ctx.Settings.CollectSmallObjects)
 		{
-			std::unordered_map<uint32_t, uint64_t> BlockUsage;
+			BlockStore::BlockUsageMap BlockUsage;
 			{
 				RwLock::SharedLockScope __(m_CasContainerStrategy.m_LocationMapLock);
 				if (Ctx.IsCancelledFlag.load())
@@ -591,14 +600,14 @@ public:
 
 					uint32_t BlockIndex = Loc.GetBlockIndex();
 					uint64_t ChunkSize	= RoundUp(Loc.GetSize(), m_CasContainerStrategy.m_PayloadAlignment);
-					auto	 It			= BlockUsage.find(BlockIndex);
-					if (It == BlockUsage.end())
+					if (auto It = BlockUsage.find(BlockIndex); It != BlockUsage.end())
 					{
-						BlockUsage.insert_or_assign(BlockIndex, ChunkSize);
+						It->second.EntryCount++;
+						It->second.DiskUsage += ChunkSize;
 					}
 					else
 					{
-						It->second += ChunkSize;
+						BlockUsage.insert_or_assign(BlockIndex, BlockStore::BlockUsageInfo{.DiskUsage = ChunkSize, .EntryCount = 1});
 					}
 				}
 			}
@@ -607,7 +616,7 @@ public:
 				BlockStoreCompactState BlockCompactState;
 				std::vector<IoHash>	   BlockCompactStateKeys;
 
-				std::vector<uint32_t> BlocksToCompact =
+				BlockStore::BlockEntryCountMap BlocksToCompact =
 					m_CasContainerStrategy.m_BlockStore.GetBlocksToCompact(BlockUsage, Ctx.Settings.CompactBlockUsageThresholdPercent);
 				BlockCompactState.IncludeBlocks(BlocksToCompact);
 
-- 
cgit v1.2.3