Merge branch 'main' into zs/get-all-chunk-infoszs/get-all-chunk-infos

1 files changed, 18 insertions, 7 deletions

diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 5de82f219..96ab65a5f 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -25,6 +25,9 @@
 #	include <zenstore/cidstore.h>
 #	include <algorithm>
 #	include <random>
+ZEN_THIRD_PARTY_INCLUDES_START
+#	include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
 #endif
 
 //////////////////////////////////////////////////////////////////////////
@@ -114,8 +117,14 @@ namespace {
 
 //////////////////////////////////////////////////////////////////////////
 
+static const float IndexMinLoadFactor = 0.2f;
+static const float IndexMaxLoadFactor = 0.7f;
+
 CasContainerStrategy::CasContainerStrategy(GcManager& Gc) : m_Log(logging::Get("containercas")), m_Gc(Gc)
 {
+	m_LocationMap.min_load_factor(IndexMinLoadFactor);
+	m_LocationMap.max_load_factor(IndexMaxLoadFactor);
+
 	m_Gc.AddGcStorage(this);
 	m_Gc.AddGcReferenceStore(*this);
 }
@@ -576,7 +585,7 @@ public:
 
 		if (Ctx.Settings.CollectSmallObjects)
 		{
-			std::unordered_map<uint32_t, uint64_t> BlockUsage;
+			BlockStore::BlockUsageMap BlockUsage;
 			{
 				RwLock::SharedLockScope __(m_CasContainerStrategy.m_LocationMapLock);
 				if (Ctx.IsCancelledFlag.load())
@@ -591,14 +600,14 @@ public:
 
 					uint32_t BlockIndex = Loc.GetBlockIndex();
 					uint64_t ChunkSize	= RoundUp(Loc.GetSize(), m_CasContainerStrategy.m_PayloadAlignment);
-					auto	 It			= BlockUsage.find(BlockIndex);
-					if (It == BlockUsage.end())
+					if (auto It = BlockUsage.find(BlockIndex); It != BlockUsage.end())
 					{
-						BlockUsage.insert_or_assign(BlockIndex, ChunkSize);
+						It->second.EntryCount++;
+						It->second.DiskUsage += ChunkSize;
 					}
 					else
 					{
-						It->second += ChunkSize;
+						BlockUsage.insert_or_assign(BlockIndex, BlockStore::BlockUsageInfo{.DiskUsage = ChunkSize, .EntryCount = 1});
 					}
 				}
 			}
@@ -607,7 +616,7 @@ public:
 				BlockStoreCompactState BlockCompactState;
 				std::vector<IoHash>	   BlockCompactStateKeys;
 
-				std::vector<uint32_t> BlocksToCompact =
+				BlockStore::BlockEntryCountMap BlocksToCompact =
 					m_CasContainerStrategy.m_BlockStore.GetBlocksToCompact(BlockUsage, Ctx.Settings.CompactBlockUsageThresholdPercent);
 				BlockCompactState.IncludeBlocks(BlocksToCompact);
 
@@ -980,13 +989,14 @@ CasContainerStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint
 			Entries.resize(128 * 1024 / sizeof(CasDiskIndexEntry));
 
 			uint64_t RemainingEntries = Header.EntryCount;
+			uint64_t ReadOffset		  = sizeof(CasDiskIndexHeader);
 
 			do
 			{
 				const uint64_t NumToRead = Min(RemainingEntries, Entries.size());
 				Entries.resize(NumToRead);
 
-				ObjectIndexFile.Read(Entries.data(), Entries.size() * sizeof(CasDiskIndexEntry), sizeof(CasDiskIndexHeader));
+				ObjectIndexFile.Read(Entries.data(), Entries.size() * sizeof(CasDiskIndexEntry), ReadOffset);
 
 				std::string InvalidEntryReason;
 				for (const CasDiskIndexEntry& Entry : Entries)
@@ -1002,6 +1012,7 @@ CasContainerStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint
 				}
 
 				RemainingEntries -= NumToRead;
+				ReadOffset += NumToRead * sizeof(CasDiskIndexEntry);
 			} while (RemainingEntries);
 
 			OutVersion = CasDiskIndexHeader::CurrentVersion;