diff options
| author | Dan Engelbrecht <[email protected]> | 2023-12-04 08:37:05 -0500 |
|---|---|---|
| committer | GitHub <[email protected]> | 2023-12-04 14:37:05 +0100 |
| commit | 8269e0616cf4333fd1007ccd8a7b1dac09743e11 (patch) | |
| tree | 7f2e974acb09b66cd556ea0d43c62281aa73941b /src/zenstore/compactcas.cpp | |
| parent | memory usage estimation for memcached entries (#586) (diff) | |
| download | zen-8269e0616cf4333fd1007ccd8a7b1dac09743e11.tar.xz zen-8269e0616cf4333fd1007ccd8a7b1dac09743e11.zip | |
reserve vectors in gcv2 upfront / load factor for robin_map (#582)
* reserve vectors in gcv2 upfront
* set max load factor for robin_map indexes to reduce memory usage
* set min load factor for robin_map indexes to allow them to shrink
Diffstat (limited to 'src/zenstore/compactcas.cpp')
| -rw-r--r-- | src/zenstore/compactcas.cpp | 21 |
1 files changed, 15 insertions, 6 deletions
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 5de82f219..42302c4a9 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -25,6 +25,9 @@ # include <zenstore/cidstore.h> # include <algorithm> # include <random> +ZEN_THIRD_PARTY_INCLUDES_START +# include <tsl/robin_map.h> +ZEN_THIRD_PARTY_INCLUDES_END #endif ////////////////////////////////////////////////////////////////////////// @@ -114,8 +117,14 @@ namespace { ////////////////////////////////////////////////////////////////////////// +static const float IndexMinLoadFactor = 0.2f; +static const float IndexMaxLoadFactor = 0.7f; + CasContainerStrategy::CasContainerStrategy(GcManager& Gc) : m_Log(logging::Get("containercas")), m_Gc(Gc) { + m_LocationMap.min_load_factor(IndexMinLoadFactor); + m_LocationMap.max_load_factor(IndexMaxLoadFactor); + m_Gc.AddGcStorage(this); m_Gc.AddGcReferenceStore(*this); } @@ -576,7 +585,7 @@ public: if (Ctx.Settings.CollectSmallObjects) { - std::unordered_map<uint32_t, uint64_t> BlockUsage; + BlockStore::BlockUsageMap BlockUsage; { RwLock::SharedLockScope __(m_CasContainerStrategy.m_LocationMapLock); if (Ctx.IsCancelledFlag.load()) @@ -591,14 +600,14 @@ public: uint32_t BlockIndex = Loc.GetBlockIndex(); uint64_t ChunkSize = RoundUp(Loc.GetSize(), m_CasContainerStrategy.m_PayloadAlignment); - auto It = BlockUsage.find(BlockIndex); - if (It == BlockUsage.end()) + if (auto It = BlockUsage.find(BlockIndex); It != BlockUsage.end()) { - BlockUsage.insert_or_assign(BlockIndex, ChunkSize); + It->second.EntryCount++; + It->second.DiskUsage += ChunkSize; } else { - It->second += ChunkSize; + BlockUsage.insert_or_assign(BlockIndex, BlockStore::BlockUsageInfo{.DiskUsage = ChunkSize, .EntryCount = 1}); } } } @@ -607,7 +616,7 @@ public: BlockStoreCompactState BlockCompactState; std::vector<IoHash> BlockCompactStateKeys; - std::vector<uint32_t> BlocksToCompact = + BlockStore::BlockEntryCountMap BlocksToCompact = m_CasContainerStrategy.m_BlockStore.GetBlocksToCompact(BlockUsage, Ctx.Settings.CompactBlockUsageThresholdPercent); BlockCompactState.IncludeBlocks(BlocksToCompact); |