aboutsummaryrefslogtreecommitdiff
path: root/src/zenstore/cache
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2025-03-06 17:27:59 +0100
committerStefan Boberg <[email protected]>2025-03-06 17:27:59 +0100
commit66e5d1f4e288e0c32f854ebe3b63584b42b83554 (patch)
treed67e9d358419b5baccd429d54988414e0d7cd7a6 /src/zenstore/cache
parentreduced memory churn using fixed_xxx containers (#236) (diff)
downloadzen-66e5d1f4e288e0c32f854ebe3b63584b42b83554.tar.xz
zen-66e5d1f4e288e0c32f854ebe3b63584b42b83554.zip
switched std::vector -> eastl::vector
Diffstat (limited to 'src/zenstore/cache')
-rw-r--r--src/zenstore/cache/cachedisklayer.cpp311
-rw-r--r--src/zenstore/cache/cacherpc.cpp104
-rw-r--r--src/zenstore/cache/structuredcachestore.cpp134
3 files changed, 276 insertions, 273 deletions
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 61552fafc..2f648bc2d 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -262,36 +262,36 @@ public:
return OutVersion == CurrentDiskBucketVersion;
}
- void ParseManifest(RwLock::ExclusiveLockScope& BucketLock,
- ZenCacheDiskLayer::CacheBucket& Bucket,
- std::filesystem::path ManifestPath,
- ZenCacheDiskLayer::CacheBucket::IndexMap& Index,
- std::vector<AccessTime>& AccessTimes,
- std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads);
+ void ParseManifest(RwLock::ExclusiveLockScope& BucketLock,
+ ZenCacheDiskLayer::CacheBucket& Bucket,
+ std::filesystem::path ManifestPath,
+ ZenCacheDiskLayer::CacheBucket::IndexMap& Index,
+ eastl::vector<AccessTime>& AccessTimes,
+ eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads);
Oid GenerateNewManifest(std::filesystem::path ManifestPath);
IoBuffer MakeSidecarManifest(const Oid& BucketId, uint64_t EntryCount);
uint64_t GetSidecarSize() const { return sizeof(BucketMetaHeader) + m_ManifestEntryCount * sizeof(ManifestData); }
- void WriteSidecarFile(RwLock::SharedLockScope& BucketLock,
- const std::filesystem::path& SidecarPath,
- uint64_t SnapshotLogPosition,
- const ZenCacheDiskLayer::CacheBucket::IndexMap& Index,
- const std::vector<AccessTime>& AccessTimes,
- const std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads,
- const std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas);
- bool ReadSidecarFile(RwLock::ExclusiveLockScope& BucketLock,
- ZenCacheDiskLayer::CacheBucket& Bucket,
- std::filesystem::path SidecarPath,
- ZenCacheDiskLayer::CacheBucket::IndexMap& Index,
- std::vector<AccessTime>& AccessTimes,
- std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads);
-
- IoBuffer MakeManifest(const Oid& BucketId,
- ZenCacheDiskLayer::CacheBucket::IndexMap&& Index,
- std::vector<AccessTime>&& AccessTimes,
- std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&& Payloads,
- std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas);
+ void WriteSidecarFile(RwLock::SharedLockScope& BucketLock,
+ const std::filesystem::path& SidecarPath,
+ uint64_t SnapshotLogPosition,
+ const ZenCacheDiskLayer::CacheBucket::IndexMap& Index,
+ const eastl::vector<AccessTime>& AccessTimes,
+ const eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads,
+ const eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas);
+ bool ReadSidecarFile(RwLock::ExclusiveLockScope& BucketLock,
+ ZenCacheDiskLayer::CacheBucket& Bucket,
+ std::filesystem::path SidecarPath,
+ ZenCacheDiskLayer::CacheBucket::IndexMap& Index,
+ eastl::vector<AccessTime>& AccessTimes,
+ eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads);
+
+ IoBuffer MakeManifest(const Oid& BucketId,
+ ZenCacheDiskLayer::CacheBucket::IndexMap&& Index,
+ eastl::vector<AccessTime>&& AccessTimes,
+ eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&& Payloads,
+ eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas);
CbObject Manifest;
@@ -329,12 +329,12 @@ private:
};
void
-BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& BucketLock,
- ZenCacheDiskLayer::CacheBucket& Bucket,
- std::filesystem::path ManifestPath,
- ZenCacheDiskLayer::CacheBucket::IndexMap& Index,
- std::vector<AccessTime>& AccessTimes,
- std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads)
+BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& BucketLock,
+ ZenCacheDiskLayer::CacheBucket& Bucket,
+ std::filesystem::path ManifestPath,
+ ZenCacheDiskLayer::CacheBucket::IndexMap& Index,
+ eastl::vector<AccessTime>& AccessTimes,
+ eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads)
{
if (Manifest["UsingMetaFile"sv].AsBool())
{
@@ -348,8 +348,8 @@ BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& Buck
Stopwatch Timer;
const auto _ = MakeGuard([&] { ZEN_INFO("parsed store manifest '{}' in {}", ManifestPath, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
- const uint64_t Count = Manifest["Count"sv].AsUInt64(0);
- std::vector<PayloadIndex> KeysIndexes;
+ const uint64_t Count = Manifest["Count"sv].AsUInt64(0);
+ eastl::vector<PayloadIndex> KeysIndexes;
KeysIndexes.reserve(Count);
CbArrayView KeyArray = Manifest["Keys"sv].AsArrayView();
@@ -425,11 +425,11 @@ BucketManifestSerializer::GenerateNewManifest(std::filesystem::path ManifestPath
}
IoBuffer
-BucketManifestSerializer::MakeManifest(const Oid& BucketId,
- ZenCacheDiskLayer::CacheBucket::IndexMap&& Index,
- std::vector<AccessTime>&& AccessTimes,
- std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&& Payloads,
- std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas)
+BucketManifestSerializer::MakeManifest(const Oid& BucketId,
+ ZenCacheDiskLayer::CacheBucket::IndexMap&& Index,
+ eastl::vector<AccessTime>&& AccessTimes,
+ eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&& Payloads,
+ eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas)
{
using namespace std::literals;
@@ -519,12 +519,12 @@ BucketManifestSerializer::MakeSidecarManifest(const Oid& BucketId, uint64_t Entr
}
bool
-BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& BucketLock,
- ZenCacheDiskLayer::CacheBucket& Bucket,
- std::filesystem::path SidecarPath,
- ZenCacheDiskLayer::CacheBucket::IndexMap& Index,
- std::vector<AccessTime>& AccessTimes,
- std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads)
+BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& BucketLock,
+ ZenCacheDiskLayer::CacheBucket& Bucket,
+ std::filesystem::path SidecarPath,
+ ZenCacheDiskLayer::CacheBucket::IndexMap& Index,
+ eastl::vector<AccessTime>& AccessTimes,
+ eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads)
{
ZEN_TRACE_CPU("Z$::ReadSidecarFile");
@@ -621,12 +621,12 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& B
void
BucketManifestSerializer::WriteSidecarFile(RwLock::SharedLockScope&,
- const std::filesystem::path& SidecarPath,
- uint64_t SnapshotLogPosition,
- const ZenCacheDiskLayer::CacheBucket::IndexMap& Index,
- const std::vector<AccessTime>& AccessTimes,
- const std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads,
- const std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas)
+ const std::filesystem::path& SidecarPath,
+ uint64_t SnapshotLogPosition,
+ const ZenCacheDiskLayer::CacheBucket::IndexMap& Index,
+ const eastl::vector<AccessTime>& AccessTimes,
+ const eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads,
+ const eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas)
{
ZEN_TRACE_CPU("Z$::WriteSidecarFile");
@@ -653,8 +653,8 @@ BucketManifestSerializer::WriteSidecarFile(RwLock::SharedLockScope&,
// BasicFileWriter SidecarWriter(SidecarFile, 128 * 1024);
- std::vector<ManifestData> ManifestDataBuffer;
- const size_t MaxManifestDataBufferCount = Min(Index.size(), 8192u); // 512 Kb
+ eastl::vector<ManifestData> ManifestDataBuffer;
+ const size_t MaxManifestDataBufferCount = Min(Index.size(), 8192u); // 512 Kb
ManifestDataBuffer.reserve(MaxManifestDataBufferCount);
for (auto& Kv : Index)
{
@@ -1221,20 +1221,20 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc,
struct ZenCacheDiskLayer::CacheBucket::PutBatchHandle
{
- PutBatchHandle(std::vector<bool>& OutResults) : OutResults(OutResults) {}
+ PutBatchHandle(eastl::vector<bool>& OutResults) : OutResults(OutResults) {}
struct Entry
{
- std::vector<IoHash> HashKeyAndReferences;
+ eastl::vector<IoHash> HashKeyAndReferences;
};
- std::vector<IoBuffer> Buffers;
- std::vector<Entry> Entries;
- std::vector<size_t> EntryResultIndexes;
+ eastl::vector<IoBuffer> Buffers;
+ eastl::vector<Entry> Entries;
+ eastl::vector<size_t> EntryResultIndexes;
- std::vector<bool>& OutResults;
+ eastl::vector<bool>& OutResults;
};
ZenCacheDiskLayer::CacheBucket::PutBatchHandle*
-ZenCacheDiskLayer::CacheBucket::BeginPutBatch(std::vector<bool>& OutResults)
+ZenCacheDiskLayer::CacheBucket::BeginPutBatch(eastl::vector<bool>& OutResults)
{
ZEN_TRACE_CPU("Z$::Bucket::BeginPutBatch");
return new PutBatchHandle(OutResults);
@@ -1250,7 +1250,7 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept
ZEN_ASSERT(Batch);
if (!Batch->Buffers.empty())
{
- std::vector<uint8_t> EntryFlags;
+ eastl::vector<uint8_t> EntryFlags;
for (const IoBuffer& Buffer : Batch->Buffers)
{
uint8_t Flags = 0;
@@ -1268,13 +1268,13 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept
size_t IndexOffset = 0;
m_BlockStore.WriteChunks(Batch->Buffers, m_Configuration.PayloadAlignment, [&](std::span<BlockStoreLocation> Locations) {
ZEN_MEMSCOPE(GetCacheDiskTag());
- std::vector<DiskIndexEntry> DiskEntries;
+ eastl::vector<DiskIndexEntry> DiskEntries;
{
RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
for (size_t Index = 0; Index < Locations.size(); Index++)
{
DiskLocation Location(Locations[Index], m_Configuration.PayloadAlignment, EntryFlags[IndexOffset + Index]);
- const std::vector<IoHash>& HashKeyAndReferences = Batch->Entries[IndexOffset + Index].HashKeyAndReferences;
+ const eastl::vector<IoHash>& HashKeyAndReferences = Batch->Entries[IndexOffset + Index].HashKeyAndReferences;
ZEN_ASSERT(HashKeyAndReferences.size() > 1);
const IoHash HashKey = HashKeyAndReferences[0];
DiskEntries.push_back({.Key = HashKey, .Location = Location});
@@ -1337,8 +1337,8 @@ struct ZenCacheDiskLayer::CacheBucket::GetBatchHandle
~GetBatchHandle() {}
- std::vector<IoHash> Keys;
- std::vector<size_t> ResultIndexes;
+ eastl::vector<IoHash> Keys;
+ eastl::vector<size_t> ResultIndexes;
ZenCacheValueVec_t& OutResults;
};
@@ -1829,7 +1829,7 @@ ZenCacheDiskLayer::CacheBucket::MemCacheTrim(GcClock::TimePoint ExpireTime)
uint64_t Trimmed = 0;
GcClock::Tick ExpireTicks = ExpireTime.time_since_epoch().count();
- std::vector<IoBuffer> PurgedBuffers;
+ eastl::vector<IoBuffer> PurgedBuffers;
{
RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
@@ -1868,12 +1868,12 @@ ZenCacheDiskLayer::CacheBucket::MemCacheTrim(GcClock::TimePoint ExpireTime)
}
void
-ZenCacheDiskLayer::CacheBucket::GetUsageByAccess(GcClock::TimePoint Now, GcClock::Duration MaxAge, std::vector<uint64_t>& InOutUsageSlots)
+ZenCacheDiskLayer::CacheBucket::GetUsageByAccess(GcClock::TimePoint Now, GcClock::Duration MaxAge, eastl::vector<uint64_t>& InOutUsageSlots)
{
ZEN_TRACE_CPU("Z$::Bucket::GetUsageByAccess");
- std::vector<uint64_t> PayloadSizes;
- std::vector<AccessTime> AccessTimes;
+ eastl::vector<uint64_t> PayloadSizes;
+ eastl::vector<AccessTime> AccessTimes;
size_t SlotCount = InOutUsageSlots.capacity();
@@ -1925,7 +1925,7 @@ ZenCacheDiskLayer::CacheBucket::Drop()
RwLock::ExclusiveLockScope _(m_IndexLock);
- std::vector<std::unique_ptr<RwLock::ExclusiveLockScope>> ShardLocks;
+ eastl::vector<std::unique_ptr<RwLock::ExclusiveLockScope>> ShardLocks;
ShardLocks.reserve(256);
for (RwLock& Lock : m_ShardedLocks)
{
@@ -1989,10 +1989,10 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl
if (UseLegacyScheme)
{
- std::vector<AccessTime> AccessTimes;
- std::vector<BucketPayload> Payloads;
- std::vector<BucketMetaData> MetaDatas;
- IndexMap Index;
+ eastl::vector<AccessTime> AccessTimes;
+ eastl::vector<BucketPayload> Payloads;
+ eastl::vector<BucketMetaData> MetaDatas;
+ IndexMap Index;
{
RwLock::SharedLockScope IndexLock(m_IndexLock);
@@ -2104,14 +2104,14 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
NiceRate(VerifiedChunkBytes, DurationMs));
});
- RwLock BadKeysLock;
- std::vector<IoHash> BadKeys;
- auto ReportBadKey = [&](const IoHash& Key) { BadKeysLock.WithExclusiveLock([&]() { BadKeys.push_back(Key); }); };
+ RwLock BadKeysLock;
+ eastl::vector<IoHash> BadKeys;
+ auto ReportBadKey = [&](const IoHash& Key) { BadKeysLock.WithExclusiveLock([&]() { BadKeys.push_back(Key); }); };
try
{
- std::vector<BlockStoreLocation> ChunkLocations;
- std::vector<IoHash> ChunkIndexToChunkHash;
+ eastl::vector<BlockStoreLocation> ChunkLocations;
+ eastl::vector<IoHash> ChunkIndexToChunkHash;
RwLock::SharedLockScope _(m_IndexLock);
@@ -2255,7 +2255,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
{
// Deal with bad chunks by removing them from our lookup map
- std::vector<DiskIndexEntry> LogEntries;
+ eastl::vector<DiskIndexEntry> LogEntries;
LogEntries.reserve(BadKeys.size());
{
@@ -2299,11 +2299,11 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
// Clean up m_AccessTimes and m_Payloads vectors
{
- std::vector<BucketPayload> Payloads;
- std::vector<AccessTime> AccessTimes;
- std::vector<BucketMetaData> MetaDatas;
- std::vector<MemCacheData> MemCachedPayloads;
- IndexMap Index;
+ eastl::vector<BucketPayload> Payloads;
+ eastl::vector<AccessTime> AccessTimes;
+ eastl::vector<BucketMetaData> MetaDatas;
+ eastl::vector<MemCacheData> MemCachedPayloads;
+ IndexMap Index;
{
RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
@@ -2348,8 +2348,8 @@ ZenCacheDiskLayer::CacheBucket::EntryCount() const
CacheValueDetails::ValueDetails
ZenCacheDiskLayer::CacheBucket::GetValueDetails(RwLock::SharedLockScope& IndexLock, const IoHash& Key, PayloadIndex Index) const
{
- std::vector<IoHash> Attachments;
- const BucketPayload& Payload = m_Payloads[Index];
+ eastl::vector<IoHash> Attachments;
+ const BucketPayload& Payload = m_Payloads[Index];
if (Payload.Location.IsFlagSet(DiskLocation::kStructured))
{
IoBuffer Value = Payload.Location.IsFlagSet(DiskLocation::kStandaloneFile) ? GetStandaloneCacheValue(Payload.Location, Key)
@@ -2661,7 +2661,7 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey,
OptionalBatchHandle->Entries.push_back({});
OptionalBatchHandle->EntryResultIndexes.push_back(OptionalBatchHandle->OutResults.size());
OptionalBatchHandle->OutResults.push_back(false);
- std::vector<IoHash>& HashKeyAndReferences = OptionalBatchHandle->Entries.back().HashKeyAndReferences;
+ eastl::vector<IoHash>& HashKeyAndReferences = OptionalBatchHandle->Entries.back().HashKeyAndReferences;
HashKeyAndReferences.reserve(1 + HashKeyAndReferences.size());
HashKeyAndReferences.push_back(HashKey);
HashKeyAndReferences.insert(HashKeyAndReferences.end(), HashKeyAndReferences.begin(), HashKeyAndReferences.end());
@@ -2731,7 +2731,7 @@ class DiskBucketStoreCompactor : public GcStoreCompactor
using CacheBucket = ZenCacheDiskLayer::CacheBucket;
public:
- DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys)
+ DiskBucketStoreCompactor(CacheBucket& Bucket, eastl::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys)
: m_Bucket(Bucket)
, m_ExpiredStandaloneKeys(std::move(ExpiredStandaloneKeys))
{
@@ -2871,7 +2871,7 @@ public:
{
BlockStoreCompactState BlockCompactState;
- std::vector<IoHash> BlockCompactStateKeys;
+ eastl::vector<IoHash> BlockCompactStateKeys;
BlockCompactStateKeys.reserve(InlineEntryCount);
BlockStore::BlockEntryCountMap BlocksToCompact =
@@ -2913,7 +2913,7 @@ public:
BlockCompactState,
m_Bucket.m_Configuration.PayloadAlignment,
[&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) {
- std::vector<DiskIndexEntry> MovedEntries;
+ eastl::vector<DiskIndexEntry> MovedEntries;
MovedEntries.reserve(MovedArray.size());
RwLock::ExclusiveLockScope _(m_Bucket.m_IndexLock);
for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray)
@@ -2965,8 +2965,8 @@ public:
virtual std::string GetGcName(GcCtx& Ctx) override { return m_Bucket.GetGcName(Ctx); }
private:
- ZenCacheDiskLayer::CacheBucket& m_Bucket;
- std::vector<std::pair<IoHash, uint64_t>> m_ExpiredStandaloneKeys;
+ ZenCacheDiskLayer::CacheBucket& m_Bucket;
+ eastl::vector<std::pair<IoHash, uint64_t>> m_ExpiredStandaloneKeys;
};
GcStoreCompactor*
@@ -3012,9 +3012,9 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count();
- std::vector<DiskIndexEntry> ExpiredEntries;
- std::vector<std::pair<IoHash, uint64_t>> ExpiredStandaloneKeys;
- uint64_t RemovedStandaloneSize = 0;
+ eastl::vector<DiskIndexEntry> ExpiredEntries;
+ eastl::vector<std::pair<IoHash, uint64_t>> ExpiredStandaloneKeys;
+ uint64_t RemovedStandaloneSize = 0;
{
RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
if (Ctx.IsCancelledFlag.load())
@@ -3078,11 +3078,11 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
if (Ctx.Settings.IsDeleteMode && !ExpiredEntries.empty())
{
- std::vector<BucketPayload> Payloads;
- std::vector<AccessTime> AccessTimes;
- std::vector<BucketMetaData> MetaDatas;
- std::vector<MemCacheData> MemCachedPayloads;
- IndexMap Index;
+ eastl::vector<BucketPayload> Payloads;
+ eastl::vector<AccessTime> AccessTimes;
+ eastl::vector<BucketMetaData> MetaDatas;
+ eastl::vector<MemCacheData> MemCachedPayloads;
+ IndexMap Index;
{
RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, Index);
@@ -3101,7 +3101,7 @@ bool
ZenCacheDiskLayer::CacheBucket::ReadAttachmentsFromMetaData(uint32_t BlockIndex,
std::span<const IoHash> InlineKeys,
std::span<const std::size_t> ChunkIndexes,
- std::vector<IoHash>& OutReferences) const
+ eastl::vector<IoHash>& OutReferences) const
{
ZEN_TRACE_CPU("Z$::Bucket::ReadAttachmentsFromMetaData");
IoBuffer MetaDataPayload = m_BlockStore.GetMetaData(BlockIndex);
@@ -3146,13 +3146,13 @@ ZenCacheDiskLayer::CacheBucket::ReadAttachmentsFromMetaData(uint32_t BlockI
}
bool
-ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger,
- std::atomic_bool& IsCancelledFlag,
- bool StateIsAlreadyLocked,
- bool ReadCacheAttachmentMetaData,
- bool WriteCacheAttachmentMetaData,
- std::vector<IoHash>& OutReferences,
- ReferencesStats* OptionalOutReferencesStats)
+ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger,
+ std::atomic_bool& IsCancelledFlag,
+ bool StateIsAlreadyLocked,
+ bool ReadCacheAttachmentMetaData,
+ bool WriteCacheAttachmentMetaData,
+ eastl::vector<IoHash>& OutReferences,
+ ReferencesStats* OptionalOutReferencesStats)
{
ZEN_TRACE_CPU("Z$::Bucket::GetReferencesLocked");
@@ -3168,11 +3168,11 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger,
return false;
};
- std::vector<std::pair<IoHash, DiskLocation>> StandaloneKeys;
+ eastl::vector<std::pair<IoHash, DiskLocation>> StandaloneKeys;
{
- std::vector<IoHash> InlineKeys;
- std::vector<BlockStoreLocation> InlineLocations;
- std::vector<std::vector<std::size_t>> InlineBlockChunkIndexes;
+ eastl::vector<IoHash> InlineKeys;
+ eastl::vector<BlockStoreLocation> InlineLocations;
+ eastl::vector<eastl::vector<std::size_t>> InlineBlockChunkIndexes;
{
std::unordered_map<uint32_t, std::size_t> BlockIndexToChunkIndexes;
@@ -3229,7 +3229,7 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger,
else
{
BlockIndexToChunkIndexes.insert_or_assign(ChunkLocation.BlockIndex, InlineBlockChunkIndexes.size());
- InlineBlockChunkIndexes.emplace_back(std::vector<size_t>{ChunkIndex});
+ InlineBlockChunkIndexes.emplace_back(eastl::vector<size_t>{ChunkIndex});
}
}
}
@@ -3237,17 +3237,17 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger,
OutReferences.reserve(OutReferences.size() + InlineKeys.size() +
StandaloneKeys.size()); // Make space for at least one attachment per record
- for (const std::vector<std::size_t>& ChunkIndexes : InlineBlockChunkIndexes)
+ for (const eastl::vector<std::size_t>& ChunkIndexes : InlineBlockChunkIndexes)
{
ZEN_ASSERT(!ChunkIndexes.empty());
uint32_t BlockIndex = InlineLocations[ChunkIndexes[0]].BlockIndex;
if (!ReadCacheAttachmentMetaData || !ReadAttachmentsFromMetaData(BlockIndex, InlineKeys, ChunkIndexes, OutReferences))
{
- std::vector<IoHash> Keys;
- std::vector<uint32_t> AttachmentCounts;
- size_t PrecachedReferencesStart = OutReferences.size();
- size_t NextPrecachedReferencesStart = PrecachedReferencesStart;
+ eastl::vector<IoHash> Keys;
+ eastl::vector<uint32_t> AttachmentCounts;
+ size_t PrecachedReferencesStart = OutReferences.size();
+ size_t NextPrecachedReferencesStart = PrecachedReferencesStart;
bool WriteMetaData = WriteCacheAttachmentMetaData && !m_BlockStore.IsWriting(BlockIndex);
if (WriteMetaData)
@@ -3372,7 +3372,8 @@ public:
NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
});
- m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences = std::make_unique<std::vector<IoHash>>(); });
+ m_CacheBucket.m_IndexLock.WithExclusiveLock(
+ [&]() { m_CacheBucket.m_TrackedReferences = std::make_unique<eastl::vector<IoHash>>(); });
bool Continue = m_CacheBucket.GetReferences(Ctx.Logger,
Ctx.IsCancelledFlag,
@@ -3448,12 +3449,12 @@ public:
return UnusedReferences;
}
- CacheBucket& m_CacheBucket;
- std::vector<IoHash> m_PrecachedReferences;
- std::vector<IoHash> m_AddedReferences;
+ CacheBucket& m_CacheBucket;
+ eastl::vector<IoHash> m_PrecachedReferences;
+ eastl::vector<IoHash> m_AddedReferences;
};
-std::vector<GcReferenceChecker*>
+eastl::vector<GcReferenceChecker*>
ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx)
{
ZEN_TRACE_CPU("Z$::Bucket::CreateReferenceCheckers");
@@ -3480,7 +3481,7 @@ ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx)
return {new DiskBucketReferenceChecker(*this)};
}
-std::vector<GcReferenceValidator*>
+eastl::vector<GcReferenceValidator*>
ZenCacheDiskLayer::CacheBucket::CreateReferenceValidators(GcCtx& /*Ctx*/)
{
return {};
@@ -3488,11 +3489,11 @@ ZenCacheDiskLayer::CacheBucket::CreateReferenceValidators(GcCtx& /*Ctx*/)
void
ZenCacheDiskLayer::CacheBucket::CompactState(RwLock::ExclusiveLockScope&,
- std::vector<BucketPayload>& Payloads,
- std::vector<AccessTime>& AccessTimes,
- std::vector<BucketMetaData>& MetaDatas,
- std::vector<MemCacheData>& MemCachedPayloads,
- IndexMap& Index)
+ eastl::vector<BucketPayload>& Payloads,
+ eastl::vector<AccessTime>& AccessTimes,
+ eastl::vector<BucketMetaData>& MetaDatas,
+ eastl::vector<MemCacheData>& MemCachedPayloads,
+ IndexMap& Index)
{
ZEN_TRACE_CPU("Z$::Bucket::CompactState");
@@ -3650,7 +3651,7 @@ ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket)
struct ZenCacheDiskLayer::PutBatchHandle
{
- PutBatchHandle(std::vector<bool>& OutResults) : OutResults(OutResults) {}
+ PutBatchHandle(eastl::vector<bool>& OutResults) : OutResults(OutResults) {}
struct BucketHandle
{
CacheBucket* Bucket;
@@ -3709,13 +3710,13 @@ struct ZenCacheDiskLayer::PutBatchHandle
return NewBucketHandle;
}
- RwLock Lock;
- std::vector<BucketHandle> BucketHandles;
- std::vector<bool>& OutResults;
+ RwLock Lock;
+ eastl::vector<BucketHandle> BucketHandles;
+ eastl::vector<bool>& OutResults;
};
ZenCacheDiskLayer::PutBatchHandle*
-ZenCacheDiskLayer::BeginPutBatch(std::vector<bool>& OutResults)
+ZenCacheDiskLayer::BeginPutBatch(eastl::vector<bool>& OutResults)
{
return new PutBatchHandle(OutResults);
}
@@ -3879,8 +3880,8 @@ ZenCacheDiskLayer::DiscoverBuckets()
// Initialize buckets
- std::vector<std::filesystem::path> BadBucketDirectories;
- std::vector<std::filesystem::path> FoundBucketDirectories;
+ eastl::vector<std::filesystem::path> BadBucketDirectories;
+ eastl::vector<std::filesystem::path> FoundBucketDirectories;
RwLock::ExclusiveLockScope _(m_Lock);
@@ -4003,7 +4004,7 @@ ZenCacheDiskLayer::Drop()
RwLock::ExclusiveLockScope _(m_Lock);
- std::vector<std::unique_ptr<CacheBucket>> Buckets;
+ eastl::vector<std::unique_ptr<CacheBucket>> Buckets;
Buckets.reserve(m_Buckets.size());
while (!m_Buckets.empty())
{
@@ -4025,15 +4026,15 @@ ZenCacheDiskLayer::Flush()
ZEN_MEMSCOPE(GetCacheDiskTag());
ZEN_TRACE_CPU("Z$::Flush");
- std::vector<CacheBucket*> Buckets;
- Stopwatch Timer;
- const auto _ = MakeGuard([&] {
- if (Buckets.empty())
- {
- return;
- }
- ZEN_INFO("Flushed {} buckets at '{}' in {}", Buckets.size(), m_RootDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
- });
+ eastl::vector<CacheBucket*> Buckets;
+ Stopwatch Timer;
+ const auto _ = MakeGuard([&] {
+ if (Buckets.empty())
+ {
+ return;
+ }
+ ZEN_INFO("Flushed {} buckets at '{}' in {}", Buckets.size(), m_RootDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ });
{
RwLock::SharedLockScope __(m_Lock);
@@ -4086,7 +4087,7 @@ ZenCacheDiskLayer::ScrubStorage(ScrubContext& Ctx)
RwLock::SharedLockScope _(m_Lock);
{
- std::vector<std::future<void>> Results;
+ eastl::vector<std::future<void>> Results;
Results.reserve(m_Buckets.size());
for (auto& Kv : m_Buckets)
@@ -4211,10 +4212,10 @@ ZenCacheDiskLayer::GetValueDetails(const std::string_view BucketFilter, const st
return Details;
}
-std::vector<RwLock::SharedLockScope>
+eastl::vector<RwLock::SharedLockScope>
ZenCacheDiskLayer::GetGcReferencerLocks()
{
- std::vector<RwLock::SharedLockScope> Locks;
+ eastl::vector<RwLock::SharedLockScope> Locks;
Locks.emplace_back(RwLock::SharedLockScope(m_Lock));
for (auto& Kv : m_Buckets)
{
@@ -4230,7 +4231,7 @@ ZenCacheDiskLayer::EnableUpdateCapture()
if (m_UpdateCaptureRefCounter == 0)
{
ZEN_ASSERT(!m_CapturedBuckets);
- m_CapturedBuckets = std::make_unique<std::vector<std::string>>();
+ m_CapturedBuckets = std::make_unique<eastl::vector<std::string>>();
}
else
{
@@ -4254,7 +4255,7 @@ ZenCacheDiskLayer::DisableUpdateCapture()
});
}
-std::vector<std::string>
+eastl::vector<std::string>
ZenCacheDiskLayer::GetCapturedBucketsLocked()
{
if (m_CapturedBuckets)
@@ -4332,11 +4333,11 @@ ZenCacheDiskLayer::MemCacheTrim()
const std::chrono::seconds MaxAge = std::chrono::seconds(m_Configuration.MemCacheMaxAgeSeconds);
- static const size_t UsageSlotCount = 2048;
- std::vector<uint64_t> UsageSlots;
+ static const size_t UsageSlotCount = 2048;
+ eastl::vector<uint64_t> UsageSlots;
UsageSlots.reserve(UsageSlotCount);
- std::vector<CacheBucket*> Buckets;
+ eastl::vector<CacheBucket*> Buckets;
{
RwLock::SharedLockScope __(m_Lock);
Buckets.reserve(m_Buckets.size());
@@ -4371,7 +4372,7 @@ ZenCacheDiskLayer::MemCacheTrim()
}
uint64_t
-ZenCacheDiskLayer::MemCacheTrim(std::vector<CacheBucket*>& Buckets, GcClock::TimePoint ExpireTime)
+ZenCacheDiskLayer::MemCacheTrim(eastl::vector<CacheBucket*>& Buckets, GcClock::TimePoint ExpireTime)
{
if (m_Configuration.MemCacheTargetFootprintBytes == 0)
{
diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp
index 97e26a38d..481a8906e 100644
--- a/src/zenstore/cache/cacherpc.cpp
+++ b/src/zenstore/cache/cacherpc.cpp
@@ -107,7 +107,7 @@ namespace cache::detail {
struct RecordBody
{
IoBuffer CacheValue;
- std::vector<RecordValue> Values;
+ eastl::vector<RecordValue> Values;
const UpstreamEndpointInfo* Source = nullptr;
CachePolicy DownstreamPolicy;
bool Exists = false;
@@ -362,12 +362,12 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag
uint64_t RecordObjectSize = Record.GetSize();
uint64_t TransferredSize = RecordObjectSize;
- AttachmentCount Count;
- size_t NumAttachments = Package->GetAttachments().size();
- std::vector<IoHash> ValidAttachments;
- std::vector<IoHash> ReferencedAttachments;
- std::vector<IoBuffer> WriteAttachmentBuffers;
- std::vector<IoHash> WriteRawHashes;
+ AttachmentCount Count;
+ size_t NumAttachments = Package->GetAttachments().size();
+ eastl::vector<IoHash> ValidAttachments;
+ eastl::vector<IoHash> ReferencedAttachments;
+ eastl::vector<IoBuffer> WriteAttachmentBuffers;
+ eastl::vector<IoHash> WriteRawHashes;
ValidAttachments.reserve(NumAttachments);
WriteAttachmentBuffers.reserve(NumAttachments);
WriteRawHashes.reserve(NumAttachments);
@@ -429,7 +429,7 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag
if (!WriteAttachmentBuffers.empty())
{
- std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes);
+ eastl::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes);
for (size_t Index = 0; Index < InsertResults.size(); Index++)
{
if (InsertResults[Index].New)
@@ -563,11 +563,11 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
Request.RecordObject = CbObjectView(Request.RecordCacheValue.GetData());
ParseValues(Request);
- Request.Complete = true;
- size_t ValueCount = Request.Values.size();
- std::vector<IoHash> CidHashes;
- std::vector<size_t> RequestValueIndexes;
- const bool DoBatch = ValueCount > 7;
+ Request.Complete = true;
+ size_t ValueCount = Request.Values.size();
+ eastl::vector<IoHash> CidHashes;
+ eastl::vector<size_t> RequestValueIndexes;
+ const bool DoBatch = ValueCount > 7;
if (DoBatch)
{
CidHashes.reserve(ValueCount);
@@ -703,7 +703,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
if (!UpstreamIndexes.empty())
{
- std::vector<CacheKeyRequest*> UpstreamRequests;
+ eastl::vector<CacheKeyRequest*> UpstreamRequests;
UpstreamRequests.reserve(UpstreamIndexes.size());
for (size_t Index : UpstreamIndexes)
{
@@ -753,7 +753,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal) && AreDiskWritesAllowed();
if (StoreLocal)
{
- std::vector<IoHash> ReferencedAttachments;
+ eastl::vector<IoHash> ReferencedAttachments;
ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) {
const IoHash ValueHash = HashView.AsHash();
ReferencedAttachments.push_back(ValueHash);
@@ -912,7 +912,7 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con
const bool HasUpstream = m_UpstreamCache.IsActive();
CbArrayView RequestsArray = Params["Requests"sv].AsArrayView();
- std::vector<bool> BatchResults;
+ eastl::vector<bool> BatchResults;
eastl::fixed_vector<size_t, 32> BatchResultIndexes;
eastl::fixed_vector<bool, 32> Results;
eastl::fixed_vector<CacheKey, 32> UpstreamCacheKeys;
@@ -1200,8 +1200,8 @@ CacheRpcHandler::HandleRpcGetCacheValues(const CacheRequestContext& Context, CbO
if (!RemoteRequestIndexes.empty())
{
- std::vector<CacheValueRequest> RequestedRecordsData;
- std::vector<CacheValueRequest*> CacheValueRequests;
+ eastl::vector<CacheValueRequest> RequestedRecordsData;
+ eastl::vector<CacheValueRequest*> CacheValueRequests;
RequestedRecordsData.reserve(RemoteRequestIndexes.size());
CacheValueRequests.reserve(RemoteRequestIndexes.size());
for (size_t Index : RemoteRequestIndexes)
@@ -1320,14 +1320,14 @@ CacheRpcHandler::HandleRpcGetCacheChunks(const CacheRequestContext& Context, Rpc
ZEN_TRACE_CPU("Z$::RpcGetCacheChunks");
using namespace cache::detail;
- std::string Namespace;
- std::vector<CacheKeyRequest> RecordKeys; // Data about a Record necessary to identify it to the upstream
- std::vector<RecordBody> Records; // Scratch-space data about a Record when fulfilling RecordRequests
- std::vector<CacheChunkRequest> RequestKeys; // Data about a ChunkRequest necessary to identify it to the upstream
- std::vector<ChunkRequest> Requests; // Intermediate and result data about a ChunkRequest
- std::vector<ChunkRequest*> RecordRequests; // The ChunkRequests that are requesting a subvalue from a Record Key
- std::vector<ChunkRequest*> ValueRequests; // The ChunkRequests that are requesting a Value Key
- std::vector<CacheChunkRequest*> UpstreamChunks; // ChunkRequests that we need to send to the upstream
+ std::string Namespace;
+ eastl::vector<CacheKeyRequest> RecordKeys; // Data about a Record necessary to identify it to the upstream
+ eastl::vector<RecordBody> Records; // Scratch-space data about a Record when fulfilling RecordRequests
+ eastl::vector<CacheChunkRequest> RequestKeys; // Data about a ChunkRequest necessary to identify it to the upstream
+ eastl::vector<ChunkRequest> Requests; // Intermediate and result data about a ChunkRequest
+ eastl::vector<ChunkRequest*> RecordRequests; // The ChunkRequests that are requesting a subvalue from a Record Key
+ eastl::vector<ChunkRequest*> ValueRequests; // The ChunkRequests that are requesting a Value Key
+ eastl::vector<CacheChunkRequest*> UpstreamChunks; // ChunkRequests that we need to send to the upstream
// Parse requests from the CompactBinary body of the RpcRequest and divide it into RecordRequests and ValueRequests
if (!ParseGetCacheChunksRequest(Namespace, RecordKeys, Records, RequestKeys, Requests, RecordRequests, ValueRequests, RpcRequest))
@@ -1350,14 +1350,14 @@ CacheRpcHandler::HandleRpcGetCacheChunks(const CacheRequestContext& Context, Rpc
}
bool
-CacheRpcHandler::ParseGetCacheChunksRequest(std::string& Namespace,
- std::vector<CacheKeyRequest>& RecordKeys,
- std::vector<cache::detail::RecordBody>& Records,
- std::vector<CacheChunkRequest>& RequestKeys,
- std::vector<cache::detail::ChunkRequest>& Requests,
- std::vector<cache::detail::ChunkRequest*>& RecordRequests,
- std::vector<cache::detail::ChunkRequest*>& ValueRequests,
- CbObjectView RpcRequest)
+CacheRpcHandler::ParseGetCacheChunksRequest(std::string& Namespace,
+ eastl::vector<CacheKeyRequest>& RecordKeys,
+ eastl::vector<cache::detail::RecordBody>& Records,
+ eastl::vector<CacheChunkRequest>& RequestKeys,
+ eastl::vector<cache::detail::ChunkRequest>& Requests,
+ eastl::vector<cache::detail::ChunkRequest*>& RecordRequests,
+ eastl::vector<cache::detail::ChunkRequest*>& ValueRequests,
+ CbObjectView RpcRequest)
{
ZEN_TRACE_CPU("Z$::ParseGetCacheChunksRequest");
@@ -1470,12 +1470,12 @@ CacheRpcHandler::ParseGetCacheChunksRequest(std::string& Namespace,
}
void
-CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context,
- std::string_view Namespace,
- std::vector<CacheKeyRequest>& RecordKeys,
- std::vector<cache::detail::RecordBody>& Records,
- std::vector<cache::detail::ChunkRequest*>& RecordRequests,
- std::vector<CacheChunkRequest*>& OutUpstreamChunks)
+CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context,
+ std::string_view Namespace,
+ eastl::vector<CacheKeyRequest>& RecordKeys,
+ eastl::vector<cache::detail::RecordBody>& Records,
+ eastl::vector<cache::detail::ChunkRequest*>& RecordRequests,
+ eastl::vector<CacheChunkRequest*>& OutUpstreamChunks)
{
ZEN_TRACE_CPU("Z$::GetLocalCacheRecords");
@@ -1483,7 +1483,7 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context,
const bool HasUpstream = m_UpstreamCache.IsActive();
// TODO: BatchGet records?
- std::vector<CacheKeyRequest*> UpstreamRecordRequests;
+ eastl::vector<CacheKeyRequest*> UpstreamRecordRequests;
for (size_t RecordIndex = 0; RecordIndex < Records.size(); ++RecordIndex)
{
ZEN_TRACE_CPU("Z$::GetLocalCacheRecords::Record");
@@ -1536,7 +1536,7 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context,
bool StoreLocal = EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal) && AreDiskWritesAllowed();
if (StoreLocal)
{
- std::vector<IoHash> ReferencedAttachments;
+ eastl::vector<IoHash> ReferencedAttachments;
ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) {
const IoHash ValueHash = HashView.AsHash();
ReferencedAttachments.push_back(ValueHash);
@@ -1637,10 +1637,10 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context,
}
void
-CacheRpcHandler::GetLocalCacheValues(const CacheRequestContext& Context,
- std::string_view Namespace,
- std::vector<cache::detail::ChunkRequest*>& ValueRequests,
- std::vector<CacheChunkRequest*>& OutUpstreamChunks)
+CacheRpcHandler::GetLocalCacheValues(const CacheRequestContext& Context,
+ std::string_view Namespace,
+ eastl::vector<cache::detail::ChunkRequest*>& ValueRequests,
+ eastl::vector<CacheChunkRequest*>& OutUpstreamChunks)
{
ZEN_TRACE_CPU("Z$::GetLocalCacheValues");
@@ -1717,11 +1717,11 @@ CacheRpcHandler::GetLocalCacheValues(const CacheRequestContext& Context,
}
void
-CacheRpcHandler::GetUpstreamCacheChunks(const CacheRequestContext& Context,
- std::string_view Namespace,
- std::vector<CacheChunkRequest*>& UpstreamChunks,
- std::vector<CacheChunkRequest>& RequestKeys,
- std::vector<cache::detail::ChunkRequest>& Requests)
+CacheRpcHandler::GetUpstreamCacheChunks(const CacheRequestContext& Context,
+ std::string_view Namespace,
+ eastl::vector<CacheChunkRequest*>& UpstreamChunks,
+ eastl::vector<CacheChunkRequest>& RequestKeys,
+ eastl::vector<cache::detail::ChunkRequest>& Requests)
{
if (UpstreamChunks.empty())
{
@@ -1790,7 +1790,7 @@ CbPackage
CacheRpcHandler::WriteGetCacheChunksResponse([[maybe_unused]] const CacheRequestContext& Context,
std::string_view Namespace,
RpcAcceptOptions AcceptOptions,
- std::vector<cache::detail::ChunkRequest>& Requests)
+ eastl::vector<cache::detail::ChunkRequest>& Requests)
{
ZEN_TRACE_CPU("Z$::WriteGetCacheChunksResponse");
diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index 7d277329e..87c95f8a2 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -154,7 +154,7 @@ struct ZenCacheNamespace::PutBatchHandle
};
ZenCacheNamespace::PutBatchHandle*
-ZenCacheNamespace::BeginPutBatch(std::vector<bool>& OutResult)
+ZenCacheNamespace::BeginPutBatch(eastl::vector<bool>& OutResult)
{
ZenCacheNamespace::PutBatchHandle* Handle = new ZenCacheNamespace::PutBatchHandle;
Handle->DiskLayerHandle = m_DiskLayer.BeginPutBatch(OutResult);
@@ -366,7 +366,7 @@ ZenCacheNamespace::GetValueDetails(const std::string_view BucketFilter, const st
return m_DiskLayer.GetValueDetails(BucketFilter, ValueFilter);
}
-std::vector<RwLock::SharedLockScope>
+eastl::vector<RwLock::SharedLockScope>
ZenCacheNamespace::GetGcReferencerLocks()
{
return m_DiskLayer.GetGcReferencerLocks();
@@ -426,7 +426,7 @@ ZenCacheStore::ZenCacheStore(GcManager& Gc,
DirectoryContent DirContent;
GetDirectoryContent(m_BasePath, DirectoryContentFlags::IncludeDirs, DirContent);
- std::vector<std::string> Namespaces;
+ eastl::vector<std::string> Namespaces;
for (const std::filesystem::path& DirPath : DirContent.Directories)
{
std::string DirName = PathToUtf8(DirPath.filename());
@@ -480,7 +480,7 @@ ZenCacheStore::LogWorker()
auto Log = [&ZCacheLog]() -> LoggerRef { return ZCacheLog; };
- std::vector<AccessLogItem> Items;
+ eastl::vector<AccessLogItem> Items;
while (true)
{
try
@@ -553,7 +553,7 @@ ZenCacheStore::LogWorker()
}
}
-ZenCacheStore::PutBatch::PutBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, std::vector<bool>& OutResult)
+ZenCacheStore::PutBatch::PutBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, eastl::vector<bool>& OutResult)
: m_CacheStore(CacheStore)
{
ZEN_MEMSCOPE(GetCacheStoreTag());
@@ -904,10 +904,10 @@ ZenCacheStore::FindNamespace(std::string_view Namespace) const
return nullptr;
}
-std::vector<std::string>
+eastl::vector<std::string>
ZenCacheStore::GetNamespaces()
{
- std::vector<std::string> Namespaces;
+ eastl::vector<std::string> Namespaces;
IterateNamespaces([&](std::string_view Namespace, ZenCacheNamespace&) { Namespaces.push_back(std::string(Namespace)); });
@@ -917,7 +917,7 @@ ZenCacheStore::GetNamespaces()
void
ZenCacheStore::IterateNamespaces(const std::function<void(std::string_view Namespace, ZenCacheNamespace& Store)>& Callback) const
{
- std::vector<std::pair<std::string, ZenCacheNamespace&>> Namespaces;
+ eastl::vector<std::pair<std::string, ZenCacheNamespace&>> Namespaces;
{
RwLock::SharedLockScope _(m_NamespacesLock);
Namespaces.reserve(m_Namespaces.size());
@@ -1042,18 +1042,18 @@ ZenCacheStore::GetBucketInfo(std::string_view NamespaceName, std::string_view Bu
return {};
}
-std::vector<RwLock::SharedLockScope>
+eastl::vector<RwLock::SharedLockScope>
ZenCacheStore::LockState(GcCtx& Ctx)
{
ZEN_TRACE_CPU("CacheStore::LockState");
auto Log = [&Ctx]() { return Ctx.Logger; };
- std::vector<RwLock::SharedLockScope> Locks;
+ eastl::vector<RwLock::SharedLockScope> Locks;
Locks.emplace_back(RwLock::SharedLockScope(m_NamespacesLock));
for (auto& NamespaceIt : m_Namespaces)
{
- std::vector<RwLock::SharedLockScope> NamespaceLocks = NamespaceIt.second->GetGcReferencerLocks();
+ eastl::vector<RwLock::SharedLockScope> NamespaceLocks = NamespaceIt.second->GetGcReferencerLocks();
for (auto It = std::make_move_iterator(NamespaceLocks.begin()); It != std::make_move_iterator(NamespaceLocks.end()); It++)
{
Locks.emplace_back(std::move(*It));
@@ -1065,12 +1065,12 @@ ZenCacheStore::LockState(GcCtx& Ctx)
void
ZenCacheStore::EnableUpdateCapture()
{
- std::vector<ZenCacheNamespace*> Namespaces;
+ eastl::vector<ZenCacheNamespace*> Namespaces;
m_NamespacesLock.WithExclusiveLock([&]() {
if (m_UpdateCaptureRefCounter == 0)
{
ZEN_ASSERT(!m_CapturedNamespaces);
- m_CapturedNamespaces = std::make_unique<std::vector<std::string>>();
+ m_CapturedNamespaces = std::make_unique<eastl::vector<std::string>>();
}
else
{
@@ -1093,7 +1093,7 @@ ZenCacheStore::EnableUpdateCapture()
void
ZenCacheStore::DisableUpdateCapture()
{
- std::vector<ZenCacheNamespace*> Namespaces;
+ eastl::vector<ZenCacheNamespace*> Namespaces;
m_NamespacesLock.WithExclusiveLock([&]() {
ZEN_ASSERT(m_CapturedNamespaces);
ZEN_ASSERT(m_UpdateCaptureRefCounter > 0);
@@ -1114,7 +1114,7 @@ ZenCacheStore::DisableUpdateCapture()
}
}
-std::vector<std::string>
+eastl::vector<std::string>
ZenCacheStore::GetCapturedNamespacesLocked()
{
if (m_CapturedNamespaces)
@@ -1173,7 +1173,7 @@ public:
Stopwatch Timer;
- std::vector<ZenCacheDiskLayer::CacheBucket*> AddedBuckets;
+ eastl::vector<ZenCacheDiskLayer::CacheBucket*> AddedBuckets;
const auto _ = MakeGuard([&] {
if (!Ctx.Settings.Verbose)
@@ -1187,7 +1187,7 @@ public:
AddedBuckets.size());
});
- std::vector<std::string> AddedNamespaces = m_CacheStore.GetCapturedNamespacesLocked();
+ eastl::vector<std::string> AddedNamespaces = m_CacheStore.GetCapturedNamespacesLocked();
for (const std::string& AddedNamespace : AddedNamespaces)
{
@@ -1202,8 +1202,8 @@ public:
}
for (auto& NamepaceKV : m_CacheStore.m_Namespaces)
{
- ZenCacheNamespace& Namespace = *NamepaceKV.second;
- std::vector<std::string> NamespaceAddedBuckets = Namespace.m_DiskLayer.GetCapturedBucketsLocked();
+ ZenCacheNamespace& Namespace = *NamepaceKV.second;
+ eastl::vector<std::string> NamespaceAddedBuckets = Namespace.m_DiskLayer.GetCapturedBucketsLocked();
for (const std::string& AddedBucketName : NamespaceAddedBuckets)
{
if (auto It = Namespace.m_DiskLayer.m_Buckets.find(AddedBucketName); It != Namespace.m_DiskLayer.m_Buckets.end())
@@ -1258,31 +1258,31 @@ public:
}
private:
- ZenCacheStore& m_CacheStore;
- std::vector<IoHash> m_References;
+ ZenCacheStore& m_CacheStore;
+ eastl::vector<IoHash> m_References;
};
-std::vector<GcReferenceChecker*>
+eastl::vector<GcReferenceChecker*>
ZenCacheStore::CreateReferenceCheckers(GcCtx& Ctx)
{
ZEN_TRACE_CPU("CacheStore::CreateReferenceCheckers");
auto Log = [&Ctx]() { return Ctx.Logger; };
- Stopwatch Timer;
- const auto _ = MakeGuard([&] {
- if (!Ctx.Settings.Verbose)
- {
- return;
- }
- ZEN_INFO("GCV2: cachestore [CREATE CHECKERS] '{}': completed in {}", m_BasePath, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
- });
- std::vector<GcReferenceChecker*> Checkers;
+ Stopwatch Timer;
+ const auto _ = MakeGuard([&] {
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: cachestore [CREATE CHECKERS] '{}': completed in {}", m_BasePath, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ });
+ eastl::vector<GcReferenceChecker*> Checkers;
Checkers.emplace_back(new CacheStoreReferenceChecker(*this));
return Checkers;
}
-std::vector<GcReferenceValidator*>
+eastl::vector<GcReferenceValidator*>
ZenCacheStore::CreateReferenceValidators(GcCtx& /*Ctx*/)
{
return {};
@@ -1307,9 +1307,9 @@ namespace testutils {
std::pair<Oid, IoBuffer> CreateBinaryBlob(size_t Size) { return {Oid::NewOid(), CreateRandomBlob(Size)}; }
- std::vector<std::pair<Oid, CompressedBuffer>> CreateCompressedAttachment(CidStore& Store, const std::span<const size_t>& Sizes)
+ eastl::vector<std::pair<Oid, CompressedBuffer>> CreateCompressedAttachment(CidStore& Store, const std::span<const size_t>& Sizes)
{
- std::vector<std::pair<Oid, CompressedBuffer>> Result;
+ eastl::vector<std::pair<Oid, CompressedBuffer>> Result;
Result.reserve(Sizes.size());
for (size_t Size : Sizes)
{
@@ -1386,7 +1386,7 @@ TEST_CASE("cachestore.size")
auto JobQueue = MakeJobQueue(1, "testqueue");
const auto CreateCacheValue = [](size_t Size) -> CbObject {
- std::vector<uint8_t> Buf;
+ eastl::vector<uint8_t> Buf;
Buf.resize(Size);
CbObjectWriter Writer;
@@ -1410,7 +1410,7 @@ TEST_CASE("cachestore.size")
IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer();
Buffer.SetContentType(ZenContentType::kCbObject);
- std::vector<std::pair<std::string, IoHash>> Keys;
+ eastl::vector<std::pair<std::string, IoHash>> Keys;
for (size_t Key = 0; Key < Count; ++Key)
{
@@ -1710,7 +1710,7 @@ TEST_CASE("cachestore.namespaces")
using namespace testutils;
const auto CreateCacheValue = [](size_t Size) -> CbObject {
- std::vector<uint8_t> Buf;
+ eastl::vector<uint8_t> Buf;
Buf.resize(Size);
CbObjectWriter Writer;
@@ -1778,7 +1778,7 @@ TEST_CASE("cachestore.drop.bucket")
using namespace testutils;
const auto CreateCacheValue = [](size_t Size) -> CbObject {
- std::vector<uint8_t> Buf;
+ eastl::vector<uint8_t> Buf;
Buf.resize(Size);
CbObjectWriter Writer;
@@ -1855,7 +1855,7 @@ TEST_CASE("cachestore.drop.namespace")
const CacheRequestContext Context;
const auto CreateCacheValue = [](size_t Size) -> CbObject {
- std::vector<uint8_t> Buf;
+ eastl::vector<uint8_t> Buf;
Buf.resize(Size);
CbObjectWriter Writer;
@@ -1945,7 +1945,7 @@ TEST_CASE("cachestore.blocked.disklayer.put")
GcStorageSize CacheSize;
const auto CreateCacheValue = [](size_t Size) -> CbObject {
- std::vector<uint8_t> Buf;
+ eastl::vector<uint8_t> Buf;
Buf.resize(Size, Size & 0xff);
CbObjectWriter Writer;
@@ -1999,11 +1999,11 @@ TEST_CASE("cachestore.scrub")
struct CacheRecord
{
- IoBuffer Record;
- std::vector<CompressedBuffer> Attachments;
+ IoBuffer Record;
+ eastl::vector<CompressedBuffer> Attachments;
};
- auto CreateCacheRecord = [](bool Structured, std::string_view Bucket, const IoHash& Key, const std::vector<size_t>& AttachmentSizes) {
+ auto CreateCacheRecord = [](bool Structured, std::string_view Bucket, const IoHash& Key, const eastl::vector<size_t>& AttachmentSizes) {
CacheRecord Result;
if (Structured)
{
@@ -2054,27 +2054,29 @@ TEST_CASE("cachestore.scrub")
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
CidStore.Initialize(CidConfig);
- auto CreateRecords =
- [&](bool IsStructured, std::string_view BucketName, const std::vector<IoHash>& Cids, const std::vector<size_t>& AttachmentSizes) {
- for (const IoHash& Cid : Cids)
+ auto CreateRecords = [&](bool IsStructured,
+ std::string_view BucketName,
+ const eastl::vector<IoHash>& Cids,
+ const eastl::vector<size_t>& AttachmentSizes) {
+ for (const IoHash& Cid : Cids)
+ {
+ CacheRecord Record = CreateCacheRecord(IsStructured, BucketName, Cid, AttachmentSizes);
+ eastl::vector<IoHash> AttachmentHashes;
+ for (const CompressedBuffer& Attachment : Record.Attachments)
{
- CacheRecord Record = CreateCacheRecord(IsStructured, BucketName, Cid, AttachmentSizes);
- std::vector<IoHash> AttachmentHashes;
- for (const CompressedBuffer& Attachment : Record.Attachments)
- {
- AttachmentHashes.push_back(Attachment.DecodeRawHash());
- CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), AttachmentHashes.back());
- }
- Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes);
+ AttachmentHashes.push_back(Attachment.DecodeRawHash());
+ CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), AttachmentHashes.back());
}
- };
+ Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes);
+ }
+ };
- std::vector<size_t> AttachmentSizes = {16, 1000, 2000, 4000, 8000, 64000, 80000};
+ eastl::vector<size_t> AttachmentSizes = {16, 1000, 2000, 4000, 8000, 64000, 80000};
- std::vector<IoHash> UnstructuredCids{CreateKey(4), CreateKey(5), CreateKey(6)};
+ eastl::vector<IoHash> UnstructuredCids{CreateKey(4), CreateKey(5), CreateKey(6)};
CreateRecords(false, "mybucket"sv, UnstructuredCids, AttachmentSizes);
- std::vector<IoHash> StructuredCids{CreateKey(1), CreateKey(2), CreateKey(3)};
+ eastl::vector<IoHash> StructuredCids{CreateKey(1), CreateKey(2), CreateKey(3)};
CreateRecords(true, "mybucket"sv, StructuredCids, AttachmentSizes);
WorkerThreadPool ThreadPool{1};
@@ -2095,15 +2097,15 @@ TEST_CASE("cachestore.newgc.basics")
struct CacheEntry
{
- IoBuffer Data;
- std::vector<std::pair<Oid, CompressedBuffer>> Attachments;
+ IoBuffer Data;
+ eastl::vector<std::pair<Oid, CompressedBuffer>> Attachments;
};
std::unordered_map<IoHash, CacheEntry> CacheEntries;
auto CreateCacheRecord =
[&](ZenCacheNamespace& Zcs, CidStore& CidStore, std::string_view Bucket, std::span<std::pair<Oid, CompressedBuffer>> Attachments) {
- std::vector<IoHash> AttachmentKeys;
+ eastl::vector<IoHash> AttachmentKeys;
for (const auto& Attachment : Attachments)
{
AttachmentKeys.push_back(Attachment.second.DecodeRawHash());
@@ -2194,8 +2196,8 @@ TEST_CASE("cachestore.newgc.basics")
return true;
};
- std::vector<IoHash> CacheRecords;
- std::vector<IoHash> UnstructuredCacheValues;
+ eastl::vector<IoHash> CacheRecords;
+ eastl::vector<IoHash> UnstructuredCacheValues;
const auto TearDrinkerBucket = "teardrinker"sv;
{
@@ -2207,11 +2209,11 @@ TEST_CASE("cachestore.newgc.basics")
// Create some basic data
{
// Structured record with attachments
- auto Attachments1 = CreateCompressedAttachment(CidStore, std::vector<size_t>{77, 1024 * 1024 * 2, 99, 1024 * 1024 * 2 + 87});
+ auto Attachments1 = CreateCompressedAttachment(CidStore, eastl::vector<size_t>{77, 1024 * 1024 * 2, 99, 1024 * 1024 * 2 + 87});
CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments1));
// Structured record with reuse of attachments
- auto Attachments2 = CreateCompressedAttachment(CidStore, std::vector<size_t>{971});
+ auto Attachments2 = CreateCompressedAttachment(CidStore, eastl::vector<size_t>{971});
Attachments2.push_back(Attachments1[0]);
Attachments2.push_back(Attachments1[1]);
CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments2));
@@ -2625,7 +2627,7 @@ TEST_CASE("cachestore.newgc.basics")
CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
auto Attachments =
- CreateCompressedAttachment(CidStore, std::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187});
+ CreateCompressedAttachment(CidStore, eastl::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187});
IoHash CacheRecord = CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments);
{
// Do get so it ends up in memcache