aboutsummaryrefslogtreecommitdiff
path: root/src/zenstore/cache/structuredcachestore.cpp
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2025-03-06 17:27:59 +0100
committerStefan Boberg <[email protected]>2025-03-06 17:27:59 +0100
commit66e5d1f4e288e0c32f854ebe3b63584b42b83554 (patch)
treed67e9d358419b5baccd429d54988414e0d7cd7a6 /src/zenstore/cache/structuredcachestore.cpp
parentreduced memory churn using fixed_xxx containers (#236) (diff)
downloadzen-66e5d1f4e288e0c32f854ebe3b63584b42b83554.tar.xz
zen-66e5d1f4e288e0c32f854ebe3b63584b42b83554.zip
switched std::vector -> eastl::vector
Diffstat (limited to 'src/zenstore/cache/structuredcachestore.cpp')
-rw-r--r--src/zenstore/cache/structuredcachestore.cpp134
1 files changed, 68 insertions, 66 deletions
diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index 7d277329e..87c95f8a2 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -154,7 +154,7 @@ struct ZenCacheNamespace::PutBatchHandle
};
ZenCacheNamespace::PutBatchHandle*
-ZenCacheNamespace::BeginPutBatch(std::vector<bool>& OutResult)
+ZenCacheNamespace::BeginPutBatch(eastl::vector<bool>& OutResult)
{
ZenCacheNamespace::PutBatchHandle* Handle = new ZenCacheNamespace::PutBatchHandle;
Handle->DiskLayerHandle = m_DiskLayer.BeginPutBatch(OutResult);
@@ -366,7 +366,7 @@ ZenCacheNamespace::GetValueDetails(const std::string_view BucketFilter, const st
return m_DiskLayer.GetValueDetails(BucketFilter, ValueFilter);
}
-std::vector<RwLock::SharedLockScope>
+eastl::vector<RwLock::SharedLockScope>
ZenCacheNamespace::GetGcReferencerLocks()
{
return m_DiskLayer.GetGcReferencerLocks();
@@ -426,7 +426,7 @@ ZenCacheStore::ZenCacheStore(GcManager& Gc,
DirectoryContent DirContent;
GetDirectoryContent(m_BasePath, DirectoryContentFlags::IncludeDirs, DirContent);
- std::vector<std::string> Namespaces;
+ eastl::vector<std::string> Namespaces;
for (const std::filesystem::path& DirPath : DirContent.Directories)
{
std::string DirName = PathToUtf8(DirPath.filename());
@@ -480,7 +480,7 @@ ZenCacheStore::LogWorker()
auto Log = [&ZCacheLog]() -> LoggerRef { return ZCacheLog; };
- std::vector<AccessLogItem> Items;
+ eastl::vector<AccessLogItem> Items;
while (true)
{
try
@@ -553,7 +553,7 @@ ZenCacheStore::LogWorker()
}
}
-ZenCacheStore::PutBatch::PutBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, std::vector<bool>& OutResult)
+ZenCacheStore::PutBatch::PutBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, eastl::vector<bool>& OutResult)
: m_CacheStore(CacheStore)
{
ZEN_MEMSCOPE(GetCacheStoreTag());
@@ -904,10 +904,10 @@ ZenCacheStore::FindNamespace(std::string_view Namespace) const
return nullptr;
}
-std::vector<std::string>
+eastl::vector<std::string>
ZenCacheStore::GetNamespaces()
{
- std::vector<std::string> Namespaces;
+ eastl::vector<std::string> Namespaces;
IterateNamespaces([&](std::string_view Namespace, ZenCacheNamespace&) { Namespaces.push_back(std::string(Namespace)); });
@@ -917,7 +917,7 @@ ZenCacheStore::GetNamespaces()
void
ZenCacheStore::IterateNamespaces(const std::function<void(std::string_view Namespace, ZenCacheNamespace& Store)>& Callback) const
{
- std::vector<std::pair<std::string, ZenCacheNamespace&>> Namespaces;
+ eastl::vector<std::pair<std::string, ZenCacheNamespace&>> Namespaces;
{
RwLock::SharedLockScope _(m_NamespacesLock);
Namespaces.reserve(m_Namespaces.size());
@@ -1042,18 +1042,18 @@ ZenCacheStore::GetBucketInfo(std::string_view NamespaceName, std::string_view Bu
return {};
}
-std::vector<RwLock::SharedLockScope>
+eastl::vector<RwLock::SharedLockScope>
ZenCacheStore::LockState(GcCtx& Ctx)
{
ZEN_TRACE_CPU("CacheStore::LockState");
auto Log = [&Ctx]() { return Ctx.Logger; };
- std::vector<RwLock::SharedLockScope> Locks;
+ eastl::vector<RwLock::SharedLockScope> Locks;
Locks.emplace_back(RwLock::SharedLockScope(m_NamespacesLock));
for (auto& NamespaceIt : m_Namespaces)
{
- std::vector<RwLock::SharedLockScope> NamespaceLocks = NamespaceIt.second->GetGcReferencerLocks();
+ eastl::vector<RwLock::SharedLockScope> NamespaceLocks = NamespaceIt.second->GetGcReferencerLocks();
for (auto It = std::make_move_iterator(NamespaceLocks.begin()); It != std::make_move_iterator(NamespaceLocks.end()); It++)
{
Locks.emplace_back(std::move(*It));
@@ -1065,12 +1065,12 @@ ZenCacheStore::LockState(GcCtx& Ctx)
void
ZenCacheStore::EnableUpdateCapture()
{
- std::vector<ZenCacheNamespace*> Namespaces;
+ eastl::vector<ZenCacheNamespace*> Namespaces;
m_NamespacesLock.WithExclusiveLock([&]() {
if (m_UpdateCaptureRefCounter == 0)
{
ZEN_ASSERT(!m_CapturedNamespaces);
- m_CapturedNamespaces = std::make_unique<std::vector<std::string>>();
+ m_CapturedNamespaces = std::make_unique<eastl::vector<std::string>>();
}
else
{
@@ -1093,7 +1093,7 @@ ZenCacheStore::EnableUpdateCapture()
void
ZenCacheStore::DisableUpdateCapture()
{
- std::vector<ZenCacheNamespace*> Namespaces;
+ eastl::vector<ZenCacheNamespace*> Namespaces;
m_NamespacesLock.WithExclusiveLock([&]() {
ZEN_ASSERT(m_CapturedNamespaces);
ZEN_ASSERT(m_UpdateCaptureRefCounter > 0);
@@ -1114,7 +1114,7 @@ ZenCacheStore::DisableUpdateCapture()
}
}
-std::vector<std::string>
+eastl::vector<std::string>
ZenCacheStore::GetCapturedNamespacesLocked()
{
if (m_CapturedNamespaces)
@@ -1173,7 +1173,7 @@ public:
Stopwatch Timer;
- std::vector<ZenCacheDiskLayer::CacheBucket*> AddedBuckets;
+ eastl::vector<ZenCacheDiskLayer::CacheBucket*> AddedBuckets;
const auto _ = MakeGuard([&] {
if (!Ctx.Settings.Verbose)
@@ -1187,7 +1187,7 @@ public:
AddedBuckets.size());
});
- std::vector<std::string> AddedNamespaces = m_CacheStore.GetCapturedNamespacesLocked();
+ eastl::vector<std::string> AddedNamespaces = m_CacheStore.GetCapturedNamespacesLocked();
for (const std::string& AddedNamespace : AddedNamespaces)
{
@@ -1202,8 +1202,8 @@ public:
}
for (auto& NamepaceKV : m_CacheStore.m_Namespaces)
{
- ZenCacheNamespace& Namespace = *NamepaceKV.second;
- std::vector<std::string> NamespaceAddedBuckets = Namespace.m_DiskLayer.GetCapturedBucketsLocked();
+ ZenCacheNamespace& Namespace = *NamepaceKV.second;
+ eastl::vector<std::string> NamespaceAddedBuckets = Namespace.m_DiskLayer.GetCapturedBucketsLocked();
for (const std::string& AddedBucketName : NamespaceAddedBuckets)
{
if (auto It = Namespace.m_DiskLayer.m_Buckets.find(AddedBucketName); It != Namespace.m_DiskLayer.m_Buckets.end())
@@ -1258,31 +1258,31 @@ public:
}
private:
- ZenCacheStore& m_CacheStore;
- std::vector<IoHash> m_References;
+ ZenCacheStore& m_CacheStore;
+ eastl::vector<IoHash> m_References;
};
-std::vector<GcReferenceChecker*>
+eastl::vector<GcReferenceChecker*>
ZenCacheStore::CreateReferenceCheckers(GcCtx& Ctx)
{
ZEN_TRACE_CPU("CacheStore::CreateReferenceCheckers");
auto Log = [&Ctx]() { return Ctx.Logger; };
- Stopwatch Timer;
- const auto _ = MakeGuard([&] {
- if (!Ctx.Settings.Verbose)
- {
- return;
- }
- ZEN_INFO("GCV2: cachestore [CREATE CHECKERS] '{}': completed in {}", m_BasePath, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
- });
- std::vector<GcReferenceChecker*> Checkers;
+ Stopwatch Timer;
+ const auto _ = MakeGuard([&] {
+ if (!Ctx.Settings.Verbose)
+ {
+ return;
+ }
+ ZEN_INFO("GCV2: cachestore [CREATE CHECKERS] '{}': completed in {}", m_BasePath, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ });
+ eastl::vector<GcReferenceChecker*> Checkers;
Checkers.emplace_back(new CacheStoreReferenceChecker(*this));
return Checkers;
}
-std::vector<GcReferenceValidator*>
+eastl::vector<GcReferenceValidator*>
ZenCacheStore::CreateReferenceValidators(GcCtx& /*Ctx*/)
{
return {};
@@ -1307,9 +1307,9 @@ namespace testutils {
std::pair<Oid, IoBuffer> CreateBinaryBlob(size_t Size) { return {Oid::NewOid(), CreateRandomBlob(Size)}; }
- std::vector<std::pair<Oid, CompressedBuffer>> CreateCompressedAttachment(CidStore& Store, const std::span<const size_t>& Sizes)
+ eastl::vector<std::pair<Oid, CompressedBuffer>> CreateCompressedAttachment(CidStore& Store, const std::span<const size_t>& Sizes)
{
- std::vector<std::pair<Oid, CompressedBuffer>> Result;
+ eastl::vector<std::pair<Oid, CompressedBuffer>> Result;
Result.reserve(Sizes.size());
for (size_t Size : Sizes)
{
@@ -1386,7 +1386,7 @@ TEST_CASE("cachestore.size")
auto JobQueue = MakeJobQueue(1, "testqueue");
const auto CreateCacheValue = [](size_t Size) -> CbObject {
- std::vector<uint8_t> Buf;
+ eastl::vector<uint8_t> Buf;
Buf.resize(Size);
CbObjectWriter Writer;
@@ -1410,7 +1410,7 @@ TEST_CASE("cachestore.size")
IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer();
Buffer.SetContentType(ZenContentType::kCbObject);
- std::vector<std::pair<std::string, IoHash>> Keys;
+ eastl::vector<std::pair<std::string, IoHash>> Keys;
for (size_t Key = 0; Key < Count; ++Key)
{
@@ -1710,7 +1710,7 @@ TEST_CASE("cachestore.namespaces")
using namespace testutils;
const auto CreateCacheValue = [](size_t Size) -> CbObject {
- std::vector<uint8_t> Buf;
+ eastl::vector<uint8_t> Buf;
Buf.resize(Size);
CbObjectWriter Writer;
@@ -1778,7 +1778,7 @@ TEST_CASE("cachestore.drop.bucket")
using namespace testutils;
const auto CreateCacheValue = [](size_t Size) -> CbObject {
- std::vector<uint8_t> Buf;
+ eastl::vector<uint8_t> Buf;
Buf.resize(Size);
CbObjectWriter Writer;
@@ -1855,7 +1855,7 @@ TEST_CASE("cachestore.drop.namespace")
const CacheRequestContext Context;
const auto CreateCacheValue = [](size_t Size) -> CbObject {
- std::vector<uint8_t> Buf;
+ eastl::vector<uint8_t> Buf;
Buf.resize(Size);
CbObjectWriter Writer;
@@ -1945,7 +1945,7 @@ TEST_CASE("cachestore.blocked.disklayer.put")
GcStorageSize CacheSize;
const auto CreateCacheValue = [](size_t Size) -> CbObject {
- std::vector<uint8_t> Buf;
+ eastl::vector<uint8_t> Buf;
Buf.resize(Size, Size & 0xff);
CbObjectWriter Writer;
@@ -1999,11 +1999,11 @@ TEST_CASE("cachestore.scrub")
struct CacheRecord
{
- IoBuffer Record;
- std::vector<CompressedBuffer> Attachments;
+ IoBuffer Record;
+ eastl::vector<CompressedBuffer> Attachments;
};
- auto CreateCacheRecord = [](bool Structured, std::string_view Bucket, const IoHash& Key, const std::vector<size_t>& AttachmentSizes) {
+ auto CreateCacheRecord = [](bool Structured, std::string_view Bucket, const IoHash& Key, const eastl::vector<size_t>& AttachmentSizes) {
CacheRecord Result;
if (Structured)
{
@@ -2054,27 +2054,29 @@ TEST_CASE("cachestore.scrub")
CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
CidStore.Initialize(CidConfig);
- auto CreateRecords =
- [&](bool IsStructured, std::string_view BucketName, const std::vector<IoHash>& Cids, const std::vector<size_t>& AttachmentSizes) {
- for (const IoHash& Cid : Cids)
+ auto CreateRecords = [&](bool IsStructured,
+ std::string_view BucketName,
+ const eastl::vector<IoHash>& Cids,
+ const eastl::vector<size_t>& AttachmentSizes) {
+ for (const IoHash& Cid : Cids)
+ {
+ CacheRecord Record = CreateCacheRecord(IsStructured, BucketName, Cid, AttachmentSizes);
+ eastl::vector<IoHash> AttachmentHashes;
+ for (const CompressedBuffer& Attachment : Record.Attachments)
{
- CacheRecord Record = CreateCacheRecord(IsStructured, BucketName, Cid, AttachmentSizes);
- std::vector<IoHash> AttachmentHashes;
- for (const CompressedBuffer& Attachment : Record.Attachments)
- {
- AttachmentHashes.push_back(Attachment.DecodeRawHash());
- CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), AttachmentHashes.back());
- }
- Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes);
+ AttachmentHashes.push_back(Attachment.DecodeRawHash());
+ CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), AttachmentHashes.back());
}
- };
+ Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes);
+ }
+ };
- std::vector<size_t> AttachmentSizes = {16, 1000, 2000, 4000, 8000, 64000, 80000};
+ eastl::vector<size_t> AttachmentSizes = {16, 1000, 2000, 4000, 8000, 64000, 80000};
- std::vector<IoHash> UnstructuredCids{CreateKey(4), CreateKey(5), CreateKey(6)};
+ eastl::vector<IoHash> UnstructuredCids{CreateKey(4), CreateKey(5), CreateKey(6)};
CreateRecords(false, "mybucket"sv, UnstructuredCids, AttachmentSizes);
- std::vector<IoHash> StructuredCids{CreateKey(1), CreateKey(2), CreateKey(3)};
+ eastl::vector<IoHash> StructuredCids{CreateKey(1), CreateKey(2), CreateKey(3)};
CreateRecords(true, "mybucket"sv, StructuredCids, AttachmentSizes);
WorkerThreadPool ThreadPool{1};
@@ -2095,15 +2097,15 @@ TEST_CASE("cachestore.newgc.basics")
struct CacheEntry
{
- IoBuffer Data;
- std::vector<std::pair<Oid, CompressedBuffer>> Attachments;
+ IoBuffer Data;
+ eastl::vector<std::pair<Oid, CompressedBuffer>> Attachments;
};
std::unordered_map<IoHash, CacheEntry> CacheEntries;
auto CreateCacheRecord =
[&](ZenCacheNamespace& Zcs, CidStore& CidStore, std::string_view Bucket, std::span<std::pair<Oid, CompressedBuffer>> Attachments) {
- std::vector<IoHash> AttachmentKeys;
+ eastl::vector<IoHash> AttachmentKeys;
for (const auto& Attachment : Attachments)
{
AttachmentKeys.push_back(Attachment.second.DecodeRawHash());
@@ -2194,8 +2196,8 @@ TEST_CASE("cachestore.newgc.basics")
return true;
};
- std::vector<IoHash> CacheRecords;
- std::vector<IoHash> UnstructuredCacheValues;
+ eastl::vector<IoHash> CacheRecords;
+ eastl::vector<IoHash> UnstructuredCacheValues;
const auto TearDrinkerBucket = "teardrinker"sv;
{
@@ -2207,11 +2209,11 @@ TEST_CASE("cachestore.newgc.basics")
// Create some basic data
{
// Structured record with attachments
- auto Attachments1 = CreateCompressedAttachment(CidStore, std::vector<size_t>{77, 1024 * 1024 * 2, 99, 1024 * 1024 * 2 + 87});
+ auto Attachments1 = CreateCompressedAttachment(CidStore, eastl::vector<size_t>{77, 1024 * 1024 * 2, 99, 1024 * 1024 * 2 + 87});
CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments1));
// Structured record with reuse of attachments
- auto Attachments2 = CreateCompressedAttachment(CidStore, std::vector<size_t>{971});
+ auto Attachments2 = CreateCompressedAttachment(CidStore, eastl::vector<size_t>{971});
Attachments2.push_back(Attachments1[0]);
Attachments2.push_back(Attachments1[1]);
CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments2));
@@ -2625,7 +2627,7 @@ TEST_CASE("cachestore.newgc.basics")
CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
auto Attachments =
- CreateCompressedAttachment(CidStore, std::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187});
+ CreateCompressedAttachment(CidStore, eastl::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187});
IoHash CacheRecord = CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments);
{
// Do get so it ends up in memcache