diff options
| author | Stefan Boberg <[email protected]> | 2025-03-06 17:27:59 +0100 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2025-03-06 17:27:59 +0100 |
| commit | 66e5d1f4e288e0c32f854ebe3b63584b42b83554 (patch) | |
| tree | d67e9d358419b5baccd429d54988414e0d7cd7a6 /src/zenstore/cache/structuredcachestore.cpp | |
| parent | reduced memory churn using fixed_xxx containers (#236) (diff) | |
| download | zen-66e5d1f4e288e0c32f854ebe3b63584b42b83554.tar.xz zen-66e5d1f4e288e0c32f854ebe3b63584b42b83554.zip | |
switched std::vector -> eastl::vector
Diffstat (limited to 'src/zenstore/cache/structuredcachestore.cpp')
| -rw-r--r-- | src/zenstore/cache/structuredcachestore.cpp | 134 |
1 files changed, 68 insertions, 66 deletions
diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 7d277329e..87c95f8a2 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -154,7 +154,7 @@ struct ZenCacheNamespace::PutBatchHandle }; ZenCacheNamespace::PutBatchHandle* -ZenCacheNamespace::BeginPutBatch(std::vector<bool>& OutResult) +ZenCacheNamespace::BeginPutBatch(eastl::vector<bool>& OutResult) { ZenCacheNamespace::PutBatchHandle* Handle = new ZenCacheNamespace::PutBatchHandle; Handle->DiskLayerHandle = m_DiskLayer.BeginPutBatch(OutResult); @@ -366,7 +366,7 @@ ZenCacheNamespace::GetValueDetails(const std::string_view BucketFilter, const st return m_DiskLayer.GetValueDetails(BucketFilter, ValueFilter); } -std::vector<RwLock::SharedLockScope> +eastl::vector<RwLock::SharedLockScope> ZenCacheNamespace::GetGcReferencerLocks() { return m_DiskLayer.GetGcReferencerLocks(); @@ -426,7 +426,7 @@ ZenCacheStore::ZenCacheStore(GcManager& Gc, DirectoryContent DirContent; GetDirectoryContent(m_BasePath, DirectoryContentFlags::IncludeDirs, DirContent); - std::vector<std::string> Namespaces; + eastl::vector<std::string> Namespaces; for (const std::filesystem::path& DirPath : DirContent.Directories) { std::string DirName = PathToUtf8(DirPath.filename()); @@ -480,7 +480,7 @@ ZenCacheStore::LogWorker() auto Log = [&ZCacheLog]() -> LoggerRef { return ZCacheLog; }; - std::vector<AccessLogItem> Items; + eastl::vector<AccessLogItem> Items; while (true) { try @@ -553,7 +553,7 @@ ZenCacheStore::LogWorker() } } -ZenCacheStore::PutBatch::PutBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, std::vector<bool>& OutResult) +ZenCacheStore::PutBatch::PutBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, eastl::vector<bool>& OutResult) : m_CacheStore(CacheStore) { ZEN_MEMSCOPE(GetCacheStoreTag()); @@ -904,10 +904,10 @@ ZenCacheStore::FindNamespace(std::string_view Namespace) const return nullptr; } -std::vector<std::string> +eastl::vector<std::string> ZenCacheStore::GetNamespaces() { - std::vector<std::string> Namespaces; + eastl::vector<std::string> Namespaces; IterateNamespaces([&](std::string_view Namespace, ZenCacheNamespace&) { Namespaces.push_back(std::string(Namespace)); }); @@ -917,7 +917,7 @@ ZenCacheStore::GetNamespaces() void ZenCacheStore::IterateNamespaces(const std::function<void(std::string_view Namespace, ZenCacheNamespace& Store)>& Callback) const { - std::vector<std::pair<std::string, ZenCacheNamespace&>> Namespaces; + eastl::vector<std::pair<std::string, ZenCacheNamespace&>> Namespaces; { RwLock::SharedLockScope _(m_NamespacesLock); Namespaces.reserve(m_Namespaces.size()); @@ -1042,18 +1042,18 @@ ZenCacheStore::GetBucketInfo(std::string_view NamespaceName, std::string_view Bu return {}; } -std::vector<RwLock::SharedLockScope> +eastl::vector<RwLock::SharedLockScope> ZenCacheStore::LockState(GcCtx& Ctx) { ZEN_TRACE_CPU("CacheStore::LockState"); auto Log = [&Ctx]() { return Ctx.Logger; }; - std::vector<RwLock::SharedLockScope> Locks; + eastl::vector<RwLock::SharedLockScope> Locks; Locks.emplace_back(RwLock::SharedLockScope(m_NamespacesLock)); for (auto& NamespaceIt : m_Namespaces) { - std::vector<RwLock::SharedLockScope> NamespaceLocks = NamespaceIt.second->GetGcReferencerLocks(); + eastl::vector<RwLock::SharedLockScope> NamespaceLocks = NamespaceIt.second->GetGcReferencerLocks(); for (auto It = std::make_move_iterator(NamespaceLocks.begin()); It != std::make_move_iterator(NamespaceLocks.end()); It++) { Locks.emplace_back(std::move(*It)); @@ -1065,12 +1065,12 @@ ZenCacheStore::LockState(GcCtx& Ctx) void ZenCacheStore::EnableUpdateCapture() { - std::vector<ZenCacheNamespace*> Namespaces; + eastl::vector<ZenCacheNamespace*> Namespaces; m_NamespacesLock.WithExclusiveLock([&]() { if (m_UpdateCaptureRefCounter == 0) { ZEN_ASSERT(!m_CapturedNamespaces); - m_CapturedNamespaces = std::make_unique<std::vector<std::string>>(); + m_CapturedNamespaces = std::make_unique<eastl::vector<std::string>>(); } else { @@ -1093,7 +1093,7 @@ ZenCacheStore::EnableUpdateCapture() void ZenCacheStore::DisableUpdateCapture() { - std::vector<ZenCacheNamespace*> Namespaces; + eastl::vector<ZenCacheNamespace*> Namespaces; m_NamespacesLock.WithExclusiveLock([&]() { ZEN_ASSERT(m_CapturedNamespaces); ZEN_ASSERT(m_UpdateCaptureRefCounter > 0); @@ -1114,7 +1114,7 @@ ZenCacheStore::DisableUpdateCapture() } } -std::vector<std::string> +eastl::vector<std::string> ZenCacheStore::GetCapturedNamespacesLocked() { if (m_CapturedNamespaces) @@ -1173,7 +1173,7 @@ public: Stopwatch Timer; - std::vector<ZenCacheDiskLayer::CacheBucket*> AddedBuckets; + eastl::vector<ZenCacheDiskLayer::CacheBucket*> AddedBuckets; const auto _ = MakeGuard([&] { if (!Ctx.Settings.Verbose) @@ -1187,7 +1187,7 @@ public: AddedBuckets.size()); }); - std::vector<std::string> AddedNamespaces = m_CacheStore.GetCapturedNamespacesLocked(); + eastl::vector<std::string> AddedNamespaces = m_CacheStore.GetCapturedNamespacesLocked(); for (const std::string& AddedNamespace : AddedNamespaces) { @@ -1202,8 +1202,8 @@ public: } for (auto& NamepaceKV : m_CacheStore.m_Namespaces) { - ZenCacheNamespace& Namespace = *NamepaceKV.second; - std::vector<std::string> NamespaceAddedBuckets = Namespace.m_DiskLayer.GetCapturedBucketsLocked(); + ZenCacheNamespace& Namespace = *NamepaceKV.second; + eastl::vector<std::string> NamespaceAddedBuckets = Namespace.m_DiskLayer.GetCapturedBucketsLocked(); for (const std::string& AddedBucketName : NamespaceAddedBuckets) { if (auto It = Namespace.m_DiskLayer.m_Buckets.find(AddedBucketName); It != Namespace.m_DiskLayer.m_Buckets.end()) @@ -1258,31 +1258,31 @@ public: } private: - ZenCacheStore& m_CacheStore; - std::vector<IoHash> m_References; + ZenCacheStore& m_CacheStore; + eastl::vector<IoHash> m_References; }; -std::vector<GcReferenceChecker*> +eastl::vector<GcReferenceChecker*> ZenCacheStore::CreateReferenceCheckers(GcCtx& Ctx) { ZEN_TRACE_CPU("CacheStore::CreateReferenceCheckers"); auto Log = [&Ctx]() { return Ctx.Logger; }; - Stopwatch Timer; - const auto _ = MakeGuard([&] { - if (!Ctx.Settings.Verbose) - { - return; - } - ZEN_INFO("GCV2: cachestore [CREATE CHECKERS] '{}': completed in {}", m_BasePath, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }); - std::vector<GcReferenceChecker*> Checkers; + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: cachestore [CREATE CHECKERS] '{}': completed in {}", m_BasePath, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + eastl::vector<GcReferenceChecker*> Checkers; Checkers.emplace_back(new CacheStoreReferenceChecker(*this)); return Checkers; } -std::vector<GcReferenceValidator*> +eastl::vector<GcReferenceValidator*> ZenCacheStore::CreateReferenceValidators(GcCtx& /*Ctx*/) { return {}; @@ -1307,9 +1307,9 @@ namespace testutils { std::pair<Oid, IoBuffer> CreateBinaryBlob(size_t Size) { return {Oid::NewOid(), CreateRandomBlob(Size)}; } - std::vector<std::pair<Oid, CompressedBuffer>> CreateCompressedAttachment(CidStore& Store, const std::span<const size_t>& Sizes) + eastl::vector<std::pair<Oid, CompressedBuffer>> CreateCompressedAttachment(CidStore& Store, const std::span<const size_t>& Sizes) { - std::vector<std::pair<Oid, CompressedBuffer>> Result; + eastl::vector<std::pair<Oid, CompressedBuffer>> Result; Result.reserve(Sizes.size()); for (size_t Size : Sizes) { @@ -1386,7 +1386,7 @@ TEST_CASE("cachestore.size") auto JobQueue = MakeJobQueue(1, "testqueue"); const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size); CbObjectWriter Writer; @@ -1410,7 +1410,7 @@ TEST_CASE("cachestore.size") IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer(); Buffer.SetContentType(ZenContentType::kCbObject); - std::vector<std::pair<std::string, IoHash>> Keys; + eastl::vector<std::pair<std::string, IoHash>> Keys; for (size_t Key = 0; Key < Count; ++Key) { @@ -1710,7 +1710,7 @@ TEST_CASE("cachestore.namespaces") using namespace testutils; const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size); CbObjectWriter Writer; @@ -1778,7 +1778,7 @@ TEST_CASE("cachestore.drop.bucket") using namespace testutils; const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size); CbObjectWriter Writer; @@ -1855,7 +1855,7 @@ TEST_CASE("cachestore.drop.namespace") const CacheRequestContext Context; const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size); CbObjectWriter Writer; @@ -1945,7 +1945,7 @@ TEST_CASE("cachestore.blocked.disklayer.put") GcStorageSize CacheSize; const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size, Size & 0xff); CbObjectWriter Writer; @@ -1999,11 +1999,11 @@ TEST_CASE("cachestore.scrub") struct CacheRecord { - IoBuffer Record; - std::vector<CompressedBuffer> Attachments; + IoBuffer Record; + eastl::vector<CompressedBuffer> Attachments; }; - auto CreateCacheRecord = [](bool Structured, std::string_view Bucket, const IoHash& Key, const std::vector<size_t>& AttachmentSizes) { + auto CreateCacheRecord = [](bool Structured, std::string_view Bucket, const IoHash& Key, const eastl::vector<size_t>& AttachmentSizes) { CacheRecord Result; if (Structured) { @@ -2054,27 +2054,29 @@ TEST_CASE("cachestore.scrub") CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; CidStore.Initialize(CidConfig); - auto CreateRecords = - [&](bool IsStructured, std::string_view BucketName, const std::vector<IoHash>& Cids, const std::vector<size_t>& AttachmentSizes) { - for (const IoHash& Cid : Cids) + auto CreateRecords = [&](bool IsStructured, + std::string_view BucketName, + const eastl::vector<IoHash>& Cids, + const eastl::vector<size_t>& AttachmentSizes) { + for (const IoHash& Cid : Cids) + { + CacheRecord Record = CreateCacheRecord(IsStructured, BucketName, Cid, AttachmentSizes); + eastl::vector<IoHash> AttachmentHashes; + for (const CompressedBuffer& Attachment : Record.Attachments) { - CacheRecord Record = CreateCacheRecord(IsStructured, BucketName, Cid, AttachmentSizes); - std::vector<IoHash> AttachmentHashes; - for (const CompressedBuffer& Attachment : Record.Attachments) - { - AttachmentHashes.push_back(Attachment.DecodeRawHash()); - CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), AttachmentHashes.back()); - } - Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes); + AttachmentHashes.push_back(Attachment.DecodeRawHash()); + CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), AttachmentHashes.back()); } - }; + Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes); + } + }; - std::vector<size_t> AttachmentSizes = {16, 1000, 2000, 4000, 8000, 64000, 80000}; + eastl::vector<size_t> AttachmentSizes = {16, 1000, 2000, 4000, 8000, 64000, 80000}; - std::vector<IoHash> UnstructuredCids{CreateKey(4), CreateKey(5), CreateKey(6)}; + eastl::vector<IoHash> UnstructuredCids{CreateKey(4), CreateKey(5), CreateKey(6)}; CreateRecords(false, "mybucket"sv, UnstructuredCids, AttachmentSizes); - std::vector<IoHash> StructuredCids{CreateKey(1), CreateKey(2), CreateKey(3)}; + eastl::vector<IoHash> StructuredCids{CreateKey(1), CreateKey(2), CreateKey(3)}; CreateRecords(true, "mybucket"sv, StructuredCids, AttachmentSizes); WorkerThreadPool ThreadPool{1}; @@ -2095,15 +2097,15 @@ TEST_CASE("cachestore.newgc.basics") struct CacheEntry { - IoBuffer Data; - std::vector<std::pair<Oid, CompressedBuffer>> Attachments; + IoBuffer Data; + eastl::vector<std::pair<Oid, CompressedBuffer>> Attachments; }; std::unordered_map<IoHash, CacheEntry> CacheEntries; auto CreateCacheRecord = [&](ZenCacheNamespace& Zcs, CidStore& CidStore, std::string_view Bucket, std::span<std::pair<Oid, CompressedBuffer>> Attachments) { - std::vector<IoHash> AttachmentKeys; + eastl::vector<IoHash> AttachmentKeys; for (const auto& Attachment : Attachments) { AttachmentKeys.push_back(Attachment.second.DecodeRawHash()); @@ -2194,8 +2196,8 @@ TEST_CASE("cachestore.newgc.basics") return true; }; - std::vector<IoHash> CacheRecords; - std::vector<IoHash> UnstructuredCacheValues; + eastl::vector<IoHash> CacheRecords; + eastl::vector<IoHash> UnstructuredCacheValues; const auto TearDrinkerBucket = "teardrinker"sv; { @@ -2207,11 +2209,11 @@ TEST_CASE("cachestore.newgc.basics") // Create some basic data { // Structured record with attachments - auto Attachments1 = CreateCompressedAttachment(CidStore, std::vector<size_t>{77, 1024 * 1024 * 2, 99, 1024 * 1024 * 2 + 87}); + auto Attachments1 = CreateCompressedAttachment(CidStore, eastl::vector<size_t>{77, 1024 * 1024 * 2, 99, 1024 * 1024 * 2 + 87}); CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments1)); // Structured record with reuse of attachments - auto Attachments2 = CreateCompressedAttachment(CidStore, std::vector<size_t>{971}); + auto Attachments2 = CreateCompressedAttachment(CidStore, eastl::vector<size_t>{971}); Attachments2.push_back(Attachments1[0]); Attachments2.push_back(Attachments1[1]); CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments2)); @@ -2625,7 +2627,7 @@ TEST_CASE("cachestore.newgc.basics") CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); auto Attachments = - CreateCompressedAttachment(CidStore, std::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187}); + CreateCompressedAttachment(CidStore, eastl::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187}); IoHash CacheRecord = CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments); { // Do get so it ends up in memcache |