diff options
| author | Stefan Boberg <[email protected]> | 2025-03-06 17:27:59 +0100 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2025-03-06 17:27:59 +0100 |
| commit | 66e5d1f4e288e0c32f854ebe3b63584b42b83554 (patch) | |
| tree | d67e9d358419b5baccd429d54988414e0d7cd7a6 /src/zenstore | |
| parent | reduced memory churn using fixed_xxx containers (#236) (diff) | |
| download | zen-66e5d1f4e288e0c32f854ebe3b63584b42b83554.tar.xz zen-66e5d1f4e288e0c32f854ebe3b63584b42b83554.zip | |
switched std::vector -> eastl::vector
Diffstat (limited to 'src/zenstore')
23 files changed, 894 insertions, 889 deletions
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index e976c061d..7df9117db 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -293,8 +293,8 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max if (std::filesystem::is_directory(m_BlocksBasePath)) { - uint32_t NextBlockIndex = 0; - std::vector<std::filesystem::path> FoldersToScan; + uint32_t NextBlockIndex = 0; + eastl::vector<std::filesystem::path> FoldersToScan; FoldersToScan.push_back(m_BlocksBasePath); size_t FolderOffset = 0; while (FolderOffset < FoldersToScan.size()) @@ -399,7 +399,7 @@ BlockStore::GetBlocksToCompact(const BlockUsageMap& BlockUsage, uint32_t BlockUs const uint64_t SmallBlockLimit = m_MaxBlockSize / 2; - std::vector<uint32_t> SmallBlockIndexes; + eastl::vector<uint32_t> SmallBlockIndexes; for (const auto& It : m_ChunkBlocks) { @@ -598,9 +598,9 @@ BlockStore::WriteChunks(std::span<IoBuffer> Datas, uint32_t Alignment, const Wri LargestSize = Max(LargestSize, Size); } - const uint64_t MinSize = Max(LargestSize, 8u * 1024u * 1024u); - const uint64_t BufferSize = Min(TotalSize, MinSize); - std::vector<uint8_t> Buffer(BufferSize); + const uint64_t MinSize = Max(LargestSize, 8u * 1024u * 1024u); + const uint64_t BufferSize = Min(TotalSize, MinSize); + eastl::vector<uint8_t> Buffer(BufferSize); size_t Offset = 0; while (Offset < TotalCount) @@ -660,8 +660,8 @@ BlockStore::WriteChunks(std::span<IoBuffer> Datas, uint32_t Alignment, const Wri m_TotalSize.fetch_add(RangeSize, std::memory_order::relaxed); - uint32_t ChunkOffset = AlignedInsertOffset; - std::vector<BlockStoreLocation> Locations(Count); + uint32_t ChunkOffset = AlignedInsertOffset; + eastl::vector<BlockStoreLocation> Locations(Count); for (size_t Index = 0; Index < Count; Index++) { uint32_t ChunkSize = gsl::narrow<uint32_t>(Datas[Offset + Index].GetSize()); @@ -746,8 +746,8 @@ BlockStore::IterateBlock(std::span<const BlockStoreLocation> ChunkLocations, IterateSmallChunkWindowSize = Min((LargeSizeLimit + IterateSmallChunkMaxGapSize) * ChunkLocations.size(), IterateSmallChunkWindowSize); - uint32_t BlockIndex = ChunkLocations[InChunkIndexes[0]].BlockIndex; - std::vector<size_t> ChunkIndexes(InChunkIndexes.begin(), InChunkIndexes.end()); + uint32_t BlockIndex = ChunkLocations[InChunkIndexes[0]].BlockIndex; + eastl::vector<size_t> ChunkIndexes(InChunkIndexes.begin(), InChunkIndexes.end()); std::sort(ChunkIndexes.begin(), ChunkIndexes.end(), [&](size_t IndexA, size_t IndexB) -> bool { return ChunkLocations[IndexA].Offset < ChunkLocations[IndexB].Offset; }); @@ -894,7 +894,7 @@ BlockStore::IterateChunks(const std::span<const BlockStoreLocation>& ChunkLocati ZEN_LOG_SCOPE("iterating chunks from '{}'", m_BlocksBasePath); - std::vector<size_t> ChunkOrder(ChunkLocations.size()); + eastl::vector<size_t> ChunkOrder(ChunkLocations.size()); for (size_t ChunkIndex = 0; ChunkIndex < ChunkLocations.size(); ++ChunkIndex) { ChunkOrder[ChunkIndex] = ChunkIndex; @@ -998,11 +998,11 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, return Continue; }; - std::vector<uint32_t> RemovedBlocks; + eastl::vector<uint32_t> RemovedBlocks; - CompactState.IterateBlocks([&](uint32_t BlockIndex, - const std::vector<size_t>& KeepChunkIndexes, - const std::vector<BlockStoreLocation>& ChunkLocations) -> bool { + CompactState.IterateBlocks([&](uint32_t BlockIndex, + const eastl::vector<size_t>& KeepChunkIndexes, + const eastl::vector<BlockStoreLocation>& ChunkLocations) -> bool { ZEN_TRACE_CPU("BlockStore::CompactBlock"); Ref<BlockStoreFile> OldBlockFile; { @@ -1047,15 +1047,15 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, } else { - std::vector<size_t> SortedChunkIndexes(KeepChunkIndexes); + eastl::vector<size_t> SortedChunkIndexes(KeepChunkIndexes); std::sort(SortedChunkIndexes.begin(), SortedChunkIndexes.end(), [&ChunkLocations](size_t Lhs, size_t Rhs) { return ChunkLocations[Lhs].Offset < ChunkLocations[Rhs].Offset; }); BasicFileBuffer SourceFileBuffer(OldBlockFile->GetBasicFile(), Min(65536u, OldBlockSize)); - uint64_t WrittenBytesToBlock = 0; - uint64_t MovedFromBlock = 0; - std::vector<uint8_t> Chunk; + uint64_t WrittenBytesToBlock = 0; + uint64_t MovedFromBlock = 0; + eastl::vector<uint8_t> Chunk; for (const size_t& ChunkIndex : SortedChunkIndexes) { const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex]; @@ -1424,14 +1424,14 @@ namespace blockstore::impl { return AsString; }; - std::vector<std::filesystem::path> GetDirectoryContent(std::filesystem::path RootDir, bool Files, bool Directories) + eastl::vector<std::filesystem::path> GetDirectoryContent(std::filesystem::path RootDir, bool Files, bool Directories) { DirectoryContent DirectoryContent; GetDirectoryContent(RootDir, DirectoryContentFlags::Recursive | (Files ? DirectoryContentFlags::IncludeFiles : DirectoryContentFlags::None) | (Directories ? DirectoryContentFlags::IncludeDirs : DirectoryContentFlags::None), DirectoryContent); - std::vector<std::filesystem::path> Result; + eastl::vector<std::filesystem::path> Result; Result.insert(Result.end(), DirectoryContent.Directories.begin(), DirectoryContent.Directories.end()); Result.insert(Result.end(), DirectoryContent.Files.begin(), DirectoryContent.Files.end()); return Result; @@ -1449,8 +1449,8 @@ TEST_CASE("blockstore.multichunks") BlockStore Store; Store.Initialize(RootDirectory, 128, 1024); - std::vector<IoBuffer> MultiChunkData; - std::string FirstChunkData = "0123456789012345678901234567890123456789012345678901234567890123"; + eastl::vector<IoBuffer> MultiChunkData; + std::string FirstChunkData = "0123456789012345678901234567890123456789012345678901234567890123"; MultiChunkData.push_back(IoBuffer(IoBuffer::Wrap, FirstChunkData.data(), FirstChunkData.size())); std::string SecondChunkData = "12345678901234567890123456789012345678901234567890123456"; @@ -1588,16 +1588,16 @@ TEST_CASE("blockstore.iterate.chunks") WorkerThreadPool WorkerPool(4); - std::vector<BlockStoreLocation> Locations{FirstChunkLocation, - SecondChunkLocation, - VeryLargeChunkLocation, - BadLocationZeroSize, - BadLocationOutOfRange, - BadBlockIndex}; - Latch WorkLatch(1); + eastl::vector<BlockStoreLocation> Locations{FirstChunkLocation, + SecondChunkLocation, + VeryLargeChunkLocation, + BadLocationZeroSize, + BadLocationOutOfRange, + BadBlockIndex}; + Latch WorkLatch(1); Store.IterateChunks(Locations, [&](uint32_t, std::span<const size_t> ChunkIndexes) -> bool { WorkLatch.AddCount(1); - WorkerPool.ScheduleWork([&, ChunkIndexes = std::vector<size_t>(ChunkIndexes.begin(), ChunkIndexes.end())]() { + WorkerPool.ScheduleWork([&, ChunkIndexes = eastl::vector<size_t>(ChunkIndexes.begin(), ChunkIndexes.end())]() { auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); bool Continue = Store.IterateBlock( Locations, @@ -1687,10 +1687,10 @@ TEST_CASE("blockstore.thread.read.write") BlockStore Store; Store.Initialize(RootDirectory / "store", 1088, 1024); - constexpr size_t ChunkCount = 1000; - constexpr size_t Alignment = 8; - std::vector<IoBuffer> Chunks; - std::vector<IoHash> ChunkHashes; + constexpr size_t ChunkCount = 1000; + constexpr size_t Alignment = 8; + eastl::vector<IoBuffer> Chunks; + eastl::vector<IoHash> ChunkHashes; Chunks.reserve(ChunkCount); ChunkHashes.reserve(ChunkCount); for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex) @@ -1700,7 +1700,7 @@ TEST_CASE("blockstore.thread.read.write") ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size())); } - std::vector<BlockStoreLocation> ChunkLocations; + eastl::vector<BlockStoreLocation> ChunkLocations; ChunkLocations.resize(ChunkCount); WorkerThreadPool WorkerPool(8); @@ -1734,7 +1734,7 @@ TEST_CASE("blockstore.thread.read.write") Sleep(1); } - std::vector<BlockStoreLocation> SecondChunkLocations; + eastl::vector<BlockStoreLocation> SecondChunkLocations; SecondChunkLocations.resize(ChunkCount); WorkCompleted = 0; for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex) @@ -1770,10 +1770,10 @@ TEST_CASE("blockstore.compact.blocks") BlockStore Store; Store.Initialize(RootDirectory / "store", 1088, 1024); - constexpr size_t ChunkCount = 200; - constexpr size_t Alignment = 8; - std::vector<IoBuffer> Chunks; - std::vector<IoHash> ChunkHashes; + constexpr size_t ChunkCount = 200; + constexpr size_t Alignment = 8; + eastl::vector<IoBuffer> Chunks; + eastl::vector<IoHash> ChunkHashes; Chunks.reserve(ChunkCount); ChunkHashes.reserve(ChunkCount); for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex) @@ -1783,7 +1783,7 @@ TEST_CASE("blockstore.compact.blocks") ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size())); } - std::vector<BlockStoreLocation> ChunkLocations; + eastl::vector<BlockStoreLocation> ChunkLocations; ChunkLocations.resize(ChunkCount); for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex) @@ -1924,8 +1924,8 @@ TEST_CASE("blockstore.compact.blocks") CHECK(!Store.IsWriting(0)); State.IncludeBlock(0); - uint64_t SkipChunkCount = 2; - std::vector<BlockStoreLocation> DroppedLocations(ChunkLocations.begin(), ChunkLocations.begin() + 2); + uint64_t SkipChunkCount = 2; + eastl::vector<BlockStoreLocation> DroppedLocations(ChunkLocations.begin(), ChunkLocations.begin() + 2); for (auto It = ChunkLocations.begin() + 2; It != ChunkLocations.end(); It++) { const BlockStoreLocation& Location = *It; @@ -1996,7 +1996,7 @@ TEST_CASE("blockstore.compact.blocks") } SkipFlag = false; - std::vector<BlockStoreLocation> DroppedLocations; + eastl::vector<BlockStoreLocation> DroppedLocations; for (const BlockStoreLocation& Location : ChunkLocations) { if (Store.IsWriting(Location.BlockIndex)) diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 61552fafc..2f648bc2d 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -262,36 +262,36 @@ public: return OutVersion == CurrentDiskBucketVersion; } - void ParseManifest(RwLock::ExclusiveLockScope& BucketLock, - ZenCacheDiskLayer::CacheBucket& Bucket, - std::filesystem::path ManifestPath, - ZenCacheDiskLayer::CacheBucket::IndexMap& Index, - std::vector<AccessTime>& AccessTimes, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads); + void ParseManifest(RwLock::ExclusiveLockScope& BucketLock, + ZenCacheDiskLayer::CacheBucket& Bucket, + std::filesystem::path ManifestPath, + ZenCacheDiskLayer::CacheBucket::IndexMap& Index, + eastl::vector<AccessTime>& AccessTimes, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads); Oid GenerateNewManifest(std::filesystem::path ManifestPath); IoBuffer MakeSidecarManifest(const Oid& BucketId, uint64_t EntryCount); uint64_t GetSidecarSize() const { return sizeof(BucketMetaHeader) + m_ManifestEntryCount * sizeof(ManifestData); } - void WriteSidecarFile(RwLock::SharedLockScope& BucketLock, - const std::filesystem::path& SidecarPath, - uint64_t SnapshotLogPosition, - const ZenCacheDiskLayer::CacheBucket::IndexMap& Index, - const std::vector<AccessTime>& AccessTimes, - const std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads, - const std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas); - bool ReadSidecarFile(RwLock::ExclusiveLockScope& BucketLock, - ZenCacheDiskLayer::CacheBucket& Bucket, - std::filesystem::path SidecarPath, - ZenCacheDiskLayer::CacheBucket::IndexMap& Index, - std::vector<AccessTime>& AccessTimes, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads); - - IoBuffer MakeManifest(const Oid& BucketId, - ZenCacheDiskLayer::CacheBucket::IndexMap&& Index, - std::vector<AccessTime>&& AccessTimes, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&& Payloads, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas); + void WriteSidecarFile(RwLock::SharedLockScope& BucketLock, + const std::filesystem::path& SidecarPath, + uint64_t SnapshotLogPosition, + const ZenCacheDiskLayer::CacheBucket::IndexMap& Index, + const eastl::vector<AccessTime>& AccessTimes, + const eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads, + const eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas); + bool ReadSidecarFile(RwLock::ExclusiveLockScope& BucketLock, + ZenCacheDiskLayer::CacheBucket& Bucket, + std::filesystem::path SidecarPath, + ZenCacheDiskLayer::CacheBucket::IndexMap& Index, + eastl::vector<AccessTime>& AccessTimes, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads); + + IoBuffer MakeManifest(const Oid& BucketId, + ZenCacheDiskLayer::CacheBucket::IndexMap&& Index, + eastl::vector<AccessTime>&& AccessTimes, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&& Payloads, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas); CbObject Manifest; @@ -329,12 +329,12 @@ private: }; void -BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& BucketLock, - ZenCacheDiskLayer::CacheBucket& Bucket, - std::filesystem::path ManifestPath, - ZenCacheDiskLayer::CacheBucket::IndexMap& Index, - std::vector<AccessTime>& AccessTimes, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads) +BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& BucketLock, + ZenCacheDiskLayer::CacheBucket& Bucket, + std::filesystem::path ManifestPath, + ZenCacheDiskLayer::CacheBucket::IndexMap& Index, + eastl::vector<AccessTime>& AccessTimes, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads) { if (Manifest["UsingMetaFile"sv].AsBool()) { @@ -348,8 +348,8 @@ BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& Buck Stopwatch Timer; const auto _ = MakeGuard([&] { ZEN_INFO("parsed store manifest '{}' in {}", ManifestPath, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); - const uint64_t Count = Manifest["Count"sv].AsUInt64(0); - std::vector<PayloadIndex> KeysIndexes; + const uint64_t Count = Manifest["Count"sv].AsUInt64(0); + eastl::vector<PayloadIndex> KeysIndexes; KeysIndexes.reserve(Count); CbArrayView KeyArray = Manifest["Keys"sv].AsArrayView(); @@ -425,11 +425,11 @@ BucketManifestSerializer::GenerateNewManifest(std::filesystem::path ManifestPath } IoBuffer -BucketManifestSerializer::MakeManifest(const Oid& BucketId, - ZenCacheDiskLayer::CacheBucket::IndexMap&& Index, - std::vector<AccessTime>&& AccessTimes, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&& Payloads, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas) +BucketManifestSerializer::MakeManifest(const Oid& BucketId, + ZenCacheDiskLayer::CacheBucket::IndexMap&& Index, + eastl::vector<AccessTime>&& AccessTimes, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&& Payloads, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas) { using namespace std::literals; @@ -519,12 +519,12 @@ BucketManifestSerializer::MakeSidecarManifest(const Oid& BucketId, uint64_t Entr } bool -BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& BucketLock, - ZenCacheDiskLayer::CacheBucket& Bucket, - std::filesystem::path SidecarPath, - ZenCacheDiskLayer::CacheBucket::IndexMap& Index, - std::vector<AccessTime>& AccessTimes, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads) +BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& BucketLock, + ZenCacheDiskLayer::CacheBucket& Bucket, + std::filesystem::path SidecarPath, + ZenCacheDiskLayer::CacheBucket::IndexMap& Index, + eastl::vector<AccessTime>& AccessTimes, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads) { ZEN_TRACE_CPU("Z$::ReadSidecarFile"); @@ -621,12 +621,12 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& B void BucketManifestSerializer::WriteSidecarFile(RwLock::SharedLockScope&, - const std::filesystem::path& SidecarPath, - uint64_t SnapshotLogPosition, - const ZenCacheDiskLayer::CacheBucket::IndexMap& Index, - const std::vector<AccessTime>& AccessTimes, - const std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads, - const std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas) + const std::filesystem::path& SidecarPath, + uint64_t SnapshotLogPosition, + const ZenCacheDiskLayer::CacheBucket::IndexMap& Index, + const eastl::vector<AccessTime>& AccessTimes, + const eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads, + const eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas) { ZEN_TRACE_CPU("Z$::WriteSidecarFile"); @@ -653,8 +653,8 @@ BucketManifestSerializer::WriteSidecarFile(RwLock::SharedLockScope&, // BasicFileWriter SidecarWriter(SidecarFile, 128 * 1024); - std::vector<ManifestData> ManifestDataBuffer; - const size_t MaxManifestDataBufferCount = Min(Index.size(), 8192u); // 512 Kb + eastl::vector<ManifestData> ManifestDataBuffer; + const size_t MaxManifestDataBufferCount = Min(Index.size(), 8192u); // 512 Kb ManifestDataBuffer.reserve(MaxManifestDataBufferCount); for (auto& Kv : Index) { @@ -1221,20 +1221,20 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, struct ZenCacheDiskLayer::CacheBucket::PutBatchHandle { - PutBatchHandle(std::vector<bool>& OutResults) : OutResults(OutResults) {} + PutBatchHandle(eastl::vector<bool>& OutResults) : OutResults(OutResults) {} struct Entry { - std::vector<IoHash> HashKeyAndReferences; + eastl::vector<IoHash> HashKeyAndReferences; }; - std::vector<IoBuffer> Buffers; - std::vector<Entry> Entries; - std::vector<size_t> EntryResultIndexes; + eastl::vector<IoBuffer> Buffers; + eastl::vector<Entry> Entries; + eastl::vector<size_t> EntryResultIndexes; - std::vector<bool>& OutResults; + eastl::vector<bool>& OutResults; }; ZenCacheDiskLayer::CacheBucket::PutBatchHandle* -ZenCacheDiskLayer::CacheBucket::BeginPutBatch(std::vector<bool>& OutResults) +ZenCacheDiskLayer::CacheBucket::BeginPutBatch(eastl::vector<bool>& OutResults) { ZEN_TRACE_CPU("Z$::Bucket::BeginPutBatch"); return new PutBatchHandle(OutResults); @@ -1250,7 +1250,7 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept ZEN_ASSERT(Batch); if (!Batch->Buffers.empty()) { - std::vector<uint8_t> EntryFlags; + eastl::vector<uint8_t> EntryFlags; for (const IoBuffer& Buffer : Batch->Buffers) { uint8_t Flags = 0; @@ -1268,13 +1268,13 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept size_t IndexOffset = 0; m_BlockStore.WriteChunks(Batch->Buffers, m_Configuration.PayloadAlignment, [&](std::span<BlockStoreLocation> Locations) { ZEN_MEMSCOPE(GetCacheDiskTag()); - std::vector<DiskIndexEntry> DiskEntries; + eastl::vector<DiskIndexEntry> DiskEntries; { RwLock::ExclusiveLockScope IndexLock(m_IndexLock); for (size_t Index = 0; Index < Locations.size(); Index++) { DiskLocation Location(Locations[Index], m_Configuration.PayloadAlignment, EntryFlags[IndexOffset + Index]); - const std::vector<IoHash>& HashKeyAndReferences = Batch->Entries[IndexOffset + Index].HashKeyAndReferences; + const eastl::vector<IoHash>& HashKeyAndReferences = Batch->Entries[IndexOffset + Index].HashKeyAndReferences; ZEN_ASSERT(HashKeyAndReferences.size() > 1); const IoHash HashKey = HashKeyAndReferences[0]; DiskEntries.push_back({.Key = HashKey, .Location = Location}); @@ -1337,8 +1337,8 @@ struct ZenCacheDiskLayer::CacheBucket::GetBatchHandle ~GetBatchHandle() {} - std::vector<IoHash> Keys; - std::vector<size_t> ResultIndexes; + eastl::vector<IoHash> Keys; + eastl::vector<size_t> ResultIndexes; ZenCacheValueVec_t& OutResults; }; @@ -1829,7 +1829,7 @@ ZenCacheDiskLayer::CacheBucket::MemCacheTrim(GcClock::TimePoint ExpireTime) uint64_t Trimmed = 0; GcClock::Tick ExpireTicks = ExpireTime.time_since_epoch().count(); - std::vector<IoBuffer> PurgedBuffers; + eastl::vector<IoBuffer> PurgedBuffers; { RwLock::ExclusiveLockScope IndexLock(m_IndexLock); @@ -1868,12 +1868,12 @@ ZenCacheDiskLayer::CacheBucket::MemCacheTrim(GcClock::TimePoint ExpireTime) } void -ZenCacheDiskLayer::CacheBucket::GetUsageByAccess(GcClock::TimePoint Now, GcClock::Duration MaxAge, std::vector<uint64_t>& InOutUsageSlots) +ZenCacheDiskLayer::CacheBucket::GetUsageByAccess(GcClock::TimePoint Now, GcClock::Duration MaxAge, eastl::vector<uint64_t>& InOutUsageSlots) { ZEN_TRACE_CPU("Z$::Bucket::GetUsageByAccess"); - std::vector<uint64_t> PayloadSizes; - std::vector<AccessTime> AccessTimes; + eastl::vector<uint64_t> PayloadSizes; + eastl::vector<AccessTime> AccessTimes; size_t SlotCount = InOutUsageSlots.capacity(); @@ -1925,7 +1925,7 @@ ZenCacheDiskLayer::CacheBucket::Drop() RwLock::ExclusiveLockScope _(m_IndexLock); - std::vector<std::unique_ptr<RwLock::ExclusiveLockScope>> ShardLocks; + eastl::vector<std::unique_ptr<RwLock::ExclusiveLockScope>> ShardLocks; ShardLocks.reserve(256); for (RwLock& Lock : m_ShardedLocks) { @@ -1989,10 +1989,10 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl if (UseLegacyScheme) { - std::vector<AccessTime> AccessTimes; - std::vector<BucketPayload> Payloads; - std::vector<BucketMetaData> MetaDatas; - IndexMap Index; + eastl::vector<AccessTime> AccessTimes; + eastl::vector<BucketPayload> Payloads; + eastl::vector<BucketMetaData> MetaDatas; + IndexMap Index; { RwLock::SharedLockScope IndexLock(m_IndexLock); @@ -2104,14 +2104,14 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) NiceRate(VerifiedChunkBytes, DurationMs)); }); - RwLock BadKeysLock; - std::vector<IoHash> BadKeys; - auto ReportBadKey = [&](const IoHash& Key) { BadKeysLock.WithExclusiveLock([&]() { BadKeys.push_back(Key); }); }; + RwLock BadKeysLock; + eastl::vector<IoHash> BadKeys; + auto ReportBadKey = [&](const IoHash& Key) { BadKeysLock.WithExclusiveLock([&]() { BadKeys.push_back(Key); }); }; try { - std::vector<BlockStoreLocation> ChunkLocations; - std::vector<IoHash> ChunkIndexToChunkHash; + eastl::vector<BlockStoreLocation> ChunkLocations; + eastl::vector<IoHash> ChunkIndexToChunkHash; RwLock::SharedLockScope _(m_IndexLock); @@ -2255,7 +2255,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) { // Deal with bad chunks by removing them from our lookup map - std::vector<DiskIndexEntry> LogEntries; + eastl::vector<DiskIndexEntry> LogEntries; LogEntries.reserve(BadKeys.size()); { @@ -2299,11 +2299,11 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) // Clean up m_AccessTimes and m_Payloads vectors { - std::vector<BucketPayload> Payloads; - std::vector<AccessTime> AccessTimes; - std::vector<BucketMetaData> MetaDatas; - std::vector<MemCacheData> MemCachedPayloads; - IndexMap Index; + eastl::vector<BucketPayload> Payloads; + eastl::vector<AccessTime> AccessTimes; + eastl::vector<BucketMetaData> MetaDatas; + eastl::vector<MemCacheData> MemCachedPayloads; + IndexMap Index; { RwLock::ExclusiveLockScope IndexLock(m_IndexLock); @@ -2348,8 +2348,8 @@ ZenCacheDiskLayer::CacheBucket::EntryCount() const CacheValueDetails::ValueDetails ZenCacheDiskLayer::CacheBucket::GetValueDetails(RwLock::SharedLockScope& IndexLock, const IoHash& Key, PayloadIndex Index) const { - std::vector<IoHash> Attachments; - const BucketPayload& Payload = m_Payloads[Index]; + eastl::vector<IoHash> Attachments; + const BucketPayload& Payload = m_Payloads[Index]; if (Payload.Location.IsFlagSet(DiskLocation::kStructured)) { IoBuffer Value = Payload.Location.IsFlagSet(DiskLocation::kStandaloneFile) ? GetStandaloneCacheValue(Payload.Location, Key) @@ -2661,7 +2661,7 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, OptionalBatchHandle->Entries.push_back({}); OptionalBatchHandle->EntryResultIndexes.push_back(OptionalBatchHandle->OutResults.size()); OptionalBatchHandle->OutResults.push_back(false); - std::vector<IoHash>& HashKeyAndReferences = OptionalBatchHandle->Entries.back().HashKeyAndReferences; + eastl::vector<IoHash>& HashKeyAndReferences = OptionalBatchHandle->Entries.back().HashKeyAndReferences; HashKeyAndReferences.reserve(1 + HashKeyAndReferences.size()); HashKeyAndReferences.push_back(HashKey); HashKeyAndReferences.insert(HashKeyAndReferences.end(), HashKeyAndReferences.begin(), HashKeyAndReferences.end()); @@ -2731,7 +2731,7 @@ class DiskBucketStoreCompactor : public GcStoreCompactor using CacheBucket = ZenCacheDiskLayer::CacheBucket; public: - DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys) + DiskBucketStoreCompactor(CacheBucket& Bucket, eastl::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys) : m_Bucket(Bucket) , m_ExpiredStandaloneKeys(std::move(ExpiredStandaloneKeys)) { @@ -2871,7 +2871,7 @@ public: { BlockStoreCompactState BlockCompactState; - std::vector<IoHash> BlockCompactStateKeys; + eastl::vector<IoHash> BlockCompactStateKeys; BlockCompactStateKeys.reserve(InlineEntryCount); BlockStore::BlockEntryCountMap BlocksToCompact = @@ -2913,7 +2913,7 @@ public: BlockCompactState, m_Bucket.m_Configuration.PayloadAlignment, [&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) { - std::vector<DiskIndexEntry> MovedEntries; + eastl::vector<DiskIndexEntry> MovedEntries; MovedEntries.reserve(MovedArray.size()); RwLock::ExclusiveLockScope _(m_Bucket.m_IndexLock); for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray) @@ -2965,8 +2965,8 @@ public: virtual std::string GetGcName(GcCtx& Ctx) override { return m_Bucket.GetGcName(Ctx); } private: - ZenCacheDiskLayer::CacheBucket& m_Bucket; - std::vector<std::pair<IoHash, uint64_t>> m_ExpiredStandaloneKeys; + ZenCacheDiskLayer::CacheBucket& m_Bucket; + eastl::vector<std::pair<IoHash, uint64_t>> m_ExpiredStandaloneKeys; }; GcStoreCompactor* @@ -3012,9 +3012,9 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count(); - std::vector<DiskIndexEntry> ExpiredEntries; - std::vector<std::pair<IoHash, uint64_t>> ExpiredStandaloneKeys; - uint64_t RemovedStandaloneSize = 0; + eastl::vector<DiskIndexEntry> ExpiredEntries; + eastl::vector<std::pair<IoHash, uint64_t>> ExpiredStandaloneKeys; + uint64_t RemovedStandaloneSize = 0; { RwLock::ExclusiveLockScope IndexLock(m_IndexLock); if (Ctx.IsCancelledFlag.load()) @@ -3078,11 +3078,11 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) if (Ctx.Settings.IsDeleteMode && !ExpiredEntries.empty()) { - std::vector<BucketPayload> Payloads; - std::vector<AccessTime> AccessTimes; - std::vector<BucketMetaData> MetaDatas; - std::vector<MemCacheData> MemCachedPayloads; - IndexMap Index; + eastl::vector<BucketPayload> Payloads; + eastl::vector<AccessTime> AccessTimes; + eastl::vector<BucketMetaData> MetaDatas; + eastl::vector<MemCacheData> MemCachedPayloads; + IndexMap Index; { RwLock::ExclusiveLockScope IndexLock(m_IndexLock); CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, Index); @@ -3101,7 +3101,7 @@ bool ZenCacheDiskLayer::CacheBucket::ReadAttachmentsFromMetaData(uint32_t BlockIndex, std::span<const IoHash> InlineKeys, std::span<const std::size_t> ChunkIndexes, - std::vector<IoHash>& OutReferences) const + eastl::vector<IoHash>& OutReferences) const { ZEN_TRACE_CPU("Z$::Bucket::ReadAttachmentsFromMetaData"); IoBuffer MetaDataPayload = m_BlockStore.GetMetaData(BlockIndex); @@ -3146,13 +3146,13 @@ ZenCacheDiskLayer::CacheBucket::ReadAttachmentsFromMetaData(uint32_t BlockI } bool -ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger, - std::atomic_bool& IsCancelledFlag, - bool StateIsAlreadyLocked, - bool ReadCacheAttachmentMetaData, - bool WriteCacheAttachmentMetaData, - std::vector<IoHash>& OutReferences, - ReferencesStats* OptionalOutReferencesStats) +ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger, + std::atomic_bool& IsCancelledFlag, + bool StateIsAlreadyLocked, + bool ReadCacheAttachmentMetaData, + bool WriteCacheAttachmentMetaData, + eastl::vector<IoHash>& OutReferences, + ReferencesStats* OptionalOutReferencesStats) { ZEN_TRACE_CPU("Z$::Bucket::GetReferencesLocked"); @@ -3168,11 +3168,11 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger, return false; }; - std::vector<std::pair<IoHash, DiskLocation>> StandaloneKeys; + eastl::vector<std::pair<IoHash, DiskLocation>> StandaloneKeys; { - std::vector<IoHash> InlineKeys; - std::vector<BlockStoreLocation> InlineLocations; - std::vector<std::vector<std::size_t>> InlineBlockChunkIndexes; + eastl::vector<IoHash> InlineKeys; + eastl::vector<BlockStoreLocation> InlineLocations; + eastl::vector<eastl::vector<std::size_t>> InlineBlockChunkIndexes; { std::unordered_map<uint32_t, std::size_t> BlockIndexToChunkIndexes; @@ -3229,7 +3229,7 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger, else { BlockIndexToChunkIndexes.insert_or_assign(ChunkLocation.BlockIndex, InlineBlockChunkIndexes.size()); - InlineBlockChunkIndexes.emplace_back(std::vector<size_t>{ChunkIndex}); + InlineBlockChunkIndexes.emplace_back(eastl::vector<size_t>{ChunkIndex}); } } } @@ -3237,17 +3237,17 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger, OutReferences.reserve(OutReferences.size() + InlineKeys.size() + StandaloneKeys.size()); // Make space for at least one attachment per record - for (const std::vector<std::size_t>& ChunkIndexes : InlineBlockChunkIndexes) + for (const eastl::vector<std::size_t>& ChunkIndexes : InlineBlockChunkIndexes) { ZEN_ASSERT(!ChunkIndexes.empty()); uint32_t BlockIndex = InlineLocations[ChunkIndexes[0]].BlockIndex; if (!ReadCacheAttachmentMetaData || !ReadAttachmentsFromMetaData(BlockIndex, InlineKeys, ChunkIndexes, OutReferences)) { - std::vector<IoHash> Keys; - std::vector<uint32_t> AttachmentCounts; - size_t PrecachedReferencesStart = OutReferences.size(); - size_t NextPrecachedReferencesStart = PrecachedReferencesStart; + eastl::vector<IoHash> Keys; + eastl::vector<uint32_t> AttachmentCounts; + size_t PrecachedReferencesStart = OutReferences.size(); + size_t NextPrecachedReferencesStart = PrecachedReferencesStart; bool WriteMetaData = WriteCacheAttachmentMetaData && !m_BlockStore.IsWriting(BlockIndex); if (WriteMetaData) @@ -3372,7 +3372,8 @@ public: NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); - m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences = std::make_unique<std::vector<IoHash>>(); }); + m_CacheBucket.m_IndexLock.WithExclusiveLock( + [&]() { m_CacheBucket.m_TrackedReferences = std::make_unique<eastl::vector<IoHash>>(); }); bool Continue = m_CacheBucket.GetReferences(Ctx.Logger, Ctx.IsCancelledFlag, @@ -3448,12 +3449,12 @@ public: return UnusedReferences; } - CacheBucket& m_CacheBucket; - std::vector<IoHash> m_PrecachedReferences; - std::vector<IoHash> m_AddedReferences; + CacheBucket& m_CacheBucket; + eastl::vector<IoHash> m_PrecachedReferences; + eastl::vector<IoHash> m_AddedReferences; }; -std::vector<GcReferenceChecker*> +eastl::vector<GcReferenceChecker*> ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx) { ZEN_TRACE_CPU("Z$::Bucket::CreateReferenceCheckers"); @@ -3480,7 +3481,7 @@ ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx) return {new DiskBucketReferenceChecker(*this)}; } -std::vector<GcReferenceValidator*> +eastl::vector<GcReferenceValidator*> ZenCacheDiskLayer::CacheBucket::CreateReferenceValidators(GcCtx& /*Ctx*/) { return {}; @@ -3488,11 +3489,11 @@ ZenCacheDiskLayer::CacheBucket::CreateReferenceValidators(GcCtx& /*Ctx*/) void ZenCacheDiskLayer::CacheBucket::CompactState(RwLock::ExclusiveLockScope&, - std::vector<BucketPayload>& Payloads, - std::vector<AccessTime>& AccessTimes, - std::vector<BucketMetaData>& MetaDatas, - std::vector<MemCacheData>& MemCachedPayloads, - IndexMap& Index) + eastl::vector<BucketPayload>& Payloads, + eastl::vector<AccessTime>& AccessTimes, + eastl::vector<BucketMetaData>& MetaDatas, + eastl::vector<MemCacheData>& MemCachedPayloads, + IndexMap& Index) { ZEN_TRACE_CPU("Z$::Bucket::CompactState"); @@ -3650,7 +3651,7 @@ ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket) struct ZenCacheDiskLayer::PutBatchHandle { - PutBatchHandle(std::vector<bool>& OutResults) : OutResults(OutResults) {} + PutBatchHandle(eastl::vector<bool>& OutResults) : OutResults(OutResults) {} struct BucketHandle { CacheBucket* Bucket; @@ -3709,13 +3710,13 @@ struct ZenCacheDiskLayer::PutBatchHandle return NewBucketHandle; } - RwLock Lock; - std::vector<BucketHandle> BucketHandles; - std::vector<bool>& OutResults; + RwLock Lock; + eastl::vector<BucketHandle> BucketHandles; + eastl::vector<bool>& OutResults; }; ZenCacheDiskLayer::PutBatchHandle* -ZenCacheDiskLayer::BeginPutBatch(std::vector<bool>& OutResults) +ZenCacheDiskLayer::BeginPutBatch(eastl::vector<bool>& OutResults) { return new PutBatchHandle(OutResults); } @@ -3879,8 +3880,8 @@ ZenCacheDiskLayer::DiscoverBuckets() // Initialize buckets - std::vector<std::filesystem::path> BadBucketDirectories; - std::vector<std::filesystem::path> FoundBucketDirectories; + eastl::vector<std::filesystem::path> BadBucketDirectories; + eastl::vector<std::filesystem::path> FoundBucketDirectories; RwLock::ExclusiveLockScope _(m_Lock); @@ -4003,7 +4004,7 @@ ZenCacheDiskLayer::Drop() RwLock::ExclusiveLockScope _(m_Lock); - std::vector<std::unique_ptr<CacheBucket>> Buckets; + eastl::vector<std::unique_ptr<CacheBucket>> Buckets; Buckets.reserve(m_Buckets.size()); while (!m_Buckets.empty()) { @@ -4025,15 +4026,15 @@ ZenCacheDiskLayer::Flush() ZEN_MEMSCOPE(GetCacheDiskTag()); ZEN_TRACE_CPU("Z$::Flush"); - std::vector<CacheBucket*> Buckets; - Stopwatch Timer; - const auto _ = MakeGuard([&] { - if (Buckets.empty()) - { - return; - } - ZEN_INFO("Flushed {} buckets at '{}' in {}", Buckets.size(), m_RootDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }); + eastl::vector<CacheBucket*> Buckets; + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (Buckets.empty()) + { + return; + } + ZEN_INFO("Flushed {} buckets at '{}' in {}", Buckets.size(), m_RootDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); { RwLock::SharedLockScope __(m_Lock); @@ -4086,7 +4087,7 @@ ZenCacheDiskLayer::ScrubStorage(ScrubContext& Ctx) RwLock::SharedLockScope _(m_Lock); { - std::vector<std::future<void>> Results; + eastl::vector<std::future<void>> Results; Results.reserve(m_Buckets.size()); for (auto& Kv : m_Buckets) @@ -4211,10 +4212,10 @@ ZenCacheDiskLayer::GetValueDetails(const std::string_view BucketFilter, const st return Details; } -std::vector<RwLock::SharedLockScope> +eastl::vector<RwLock::SharedLockScope> ZenCacheDiskLayer::GetGcReferencerLocks() { - std::vector<RwLock::SharedLockScope> Locks; + eastl::vector<RwLock::SharedLockScope> Locks; Locks.emplace_back(RwLock::SharedLockScope(m_Lock)); for (auto& Kv : m_Buckets) { @@ -4230,7 +4231,7 @@ ZenCacheDiskLayer::EnableUpdateCapture() if (m_UpdateCaptureRefCounter == 0) { ZEN_ASSERT(!m_CapturedBuckets); - m_CapturedBuckets = std::make_unique<std::vector<std::string>>(); + m_CapturedBuckets = std::make_unique<eastl::vector<std::string>>(); } else { @@ -4254,7 +4255,7 @@ ZenCacheDiskLayer::DisableUpdateCapture() }); } -std::vector<std::string> +eastl::vector<std::string> ZenCacheDiskLayer::GetCapturedBucketsLocked() { if (m_CapturedBuckets) @@ -4332,11 +4333,11 @@ ZenCacheDiskLayer::MemCacheTrim() const std::chrono::seconds MaxAge = std::chrono::seconds(m_Configuration.MemCacheMaxAgeSeconds); - static const size_t UsageSlotCount = 2048; - std::vector<uint64_t> UsageSlots; + static const size_t UsageSlotCount = 2048; + eastl::vector<uint64_t> UsageSlots; UsageSlots.reserve(UsageSlotCount); - std::vector<CacheBucket*> Buckets; + eastl::vector<CacheBucket*> Buckets; { RwLock::SharedLockScope __(m_Lock); Buckets.reserve(m_Buckets.size()); @@ -4371,7 +4372,7 @@ ZenCacheDiskLayer::MemCacheTrim() } uint64_t -ZenCacheDiskLayer::MemCacheTrim(std::vector<CacheBucket*>& Buckets, GcClock::TimePoint ExpireTime) +ZenCacheDiskLayer::MemCacheTrim(eastl::vector<CacheBucket*>& Buckets, GcClock::TimePoint ExpireTime) { if (m_Configuration.MemCacheTargetFootprintBytes == 0) { diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index 97e26a38d..481a8906e 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -107,7 +107,7 @@ namespace cache::detail { struct RecordBody { IoBuffer CacheValue; - std::vector<RecordValue> Values; + eastl::vector<RecordValue> Values; const UpstreamEndpointInfo* Source = nullptr; CachePolicy DownstreamPolicy; bool Exists = false; @@ -362,12 +362,12 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag uint64_t RecordObjectSize = Record.GetSize(); uint64_t TransferredSize = RecordObjectSize; - AttachmentCount Count; - size_t NumAttachments = Package->GetAttachments().size(); - std::vector<IoHash> ValidAttachments; - std::vector<IoHash> ReferencedAttachments; - std::vector<IoBuffer> WriteAttachmentBuffers; - std::vector<IoHash> WriteRawHashes; + AttachmentCount Count; + size_t NumAttachments = Package->GetAttachments().size(); + eastl::vector<IoHash> ValidAttachments; + eastl::vector<IoHash> ReferencedAttachments; + eastl::vector<IoBuffer> WriteAttachmentBuffers; + eastl::vector<IoHash> WriteRawHashes; ValidAttachments.reserve(NumAttachments); WriteAttachmentBuffers.reserve(NumAttachments); WriteRawHashes.reserve(NumAttachments); @@ -429,7 +429,7 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag if (!WriteAttachmentBuffers.empty()) { - std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + eastl::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); for (size_t Index = 0; Index < InsertResults.size(); Index++) { if (InsertResults[Index].New) @@ -563,11 +563,11 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb Request.RecordObject = CbObjectView(Request.RecordCacheValue.GetData()); ParseValues(Request); - Request.Complete = true; - size_t ValueCount = Request.Values.size(); - std::vector<IoHash> CidHashes; - std::vector<size_t> RequestValueIndexes; - const bool DoBatch = ValueCount > 7; + Request.Complete = true; + size_t ValueCount = Request.Values.size(); + eastl::vector<IoHash> CidHashes; + eastl::vector<size_t> RequestValueIndexes; + const bool DoBatch = ValueCount > 7; if (DoBatch) { CidHashes.reserve(ValueCount); @@ -703,7 +703,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb if (!UpstreamIndexes.empty()) { - std::vector<CacheKeyRequest*> UpstreamRequests; + eastl::vector<CacheKeyRequest*> UpstreamRequests; UpstreamRequests.reserve(UpstreamIndexes.size()); for (size_t Index : UpstreamIndexes) { @@ -753,7 +753,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal) && AreDiskWritesAllowed(); if (StoreLocal) { - std::vector<IoHash> ReferencedAttachments; + eastl::vector<IoHash> ReferencedAttachments; ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) { const IoHash ValueHash = HashView.AsHash(); ReferencedAttachments.push_back(ValueHash); @@ -912,7 +912,7 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con const bool HasUpstream = m_UpstreamCache.IsActive(); CbArrayView RequestsArray = Params["Requests"sv].AsArrayView(); - std::vector<bool> BatchResults; + eastl::vector<bool> BatchResults; eastl::fixed_vector<size_t, 32> BatchResultIndexes; eastl::fixed_vector<bool, 32> Results; eastl::fixed_vector<CacheKey, 32> UpstreamCacheKeys; @@ -1200,8 +1200,8 @@ CacheRpcHandler::HandleRpcGetCacheValues(const CacheRequestContext& Context, CbO if (!RemoteRequestIndexes.empty()) { - std::vector<CacheValueRequest> RequestedRecordsData; - std::vector<CacheValueRequest*> CacheValueRequests; + eastl::vector<CacheValueRequest> RequestedRecordsData; + eastl::vector<CacheValueRequest*> CacheValueRequests; RequestedRecordsData.reserve(RemoteRequestIndexes.size()); CacheValueRequests.reserve(RemoteRequestIndexes.size()); for (size_t Index : RemoteRequestIndexes) @@ -1320,14 +1320,14 @@ CacheRpcHandler::HandleRpcGetCacheChunks(const CacheRequestContext& Context, Rpc ZEN_TRACE_CPU("Z$::RpcGetCacheChunks"); using namespace cache::detail; - std::string Namespace; - std::vector<CacheKeyRequest> RecordKeys; // Data about a Record necessary to identify it to the upstream - std::vector<RecordBody> Records; // Scratch-space data about a Record when fulfilling RecordRequests - std::vector<CacheChunkRequest> RequestKeys; // Data about a ChunkRequest necessary to identify it to the upstream - std::vector<ChunkRequest> Requests; // Intermediate and result data about a ChunkRequest - std::vector<ChunkRequest*> RecordRequests; // The ChunkRequests that are requesting a subvalue from a Record Key - std::vector<ChunkRequest*> ValueRequests; // The ChunkRequests that are requesting a Value Key - std::vector<CacheChunkRequest*> UpstreamChunks; // ChunkRequests that we need to send to the upstream + std::string Namespace; + eastl::vector<CacheKeyRequest> RecordKeys; // Data about a Record necessary to identify it to the upstream + eastl::vector<RecordBody> Records; // Scratch-space data about a Record when fulfilling RecordRequests + eastl::vector<CacheChunkRequest> RequestKeys; // Data about a ChunkRequest necessary to identify it to the upstream + eastl::vector<ChunkRequest> Requests; // Intermediate and result data about a ChunkRequest + eastl::vector<ChunkRequest*> RecordRequests; // The ChunkRequests that are requesting a subvalue from a Record Key + eastl::vector<ChunkRequest*> ValueRequests; // The ChunkRequests that are requesting a Value Key + eastl::vector<CacheChunkRequest*> UpstreamChunks; // ChunkRequests that we need to send to the upstream // Parse requests from the CompactBinary body of the RpcRequest and divide it into RecordRequests and ValueRequests if (!ParseGetCacheChunksRequest(Namespace, RecordKeys, Records, RequestKeys, Requests, RecordRequests, ValueRequests, RpcRequest)) @@ -1350,14 +1350,14 @@ CacheRpcHandler::HandleRpcGetCacheChunks(const CacheRequestContext& Context, Rpc } bool -CacheRpcHandler::ParseGetCacheChunksRequest(std::string& Namespace, - std::vector<CacheKeyRequest>& RecordKeys, - std::vector<cache::detail::RecordBody>& Records, - std::vector<CacheChunkRequest>& RequestKeys, - std::vector<cache::detail::ChunkRequest>& Requests, - std::vector<cache::detail::ChunkRequest*>& RecordRequests, - std::vector<cache::detail::ChunkRequest*>& ValueRequests, - CbObjectView RpcRequest) +CacheRpcHandler::ParseGetCacheChunksRequest(std::string& Namespace, + eastl::vector<CacheKeyRequest>& RecordKeys, + eastl::vector<cache::detail::RecordBody>& Records, + eastl::vector<CacheChunkRequest>& RequestKeys, + eastl::vector<cache::detail::ChunkRequest>& Requests, + eastl::vector<cache::detail::ChunkRequest*>& RecordRequests, + eastl::vector<cache::detail::ChunkRequest*>& ValueRequests, + CbObjectView RpcRequest) { ZEN_TRACE_CPU("Z$::ParseGetCacheChunksRequest"); @@ -1470,12 +1470,12 @@ CacheRpcHandler::ParseGetCacheChunksRequest(std::string& Namespace, } void -CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, - std::string_view Namespace, - std::vector<CacheKeyRequest>& RecordKeys, - std::vector<cache::detail::RecordBody>& Records, - std::vector<cache::detail::ChunkRequest*>& RecordRequests, - std::vector<CacheChunkRequest*>& OutUpstreamChunks) +CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, + std::string_view Namespace, + eastl::vector<CacheKeyRequest>& RecordKeys, + eastl::vector<cache::detail::RecordBody>& Records, + eastl::vector<cache::detail::ChunkRequest*>& RecordRequests, + eastl::vector<CacheChunkRequest*>& OutUpstreamChunks) { ZEN_TRACE_CPU("Z$::GetLocalCacheRecords"); @@ -1483,7 +1483,7 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, const bool HasUpstream = m_UpstreamCache.IsActive(); // TODO: BatchGet records? - std::vector<CacheKeyRequest*> UpstreamRecordRequests; + eastl::vector<CacheKeyRequest*> UpstreamRecordRequests; for (size_t RecordIndex = 0; RecordIndex < Records.size(); ++RecordIndex) { ZEN_TRACE_CPU("Z$::GetLocalCacheRecords::Record"); @@ -1536,7 +1536,7 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, bool StoreLocal = EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal) && AreDiskWritesAllowed(); if (StoreLocal) { - std::vector<IoHash> ReferencedAttachments; + eastl::vector<IoHash> ReferencedAttachments; ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) { const IoHash ValueHash = HashView.AsHash(); ReferencedAttachments.push_back(ValueHash); @@ -1637,10 +1637,10 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, } void -CacheRpcHandler::GetLocalCacheValues(const CacheRequestContext& Context, - std::string_view Namespace, - std::vector<cache::detail::ChunkRequest*>& ValueRequests, - std::vector<CacheChunkRequest*>& OutUpstreamChunks) +CacheRpcHandler::GetLocalCacheValues(const CacheRequestContext& Context, + std::string_view Namespace, + eastl::vector<cache::detail::ChunkRequest*>& ValueRequests, + eastl::vector<CacheChunkRequest*>& OutUpstreamChunks) { ZEN_TRACE_CPU("Z$::GetLocalCacheValues"); @@ -1717,11 +1717,11 @@ CacheRpcHandler::GetLocalCacheValues(const CacheRequestContext& Context, } void -CacheRpcHandler::GetUpstreamCacheChunks(const CacheRequestContext& Context, - std::string_view Namespace, - std::vector<CacheChunkRequest*>& UpstreamChunks, - std::vector<CacheChunkRequest>& RequestKeys, - std::vector<cache::detail::ChunkRequest>& Requests) +CacheRpcHandler::GetUpstreamCacheChunks(const CacheRequestContext& Context, + std::string_view Namespace, + eastl::vector<CacheChunkRequest*>& UpstreamChunks, + eastl::vector<CacheChunkRequest>& RequestKeys, + eastl::vector<cache::detail::ChunkRequest>& Requests) { if (UpstreamChunks.empty()) { @@ -1790,7 +1790,7 @@ CbPackage CacheRpcHandler::WriteGetCacheChunksResponse([[maybe_unused]] const CacheRequestContext& Context, std::string_view Namespace, RpcAcceptOptions AcceptOptions, - std::vector<cache::detail::ChunkRequest>& Requests) + eastl::vector<cache::detail::ChunkRequest>& Requests) { ZEN_TRACE_CPU("Z$::WriteGetCacheChunksResponse"); diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 7d277329e..87c95f8a2 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -154,7 +154,7 @@ struct ZenCacheNamespace::PutBatchHandle }; ZenCacheNamespace::PutBatchHandle* -ZenCacheNamespace::BeginPutBatch(std::vector<bool>& OutResult) +ZenCacheNamespace::BeginPutBatch(eastl::vector<bool>& OutResult) { ZenCacheNamespace::PutBatchHandle* Handle = new ZenCacheNamespace::PutBatchHandle; Handle->DiskLayerHandle = m_DiskLayer.BeginPutBatch(OutResult); @@ -366,7 +366,7 @@ ZenCacheNamespace::GetValueDetails(const std::string_view BucketFilter, const st return m_DiskLayer.GetValueDetails(BucketFilter, ValueFilter); } -std::vector<RwLock::SharedLockScope> +eastl::vector<RwLock::SharedLockScope> ZenCacheNamespace::GetGcReferencerLocks() { return m_DiskLayer.GetGcReferencerLocks(); @@ -426,7 +426,7 @@ ZenCacheStore::ZenCacheStore(GcManager& Gc, DirectoryContent DirContent; GetDirectoryContent(m_BasePath, DirectoryContentFlags::IncludeDirs, DirContent); - std::vector<std::string> Namespaces; + eastl::vector<std::string> Namespaces; for (const std::filesystem::path& DirPath : DirContent.Directories) { std::string DirName = PathToUtf8(DirPath.filename()); @@ -480,7 +480,7 @@ ZenCacheStore::LogWorker() auto Log = [&ZCacheLog]() -> LoggerRef { return ZCacheLog; }; - std::vector<AccessLogItem> Items; + eastl::vector<AccessLogItem> Items; while (true) { try @@ -553,7 +553,7 @@ ZenCacheStore::LogWorker() } } -ZenCacheStore::PutBatch::PutBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, std::vector<bool>& OutResult) +ZenCacheStore::PutBatch::PutBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, eastl::vector<bool>& OutResult) : m_CacheStore(CacheStore) { ZEN_MEMSCOPE(GetCacheStoreTag()); @@ -904,10 +904,10 @@ ZenCacheStore::FindNamespace(std::string_view Namespace) const return nullptr; } -std::vector<std::string> +eastl::vector<std::string> ZenCacheStore::GetNamespaces() { - std::vector<std::string> Namespaces; + eastl::vector<std::string> Namespaces; IterateNamespaces([&](std::string_view Namespace, ZenCacheNamespace&) { Namespaces.push_back(std::string(Namespace)); }); @@ -917,7 +917,7 @@ ZenCacheStore::GetNamespaces() void ZenCacheStore::IterateNamespaces(const std::function<void(std::string_view Namespace, ZenCacheNamespace& Store)>& Callback) const { - std::vector<std::pair<std::string, ZenCacheNamespace&>> Namespaces; + eastl::vector<std::pair<std::string, ZenCacheNamespace&>> Namespaces; { RwLock::SharedLockScope _(m_NamespacesLock); Namespaces.reserve(m_Namespaces.size()); @@ -1042,18 +1042,18 @@ ZenCacheStore::GetBucketInfo(std::string_view NamespaceName, std::string_view Bu return {}; } -std::vector<RwLock::SharedLockScope> +eastl::vector<RwLock::SharedLockScope> ZenCacheStore::LockState(GcCtx& Ctx) { ZEN_TRACE_CPU("CacheStore::LockState"); auto Log = [&Ctx]() { return Ctx.Logger; }; - std::vector<RwLock::SharedLockScope> Locks; + eastl::vector<RwLock::SharedLockScope> Locks; Locks.emplace_back(RwLock::SharedLockScope(m_NamespacesLock)); for (auto& NamespaceIt : m_Namespaces) { - std::vector<RwLock::SharedLockScope> NamespaceLocks = NamespaceIt.second->GetGcReferencerLocks(); + eastl::vector<RwLock::SharedLockScope> NamespaceLocks = NamespaceIt.second->GetGcReferencerLocks(); for (auto It = std::make_move_iterator(NamespaceLocks.begin()); It != std::make_move_iterator(NamespaceLocks.end()); It++) { Locks.emplace_back(std::move(*It)); @@ -1065,12 +1065,12 @@ ZenCacheStore::LockState(GcCtx& Ctx) void ZenCacheStore::EnableUpdateCapture() { - std::vector<ZenCacheNamespace*> Namespaces; + eastl::vector<ZenCacheNamespace*> Namespaces; m_NamespacesLock.WithExclusiveLock([&]() { if (m_UpdateCaptureRefCounter == 0) { ZEN_ASSERT(!m_CapturedNamespaces); - m_CapturedNamespaces = std::make_unique<std::vector<std::string>>(); + m_CapturedNamespaces = std::make_unique<eastl::vector<std::string>>(); } else { @@ -1093,7 +1093,7 @@ ZenCacheStore::EnableUpdateCapture() void ZenCacheStore::DisableUpdateCapture() { - std::vector<ZenCacheNamespace*> Namespaces; + eastl::vector<ZenCacheNamespace*> Namespaces; m_NamespacesLock.WithExclusiveLock([&]() { ZEN_ASSERT(m_CapturedNamespaces); ZEN_ASSERT(m_UpdateCaptureRefCounter > 0); @@ -1114,7 +1114,7 @@ ZenCacheStore::DisableUpdateCapture() } } -std::vector<std::string> +eastl::vector<std::string> ZenCacheStore::GetCapturedNamespacesLocked() { if (m_CapturedNamespaces) @@ -1173,7 +1173,7 @@ public: Stopwatch Timer; - std::vector<ZenCacheDiskLayer::CacheBucket*> AddedBuckets; + eastl::vector<ZenCacheDiskLayer::CacheBucket*> AddedBuckets; const auto _ = MakeGuard([&] { if (!Ctx.Settings.Verbose) @@ -1187,7 +1187,7 @@ public: AddedBuckets.size()); }); - std::vector<std::string> AddedNamespaces = m_CacheStore.GetCapturedNamespacesLocked(); + eastl::vector<std::string> AddedNamespaces = m_CacheStore.GetCapturedNamespacesLocked(); for (const std::string& AddedNamespace : AddedNamespaces) { @@ -1202,8 +1202,8 @@ public: } for (auto& NamepaceKV : m_CacheStore.m_Namespaces) { - ZenCacheNamespace& Namespace = *NamepaceKV.second; - std::vector<std::string> NamespaceAddedBuckets = Namespace.m_DiskLayer.GetCapturedBucketsLocked(); + ZenCacheNamespace& Namespace = *NamepaceKV.second; + eastl::vector<std::string> NamespaceAddedBuckets = Namespace.m_DiskLayer.GetCapturedBucketsLocked(); for (const std::string& AddedBucketName : NamespaceAddedBuckets) { if (auto It = Namespace.m_DiskLayer.m_Buckets.find(AddedBucketName); It != Namespace.m_DiskLayer.m_Buckets.end()) @@ -1258,31 +1258,31 @@ public: } private: - ZenCacheStore& m_CacheStore; - std::vector<IoHash> m_References; + ZenCacheStore& m_CacheStore; + eastl::vector<IoHash> m_References; }; -std::vector<GcReferenceChecker*> +eastl::vector<GcReferenceChecker*> ZenCacheStore::CreateReferenceCheckers(GcCtx& Ctx) { ZEN_TRACE_CPU("CacheStore::CreateReferenceCheckers"); auto Log = [&Ctx]() { return Ctx.Logger; }; - Stopwatch Timer; - const auto _ = MakeGuard([&] { - if (!Ctx.Settings.Verbose) - { - return; - } - ZEN_INFO("GCV2: cachestore [CREATE CHECKERS] '{}': completed in {}", m_BasePath, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }); - std::vector<GcReferenceChecker*> Checkers; + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: cachestore [CREATE CHECKERS] '{}': completed in {}", m_BasePath, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + eastl::vector<GcReferenceChecker*> Checkers; Checkers.emplace_back(new CacheStoreReferenceChecker(*this)); return Checkers; } -std::vector<GcReferenceValidator*> +eastl::vector<GcReferenceValidator*> ZenCacheStore::CreateReferenceValidators(GcCtx& /*Ctx*/) { return {}; @@ -1307,9 +1307,9 @@ namespace testutils { std::pair<Oid, IoBuffer> CreateBinaryBlob(size_t Size) { return {Oid::NewOid(), CreateRandomBlob(Size)}; } - std::vector<std::pair<Oid, CompressedBuffer>> CreateCompressedAttachment(CidStore& Store, const std::span<const size_t>& Sizes) + eastl::vector<std::pair<Oid, CompressedBuffer>> CreateCompressedAttachment(CidStore& Store, const std::span<const size_t>& Sizes) { - std::vector<std::pair<Oid, CompressedBuffer>> Result; + eastl::vector<std::pair<Oid, CompressedBuffer>> Result; Result.reserve(Sizes.size()); for (size_t Size : Sizes) { @@ -1386,7 +1386,7 @@ TEST_CASE("cachestore.size") auto JobQueue = MakeJobQueue(1, "testqueue"); const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size); CbObjectWriter Writer; @@ -1410,7 +1410,7 @@ TEST_CASE("cachestore.size") IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer(); Buffer.SetContentType(ZenContentType::kCbObject); - std::vector<std::pair<std::string, IoHash>> Keys; + eastl::vector<std::pair<std::string, IoHash>> Keys; for (size_t Key = 0; Key < Count; ++Key) { @@ -1710,7 +1710,7 @@ TEST_CASE("cachestore.namespaces") using namespace testutils; const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size); CbObjectWriter Writer; @@ -1778,7 +1778,7 @@ TEST_CASE("cachestore.drop.bucket") using namespace testutils; const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size); CbObjectWriter Writer; @@ -1855,7 +1855,7 @@ TEST_CASE("cachestore.drop.namespace") const CacheRequestContext Context; const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size); CbObjectWriter Writer; @@ -1945,7 +1945,7 @@ TEST_CASE("cachestore.blocked.disklayer.put") GcStorageSize CacheSize; const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size, Size & 0xff); CbObjectWriter Writer; @@ -1999,11 +1999,11 @@ TEST_CASE("cachestore.scrub") struct CacheRecord { - IoBuffer Record; - std::vector<CompressedBuffer> Attachments; + IoBuffer Record; + eastl::vector<CompressedBuffer> Attachments; }; - auto CreateCacheRecord = [](bool Structured, std::string_view Bucket, const IoHash& Key, const std::vector<size_t>& AttachmentSizes) { + auto CreateCacheRecord = [](bool Structured, std::string_view Bucket, const IoHash& Key, const eastl::vector<size_t>& AttachmentSizes) { CacheRecord Result; if (Structured) { @@ -2054,27 +2054,29 @@ TEST_CASE("cachestore.scrub") CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; CidStore.Initialize(CidConfig); - auto CreateRecords = - [&](bool IsStructured, std::string_view BucketName, const std::vector<IoHash>& Cids, const std::vector<size_t>& AttachmentSizes) { - for (const IoHash& Cid : Cids) + auto CreateRecords = [&](bool IsStructured, + std::string_view BucketName, + const eastl::vector<IoHash>& Cids, + const eastl::vector<size_t>& AttachmentSizes) { + for (const IoHash& Cid : Cids) + { + CacheRecord Record = CreateCacheRecord(IsStructured, BucketName, Cid, AttachmentSizes); + eastl::vector<IoHash> AttachmentHashes; + for (const CompressedBuffer& Attachment : Record.Attachments) { - CacheRecord Record = CreateCacheRecord(IsStructured, BucketName, Cid, AttachmentSizes); - std::vector<IoHash> AttachmentHashes; - for (const CompressedBuffer& Attachment : Record.Attachments) - { - AttachmentHashes.push_back(Attachment.DecodeRawHash()); - CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), AttachmentHashes.back()); - } - Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes); + AttachmentHashes.push_back(Attachment.DecodeRawHash()); + CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), AttachmentHashes.back()); } - }; + Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes); + } + }; - std::vector<size_t> AttachmentSizes = {16, 1000, 2000, 4000, 8000, 64000, 80000}; + eastl::vector<size_t> AttachmentSizes = {16, 1000, 2000, 4000, 8000, 64000, 80000}; - std::vector<IoHash> UnstructuredCids{CreateKey(4), CreateKey(5), CreateKey(6)}; + eastl::vector<IoHash> UnstructuredCids{CreateKey(4), CreateKey(5), CreateKey(6)}; CreateRecords(false, "mybucket"sv, UnstructuredCids, AttachmentSizes); - std::vector<IoHash> StructuredCids{CreateKey(1), CreateKey(2), CreateKey(3)}; + eastl::vector<IoHash> StructuredCids{CreateKey(1), CreateKey(2), CreateKey(3)}; CreateRecords(true, "mybucket"sv, StructuredCids, AttachmentSizes); WorkerThreadPool ThreadPool{1}; @@ -2095,15 +2097,15 @@ TEST_CASE("cachestore.newgc.basics") struct CacheEntry { - IoBuffer Data; - std::vector<std::pair<Oid, CompressedBuffer>> Attachments; + IoBuffer Data; + eastl::vector<std::pair<Oid, CompressedBuffer>> Attachments; }; std::unordered_map<IoHash, CacheEntry> CacheEntries; auto CreateCacheRecord = [&](ZenCacheNamespace& Zcs, CidStore& CidStore, std::string_view Bucket, std::span<std::pair<Oid, CompressedBuffer>> Attachments) { - std::vector<IoHash> AttachmentKeys; + eastl::vector<IoHash> AttachmentKeys; for (const auto& Attachment : Attachments) { AttachmentKeys.push_back(Attachment.second.DecodeRawHash()); @@ -2194,8 +2196,8 @@ TEST_CASE("cachestore.newgc.basics") return true; }; - std::vector<IoHash> CacheRecords; - std::vector<IoHash> UnstructuredCacheValues; + eastl::vector<IoHash> CacheRecords; + eastl::vector<IoHash> UnstructuredCacheValues; const auto TearDrinkerBucket = "teardrinker"sv; { @@ -2207,11 +2209,11 @@ TEST_CASE("cachestore.newgc.basics") // Create some basic data { // Structured record with attachments - auto Attachments1 = CreateCompressedAttachment(CidStore, std::vector<size_t>{77, 1024 * 1024 * 2, 99, 1024 * 1024 * 2 + 87}); + auto Attachments1 = CreateCompressedAttachment(CidStore, eastl::vector<size_t>{77, 1024 * 1024 * 2, 99, 1024 * 1024 * 2 + 87}); CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments1)); // Structured record with reuse of attachments - auto Attachments2 = CreateCompressedAttachment(CidStore, std::vector<size_t>{971}); + auto Attachments2 = CreateCompressedAttachment(CidStore, eastl::vector<size_t>{971}); Attachments2.push_back(Attachments1[0]); Attachments2.push_back(Attachments1[1]); CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments2)); @@ -2625,7 +2627,7 @@ TEST_CASE("cachestore.newgc.basics") CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); auto Attachments = - CreateCompressedAttachment(CidStore, std::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187}); + CreateCompressedAttachment(CidStore, eastl::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187}); IoHash CacheRecord = CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments); { // Do get so it ends up in memcache diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index 73c10a6db..e85383a2a 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -60,21 +60,21 @@ public: CasImpl(GcManager& Gc); virtual ~CasImpl(); - virtual void Initialize(const CidStoreConfiguration& InConfig) override; - virtual CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, InsertMode Mode) override; - virtual std::vector<InsertResult> InsertChunks(std::span<IoBuffer> Data, - std::span<IoHash> ChunkHashes, - InsertMode Mode = InsertMode::kMayBeMovedInPlace) override; - virtual IoBuffer FindChunk(const IoHash& ChunkHash) override; - virtual bool ContainsChunk(const IoHash& ChunkHash) override; - virtual void FilterChunks(HashKeySet& InOutChunks) override; - virtual bool IterateChunks(std::span<IoHash> DecompressedIds, - const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool, - uint64_t LargeSizeLimit) override; - virtual void Flush() override; - virtual void ScrubStorage(ScrubContext& Ctx) override; - virtual CidStoreSize TotalSize() const override; + virtual void Initialize(const CidStoreConfiguration& InConfig) override; + virtual CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, InsertMode Mode) override; + virtual eastl::vector<InsertResult> InsertChunks(std::span<IoBuffer> Data, + std::span<IoHash> ChunkHashes, + InsertMode Mode = InsertMode::kMayBeMovedInPlace) override; + virtual IoBuffer FindChunk(const IoHash& ChunkHash) override; + virtual bool ContainsChunk(const IoHash& ChunkHash) override; + virtual void FilterChunks(HashKeySet& InOutChunks) override; + virtual bool IterateChunks(std::span<IoHash> DecompressedIds, + const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit) override; + virtual void Flush() override; + virtual void ScrubStorage(ScrubContext& Ctx) override; + virtual CidStoreSize TotalSize() const override; private: CasContainerStrategy m_TinyStrategy; @@ -126,8 +126,8 @@ CasImpl::Initialize(const CidStoreConfiguration& InConfig) // Initialize payload storage { - WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Burst); - std::vector<std::future<void>> Work; + WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Burst); + eastl::vector<std::future<void>> Work; Work.emplace_back( WorkerPool.EnqueueTask(std::packaged_task<void()>{[&]() { m_LargeStrategy.Initialize(m_Config.RootDirectory, IsNewStore); }})); Work.emplace_back(WorkerPool.EnqueueTask(std::packaged_task<void()>{[&]() { @@ -259,11 +259,11 @@ CasImpl::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, InsertMode Mode) } static void -GetCompactCasResults(CasContainerStrategy& Strategy, - std::span<IoBuffer> Data, - std::span<IoHash> ChunkHashes, - std::span<size_t> Indexes, - std::vector<CasStore::InsertResult> Results) +GetCompactCasResults(CasContainerStrategy& Strategy, + std::span<IoBuffer> Data, + std::span<IoHash> ChunkHashes, + std::span<size_t> Indexes, + eastl::vector<CasStore::InsertResult> Results) { const size_t Count = Indexes.size(); if (Count == 1) @@ -272,8 +272,8 @@ GetCompactCasResults(CasContainerStrategy& Strategy, Results[Index] = Strategy.InsertChunk(Data[Index], ChunkHashes[Index]); return; } - std::vector<IoBuffer> Chunks; - std::vector<IoHash> Hashes; + eastl::vector<IoBuffer> Chunks; + eastl::vector<IoHash> Hashes; Chunks.reserve(Count); Hashes.reserve(Count); for (size_t Index : Indexes) @@ -292,12 +292,12 @@ GetCompactCasResults(CasContainerStrategy& Strategy, }; static void -GetFileCasResults(FileCasStrategy& Strategy, - CasStore::InsertMode Mode, - std::span<IoBuffer> Data, - std::span<IoHash> ChunkHashes, - std::span<size_t> Indexes, - std::vector<CasStore::InsertResult> Results) +GetFileCasResults(FileCasStrategy& Strategy, + CasStore::InsertMode Mode, + std::span<IoBuffer> Data, + std::span<IoHash> ChunkHashes, + std::span<size_t> Indexes, + eastl::vector<CasStore::InsertResult> Results) { for (size_t Index : Indexes) { @@ -305,7 +305,7 @@ GetFileCasResults(FileCasStrategy& Strategy, } }; -std::vector<CasStore::InsertResult> +eastl::vector<CasStore::InsertResult> CasImpl::InsertChunks(std::span<IoBuffer> Data, std::span<IoHash> ChunkHashes, CasStore::InsertMode Mode) { ZEN_MEMSCOPE(GetCasTag()); @@ -314,14 +314,14 @@ CasImpl::InsertChunks(std::span<IoBuffer> Data, std::span<IoHash> ChunkHashes, C if (Data.size() == 1) { - std::vector<CasStore::InsertResult> Result(1); + eastl::vector<CasStore::InsertResult> Result(1); Result[0] = InsertChunk(Data[0], ChunkHashes[0], Mode); return Result; } - std::vector<size_t> TinyIndexes; - std::vector<size_t> SmallIndexes; - std::vector<size_t> LargeIndexes; + eastl::vector<size_t> TinyIndexes; + eastl::vector<size_t> SmallIndexes; + eastl::vector<size_t> LargeIndexes; for (size_t Index = 0; Index < Data.size(); Index++) { @@ -341,7 +341,7 @@ CasImpl::InsertChunks(std::span<IoBuffer> Data, std::span<IoHash> ChunkHashes, C } } - std::vector<CasStore::InsertResult> Result(Data.size()); + eastl::vector<CasStore::InsertResult> Result(Data.size()); if (!TinyIndexes.empty()) { diff --git a/src/zenstore/cas.h b/src/zenstore/cas.h index e279dd2cc..5e7085d9b 100644 --- a/src/zenstore/cas.h +++ b/src/zenstore/cas.h @@ -39,19 +39,19 @@ public: virtual void Initialize(const CidStoreConfiguration& Config) = 0; virtual InsertResult InsertChunk(IoBuffer Data, const IoHash& ChunkHash, InsertMode Mode = InsertMode::kMayBeMovedInPlace) = 0; - virtual std::vector<InsertResult> InsertChunks(std::span<IoBuffer> Data, - std::span<IoHash> ChunkHashes, - InsertMode Mode = InsertMode::kMayBeMovedInPlace) = 0; - virtual IoBuffer FindChunk(const IoHash& ChunkHash) = 0; - virtual bool ContainsChunk(const IoHash& ChunkHash) = 0; - virtual void FilterChunks(HashKeySet& InOutChunks) = 0; - virtual bool IterateChunks(std::span<IoHash> DecompressedIds, - const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool, - uint64_t LargeSizeLimit) = 0; - virtual void Flush() = 0; - virtual void ScrubStorage(ScrubContext& Ctx) = 0; - virtual CidStoreSize TotalSize() const = 0; + virtual eastl::vector<InsertResult> InsertChunks(std::span<IoBuffer> Data, + std::span<IoHash> ChunkHashes, + InsertMode Mode = InsertMode::kMayBeMovedInPlace) = 0; + virtual IoBuffer FindChunk(const IoHash& ChunkHash) = 0; + virtual bool ContainsChunk(const IoHash& ChunkHash) = 0; + virtual void FilterChunks(HashKeySet& InOutChunks) = 0; + virtual bool IterateChunks(std::span<IoHash> DecompressedIds, + const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit) = 0; + virtual void Flush() = 0; + virtual void ScrubStorage(ScrubContext& Ctx) = 0; + virtual CidStoreSize TotalSize() const = 0; protected: CidStoreConfiguration m_Config; diff --git a/src/zenstore/caslog.cpp b/src/zenstore/caslog.cpp index 6c7b1b297..fe3970a02 100644 --- a/src/zenstore/caslog.cpp +++ b/src/zenstore/caslog.cpp @@ -193,7 +193,7 @@ CasLogFile::Replay(std::function<void(const void*)>&& Handler, uint64_t SkipEntr const uint64_t MaxBufferSize = 1024 * 1024; - std::vector<uint8_t> ReadBuffer; + eastl::vector<uint8_t> ReadBuffer; ReadBuffer.resize((Min(LogDataSize, MaxBufferSize) / m_RecordSize) * m_RecordSize); uint64_t ReadOffset = 0; diff --git a/src/zenstore/cidstore.cpp b/src/zenstore/cidstore.cpp index 2ab769d04..ca505cc7a 100644 --- a/src/zenstore/cidstore.cpp +++ b/src/zenstore/cidstore.cpp @@ -45,16 +45,16 @@ struct CidStore::Impl return {.New = Result.New}; } - std::vector<CidStore::InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas, std::span<IoHash> RawHashes, CidStore::InsertMode Mode) + eastl::vector<CidStore::InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas, std::span<IoHash> RawHashes, CidStore::InsertMode Mode) { if (ChunkDatas.size() == 1) { - std::vector<CidStore::InsertResult> Result(1); + eastl::vector<CidStore::InsertResult> Result(1); Result[0] = AddChunk(ChunkDatas[0], RawHashes[0], Mode); return Result; } ZEN_ASSERT(ChunkDatas.size() == RawHashes.size()); - std::vector<IoBuffer> Chunks; + eastl::vector<IoBuffer> Chunks; Chunks.reserve(ChunkDatas.size()); #if ZEN_BUILD_DEBUG size_t Offset = 0; @@ -74,10 +74,10 @@ struct CidStore::Impl metrics::RequestStats::Scope $(m_AddChunkOps, TotalSize); - std::vector<CasStore::InsertResult> CasResults = + eastl::vector<CasStore::InsertResult> CasResults = m_CasStore.InsertChunks(Chunks, RawHashes, static_cast<CasStore::InsertMode>(Mode)); ZEN_ASSERT(CasResults.size() == ChunkDatas.size()); - std::vector<CidStore::InsertResult> Result; + eastl::vector<CidStore::InsertResult> Result; for (const CasStore::InsertResult& CasResult : CasResults) { if (CasResult.New) @@ -197,7 +197,7 @@ CidStore::AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode return m_Impl->AddChunk(ChunkData, RawHash, Mode); } -std::vector<CidStore::InsertResult> +eastl::vector<CidStore::InsertResult> CidStore::AddChunks(std::span<IoBuffer> ChunkDatas, std::span<IoHash> RawHashes, InsertMode Mode) { return m_Impl->AddChunks(ChunkDatas, RawHashes, Mode); diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 2be0542db..2e331ab34 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -225,14 +225,14 @@ CasContainerStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) return InsertChunk(Chunk.Data(), Chunk.Size(), ChunkHash); } -std::vector<CasStore::InsertResult> +eastl::vector<CasStore::InsertResult> CasContainerStrategy::InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash> ChunkHashes) { ZEN_MEMSCOPE(GetCasContainerTag()); ZEN_ASSERT(Chunks.size() == ChunkHashes.size()); - std::vector<CasStore::InsertResult> Result(Chunks.size()); - std::vector<size_t> NewChunkIndexes; + eastl::vector<CasStore::InsertResult> Result(Chunks.size()); + eastl::vector<size_t> NewChunkIndexes; { RwLock::SharedLockScope _(m_LocationMapLock); for (size_t ChunkIndex = 0; ChunkIndex < ChunkHashes.size(); ChunkIndex++) @@ -252,7 +252,7 @@ CasContainerStrategy::InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash> return Result; } - std::vector<IoBuffer> Datas; + eastl::vector<IoBuffer> Datas; for (size_t ChunkIndex : NewChunkIndexes) { const IoBuffer& Chunk = Chunks[ChunkIndex]; @@ -265,7 +265,7 @@ CasContainerStrategy::InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash> size_t ChunkOffset = 0; m_BlockStore.WriteChunks(Datas, m_PayloadAlignment, [&](std::span<BlockStoreLocation> Locations) { ZEN_MEMSCOPE(GetCasContainerTag()); - std::vector<CasDiskIndexEntry> IndexEntries; + eastl::vector<CasDiskIndexEntry> IndexEntries; for (const BlockStoreLocation& Location : Locations) { size_t ChunkIndex = NewChunkIndexes[ChunkOffset++]; @@ -330,9 +330,9 @@ CasContainerStrategy::IterateChunks(std::span<IoHash> ChunkHashes, { ZEN_MEMSCOPE(GetCasContainerTag()); - const size_t ChunkCount = ChunkHashes.size(); - std::vector<size_t> FoundChunkIndexes; - std::vector<BlockStoreLocation> FoundChunkLocations; + const size_t ChunkCount = ChunkHashes.size(); + eastl::vector<size_t> FoundChunkIndexes; + eastl::vector<BlockStoreLocation> FoundChunkLocations; FoundChunkIndexes.reserve(ChunkCount); FoundChunkLocations.reserve(ChunkCount); { @@ -395,7 +395,7 @@ CasContainerStrategy::IterateChunks(std::span<IoHash> ChunkHashes, if (OptionalWorkerPool && (ChunkIndexes.size() > 3)) { WorkLatch.AddCount(1); - OptionalWorkerPool->ScheduleWork([&, ChunkIndexes = std::vector<size_t>(ChunkIndexes.begin(), ChunkIndexes.end())]() { + OptionalWorkerPool->ScheduleWork([&, ChunkIndexes = eastl::vector<size_t>(ChunkIndexes.begin(), ChunkIndexes.end())]() { auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); if (!AsyncContinue) { @@ -455,11 +455,11 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx) ZEN_INFO("scrubbing '{}'", m_BlocksBasePath); - RwLock BadKeysLock; - std::vector<IoHash> BadKeys; - std::atomic_uint64_t ChunkCount{0}, ChunkBytes{0}; - std::vector<BlockStoreLocation> ChunkLocations; - std::vector<IoHash> ChunkIndexToChunkHash; + RwLock BadKeysLock; + eastl::vector<IoHash> BadKeys; + std::atomic_uint64_t ChunkCount{0}, ChunkBytes{0}; + eastl::vector<BlockStoreLocation> ChunkLocations; + eastl::vector<IoHash> ChunkIndexToChunkHash; try { @@ -550,7 +550,7 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx) { // Deal with bad chunks by removing them from our lookup map - std::vector<CasDiskIndexEntry> LogEntries; + eastl::vector<CasDiskIndexEntry> LogEntries; LogEntries.reserve(BadKeys.size()); { RwLock::ExclusiveLockScope IndexLock(m_LocationMapLock); @@ -642,7 +642,7 @@ public: { BlockStoreCompactState BlockCompactState; - std::vector<IoHash> BlockCompactStateKeys; + eastl::vector<IoHash> BlockCompactStateKeys; BlockStore::BlockEntryCountMap BlocksToCompact = m_CasContainerStrategy.m_BlockStore.GetBlocksToCompact(BlockUsage, Ctx.Settings.CompactBlockUsageThresholdPercent); @@ -678,8 +678,8 @@ public: BlockCompactState, m_CasContainerStrategy.m_PayloadAlignment, [&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) { - std::vector<CasDiskIndexEntry> MovedEntries; - RwLock::ExclusiveLockScope _(m_CasContainerStrategy.m_LocationMapLock); + eastl::vector<CasDiskIndexEntry> MovedEntries; + RwLock::ExclusiveLockScope _(m_CasContainerStrategy.m_LocationMapLock); for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray) { size_t ChunkIndex = Moved.first; @@ -736,7 +736,7 @@ public: class CasContainerReferencePruner : public GcReferencePruner { public: - CasContainerReferencePruner(CasContainerStrategy& Owner, std::vector<IoHash>&& Cids) + CasContainerReferencePruner(CasContainerStrategy& Owner, eastl::vector<IoHash>&& Cids) : m_CasContainerStrategy(Owner) , m_Cids(std::move(Cids)) { @@ -780,7 +780,7 @@ public: { if (Ctx.Settings.IsDeleteMode) { - std::vector<CasDiskIndexEntry> ExpiredEntries; + eastl::vector<CasDiskIndexEntry> ExpiredEntries; ExpiredEntries.reserve(UnusedCids.size()); { @@ -819,7 +819,7 @@ public: private: CasContainerStrategy& m_CasContainerStrategy; - std::vector<IoHash> m_Cids; + eastl::vector<IoHash> m_Cids; }; std::string @@ -847,7 +847,7 @@ CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats&) NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); - std::vector<IoHash> CidsToCheck; + eastl::vector<IoHash> CidsToCheck; { RwLock::SharedLockScope __(m_LocationMapLock); if (m_LocationMap.empty()) @@ -878,9 +878,9 @@ CasContainerStrategy::CompactIndex(RwLock::ExclusiveLockScope&) ZEN_MEMSCOPE(GetCasContainerTag()); ZEN_TRACE_CPU("CasContainer::CompactIndex"); - size_t EntryCount = m_LocationMap.size(); - LocationMap_t LocationMap; - std::vector<BlockStoreDiskLocation> Locations; + size_t EntryCount = m_LocationMap.size(); + LocationMap_t LocationMap; + eastl::vector<BlockStoreDiskLocation> Locations; Locations.reserve(EntryCount); LocationMap.reserve(EntryCount); for (auto It : m_LocationMap) @@ -945,8 +945,8 @@ CasContainerStrategy::MakeIndexSnapshot() } // Write the current state of the location map to a new index state - std::vector<CasDiskIndexEntry> Entries; - uint64_t IndexLogPosition = 0; + eastl::vector<CasDiskIndexEntry> Entries; + uint64_t IndexLogPosition = 0; { RwLock::SharedLockScope ___(m_LocationMapLock); @@ -1042,7 +1042,7 @@ CasContainerStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint m_Locations.reserve(ExpectedEntryCount); m_LocationMap.reserve(ExpectedEntryCount); - std::vector<CasDiskIndexEntry> Entries; + eastl::vector<CasDiskIndexEntry> Entries; Entries.resize(128 * 1024 / sizeof(CasDiskIndexEntry)); uint64_t RemainingEntries = Header.EntryCount; @@ -1258,7 +1258,7 @@ TEST_CASE("compactcas.compact.gc") const int kIterationCount = 1000; - std::vector<IoHash> Keys(kIterationCount); + eastl::vector<IoHash> Keys(kIterationCount); { GcManager Gc; @@ -1478,7 +1478,7 @@ TEST_CASE("compactcas.threadedinsert") } tsl::robin_set<IoHash, IoHash::Hasher> ChunksToDelete; - std::vector<IoHash> KeepHashes(GcChunkHashes.begin(), GcChunkHashes.end()); + eastl::vector<IoHash> KeepHashes(GcChunkHashes.begin(), GcChunkHashes.end()); size_t C = 0; while (C < KeepHashes.size()) @@ -1503,7 +1503,7 @@ TEST_CASE("compactcas.threadedinsert") auto DoGC = [](CasContainerStrategy& Cas, const tsl::robin_set<IoHash, IoHash::Hasher>& ChunksToDelete, - const std::vector<IoHash>& KeepHashes, + const eastl::vector<IoHash>& KeepHashes, tsl::robin_set<IoHash, IoHash::Hasher>& GcChunkHashes) { std::atomic_bool IsCancelledFlag = false; GcCtx Ctx = {.Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), @@ -1519,8 +1519,8 @@ TEST_CASE("compactcas.threadedinsert") GcStats Stats; GcStoreCompactor* Compactor = Pruner->RemoveUnreferencedData(Ctx, Stats, [&](std::span<IoHash> References) -> std::span<IoHash> { - std::vector<IoHash> Unreferenced; - HashKeySet Retain; + eastl::vector<IoHash> Unreferenced; + HashKeySet Retain; Retain.AddHashesToSet(KeepHashes); for (const IoHash& ChunkHash : References) { diff --git a/src/zenstore/compactcas.h b/src/zenstore/compactcas.h index 07e620086..f72a707ca 100644 --- a/src/zenstore/compactcas.h +++ b/src/zenstore/compactcas.h @@ -51,21 +51,21 @@ struct CasContainerStrategy final : public GcStorage, public GcReferenceStore CasContainerStrategy(GcManager& Gc); ~CasContainerStrategy(); - CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash); - std::vector<CasStore::InsertResult> InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash> ChunkHashes); - IoBuffer FindChunk(const IoHash& ChunkHash); - bool HaveChunk(const IoHash& ChunkHash); - void FilterChunks(HashKeySet& InOutChunks); - bool IterateChunks(std::span<IoHash> ChunkHashes, - const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool, - uint64_t LargeSizeLimit); - void Initialize(const std::filesystem::path& RootDirectory, - const std::string_view ContainerBaseName, - uint32_t MaxBlockSize, - uint32_t Alignment, - bool IsNewStore); - void Flush(); + CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash); + eastl::vector<CasStore::InsertResult> InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash> ChunkHashes); + IoBuffer FindChunk(const IoHash& ChunkHash); + bool HaveChunk(const IoHash& ChunkHash); + void FilterChunks(HashKeySet& InOutChunks); + bool IterateChunks(std::span<IoHash> ChunkHashes, + const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit); + void Initialize(const std::filesystem::path& RootDirectory, + const std::string_view ContainerBaseName, + uint32_t MaxBlockSize, + uint32_t Alignment, + bool IsNewStore); + void Flush(); // GcStorage @@ -100,7 +100,7 @@ private: RwLock m_LocationMapLock; typedef tsl::robin_map<IoHash, size_t, IoHash::Hasher> LocationMap_t; LocationMap_t m_LocationMap; - std::vector<BlockStoreDiskLocation> m_Locations; + eastl::vector<BlockStoreDiskLocation> m_Locations; friend class CasContainerReferencePruner; friend class CasContainerStoreCompactor; diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 34db51aa9..36942b515 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -204,7 +204,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN } return false; } - std::vector<std::filesystem::path> ShardedRoots; + eastl::vector<std::filesystem::path> ShardedRoots; } CasVisitor; FileSystemTraversal Traversal; @@ -619,8 +619,8 @@ FileCasStrategy::IterateChunks(std::span<IoHash> ChunkHashes, const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, WorkerThreadPool* OptionalWorkerPool) { - std::vector<size_t> FoundChunkIndexes; - std::vector<uint64_t> FoundChunkExpectedSizes; + eastl::vector<size_t> FoundChunkIndexes; + eastl::vector<uint64_t> FoundChunkExpectedSizes; { RwLock::SharedLockScope _(m_Lock); for (size_t ChunkIndex = 0; ChunkIndex < ChunkHashes.size(); ChunkIndex++) @@ -697,8 +697,8 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, IoBuffer&& ZEN_ASSERT(m_IsInitialized); - std::vector<IoHash> RawHashes; - std::vector<uint64_t> ExpectedSizes; + eastl::vector<IoHash> RawHashes; + eastl::vector<uint64_t> ExpectedSizes; { RwLock::SharedLockScope _(m_Lock); @@ -748,14 +748,14 @@ FileCasStrategy::ScrubStorage(ScrubContext& Ctx) ZEN_ASSERT(m_IsInitialized); - std::vector<IoHash> BadHashes; - uint64_t ChunkCount{0}, ChunkBytes{0}; + eastl::vector<IoHash> BadHashes; + uint64_t ChunkCount{0}, ChunkBytes{0}; int DiscoveredFilesNotInIndex = 0; { - std::vector<FileCasStrategy::FileCasIndexEntry> ScannedEntries = FileCasStrategy::ScanFolderForCasFiles(m_RootDirectory); - RwLock::ExclusiveLockScope _(m_Lock); + eastl::vector<FileCasStrategy::FileCasIndexEntry> ScannedEntries = FileCasStrategy::ScanFolderForCasFiles(m_RootDirectory); + RwLock::ExclusiveLockScope _(m_Lock); for (const FileCasStrategy::FileCasIndexEntry& Entry : ScannedEntries) { if (m_Index.insert_or_assign(Entry.Key, IndexEntry{.Size = Entry.Size}).second) @@ -959,8 +959,8 @@ FileCasStrategy::MakeIndexSnapshot() } // Write the current state of the location map to a new index state - std::vector<FileCasIndexEntry> Entries; - uint64_t IndexLogPosition = 0; + eastl::vector<FileCasIndexEntry> Entries; + uint64_t IndexLogPosition = 0; { RwLock::SharedLockScope __(m_Lock); @@ -1031,7 +1031,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& using namespace filecas::impl; - std::vector<FileCasIndexEntry> Entries; + eastl::vector<FileCasIndexEntry> Entries; if (std::filesystem::is_regular_file(IndexPath)) { Stopwatch Timer; @@ -1093,7 +1093,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& std::filesystem::path LogPath = GetLogPath(m_RootDirectory); - std::vector<FileCasStrategy::FileCasIndexEntry> ScannedEntries = FileCasStrategy::ScanFolderForCasFiles(m_RootDirectory); + eastl::vector<FileCasStrategy::FileCasIndexEntry> ScannedEntries = FileCasStrategy::ScanFolderForCasFiles(m_RootDirectory); CasLog.Open(LogPath, CasLogFile::Mode::kTruncate); std::string InvalidEntryReason; for (const FileCasStrategy::FileCasIndexEntry& Entry : ScannedEntries) @@ -1163,17 +1163,19 @@ FileCasStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t SkipEntr return 0; } -std::vector<FileCasStrategy::FileCasIndexEntry> +eastl::vector<FileCasStrategy::FileCasIndexEntry> FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir) { ZEN_TRACE_CPU("FileCas::ScanFolderForCasFiles"); using namespace filecas::impl; - std::vector<FileCasIndexEntry> Entries; + eastl::vector<FileCasIndexEntry> Entries; struct Visitor : public FileSystemTraversal::TreeVisitor { - Visitor(const std::filesystem::path& RootDir, std::vector<FileCasIndexEntry>& Entries) : RootDirectory(RootDir), Entries(Entries) {} + Visitor(const std::filesystem::path& RootDir, eastl::vector<FileCasIndexEntry>& Entries) : RootDirectory(RootDir), Entries(Entries) + { + } virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override { std::filesystem::path RelPath = std::filesystem::relative(Parent, RootDirectory); @@ -1202,8 +1204,8 @@ FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir) virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; } - const std::filesystem::path& RootDirectory; - std::vector<FileCasIndexEntry>& Entries; + const std::filesystem::path& RootDirectory; + eastl::vector<FileCasIndexEntry>& Entries; } CasVisitor{RootDir, Entries}; FileSystemTraversal Traversal; @@ -1214,7 +1216,7 @@ FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir) class FileCasStoreCompactor : public GcStoreCompactor { public: - FileCasStoreCompactor(FileCasStrategy& Owner, std::vector<IoHash>&& ReferencesToClean) + FileCasStoreCompactor(FileCasStrategy& Owner, eastl::vector<IoHash>&& ReferencesToClean) : m_FileCasStrategy(Owner) , m_ReferencesToClean(std::move(ReferencesToClean)) { @@ -1352,14 +1354,14 @@ public: virtual std::string GetGcName(GcCtx& Ctx) override { return m_FileCasStrategy.GetGcName(Ctx); } private: - FileCasStrategy& m_FileCasStrategy; - std::vector<IoHash> m_ReferencesToClean; + FileCasStrategy& m_FileCasStrategy; + eastl::vector<IoHash> m_ReferencesToClean; }; class FileCasReferencePruner : public GcReferencePruner { public: - FileCasReferencePruner(FileCasStrategy& Owner, std::vector<IoHash>&& Cids) : m_FileCasStrategy(Owner), m_Cids(std::move(Cids)) {} + FileCasReferencePruner(FileCasStrategy& Owner, eastl::vector<IoHash>&& Cids) : m_FileCasStrategy(Owner), m_Cids(std::move(Cids)) {} virtual std::string GetGcName(GcCtx& Ctx) override { return m_FileCasStrategy.GetGcName(Ctx); } @@ -1397,7 +1399,7 @@ public: if (Ctx.Settings.IsDeleteMode) { - std::vector<FileCasStrategy::FileCasIndexEntry> ExpiredEntries; + eastl::vector<FileCasStrategy::FileCasIndexEntry> ExpiredEntries; ExpiredEntries.reserve(UnusedCids.size()); { RwLock::ExclusiveLockScope __(m_FileCasStrategy.m_Lock); @@ -1432,12 +1434,12 @@ public: } } - return new FileCasStoreCompactor(m_FileCasStrategy, std::vector<IoHash>(UnusedCids.begin(), UnusedCids.end())); + return new FileCasStoreCompactor(m_FileCasStrategy, eastl::vector<IoHash>(UnusedCids.begin(), UnusedCids.end())); } private: - FileCasStrategy& m_FileCasStrategy; - std::vector<IoHash> m_Cids; + FileCasStrategy& m_FileCasStrategy; + eastl::vector<IoHash> m_Cids; }; std::string @@ -1455,15 +1457,15 @@ FileCasStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats&) auto Log = [&Ctx]() { return Ctx.Logger; }; - Stopwatch Timer; - const auto _ = MakeGuard([&] { - if (!Ctx.Settings.Verbose) - { - return; - } - ZEN_INFO("GCV2: filecas [CREATE PRUNER] '{}' in {}", m_RootDirectory, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }); - std::vector<IoHash> CidsToCheck; + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: filecas [CREATE PRUNER] '{}' in {}", m_RootDirectory, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + eastl::vector<IoHash> CidsToCheck; { RwLock::SharedLockScope __(m_Lock); if (m_Index.empty()) @@ -1740,7 +1742,7 @@ TEST_CASE("cas.file.move") # if 0 SUBCASE("stresstest") { - std::vector<IoHash> PayloadHashes; + eastl::vector<IoHash> PayloadHashes; const int kWorkers = 64; const int kItemCount = 128; @@ -1761,7 +1763,7 @@ TEST_CASE("cas.file.move") std::barrier Sync{kWorkers}; auto PopulateAll = [&](int w) { - std::vector<IoBuffer> Buffers; + eastl::vector<IoBuffer> Buffers; for (int i = 0; i < kItemCount; ++i) { @@ -1773,7 +1775,7 @@ TEST_CASE("cas.file.move") } }; - std::vector<std::jthread> Threads; + eastl::vector<std::jthread> Threads; for (int i = 0; i < kWorkers; ++i) { diff --git a/src/zenstore/filecas.h b/src/zenstore/filecas.h index 21d8c3b9e..c3adf125c 100644 --- a/src/zenstore/filecas.h +++ b/src/zenstore/filecas.h @@ -80,8 +80,8 @@ private: uint32_t Flags = 0; uint64_t Size = 0; }; - static bool ValidateEntry(const FileCasIndexEntry& Entry, std::string& OutReason); - static std::vector<FileCasStrategy::FileCasIndexEntry> ScanFolderForCasFiles(const std::filesystem::path& RootDir); + static bool ValidateEntry(const FileCasIndexEntry& Entry, std::string& OutReason); + static eastl::vector<FileCasStrategy::FileCasIndexEntry> ScanFolderForCasFiles(const std::filesystem::path& RootDir); static_assert(sizeof(FileCasIndexEntry) == 32); diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index 7ac10d613..258ec80ca 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -536,7 +536,7 @@ CompareForGC(const IoHash& Lhs, const IoHash& Rhs) } bool -FilterReferences(GcCtx& Ctx, std::string_view Context, std::vector<IoHash>& InOutReferences) +FilterReferences(GcCtx& Ctx, std::string_view Context, eastl::vector<IoHash>& InOutReferences) { if (InOutReferences.empty()) { @@ -564,7 +564,7 @@ FilterReferences(GcCtx& Ctx, std::string_view Context, std::vector<IoHash>& InOu if (Filter) { - std::erase_if(InOutReferences, [&Ctx](const IoHash& Key) { + zen::erase_if(InOutReferences, [&Ctx](const IoHash& Key) { return ((Ctx.Settings.AttachmentRangeMax < Key) || (Key < Ctx.Settings.AttachmentRangeMin)); }); RemovedCount = TotalCount - InOutReferences.size(); @@ -651,7 +651,7 @@ void GcManager::RemoveGcReferencer(GcReferencer& Referencer) { RwLock::ExclusiveLockScope _(m_Lock); - std::erase_if(m_GcReferencers, [&](GcReferencer* $) { return $ == &Referencer; }); + zen::erase_if(m_GcReferencers, [&](GcReferencer* $) { return $ == &Referencer; }); } void @@ -666,7 +666,7 @@ GcManager::RemoveGcReferenceLocker(GcReferenceLocker& ReferenceLocker) { ZEN_MEMSCOPE(GetGcTag()); RwLock::ExclusiveLockScope _(m_Lock); - std::erase_if(m_GcReferencerLockers, [&](GcReferenceLocker* $) { return $ == &ReferenceLocker; }); + zen::erase_if(m_GcReferencerLockers, [&](GcReferenceLocker* $) { return $ == &ReferenceLocker; }); } void @@ -680,7 +680,7 @@ void GcManager::RemoveGcReferenceStore(GcReferenceStore& ReferenceStore) { RwLock::ExclusiveLockScope _(m_Lock); - std::erase_if(m_GcReferenceStores, [&](GcReferenceStore* $) { return $ == &ReferenceStore; }); + zen::erase_if(m_GcReferenceStores, [&](GcReferenceStore* $) { return $ == &ReferenceStore; }); } #define SCOPED_TIMER(closure) \ @@ -892,7 +892,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); // The Referencer will create a reference checker that guarantees that the references do not change // as long as it lives - std::vector<GcReferenceChecker*> Checkers; + eastl::vector<GcReferenceChecker*> Checkers; try { { @@ -973,8 +973,8 @@ GcManager::CollectGarbage(const GcSettings& Settings) &ReferenceValidators]() { ZEN_MEMSCOPE(GetGcTag()); - auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); - std::vector<GcReferenceValidator*> Validators; + auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); + eastl::vector<GcReferenceValidator*> Validators; try { { @@ -1069,7 +1069,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) WorkerThreadPool& LockedPhaseThreadPool = Settings.SingleThread ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Background); - std::vector<RwLock::SharedLockScope> LockerScopes; + eastl::vector<RwLock::SharedLockScope> LockerScopes; SCOPED_TIMER(uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); Result.WriteBlockMS = std::chrono::milliseconds(ElapsedMS);); { if (!ReferenceCheckers.empty()) @@ -1093,7 +1093,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) }); for (GcReferenceLocker* ReferenceLocker : m_GcReferencerLockers) { - std::vector<RwLock::SharedLockScope> LockScopes = ReferenceLocker->LockState(Ctx); + eastl::vector<RwLock::SharedLockScope> LockScopes = ReferenceLocker->LockState(Ctx); for (auto It = std::make_move_iterator(LockScopes.begin()); It != std::make_move_iterator(LockScopes.end()); It++) @@ -1373,7 +1373,7 @@ void GcManager::RemoveGcStorage(GcStorage* Storage) { RwLock::ExclusiveLockScope _(m_Lock); - std::erase_if(m_GcStorage, [&](GcStorage* $) { return $ == Storage; }); + zen::erase_if(m_GcStorage, [&](GcStorage* $) { return $ == Storage; }); } void @@ -1442,13 +1442,13 @@ DiskUsageWindow::KeepRange(GcClock::Tick StartTick, GcClock::Tick EndTick) } } -std::vector<uint64_t> +eastl::vector<uint64_t> DiskUsageWindow::GetDiskDeltas(GcClock::Tick StartTick, GcClock::Tick EndTick, GcClock::Tick DeltaWidth, uint64_t& OutMaxDelta) const { ZEN_ASSERT(StartTick != -1); ZEN_ASSERT(DeltaWidth > 0); - std::vector<uint64_t> Result; + eastl::vector<uint64_t> Result; Result.reserve((EndTick - StartTick + DeltaWidth - 1) / DeltaWidth); size_t WindowSize = m_LogWindow.size(); @@ -1729,7 +1729,7 @@ GcScheduler::AppendGCLog(std::string_view Id, GcClock::TimePoint StartTime, cons ZEN_MEMSCOPE(GetGcTag()); try { - std::vector<uint8_t> Blob; + eastl::vector<uint8_t> Blob; { CbObjectWriter Writer; Writer.BeginObject(Id); @@ -2055,7 +2055,7 @@ GcScheduler::SchedulerThread() const int64_t PressureGraphLength = 30; const std::chrono::duration LoadGraphTime = PressureGraphLength * m_Config.MonitorInterval; - std::vector<uint64_t> DiskDeltas; + eastl::vector<uint64_t> DiskDeltas; uint64_t MaxLoad = 0; { @@ -2721,7 +2721,7 @@ TEST_CASE("gc.diskusagewindow") { uint64_t MaxDelta = 0; // 0-10, 10-20, 20-30, 30-40, 40-50, 50-60, 60-70, 70-80 - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(0, 80, 10, MaxDelta); + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(0, 80, 10, MaxDelta); CHECK(DiskDeltas.size() == 8); CHECK(MaxDelta == 15); CHECK(DiskDeltas[0] == 0); @@ -2736,8 +2736,8 @@ TEST_CASE("gc.diskusagewindow") SUBCASE("Sub range") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(20, 40, 10, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(20, 40, 10, MaxDelta); CHECK(DiskDeltas.size() == 2); CHECK(MaxDelta == 10); CHECK(DiskDeltas[0] == 10); // [20:30] @@ -2745,8 +2745,8 @@ TEST_CASE("gc.diskusagewindow") } SUBCASE("Unaligned sub range 1") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(21, 51, 10, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(21, 51, 10, MaxDelta); CHECK(DiskDeltas.size() == 3); CHECK(MaxDelta == 10); CHECK(DiskDeltas[0] == 0); // [21:31] @@ -2755,8 +2755,8 @@ TEST_CASE("gc.diskusagewindow") } SUBCASE("Unaligned end range") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(29, 79, 10, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(29, 79, 10, MaxDelta); CHECK(DiskDeltas.size() == 5); CHECK(MaxDelta == 15); CHECK(DiskDeltas[0] == 0); // [29:39] @@ -2767,8 +2767,8 @@ TEST_CASE("gc.diskusagewindow") } SUBCASE("Ahead of window") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(-40, 0, 10, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(-40, 0, 10, MaxDelta); CHECK(DiskDeltas.size() == 4); CHECK(MaxDelta == 0); CHECK(DiskDeltas[0] == 0); // [-40:-30] @@ -2778,8 +2778,8 @@ TEST_CASE("gc.diskusagewindow") } SUBCASE("After of window") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(90, 120, 10, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(90, 120, 10, MaxDelta); CHECK(DiskDeltas.size() == 3); CHECK(MaxDelta == 0); CHECK(DiskDeltas[0] == 0); // [90:100] @@ -2788,8 +2788,8 @@ TEST_CASE("gc.diskusagewindow") } SUBCASE("Encapsulating window") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(-20, 100, 10, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(-20, 100, 10, MaxDelta); CHECK(DiskDeltas.size() == 12); CHECK(MaxDelta == 15); CHECK(DiskDeltas[0] == 0); // [-20:-10] @@ -2808,8 +2808,8 @@ TEST_CASE("gc.diskusagewindow") SUBCASE("Full range half stride") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(0, 80, 20, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(0, 80, 20, MaxDelta); CHECK(DiskDeltas.size() == 4); CHECK(MaxDelta == 20); CHECK(DiskDeltas[0] == 10); // [0:20] @@ -2820,8 +2820,8 @@ TEST_CASE("gc.diskusagewindow") SUBCASE("Partial odd stride") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(13, 67, 18, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(13, 67, 18, MaxDelta); CHECK(DiskDeltas.size() == 3); CHECK(MaxDelta == 15); CHECK(DiskDeltas[0] == 10); // [13:31] @@ -2850,72 +2850,72 @@ TEST_CASE("gc.keepunusedreferences") IoHash::FromHexString("ab3917854bfef7e7af2c372d795bb907a15cab15"), IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")}; { - std::vector<IoHash> UsedReferences; - std::vector<IoHash> References; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences; + eastl::vector<IoHash> References; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.empty()); } { - std::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::vector<IoHash> References; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + eastl::vector<IoHash> References; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.empty()); } { - std::vector<IoHash> UsedReferences{}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 5); } { - std::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.empty()); } { - std::vector<IoHash> UsedReferences{Hashes[0], Hashes[2], Hashes[4]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[0], Hashes[2], Hashes[4]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 2); CHECK(UnusedReferences[0] == Hashes[1]); CHECK(UnusedReferences[1] == Hashes[3]); } { - std::vector<IoHash> UsedReferences{Hashes[2], Hashes[3], Hashes[4]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[2], Hashes[3], Hashes[4]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 2); CHECK(UnusedReferences[0] == Hashes[0]); CHECK(UnusedReferences[1] == Hashes[1]); } { - std::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 2); CHECK(UnusedReferences[0] == Hashes[3]); CHECK(UnusedReferences[1] == Hashes[4]); } { - std::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2], Hashes[4]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2], Hashes[4]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 1); CHECK(UnusedReferences[0] == Hashes[3]); } { - std::vector<IoHash> UsedReferences{Hashes[1], Hashes[3]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[1], Hashes[3]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 3); CHECK(UnusedReferences[0] == Hashes[0]); CHECK(UnusedReferences[1] == Hashes[2]); @@ -2923,9 +2923,9 @@ TEST_CASE("gc.keepunusedreferences") } { - std::vector<IoHash> UsedReferences{Hashes[0]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[0]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 4); CHECK(UnusedReferences[0] == Hashes[1]); CHECK(UnusedReferences[1] == Hashes[2]); @@ -2934,9 +2934,9 @@ TEST_CASE("gc.keepunusedreferences") } { - std::vector<IoHash> UsedReferences{Hashes[1]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[1]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 4); CHECK(UnusedReferences[0] == Hashes[0]); CHECK(UnusedReferences[1] == Hashes[2]); @@ -2945,9 +2945,9 @@ TEST_CASE("gc.keepunusedreferences") } { - std::vector<IoHash> UsedReferences{Hashes[3]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[3]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 4); CHECK(UnusedReferences[0] == Hashes[0]); CHECK(UnusedReferences[1] == Hashes[1]); @@ -2956,9 +2956,9 @@ TEST_CASE("gc.keepunusedreferences") } { - std::vector<IoHash> UsedReferences{Hashes[4]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[4]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 4); CHECK(UnusedReferences[0] == Hashes[0]); CHECK(UnusedReferences[1] == Hashes[1]); diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h index 97357e5cb..fe38d40ae 100644 --- a/src/zenstore/include/zenstore/blockstore.h +++ b/src/zenstore/include/zenstore/blockstore.h @@ -124,8 +124,8 @@ public: std::unordered_set<uint32_t> m_BlockIndexes; }; - typedef std::vector<std::pair<size_t, BlockStoreLocation>> MovedChunksArray; - typedef std::vector<size_t> ChunkIndexArray; + typedef eastl::vector<std::pair<size_t, BlockStoreLocation>> MovedChunksArray; + typedef eastl::vector<size_t> ChunkIndexArray; typedef std::function<bool(const MovedChunksArray& MovedChunks, uint64_t FreedDiskSpace)> CompactCallback; typedef std::function<uint64_t()> ClaimDiskReserveCallback; @@ -190,11 +190,11 @@ private: tsl::robin_map<uint32_t, Ref<BlockStoreFile>> m_ChunkBlocks; - mutable RwLock m_InsertLock; // used to serialize inserts - Ref<BlockStoreFile> m_WriteBlock; - std::uint32_t m_CurrentInsertOffset = 0; - std::atomic_uint32_t m_WriteBlockIndex{}; - std::vector<uint32_t> m_ActiveWriteBlocks; + mutable RwLock m_InsertLock; // used to serialize inserts + Ref<BlockStoreFile> m_WriteBlock; + std::uint32_t m_CurrentInsertOffset = 0; + std::atomic_uint32_t m_WriteBlockIndex{}; + eastl::vector<uint32_t> m_ActiveWriteBlocks; uint64_t m_MaxBlockSize = 1u << 28; uint64_t m_MaxBlockCount = BlockStoreDiskLocation::MaxBlockIndex + 1; @@ -216,7 +216,7 @@ public: uint32_t BlockIndex = BlockUsageIt.first; ZEN_ASSERT(m_BlockIndexToChunkMapIndex.find(BlockIndex) == m_BlockIndexToChunkMapIndex.end()); - m_KeepChunks.emplace_back(std::vector<size_t>()); + m_KeepChunks.emplace_back(eastl::vector<size_t>()); m_KeepChunks.back().reserve(BlockUsageIt.second); m_BlockIndexToChunkMapIndex.insert_or_assign(BlockIndex, m_KeepChunks.size() - 1); EntryCountTotal += BlockUsageIt.second; @@ -228,7 +228,7 @@ public: { if (m_BlockIndexToChunkMapIndex.find(BlockIndex) == m_BlockIndexToChunkMapIndex.end()) { - m_KeepChunks.emplace_back(std::vector<size_t>()); + m_KeepChunks.emplace_back(eastl::vector<size_t>()); m_BlockIndexToChunkMapIndex.insert_or_assign(BlockIndex, m_KeepChunks.size() - 1); } } @@ -241,8 +241,8 @@ public: return false; } - std::vector<size_t>& KeepChunks = m_KeepChunks[It->second]; - size_t Index = m_ChunkLocations.size(); + eastl::vector<size_t>& KeepChunks = m_KeepChunks[It->second]; + size_t Index = m_ChunkLocations.size(); KeepChunks.push_back(Index); m_ChunkLocations.push_back(Location); return true; @@ -250,11 +250,11 @@ public: const BlockStoreLocation& GetLocation(size_t Index) const { return m_ChunkLocations[Index]; } - void IterateBlocks(std::function<bool(uint32_t BlockIndex, - const std::vector<size_t>& KeepChunkIndexes, - const std::vector<BlockStoreLocation>& ChunkLocations)> Callback) const + void IterateBlocks(std::function<bool(uint32_t BlockIndex, + const eastl::vector<size_t>& KeepChunkIndexes, + const eastl::vector<BlockStoreLocation>& ChunkLocations)> Callback) const { - std::vector<uint32_t> BlockOrder; + eastl::vector<uint32_t> BlockOrder; BlockOrder.reserve(m_BlockIndexToChunkMapIndex.size()); for (auto It : m_BlockIndexToChunkMapIndex) { @@ -275,9 +275,9 @@ public: } private: - tsl::robin_map<uint32_t, size_t> m_BlockIndexToChunkMapIndex; // Maps to which vector in BlockKeepChunks to use for a block - std::vector<std::vector<size_t>> m_KeepChunks; // One vector per block index with index into ChunkLocations - std::vector<BlockStoreLocation> m_ChunkLocations; + tsl::robin_map<uint32_t, size_t> m_BlockIndexToChunkMapIndex; // Maps to which vector in BlockKeepChunks to use for a block + eastl::vector<eastl::vector<size_t>> m_KeepChunks; // One vector per block index with index into ChunkLocations + eastl::vector<BlockStoreLocation> m_ChunkLocations; }; void blockstore_forcelink(); diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h index 05400c784..c5c0ae36e 100644 --- a/src/zenstore/include/zenstore/cache/cachedisklayer.h +++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h @@ -132,11 +132,11 @@ public: struct Info { - std::filesystem::path RootDir; - Configuration Config; - std::vector<std::string> BucketNames; - uint64_t EntryCount = 0; - GcStorageSize StorageSize; + std::filesystem::path RootDir; + Configuration Config; + eastl::vector<std::string> BucketNames; + uint64_t EntryCount = 0; + GcStorageSize StorageSize; }; struct BucketStats @@ -161,9 +161,9 @@ public: struct DiskStats { - std::vector<NamedBucketStats> BucketStats; - uint64_t DiskSize; - uint64_t MemorySize; + eastl::vector<NamedBucketStats> BucketStats; + uint64_t DiskSize; + uint64_t MemorySize; }; explicit ZenCacheDiskLayer(GcManager& Gc, JobQueue& JobQueue, const std::filesystem::path& RootDir, const Configuration& Config); @@ -176,7 +176,7 @@ public: void Get(std::string_view Bucket, const IoHash& HashKey, GetBatchHandle& BatchHandle); struct PutBatchHandle; - PutBatchHandle* BeginPutBatch(std::vector<bool>& OutResult); + PutBatchHandle* BeginPutBatch(eastl::vector<bool>& OutResult); void EndPutBatch(PutBatchHandle* Batch) noexcept; void Put(std::string_view Bucket, @@ -200,11 +200,11 @@ public: CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const; - std::vector<RwLock::SharedLockScope> GetGcReferencerLocks(); + eastl::vector<RwLock::SharedLockScope> GetGcReferencerLocks(); - void EnableUpdateCapture(); - void DisableUpdateCapture(); - std::vector<std::string> GetCapturedBucketsLocked(); + void EnableUpdateCapture(); + void DisableUpdateCapture(); + eastl::vector<std::string> GetCapturedBucketsLocked(); #if ZEN_WITH_TESTS void SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time); @@ -232,7 +232,7 @@ public: void Get(const IoHash& HashKey, GetBatchHandle& BatchHandle); struct PutBatchHandle; - PutBatchHandle* BeginPutBatch(std::vector<bool>& OutResult); + PutBatchHandle* BeginPutBatch(eastl::vector<bool>& OutResult); void EndPutBatch(PutBatchHandle* Batch) noexcept; void Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References, PutBatchHandle* OptionalBatchHandle); uint64_t MemCacheTrim(GcClock::TimePoint ExpireTime); @@ -243,23 +243,23 @@ public: struct ReferencesStats { - std::vector<uint64_t> ValueSizes; - uint64_t StructuredValuesCount = 0; - uint64_t StandaloneValuesCount = 0; + eastl::vector<uint64_t> ValueSizes; + uint64_t StructuredValuesCount = 0; + uint64_t StandaloneValuesCount = 0; }; - bool GetReferences(const LoggerRef& Logger, - std::atomic_bool& IsCancelledFlag, - bool StateIsAlreadyLocked, - bool ReadCacheAttachmentMetaData, - bool WriteCacheAttachmentMetaData, - std::vector<IoHash>& OutReferences, - ReferencesStats* OptionalOutReferencesStats); + bool GetReferences(const LoggerRef& Logger, + std::atomic_bool& IsCancelledFlag, + bool StateIsAlreadyLocked, + bool ReadCacheAttachmentMetaData, + bool WriteCacheAttachmentMetaData, + eastl::vector<IoHash>& OutReferences, + ReferencesStats* OptionalOutReferencesStats); bool ReadAttachmentsFromMetaData(uint32_t BlockIndex, std::span<const IoHash> InlineKeys, std::span<const std::size_t> ChunkIndexes, - std::vector<IoHash>& OutReferences) const; + eastl::vector<IoHash>& OutReferences) const; inline GcStorageSize StorageSize() const { @@ -272,7 +272,7 @@ public: CacheValueDetails::BucketDetails GetValueDetails(RwLock::SharedLockScope& IndexLock, const std::string_view ValueFilter) const; void EnumerateBucketContents(std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const; - void GetUsageByAccess(GcClock::TimePoint Now, GcClock::Duration MaxAge, std::vector<uint64_t>& InOutUsageSlots); + void GetUsageByAccess(GcClock::TimePoint Now, GcClock::Duration MaxAge, eastl::vector<uint64_t>& InOutUsageSlots); #if ZEN_WITH_TESTS void SetAccessTime(const IoHash& HashKey, GcClock::TimePoint Time); #endif // ZEN_WITH_TESTS @@ -362,23 +362,23 @@ public: metrics::RequestStats m_PutOps; metrics::RequestStats m_GetOps; - mutable RwLock m_IndexLock; - IndexMap m_Index; - std::vector<AccessTime> m_AccessTimes; - std::vector<BucketPayload> m_Payloads; - std::vector<BucketMetaData> m_MetaDatas; - std::vector<MetaDataIndex> m_FreeMetaDatas; - std::vector<MemCacheData> m_MemCachedPayloads; - std::vector<MemCachedIndex> m_FreeMemCachedPayloads; - std::unique_ptr<HashSet> m_TrackedCacheKeys; - std::unique_ptr<std::vector<IoHash>> m_TrackedReferences; - std::atomic_uint64_t m_StandaloneSize{}; - std::atomic_uint64_t m_MemCachedSize{}; - - virtual std::string GetGcName(GcCtx& Ctx) override; - virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; - virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; - virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override; + mutable RwLock m_IndexLock; + IndexMap m_Index; + eastl::vector<AccessTime> m_AccessTimes; + eastl::vector<BucketPayload> m_Payloads; + eastl::vector<BucketMetaData> m_MetaDatas; + eastl::vector<MetaDataIndex> m_FreeMetaDatas; + eastl::vector<MemCacheData> m_MemCachedPayloads; + eastl::vector<MemCachedIndex> m_FreeMemCachedPayloads; + std::unique_ptr<HashSet> m_TrackedCacheKeys; + std::unique_ptr<eastl::vector<IoHash>> m_TrackedReferences; + std::atomic_uint64_t m_StandaloneSize{}; + std::atomic_uint64_t m_MemCachedSize{}; + + virtual std::string GetGcName(GcCtx& Ctx) override; + virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; + virtual eastl::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; + virtual eastl::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override; void BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const; void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References); @@ -421,12 +421,12 @@ public: bool FlushLockPosition, const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; }); - void CompactState(RwLock::ExclusiveLockScope& IndexLock, - std::vector<BucketPayload>& Payloads, - std::vector<AccessTime>& AccessTimes, - std::vector<BucketMetaData>& MetaDatas, - std::vector<MemCacheData>& MemCachedPayloads, - IndexMap& Index); + void CompactState(RwLock::ExclusiveLockScope& IndexLock, + eastl::vector<BucketPayload>& Payloads, + eastl::vector<AccessTime>& AccessTimes, + eastl::vector<BucketMetaData>& MetaDatas, + eastl::vector<MemCacheData>& MemCachedPayloads, + IndexMap& Index); void AddMemCacheUsage(uint64_t ValueSize) { @@ -486,24 +486,24 @@ private: StartAsyncMemCacheTrim(); } - uint64_t MemCacheTrim(std::vector<CacheBucket*>& Buckets, GcClock::TimePoint ExpireTime); + uint64_t MemCacheTrim(eastl::vector<CacheBucket*>& Buckets, GcClock::TimePoint ExpireTime); bool StartAsyncMemCacheTrim(); void MemCacheTrim(); typedef eastl::unordered_map<std::string, std::unique_ptr<CacheBucket>, std::hash<std::string>, std::equal_to<std::string>> BucketMap_t; - GcManager& m_Gc; - JobQueue& m_JobQueue; - std::filesystem::path m_RootDir; - Configuration m_Configuration; - std::atomic_uint64_t m_TotalMemCachedSize{}; - std::atomic_bool m_IsMemCacheTrimming = false; - std::atomic<GcClock::Tick> m_NextAllowedTrimTick; - mutable RwLock m_Lock; - BucketMap_t m_Buckets; - std::vector<std::unique_ptr<CacheBucket>> m_DroppedBuckets; - uint32_t m_UpdateCaptureRefCounter = 0; - std::unique_ptr<std::vector<std::string>> m_CapturedBuckets; + GcManager& m_Gc; + JobQueue& m_JobQueue; + std::filesystem::path m_RootDir; + Configuration m_Configuration; + std::atomic_uint64_t m_TotalMemCachedSize{}; + std::atomic_bool m_IsMemCacheTrimming = false; + std::atomic<GcClock::Tick> m_NextAllowedTrimTick; + mutable RwLock m_Lock; + BucketMap_t m_Buckets; + eastl::vector<std::unique_ptr<CacheBucket>> m_DroppedBuckets; + uint32_t m_UpdateCaptureRefCounter = 0; + std::unique_ptr<eastl::vector<std::string>> m_CapturedBuckets; ZenCacheDiskLayer(const ZenCacheDiskLayer&) = delete; ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete; diff --git a/src/zenstore/include/zenstore/cache/cacherpc.h b/src/zenstore/include/zenstore/cache/cacherpc.h index da8cf69fe..f180c80d1 100644 --- a/src/zenstore/include/zenstore/cache/cacherpc.h +++ b/src/zenstore/include/zenstore/cache/cacherpc.h @@ -6,9 +6,9 @@ #include <zencore/logging.h> #include <zenutil/cache/cache.h> +#include <EASTL/vector.h> #include <atomic> #include <string_view> -#include <vector> namespace zen { @@ -110,38 +110,38 @@ private: PutResult PutCacheRecord(PutRequestData& Request, const CbPackage* Package); /** HandleRpcGetCacheChunks Helper: Parse the Body object into RecordValue Requests and Value Requests. */ - bool ParseGetCacheChunksRequest(std::string& Namespace, - std::vector<CacheKeyRequest>& RecordKeys, - std::vector<cache::detail::RecordBody>& Records, - std::vector<CacheChunkRequest>& RequestKeys, - std::vector<cache::detail::ChunkRequest>& Requests, - std::vector<cache::detail::ChunkRequest*>& RecordRequests, - std::vector<cache::detail::ChunkRequest*>& ValueRequests, - CbObjectView RpcRequest); + bool ParseGetCacheChunksRequest(std::string& Namespace, + eastl::vector<CacheKeyRequest>& RecordKeys, + eastl::vector<cache::detail::RecordBody>& Records, + eastl::vector<CacheChunkRequest>& RequestKeys, + eastl::vector<cache::detail::ChunkRequest>& Requests, + eastl::vector<cache::detail::ChunkRequest*>& RecordRequests, + eastl::vector<cache::detail::ChunkRequest*>& ValueRequests, + CbObjectView RpcRequest); /** HandleRpcGetCacheChunks Helper: Load records to get ContentId for RecordRequests, and load their payloads if they exist locally. */ - void GetLocalCacheRecords(const CacheRequestContext& Context, - std::string_view Namespace, - std::vector<CacheKeyRequest>& RecordKeys, - std::vector<cache::detail::RecordBody>& Records, - std::vector<cache::detail::ChunkRequest*>& RecordRequests, - std::vector<CacheChunkRequest*>& OutUpstreamChunks); + void GetLocalCacheRecords(const CacheRequestContext& Context, + std::string_view Namespace, + eastl::vector<CacheKeyRequest>& RecordKeys, + eastl::vector<cache::detail::RecordBody>& Records, + eastl::vector<cache::detail::ChunkRequest*>& RecordRequests, + eastl::vector<CacheChunkRequest*>& OutUpstreamChunks); /** HandleRpcGetCacheChunks Helper: For ValueRequests, load their payloads if they exist locally. */ - void GetLocalCacheValues(const CacheRequestContext& Context, - std::string_view Namespace, - std::vector<cache::detail::ChunkRequest*>& ValueRequests, - std::vector<CacheChunkRequest*>& OutUpstreamChunks); + void GetLocalCacheValues(const CacheRequestContext& Context, + std::string_view Namespace, + eastl::vector<cache::detail::ChunkRequest*>& ValueRequests, + eastl::vector<CacheChunkRequest*>& OutUpstreamChunks); /** HandleRpcGetCacheChunks Helper: Load payloads from upstream that did not exist locally. */ - void GetUpstreamCacheChunks(const CacheRequestContext& Context, - std::string_view Namespace, - std::vector<CacheChunkRequest*>& UpstreamChunks, - std::vector<CacheChunkRequest>& RequestKeys, - std::vector<cache::detail::ChunkRequest>& Requests); + void GetUpstreamCacheChunks(const CacheRequestContext& Context, + std::string_view Namespace, + eastl::vector<CacheChunkRequest*>& UpstreamChunks, + eastl::vector<CacheChunkRequest>& RequestKeys, + eastl::vector<cache::detail::ChunkRequest>& Requests); /** HandleRpcGetCacheChunks Helper: Send response message containing all chunk results. */ - CbPackage WriteGetCacheChunksResponse(const CacheRequestContext& Context, - std::string_view Namespace, - RpcAcceptOptions AcceptOptions, - std::vector<cache::detail::ChunkRequest>& Requests); + CbPackage WriteGetCacheChunksResponse(const CacheRequestContext& Context, + std::string_view Namespace, + RpcAcceptOptions AcceptOptions, + eastl::vector<cache::detail::ChunkRequest>& Requests); LoggerRef Log() { return m_Log; } LoggerRef m_Log; diff --git a/src/zenstore/include/zenstore/cache/cacheshared.h b/src/zenstore/include/zenstore/cache/cacheshared.h index 521c78bb1..bafa3d09e 100644 --- a/src/zenstore/include/zenstore/cache/cacheshared.h +++ b/src/zenstore/include/zenstore/cache/cacheshared.h @@ -23,7 +23,7 @@ namespace access_tracking { struct AccessTimes { - std::unordered_map<std::string, std::vector<KeyAccessTime>> Buckets; + std::unordered_map<std::string, eastl::vector<KeyAccessTime>> Buckets; }; }; // namespace access_tracking @@ -40,12 +40,12 @@ struct CacheValueDetails { struct ValueDetails { - uint64_t Size; - uint64_t RawSize; - IoHash RawHash; - GcClock::Tick LastAccess{}; - std::vector<IoHash> Attachments; - ZenContentType ContentType; + uint64_t Size; + uint64_t RawSize; + IoHash RawHash; + GcClock::Tick LastAccess{}; + eastl::vector<IoHash> Attachments; + ZenContentType ContentType; }; struct BucketDetails @@ -63,10 +63,10 @@ struct CacheValueDetails struct CacheContentStats { - std::vector<uint64_t> ValueSizes; - uint64_t StructuredValuesCount = 0; - uint64_t StandaloneValuesCount = 0; - std::vector<IoHash> Attachments; + eastl::vector<uint64_t> ValueSizes; + uint64_t StructuredValuesCount = 0; + uint64_t StandaloneValuesCount = 0; + eastl::vector<IoHash> Attachments; }; bool IsKnownBadBucketName(std::string_view BucketName); diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h index 5e056cf2d..faaba407f 100644 --- a/src/zenstore/include/zenstore/cache/structuredcachestore.h +++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h @@ -62,10 +62,10 @@ public: }; struct Info { - std::filesystem::path RootDir; - Configuration Config; - std::vector<std::string> BucketNames; - ZenCacheDiskLayer::Info DiskLayerInfo; + std::filesystem::path RootDir; + Configuration Config; + eastl::vector<std::string> BucketNames; + ZenCacheDiskLayer::Info DiskLayerInfo; }; struct NamespaceStats @@ -82,7 +82,7 @@ public: ~ZenCacheNamespace(); struct PutBatchHandle; - PutBatchHandle* BeginPutBatch(std::vector<bool>& OutResults); + PutBatchHandle* BeginPutBatch(eastl::vector<bool>& OutResults); void EndPutBatch(PutBatchHandle* Batch) noexcept; struct GetBatchHandle; @@ -115,7 +115,7 @@ public: CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const; - std::vector<RwLock::SharedLockScope> GetGcReferencerLocks(); + eastl::vector<RwLock::SharedLockScope> GetGcReferencerLocks(); void EnableUpdateCapture(); void DisableUpdateCapture(); @@ -171,11 +171,11 @@ public: struct Info { - std::filesystem::path BasePath; - Configuration Config; - std::vector<std::string> NamespaceNames; - uint64_t DiskEntryCount = 0; - GcStorageSize StorageSize; + std::filesystem::path BasePath; + Configuration Config; + eastl::vector<std::string> NamespaceNames; + uint64_t DiskEntryCount = 0; + GcStorageSize StorageSize; }; struct NamedNamespaceStats @@ -186,14 +186,14 @@ public: struct CacheStoreStats { - uint64_t HitCount = 0; - uint64_t MissCount = 0; - uint64_t WriteCount = 0; - uint64_t RejectedWriteCount = 0; - uint64_t RejectedReadCount = 0; - metrics::RequestStatsSnapshot PutOps; - metrics::RequestStatsSnapshot GetOps; - std::vector<NamedNamespaceStats> NamespaceStats; + uint64_t HitCount = 0; + uint64_t MissCount = 0; + uint64_t WriteCount = 0; + uint64_t RejectedWriteCount = 0; + uint64_t RejectedReadCount = 0; + metrics::RequestStatsSnapshot PutOps; + metrics::RequestStatsSnapshot GetOps; + eastl::vector<NamedNamespaceStats> NamespaceStats; }; ZenCacheStore(GcManager& Gc, @@ -206,7 +206,7 @@ public: class PutBatch { public: - PutBatch(ZenCacheStore& CacheStore, std::string_view Namespace, std::vector<bool>& OutResult); + PutBatch(ZenCacheStore& CacheStore, std::string_view Namespace, eastl::vector<bool>& OutResult); ~PutBatch(); private: @@ -268,7 +268,7 @@ public: Info GetInfo() const; std::optional<ZenCacheNamespace::Info> GetNamespaceInfo(std::string_view Namespace); std::optional<ZenCacheNamespace::BucketInfo> GetBucketInfo(std::string_view Namespace, std::string_view Bucket); - std::vector<std::string> GetNamespaces(); + eastl::vector<std::string> GetNamespaces(); void EnumerateBucketContents(std::string_view Namespace, std::string_view Bucket, @@ -277,16 +277,16 @@ public: // StatsProvider virtual void ReportMetrics(StatsMetrics& Statsd) override; - virtual std::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) override; + virtual eastl::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) override; - virtual std::string GetGcName(GcCtx& Ctx) override; - virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; - virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; - virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override; + virtual std::string GetGcName(GcCtx& Ctx) override; + virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; + virtual eastl::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; + virtual eastl::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override; - void EnableUpdateCapture(); - void DisableUpdateCapture(); - std::vector<std::string> GetCapturedNamespacesLocked(); + void EnableUpdateCapture(); + void DisableUpdateCapture(); + eastl::vector<std::string> GetCapturedNamespacesLocked(); bool GetContentStats(std::string_view Namespace, std::string_view BucketName, CacheContentStats& OutContentStats) const; @@ -297,13 +297,13 @@ private: typedef std::unordered_map<std::string, std::unique_ptr<ZenCacheNamespace>> NamespaceMap; - CacheStoreStats m_LastReportedMetrics; - const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; - mutable RwLock m_NamespacesLock; - NamespaceMap m_Namespaces; - std::vector<std::unique_ptr<ZenCacheNamespace>> m_DroppedNamespaces; - uint32_t m_UpdateCaptureRefCounter = 0; - std::unique_ptr<std::vector<std::string>> m_CapturedNamespaces; + CacheStoreStats m_LastReportedMetrics; + const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; + mutable RwLock m_NamespacesLock; + NamespaceMap m_Namespaces; + eastl::vector<std::unique_ptr<ZenCacheNamespace>> m_DroppedNamespaces; + uint32_t m_UpdateCaptureRefCounter = 0; + std::unique_ptr<eastl::vector<std::string>> m_CapturedNamespaces; GcManager& m_Gc; JobQueue& m_JobQueue; @@ -327,14 +327,14 @@ private: ZenCacheValue Value; }; - void LogWorker(); - RwLock m_LogQueueLock; - std::vector<AccessLogItem> m_LogQueue; - std::atomic_bool m_ExitLogging; - Event m_LogEvent; - std::thread m_AsyncLoggingThread; - std::atomic_bool m_WriteLogEnabled; - std::atomic_bool m_AccessLogEnabled; + void LogWorker(); + RwLock m_LogQueueLock; + eastl::vector<AccessLogItem> m_LogQueue; + std::atomic_bool m_ExitLogging; + Event m_LogEvent; + std::thread m_AsyncLoggingThread; + std::atomic_bool m_WriteLogEnabled; + std::atomic_bool m_AccessLogEnabled; friend class CacheStoreReferenceChecker; }; diff --git a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h index 152031c3a..ee5f93019 100644 --- a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h +++ b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h @@ -10,10 +10,10 @@ #include <zencore/zencore.h> #include <zenutil/cache/cache.h> +#include <EASTL/vector.h> #include <functional> #include <memory> #include <string> -#include <vector> namespace zen { @@ -22,11 +22,11 @@ class CbPackage; struct UpstreamCacheRecord { - ZenContentType Type = ZenContentType::kBinary; - std::string Namespace; - CacheKey Key; - std::vector<IoHash> ValueContentIds; - CacheRequestContext Context; + ZenContentType Type = ZenContentType::kBinary; + std::string Namespace; + CacheKey Key; + eastl::vector<IoHash> ValueContentIds; + CacheRequestContext Context; }; struct UpstreamError diff --git a/src/zenstore/include/zenstore/cidstore.h b/src/zenstore/include/zenstore/cidstore.h index b3d00fec0..ac73d71ea 100644 --- a/src/zenstore/include/zenstore/cidstore.h +++ b/src/zenstore/include/zenstore/cidstore.h @@ -74,22 +74,22 @@ public: kMayBeMovedInPlace }; - void Initialize(const CidStoreConfiguration& Config); - InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode Mode = InsertMode::kMayBeMovedInPlace); - std::vector<InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas, - std::span<IoHash> RawHashes, - InsertMode Mode = InsertMode::kMayBeMovedInPlace); - virtual IoBuffer FindChunkByCid(const IoHash& DecompressedId) override; - bool IterateChunks(std::span<IoHash> DecompressedIds, - const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool, - uint64_t LargeSizeLimit); - bool ContainsChunk(const IoHash& DecompressedId); - void FilterChunks(HashKeySet& InOutChunks); - void Flush(); - void ScrubStorage(ScrubContext& Ctx); - CidStoreSize TotalSize() const; - CidStoreStats Stats() const; + void Initialize(const CidStoreConfiguration& Config); + InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode Mode = InsertMode::kMayBeMovedInPlace); + eastl::vector<InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas, + std::span<IoHash> RawHashes, + InsertMode Mode = InsertMode::kMayBeMovedInPlace); + virtual IoBuffer FindChunkByCid(const IoHash& DecompressedId) override; + bool IterateChunks(std::span<IoHash> DecompressedIds, + const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit); + bool ContainsChunk(const IoHash& DecompressedId); + void FilterChunks(HashKeySet& InOutChunks); + void Flush(); + void ScrubStorage(ScrubContext& Ctx); + CidStoreSize TotalSize() const; + CidStoreStats Stats() const; virtual void ReportMetrics(StatsMetrics& Statsd) override; diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h index 3daae0a93..f98b0a537 100644 --- a/src/zenstore/include/zenstore/gc.h +++ b/src/zenstore/include/zenstore/gc.h @@ -121,9 +121,9 @@ struct GcReferenceStoreStats struct GcResult { - std::vector<std::pair<std::string, GcReferencerStats>> ReferencerStats; - std::vector<std::pair<std::string, GcReferenceStoreStats>> ReferenceStoreStats; - std::vector<std::pair<std::string, GcReferenceValidatorStats>> ReferenceValidatorStats; + eastl::vector<std::pair<std::string, GcReferencerStats>> ReferencerStats; + eastl::vector<std::pair<std::string, GcReferenceStoreStats>> ReferenceStoreStats; + eastl::vector<std::pair<std::string, GcReferenceValidatorStats>> ReferenceValidatorStats; GcReferencerStats ReferencerStatSum; GcReferenceStoreStats ReferenceStoreStatSum; @@ -232,7 +232,7 @@ public: }; std::span<IoHash> KeepUnusedReferences(std::span<const IoHash> SortedUsedReferences, std::span<IoHash> SortedReferences); -bool FilterReferences(GcCtx& Ctx, std::string_view Context, std::vector<IoHash>& InOutReferences); +bool FilterReferences(GcCtx& Ctx, std::string_view Context, eastl::vector<IoHash>& InOutReferences); /** * @brief An interface to implement a lock for Stop The World (from writing new data) @@ -247,7 +247,7 @@ public: // Take all the locks needed to execute UpdateLockedState for the all the GcReferenceChecker in your domain // Once all the GcReferenceChecker has executed UpdateLockedState and GetUnusedReferences for all // domains has completed, the locks will be disposed and writes are allowed once again - virtual std::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) = 0; + virtual eastl::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) = 0; }; /** @@ -268,11 +268,11 @@ public: // Create 0-n GcReferenceChecker for this GcReferencer. Caller will manage lifetime of // returned instances - virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) = 0; + virtual eastl::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) = 0; // Create 0-n GcReferenceValidator for this GcReferencer. Caller will manage lifetime of // returned instances - virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) = 0; + virtual eastl::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) = 0; }; /** @@ -377,17 +377,17 @@ public: void SetDiskWriteBlocker(const DiskWriteBlocker* Monitor) { m_DiskWriteBlocker = Monitor; } private: - bool CheckGCCancel() { return m_CancelGC.load(); } - LoggerRef Log() { return m_Log; } - LoggerRef m_Log; - mutable RwLock m_Lock; - std::vector<GcStorage*> m_GcStorage; - CidStore* m_CidStore = nullptr; - const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; - - std::vector<GcReferencer*> m_GcReferencers; - std::vector<GcReferenceLocker*> m_GcReferencerLockers; - std::vector<GcReferenceStore*> m_GcReferenceStores; + bool CheckGCCancel() { return m_CancelGC.load(); } + LoggerRef Log() { return m_Log; } + LoggerRef m_Log; + mutable RwLock m_Lock; + eastl::vector<GcStorage*> m_GcStorage; + CidStore* m_CidStore = nullptr; + const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; + + eastl::vector<GcReferencer*> m_GcReferencers; + eastl::vector<GcReferenceLocker*> m_GcReferencerLockers; + eastl::vector<GcReferenceStore*> m_GcReferenceStores; std::atomic_bool m_CancelGC{false}; }; @@ -466,15 +466,15 @@ public: uint64_t DiskUsage; }; - std::vector<DiskUsageEntry> m_LogWindow; - inline void Append(const DiskUsageEntry& Entry) { m_LogWindow.push_back(Entry); } - inline void Append(DiskUsageEntry&& Entry) { m_LogWindow.emplace_back(std::move(Entry)); } - void KeepRange(GcClock::Tick StartTick, GcClock::Tick EndTick); - std::vector<uint64_t> GetDiskDeltas(GcClock::Tick StartTick, - GcClock::Tick EndTick, - GcClock::Tick DeltaWidth, - uint64_t& OutMaxDelta) const; - GcClock::Tick FindTimepointThatRemoves(uint64_t Amount, GcClock::Tick EndTick) const; + eastl::vector<DiskUsageEntry> m_LogWindow; + inline void Append(const DiskUsageEntry& Entry) { m_LogWindow.push_back(Entry); } + inline void Append(DiskUsageEntry&& Entry) { m_LogWindow.emplace_back(std::move(Entry)); } + void KeepRange(GcClock::Tick StartTick, GcClock::Tick EndTick); + eastl::vector<uint64_t> GetDiskDeltas(GcClock::Tick StartTick, + GcClock::Tick EndTick, + GcClock::Tick DeltaWidth, + uint64_t& OutMaxDelta) const; + GcClock::Tick FindTimepointThatRemoves(uint64_t Amount, GcClock::Tick EndTick) const; }; /** diff --git a/src/zenstore/include/zenstore/workspaces.h b/src/zenstore/include/zenstore/workspaces.h index 3e9edf9f9..f2eb119a9 100644 --- a/src/zenstore/include/zenstore/workspaces.h +++ b/src/zenstore/include/zenstore/workspaces.h @@ -64,21 +64,21 @@ public: Workspaces(); ~Workspaces(); - std::optional<std::vector<ShareFile>> GetWorkspaceShareFiles(const Oid& WorkspaceId, - const Oid& ShareId, - bool ForceRefresh, - WorkerThreadPool& WorkerPool); + std::optional<eastl::vector<ShareFile>> GetWorkspaceShareFiles(const Oid& WorkspaceId, + const Oid& ShareId, + bool ForceRefresh, + WorkerThreadPool& WorkerPool); ShareFile GetWorkspaceShareChunkInfo(const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId, WorkerThreadPool& WorkerPool); - std::vector<IoBuffer> GetWorkspaceShareChunks(const Oid& WorkspaceId, - const Oid& ShareId, - const std::span<const ChunkRequest> ChunkRequests, - WorkerThreadPool& WorkerPool); + eastl::vector<IoBuffer> GetWorkspaceShareChunks(const Oid& WorkspaceId, + const Oid& ShareId, + const std::span<const ChunkRequest> ChunkRequests, + WorkerThreadPool& WorkerPool); - std::vector<Oid> GetWorkspaces() const; + eastl::vector<Oid> GetWorkspaces() const; std::optional<WorkspaceConfiguration> GetWorkspaceConfiguration(const Oid& WorkspaceId) const; - std::optional<std::vector<Oid>> GetWorkspaceShares(const Oid& WorkspaceId) const; + std::optional<eastl::vector<Oid>> GetWorkspaceShares(const Oid& WorkspaceId) const; std::optional<WorkspaceShareConfiguration> GetWorkspaceShareConfiguration(const Oid& WorkspaceId, const Oid& ShareId) const; void RefreshState(const std::filesystem::path& WorkspaceStatePath); @@ -106,37 +106,37 @@ public: const std::filesystem::path& WorkspaceStatePath, const std::filesystem::path& WorkspaceRoot); - static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& Log, - const std::filesystem::path& WorkspaceStatePath, - std::string_view ShareAlias, - WorkspaceConfiguration& OutWorkspace); - static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& InLog, - const std::filesystem::path& WorkspaceStatePath, - const Oid& WorkspaceId, - const Oid& WorkspaceShareId); - static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& Log, - const std::filesystem::path& WorkspaceRoot, - const Oid& WorkspaceShareId); - static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& Log, - const std::filesystem::path& WorkspaceRoot, - const std::filesystem::path& SharePath); - static std::vector<WorkspaceConfiguration> ReadConfig(const LoggerRef& Log, - const std::filesystem::path& WorkspaceStatePath, - std::string& OutError); - static std::vector<WorkspaceShareConfiguration> ReadWorkspaceConfig(const LoggerRef& Log, - const std::filesystem::path& WorkspaceRoot, - std::string& OutError); + static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& Log, + const std::filesystem::path& WorkspaceStatePath, + std::string_view ShareAlias, + WorkspaceConfiguration& OutWorkspace); + static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& InLog, + const std::filesystem::path& WorkspaceStatePath, + const Oid& WorkspaceId, + const Oid& WorkspaceShareId); + static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& Log, + const std::filesystem::path& WorkspaceRoot, + const Oid& WorkspaceShareId); + static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& Log, + const std::filesystem::path& WorkspaceRoot, + const std::filesystem::path& SharePath); + static eastl::vector<WorkspaceConfiguration> ReadConfig(const LoggerRef& Log, + const std::filesystem::path& WorkspaceStatePath, + std::string& OutError); + static eastl::vector<WorkspaceShareConfiguration> ReadWorkspaceConfig(const LoggerRef& Log, + const std::filesystem::path& WorkspaceRoot, + std::string& OutError); static Oid PathToId(const std::filesystem::path& Path); private: - static void WriteConfig(const LoggerRef& Log, - const std::filesystem::path& WorkspaceStatePath, - const std::vector<WorkspaceConfiguration>& WorkspaceConfigurations); + static void WriteConfig(const LoggerRef& Log, + const std::filesystem::path& WorkspaceStatePath, + const eastl::vector<WorkspaceConfiguration>& WorkspaceConfigurations); - static void WriteWorkspaceConfig(const LoggerRef& Log, - const std::filesystem::path& WorkspaceRoot, - const std::vector<WorkspaceShareConfiguration>& WorkspaceShareConfigurations); + static void WriteWorkspaceConfig(const LoggerRef& Log, + const std::filesystem::path& WorkspaceRoot, + const eastl::vector<WorkspaceShareConfiguration>& WorkspaceShareConfigurations); void RefreshWorkspaceShares(const Oid& WorkspaceId); bool RemoveWorkspace(RwLock::ExclusiveLockScope& Lock, const Oid& WorkspaceId); diff --git a/src/zenstore/workspaces.cpp b/src/zenstore/workspaces.cpp index 02a83d2a6..c33997d13 100644 --- a/src/zenstore/workspaces.cpp +++ b/src/zenstore/workspaces.cpp @@ -51,7 +51,7 @@ namespace { return Json.ToString(); } - std::vector<Workspaces::WorkspaceConfiguration> WorkspacesFromJson(const IoBuffer& WorkspaceJson, std::string& OutError) + eastl::vector<Workspaces::WorkspaceConfiguration> WorkspacesFromJson(const IoBuffer& WorkspaceJson, std::string& OutError) { using namespace std::literals; @@ -59,7 +59,7 @@ namespace { LoadCompactBinaryFromJson(std::string_view((const char*)(WorkspaceJson.Data()), WorkspaceJson.GetSize()), OutError); if (OutError.empty()) { - std::vector<Workspaces::WorkspaceConfiguration> Workspaces; + eastl::vector<Workspaces::WorkspaceConfiguration> Workspaces; if (CbObjectView RootObject = RootField.AsObjectView(); RootObject) { for (CbFieldView WorkspaceField : RootObject["workspaces"].AsArrayView()) @@ -111,7 +111,7 @@ namespace { return Json.ToString(); } - std::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceSharesFromJson(const IoBuffer& WorkspaceJson, std::string& OutError) + eastl::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceSharesFromJson(const IoBuffer& WorkspaceJson, std::string& OutError) { using namespace std::literals; @@ -119,7 +119,7 @@ namespace { LoadCompactBinaryFromJson(std::string_view((const char*)(WorkspaceJson.Data()), WorkspaceJson.GetSize()), OutError); if (OutError.empty()) { - std::vector<Workspaces::WorkspaceShareConfiguration> Shares; + eastl::vector<Workspaces::WorkspaceShareConfiguration> Shares; if (CbObjectView RootObject = RootField.AsObjectView(); RootObject) { for (CbFieldView ShareField : RootObject["shares"].AsArrayView()) @@ -168,7 +168,7 @@ public: }; FolderStructure() {} - FolderStructure(std::vector<FileEntry>&& InEntries, std::vector<Oid>&& Ids); + FolderStructure(eastl::vector<FileEntry>&& InEntries, eastl::vector<Oid>&& Ids); const FileEntry* FindEntry(const Oid& Id) const { @@ -190,7 +190,7 @@ public: } private: - const std::vector<FileEntry> Entries; + const eastl::vector<FileEntry> Entries; tsl::robin_map<Oid, size_t, Oid::Hasher> IdLookup; }; @@ -226,7 +226,7 @@ public: Workspace(const LoggerRef& Log, const Workspaces::WorkspaceConfiguration& Config); const Workspaces::WorkspaceConfiguration& GetConfig() const; - std::vector<Ref<WorkspaceShare>> GetShares() const; + eastl::vector<Ref<WorkspaceShare>> GetShares() const; Ref<WorkspaceShare> GetShare(const Oid& ShareId) const; void SetShare(const Oid& ShareId, Ref<WorkspaceShare>&& Share); @@ -241,7 +241,7 @@ private: ////////////////////////////////////////////////////////////////////////// -FolderStructure::FolderStructure(std::vector<FileEntry>&& InEntries, std::vector<Oid>&& Ids) : Entries(std::move(InEntries)) +FolderStructure::FolderStructure(eastl::vector<FileEntry>&& InEntries, eastl::vector<Oid>&& Ids) : Entries(std::move(InEntries)) { IdLookup.reserve(Entries.size()); for (size_t Index = 0; Index < Entries.size(); Index++) @@ -265,9 +265,9 @@ namespace { virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override { - std::vector<FolderStructure::FileEntry> FileEntries; - std::vector<Oid> PathIds; - const size_t FileCount = Content.FileNames.size(); + eastl::vector<FolderStructure::FileEntry> FileEntries; + eastl::vector<Oid> PathIds; + const size_t FileCount = Content.FileNames.size(); FileEntries.reserve(FileCount); PathIds.reserve(FileCount); @@ -290,13 +290,13 @@ namespace { }); } - LoggerRef& Log() { return m_Log; } - LoggerRef& m_Log; - const std::filesystem::path Path; - RwLock WorkLock; - std::vector<FolderStructure::FileEntry> FoundFiles; - std::vector<Oid> FoundFileIds; - WorkerThreadPool& WorkerPool; + LoggerRef& Log() { return m_Log; } + LoggerRef& m_Log; + const std::filesystem::path Path; + RwLock WorkLock; + eastl::vector<FolderStructure::FileEntry> FoundFiles; + eastl::vector<Oid> FoundFileIds; + WorkerThreadPool& WorkerPool; }; void FolderScanner::Traverse() @@ -376,10 +376,10 @@ Workspace::GetConfig() const { return m_Config; } -std::vector<Ref<WorkspaceShare>> +eastl::vector<Ref<WorkspaceShare>> Workspace::GetShares() const { - std::vector<Ref<WorkspaceShare>> Shares; + eastl::vector<Ref<WorkspaceShare>> Shares; Shares.reserve(m_Shares.size()); for (auto It : m_Shares) { @@ -446,8 +446,8 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) std::filesystem::path ConfigPath = RootPath / WorkspaceConfigName; if (std::filesystem::exists(ConfigPath)) { - std::string Error; - std::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceShares = ReadWorkspaceConfig(m_Log, RootPath, Error); + std::string Error; + eastl::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceShares = ReadWorkspaceConfig(m_Log, RootPath, Error); if (!Error.empty()) { ZEN_WARN("Failed to read workspace state from {}. Reason: '{}'", ConfigPath, Error); @@ -529,7 +529,7 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) } } -std::optional<std::vector<Workspaces::ShareFile>> +std::optional<eastl::vector<Workspaces::ShareFile>> Workspaces::GetWorkspaceShareFiles(const Oid& WorkspaceId, const Oid& ShareId, bool ForceRefresh, WorkerThreadPool& WorkerPool) { std::pair<Ref<Workspace>, Ref<WorkspaceShare>> WorkspaceAndShare = FindWorkspaceShare(WorkspaceId, ShareId, ForceRefresh, WorkerPool); @@ -538,8 +538,8 @@ Workspaces::GetWorkspaceShareFiles(const Oid& WorkspaceId, const Oid& ShareId, b return {}; } - const FolderStructure& Structure = WorkspaceAndShare.second->GetStructure(); - std::vector<Workspaces::ShareFile> Files; + const FolderStructure& Structure = WorkspaceAndShare.second->GetStructure(); + eastl::vector<Workspaces::ShareFile> Files; Files.reserve(Structure.EntryCount()); Structure.IterateEntries([&Files](const Oid& Id, const FolderStructure::FileEntry& Entry) { std::string GenericPath(reinterpret_cast<const char*>(Entry.RelativePath.generic_u8string().c_str())); @@ -568,7 +568,7 @@ Workspaces::GetWorkspaceShareChunkInfo(const Oid& WorkspaceId, const Oid& ShareI return {}; } -std::vector<IoBuffer> +eastl::vector<IoBuffer> Workspaces::GetWorkspaceShareChunks(const Oid& WorkspaceId, const Oid& ShareId, const std::span<const ChunkRequest> ChunkRequests, @@ -612,10 +612,10 @@ Workspaces::GetWorkspaceShareChunks(const Oid& WorkspaceId, if (ChunkRequests.size() == 1) { - return std::vector<IoBuffer>({GetOne(RootPath, *WorkspaceAndShare.second, ChunkRequests[0])}); + return eastl::vector<IoBuffer>({GetOne(RootPath, *WorkspaceAndShare.second, ChunkRequests[0])}); } - std::vector<IoBuffer> Chunks; + eastl::vector<IoBuffer> Chunks; Chunks.resize(ChunkRequests.size()); Latch WorkLatch(1); @@ -640,10 +640,10 @@ Workspaces::GetWorkspaceShareChunks(const Oid& WorkspaceId, return Chunks; } -std::vector<Oid> +eastl::vector<Oid> Workspaces::GetWorkspaces() const { - std::vector<Oid> Workspaces; + eastl::vector<Oid> Workspaces; RwLock::SharedLockScope Lock(m_Lock); for (auto It : m_Workspaces) { @@ -667,14 +667,14 @@ Workspaces::GetWorkspaceConfiguration(const Oid& WorkspaceId) const return {}; } -std::optional<std::vector<Oid>> +std::optional<eastl::vector<Oid>> Workspaces::GetWorkspaceShares(const Oid& WorkspaceId) const { RwLock::SharedLockScope Lock(m_Lock); Ref<Workspace> Workspace = FindWorkspace(Lock, WorkspaceId); if (Workspace) { - std::vector<Oid> Shares; + eastl::vector<Oid> Shares; for (auto Share : Workspace->GetShares()) { Shares.push_back(Share->GetConfig().Id); @@ -705,8 +705,8 @@ Workspaces::RefreshState(const std::filesystem::path& WorkspaceStatePath) { using namespace std::literals; - std::string Error; - std::vector<Workspaces::WorkspaceConfiguration> Workspaces = ReadConfig(Log(), WorkspaceStatePath, Error); + std::string Error; + eastl::vector<Workspaces::WorkspaceConfiguration> Workspaces = ReadConfig(Log(), WorkspaceStatePath, Error); if (!Error.empty()) { @@ -798,7 +798,7 @@ Workspaces::GetShareAlias(std::string_view Alias) const return {}; } -std::vector<Workspaces::WorkspaceConfiguration> +eastl::vector<Workspaces::WorkspaceConfiguration> Workspaces::ReadConfig(const LoggerRef& InLog, const std::filesystem::path& WorkspaceStatePath, std::string& OutError) { auto Log = [&InLog]() { return InLog; }; @@ -810,7 +810,7 @@ Workspaces::ReadConfig(const LoggerRef& InLog, const std::filesystem::path& Work const std::filesystem::path ConfigPath = WorkspaceStatePath / WorkspacesConfigName; if (std::filesystem::exists(ConfigPath)) { - std::vector<Workspaces::WorkspaceConfiguration> Workspaces = + eastl::vector<Workspaces::WorkspaceConfiguration> Workspaces = WorkspacesFromJson(IoBufferBuilder::MakeFromFile(ConfigPath), OutError); if (OutError.empty()) { @@ -821,9 +821,9 @@ Workspaces::ReadConfig(const LoggerRef& InLog, const std::filesystem::path& Work } void -Workspaces::WriteConfig(const LoggerRef& InLog, - const std::filesystem::path& WorkspaceStatePath, - const std::vector<WorkspaceConfiguration>& WorkspaceConfigurations) +Workspaces::WriteConfig(const LoggerRef& InLog, + const std::filesystem::path& WorkspaceStatePath, + const eastl::vector<WorkspaceConfiguration>& WorkspaceConfigurations) { auto Log = [&InLog]() { return InLog; }; @@ -837,7 +837,7 @@ Workspaces::WriteConfig(const LoggerRef& InLog, TemporaryFile::SafeWriteFile(WorkspaceStatePath / WorkspacesConfigName, MemoryView(ConfigJson.data(), ConfigJson.size())); } -std::vector<Workspaces::WorkspaceShareConfiguration> +eastl::vector<Workspaces::WorkspaceShareConfiguration> Workspaces::ReadWorkspaceConfig(const LoggerRef& InLog, const std::filesystem::path& WorkspaceRoot, std::string& OutError) { auto Log = [&InLog]() { return InLog; }; @@ -849,7 +849,7 @@ Workspaces::ReadWorkspaceConfig(const LoggerRef& InLog, const std::filesystem::p std::filesystem::path ConfigPath = WorkspaceRoot / WorkspaceConfigName; if (std::filesystem::exists(ConfigPath)) { - std::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceShares = + eastl::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceShares = WorkspaceSharesFromJson(IoBufferBuilder::MakeFromFile(ConfigPath), OutError); if (OutError.empty()) { @@ -860,9 +860,9 @@ Workspaces::ReadWorkspaceConfig(const LoggerRef& InLog, const std::filesystem::p } void -Workspaces::WriteWorkspaceConfig(const LoggerRef& InLog, - const std::filesystem::path& WorkspaceRoot, - const std::vector<WorkspaceShareConfiguration>& WorkspaceShareConfigurations) +Workspaces::WriteWorkspaceConfig(const LoggerRef& InLog, + const std::filesystem::path& WorkspaceRoot, + const eastl::vector<WorkspaceShareConfiguration>& WorkspaceShareConfigurations) { auto Log = [&InLog]() { return InLog; }; @@ -891,8 +891,8 @@ Workspaces::AddWorkspace(const LoggerRef& Log, const std::filesystem::path& Work throw std::invalid_argument( fmt::format("workspace root path '{}' does not exist for workspace '{}'", Configuration.RootPath, Configuration.Id)); } - std::string Error; - std::vector<WorkspaceConfiguration> WorkspaceConfigurations = ReadConfig(Log, WorkspaceStatePath, Error); + std::string Error; + eastl::vector<WorkspaceConfiguration> WorkspaceConfigurations = ReadConfig(Log, WorkspaceStatePath, Error); if (!Error.empty()) { throw std::invalid_argument( @@ -931,8 +931,8 @@ Workspaces::AddWorkspace(const LoggerRef& Log, const std::filesystem::path& Work bool Workspaces::RemoveWorkspace(const LoggerRef& Log, const std::filesystem::path& WorkspaceStatePath, const Oid& WorkspaceId) { - std::string Error; - std::vector<WorkspaceConfiguration> WorkspaceConfigurations = ReadConfig(Log, WorkspaceStatePath, Error); + std::string Error; + eastl::vector<WorkspaceConfiguration> WorkspaceConfigurations = ReadConfig(Log, WorkspaceStatePath, Error); if (!Error.empty()) { throw std::invalid_argument( @@ -976,8 +976,8 @@ Workspaces::AddWorkspaceShare(const LoggerRef& Log, fmt::format("invalid workspace share alias '{}' for workspace share {}", Configuration.Alias, Configuration.Id)); } - std::string Error; - std::vector<WorkspaceShareConfiguration> WorkspaceShareConfigurations = ReadWorkspaceConfig(Log, WorkspaceRoot, Error); + std::string Error; + eastl::vector<WorkspaceShareConfiguration> WorkspaceShareConfigurations = ReadWorkspaceConfig(Log, WorkspaceRoot, Error); if (!Error.empty()) { throw std::invalid_argument(fmt::format("failed to read workspace configuration from '{}'. Reason: '{}'", WorkspaceRoot, Error)); @@ -1018,8 +1018,8 @@ Workspaces::AddWorkspaceShare(const LoggerRef& Log, bool Workspaces::RemoveWorkspaceShare(const LoggerRef& Log, const std::filesystem::path& WorkspaceRoot, const Oid& WorkspaceShareId) { - std::string Error; - std::vector<WorkspaceShareConfiguration> WorkspaceShareConfigurations = ReadWorkspaceConfig(Log, WorkspaceRoot, Error); + std::string Error; + eastl::vector<WorkspaceShareConfiguration> WorkspaceShareConfigurations = ReadWorkspaceConfig(Log, WorkspaceRoot, Error); if (!Error.empty()) { throw std::invalid_argument(fmt::format("failed to read workspace configuration from '{}'. Reason: '{}'", WorkspaceRoot, Error)); @@ -1041,8 +1041,8 @@ Workspaces::FindWorkspace(const LoggerRef& InLog, const std::filesystem::path& W { auto Log = [&InLog]() { return InLog; }; - std::string Error; - std::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); + std::string Error; + eastl::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); if (!Error.empty()) { throw std::invalid_argument( @@ -1066,8 +1066,8 @@ Workspaces::FindWorkspace(const LoggerRef& InLog, { auto Log = [&InLog]() { return InLog; }; - std::string Error; - std::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); + std::string Error; + eastl::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); if (!Error.empty()) { throw std::invalid_argument( @@ -1092,8 +1092,8 @@ Workspaces::FindWorkspaceShare(const LoggerRef& InLog, { auto Log = [&InLog]() { return InLog; }; - std::string Error; - std::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); + std::string Error; + eastl::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); if (!Error.empty()) { throw std::invalid_argument( @@ -1102,7 +1102,7 @@ Workspaces::FindWorkspaceShare(const LoggerRef& InLog, for (const WorkspaceConfiguration& WorkspaceConfig : Workspaces) { - std::vector<WorkspaceShareConfiguration> Shares = ReadWorkspaceConfig(InLog, WorkspaceConfig.RootPath, Error); + eastl::vector<WorkspaceShareConfiguration> Shares = ReadWorkspaceConfig(InLog, WorkspaceConfig.RootPath, Error); if (!Error.empty()) { ZEN_WARN("Invalid workspace config in workspace root '{}'", WorkspaceConfig.RootPath); @@ -1139,9 +1139,9 @@ Workspaces::FindWorkspaceShare(const LoggerRef& InLog, Workspaces::WorkspaceShareConfiguration Workspaces::FindWorkspaceShare(const LoggerRef& InLog, const std::filesystem::path& WorkspaceRoot, const Oid& WorkspaceShareId) { - auto Log = [&InLog]() { return InLog; }; - std::string Error; - std::vector<WorkspaceShareConfiguration> Shares = ReadWorkspaceConfig(InLog, WorkspaceRoot, Error); + auto Log = [&InLog]() { return InLog; }; + std::string Error; + eastl::vector<WorkspaceShareConfiguration> Shares = ReadWorkspaceConfig(InLog, WorkspaceRoot, Error); if (!Error.empty()) { ZEN_WARN("Invalid workspace config in workspace root '{}'", WorkspaceRoot); @@ -1162,9 +1162,9 @@ Workspaces::FindWorkspaceShare(const LoggerRef& InLog, const std::filesystem::pa Workspaces::WorkspaceShareConfiguration Workspaces::FindWorkspaceShare(const LoggerRef& InLog, const std::filesystem::path& WorkspaceRoot, const std::filesystem::path& SharePath) { - auto Log = [&InLog]() { return InLog; }; - std::string Error; - std::vector<WorkspaceShareConfiguration> Shares = ReadWorkspaceConfig(InLog, WorkspaceRoot, Error); + auto Log = [&InLog]() { return InLog; }; + std::string Error; + eastl::vector<WorkspaceShareConfiguration> Shares = ReadWorkspaceConfig(InLog, WorkspaceRoot, Error); if (!Error.empty()) { ZEN_WARN("Invalid workspace config in workspace root '{}'", WorkspaceRoot); @@ -1200,7 +1200,7 @@ Workspaces::RemoveWorkspace(RwLock::ExclusiveLockScope&, const Oid& WorkspaceId) { if (auto It = m_Workspaces.find(WorkspaceId); It != m_Workspaces.end()) { - std::vector<std::string> Aliases; + eastl::vector<std::string> Aliases; for (const auto& AliasIt : m_ShareAliases) { if (AliasIt.second.WorkspaceId == WorkspaceId) @@ -1296,10 +1296,10 @@ Workspaces::FindWorkspace(const RwLock::SharedLockScope&, const Oid& WorkspaceId #if ZEN_WITH_TESTS namespace { - std::vector<std::pair<std::filesystem::path, IoBuffer>> GenerateFolderContent(const std::filesystem::path& RootPath) + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> GenerateFolderContent(const std::filesystem::path& RootPath) { CreateDirectories(RootPath); - std::vector<std::pair<std::filesystem::path, IoBuffer>> Result; + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> Result; Result.push_back(std::make_pair(RootPath / "root_blob_1.bin", CreateRandomBlob(4122))); Result.push_back(std::make_pair(RootPath / "root_blob_2.bin", CreateRandomBlob(2122))); @@ -1328,10 +1328,10 @@ namespace { return Result; } - std::vector<std::pair<std::filesystem::path, IoBuffer>> GenerateFolderContent2(const std::filesystem::path& RootPath) + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> GenerateFolderContent2(const std::filesystem::path& RootPath) { CreateDirectories(RootPath); - std::vector<std::pair<std::filesystem::path, IoBuffer>> Result; + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> Result; Result.push_back(std::make_pair(RootPath / "root_blob_3.bin", CreateRandomBlob(312))); std::filesystem::path FirstFolder(RootPath / "first_folder"); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob3.bin", CreateRandomBlob(722))); @@ -1381,9 +1381,9 @@ TEST_CASE("workspace.share.paths") WorkerThreadPool WorkerPool(std::thread::hardware_concurrency()); - ScopedTemporaryDirectory TempDir; - std::filesystem::path RootPath = TempDir.Path() / "workspace"; - std::vector<std::pair<std::filesystem::path, IoBuffer>> Content = GenerateFolderContent(RootPath); + ScopedTemporaryDirectory TempDir; + std::filesystem::path RootPath = TempDir.Path() / "workspace"; + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> Content = GenerateFolderContent(RootPath); CHECK(Workspaces::AddWorkspace(Log(), TempDir.Path(), Workspaces::WorkspaceConfiguration{.Id = Workspaces::PathToId(RootPath), .RootPath = RootPath})); @@ -1409,9 +1409,9 @@ TEST_CASE("workspace.share.basic") WorkerThreadPool WorkerPool(std::thread::hardware_concurrency()); - ScopedTemporaryDirectory TempDir; - std::filesystem::path RootPath = TempDir.Path() / "workspace"; - std::vector<std::pair<std::filesystem::path, IoBuffer>> Content = GenerateFolderContent(RootPath); + ScopedTemporaryDirectory TempDir; + std::filesystem::path RootPath = TempDir.Path() / "workspace"; + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> Content = GenerateFolderContent(RootPath); Workspaces::AddWorkspace(Log(), TempDir.Path(), @@ -1421,18 +1421,18 @@ TEST_CASE("workspace.share.basic") Workspaces WS; WS.RefreshState(TempDir.Path()); - std::filesystem::path SharePath = RootPath / "second_folder"; - std::vector<std::filesystem::path> Paths = {{std::filesystem::relative(Content[4].first, SharePath)}, - {std::filesystem::relative(Content[6].first, SharePath)}, - {std::filesystem::relative(Content[7].first, SharePath)}, - {"the_file_that_is_not_there.txt"}}; - std::vector<IoBuffer> Chunks = + std::filesystem::path SharePath = RootPath / "second_folder"; + eastl::vector<std::filesystem::path> Paths = {{std::filesystem::relative(Content[4].first, SharePath)}, + {std::filesystem::relative(Content[6].first, SharePath)}, + {std::filesystem::relative(Content[7].first, SharePath)}, + {"the_file_that_is_not_there.txt"}}; + eastl::vector<IoBuffer> Chunks = WS.GetWorkspaceShareChunks(Workspaces::PathToId(RootPath), Workspaces::PathToId("second_folder"), - std::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths[0])}, - {.ChunkId = Workspaces::PathToId(Paths[1])}, - {.ChunkId = Workspaces::PathToId(Paths[2])}, - {.ChunkId = Workspaces::PathToId(Paths[3])}}, + eastl::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths[0])}, + {.ChunkId = Workspaces::PathToId(Paths[1])}, + {.ChunkId = Workspaces::PathToId(Paths[2])}, + {.ChunkId = Workspaces::PathToId(Paths[3])}}, WorkerPool); CHECK(Chunks.size() == 4); CHECK(Chunks[0].GetView().EqualBytes(Content[4].second.GetView())); @@ -1440,20 +1440,20 @@ TEST_CASE("workspace.share.basic") CHECK(Chunks[2].GetView().EqualBytes(Content[7].second.GetView())); CHECK(Chunks[3].GetSize() == 0); - std::vector<std::pair<std::filesystem::path, IoBuffer>> Content2 = GenerateFolderContent2(RootPath); - std::vector<std::filesystem::path> Paths2 = {{std::filesystem::relative(Content2[2].first, SharePath)}, - {std::filesystem::relative(Content2[3].first, SharePath)}}; + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> Content2 = GenerateFolderContent2(RootPath); + eastl::vector<std::filesystem::path> Paths2 = {{std::filesystem::relative(Content2[2].first, SharePath)}, + {std::filesystem::relative(Content2[3].first, SharePath)}}; - std::vector<IoBuffer> Chunks2 = WS.GetWorkspaceShareChunks( + eastl::vector<IoBuffer> Chunks2 = WS.GetWorkspaceShareChunks( Workspaces::PathToId(RootPath), Workspaces::PathToId("second_folder"), - std::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths2[0])}, {.ChunkId = Workspaces::PathToId(Paths2[1])}}, + eastl::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths2[0])}, {.ChunkId = Workspaces::PathToId(Paths2[1])}}, WorkerPool); CHECK(Chunks2.size() == 2); CHECK(Chunks2[0].GetSize() == 0); CHECK(Chunks2[1].GetSize() == 0); - std::optional<std::vector<Workspaces::ShareFile>> Files = + std::optional<eastl::vector<Workspaces::ShareFile>> Files = WS.GetWorkspaceShareFiles(Workspaces::PathToId(RootPath), Workspaces::PathToId("second_folder"), true, WorkerPool); CHECK(Files.has_value()); CHECK(Files.value().size() == 6); @@ -1461,7 +1461,7 @@ TEST_CASE("workspace.share.basic") Chunks2 = WS.GetWorkspaceShareChunks( Workspaces::PathToId(RootPath), Workspaces::PathToId("second_folder"), - std::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths2[0])}, {.ChunkId = Workspaces::PathToId(Paths2[1])}}, + eastl::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths2[0])}, {.ChunkId = Workspaces::PathToId(Paths2[1])}}, WorkerPool); CHECK(Chunks2.size() == 2); CHECK(Chunks2[0].GetView().EqualBytes(Content2[2].second.GetView())); @@ -1489,7 +1489,7 @@ TEST_CASE("workspace.share.basic") Chunks2 = WS.GetWorkspaceShareChunks( Workspaces::PathToId(RootPath), Workspaces::PathToId("second_folder"), - std::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths2[0])}, {.ChunkId = Workspaces::PathToId(Paths2[1])}}, + eastl::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths2[0])}, {.ChunkId = Workspaces::PathToId(Paths2[1])}}, WorkerPool); CHECK(Chunks2.empty()); @@ -1503,9 +1503,9 @@ TEST_CASE("workspace.share.alias") WorkerThreadPool WorkerPool(std::thread::hardware_concurrency()); - ScopedTemporaryDirectory TempDir; - std::filesystem::path RootPath = TempDir.Path() / "workspace"; - std::vector<std::pair<std::filesystem::path, IoBuffer>> Content = GenerateFolderContent(RootPath); + ScopedTemporaryDirectory TempDir; + std::filesystem::path RootPath = TempDir.Path() / "workspace"; + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> Content = GenerateFolderContent(RootPath); CHECK(Workspaces::AddWorkspace(Log(), TempDir.Path(), {Workspaces::PathToId(RootPath), RootPath})); CHECK(Workspaces::AddWorkspaceShare(Log(), RootPath, {Workspaces::PathToId("second_folder"), "second_folder", "my_share"})); @@ -1517,18 +1517,18 @@ TEST_CASE("workspace.share.alias") CHECK(Alias.has_value()); CHECK(!WS.GetShareAlias("my_share2").has_value()); - std::filesystem::path SharePath = RootPath / "second_folder"; - std::vector<std::filesystem::path> Paths = {{std::filesystem::relative(Content[4].first, SharePath)}, - {std::filesystem::relative(Content[6].first, SharePath)}, - {std::filesystem::relative(Content[7].first, SharePath)}, - {"the_file_that_is_not_there.txt"}}; - std::vector<IoBuffer> Chunks = + std::filesystem::path SharePath = RootPath / "second_folder"; + eastl::vector<std::filesystem::path> Paths = {{std::filesystem::relative(Content[4].first, SharePath)}, + {std::filesystem::relative(Content[6].first, SharePath)}, + {std::filesystem::relative(Content[7].first, SharePath)}, + {"the_file_that_is_not_there.txt"}}; + eastl::vector<IoBuffer> Chunks = WS.GetWorkspaceShareChunks(Alias->WorkspaceId, Alias->ShareId, - std::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths[0])}, - {.ChunkId = Workspaces::PathToId(Paths[1])}, - {.ChunkId = Workspaces::PathToId(Paths[2])}, - {.ChunkId = Workspaces::PathToId(Paths[3])}}, + eastl::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths[0])}, + {.ChunkId = Workspaces::PathToId(Paths[1])}, + {.ChunkId = Workspaces::PathToId(Paths[2])}, + {.ChunkId = Workspaces::PathToId(Paths[3])}}, WorkerPool); CHECK(Chunks.size() == 4); CHECK(Chunks[0].GetView().EqualBytes(Content[4].second.GetView())); |