diff options
| author | Stefan Boberg <[email protected]> | 2025-03-06 17:27:59 +0100 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2025-03-06 17:27:59 +0100 |
| commit | 66e5d1f4e288e0c32f854ebe3b63584b42b83554 (patch) | |
| tree | d67e9d358419b5baccd429d54988414e0d7cd7a6 /src | |
| parent | reduced memory churn using fixed_xxx containers (#236) (diff) | |
| download | zen-66e5d1f4e288e0c32f854ebe3b63584b42b83554.tar.xz zen-66e5d1f4e288e0c32f854ebe3b63584b42b83554.zip | |
switched std::vector -> eastl::vector
Diffstat (limited to 'src')
129 files changed, 2478 insertions, 2456 deletions
diff --git a/src/transports/winsock/winsock.cpp b/src/transports/winsock/winsock.cpp index 1c3ee909a..2adf547fa 100644 --- a/src/transports/winsock/winsock.cpp +++ b/src/transports/winsock/winsock.cpp @@ -12,6 +12,8 @@ #include <zenbase/concepts.h> #include <zenbase/refcount.h> +#include <vector> + #ifndef _WIN32_WINNT # define _WIN32_WINNT 0x0A00 #endif diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index eb0650c4d..af51ef0e7 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -98,8 +98,8 @@ namespace { const std::string UGSFolderName = ".ugs"; const std::string LegacyZenTempFolderName = ".zen-tmp"; - const std::vector<std::string_view> DefaultExcludeFolders({UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName}); - const std::vector<std::string_view> DefaultExcludeExtensions({}); + const eastl::vector<std::string_view> DefaultExcludeFolders({UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName}); + const eastl::vector<std::string_view> DefaultExcludeExtensions({}); static bool IsVerbose = false; static bool UsePlainProgress = false; @@ -505,19 +505,19 @@ namespace { } }; - std::vector<uint32_t> CalculateAbsoluteChunkOrders(const std::span<const IoHash> LocalChunkHashes, - const std::span<const uint32_t> LocalChunkOrder, - const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToLocalChunkIndex, - const std::span<const uint32_t>& LooseChunkIndexes, - const std::span<const ChunkBlockDescription>& BlockDescriptions) + eastl::vector<uint32_t> CalculateAbsoluteChunkOrders(const std::span<const IoHash> LocalChunkHashes, + const std::span<const uint32_t> LocalChunkOrder, + const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToLocalChunkIndex, + const std::span<const uint32_t>& LooseChunkIndexes, + const std::span<const ChunkBlockDescription>& BlockDescriptions) { ZEN_TRACE_CPU("CalculateAbsoluteChunkOrders"); #if EXTRA_VERIFY - std::vector<IoHash> TmpAbsoluteChunkHashes; + eastl::vector<IoHash> TmpAbsoluteChunkHashes; TmpAbsoluteChunkHashes.reserve(LocalChunkHashes.size()); #endif // EXTRA_VERIFY - std::vector<uint32_t> LocalChunkIndexToAbsoluteChunkIndex; + eastl::vector<uint32_t> LocalChunkIndexToAbsoluteChunkIndex; LocalChunkIndexToAbsoluteChunkIndex.resize(LocalChunkHashes.size(), (uint32_t)-1); std::uint32_t AbsoluteChunkCount = 0; for (uint32_t ChunkIndex : LooseChunkIndexes) @@ -544,7 +544,7 @@ namespace { AbsoluteChunkCount++; } } - std::vector<uint32_t> AbsoluteChunkOrder; + eastl::vector<uint32_t> AbsoluteChunkOrder; AbsoluteChunkOrder.reserve(LocalChunkHashes.size()); for (const uint32_t LocalChunkIndex : LocalChunkOrder) { @@ -575,14 +575,14 @@ namespace { const std::span<const IoHash> LooseChunkHashes, const std::span<const uint64_t> LooseChunkRawSizes, const std::span<const ChunkBlockDescription>& BlockDescriptions, - std::vector<IoHash>& OutLocalChunkHashes, - std::vector<uint64_t>& OutLocalChunkRawSizes, - std::vector<uint32_t>& OutLocalChunkOrders) + eastl::vector<IoHash>& OutLocalChunkHashes, + eastl::vector<uint64_t>& OutLocalChunkRawSizes, + eastl::vector<uint32_t>& OutLocalChunkOrders) { ZEN_TRACE_CPU("CalculateLocalChunkOrders"); - std::vector<IoHash> AbsoluteChunkHashes; - std::vector<uint64_t> AbsoluteChunkRawSizes; + eastl::vector<IoHash> AbsoluteChunkHashes; + eastl::vector<uint64_t> AbsoluteChunkRawSizes; AbsoluteChunkHashes.insert(AbsoluteChunkHashes.end(), LooseChunkHashes.begin(), LooseChunkHashes.end()); AbsoluteChunkRawSizes.insert(AbsoluteChunkRawSizes.end(), LooseChunkRawSizes.begin(), LooseChunkRawSizes.end()); for (const ChunkBlockDescription& Block : BlockDescriptions) @@ -651,7 +651,7 @@ namespace { std::span<const uint32_t> ChunkCounts, std::span<const IoHash> LocalChunkHashes, std::span<const uint64_t> LocalChunkRawSizes, - std::vector<uint32_t> AbsoluteChunkOrders, + eastl::vector<uint32_t> AbsoluteChunkOrders, const std::span<const uint32_t> LooseLocalChunkIndexes, const std::span<IoHash> BlockHashes) { @@ -721,18 +721,18 @@ namespace { } } - void ReadBuildContentFromCompactBinary(CbObjectView BuildPartManifest, - SourcePlatform& OutPlatform, - std::vector<std::filesystem::path>& OutPaths, - std::vector<IoHash>& OutRawHashes, - std::vector<uint64_t>& OutRawSizes, - std::vector<uint32_t>& OutAttributes, - std::vector<IoHash>& OutSequenceRawHashes, - std::vector<uint32_t>& OutChunkCounts, - std::vector<uint32_t>& OutAbsoluteChunkOrders, - std::vector<IoHash>& OutLooseChunkHashes, - std::vector<uint64_t>& OutLooseChunkRawSizes, - std::vector<IoHash>& OutBlockRawHashes) + void ReadBuildContentFromCompactBinary(CbObjectView BuildPartManifest, + SourcePlatform& OutPlatform, + eastl::vector<std::filesystem::path>& OutPaths, + eastl::vector<IoHash>& OutRawHashes, + eastl::vector<uint64_t>& OutRawSizes, + eastl::vector<uint32_t>& OutAttributes, + eastl::vector<IoHash>& OutSequenceRawHashes, + eastl::vector<uint32_t>& OutChunkCounts, + eastl::vector<uint32_t>& OutAbsoluteChunkOrders, + eastl::vector<IoHash>& OutLooseChunkHashes, + eastl::vector<uint64_t>& OutLooseChunkRawSizes, + eastl::vector<IoHash>& OutBlockRawHashes) { OutPlatform = FromString(BuildPartManifest["platform"sv].AsString(), SourcePlatform::_Count); @@ -752,14 +752,14 @@ namespace { throw std::runtime_error(fmt::format("Number of raw sizes entries does not match number of paths")); } - std::vector<uint32_t> ModeArray; + eastl::vector<uint32_t> ModeArray; compactbinary_helpers::ReadArray("mode"sv, FilesObject, ModeArray); if (ModeArray.size() != PathCount && ModeArray.size() != 0) { throw std::runtime_error(fmt::format("Number of attribute entries does not match number of paths")); } - std::vector<uint32_t> AttributeArray; + eastl::vector<uint32_t> AttributeArray; compactbinary_helpers::ReadArray("attributes"sv, FilesObject, ModeArray); if (AttributeArray.size() != PathCount && AttributeArray.size() != 0) { @@ -804,13 +804,13 @@ namespace { { // Legacy zen style - std::vector<uint32_t> LegacyChunkCounts; + eastl::vector<uint32_t> LegacyChunkCounts; compactbinary_helpers::ReadArray("chunkcounts"sv, FilesObject, LegacyChunkCounts); if (LegacyChunkCounts.size() != PathCount) { throw std::runtime_error(fmt::format("Number of chunk count entries does not match number of paths")); } - std::vector<uint32_t> LegacyAbsoluteChunkOrders; + eastl::vector<uint32_t> LegacyAbsoluteChunkOrders; compactbinary_helpers::ReadArray("chunkorders"sv, FilesObject, LegacyAbsoluteChunkOrders); CbArrayView ChunkOrdersArray = BuildPartManifest["chunkorders"sv].AsArrayView(); @@ -881,12 +881,12 @@ namespace { } } - bool ReadStateObject(CbObjectView StateView, - Oid& OutBuildId, - std::vector<Oid>& BuildPartsIds, - std::vector<std::string>& BuildPartsNames, - std::vector<ChunkedFolderContent>& OutPartContents, - FolderContent& OutLocalFolderState) + bool ReadStateObject(CbObjectView StateView, + Oid& OutBuildId, + eastl::vector<Oid>& BuildPartsIds, + eastl::vector<std::string>& BuildPartsNames, + eastl::vector<ChunkedFolderContent>& OutPartContents, + FolderContent& OutLocalFolderState) { try { @@ -909,10 +909,10 @@ namespace { } } - CbObject CreateStateObject(const Oid& BuildId, - std::vector<std::pair<Oid, std::string>> AllBuildParts, - std::span<const ChunkedFolderContent> PartContents, - const FolderContent& LocalFolderState) + CbObject CreateStateObject(const Oid& BuildId, + eastl::vector<std::pair<Oid, std::string>> AllBuildParts, + std::span<const ChunkedFolderContent> PartContents, + const FolderContent& LocalFolderState) { CbObjectWriter CurrentStateWriter; CurrentStateWriter.BeginArray("builds"sv); @@ -973,7 +973,7 @@ namespace { const uint64_t BlockIndexStart = Offset / BlockSize; const uint64_t BlockIndexEnd = (Offset + Size - 1) / BlockSize; - std::vector<SharedBuffer> BufferRanges; + eastl::vector<SharedBuffer> BufferRanges; BufferRanges.reserve(BlockIndexEnd - BlockIndexStart + 1); uint64_t ReadOffset = Offset; @@ -1073,11 +1073,11 @@ namespace { } private: - const std::filesystem::path m_Path; - const ChunkedFolderContent& m_LocalContent; - const ChunkedContentLookup& m_LocalLookup; - std::vector<std::pair<uint32_t, std::unique_ptr<BufferedOpenFile>>> m_OpenFiles; - DiskStatistics& m_DiskStats; + const std::filesystem::path m_Path; + const ChunkedFolderContent& m_LocalContent; + const ChunkedContentLookup& m_LocalLookup; + eastl::vector<std::pair<uint32_t, std::unique_ptr<BufferedOpenFile>>> m_OpenFiles; + DiskStatistics& m_DiskStats; }; CompositeBuffer ValidateBlob(IoBuffer&& Payload, const IoHash& BlobHash, uint64_t& OutCompressedSize, uint64_t& OutDecompressedSize) @@ -1191,16 +1191,16 @@ namespace { return Chunk; }; - CompressedBuffer GenerateBlock(const std::filesystem::path& Path, - const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const std::vector<uint32_t>& ChunksInBlock, - ChunkBlockDescription& OutBlockDescription, - DiskStatistics& DiskStats) + CompressedBuffer GenerateBlock(const std::filesystem::path& Path, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const eastl::vector<uint32_t>& ChunksInBlock, + ChunkBlockDescription& OutBlockDescription, + DiskStatistics& DiskStats) { ReadFileCache OpenFileCache(DiskStats, Path, Content, Lookup, 4); - std::vector<std::pair<IoHash, FetchChunkFunc>> BlockContent; + eastl::vector<std::pair<IoHash, FetchChunkFunc>> BlockContent; BlockContent.reserve(ChunksInBlock.size()); for (uint32_t ChunkIndex : ChunksInBlock) { @@ -1241,18 +1241,18 @@ namespace { } CbObject BuildPart = Storage.GetBuildPart(BuildId, BuildPartId); ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize())); - std::vector<IoHash> ChunkAttachments; + eastl::vector<IoHash> ChunkAttachments; for (CbFieldView LooseFileView : BuildPart["chunkAttachments"sv].AsObjectView()["rawHashes"sv]) { ChunkAttachments.push_back(LooseFileView.AsBinaryAttachment()); } - std::vector<IoHash> BlockAttachments; + eastl::vector<IoHash> BlockAttachments; for (CbFieldView BlocksView : BuildPart["blockAttachments"sv].AsObjectView()["rawHashes"sv]) { BlockAttachments.push_back(BlocksView.AsBinaryAttachment()); } - std::vector<ChunkBlockDescription> VerifyBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockAttachments); + eastl::vector<ChunkBlockDescription> VerifyBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockAttachments); if (VerifyBlockDescriptions.size() != BlockAttachments.size()) { throw std::runtime_error(fmt::format("Uploaded blocks metadata could not all be found, {} blocks metadata is missing", @@ -1401,11 +1401,11 @@ namespace { ProgressBar.Finish(); } - void ArrangeChunksIntoBlocks(const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - uint64_t MaxBlockSize, - std::vector<uint32_t>& ChunkIndexes, - std::vector<std::vector<uint32_t>>& OutBlocks) + void ArrangeChunksIntoBlocks(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + uint64_t MaxBlockSize, + eastl::vector<uint32_t>& ChunkIndexes, + eastl::vector<eastl::vector<uint32_t>>& OutBlocks) { std::sort(ChunkIndexes.begin(), ChunkIndexes.end(), [&Content, &Lookup](uint32_t Lhs, uint32_t Rhs) { const ChunkedContentLookup::ChunkSequenceLocation& LhsLocation = GetChunkSequenceLocations(Lookup, Lhs)[0]; @@ -1464,7 +1464,7 @@ namespace { ScanChunkIndexOffset--; } - std::vector<uint32_t> ChunksInBlock; + eastl::vector<uint32_t> ChunksInBlock; ChunksInBlock.reserve(ChunkIndexOffset - ChunkIndexStart); for (uint32_t AddIndexOffset = ChunkIndexStart; AddIndexOffset < ChunkIndexOffset; AddIndexOffset++) { @@ -1483,7 +1483,7 @@ namespace { } if (ChunkIndexStart < ChunkIndexes.size()) { - std::vector<uint32_t> ChunksInBlock; + eastl::vector<uint32_t> ChunksInBlock; ChunksInBlock.reserve(ChunkIndexes.size() - ChunkIndexStart); for (uint32_t AddIndexOffset = ChunkIndexStart; AddIndexOffset < ChunkIndexes.size(); AddIndexOffset++) { @@ -1567,24 +1567,24 @@ namespace { struct GeneratedBlocks { - std::vector<ChunkBlockDescription> BlockDescriptions; - std::vector<uint64_t> BlockSizes; - std::vector<CompositeBuffer> BlockBuffers; - std::vector<CbObject> BlockMetaDatas; - std::vector<bool> MetaDataHasBeenUploaded; + eastl::vector<ChunkBlockDescription> BlockDescriptions; + eastl::vector<uint64_t> BlockSizes; + eastl::vector<CompositeBuffer> BlockBuffers; + eastl::vector<CbObject> BlockMetaDatas; + eastl::vector<bool> MetaDataHasBeenUploaded; tsl::robin_map<IoHash, size_t, IoHash::Hasher> BlockHashToBlockIndex; }; - void GenerateBuildBlocks(const std::filesystem::path& Path, - const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - BuildStorage& Storage, - const Oid& BuildId, - const std::vector<std::vector<uint32_t>>& NewBlockChunks, - GeneratedBlocks& OutBlocks, - DiskStatistics& DiskStats, - UploadStatistics& UploadStats, - GenerateBlocksStatistics& GenerateBlocksStats) + void GenerateBuildBlocks(const std::filesystem::path& Path, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + BuildStorage& Storage, + const Oid& BuildId, + const eastl::vector<eastl::vector<uint32_t>>& NewBlockChunks, + GeneratedBlocks& OutBlocks, + DiskStatistics& DiskStats, + UploadStatistics& UploadStats, + GenerateBlocksStatistics& GenerateBlocksStats) { const std::size_t NewBlockCount = NewBlockChunks.size(); if (NewBlockCount > 0) @@ -1616,7 +1616,7 @@ namespace { { break; } - const std::vector<uint32_t>& ChunksInBlock = NewBlockChunks[BlockIndex]; + const eastl::vector<uint32_t>& ChunksInBlock = NewBlockChunks[BlockIndex]; Work.ScheduleWork( GenerateBlobsPool, [&, BlockIndex](std::atomic<bool>&) { @@ -1743,20 +1743,20 @@ namespace { } } - void UploadPartBlobs(BuildStorage& Storage, - const Oid& BuildId, - const std::filesystem::path& Path, - const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - std::span<IoHash> RawHashes, - const std::vector<std::vector<uint32_t>>& NewBlockChunks, - GeneratedBlocks& NewBlocks, - std::span<const uint32_t> LooseChunkIndexes, - const std::uint64_t LargeAttachmentSize, - DiskStatistics& DiskStats, - UploadStatistics& UploadStats, - GenerateBlocksStatistics& GenerateBlocksStats, - LooseChunksStatistics& LooseChunksStats) + void UploadPartBlobs(BuildStorage& Storage, + const Oid& BuildId, + const std::filesystem::path& Path, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + std::span<IoHash> RawHashes, + const eastl::vector<eastl::vector<uint32_t>>& NewBlockChunks, + GeneratedBlocks& NewBlocks, + std::span<const uint32_t> LooseChunkIndexes, + const std::uint64_t LargeAttachmentSize, + DiskStatistics& DiskStats, + UploadStatistics& UploadStats, + GenerateBlocksStatistics& GenerateBlocksStats, + LooseChunksStatistics& LooseChunksStats) { { ProgressBar ProgressBar(UsePlainProgress); @@ -1783,8 +1783,8 @@ namespace { ChunkIndexToLooseChunkOrderIndex.insert_or_assign(LooseChunkIndexes[OrderIndex], OrderIndex); } - std::vector<size_t> BlockIndexes; - std::vector<uint32_t> LooseChunkOrderIndexes; + eastl::vector<size_t> BlockIndexes; + eastl::vector<uint32_t> LooseChunkOrderIndexes; uint64_t TotalLooseChunksSize = 0; uint64_t TotalBlocksSize = 0; @@ -1859,7 +1859,7 @@ namespace { if (PayloadSize >= LargeAttachmentSize) { UploadStats.MultipartAttachmentCount++; - std::vector<std::function<void()>> MultipartWork = Storage.PutLargeBuildBlob( + eastl::vector<std::function<void()>> MultipartWork = Storage.PutLargeBuildBlob( BuildId, RawHash, ZenContentType::kCompressedBinary, @@ -1919,7 +1919,7 @@ namespace { Work.DefaultErrorFunction()); }; - std::vector<size_t> GenerateBlockIndexes; + eastl::vector<size_t> GenerateBlockIndexes; std::atomic<uint64_t> GeneratedBlockCount = 0; std::atomic<uint64_t> GeneratedBlockByteCount = 0; @@ -1942,7 +1942,7 @@ namespace { } } - std::vector<uint32_t> CompressLooseChunkOrderIndexes; + eastl::vector<uint32_t> CompressLooseChunkOrderIndexes; // Start upload of any pre-compressed loose chunks for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes) @@ -2075,12 +2075,12 @@ namespace { } } - std::vector<size_t> FindReuseBlocks(const std::vector<ChunkBlockDescription>& KnownBlocks, - std::span<const IoHash> ChunkHashes, - std::span<const uint32_t> ChunkIndexes, - uint8_t MinPercentLimit, - std::vector<uint32_t>& OutUnusedChunkIndexes, - FindBlocksStatistics& FindBlocksStats) + eastl::vector<size_t> FindReuseBlocks(const eastl::vector<ChunkBlockDescription>& KnownBlocks, + std::span<const IoHash> ChunkHashes, + std::span<const uint32_t> ChunkIndexes, + uint8_t MinPercentLimit, + eastl::vector<uint32_t>& OutUnusedChunkIndexes, + FindBlocksStatistics& FindBlocksStats) { // Find all blocks with a usage level higher than MinPercentLimit // Pick out the blocks with usage higher or equal to MinPercentLimit @@ -2090,10 +2090,10 @@ namespace { // If the block still reaches MinPercentLimit, keep it and remove the matching chunks from the not found list // Repeat for following all remaining block that initially matched MinPercentLimit - std::vector<size_t> FilteredReuseBlockIndexes; + eastl::vector<size_t> FilteredReuseBlockIndexes; - uint32_t ChunkCount = gsl::narrow<uint32_t>(ChunkHashes.size()); - std::vector<bool> ChunkFound(ChunkCount, false); + uint32_t ChunkCount = gsl::narrow<uint32_t>(ChunkHashes.size()); + eastl::vector<bool> ChunkFound(ChunkCount, false); if (ChunkCount > 0) { @@ -2108,10 +2108,10 @@ namespace { ChunkHashToChunkIndex.insert_or_assign(ChunkHashes[ChunkIndex], ChunkIndex); } - std::vector<size_t> BlockSizes(KnownBlocks.size(), 0); - std::vector<size_t> BlockUseSize(KnownBlocks.size(), 0); + eastl::vector<size_t> BlockSizes(KnownBlocks.size(), 0); + eastl::vector<size_t> BlockUseSize(KnownBlocks.size(), 0); - std::vector<size_t> ReuseBlockIndexes; + eastl::vector<size_t> ReuseBlockIndexes; for (size_t KnownBlockIndex = 0; KnownBlockIndex < KnownBlocks.size(); KnownBlockIndex++) { @@ -2174,7 +2174,7 @@ namespace { for (size_t KnownBlockIndex : ReuseBlockIndexes) { - std::vector<uint32_t> FoundChunkIndexes; + eastl::vector<uint32_t> FoundChunkIndexes; size_t BlockSize = 0; size_t AdjustedReuseSize = 0; const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; @@ -2298,10 +2298,10 @@ namespace { }; auto ParseManifest = [](const std::filesystem::path& Path, - const std::filesystem::path& ManifestPath) -> std::vector<std::filesystem::path> { - std::vector<std::filesystem::path> AssetPaths; - std::filesystem::path AbsoluteManifestPath = ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath; - IoBuffer ManifestContent = ReadFile(AbsoluteManifestPath).Flatten(); + const std::filesystem::path& ManifestPath) -> eastl::vector<std::filesystem::path> { + eastl::vector<std::filesystem::path> AssetPaths; + std::filesystem::path AbsoluteManifestPath = ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath; + IoBuffer ManifestContent = ReadFile(AbsoluteManifestPath).Flatten(); std::string_view ManifestString((const char*)ManifestContent.GetView().GetData(), ManifestContent.GetSize()); std::string_view::size_type Offset = 0; while (Offset < ManifestContent.GetSize()) @@ -2341,7 +2341,7 @@ namespace { tsl::robin_set<std::string> ExcludeAssetPaths; if (std::filesystem::is_regular_file(ExcludeManifestPath)) { - std::vector<std::filesystem::path> AssetPaths = ParseManifest(Path, ExcludeManifestPath); + eastl::vector<std::filesystem::path> AssetPaths = ParseManifest(Path, ExcludeManifestPath); ExcludeAssetPaths.reserve(AssetPaths.size()); for (const std::filesystem::path& AssetPath : AssetPaths) { @@ -2377,8 +2377,8 @@ namespace { } else { - Stopwatch ManifestParseTimer; - std::vector<std::filesystem::path> AssetPaths = ParseManifest(Path, ManifestPath); + Stopwatch ManifestParseTimer; + eastl::vector<std::filesystem::path> AssetPaths = ParseManifest(Path, ManifestPath); for (const std::filesystem::path& AssetPath : AssetPaths) { Content.Paths.push_back(AssetPath); @@ -2503,15 +2503,15 @@ namespace { GenerateBlocksStatistics GenerateBlocksStats; LooseChunksStatistics LooseChunksStats; - std::vector<ChunkBlockDescription> KnownBlocks; - std::vector<size_t> ReuseBlockIndexes; - std::vector<uint32_t> NewBlockChunkIndexes; - Stopwatch BlockArrangeTimer; + eastl::vector<ChunkBlockDescription> KnownBlocks; + eastl::vector<size_t> ReuseBlockIndexes; + eastl::vector<uint32_t> NewBlockChunkIndexes; + Stopwatch BlockArrangeTimer; - std::vector<std::uint32_t> LooseChunkIndexes; + eastl::vector<std::uint32_t> LooseChunkIndexes; { - bool EnableBlocks = true; - std::vector<std::uint32_t> BlockChunkIndexes; + bool EnableBlocks = true; + eastl::vector<std::uint32_t> BlockChunkIndexes; for (uint32_t ChunkIndex = 0; ChunkIndex < LocalContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++) { const uint64_t ChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; @@ -2560,7 +2560,7 @@ namespace { } } - std::vector<std::vector<uint32_t>> NewBlockChunks; + eastl::vector<eastl::vector<uint32_t>> NewBlockChunks; ArrangeChunksIntoBlocks(LocalContent, LocalLookup, DefaultChunksBlockParams.MaxBlockSize, NewBlockChunkIndexes, NewBlockChunks); FindBlocksStats.NewBlocksCount = NewBlockChunks.size(); @@ -2655,8 +2655,8 @@ namespace { }); PartManifestWriter.AddObject("chunker"sv, ChunkerParameters); - std::vector<IoHash> AllChunkBlockHashes; - std::vector<ChunkBlockDescription> AllChunkBlockDescriptions; + eastl::vector<IoHash> AllChunkBlockHashes; + eastl::vector<ChunkBlockDescription> AllChunkBlockDescriptions; AllChunkBlockHashes.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size()); AllChunkBlockDescriptions.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size()); for (size_t ReuseBlockIndex : ReuseBlockIndexes) @@ -2673,7 +2673,7 @@ namespace { } #if EXTRA_VERIFY tsl::robin_map<IoHash, size_t, IoHash::Hasher> ChunkHashToAbsoluteChunkIndex; - std::vector<IoHash> AbsoluteChunkHashes; + eastl::vector<IoHash> AbsoluteChunkHashes; AbsoluteChunkHashes.reserve(LocalContent.ChunkedContent.ChunkHashes.size()); for (uint32_t ChunkIndex : LooseChunkIndexes) { @@ -2700,11 +2700,11 @@ namespace { ZEN_ASSERT(LocalLookup.ChunkHashToChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]) == ChunkIndex); } #endif // EXTRA_VERIFY - std::vector<uint32_t> AbsoluteChunkOrders = CalculateAbsoluteChunkOrders(LocalContent.ChunkedContent.ChunkHashes, - LocalContent.ChunkedContent.ChunkOrders, - LocalLookup.ChunkHashToChunkIndex, - LooseChunkIndexes, - AllChunkBlockDescriptions); + eastl::vector<uint32_t> AbsoluteChunkOrders = CalculateAbsoluteChunkOrders(LocalContent.ChunkedContent.ChunkHashes, + LocalContent.ChunkedContent.ChunkOrders, + LocalLookup.ChunkHashToChunkIndex, + LooseChunkIndexes, + AllChunkBlockDescriptions); #if EXTRA_VERIFY for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); ChunkOrderIndex++) @@ -2735,10 +2735,10 @@ namespace { { ChunkedFolderContent VerifyFolderContent; - std::vector<uint32_t> OutAbsoluteChunkOrders; - std::vector<IoHash> OutLooseChunkHashes; - std::vector<uint64_t> OutLooseChunkRawSizes; - std::vector<IoHash> OutBlockRawHashes; + eastl::vector<uint32_t> OutAbsoluteChunkOrders; + eastl::vector<IoHash> OutLooseChunkHashes; + eastl::vector<uint64_t> OutLooseChunkRawSizes; + eastl::vector<IoHash> OutBlockRawHashes; 4 ReadBuildContentFromCompactBinary(PartManifestWriter.Save(), VerifyFolderContent.Platform, VerifyFolderContent.Paths, @@ -2797,8 +2797,9 @@ namespace { PartManifest = PartManifestWriter.Save(); } - Stopwatch PutBuildPartResultTimer; - std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult = Storage.PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest); + Stopwatch PutBuildPartResultTimer; + std::pair<IoHash, eastl::vector<IoHash>> PutBuildPartResult = + Storage.PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest); ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.", NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()), NiceBytes(PartManifest.GetSize()), @@ -2863,7 +2864,7 @@ namespace { ZEN_CONSOLE_VERBOSE("PutBuildPart uploading all attachments, needs are: {}", FormatArray<IoHash>(PutBuildPartResult.second, "\n "sv)); - std::vector<IoHash> ForceUploadChunkHashes; + eastl::vector<IoHash> ForceUploadChunkHashes; ForceUploadChunkHashes.reserve(LooseChunkIndexes.size()); for (uint32_t ChunkIndex : LooseChunkIndexes) @@ -2889,8 +2890,8 @@ namespace { while (!AbortFlag) { - Stopwatch FinalizeBuildPartTimer; - std::vector<IoHash> Needs = Storage.FinalizeBuildPart(BuildId, BuildPartId, PartHash); + Stopwatch FinalizeBuildPartTimer; + eastl::vector<IoHash> Needs = Storage.FinalizeBuildPart(BuildId, BuildPartId, PartHash); ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.", NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()), Needs.size()); @@ -2911,8 +2912,8 @@ namespace { if (!NewBlocks.BlockDescriptions.empty() && !AbortFlag) { - uint64_t UploadBlockMetadataCount = 0; - std::vector<IoHash> BlockHashes; + uint64_t UploadBlockMetadataCount = 0; + eastl::vector<IoHash> BlockHashes; BlockHashes.reserve(NewBlocks.BlockDescriptions.size()); Stopwatch UploadBlockMetadataTimer; for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++) @@ -3115,8 +3116,8 @@ namespace { const uint32_t PathCount = gsl::narrow<uint32_t>(Content.Paths.size()); - RwLock ErrorLock; - std::vector<std::string> Errors; + RwLock ErrorLock; + eastl::vector<std::string> Errors; auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders](const std::string_view& RelativePath) -> bool { for (const std::string_view& ExcludeFolder : ExcludeFolders) @@ -3332,18 +3333,18 @@ namespace { OpenFileWriter = {}; OutputFile = {}; } - std::vector<uint32_t> SeenTargetIndexes; + eastl::vector<uint32_t> SeenTargetIndexes; std::unique_ptr<BasicFile> OutputFile; std::unique_ptr<BasicFileWriter> OpenFileWriter; }; - std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> GetRemainingChunkTargets( + eastl::vector<const ChunkedContentLookup::ChunkSequenceLocation*> GetRemainingChunkTargets( std::span<const std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters, const ChunkedContentLookup& Lookup, uint32_t ChunkIndex) { - std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkSources = GetChunkSequenceLocations(Lookup, ChunkIndex); - std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs; + std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkSources = GetChunkSequenceLocations(Lookup, ChunkIndex); + eastl::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs; if (!ChunkSources.empty()) { ChunkTargetPtrs.reserve(ChunkSources.size()); @@ -3369,13 +3370,13 @@ namespace { { ZEN_TRACE_CPU("WriteBlockToDisk"); - std::vector<CompositeBuffer> ChunkBuffers; + eastl::vector<CompositeBuffer> ChunkBuffers; struct WriteOpData { const ChunkedContentLookup::ChunkSequenceLocation* Target; size_t ChunkBufferIndex; }; - std::vector<WriteOpData> WriteOps; + eastl::vector<WriteOpData> WriteOps; SharedBuffer BlockBuffer = DecompressedBlockBuffer.Flatten(); uint64_t HeaderSize = 0; @@ -3384,8 +3385,8 @@ namespace { [&](CompressedBuffer&& Chunk, const IoHash& ChunkHash) { if (auto It = Lookup.ChunkHashToChunkIndex.find(ChunkHash); It != Lookup.ChunkHashToChunkIndex.end()) { - const uint32_t ChunkIndex = It->second; - std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = + const uint32_t ChunkIndex = It->second; + eastl::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, Lookup, ChunkIndex); if (!ChunkTargetPtrs.empty()) @@ -3552,8 +3553,8 @@ namespace { } } - bool CanStreamDecompress(const ChunkedFolderContent& RemoteContent, - const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> Locations) + bool CanStreamDecompress(const ChunkedFolderContent& RemoteContent, + const eastl::vector<const ChunkedContentLookup::ChunkSequenceLocation*> Locations) { if (Locations.size() == 1) { @@ -3618,25 +3619,25 @@ namespace { } } - void DownloadLargeBlob(BuildStorage& Storage, - const std::filesystem::path& TempFolderPath, - const std::filesystem::path& CacheFolderPath, - const ChunkedFolderContent& RemoteContent, - const ChunkedContentLookup& RemoteLookup, - const Oid& BuildId, - const IoHash& ChunkHash, - const std::uint64_t PreferredMultipartChunkSize, - const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>& ChunkTargetPtrs, - std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters, - ParallellWork& Work, - WorkerThreadPool& WritePool, - WorkerThreadPool& NetworkPool, - std::atomic<uint64_t>& WriteToDiskBytes, - std::atomic<uint64_t>& BytesDownloaded, - std::atomic<uint64_t>& LooseChunksBytes, - std::atomic<uint64_t>& DownloadedChunks, - std::atomic<uint32_t>& ChunksComplete, - std::atomic<uint64_t>& MultipartAttachmentCount) + void DownloadLargeBlob(BuildStorage& Storage, + const std::filesystem::path& TempFolderPath, + const std::filesystem::path& CacheFolderPath, + const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& RemoteLookup, + const Oid& BuildId, + const IoHash& ChunkHash, + const std::uint64_t PreferredMultipartChunkSize, + const eastl::vector<const ChunkedContentLookup::ChunkSequenceLocation*>& ChunkTargetPtrs, + std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters, + ParallellWork& Work, + WorkerThreadPool& WritePool, + WorkerThreadPool& NetworkPool, + std::atomic<uint64_t>& WriteToDiskBytes, + std::atomic<uint64_t>& BytesDownloaded, + std::atomic<uint64_t>& LooseChunksBytes, + std::atomic<uint64_t>& DownloadedChunks, + std::atomic<uint32_t>& ChunksComplete, + std::atomic<uint64_t>& MultipartAttachmentCount) { ZEN_TRACE_CPU("DownloadLargeBlob"); @@ -3653,7 +3654,7 @@ namespace { throw std::runtime_error( fmt::format("Failed opening temporary file '{}': {} ({})", Workload->TempFile.GetPath(), Ec.message(), Ec.value())); } - std::vector<std::function<void()>> WorkItems = Storage.GetLargeBuildBlob( + eastl::vector<std::function<void()>> WorkItems = Storage.GetLargeBuildBlob( BuildId, ChunkHash, PreferredMultipartChunkSize, @@ -3670,7 +3671,7 @@ namespace { &DownloadedChunks, &ChunksComplete, SequenceIndexChunksLeftToWriteCounters, - ChunkTargetPtrs = std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>( + ChunkTargetPtrs = eastl::vector<const ChunkedContentLookup::ChunkSequenceLocation*>( ChunkTargetPtrs)](uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining) { BytesDownloaded += Chunk.GetSize(); LooseChunksBytes += Chunk.GetSize(); @@ -3807,17 +3808,17 @@ namespace { } } - void UpdateFolder(BuildStorage& Storage, - const Oid& BuildId, - const std::filesystem::path& Path, - const std::uint64_t LargeAttachmentSize, - const std::uint64_t PreferredMultipartChunkSize, - const ChunkedFolderContent& LocalContent, - const ChunkedFolderContent& RemoteContent, - const std::vector<ChunkBlockDescription>& BlockDescriptions, - const std::vector<IoHash>& LooseChunkHashes, - bool WipeTargetFolder, - FolderContent& OutLocalFolderState) + void UpdateFolder(BuildStorage& Storage, + const Oid& BuildId, + const std::filesystem::path& Path, + const std::uint64_t LargeAttachmentSize, + const std::uint64_t PreferredMultipartChunkSize, + const ChunkedFolderContent& LocalContent, + const ChunkedFolderContent& RemoteContent, + const eastl::vector<ChunkBlockDescription>& BlockDescriptions, + const eastl::vector<IoHash>& LooseChunkHashes, + bool WipeTargetFolder, + FolderContent& OutLocalFolderState) { ZEN_TRACE_CPU("UpdateFolder"); @@ -3843,12 +3844,12 @@ namespace { Stopwatch CacheMappingTimer; - uint64_t CacheMappedBytesForReuse = 0; - std::vector<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters(RemoteContent.ChunkedContent.SequenceRawHashes.size()); - // std::vector<bool> RemoteSequenceIndexIsCachedFlags(RemoteContent.ChunkedContent.SequenceRawHashes.size(), false); - std::vector<bool> RemoteChunkIndexIsCachedFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); + uint64_t CacheMappedBytesForReuse = 0; + eastl::vector<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters(RemoteContent.ChunkedContent.SequenceRawHashes.size()); + // eastl::vector<bool> RemoteSequenceIndexIsCachedFlags(RemoteContent.ChunkedContent.SequenceRawHashes.size(), false); + eastl::vector<bool> RemoteChunkIndexIsCachedFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); // Guard if he same chunks is in multiple blocks (can happen due to block reuse, cache reuse blocks writes directly) - std::vector<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); + eastl::vector<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); // Pick up all whole files we can use from current local state for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); @@ -3870,19 +3871,19 @@ namespace { // Pick up all chunks in current local state struct CacheCopyData { - uint32_t LocalSequenceIndex; - std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> TargetChunkLocationPtrs; + uint32_t LocalSequenceIndex; + eastl::vector<const ChunkedContentLookup::ChunkSequenceLocation*> TargetChunkLocationPtrs; struct ChunkTarget { uint32_t TargetChunkLocationCount; uint64_t ChunkRawSize; uint64_t CacheFileOffset; }; - std::vector<ChunkTarget> ChunkTargets; + eastl::vector<ChunkTarget> ChunkTargets; }; tsl::robin_map<IoHash, size_t, IoHash::Hasher> RawHashToCacheCopyDataIndex; - std::vector<CacheCopyData> CacheCopyDatas; + eastl::vector<CacheCopyData> CacheCopyDatas; for (uint32_t LocalSequenceIndex = 0; LocalSequenceIndex < LocalContent.ChunkedContent.SequenceRawHashes.size(); LocalSequenceIndex++) @@ -3905,7 +3906,7 @@ namespace { const uint32_t RemoteChunkIndex = RemoteChunkIt->second; if (!RemoteChunkIndexIsCachedFlags[RemoteChunkIndex]) { - std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = + eastl::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); if (!ChunkTargetPtrs.empty()) @@ -3929,7 +3930,7 @@ namespace { CacheCopyDatas.push_back( CacheCopyData{.LocalSequenceIndex = LocalSequenceIndex, .TargetChunkLocationPtrs = ChunkTargetPtrs, - .ChunkTargets = std::vector<CacheCopyData::ChunkTarget>{Target}}); + .ChunkTargets = eastl::vector<CacheCopyData::ChunkTarget>{Target}}); } CacheMappedBytesForReuse += LocalChunkRawSize; RemoteChunkIndexIsCachedFlags[RemoteChunkIndex] = true; @@ -3957,7 +3958,7 @@ namespace { } else { - std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = + eastl::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); if (!ChunkTargetPtrs.empty()) { @@ -4001,7 +4002,7 @@ namespace { bool NeedsCopy = true; if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false)) { - std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = + eastl::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); if (ChunkTargetPtrs.empty()) @@ -4181,7 +4182,7 @@ namespace { uint64_t ChunkSize; }; - std::vector<WriteOp> WriteOps; + eastl::vector<WriteOp> WriteOps; WriteOps.reserve(AllTargets.size()); for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) @@ -4444,7 +4445,7 @@ namespace { ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() == 0); } - std::vector<std::pair<IoHash, uint32_t>> Targets; + eastl::vector<std::pair<IoHash, uint32_t>> Targets; Targets.reserve(RemoteContent.Paths.size()); for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) { @@ -4487,7 +4488,7 @@ namespace { { RemotePathToRemoteIndex.insert({RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex}); } - std::vector<std::filesystem::path> LocalFilesToRemove; + eastl::vector<std::filesystem::path> LocalFilesToRemove; for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) { if (!RemotePathToRemoteIndex.contains(LocalContent.Paths[LocalPathIndex].generic_string())) @@ -4647,17 +4648,17 @@ namespace { } } - std::vector<std::pair<Oid, std::string>> ResolveBuildPartNames(BuildStorage& Storage, - const Oid& BuildId, - const std::vector<Oid>& BuildPartIds, - std::span<const std::string> BuildPartNames, - std::uint64_t& OutPreferredMultipartChunkSize) + eastl::vector<std::pair<Oid, std::string>> ResolveBuildPartNames(BuildStorage& Storage, + const Oid& BuildId, + const eastl::vector<Oid>& BuildPartIds, + std::span<const std::string> BuildPartNames, + std::uint64_t& OutPreferredMultipartChunkSize) { - std::vector<std::pair<Oid, std::string>> Result; + eastl::vector<std::pair<Oid, std::string>> Result; { Stopwatch GetBuildTimer; - std::vector<std::pair<Oid, std::string>> AvailableParts; + eastl::vector<std::pair<Oid, std::string>> AvailableParts; CbObject BuildObject = Storage.GetBuild(BuildId); ZEN_CONSOLE("GetBuild took {}. Payload size: {}", @@ -4733,13 +4734,13 @@ namespace { return Result; } - ChunkedFolderContent GetRemoteContent(BuildStorage& Storage, - const Oid& BuildId, - const std::vector<std::pair<Oid, std::string>>& BuildParts, - std::unique_ptr<ChunkingController>& OutChunkController, - std::vector<ChunkedFolderContent>& OutPartContents, - std::vector<ChunkBlockDescription>& OutBlockDescriptions, - std::vector<IoHash>& OutLooseChunkHashes) + ChunkedFolderContent GetRemoteContent(BuildStorage& Storage, + const Oid& BuildId, + const eastl::vector<std::pair<Oid, std::string>>& BuildParts, + std::unique_ptr<ChunkingController>& OutChunkController, + eastl::vector<ChunkedFolderContent>& OutPartContents, + eastl::vector<ChunkBlockDescription>& OutBlockDescriptions, + eastl::vector<IoHash>& OutLooseChunkHashes) { ZEN_TRACE_CPU("GetRemoteContent"); @@ -4758,16 +4759,16 @@ namespace { OutChunkController = CreateChunkingController(ChunkerName, Parameters); } - auto ParseBuildPartManifest = [](BuildStorage& Storage, - const Oid& BuildId, - const Oid& BuildPartId, - CbObject BuildPartManifest, - ChunkedFolderContent& OutRemoteContent, - std::vector<ChunkBlockDescription>& OutBlockDescriptions, - std::vector<IoHash>& OutLooseChunkHashes) { - std::vector<uint32_t> AbsoluteChunkOrders; - std::vector<uint64_t> LooseChunkRawSizes; - std::vector<IoHash> BlockRawHashes; + auto ParseBuildPartManifest = [](BuildStorage& Storage, + const Oid& BuildId, + const Oid& BuildPartId, + CbObject BuildPartManifest, + ChunkedFolderContent& OutRemoteContent, + eastl::vector<ChunkBlockDescription>& OutBlockDescriptions, + eastl::vector<IoHash>& OutLooseChunkHashes) { + eastl::vector<uint32_t> AbsoluteChunkOrders; + eastl::vector<uint64_t> LooseChunkRawSizes; + eastl::vector<IoHash> BlockRawHashes; ReadBuildContentFromCompactBinary(BuildPartManifest, OutRemoteContent.Platform, @@ -4800,9 +4801,9 @@ namespace { if (AttemptFallback) { ZEN_CONSOLE("{} Attemping fallback options.", ErrorDescription); - std::vector<ChunkBlockDescription> AugmentedBlockDescriptions; + eastl::vector<ChunkBlockDescription> AugmentedBlockDescriptions; AugmentedBlockDescriptions.reserve(BlockRawHashes.size()); - std::vector<ChunkBlockDescription> FoundBlocks = Storage.FindBlocks(BuildId); + eastl::vector<ChunkBlockDescription> FoundBlocks = Storage.FindBlocks(BuildId); for (const IoHash& BlockHash : BlockRawHashes) { @@ -4884,8 +4885,8 @@ namespace { ChunkedFolderContent RemoteContent; if (BuildParts.size() > 1) { - std::vector<ChunkBlockDescription> OverlayBlockDescriptions; - std::vector<IoHash> OverlayLooseChunkHashes; + eastl::vector<ChunkBlockDescription> OverlayBlockDescriptions; + eastl::vector<IoHash> OverlayLooseChunkHashes; for (size_t PartIndex = 1; PartIndex < BuildParts.size(); PartIndex++) { const Oid& OverlayBuildPartId = BuildParts[PartIndex].first; @@ -4898,9 +4899,9 @@ namespace { NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()), NiceBytes(OverlayBuildPartManifest.GetSize())); - ChunkedFolderContent OverlayPartContent; - std::vector<ChunkBlockDescription> OverlayPartBlockDescriptions; - std::vector<IoHash> OverlayPartLooseChunkHashes; + ChunkedFolderContent OverlayPartContent; + eastl::vector<ChunkBlockDescription> OverlayPartBlockDescriptions; + eastl::vector<IoHash> OverlayPartLooseChunkHashes; ParseBuildPartManifest(Storage, BuildId, @@ -5007,8 +5008,8 @@ namespace { FolderContent LocalFolderState; - bool ScanContent = true; - std::vector<uint32_t> PathIndexesOufOfDate; + bool ScanContent = true; + eastl::vector<uint32_t> PathIndexesOufOfDate; if (std::filesystem::is_regular_file(Path / ZenStateFilePath)) { try @@ -5017,10 +5018,10 @@ namespace { CbObject CurrentStateObject = LoadCompactBinaryObject(Path / ZenStateFilePath).Object; if (CurrentStateObject) { - Oid CurrentBuildId; - std::vector<Oid> SavedBuildPartIds; - std::vector<std::string> SavedBuildPartsNames; - std::vector<ChunkedFolderContent> SavedPartContents; + Oid CurrentBuildId; + eastl::vector<Oid> SavedBuildPartIds; + eastl::vector<std::string> SavedBuildPartsNames; + eastl::vector<ChunkedFolderContent> SavedPartContents; if (ReadStateObject(CurrentStateObject, CurrentBuildId, SavedBuildPartIds, @@ -5043,8 +5044,8 @@ namespace { if (!LocalFolderState.AreKnownFilesEqual(CurrentLocalFolderContent)) { - const size_t LocaStatePathCount = LocalFolderState.Paths.size(); - std::vector<std::filesystem::path> DeletedPaths; + const size_t LocaStatePathCount = LocalFolderState.Paths.size(); + eastl::vector<std::filesystem::path> DeletedPaths; FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, CurrentLocalFolderContent, DeletedPaths); if (!DeletedPaths.empty()) { @@ -5107,7 +5108,7 @@ namespace { { LocalFolderPaths.insert(LocalFolderPath.generic_string()); } - std::vector<std::filesystem::path> DeletedPaths; + eastl::vector<std::filesystem::path> DeletedPaths; for (const std::filesystem::path& LocalContentPath : LocalContent.Paths) { if (!LocalFolderPaths.contains(LocalContentPath.generic_string())) @@ -5182,7 +5183,7 @@ namespace { void DownloadFolder(BuildStorage& Storage, const Oid& BuildId, - const std::vector<Oid>& BuildPartIds, + const eastl::vector<Oid>& BuildPartIds, std::span<const std::string> BuildPartNames, const std::filesystem::path& Path, bool AllowMultiparts, @@ -5207,15 +5208,15 @@ namespace { std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - std::vector<std::pair<Oid, std::string>> AllBuildParts = + eastl::vector<std::pair<Oid, std::string>> AllBuildParts = ResolveBuildPartNames(Storage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); - std::vector<ChunkedFolderContent> PartContents; + eastl::vector<ChunkedFolderContent> PartContents; std::unique_ptr<ChunkingController> ChunkController; - std::vector<ChunkBlockDescription> BlockDescriptions; - std::vector<IoHash> LooseChunkHashes; + eastl::vector<ChunkBlockDescription> BlockDescriptions; + eastl::vector<IoHash> LooseChunkHashes; ChunkedFolderContent RemoteContent = GetRemoteContent(Storage, BuildId, AllBuildParts, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes); @@ -5343,7 +5344,7 @@ namespace { { std::unique_ptr<ChunkingController> ChunkController = CreateBasicChunkingController(); - std::vector<std::string_view> ExcludeExtensions = DefaultExcludeExtensions; + eastl::vector<std::string_view> ExcludeExtensions = DefaultExcludeExtensions; if (OnlyChunked) { ExcludeExtensions.insert(ExcludeExtensions.end(), @@ -5408,10 +5409,10 @@ namespace { } } - std::vector<IoHash> AddedHashes; - std::vector<IoHash> RemovedHashes; - uint64_t RemovedSize = 0; - uint64_t AddedSize = 0; + eastl::vector<IoHash> AddedHashes; + eastl::vector<IoHash> RemovedHashes; + uint64_t RemovedSize = 0; + uint64_t AddedSize = 0; tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> BaseRawHashLookup; for (size_t PathIndex = 0; PathIndex < BaseFolderContent.RawHashes.size(); PathIndex++) @@ -5775,9 +5776,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) using namespace std::literals; - std::vector<char*> SubCommandArguments; - cxxopts::Options* SubOption = nullptr; - int ParentCommandArgCount = GetSubCommand(m_Options, argc, argv, m_SubCommands, SubOption, SubCommandArguments); + eastl::vector<char*> SubCommandArguments; + cxxopts::Options* SubOption = nullptr; + int ParentCommandArgCount = GetSubCommand(m_Options, argc, argv, m_SubCommands, SubOption, SubCommandArguments); if (!ParseOptions(ParentCommandArgCount, argv)) { return 0; @@ -6151,7 +6152,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); } - std::vector<Oid> BuildPartIds; + eastl::vector<Oid> BuildPartIds; for (const std::string& BuildPartId : m_BuildPartIds) { BuildPartIds.push_back(Oid::TryFromHexString(BuildPartId)); diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp index 6ec6a80db..f12dbcf63 100644 --- a/src/zen/cmds/cache_cmd.cpp +++ b/src/zen/cmds/cache_cmd.cpp @@ -443,8 +443,8 @@ CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a } } - std::vector<std::uniform_int_distribution<uint64_t>> Variations; - std::vector<size_t> SizeRanges; + eastl::vector<std::uniform_int_distribution<uint64_t>> Variations; + eastl::vector<size_t> SizeRanges; SizeRanges.push_back(m_MinSize); Variations.push_back(std::uniform_int_distribution<uint64_t>(0, m_MinSize - 1)); while (SizeRanges.back() < m_MaxSize) @@ -462,7 +462,7 @@ CacheGenerateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a std::mt19937 Generator(RandomDevice()); std::uniform_int_distribution<uint64_t> SizeRangeDistribution(0, SizeRanges.size() - 1); - std::vector<uint64_t> Sizes; + eastl::vector<uint64_t> Sizes; Sizes.reserve(m_Count); for (uint64_t n = 0; n != m_Count; ++n) { diff --git a/src/zen/cmds/dedup_cmd.cpp b/src/zen/cmds/dedup_cmd.cpp index 5f74305c7..ebf7d744c 100644 --- a/src/zen/cmds/dedup_cmd.cpp +++ b/src/zen/cmds/dedup_cmd.cpp @@ -144,7 +144,7 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) DirEntryList_t* DirEntries; }; - std::vector<SizeList> SizeLists{FileSizeMap.size()}; + eastl::vector<SizeList> SizeLists{FileSizeMap.size()}; { int i = 0; @@ -162,10 +162,10 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("Bucket summary:"); - std::vector<size_t> BucketId; - std::vector<size_t> BucketOffsets; - std::vector<size_t> BucketSizes; - std::vector<size_t> BucketFileCounts; + eastl::vector<size_t> BucketId; + eastl::vector<size_t> BucketOffsets; + eastl::vector<size_t> BucketSizes; + eastl::vector<size_t> BucketFileCounts; size_t TotalFileSizes = 0; size_t TotalFileCount = 0; diff --git a/src/zen/cmds/dedup_cmd.h b/src/zen/cmds/dedup_cmd.h index c4f0068e4..022b1cb37 100644 --- a/src/zen/cmds/dedup_cmd.h +++ b/src/zen/cmds/dedup_cmd.h @@ -4,6 +4,8 @@ #include "../zen.h" +#include <vector> + namespace zen { /** Deduplicate files in a tree using block cloning diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 6bc499f03..4ded168b6 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -243,9 +243,9 @@ namespace { } } - std::vector<std::string> GetProjectIds(HttpClient& Http) + eastl::vector<std::string> GetProjectIds(HttpClient& Http) { - std::vector<std::string> AvailableProjects; + eastl::vector<std::string> AvailableProjects; if (HttpClient::Response Result = Http.Get("/prj")) { CbObject CompactBinary = Result.AsObject(); @@ -264,9 +264,9 @@ namespace { return AvailableProjects; } - std::vector<std::string> GetOlogIds(HttpClient& Http, std::string_view ProjectId) + eastl::vector<std::string> GetOlogIds(HttpClient& Http, std::string_view ProjectId) { - std::vector<std::string> AvailableOplogs; + eastl::vector<std::string> AvailableOplogs; if (HttpClient::Response Result = Http.Get(fmt::format("/prj/{}", ProjectId))) { CbObject CompactBinary = Result.AsObject(); @@ -285,15 +285,15 @@ namespace { return AvailableOplogs; } - std::vector<std::string> MatchId(const std::vector<std::string>& Ids, std::string_view FindId) + eastl::vector<std::string> MatchId(const eastl::vector<std::string>& Ids, std::string_view FindId) { - bool FindHasDotDelimiter = FindId.find('.') != std::string_view::npos; - std::vector<std::string> PossibleIds; + bool FindHasDotDelimiter = FindId.find('.') != std::string_view::npos; + eastl::vector<std::string> PossibleIds; for (const std::string& Id : Ids) { if (Id == FindId) { - return std::vector<std::string>{Id}; + return eastl::vector<std::string>{Id}; } else if (Id.starts_with(FindId)) { @@ -306,7 +306,7 @@ namespace { return PossibleIds; } - std::string FmtArray(const std::vector<std::string>& Values, std::string_view Separator = ", "sv, std::string_view Quotes = "'"sv) + std::string FmtArray(const eastl::vector<std::string>& Values, std::string_view Separator = ", "sv, std::string_view Quotes = "'"sv) { ExtendableStringBuilder<512> SB; for (const std::string& Value : Values) @@ -328,11 +328,11 @@ namespace { return SB.ToString(); } - std::string FmtProjectIdArray(const std::vector<std::string>& ProjectIds, - std::string_view Separator = ", "sv, - std::string_view Quotes = "'"sv) + std::string FmtProjectIdArray(const eastl::vector<std::string>& ProjectIds, + std::string_view Separator = ", "sv, + std::string_view Quotes = "'"sv) { - std::vector<std::string> PrettyProjectIds; + eastl::vector<std::string> PrettyProjectIds; PrettyProjectIds.reserve(ProjectIds.size()); for (const std::string& ProjectId : ProjectIds) { @@ -360,7 +360,7 @@ namespace { std::string ResolveProject(HttpClient& Http, std::string_view OptionalProjectName) { - std::vector<std::string> AvailableProjects = GetProjectIds(Http); + eastl::vector<std::string> AvailableProjects = GetProjectIds(Http); if (AvailableProjects.empty()) { if (OptionalProjectName.empty()) @@ -387,7 +387,7 @@ namespace { } } - std::vector<std::string> MatchingProjectIds = MatchId(AvailableProjects, OptionalProjectName); + eastl::vector<std::string> MatchingProjectIds = MatchId(AvailableProjects, OptionalProjectName); if (MatchingProjectIds.empty()) { ZEN_CONSOLE("Unable to match project name '{}' at {}, available projects: {}", @@ -411,7 +411,7 @@ namespace { std::string ResolveOplog(HttpClient& Http, std::string_view ProjectName, std::string_view OptionalOplogName) { - std::vector<std::string> AvailableOplogs = GetOlogIds(Http, ProjectName); + eastl::vector<std::string> AvailableOplogs = GetOlogIds(Http, ProjectName); if (AvailableOplogs.empty()) { if (OptionalOplogName.empty()) @@ -441,7 +441,7 @@ namespace { } } - std::vector<std::string> MatchingOplogIds = MatchId(AvailableOplogs, OptionalOplogName); + eastl::vector<std::string> MatchingOplogIds = MatchId(AvailableOplogs, OptionalOplogName); if (MatchingOplogIds.empty()) { ZEN_CONSOLE("Unable to match oplog name '{}' for project '{}', available oplogs at {}: {}", diff --git a/src/zen/cmds/rpcreplay_cmd.cpp b/src/zen/cmds/rpcreplay_cmd.cpp index 5b88a1f73..fb460b143 100644 --- a/src/zen/cmds/rpcreplay_cmd.cpp +++ b/src/zen/cmds/rpcreplay_cmd.cpp @@ -226,7 +226,7 @@ RpcReplayCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_ProcessCount > 1) { - std::vector<std::unique_ptr<ProcessHandle>> WorkerProcesses; + eastl::vector<std::unique_ptr<ProcessHandle>> WorkerProcesses; WorkerProcesses.resize(m_ProcessCount); ProcessMonitor Monitor; @@ -385,8 +385,8 @@ RpcReplayCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (RequestInfo.ContentType == ZenContentType::kCbPackage) { RequestPackage.SetObject(RequestCopyWriter.Save()); - std::vector<IoBuffer> Buffers = FormatPackageMessage(RequestPackage); - std::vector<SharedBuffer> SharedBuffers(Buffers.begin(), Buffers.end()); + eastl::vector<IoBuffer> Buffers = FormatPackageMessage(RequestPackage); + eastl::vector<SharedBuffer> SharedBuffers(Buffers.begin(), Buffers.end()); Payload = CompositeBuffer(std::move(SharedBuffers)).Flatten().AsIoBuffer(); } else diff --git a/src/zen/cmds/run_cmd.cpp b/src/zen/cmds/run_cmd.cpp index a99ba9704..f40b5e97b 100644 --- a/src/zen/cmds/run_cmd.cpp +++ b/src/zen/cmds/run_cmd.cpp @@ -122,8 +122,8 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::chrono::duration<long, std::milli> Duration{}; }; - std::vector<RunResults> Results; - int ErrorCount = 0; + eastl::vector<RunResults> Results; + int ErrorCount = 0; std::filesystem::path ExecutablePath = SearchPathForExecutable(GlobalOptions.PassthroughArgV[0]); std::string CommandArguments = GlobalOptions.PassthroughArgs; diff --git a/src/zen/cmds/serve_cmd.cpp b/src/zen/cmds/serve_cmd.cpp index f87725e36..ffe322c22 100644 --- a/src/zen/cmds/serve_cmd.cpp +++ b/src/zen/cmds/serve_cmd.cpp @@ -142,8 +142,8 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) uint64_t FileSize; }; - std::filesystem::path RootPath; - std::vector<FileEntry> Files; + std::filesystem::path RootPath; + eastl::vector<FileEntry> Files; }; FsVisitor Visitor; diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp index ac2f42a86..e6f74a2f2 100644 --- a/src/zen/cmds/up_cmd.cpp +++ b/src/zen/cmds/up_cmd.cpp @@ -57,7 +57,7 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) uint16_t DesiredPort = 0; uint16_t EffectivePort = 0; }; - std::vector<EntryInfo> RunningEntries; + eastl::vector<EntryInfo> RunningEntries; State.Snapshot([&RunningEntries, DesiredPort = this->m_Port](const zen::ZenServerState::ZenServerEntry& Entry) { if (DesiredPort == 0 || Entry.DesiredListenPort.load() == DesiredPort) { diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp index 166d4218d..f6f86e078 100644 --- a/src/zen/cmds/workspaces_cmd.cpp +++ b/src/zen/cmds/workspaces_cmd.cpp @@ -74,8 +74,8 @@ namespace { ZEN_CONSOLE("{}Id: {}", Prefix, Workspace.Id); ZEN_CONSOLE("{} Root: {}", Prefix, Workspace.RootPath); ZEN_CONSOLE("{} AllowHttpShares: {}", Prefix, Workspace.AllowShareCreationFromHttp); - std::string Error; - std::vector<Workspaces::WorkspaceShareConfiguration> Shares = Workspaces::ReadWorkspaceConfig(Log(), Workspace.RootPath, Error); + std::string Error; + eastl::vector<Workspaces::WorkspaceShareConfiguration> Shares = Workspaces::ReadWorkspaceConfig(Log(), Workspace.RootPath, Error); if (!Error.empty()) { ZEN_CONSOLE("{}Failed to read shares from workspace {}. Reason: '{}'", Prefix, Workspace.Id, Error); @@ -133,9 +133,9 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) using namespace std::literals; - std::vector<char*> SubCommandArguments; - cxxopts::Options* SubOption = nullptr; - int ParentCommandArgCount = GetSubCommand(m_Options, argc, argv, m_SubCommands, SubOption, SubCommandArguments); + eastl::vector<char*> SubCommandArguments; + cxxopts::Options* SubOption = nullptr; + int ParentCommandArgCount = GetSubCommand(m_Options, argc, argv, m_SubCommands, SubOption, SubCommandArguments); if (!ParseOptions(ParentCommandArgCount, argv)) { return 0; @@ -211,8 +211,8 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (m_Id.empty()) { - std::string Error; - static std::vector<Workspaces::WorkspaceConfiguration> Configs = Workspaces::ReadConfig(Log(), StatePath, Error); + std::string Error; + static eastl::vector<Workspaces::WorkspaceConfiguration> Configs = Workspaces::ReadConfig(Log(), StatePath, Error); if (!Error.empty()) { ZEN_CONSOLE("Failed to read workspaces state from '{}'. Reason: '{}'", StatePath, Error); @@ -384,9 +384,9 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** using namespace std::literals; - std::vector<char*> SubCommandArguments; - cxxopts::Options* SubOption = nullptr; - int ParentCommandArgCount = GetSubCommand(m_Options, argc, argv, m_SubCommands, SubOption, SubCommandArguments); + eastl::vector<char*> SubCommandArguments; + cxxopts::Options* SubOption = nullptr; + int ParentCommandArgCount = GetSubCommand(m_Options, argc, argv, m_SubCommands, SubOption, SubCommandArguments); if (!ParseOptions(ParentCommandArgCount, argv)) { return 0; @@ -709,7 +709,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** std::span<const std::string> ChunkIds) -> std::vector<std::string> { std::vector<std::string> Oids; Oids.reserve(ChunkIds.size()); - std::vector<size_t> NeedsConvertIndexes; + eastl::vector<size_t> NeedsConvertIndexes; for (const std::string& StringChunkId : ChunkIds) { Oid ChunkId = Oid::TryFromHexString(StringChunkId); @@ -766,7 +766,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** } HttpClient Http(m_HostName); - m_ChunkId = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, std::vector<std::string>{m_ChunkId})[0]; + m_ChunkId = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, eastl::vector<std::string>{m_ChunkId})[0]; HttpClient::KeyValueMap Params; if (m_Offset != 0) @@ -810,7 +810,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** HttpClient Http(m_HostName); m_ChunkIds = ChunksToOidStrings(Http, m_WorkspaceId, m_ShareId, m_ChunkIds); - std::vector<RequestChunkEntry> ChunkRequests; + eastl::vector<RequestChunkEntry> ChunkRequests; ChunkRequests.resize(m_ChunkIds.size()); for (size_t Index = 0; Index < m_ChunkIds.size(); Index++) { @@ -824,7 +824,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** if (HttpClient::Response Result = Http.Post(fmt::format("/ws/{}/batch", GetShareIdentityUrl(m_GetChunkBatchOptions)), Payload)) { ZEN_CONSOLE("{}: Bytes: {}", Result, NiceBytes(Result.ResponsePayload.GetSize())); - std::vector<IoBuffer> Results = ParseChunkBatchResponse(Result.ResponsePayload); + eastl::vector<IoBuffer> Results = ParseChunkBatchResponse(Result.ResponsePayload); if (Results.size() != m_ChunkIds.size()) { throw std::runtime_error( diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 713a0d66d..4c6aa0087 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -128,7 +128,7 @@ ZenCmdBase::GetSubCommand(cxxopts::Options&, char** argv, std::span<cxxopts::Options*> SubOptions, cxxopts::Options*& OutSubOption, - std::vector<char*>& OutSubCommandArguments) + eastl::vector<char*>& OutSubCommandArguments) { for (int i = 1; i < argc; ++i) { @@ -518,9 +518,9 @@ main(int argc, char** argv) // Split command line into options, commands and any pass-through arguments - std::string Passthrough; - std::string PassthroughArgs; - std::vector<std::string> PassthroughArgV; + std::string Passthrough; + std::string PassthroughArgs; + eastl::vector<std::string> PassthroughArgV; for (int i = 1; i < argc; ++i) { @@ -576,7 +576,7 @@ main(int argc, char** argv) // Split command line into global vs command options. We do this by simply // scanning argv for a string we recognise as a command and split it there - std::vector<char*> CommandArgVec; + eastl::vector<char*> CommandArgVec; CommandArgVec.push_back(argv[0]); for (int i = 1; i < argc; ++i) diff --git a/src/zen/zen.h b/src/zen/zen.h index 835c2b6ac..510429b91 100644 --- a/src/zen/zen.h +++ b/src/zen/zen.h @@ -6,6 +6,8 @@ #include <zencore/timer.h> #include <zencore/zencore.h> +#include <EASTL/vector.h> + ZEN_THIRD_PARTY_INCLUDES_START #include <cxxopts.hpp> ZEN_THIRD_PARTY_INCLUDES_END @@ -22,9 +24,9 @@ struct ZenCliOptions bool IsVerbose = false; // Arguments after " -- " on command line are passed through and not parsed - std::string PassthroughCommandLine; - std::string PassthroughArgs; - std::vector<std::string> PassthroughArgV; + std::string PassthroughCommandLine; + std::string PassthroughArgs; + eastl::vector<std::string> PassthroughArgV; }; struct ZenCmdCategory @@ -55,7 +57,7 @@ public: char** argv, std::span<cxxopts::Options*> SubOptions, cxxopts::Options*& OutSubOption, - std::vector<char*>& OutSubCommandArguments); + eastl::vector<char*>& OutSubCommandArguments); static std::string FormatHttpResponse(const cpr::Response& Response); static int MapHttpToCommandReturnCode(const cpr::Response& Response); static std::string ResolveTargetHostSpec(const std::string& InHostSpec); diff --git a/src/zencore/callstack.cpp b/src/zencore/callstack.cpp index 9b06d4575..79e69e092 100644 --- a/src/zencore/callstack.cpp +++ b/src/zencore/callstack.cpp @@ -137,10 +137,10 @@ GetCallstack(int FramesToSkip, int FramesToCapture, void* OutAddresses[]) #endif } -std::vector<std::string> +eastl::vector<std::string> GetFrameSymbols(uint32_t FrameCount, void** Frames) { - std::vector<std::string> FrameSymbols; + eastl::vector<std::string> FrameSymbols; if (FrameCount > 0) { FrameSymbols.resize(FrameCount); @@ -210,7 +210,7 @@ TEST_CASE("Callstack.Basic") void* Addresses[4]; uint32_t FrameCount = GetCallstack(1, 4, Addresses); CHECK(FrameCount > 0); - std::vector<std::string> Symbols = GetFrameSymbols(FrameCount, Addresses); + eastl::vector<std::string> Symbols = GetFrameSymbols(FrameCount, Addresses); for (const std::string& Symbol : Symbols) { CHECK(!Symbol.empty()); diff --git a/src/zencore/compactbinary.cpp b/src/zencore/compactbinary.cpp index b43cc18f1..23edec277 100644 --- a/src/zencore/compactbinary.cpp +++ b/src/zencore/compactbinary.cpp @@ -1484,13 +1484,13 @@ SaveCompactBinary(BinaryWriter& Ar, const CbObjectView& Object) ////////////////////////////////////////////////////////////////////////// -std::vector<CbFieldView> +eastl::vector<CbFieldView> ReadCompactBinaryStream(MemoryView Data) { - std::vector<CbFieldView> Result; - const uint8_t* Buffer = reinterpret_cast<const uint8_t*>(Data.GetData()); - uint64_t Offset = 0; - const uint64_t Size = Data.GetSize(); + eastl::vector<CbFieldView> Result; + const uint8_t* Buffer = reinterpret_cast<const uint8_t*>(Data.GetData()); + uint64_t Offset = 0; + const uint64_t Size = Data.GetSize(); while (Offset < Size) { if (ValidateCompactBinary(MemoryView(Buffer + Offset, Size - Offset), CbValidateMode::Default) != CbValidateError::None) diff --git a/src/zencore/compactbinaryjson.cpp b/src/zencore/compactbinaryjson.cpp index d8c8a8584..ac642d30e 100644 --- a/src/zencore/compactbinaryjson.cpp +++ b/src/zencore/compactbinaryjson.cpp @@ -10,8 +10,8 @@ #include <zencore/string.h> #include <zencore/testing.h> +#include <EASTL/vector.h> #include <fmt/format.h> -#include <vector> ZEN_THIRD_PARTY_INCLUDES_START #include <json11.hpp> @@ -319,8 +319,8 @@ CompactBinaryToJson(const CbArrayView& Array, StringBuilderBase& Builder) void CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder) { - std::vector<CbFieldView> Fields = ReadCompactBinaryStream(Data); - CbJsonWriter Writer(InBuilder); + eastl::vector<CbFieldView> Fields = ReadCompactBinaryStream(Data); + CbJsonWriter Writer(InBuilder); if (!Fields.empty()) { if (Fields.size() == 1) @@ -616,7 +616,7 @@ TEST_CASE("uson.json") SUBCASE("stream") { - const auto MakeObject = [&](std::string_view Name, const std::vector<int>& Fields) -> CbObject { + const auto MakeObject = [&](std::string_view Name, const eastl::vector<int>& Fields) -> CbObject { CbWriter Writer; Writer.SetName(Name); Writer.BeginObject(); @@ -628,7 +628,7 @@ TEST_CASE("uson.json") return Writer.Save().AsObject(); }; - std::vector<uint8_t> Buffer; + eastl::vector<uint8_t> Buffer; auto AppendToBuffer = [&](const void* Data, size_t Count) { const uint8_t* AppendBytes = reinterpret_cast<const uint8_t*>(Data); diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp index 6f53bba69..53a6f2f50 100644 --- a/src/zencore/compactbinaryvalidation.cpp +++ b/src/zencore/compactbinaryvalidation.cpp @@ -8,6 +8,8 @@ #include <zencore/string.h> #include <zencore/testing.h> +#include <EASTL/sort.h> + #include <algorithm> namespace zen { @@ -209,7 +211,7 @@ ValidateCbObject(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, if (Size > 0) { - std::vector<std::string_view> Names; + eastl::vector<std::string_view> Names; const bool bUniformObject = CbFieldTypeOps::GetType(ObjectType) == CbFieldType::UniformObject; const CbFieldType ExternalType = bUniformObject ? ValidateCbFieldType(ObjectView, Mode, Error) : CbFieldType::HasFieldType; @@ -234,7 +236,7 @@ ValidateCbObject(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, if (EnumHasAnyFlags(Mode, CbValidateMode::Names) && Names.size() > 1) { - std::sort(begin(Names), end(Names), [](std::string_view L, std::string_view R) { return L.compare(R) < 0; }); + std::sort(Names.begin(), Names.end(), [](std::string_view L, std::string_view R) { return L.compare(R) < 0; }); for (const std::string_view *NamesIt = Names.data(), *NamesEnd = NamesIt + Names.size() - 1; NamesIt != NamesEnd; ++NamesIt) { @@ -596,8 +598,8 @@ ValidateObjectAttachment(MemoryView View, CbValidateMode Mode) CbValidateError ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode) { - std::vector<IoHash> Attachments; - CbValidateError Error = CbValidateError::None; + eastl::vector<IoHash> Attachments; + CbValidateError Error = CbValidateError::None; if (EnumHasAnyFlags(Mode, CbValidateMode::All)) { uint32_t ObjectCount = 0; @@ -641,7 +643,7 @@ ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode) if (Attachments.size() && EnumHasAnyFlags(Mode, CbValidateMode::Package)) { - std::sort(begin(Attachments), end(Attachments)); + eastl::sort(Attachments.begin(), Attachments.end()); for (const IoHash *It = Attachments.data(), *End = It + Attachments.size() - 1; It != End; ++It) { if (It[0] == It[1]) diff --git a/src/zencore/compactbinaryyaml.cpp b/src/zencore/compactbinaryyaml.cpp index 3a9705684..7f71656ab 100644 --- a/src/zencore/compactbinaryyaml.cpp +++ b/src/zencore/compactbinaryyaml.cpp @@ -10,9 +10,9 @@ #include <zencore/string.h> #include <zencore/testing.h> +#include <EASTL/vector.h> #include <fmt/format.h> #include <string_view> -#include <vector> ZEN_THIRD_PARTY_INCLUDES_START #include <ryml/ryml.hpp> @@ -210,9 +210,9 @@ private: StringBuilderBase& m_StrBuilder; bool m_IsFirst = true; - ryml::Tree m_Tree; - std::vector<ryml::NodeRef> m_NodeStack; - ryml::NodeRef& Top() { return m_NodeStack.back(); } + ryml::Tree m_Tree; + eastl::vector<ryml::NodeRef> m_NodeStack; + ryml::NodeRef& Top() { return m_NodeStack.back(); } }; void diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp index 1844f6a63..bc9ce0c30 100644 --- a/src/zencore/compress.cpp +++ b/src/zencore/compress.cpp @@ -349,7 +349,7 @@ BlockEncoder::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize) UniqueBuffer CompressedData = UniqueBuffer::Alloc(CompressedDataSize); // Compress the raw data in blocks and store the raw data for incompressible blocks. - std::vector<uint32_t> CompressedBlockSizes; + eastl::vector<uint32_t> CompressedBlockSizes; CompressedBlockSizes.reserve(BlockCount); uint64_t CompressedSize = 0; { @@ -472,7 +472,7 @@ BlockEncoder::CompressToStream(const CompositeBuffer& RawData, const uint64_t MetaSize = BlockCount * sizeof(uint32_t); const uint64_t FullHeaderSize = sizeof(BufferHeader) + MetaSize; - std::vector<uint32_t> CompressedBlockSizes; + eastl::vector<uint32_t> CompressedBlockSizes; CompressedBlockSizes.reserve(BlockCount); uint64_t CompressedSize = 0; { @@ -638,7 +638,7 @@ BlockDecoder::DecompressToComposite(const BufferHeader& Header, const CompositeB return TryDecompressTo(Header, CompressedData, Buffer, 0) ? CompositeBuffer(Buffer.MoveToShared()) : CompositeBuffer(); } - std::vector<uint32_t> CompressedBlockSizes; + eastl::vector<uint32_t> CompressedBlockSizes; CompressedBlockSizes.resize(Header.BlockCount); CompressedData.CopyTo(MakeMutableMemoryView(CompressedBlockSizes), sizeof(BufferHeader)); @@ -669,11 +669,11 @@ BlockDecoder::DecompressToComposite(const BufferHeader& Header, const CompositeB } // Decompress the compressed data in blocks and reference the uncompressed blocks. - uint64_t PendingCompressedSegmentOffset = sizeof(BufferHeader) + uint64_t(Header.BlockCount) * sizeof(uint32_t); - uint64_t PendingCompressedSegmentSize = 0; - uint64_t PendingRawSegmentOffset = 0; - uint64_t PendingRawSegmentSize = 0; - std::vector<SharedBuffer> Segments; + uint64_t PendingCompressedSegmentOffset = sizeof(BufferHeader) + uint64_t(Header.BlockCount) * sizeof(uint32_t); + uint64_t PendingCompressedSegmentSize = 0; + uint64_t PendingRawSegmentOffset = 0; + uint64_t PendingRawSegmentSize = 0; + eastl::vector<SharedBuffer> Segments; const auto CommitPendingCompressedSegment = [&PendingCompressedSegmentOffset, &PendingCompressedSegmentSize, &CompressedData, &Segments] { @@ -1578,11 +1578,11 @@ GetCompressedRange(const BufferHeader& Header, const size_t LastBlock = uint64_t((RawOffset + RawSize - 1) / BlockSize); uint64_t CompressedOffset = sizeof(BufferHeader) + uint64_t(Header.BlockCount) * sizeof(uint32_t); - const uint64_t NewBlockCount = LastBlock - FirstBlock + 1; - const uint64_t NewMetaSize = NewBlockCount * sizeof(uint32_t); - uint64_t NewCompressedSize = 0; - uint64_t NewTotalRawSize = 0; - std::vector<uint32_t> NewCompressedBlockSizes; + const uint64_t NewBlockCount = LastBlock - FirstBlock + 1; + const uint64_t NewMetaSize = NewBlockCount * sizeof(uint32_t); + uint64_t NewCompressedSize = 0; + uint64_t NewTotalRawSize = 0; + eastl::vector<uint32_t> NewCompressedBlockSizes; NewCompressedBlockSizes.reserve(NewBlockCount); for (size_t BlockIndex = FirstBlock; BlockIndex <= LastBlock; ++BlockIndex) @@ -1666,11 +1666,11 @@ CopyCompressedRange(const BufferHeader& Header, const size_t LastBlock = uint64_t((RawOffset + RawSize - 1) / BlockSize); uint64_t CompressedOffset = sizeof(BufferHeader) + uint64_t(Header.BlockCount) * sizeof(uint32_t); - const uint64_t NewBlockCount = LastBlock - FirstBlock + 1; - const uint64_t NewMetaSize = NewBlockCount * sizeof(uint32_t); - uint64_t NewCompressedSize = 0; - uint64_t NewTotalRawSize = 0; - std::vector<uint32_t> NewCompressedBlockSizes; + const uint64_t NewBlockCount = LastBlock - FirstBlock + 1; + const uint64_t NewMetaSize = NewBlockCount * sizeof(uint32_t); + uint64_t NewCompressedSize = 0; + uint64_t NewTotalRawSize = 0; + eastl::vector<uint32_t> NewCompressedBlockSizes; NewCompressedBlockSizes.reserve(NewBlockCount); for (size_t BlockIndex = FirstBlock; BlockIndex <= LastBlock; ++BlockIndex) @@ -2359,8 +2359,8 @@ TEST_CASE("CompressedBuffer") CHECK(IoHash::HashBuffer(Decomp) == DecodedHash); } - auto GenerateData = [](uint64_t N) -> std::vector<uint64_t> { - std::vector<uint64_t> Data; + auto GenerateData = [](uint64_t N) -> eastl::vector<uint64_t> { + eastl::vector<uint64_t> Data; Data.resize(N); for (size_t Idx = 0; Idx < Data.size(); ++Idx) { @@ -2379,23 +2379,23 @@ TEST_CASE("CompressedBuffer") SUBCASE("decompress with offset and size") { - auto UncompressAndValidate = [&ValidateData](CompressedBuffer Compressed, - uint64_t OffsetCount, - uint64_t Count, - const std::vector<uint64_t>& ExpectedValues) { + auto UncompressAndValidate = [&ValidateData](CompressedBuffer Compressed, + uint64_t OffsetCount, + uint64_t Count, + const eastl::vector<uint64_t>& ExpectedValues) { SharedBuffer Uncompressed = Compressed.Decompress(OffsetCount * sizeof(uint64_t), Count * sizeof(uint64_t)); CHECK(Uncompressed.GetSize() == Count * sizeof(uint64_t)); std::span<uint64_t const> Values((const uint64_t*)Uncompressed.GetData(), Uncompressed.GetSize() / sizeof(uint64_t)); ValidateData(Values, ExpectedValues, OffsetCount); }; - const uint64_t BlockSize = 64 * sizeof(uint64_t); - const uint64_t N = 5000; - std::vector<uint64_t> ExpectedValues = GenerateData(N); - CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), - OodleCompressor::Mermaid, - OodleCompressionLevel::Optimal4, - BlockSize); + const uint64_t BlockSize = 64 * sizeof(uint64_t); + const uint64_t N = 5000; + eastl::vector<uint64_t> ExpectedValues = GenerateData(N); + CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), + OodleCompressor::Mermaid, + OodleCompressionLevel::Optimal4, + BlockSize); UncompressAndValidate(Compressed, 0, N, ExpectedValues); UncompressAndValidate(Compressed, 1, N - 1, ExpectedValues); UncompressAndValidate(Compressed, N - 1, 1, ExpectedValues); @@ -2411,7 +2411,7 @@ TEST_CASE("CompressedBuffer") { const uint64_t BlockSize = 64 * sizeof(uint64_t); const uint64_t N = 1000; - std::vector<uint64_t> ExpectedValues = GenerateData(N); + eastl::vector<uint64_t> ExpectedValues = GenerateData(N); CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), OodleCompressor::Mermaid, OodleCompressionLevel::Optimal4, @@ -2426,7 +2426,7 @@ TEST_CASE("CompressedBuffer") { const uint64_t BlockSize = 256 * sizeof(uint64_t); const uint64_t N = 100; - std::vector<uint64_t> ExpectedValues = GenerateData(N); + eastl::vector<uint64_t> ExpectedValues = GenerateData(N); CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), OodleCompressor::Mermaid, OodleCompressionLevel::Optimal4, @@ -2440,11 +2440,11 @@ TEST_CASE("CompressedBuffer") SUBCASE("decompress uncompressed buffer") { - const uint64_t N = 4242; - std::vector<uint64_t> ExpectedValues = GenerateData(N); - CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), - OodleCompressor::NotSet, - OodleCompressionLevel::None); + const uint64_t N = 4242; + eastl::vector<uint64_t> ExpectedValues = GenerateData(N); + CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), + OodleCompressor::NotSet, + OodleCompressionLevel::None); { const uint64_t OffsetCount = 0; const uint64_t Count = N; @@ -2464,9 +2464,9 @@ TEST_CASE("CompressedBuffer") SUBCASE("copy range") { - const uint64_t BlockSize = 64 * sizeof(uint64_t); - const uint64_t N = 1000; - std::vector<uint64_t> ExpectedValues = GenerateData(N); + const uint64_t BlockSize = 64 * sizeof(uint64_t); + const uint64_t N = 1000; + eastl::vector<uint64_t> ExpectedValues = GenerateData(N); CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), OodleCompressor::Mermaid, @@ -2524,9 +2524,9 @@ TEST_CASE("CompressedBuffer") SUBCASE("get range") { - const uint64_t BlockSize = 64 * sizeof(uint64_t); - const uint64_t N = 1000; - std::vector<uint64_t> ExpectedValues = GenerateData(N); + const uint64_t BlockSize = 64 * sizeof(uint64_t); + const uint64_t N = 1000; + eastl::vector<uint64_t> ExpectedValues = GenerateData(N); CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), OodleCompressor::Mermaid, @@ -2584,8 +2584,8 @@ TEST_CASE("CompressedBuffer") SUBCASE("copy uncompressed range") { - const uint64_t N = 1000; - std::vector<uint64_t> ExpectedValues = GenerateData(N); + const uint64_t N = 1000; + eastl::vector<uint64_t> ExpectedValues = GenerateData(N); CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), OodleCompressor::NotSet, @@ -2621,8 +2621,8 @@ TEST_CASE("CompressedBuffer") SUBCASE("get uncompressed range") { - const uint64_t N = 1000; - std::vector<uint64_t> ExpectedValues = GenerateData(N); + const uint64_t N = 1000; + eastl::vector<uint64_t> ExpectedValues = GenerateData(N); CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), OodleCompressor::NotSet, @@ -2659,8 +2659,8 @@ TEST_CASE("CompressedBuffer") TEST_CASE("CompressedBufferReader") { - const auto GenerateData = [](size_t N) -> std::vector<uint64_t> { - std::vector<uint64_t> Data; + const auto GenerateData = [](size_t N) -> eastl::vector<uint64_t> { + eastl::vector<uint64_t> Data; Data.resize(N); for (size_t Idx = 0; Idx < Data.size(); ++Idx) { @@ -2673,10 +2673,10 @@ TEST_CASE("CompressedBufferReader") SUBCASE("Decompress with offset and size") { - const auto UncompressAndValidate = [&](const CompressedBuffer& Compressed, - const int32_t OffsetCount, - const int32_t Count, - const std::vector<uint64_t>& ExpectedValues) { + const auto UncompressAndValidate = [&](const CompressedBuffer& Compressed, + const int32_t OffsetCount, + const int32_t Count, + const eastl::vector<uint64_t>& ExpectedValues) { Reader.SetSource(Compressed); { const SharedBuffer Uncompressed = Reader.Decompress(OffsetCount * sizeof(uint64_t), Count * sizeof(uint64_t)); @@ -2691,9 +2691,9 @@ TEST_CASE("CompressedBufferReader") } }; - constexpr uint64_t BlockSize = 64 * sizeof(uint64_t); - constexpr int32_t N = 5000; - const std::vector<uint64_t> ExpectedValues = GenerateData(N); + constexpr uint64_t BlockSize = 64 * sizeof(uint64_t); + constexpr int32_t N = 5000; + const eastl::vector<uint64_t> ExpectedValues = GenerateData(N); const CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), OodleCompressor::Mermaid, @@ -2715,9 +2715,9 @@ TEST_CASE("CompressedBufferReader") SUBCASE("Decompress with offset only") { - constexpr uint64_t BlockSize = 64 * sizeof(uint64_t); - constexpr int32_t N = 1000; - const std::vector<uint64_t> ExpectedValues = GenerateData(N); + constexpr uint64_t BlockSize = 64 * sizeof(uint64_t); + constexpr int32_t N = 1000; + const eastl::vector<uint64_t> ExpectedValues = GenerateData(N); const CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), OodleCompressor::Mermaid, @@ -2754,9 +2754,9 @@ TEST_CASE("CompressedBufferReader") SUBCASE("Decompress only block") { - constexpr uint64_t BlockSize = 256 * sizeof(uint64_t); - constexpr int32_t N = 100; - const std::vector<uint64_t> ExpectedValues = GenerateData(N); + constexpr uint64_t BlockSize = 256 * sizeof(uint64_t); + constexpr int32_t N = 100; + const eastl::vector<uint64_t> ExpectedValues = GenerateData(N); const CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), OodleCompressor::Mermaid, @@ -2781,8 +2781,8 @@ TEST_CASE("CompressedBufferReader") SUBCASE("Decompress from an uncompressed buffer.") { - constexpr int32_t N = 4242; - const std::vector<uint64_t> ExpectedValues = GenerateData(N); + constexpr int32_t N = 4242; + const eastl::vector<uint64_t> ExpectedValues = GenerateData(N); const CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer::MakeView(MakeMemoryView(ExpectedValues)), OodleCompressor::NotSet, diff --git a/src/zencore/crypto.cpp b/src/zencore/crypto.cpp index 78bea0c17..bcb765b8d 100644 --- a/src/zencore/crypto.cpp +++ b/src/zencore/crypto.cpp @@ -360,8 +360,8 @@ TEST_CASE("crypto.aes") std::string_view PlainText = "The quick brown fox jumps over the lazy dog"sv; - std::vector<uint8_t> EncryptionBuffer; - std::vector<uint8_t> DecryptionBuffer; + eastl::vector<uint8_t> EncryptionBuffer; + eastl::vector<uint8_t> DecryptionBuffer; std::optional<std::string> Reason; EncryptionBuffer.resize(PlainText.size() + Aes::BlockSize); diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 85feab2f7..69fee90e9 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -842,14 +842,14 @@ WriteFile(std::filesystem::path Path, IoBuffer Data) void WriteFile(std::filesystem::path Path, CompositeBuffer InData) { - std::vector<IoBuffer> DataVec; + eastl::vector<IoBuffer> DataVec; for (const SharedBuffer& Segment : InData.GetSegments()) { DataVec.push_back(Segment.AsIoBuffer()); } - std::vector<const IoBuffer*> DataPtrs; + eastl::vector<const IoBuffer*> DataPtrs; for (IoBuffer& Data : DataVec) { @@ -1021,9 +1021,9 @@ ScanFile(void* NativeHandle, std::function<void(const void* Data, size_t Size)>&& ProcessFunc) { ZEN_ASSERT(NativeHandle != nullptr); - uint64_t BufferSize = Min(ChunkSize, Size); - std::vector<uint8_t> ReadBuffer(BufferSize); - uint64_t ReadOffset = 0; + uint64_t BufferSize = Min(ChunkSize, Size); + eastl::vector<uint8_t> ReadBuffer(BufferSize); + uint64_t ReadOffset = 0; while (ReadOffset < Size) { const uint64_t NumberOfBytesToRead = Min(Size - ReadOffset, BufferSize); @@ -1070,7 +1070,7 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<voi return false; } - std::vector<uint8_t> ReadBuffer(ChunkSize); + eastl::vector<uint8_t> ReadBuffer(ChunkSize); for (;;) { @@ -1739,7 +1739,7 @@ GetEnvVariable(std::string_view VariableName) ZEN_ASSERT(!VariableName.empty()); #if ZEN_PLATFORM_WINDOWS - std::vector<CHAR> EnvVariableBuffer(1023 + 1); + eastl::vector<CHAR> EnvVariableBuffer(1023 + 1); DWORD RESULT = GetEnvironmentVariableA(std::string(VariableName).c_str(), EnvVariableBuffer.data(), (DWORD)EnvVariableBuffer.size()); if (RESULT == 0) { @@ -2101,8 +2101,8 @@ TEST_CASE("filesystem") CHECK(Visitor.bFoundExpected); // Scan/read file - FileContents BinRead = ReadFile(BinPath); - std::vector<uint8_t> BinScan; + FileContents BinRead = ReadFile(BinPath); + eastl::vector<uint8_t> BinScan; ScanFile(BinPath, 16 << 10, [&](const void* Data, size_t Size) { const auto* Ptr = (uint8_t*)Data; BinScan.insert(BinScan.end(), Ptr, Ptr + Size); diff --git a/src/zencore/include/zencore/callstack.h b/src/zencore/include/zencore/callstack.h index ef4ba0e91..44b01e74f 100644 --- a/src/zencore/include/zencore/callstack.h +++ b/src/zencore/include/zencore/callstack.h @@ -6,8 +6,8 @@ #include <zencore/string.h> +#include <EASTL/vector.h> #include <string> -#include <vector> namespace zen { @@ -21,9 +21,9 @@ CallstackFrames* CreateCallstack(uint32_t FrameCount, void** Frames) noexcept; CallstackFrames* CloneCallstack(const CallstackFrames* Callstack) noexcept; void FreeCallstack(CallstackFrames* Callstack) noexcept; -uint32_t GetCallstack(int FramesToSkip, int FramesToCapture, void* OutAddresses[]); -std::vector<std::string> GetFrameSymbols(uint32_t FrameCount, void** Frames); -inline std::vector<std::string> +uint32_t GetCallstack(int FramesToSkip, int FramesToCapture, void* OutAddresses[]); +eastl::vector<std::string> GetFrameSymbols(uint32_t FrameCount, void** Frames); +inline eastl::vector<std::string> GetFrameSymbols(const CallstackFrames* Callstack) { return GetFrameSymbols(Callstack ? Callstack->FrameCount : 0, Callstack ? Callstack->Frames : nullptr); diff --git a/src/zencore/include/zencore/compactbinary.h b/src/zencore/include/zencore/compactbinary.h index 0fdb56d67..b97a29277 100644 --- a/src/zencore/include/zencore/compactbinary.h +++ b/src/zencore/include/zencore/compactbinary.h @@ -15,12 +15,12 @@ #include <zencore/uid.h> #include <zencore/varint.h> +#include <EASTL/vector.h> #include <functional> #include <memory> #include <string> #include <string_view> #include <type_traits> -#include <vector> #include <gsl/gsl-lite.hpp> @@ -1527,7 +1527,7 @@ ZENCORE_API void CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuild */ ZENCORE_API void CompactBinaryToYaml(MemoryView Data, StringBuilderBase& InBuilder); -ZENCORE_API std::vector<CbFieldView> ReadCompactBinaryStream(MemoryView Data); +ZENCORE_API eastl::vector<CbFieldView> ReadCompactBinaryStream(MemoryView Data); void uson_forcelink(); // internal void cbjson_forcelink(); // internal diff --git a/src/zencore/include/zencore/compactbinarybuilder.h b/src/zencore/include/zencore/compactbinarybuilder.h index f11717453..36a15de8d 100644 --- a/src/zencore/include/zencore/compactbinarybuilder.h +++ b/src/zencore/include/zencore/compactbinarybuilder.h @@ -11,12 +11,12 @@ #include <zencore/iobuffer.h> #include <zencore/iohash.h> +#include <EASTL/vector.h> #include <atomic> #include <memory> #include <string> #include <string_view> #include <type_traits> -#include <vector> #include <EASTL/fixed_vector.h> diff --git a/src/zencore/include/zencore/compositebuffer.h b/src/zencore/include/zencore/compositebuffer.h index 1e1611de9..153c7d9ce 100644 --- a/src/zencore/include/zencore/compositebuffer.h +++ b/src/zencore/include/zencore/compositebuffer.h @@ -6,9 +6,9 @@ #include <zencore/sharedbuffer.h> #include <zencore/zencore.h> +#include <EASTL/vector.h> #include <functional> #include <span> -#include <vector> #include <EASTL/fixed_vector.h> @@ -29,7 +29,7 @@ public: /** * Construct a composite buffer by concatenating the buffers. Does not enforce ownership. * - * Buffer parameters may be SharedBuffer, CompositeBuffer, or std::vector<SharedBuffer>. + * Buffer parameters may be SharedBuffer, CompositeBuffer, or eastl::vector<SharedBuffer>. */ template<typename... BufferTypes> inline explicit CompositeBuffer(BufferTypes&&... Buffers) @@ -152,9 +152,9 @@ private: } } - static inline size_t GetBufferCount(std::vector<SharedBuffer>&& Container) { return Container.size(); } - static inline size_t GetBufferCount(std::vector<IoBuffer>&& Container) { return Container.size(); } - inline void AppendBuffers(std::vector<SharedBuffer>&& Container) + static inline size_t GetBufferCount(eastl::vector<SharedBuffer>&& Container) { return Container.size(); } + static inline size_t GetBufferCount(eastl::vector<IoBuffer>&& Container) { return Container.size(); } + inline void AppendBuffers(eastl::vector<SharedBuffer>&& Container) { m_Segments.reserve(m_Segments.size() + Container.size()); for (SharedBuffer& Buffer : Container) @@ -162,7 +162,7 @@ private: m_Segments.emplace_back(std::move(Buffer)); } } - inline void AppendBuffers(std::vector<IoBuffer>&& Container) + inline void AppendBuffers(eastl::vector<IoBuffer>&& Container) { m_Segments.reserve(m_Segments.size() + Container.size()); for (IoBuffer& Buffer : Container) diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index e020668fc..155e07ca8 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -64,8 +64,8 @@ ZENCORE_API void MaximizeOpenFileCount(); struct FileContents { - std::vector<IoBuffer> Data; - std::error_code ErrorCode; + eastl::vector<IoBuffer> Data; + std::error_code ErrorCode; IoBuffer Flatten(); }; @@ -235,12 +235,12 @@ ENUM_CLASS_FLAGS(DirectoryContentFlags) struct DirectoryContent { - std::vector<std::filesystem::path> Files; - std::vector<uint64_t> FileSizes; - std::vector<uint32_t> FileAttributes; - std::vector<uint64_t> FileModificationTicks; - std::vector<std::filesystem::path> Directories; - std::vector<uint32_t> DirectoryAttributes; + eastl::vector<std::filesystem::path> Files; + eastl::vector<uint64_t> FileSizes; + eastl::vector<uint32_t> FileAttributes; + eastl::vector<uint64_t> FileModificationTicks; + eastl::vector<std::filesystem::path> Directories; + eastl::vector<uint32_t> DirectoryAttributes; }; void GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags Flags, DirectoryContent& OutContent); @@ -250,12 +250,12 @@ struct GetDirectoryContentVisitor public: struct DirectoryContent { - std::vector<std::filesystem::path> FileNames; - std::vector<uint64_t> FileSizes; - std::vector<uint32_t> FileAttributes; - std::vector<uint64_t> FileModificationTicks; - std::vector<std::filesystem::path> DirectoryNames; - std::vector<uint32_t> DirectoryAttributes; + eastl::vector<std::filesystem::path> FileNames; + eastl::vector<uint64_t> FileSizes; + eastl::vector<uint32_t> FileAttributes; + eastl::vector<uint64_t> FileModificationTicks; + eastl::vector<std::filesystem::path> DirectoryNames; + eastl::vector<uint32_t> DirectoryAttributes; }; virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) = 0; }; diff --git a/src/zencore/include/zencore/jobqueue.h b/src/zencore/include/zencore/jobqueue.h index d5ec6255a..02bb12605 100644 --- a/src/zencore/include/zencore/jobqueue.h +++ b/src/zencore/include/zencore/jobqueue.h @@ -2,13 +2,13 @@ #pragma once +#include <EASTL/vector.h> #include <chrono> #include <functional> #include <memory> #include <optional> #include <string> #include <string_view> -#include <vector> namespace zen { @@ -48,12 +48,12 @@ public: struct State { - std::string CurrentOp; - std::string CurrentOpDetails; - ptrdiff_t TotalCount; - ptrdiff_t RemainingCount; - std::vector<std::string> Messages; - std::string AbortReason; + std::string CurrentOp; + std::string CurrentOpDetails; + ptrdiff_t TotalCount; + ptrdiff_t RemainingCount; + eastl::vector<std::string> Messages; + std::string AbortReason; }; struct JobInfo @@ -62,7 +62,7 @@ public: Status Status; }; - virtual std::vector<JobInfo> GetJobs() = 0; + virtual eastl::vector<JobInfo> GetJobs() = 0; struct JobDetails { diff --git a/src/zencore/include/zencore/memoryview.h b/src/zencore/include/zencore/memoryview.h index 92f5aea3f..2d11450cc 100644 --- a/src/zencore/include/zencore/memoryview.h +++ b/src/zencore/include/zencore/memoryview.h @@ -7,10 +7,10 @@ #include <zencore/intmath.h> #include <zencore/thread.h> +#include <EASTL/vector.h> #include <cstddef> #include <cstring> #include <span> -#include <vector> namespace zen { diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index d1394cd9a..311059411 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -88,8 +88,8 @@ public: private: using HandleType = void*; - mutable RwLock m_Lock; - std::vector<HandleType> m_ProcessHandles; + mutable RwLock m_Lock; + eastl::vector<HandleType> m_ProcessHandles; }; ZENCORE_API bool IsProcessRunning(int pid); diff --git a/src/zencore/include/zencore/stats.h b/src/zencore/include/zencore/stats.h index f232cf2f4..bce4f3a09 100644 --- a/src/zencore/include/zencore/stats.h +++ b/src/zencore/include/zencore/stats.h @@ -6,9 +6,9 @@ #include <zenbase/concepts.h> +#include <EASTL/vector.h> #include <atomic> #include <string_view> -#include <vector> namespace zen { class CbObjectWriter; @@ -111,21 +111,21 @@ private: class SampleSnapshot { public: - SampleSnapshot(std::vector<double>&& Values); + SampleSnapshot(eastl::vector<double>&& Values); ~SampleSnapshot(); - uint32_t Size() const { return (uint32_t)m_Values.size(); } - double GetQuantileValue(double Quantile); - double GetMedian() { return GetQuantileValue(0.5); } - double Get75Percentile() { return GetQuantileValue(0.75); } - double Get95Percentile() { return GetQuantileValue(0.95); } - double Get98Percentile() { return GetQuantileValue(0.98); } - double Get99Percentile() { return GetQuantileValue(0.99); } - double Get999Percentile() { return GetQuantileValue(0.999); } - const std::vector<double>& GetValues() const; + uint32_t Size() const { return (uint32_t)m_Values.size(); } + double GetQuantileValue(double Quantile); + double GetMedian() { return GetQuantileValue(0.5); } + double Get75Percentile() { return GetQuantileValue(0.75); } + double Get95Percentile() { return GetQuantileValue(0.95); } + double Get98Percentile() { return GetQuantileValue(0.98); } + double Get99Percentile() { return GetQuantileValue(0.99); } + double Get999Percentile() { return GetQuantileValue(0.999); } + const eastl::vector<double>& GetValues() const; private: - std::vector<double> m_Values; + eastl::vector<double> m_Values; }; /** Randomly selects samples from a stream. Uses Vitter's @@ -155,8 +155,8 @@ public: } private: - std::atomic<uint64_t> m_SampleCounter{0}; - std::vector<std::atomic<int64_t>> m_Values; + std::atomic<uint64_t> m_SampleCounter{0}; + eastl::vector<std::atomic<int64_t>> m_Values; }; /** Track (probabilistic) sample distribution along with min/max diff --git a/src/zencore/include/zencore/stream.h b/src/zencore/include/zencore/stream.h index 77e763518..3b7da5bcf 100644 --- a/src/zencore/include/zencore/stream.h +++ b/src/zencore/include/zencore/stream.h @@ -7,7 +7,7 @@ #include <zencore/memoryview.h> #include <zencore/thread.h> -#include <vector> +#include <EASTL/vector.h> namespace zen { @@ -42,8 +42,8 @@ public: inline MutableMemoryView GetMutableView() { return MutableMemoryView(m_Buffer.data(), m_Offset); } private: - std::vector<uint8_t> m_Buffer; - uint64_t m_Offset = 0; + eastl::vector<uint8_t> m_Buffer; + uint64_t m_Offset = 0; void Write(const void* DataPtr, size_t ByteCount, uint64_t Offset); }; diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h index 8fb781571..75fa49b4b 100644 --- a/src/zencore/include/zencore/thread.h +++ b/src/zencore/include/zencore/thread.h @@ -4,11 +4,11 @@ #include "zencore.h" +#include <EASTL/vector.h> #include <atomic> #include <filesystem> #include <shared_mutex> #include <string_view> -#include <vector> #define ZEN_USE_WINDOWS_EVENTS ZEN_PLATFORM_WINDOWS diff --git a/src/zencore/jobqueue.cpp b/src/zencore/jobqueue.cpp index b97484458..bb113afa9 100644 --- a/src/zencore/jobqueue.cpp +++ b/src/zencore/jobqueue.cpp @@ -205,17 +205,17 @@ public: } } - virtual std::vector<JobInfo> GetJobs() override + virtual eastl::vector<JobInfo> GetJobs() override { - std::vector<JobId> DeadJobs; - auto IsStale = [](JobClock::Tick Time) { - ZEN_ASSERT_SLOW(Time != JobClock::Never()); - const std::chrono::system_clock::time_point Now = std::chrono::system_clock::now(); - std::chrono::system_clock::duration Age = Now - JobClock::TimePointFromTick(Time); - return std::chrono::duration_cast<std::chrono::days>(Age) > std::chrono::days(1); + eastl::vector<JobId> DeadJobs; + auto IsStale = [](JobClock::Tick Time) { + ZEN_ASSERT_SLOW(Time != JobClock::Never()); + const std::chrono::system_clock::time_point Now = std::chrono::system_clock::now(); + std::chrono::system_clock::duration Age = Now - JobClock::TimePointFromTick(Time); + return std::chrono::duration_cast<std::chrono::days>(Age) > std::chrono::days(1); }; - std::vector<JobInfo> Jobs; + eastl::vector<JobInfo> Jobs; QueueLock.WithSharedLock([&]() { for (auto It : RunningJobs) { @@ -497,10 +497,10 @@ TEST_CASE("JobQueue") JobsLatch.CountDown(); while (true) { - bool PendingQueue = JobsLatch.Remaining() > 0; - size_t PendingCount = 0; - std::vector<JobId> RemainingJobs; - std::vector<JobQueue::JobInfo> Statuses = Queue->GetJobs(); + bool PendingQueue = JobsLatch.Remaining() > 0; + size_t PendingCount = 0; + eastl::vector<JobId> RemainingJobs; + eastl::vector<JobQueue::JobInfo> Statuses = Queue->GetJobs(); RemainingJobs.reserve(Statuses.size()); for (const auto& It : Statuses) { diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index c51e8f69d..eeeb2ffec 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -2,6 +2,7 @@ #include <zencore/process.h> +#include <zencore/eastlutil.h> #include <zencore/except.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> @@ -386,7 +387,7 @@ ProcessHandle::WaitExitCode() #if !ZEN_PLATFORM_WINDOWS || ZEN_WITH_TESTS static void -BuildArgV(std::vector<char*>& Out, char* CommandLine) +BuildArgV(eastl::vector<char*>& Out, char* CommandLine) { char* Cursor = CommandLine; while (true) @@ -664,8 +665,8 @@ CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine return CreateProcNormal(Executable, CommandLine, Options); #else - std::vector<char*> ArgV; - std::string CommandLineZ(CommandLine); + eastl::vector<char*> ArgV; + std::string CommandLineZ(CommandLine); BuildArgV(ArgV, CommandLineZ.data()); ArgV.push_back(nullptr); @@ -752,7 +753,7 @@ ProcessMonitor::IsRunning() FoundOne |= ProcIsActive; } - std::erase_if(m_ProcessHandles, [](HandleType Handle) { return Handle == 0; }); + zen::erase_if(m_ProcessHandles, [](HandleType Handle) { return Handle == 0; }); return FoundOne; } @@ -1010,8 +1011,8 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand return MakeErrorCodeFromLastError(); #endif // ZEN_PLATFORM_MAC #if ZEN_PLATFORM_LINUX - std::vector<uint32_t> RunningPids; - DirectoryContent ProcList; + eastl::vector<uint32_t> RunningPids; + DirectoryContent ProcList; GetDirectoryContent("/proc", DirectoryContentFlags::IncludeDirs, ProcList); for (const std::filesystem::path& EntryPath : ProcList.Directories) { @@ -1097,8 +1098,8 @@ TEST_CASE("BuildArgV") for (const auto& Case : Cases) { - std::vector<char*> OutArgs; - StringBuilder<64> Mutable; + eastl::vector<char*> OutArgs; + StringBuilder<64> Mutable; Mutable << Case.Input; BuildArgV(OutArgs, Mutable.Data()); diff --git a/src/zencore/stats.cpp b/src/zencore/stats.cpp index 8a424c5ad..05e55d2ca 100644 --- a/src/zencore/stats.cpp +++ b/src/zencore/stats.cpp @@ -227,7 +227,7 @@ thread_local xoshiro256 ThreadLocalRng; UniformSample::UniformSample(uint32_t ReservoirSize) { ZEN_MEMSCOPE(ELLMTag::Metrics); - m_Values = std::vector<std::atomic<int64_t>>(ReservoirSize); + m_Values = eastl::vector<std::atomic<int64_t>>(ReservoirSize); } UniformSample::~UniformSample() @@ -279,8 +279,8 @@ UniformSample::Snapshot() const { ZEN_MEMSCOPE(ELLMTag::Metrics); - uint64_t ValuesSize = Size(); - std::vector<double> Values(ValuesSize); + uint64_t ValuesSize = Size(); + eastl::vector<double> Values(ValuesSize); for (int i = 0, n = int(ValuesSize); i < n; ++i) { @@ -368,7 +368,7 @@ Histogram::Count() const ////////////////////////////////////////////////////////////////////////// -SampleSnapshot::SampleSnapshot(std::vector<double>&& Values) : m_Values(std::move(Values)) +SampleSnapshot::SampleSnapshot(eastl::vector<double>&& Values) : m_Values(std::move(Values)) { std::sort(begin(m_Values), end(m_Values)); } @@ -407,7 +407,7 @@ SampleSnapshot::GetQuantileValue(double Quantile) return Lower + (Pos - std::floor(Pos)) * (Upper - Lower); } -const std::vector<double>& +const eastl::vector<double>& SampleSnapshot::GetValues() const { return m_Values; diff --git a/src/zencore/uid.cpp b/src/zencore/uid.cpp index d7636f2ad..cd78263bf 100644 --- a/src/zencore/uid.cpp +++ b/src/zencore/uid.cpp @@ -163,7 +163,7 @@ TEST_CASE("Oid") Oid id1 = Oid::NewOid(); ZEN_UNUSED(id1); - std::vector<Oid> ids; + eastl::vector<Oid> ids; std::set<Oid> idset; std::unordered_map<Oid, int, Oid::Hasher> idmap; diff --git a/src/zencore/workthreadpool.cpp b/src/zencore/workthreadpool.cpp index d15fb2e83..9231269c5 100644 --- a/src/zencore/workthreadpool.cpp +++ b/src/zencore/workthreadpool.cpp @@ -10,8 +10,8 @@ #include <zencore/thread.h> #include <zencore/trace.h> +#include <EASTL/vector.h> #include <thread> -#include <vector> #define ZEN_USE_WINDOWS_THREADPOOL 1 @@ -141,10 +141,10 @@ struct WorkerThreadPool::ThreadStartInfo struct WorkerThreadPool::Impl { - void WorkerThreadFunction(ThreadStartInfo Info); - std::string m_WorkerThreadBaseName; - std::vector<std::thread> m_WorkerThreads; - BlockingQueue<Ref<IWork>> m_WorkQueue; + void WorkerThreadFunction(ThreadStartInfo Info); + std::string m_WorkerThreadBaseName; + eastl::vector<std::thread> m_WorkerThreads; + BlockingQueue<Ref<IWork>> m_WorkQueue; Impl(int InThreadCount, std::string_view WorkerThreadBaseName) : m_WorkerThreadBaseName(WorkerThreadBaseName) { diff --git a/src/zenhttp/auth/authmgr.cpp b/src/zenhttp/auth/authmgr.cpp index 1a9892d5c..5ba64b1ee 100644 --- a/src/zenhttp/auth/authmgr.cpp +++ b/src/zenhttp/auth/authmgr.cpp @@ -43,7 +43,7 @@ namespace details { return IoBuffer(); } - std::vector<uint8_t> DecryptionBuffer; + eastl::vector<uint8_t> DecryptionBuffer; DecryptionBuffer.resize(EncryptedBuffer.GetSize() + Aes::BlockSize); MemoryView DecryptedView = Aes::Decrypt(Key, IV, EncryptedBuffer, MakeMutableMemoryView(DecryptionBuffer), Reason); @@ -67,7 +67,7 @@ namespace details { return; } - std::vector<uint8_t> EncryptionBuffer; + eastl::vector<uint8_t> EncryptionBuffer; EncryptionBuffer.resize(FileData.GetSize() + Aes::BlockSize); MemoryView EncryptedView = Aes::Encrypt(Key, IV, FileData, MakeMutableMemoryView(EncryptionBuffer), Reason); @@ -421,7 +421,7 @@ private: { // Refresh Open ID token(s) - std::vector<OpenIdTokenMap::value_type> ExpiredTokens; + eastl::vector<OpenIdTokenMap::value_type> ExpiredTokens; { std::unique_lock _(m_TokenMutex); diff --git a/src/zenhttp/auth/oidc.cpp b/src/zenhttp/auth/oidc.cpp index 318110c7d..5bd7c2990 100644 --- a/src/zenhttp/auth/oidc.cpp +++ b/src/zenhttp/auth/oidc.cpp @@ -12,7 +12,7 @@ namespace zen { namespace details { - using StringArray = std::vector<std::string>; + using StringArray = eastl::vector<std::string>; StringArray ToStringArray(const json11::Json JsonArray) { diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index e4c6d243d..1a5955609 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -607,9 +607,9 @@ struct HttpClient::Impl : public RefCounted LoggerRef Logger() { return m_Log; } private: - LoggerRef m_Log; - RwLock m_SessionLock; - std::vector<cpr::Session*> m_Sessions; + LoggerRef m_Log; + RwLock m_SessionLock; + eastl::vector<cpr::Session*> m_Sessions; void ReleaseSession(cpr::Session*); }; @@ -758,7 +758,7 @@ HttpClient::TransactPackage(std::string_view Url, CbPackage Package, const KeyVa // First, list of offered chunks for filtering on the server end - std::vector<IoHash> AttachmentsToSend; + eastl::vector<IoHash> AttachmentsToSend; std::span<const CbAttachment> Attachments = Package.GetAttachments(); const uint32_t RequestId = ++HttpClientRequestIdCounter; @@ -1150,37 +1150,37 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold cpr::Response Response; { - std::vector<std::pair<std::string, std::string>> ReceivedHeaders; - auto HeaderCallback = [&](std::string header, intptr_t) { - std::pair<std::string, std::string> Header = GetHeader(header); - if (Header.first == "Content-Length"sv) - { - std::optional<size_t> ContentSize = ParseInt<size_t>(Header.second); - if (ContentSize.has_value()) - { - if (ContentSize.value() > 1024 * 1024) - { - PayloadFile = std::make_unique<detail::TempPayloadFile>(); - std::error_code Ec = PayloadFile->Open(TempFolderPath); - if (Ec) - { - ZEN_WARN("Failed to create temp file in '{}' for HttpClient::Download. Reason: {}", - TempFolderPath.string(), - Ec.message()); - PayloadFile.reset(); - } - } - else - { - PayloadString.reserve(ContentSize.value()); - } - } - } - if (!Header.first.empty()) - { - ReceivedHeaders.emplace_back(std::move(Header)); - } - return 1; + eastl::vector<std::pair<std::string, std::string>> ReceivedHeaders; + auto HeaderCallback = [&](std::string header, intptr_t) { + std::pair<std::string, std::string> Header = GetHeader(header); + if (Header.first == "Content-Length"sv) + { + std::optional<size_t> ContentSize = ParseInt<size_t>(Header.second); + if (ContentSize.has_value()) + { + if (ContentSize.value() > 1024 * 1024) + { + PayloadFile = std::make_unique<detail::TempPayloadFile>(); + std::error_code Ec = PayloadFile->Open(TempFolderPath); + if (Ec) + { + ZEN_WARN("Failed to create temp file in '{}' for HttpClient::Download. Reason: {}", + TempFolderPath.string(), + Ec.message()); + PayloadFile.reset(); + } + } + else + { + PayloadString.reserve(ContentSize.value()); + } + } + } + if (!Header.first.empty()) + { + ReceivedHeaders.emplace_back(std::move(Header)); + } + return 1; }; Impl::Session Sess = @@ -1221,7 +1221,7 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold std::optional<int64_t> ContentLength = ParseInt<int64_t>(It->second); if (ContentLength) { - std::vector<std::pair<std::string, std::string>> ReceivedHeaders; + eastl::vector<std::pair<std::string, std::string>> ReceivedHeaders; auto HeaderCallback = [&](std::string header, intptr_t) { std::pair<std::string, std::string> Header = GetHeader(header); diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 27a09f339..26c1f3025 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -467,7 +467,7 @@ HttpServerRequest::~HttpServerRequest() void HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, CbPackage Data) { - std::vector<IoBuffer> ResponseBuffers = FormatPackageMessage(Data); + eastl::vector<IoBuffer> ResponseBuffers = FormatPackageMessage(Data); return WriteResponse(ResponseCode, HttpContentType::kCbPackage, ResponseBuffers); } @@ -922,7 +922,7 @@ HandlePackageOffers(HttpService& Service, HttpServerRequest& Request, Ref<IHttpP { CbObject OfferMessage = LoadCompactBinaryObject(Request.ReadPayload()); - std::vector<IoHash> OfferCids; + eastl::vector<IoHash> OfferCids; for (auto& CidEntry : OfferMessage["offer"]) { diff --git a/src/zenhttp/include/zenhttp/auth/oidc.h b/src/zenhttp/include/zenhttp/auth/oidc.h index 6f9c3198e..7ca6b01c0 100644 --- a/src/zenhttp/include/zenhttp/auth/oidc.h +++ b/src/zenhttp/include/zenhttp/auth/oidc.h @@ -5,7 +5,7 @@ #include <zenbase/refcount.h> #include <zencore/string.h> -#include <vector> +#include <EASTL/vector.h> namespace zen { @@ -49,7 +49,7 @@ public: RefreshTokenResult RefreshToken(std::string_view RefreshToken); private: - using StringArray = std::vector<std::string>; + using StringArray = eastl::vector<std::string>; struct OpenIdConfiguration { diff --git a/src/zenhttp/include/zenhttp/httpcommon.h b/src/zenhttp/include/zenhttp/httpcommon.h index bc18549c9..ec7fbba88 100644 --- a/src/zenhttp/include/zenhttp/httpcommon.h +++ b/src/zenhttp/include/zenhttp/httpcommon.h @@ -23,7 +23,7 @@ struct HttpRange uint32_t End = ~uint32_t(0); }; -using HttpRanges = std::vector<HttpRange>; +using HttpRanges = eastl::vector<HttpRange>; std::string_view MapContentTypeToString(HttpContentType ContentType); extern HttpContentType (*ParseContentType)(const std::string_view& ContentTypeString); diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 217455dba..6cff5c69c 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -41,7 +41,7 @@ public: struct QueryParams { - std::vector<std::pair<std::string_view, std::string_view>> KvPairs; + eastl::vector<std::pair<std::string_view, std::string_view>> KvPairs; std::string_view GetValue(std::string_view ParamName) const { @@ -137,7 +137,7 @@ protected: class IHttpPackageHandler : public RefCounted { public: - virtual void FilterOffer(std::vector<IoHash>& OfferCids) = 0; + virtual void FilterOffer(eastl::vector<IoHash>& OfferCids) = 0; virtual void OnRequestBegin() = 0; virtual IoBuffer CreateTarget(const IoHash& Cid, uint64_t StorageSize) = 0; virtual void OnRequestComplete() = 0; diff --git a/src/zenhttp/include/zenhttp/httptest.h b/src/zenhttp/include/zenhttp/httptest.h index afe71fbce..8dfa43851 100644 --- a/src/zenhttp/include/zenhttp/httptest.h +++ b/src/zenhttp/include/zenhttp/httptest.h @@ -32,7 +32,7 @@ public: PackageHandler(HttpTestingService& Svc, uint32_t RequestId); ~PackageHandler(); - virtual void FilterOffer(std::vector<IoHash>& OfferCids) override; + virtual void FilterOffer(eastl::vector<IoHash>& OfferCids) override; virtual void OnRequestBegin() override; virtual IoBuffer CreateTarget(const IoHash& Cid, uint64_t StorageSize) override; virtual void OnRequestComplete() override; diff --git a/src/zenhttp/include/zenhttp/packageformat.h b/src/zenhttp/include/zenhttp/packageformat.h index c90b840da..f5bd53c5b 100644 --- a/src/zenhttp/include/zenhttp/packageformat.h +++ b/src/zenhttp/include/zenhttp/packageformat.h @@ -95,19 +95,19 @@ enum class RpcAcceptOptions : uint16_t gsl_DEFINE_ENUM_BITMASK_OPERATORS(RpcAcceptOptions); -std::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle = nullptr); -CompositeBuffer FormatPackageMessageBuffer(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle = nullptr); -CbPackage ParsePackageMessage( - IoBuffer Payload, - std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> CreateBuffer = [](const IoHash&, uint64_t Size) -> IoBuffer { - return IoBuffer{Size}; - }); +eastl::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle = nullptr); +CompositeBuffer FormatPackageMessageBuffer(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle = nullptr); +CbPackage ParsePackageMessage( + IoBuffer Payload, + std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> CreateBuffer = [](const IoHash&, uint64_t Size) -> IoBuffer { + return IoBuffer{Size}; + }); bool IsPackageMessage(IoBuffer Payload); bool ParsePackageMessageWithLegacyFallback(const IoBuffer& Response, CbPackage& OutPackage); -std::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data, void* TargetProcessHandle = nullptr); -CompositeBuffer FormatPackageMessageBuffer(const CbPackage& Data, void* TargetProcessHandle = nullptr); +eastl::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data, void* TargetProcessHandle = nullptr); +CompositeBuffer FormatPackageMessageBuffer(const CbPackage& Data, void* TargetProcessHandle = nullptr); /** Streaming reader for compact binary packages @@ -135,10 +135,10 @@ public: */ uint64_t ProcessPackageHeaderData(const void* Data, uint64_t DataBytes); - void Finalize(); - const std::vector<CbAttachment>& GetAttachments() { return m_Attachments; } - CbObject GetRootObject() { return m_RootObject; } - std::span<IoBuffer> GetPayloadBuffers() { return m_PayloadBuffers; } + void Finalize(); + const eastl::vector<CbAttachment>& GetAttachments() { return m_Attachments; } + CbObject GetRootObject() { return m_RootObject; } + std::span<IoBuffer> GetPayloadBuffers() { return m_PayloadBuffers; } private: enum class State @@ -150,9 +150,9 @@ private: } m_CurrentState = State::kInitialState; std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> m_CreateBuffer; - std::vector<IoBuffer> m_PayloadBuffers; - std::vector<CbAttachmentEntry> m_AttachmentEntries; - std::vector<CbAttachment> m_Attachments; + eastl::vector<IoBuffer> m_PayloadBuffers; + eastl::vector<CbAttachmentEntry> m_AttachmentEntries; + eastl::vector<CbAttachment> m_Attachments; CbObject m_RootObject; CbPackageHeader m_PackageHeader; diff --git a/src/zenhttp/packageformat.cpp b/src/zenhttp/packageformat.cpp index ae80851e4..b772ba2cf 100644 --- a/src/zenhttp/packageformat.cpp +++ b/src/zenhttp/packageformat.cpp @@ -16,8 +16,8 @@ #include <zencore/testutils.h> #include <zencore/trace.h> +#include <EASTL/vector.h> #include <span> -#include <vector> #include <EASTL/fixed_vector.h> @@ -37,7 +37,7 @@ typedef eastl::fixed_vector<IoBuffer, 16> IoBufferVec_t; IoBufferVec_t FormatPackageMessageInternal(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle); -std::vector<IoBuffer> +eastl::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data, void* TargetProcessHandle) { return FormatPackageMessage(Data, FormatFlags::kDefault, TargetProcessHandle); @@ -48,11 +48,11 @@ FormatPackageMessageBuffer(const CbPackage& Data, void* TargetProcessHandle) return FormatPackageMessageBuffer(Data, FormatFlags::kDefault, TargetProcessHandle); } -std::vector<IoBuffer> +eastl::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle) { auto Vec = FormatPackageMessageInternal(Data, Flags, TargetProcessHandle); - return std::vector<IoBuffer>(begin(Vec), end(Vec)); + return eastl::vector<IoBuffer>(begin(Vec), end(Vec)); } CompositeBuffer @@ -85,7 +85,7 @@ MarshalLocal(CbAttachmentEntry*& AttachmentInfo, static bool IsLocalRef(tsl::robin_map<void*, std::string>& FileNameMap, - std::vector<void*>& DuplicatedHandles, + eastl::vector<void*>& DuplicatedHandles, const CompositeBuffer& AttachmentBinary, bool DenyPartialLocalReferences, void* TargetProcessHandle, @@ -165,7 +165,7 @@ FormatPackageMessageInternal(const CbPackage& Data, FormatFlags Flags, void* Tar { ZEN_TRACE_CPU("FormatPackageMessage"); - std::vector<void*> DuplicatedHandles; + eastl::vector<void*> DuplicatedHandles; #if ZEN_PLATFORM_WINDOWS auto _ = MakeGuard([&DuplicatedHandles, &TargetProcessHandle]() { if (TargetProcessHandle == nullptr) @@ -394,12 +394,12 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint CbPackage Package; - std::vector<CbAttachment> Attachments; + eastl::vector<CbAttachment> Attachments; Attachments.reserve(ChunkCount); // Guessing here... tsl::robin_map<std::string, IoBuffer> PartialFileBuffers; - std::vector<std::pair<uint32_t, std::string>> MalformedAttachments; + eastl::vector<std::pair<uint32_t, std::string>> MalformedAttachments; for (uint32_t i = 0; i < ChunkCount; ++i) { @@ -830,7 +830,7 @@ TEST_CASE("CbPackage.EmptyObject") { CbPackage Pkg; Pkg.SetObject({}); - std::vector<IoBuffer> Result = FormatPackageMessage(Pkg, nullptr); + eastl::vector<IoBuffer> Result = FormatPackageMessage(Pkg, nullptr); } TEST_CASE("CbPackage.LocalRef") diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index fe59e3a6f..987d1ab64 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -12,10 +12,10 @@ #include "httpparser.h" +#include <EASTL/vector.h> #include <deque> #include <memory> #include <string_view> -#include <vector> ZEN_THIRD_PARTY_INCLUDES_START #if ZEN_PLATFORM_WINDOWS @@ -85,7 +85,7 @@ public: asio::io_service m_IoService; asio::io_service::work m_Work{m_IoService}; std::unique_ptr<asio_http::HttpAcceptor> m_Acceptor; - std::vector<std::thread> m_ThreadPool; + eastl::vector<std::thread> m_ThreadPool; LoggerRef m_RequestLog; HttpServerTracer m_RequestTracer; @@ -96,8 +96,8 @@ public: HttpService* Service; }; - RwLock m_Lock; - std::vector<ServiceEntry> m_UriHandlers; + RwLock m_Lock; + eastl::vector<ServiceEntry> m_UriHandlers; }; /** @@ -195,7 +195,7 @@ public: uint16_t ResponseCode() const { return m_ResponseCode; } uint64_t ContentLength() const { return m_ContentLength; } - const std::vector<asio::const_buffer>& AsioBuffers() const { return m_AsioBuffers; } + const eastl::vector<asio::const_buffer>& AsioBuffers() const { return m_AsioBuffers; } std::string_view GetHeaders() { @@ -218,13 +218,13 @@ public: void SuppressPayload() { m_AsioBuffers.resize(1); } private: - uint16_t m_ResponseCode = 0; - bool m_IsKeepAlive = true; - HttpContentType m_ContentType = HttpContentType::kBinary; - uint64_t m_ContentLength = 0; - std::vector<IoBuffer> m_DataBuffers; - std::vector<asio::const_buffer> m_AsioBuffers; - ExtendableStringBuilder<160> m_Headers; + uint16_t m_ResponseCode = 0; + bool m_IsKeepAlive = true; + HttpContentType m_ContentType = HttpContentType::kBinary; + uint64_t m_ContentLength = 0; + eastl::vector<IoBuffer> m_DataBuffers; + eastl::vector<asio::const_buffer> m_AsioBuffers; + ExtendableStringBuilder<160> m_Headers; }; ////////////////////////////////////////////////////////////////////////// @@ -494,7 +494,7 @@ HttpServerConnection::HandleRequest() ToString(Request.AcceptContentType())); m_Server.m_RequestTracer.WriteDebugPayload(fmt::format("request_{}_{}.bin", m_ConnectionId, RequestNumber), - std::vector<IoBuffer>{Request.ReadPayload()}); + eastl::vector<IoBuffer>{Request.ReadPayload()}); } if (!HandlePackageOffers(*Service, Request, m_PackageHandler)) @@ -554,7 +554,7 @@ HttpServerConnection::HandleRequest() Response->SuppressPayload(); } - const std::vector<asio::const_buffer>& ResponseBuffers = Response->AsioBuffers(); + const eastl::vector<asio::const_buffer>& ResponseBuffers = Response->AsioBuffers(); uint64_t ResponseLength = 0; diff --git a/src/zenhttp/servers/httpmulti.h b/src/zenhttp/servers/httpmulti.h index 53cf57568..0494598ce 100644 --- a/src/zenhttp/servers/httpmulti.h +++ b/src/zenhttp/servers/httpmulti.h @@ -24,9 +24,9 @@ public: void AddServer(Ref<HttpServer> Server); private: - bool m_IsInitialized = false; - Event m_ShutdownEvent; - std::vector<Ref<HttpServer>> m_Servers; + bool m_IsInitialized = false; + Event m_ShutdownEvent; + eastl::vector<Ref<HttpServer>> m_Servers; }; } // namespace zen diff --git a/src/zenhttp/servers/httpparser.h b/src/zenhttp/servers/httpparser.h index bdbcab4d9..918a5bb10 100644 --- a/src/zenhttp/servers/httpparser.h +++ b/src/zenhttp/servers/httpparser.h @@ -81,7 +81,7 @@ private: size_t m_CurrentHeaderNameLength = 0; char* m_CurrentHeaderValue = nullptr; // Used while parsing headers size_t m_CurrentHeaderValueLength = 0; - std::vector<HeaderEntry> m_Headers; + eastl::vector<HeaderEntry> m_Headers; int8_t m_ContentLengthHeaderIndex; int8_t m_AcceptHeaderIndex; int8_t m_ContentTypeHeaderIndex; diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp index ec12b3755..d06e5a001 100644 --- a/src/zenhttp/servers/httpplugin.cpp +++ b/src/zenhttp/servers/httpplugin.cpp @@ -112,14 +112,14 @@ struct HttpPluginServerImpl : public HttpPluginServer, TransportServer HttpService* Service; }; - bool m_IsInitialized = false; - RwLock m_Lock; - std::vector<ServiceEntry> m_UriHandlers; - std::vector<Ref<TransportPlugin>> m_Plugins; - Event m_ShutdownEvent; - bool m_IsRequestLoggingEnabled = false; - LoggerRef m_RequestLog; - std::atomic_uint32_t m_ConnectionIdCounter{0}; + bool m_IsInitialized = false; + RwLock m_Lock; + eastl::vector<ServiceEntry> m_UriHandlers; + eastl::vector<Ref<TransportPlugin>> m_Plugins; + Event m_ShutdownEvent; + bool m_IsRequestLoggingEnabled = false; + LoggerRef m_RequestLog; + std::atomic_uint32_t m_ConnectionIdCounter{0}; HttpServerTracer m_RequestTracer; @@ -175,8 +175,8 @@ public: inline uint64_t ContentLength() const { return m_ContentLength; } inline HttpContentType ContentType() const { return m_ContentType; } - const std::vector<IoBuffer>& ResponseBuffers() const { return m_ResponseBuffers; } - void SuppressPayload() { m_ResponseBuffers.resize(1); } + const eastl::vector<IoBuffer>& ResponseBuffers() const { return m_ResponseBuffers; } + void SuppressPayload() { m_ResponseBuffers.resize(1); } std::string_view GetHeaders(); @@ -185,7 +185,7 @@ private: bool m_IsKeepAlive = true; HttpContentType m_ContentType = HttpContentType::kBinary; uint64_t m_ContentLength = 0; - std::vector<IoBuffer> m_ResponseBuffers; + eastl::vector<IoBuffer> m_ResponseBuffers; ExtendableStringBuilder<160> m_Headers; }; @@ -387,7 +387,7 @@ HttpPluginConnectionHandler::HandleRequest() ToString(Request.AcceptContentType())); m_Server->m_RequestTracer.WriteDebugPayload(fmt::format("request_{}_{}.bin", m_ConnectionId, RequestNumber), - std::vector<IoBuffer>{Request.ReadPayload()}); + eastl::vector<IoBuffer>{Request.ReadPayload()}); } if (!HandlePackageOffers(*Service, Request, m_PackageHandler)) @@ -450,7 +450,7 @@ HttpPluginConnectionHandler::HandleRequest() Response->SuppressPayload(); } - const std::vector<IoBuffer>& ResponseBuffers = Response->ResponseBuffers(); + const eastl::vector<IoBuffer>& ResponseBuffers = Response->ResponseBuffers(); if (m_Server->m_RequestLog.ShouldLog(logging::level::Trace)) { diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index 3bdcdf098..e35f27d9a 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -91,16 +91,16 @@ private: RwLock m_AsyncWorkPoolInitLock; WorkerThreadPool* m_AsyncWorkPool = nullptr; - std::vector<std::wstring> m_BaseUris; // eg: http://*:nnnn/ - HTTP_SERVER_SESSION_ID m_HttpSessionId = 0; - HTTP_URL_GROUP_ID m_HttpUrlGroupId = 0; - HANDLE m_RequestQueueHandle = 0; - std::atomic_int32_t m_PendingRequests{0}; - std::atomic_int32_t m_IsShuttingDown{0}; - int32_t m_MinPendingRequests = 16; - int32_t m_MaxPendingRequests = 128; - Event m_ShutdownEvent; - HttpSysConfig m_InitialConfig; + eastl::vector<std::wstring> m_BaseUris; // eg: http://*:nnnn/ + HTTP_SERVER_SESSION_ID m_HttpSessionId = 0; + HTTP_URL_GROUP_ID m_HttpUrlGroupId = 0; + HANDLE m_RequestQueueHandle = 0; + std::atomic_int32_t m_PendingRequests{0}; + std::atomic_int32_t m_IsShuttingDown{0}; + int32_t m_MinPendingRequests = 16; + int32_t m_MaxPendingRequests = 128; + Event m_ShutdownEvent; + HttpSysConfig m_InitialConfig; }; } // namespace zen diff --git a/src/zenhttp/servers/httptracer.cpp b/src/zenhttp/servers/httptracer.cpp index 483307fb1..6a243c36b 100644 --- a/src/zenhttp/servers/httptracer.cpp +++ b/src/zenhttp/servers/httptracer.cpp @@ -20,8 +20,8 @@ HttpServerTracer::Initialize(std::filesystem::path DataDir) void HttpServerTracer::WriteDebugPayload(std::string_view Filename, const std::span<const IoBuffer> Payload) { - uint64_t PayloadSize = 0; - std::vector<const IoBuffer*> Buffers; + uint64_t PayloadSize = 0; + eastl::vector<const IoBuffer*> Buffers; for (auto& Io : Payload) { Buffers.push_back(&Io); diff --git a/src/zenhttp/testing/httptest.cpp b/src/zenhttp/testing/httptest.cpp index 090f7855e..130f6bac2 100644 --- a/src/zenhttp/testing/httptest.cpp +++ b/src/zenhttp/testing/httptest.cpp @@ -152,7 +152,7 @@ HttpTestingService::PackageHandler::~PackageHandler() } void -HttpTestingService::PackageHandler::FilterOffer(std::vector<IoHash>& OfferCids) +HttpTestingService::PackageHandler::FilterOffer(eastl::vector<IoHash>& OfferCids) { ZEN_UNUSED(OfferCids); // No-op diff --git a/src/zenhttp/transports/asiotransport.cpp b/src/zenhttp/transports/asiotransport.cpp index 96a15518c..fc7160a9f 100644 --- a/src/zenhttp/transports/asiotransport.cpp +++ b/src/zenhttp/transports/asiotransport.cpp @@ -50,7 +50,7 @@ private: asio::io_service m_IoService; asio::io_service::work m_Work{m_IoService}; std::unique_ptr<AsioTransportAcceptor> m_Acceptor; - std::vector<std::thread> m_ThreadPool; + eastl::vector<std::thread> m_ThreadPool; }; struct AsioTransportConnection : public TransportConnection, std::enable_shared_from_this<AsioTransportConnection> diff --git a/src/zenhttp/transports/dlltransport.cpp b/src/zenhttp/transports/dlltransport.cpp index e09e62ec5..4ec5735d9 100644 --- a/src/zenhttp/transports/dlltransport.cpp +++ b/src/zenhttp/transports/dlltransport.cpp @@ -7,9 +7,9 @@ #include <zencore/logging.h> #include <zencore/scopeguard.h> +#include <EASTL/vector.h> #include <exception> #include <thread> -#include <vector> ZEN_THIRD_PARTY_INCLUDES_START #include <fmt/format.h> @@ -46,9 +46,9 @@ public: virtual void ConfigureDll(std::string_view Name, const char* OptionTag, const char* OptionValue) override; private: - TransportServer* m_ServerInterface = nullptr; - RwLock m_Lock; - std::vector<LoadedDll> m_Transports; + TransportServer* m_ServerInterface = nullptr; + RwLock m_Lock; + eastl::vector<LoadedDll> m_Transports; }; DllTransportPluginImpl::DllTransportPluginImpl() diff --git a/src/zennet/statsdclient.cpp b/src/zennet/statsdclient.cpp index fe5ca4dda..80a5b2f98 100644 --- a/src/zennet/statsdclient.cpp +++ b/src/zennet/statsdclient.cpp @@ -7,7 +7,7 @@ #include <zencore/testing.h> #include <zencore/thread.h> -#include <deque> +#include <EASTL/deque.h> ZEN_THIRD_PARTY_INCLUDES_START #include <zencore/windows.h> @@ -95,7 +95,7 @@ StatsdUdpClient::Shutdown() class StatsdMemoryClient : public StatsTransportBase { - using MessageBuffer_t = std::vector<uint8_t>; + using MessageBuffer_t = eastl::vector<uint8_t>; public: StatsdMemoryClient() {} @@ -107,15 +107,15 @@ public: m_Messages.emplace_back(reinterpret_cast<const uint8_t*>(Data), reinterpret_cast<const uint8_t*>(Data) + Size); } - std::deque<MessageBuffer_t> GetMessages() + eastl::deque<MessageBuffer_t> GetMessages() { RwLock::ExclusiveLockScope _(m_Lock); return std::move(m_Messages); } private: - RwLock m_Lock; - std::deque<MessageBuffer_t> m_Messages; + RwLock m_Lock; + eastl::deque<MessageBuffer_t> m_Messages; }; ////////////////////////////////////////////////////////////////////////// @@ -210,16 +210,16 @@ public: } private: - StatsTransportBase& m_Transport; - std::thread m_MessagingThread; - Event m_ShutdownRequested; - std::atomic_flag m_AcceptMessages; - RwLock m_MessageLock; - std::vector<uint8_t> m_MessageBuffer; - size_t m_MessageSize = 0; - bool m_UseBackgroundThread = true; + StatsTransportBase& m_Transport; + std::thread m_MessagingThread; + Event m_ShutdownRequested; + std::atomic_flag m_AcceptMessages; + RwLock m_MessageLock; + eastl::vector<uint8_t> m_MessageBuffer; + size_t m_MessageSize = 0; + bool m_UseBackgroundThread = true; - std::deque<std::vector<uint8_t>> m_MessageQueue; + eastl::deque<eastl::vector<uint8_t>> m_MessageQueue; void EnqueueOrSendCurrentMessage(RwLock::ExclusiveLockScope&) { @@ -253,7 +253,7 @@ StatsMessageBuilder::FlushMessageBuffer() void StatsMessageBuilder::FlushQueue() { - std::deque<std::vector<uint8_t>> Queue; + eastl::deque<eastl::vector<uint8_t>> Queue; { RwLock::ExclusiveLockScope _(m_MessageLock); @@ -263,7 +263,7 @@ StatsMessageBuilder::FlushQueue() while (!Queue.empty()) { - std::vector<uint8_t>& Message = Queue.front(); + eastl::vector<uint8_t>& Message = Queue.front(); m_Transport.SendMessage(Message.data(), Message.size()); diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp index 6259c0f37..0dbe2cb4b 100644 --- a/src/zenserver-test/zenserver-test.cpp +++ b/src/zenserver-test/zenserver-test.cpp @@ -585,11 +585,11 @@ namespace utils { { // Convoluted way to get a compressed buffer whose result it large enough to be a separate file // but also does actually compress - const size_t PartCount = (AttachmentSize / (1u * 1024u * 64)) + 1; - const size_t PartSize = AttachmentSize / PartCount; - auto Part = SharedBuffer(CreateRandomBlob(PartSize)); - std::vector<SharedBuffer> Parts(PartCount, Part); - size_t RemainPartSize = AttachmentSize - (PartSize * PartCount); + const size_t PartCount = (AttachmentSize / (1u * 1024u * 64)) + 1; + const size_t PartSize = AttachmentSize / PartCount; + auto Part = SharedBuffer(CreateRandomBlob(PartSize)); + eastl::vector<SharedBuffer> Parts(PartCount, Part); + size_t RemainPartSize = AttachmentSize - (PartSize * PartCount); if (RemainPartSize > 0) { Parts.push_back(SharedBuffer(CreateRandomBlob(RemainPartSize))); @@ -598,9 +598,9 @@ namespace utils { return Value; }; - std::vector<std::pair<Oid, CompressedBuffer>> CreateAttachments(const std::span<const size_t>& Sizes) + eastl::vector<std::pair<Oid, CompressedBuffer>> CreateAttachments(const std::span<const size_t>& Sizes) { - std::vector<std::pair<Oid, CompressedBuffer>> Result; + eastl::vector<std::pair<Oid, CompressedBuffer>> Result; Result.reserve(Sizes.size()); for (size_t Size : Sizes) { @@ -610,9 +610,9 @@ namespace utils { return Result; } - std::vector<std::pair<Oid, CompressedBuffer>> CreateSemiRandomAttachments(const std::span<const size_t>& Sizes) + eastl::vector<std::pair<Oid, CompressedBuffer>> CreateSemiRandomAttachments(const std::span<const size_t>& Sizes) { - std::vector<std::pair<Oid, CompressedBuffer>> Result; + eastl::vector<std::pair<Oid, CompressedBuffer>> Result; Result.reserve(Sizes.size()); for (size_t Size : Sizes) { @@ -1277,7 +1277,7 @@ TEST_CASE("zcache.rpc") const zen::CacheKey& CacheKey, size_t PayloadSize, CachePolicy RecordPolicy) { - std::vector<uint8_t> Data; + eastl::vector<uint8_t> Data; Data.resize(PayloadSize); uint32_t DataSeed = *reinterpret_cast<const uint32_t*>(&CacheKey.Hash.Hash[0]); uint16_t* DataPtr = reinterpret_cast<uint16_t*>(Data.data()); @@ -1298,8 +1298,8 @@ TEST_CASE("zcache.rpc") std::string_view Bucket, size_t Num, size_t PayloadSize = 1024, - size_t KeyOffset = 1) -> std::vector<CacheKey> { - std::vector<zen::CacheKey> OutKeys; + size_t KeyOffset = 1) -> eastl::vector<CacheKey> { + eastl::vector<zen::CacheKey> OutKeys; for (uint32_t Key = 1; Key <= Num; ++Key) { @@ -1382,9 +1382,9 @@ TEST_CASE("zcache.rpc") const uint16_t BasePort = Inst.SpawnServerAndWaitUntilReady(); const std::string BaseUri = fmt::format("http://localhost:{}/z$", BasePort); - CachePolicy Policy = CachePolicy::Default; - std::vector<zen::CacheKey> Keys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 128); - GetCacheRecordResult Result = GetCacheRecords(BaseUri, "ue4.ddc"sv, Keys, Policy); + CachePolicy Policy = CachePolicy::Default; + eastl::vector<zen::CacheKey> Keys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 128); + GetCacheRecordResult Result = GetCacheRecords(BaseUri, "ue4.ddc"sv, Keys, Policy); CHECK(Result.Result.Results.size() == Keys.size()); @@ -1411,9 +1411,9 @@ TEST_CASE("zcache.rpc") const uint16_t BasePort = Inst.SpawnServerAndWaitUntilReady(); const std::string BaseUri = fmt::format("http://localhost:{}/z$", BasePort); - CachePolicy Policy = CachePolicy::Default; - std::vector<zen::CacheKey> ExistingKeys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 128); - std::vector<zen::CacheKey> Keys; + CachePolicy Policy = CachePolicy::Default; + eastl::vector<zen::CacheKey> ExistingKeys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 128); + eastl::vector<zen::CacheKey> Keys; for (const zen::CacheKey& Key : ExistingKeys) { @@ -1458,7 +1458,7 @@ TEST_CASE("zcache.rpc") ZenServerInstance LocalServer(TestEnv); SpawnServer(LocalServer, LocalCfg); - std::vector<zen::CacheKey> Keys = PutCacheRecords(UpstreamCfg.BaseUri, "ue4.ddc"sv, "mastodon"sv, 4); + eastl::vector<zen::CacheKey> Keys = PutCacheRecords(UpstreamCfg.BaseUri, "ue4.ddc"sv, "mastodon"sv, 4); CachePolicy Policy = CachePolicy::QueryLocal; GetCacheRecordResult Result = GetCacheRecords(LocalCfg.BaseUri, "ue4.ddc"sv, Keys, Policy); @@ -1483,7 +1483,7 @@ TEST_CASE("zcache.rpc") ZenServerInstance LocalServer(TestEnv); SpawnServer(LocalServer, LocalCfg); - std::vector<zen::CacheKey> Keys = PutCacheRecords(UpstreamCfg.BaseUri, "ue4.ddc"sv, "mastodon"sv, 4); + eastl::vector<zen::CacheKey> Keys = PutCacheRecords(UpstreamCfg.BaseUri, "ue4.ddc"sv, "mastodon"sv, 4); CachePolicy Policy = (CachePolicy::QueryLocal | CachePolicy::QueryRemote); GetCacheRecordResult Result = GetCacheRecords(LocalCfg.BaseUri, "ue4.ddc"sv, Keys, Policy); @@ -1510,10 +1510,10 @@ TEST_CASE("zcache.rpc") const uint16_t BasePort = Inst.SpawnServerAndWaitUntilReady(); const std::string BaseUri = fmt::format("http://localhost:{}/z$", BasePort); - std::vector<zen::CacheKey> SmallKeys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 4, 1024); - std::vector<zen::CacheKey> LargeKeys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 4, 1024 * 1024 * 16, SmallKeys.size()); + eastl::vector<zen::CacheKey> SmallKeys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 4, 1024); + eastl::vector<zen::CacheKey> LargeKeys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 4, 1024 * 1024 * 16, SmallKeys.size()); - std::vector<zen::CacheKey> Keys(SmallKeys.begin(), SmallKeys.end()); + eastl::vector<zen::CacheKey> Keys(SmallKeys.begin(), SmallKeys.end()); Keys.insert(Keys.end(), LargeKeys.begin(), LargeKeys.end()); { @@ -1658,8 +1658,8 @@ TEST_CASE("zcache.failing.upstream") ZenServerInstance Upstream2Server(TestEnv); SpawnServer(Upstream2Server, Upstream2Cfg); - std::vector<std::uint16_t> UpstreamPorts = {Upstream1Cfg.Port, Upstream2Cfg.Port}; - ZenConfig LocalCfg = ZenConfig::NewWithThreadedUpstreams(TestEnv.GetNewPortNumber(), UpstreamPorts, false); + eastl::vector<std::uint16_t> UpstreamPorts = {Upstream1Cfg.Port, Upstream2Cfg.Port}; + ZenConfig LocalCfg = ZenConfig::NewWithThreadedUpstreams(TestEnv.GetNewPortNumber(), UpstreamPorts, false); LocalCfg.Args += (" --upstream-thread-count 2"); ZenServerInstance LocalServer(TestEnv); SpawnServer(LocalServer, LocalCfg); @@ -1678,7 +1678,7 @@ TEST_CASE("zcache.failing.upstream") const zen::CacheKey& CacheKey, size_t PayloadSize, CachePolicy RecordPolicy) { - std::vector<uint32_t> Data; + eastl::vector<uint32_t> Data; Data.resize(PayloadSize / 4); for (uint32_t Idx = 0; Idx < PayloadSize / 4; ++Idx) { @@ -1694,8 +1694,8 @@ TEST_CASE("zcache.failing.upstream") std::string_view Bucket, size_t Num, size_t KeyOffset, - size_t PayloadSize = 8192) -> std::vector<CacheKey> { - std::vector<zen::CacheKey> OutKeys; + size_t PayloadSize = 8192) -> eastl::vector<CacheKey> { + eastl::vector<zen::CacheKey> OutKeys; cacherequests::PutCacheRecordsRequest Request = {.AcceptMagic = kCbPkgMagic, .Namespace = std::string(Namespace)}; for (size_t Key = 1; Key <= Num; ++Key) @@ -1785,14 +1785,15 @@ TEST_CASE("zcache.failing.upstream") std::atomic_size_t Completed = 0; - auto Keys = new std::vector<CacheKey>[ThreadCount * KeyMultiplier]; + auto Keys = new eastl::vector<CacheKey>[ThreadCount * KeyMultiplier]; RwLock KeysLock; for (size_t I = 0; I < ThreadCount * KeyMultiplier; I++) { size_t Iteration = I; Pool.ScheduleWork([&] { - std::vector<CacheKey> NewKeys = PutCacheRecords(LocalUri, "ue4.ddc"sv, "mastodon"sv, RecordsPerRequest, I * RecordsPerRequest); + eastl::vector<CacheKey> NewKeys = + PutCacheRecords(LocalUri, "ue4.ddc"sv, "mastodon"sv, RecordsPerRequest, I * RecordsPerRequest); if (NewKeys.size() != RecordsPerRequest) { ZEN_DEBUG("PutCacheRecords iteration {} failed", Iteration); @@ -1848,8 +1849,8 @@ TEST_CASE("zcache.failing.upstream") Completed = 0; for (size_t I = 0; I < ThreadCount * KeyMultiplier; I++) { - size_t Iteration = I; - std::vector<CacheKey>& LocalKeys = Keys[Iteration]; + size_t Iteration = I; + eastl::vector<CacheKey>& LocalKeys = Keys[Iteration]; if (LocalKeys.empty()) { Completed.fetch_add(1); @@ -1913,7 +1914,7 @@ TEST_CASE("zcache.rpc.partialchunks") ZenServerInstance Server(TestEnv); SpawnServer(Server, LocalCfg); - std::vector<CompressedBuffer> Attachments; + eastl::vector<CompressedBuffer> Attachments; const auto BaseUri = fmt::format("http://localhost:{}/z$", Server.GetBasePort()); @@ -1927,9 +1928,9 @@ TEST_CASE("zcache.rpc.partialchunks") const CacheKey& CacheKey, size_t AttachmentCount, size_t AttachmentsSize, - CachePolicy RecordPolicy) -> std::vector<std::pair<Oid, CompressedBuffer>> { - std::vector<std::pair<Oid, CompressedBuffer>> AttachmentBuffers; - std::vector<cacherequests::PutCacheRecordRequestValue> Attachments; + CachePolicy RecordPolicy) -> eastl::vector<std::pair<Oid, CompressedBuffer>> { + eastl::vector<std::pair<Oid, CompressedBuffer>> AttachmentBuffers; + eastl::vector<cacherequests::PutCacheRecordRequestValue> Attachments; for (size_t AttachmentIndex = 0; AttachmentIndex < AttachmentCount; AttachmentIndex++) { CompressedBuffer Value = CreateSemiRandomBlob(AttachmentsSize); @@ -1948,14 +1949,14 @@ TEST_CASE("zcache.rpc.partialchunks") size_t Num, size_t AttachmentCount, size_t AttachmentsSize = - 8192) -> std::vector<std::pair<CacheKey, std::vector<std::pair<Oid, CompressedBuffer>>>> { - std::vector<std::pair<CacheKey, std::vector<std::pair<Oid, CompressedBuffer>>>> Keys; + 8192) -> eastl::vector<std::pair<CacheKey, eastl::vector<std::pair<Oid, CompressedBuffer>>>> { + eastl::vector<std::pair<CacheKey, eastl::vector<std::pair<Oid, CompressedBuffer>>>> Keys; cacherequests::PutCacheRecordsRequest Request = {.AcceptMagic = kCbPkgMagic, .Namespace = std::string(Namespace)}; for (size_t Key = 1; Key <= Num; ++Key) { - const CacheKey NewCacheKey = GenerateKey(Bucket, KeyOffset + Key); - std::vector<std::pair<Oid, CompressedBuffer>> Attachments = + const CacheKey NewCacheKey = GenerateKey(Bucket, KeyOffset + Key); + eastl::vector<std::pair<Oid, CompressedBuffer>> Attachments = AppendCacheRecord(Request, NewCacheKey, AttachmentCount, AttachmentsSize, CachePolicy::Default); Keys.push_back(std::make_pair(NewCacheKey, std::move(Attachments))); } @@ -2209,12 +2210,12 @@ TEST_CASE("zcache.rpc.allpolicies") UserData* Data; }; - KeyData KeyDatas[NumKeys]; - std::vector<CachePutRequest> PutRequests; - std::vector<CachePutValueRequest> PutValueRequests; - std::vector<CacheGetRequest> GetRequests; - std::vector<CacheGetValueRequest> GetValueRequests; - std::vector<CacheGetChunkRequest> ChunkRequests; + KeyData KeyDatas[NumKeys]; + eastl::vector<CachePutRequest> PutRequests; + eastl::vector<CachePutValueRequest> PutValueRequests; + eastl::vector<CacheGetRequest> GetRequests; + eastl::vector<CacheGetValueRequest> GetValueRequests; + eastl::vector<CacheGetChunkRequest> ChunkRequests; for (uint32_t KeyIndex = 0; KeyIndex < NumKeys; ++KeyIndex) { @@ -2667,9 +2668,9 @@ public: ZenServerInstance& GetInstance(int Index) { return *m_Instances[Index]; } private: - std::string m_HelperId; - int m_ServerCount = 0; - std::vector<std::unique_ptr<ZenServerInstance>> m_Instances; + std::string m_HelperId; + int m_ServerCount = 0; + eastl::vector<std::unique_ptr<ZenServerInstance>> m_Instances; }; TEST_CASE("http.basics") @@ -2822,17 +2823,17 @@ TEST_CASE("project.remote") ZenServerTestHelper Servers("remote", 3); Servers.SpawnServers("--debug"); - std::vector<Oid> OpIds; - const size_t OpCount = 24; + eastl::vector<Oid> OpIds; + const size_t OpCount = 24; OpIds.reserve(OpCount); for (size_t I = 0; I < OpCount; ++I) { OpIds.emplace_back(Oid::NewOid()); } - std::unordered_map<Oid, std::vector<std::pair<Oid, CompressedBuffer>>, Oid::Hasher> Attachments; + std::unordered_map<Oid, eastl::vector<std::pair<Oid, CompressedBuffer>>, Oid::Hasher> Attachments; { - std::vector<std::size_t> AttachmentSizes( + eastl::vector<std::size_t> AttachmentSizes( {7633, 6825, 5738, 8031, 7225, 566, 3656, 6006, 24, 33466, 1093, 4269, 2257, 3685, 13489, 97194, 6151, 5482, 6217, 3511, 6738, 5061, 7537, 2759, 1916, 8210, 2235, 224024, 51582, 5251, 491, 2u * 1024u * 1024u + 124u, 74607, 18135, 3767, 154045, 4415, 5007, 8876, 96761, 3359, 8526, 4097, 4855, 48225}); @@ -2935,7 +2936,7 @@ TEST_CASE("project.remote") MakeOp(Session, Servers.GetInstance(0).GetBaseUri(), "proj0", "oplog0", OpPackage); } - std::vector<IoHash> AttachmentHashes; + eastl::vector<IoHash> AttachmentHashes; AttachmentHashes.reserve(Attachments.size()); for (const auto& AttachmentOplog : Attachments) { @@ -2982,7 +2983,7 @@ TEST_CASE("project.remote") auto ValidateOplog = [&SourceOps, &AddOp, &Servers, &Session](int ServerIndex, std::string_view Project, std::string_view Oplog) { std::unordered_map<Oid, uint32_t, Oid::Hasher> TargetOps; - std::vector<CbObject> ResultingOplog; + eastl::vector<CbObject> ResultingOplog; std::string GetOpsRequest = fmt::format("{}/prj/{}/oplog/{}/entries", Servers.GetInstance(ServerIndex).GetBaseUri(), Project, Oplog); @@ -3295,11 +3296,11 @@ TEST_CASE("project.remote") } } -std::vector<std::pair<std::filesystem::path, IoBuffer>> +eastl::vector<std::pair<std::filesystem::path, IoBuffer>> GenerateFolderContent(const std::filesystem::path& RootPath) { CreateDirectories(RootPath); - std::vector<std::pair<std::filesystem::path, IoBuffer>> Result; + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> Result; Result.push_back(std::make_pair(RootPath / "root_blob_1.bin", CreateRandomBlob(4122))); Result.push_back(std::make_pair(RootPath / "root_blob_2.bin", CreateRandomBlob(2122))); @@ -3328,10 +3329,10 @@ GenerateFolderContent(const std::filesystem::path& RootPath) return Result; } -std::vector<std::pair<std::filesystem::path, IoBuffer>> +eastl::vector<std::pair<std::filesystem::path, IoBuffer>> GenerateFolderContent2(const std::filesystem::path& RootPath) { - std::vector<std::pair<std::filesystem::path, IoBuffer>> Result; + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> Result; Result.push_back(std::make_pair(RootPath / "root_blob_3.bin", CreateRandomBlob(312))); std::filesystem::path FirstFolder(RootPath / "first_folder"); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob3.bin", CreateRandomBlob(722))); @@ -3777,8 +3778,8 @@ TEST_CASE("workspaces.share") } { - uint32_t CorrelationId = gsl::narrow<uint32_t>(Files.size()); - std::vector<RequestChunkEntry> BatchEntries; + uint32_t CorrelationId = gsl::narrow<uint32_t>(Files.size()); + eastl::vector<RequestChunkEntry> BatchEntries; for (auto It : Files) { const Oid& ChunkId = It.first; @@ -3790,7 +3791,7 @@ TEST_CASE("workspaces.share") IoBuffer BatchResponse = Client.Post(fmt::format("/ws/{}/{}/batch", WorkspaceId, ShareId), BuildChunkBatchRequest(BatchEntries)).ResponsePayload; CHECK(BatchResponse); - std::vector<IoBuffer> BatchResult = ParseChunkBatchResponse(BatchResponse); + eastl::vector<IoBuffer> BatchResult = ParseChunkBatchResponse(BatchResponse); CHECK(BatchResult.size() == Files.size()); for (const RequestChunkEntry& Request : BatchEntries) { diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp index 2888f5450..ab0372619 100644 --- a/src/zenserver/admin/admin.cpp +++ b/src/zenserver/admin/admin.cpp @@ -131,8 +131,8 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, m_Router.RegisterRoute( "jobs", [&](HttpRouterRequest& Req) { - std::vector<JobQueue::JobInfo> Jobs = m_BackgroundJobQueue.GetJobs(); - CbObjectWriter Obj; + eastl::vector<JobQueue::JobInfo> Jobs = m_BackgroundJobQueue.GetJobs(); + CbObjectWriter Obj; Obj.BeginArray("jobs"); for (const auto& Job : Jobs) { @@ -673,7 +673,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, try { - std::vector<CbObject> Manifests = ReadAllCentralManifests(m_ServerOptions.SystemRootDir); + eastl::vector<CbObject> Manifests = ReadAllCentralManifests(m_ServerOptions.SystemRootDir); Obj.BeginArray("known"); diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp index b9a9ca380..2f7a74f25 100644 --- a/src/zenserver/cache/httpstructuredcache.cpp +++ b/src/zenserver/cache/httpstructuredcache.cpp @@ -150,15 +150,15 @@ HttpStructuredCacheService::ScrubStorage(ScrubContext& Ctx) void HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) { - std::string_view Key = Request.RelativeUri(); - std::vector<std::string> Tokens; - uint32_t TokenCount = ForEachStrTok(Key, '/', [&Tokens](std::string_view Token) { - Tokens.push_back(std::string(Token)); - return true; - }); - std::string FilterNamespace; - std::string FilterBucket; - std::string FilterValue; + std::string_view Key = Request.RelativeUri(); + eastl::vector<std::string> Tokens; + uint32_t TokenCount = ForEachStrTok(Key, '/', [&Tokens](std::string_view Token) { + Tokens.push_back(std::string(Token)); + return true; + }); + std::string FilterNamespace; + std::string FilterBucket; + std::string FilterValue; switch (TokenCount) { case 1: @@ -543,7 +543,7 @@ HttpStructuredCacheService::HandleCacheRequest(HttpServerRequest& Request) } ResponseWriter.EndObject(); - std::sort(begin(Info.NamespaceNames), end(Info.NamespaceNames), [](std::string_view L, std::string_view R) { + std::sort(Info.NamespaceNames.begin(), Info.NamespaceNames.end(), [](std::string_view L, std::string_view R) { return L.compare(R) < 0; }); ResponseWriter.BeginArray("Namespaces"); @@ -602,7 +602,7 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque } ResponseWriter.EndObject(); - std::sort(begin(Info->BucketNames), end(Info->BucketNames), [](std::string_view L, std::string_view R) { + std::sort(Info->BucketNames.begin(), Info->BucketNames.end(), [](std::string_view L, std::string_view R) { return L.compare(R) < 0; }); @@ -628,7 +628,7 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque ResponseWriter.BeginArray("Buckets"); - std::vector<std::string> BucketNames; + eastl::vector<std::string> BucketNames; if (Buckets == "*") // Get all - empty FieldFilter equal getting all fields { BucketNames = Info.value().BucketNames; @@ -640,8 +640,8 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque return true; }); } - WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background); - std::vector<IoHash> AllAttachments; + WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background); + eastl::vector<IoHash> AllAttachments; for (const std::string& BucketName : BucketNames) { ResponseWriter.BeginObject(); @@ -1005,13 +1005,13 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con CbPackage Package; if (Package.TryLoad(ClientResultValue.Value)) { - CbObject CacheRecord = Package.GetObject(); - AttachmentCount Count; - size_t NumAttachments = Package.GetAttachments().size(); - std::vector<IoHash> ReferencedAttachments; - std::vector<IoBuffer> WriteAttachmentBuffers; + CbObject CacheRecord = Package.GetObject(); + AttachmentCount Count; + size_t NumAttachments = Package.GetAttachments().size(); + eastl::vector<IoHash> ReferencedAttachments; + eastl::vector<IoBuffer> WriteAttachmentBuffers; WriteAttachmentBuffers.reserve(NumAttachments); - std::vector<IoHash> WriteRawHashes; + eastl::vector<IoHash> WriteRawHashes; WriteRawHashes.reserve(NumAttachments); CacheRecord.IterateAttachments([this, @@ -1093,7 +1093,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con if (!WriteAttachmentBuffers.empty()) { - std::vector<CidStore::InsertResult> InsertResults = + eastl::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); for (const CidStore::InsertResult& Result : InsertResults) { @@ -1264,10 +1264,10 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con Body.SetContentType(ZenContentType::kCbObject); - CbObjectView CacheRecord(Body.Data()); - std::vector<IoHash> ValidAttachments; - std::vector<IoHash> ReferencedAttachments; - int32_t TotalCount = 0; + CbObjectView CacheRecord(Body.Data()); + eastl::vector<IoHash> ValidAttachments; + eastl::vector<IoHash> ReferencedAttachments; + int32_t TotalCount = 0; CacheRecord.IterateAttachments([this, &TotalCount, &ValidAttachments, &ReferencedAttachments](CbFieldView AttachmentHash) { const IoHash Hash = AttachmentHash.AsHash(); @@ -1323,13 +1323,13 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con CbObject CacheRecord = Package.GetObject(); - AttachmentCount Count; - size_t NumAttachments = Package.GetAttachments().size(); - std::vector<IoHash> ValidAttachments; - std::vector<IoHash> ReferencedAttachments; + AttachmentCount Count; + size_t NumAttachments = Package.GetAttachments().size(); + eastl::vector<IoHash> ValidAttachments; + eastl::vector<IoHash> ReferencedAttachments; ValidAttachments.reserve(NumAttachments); - std::vector<IoBuffer> WriteAttachmentBuffers; - std::vector<IoHash> WriteRawHashes; + eastl::vector<IoBuffer> WriteAttachmentBuffers; + eastl::vector<IoHash> WriteRawHashes; WriteAttachmentBuffers.reserve(NumAttachments); WriteRawHashes.reserve(NumAttachments); @@ -1380,7 +1380,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con if (!WriteAttachmentBuffers.empty()) { - std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + eastl::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); for (const CidStore::InsertResult& InsertResult : InsertResults) { if (InsertResult.New) diff --git a/src/zenserver/cache/httpstructuredcache.h b/src/zenserver/cache/httpstructuredcache.h index 13c1d6475..0a8b22ea3 100644 --- a/src/zenserver/cache/httpstructuredcache.h +++ b/src/zenserver/cache/httpstructuredcache.h @@ -10,8 +10,8 @@ #include <zenutil/cache/cache.h> #include <zenutil/openprocesscache.h> +#include <EASTL/vector.h> #include <memory> -#include <vector> namespace zen { diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 809092378..00fa99110 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -57,10 +57,10 @@ EmitCentralManifest(const std::filesystem::path& SystemRoot, Oid Identifier, CbO WriteFile(StatesPath / fmt::format("{}", Identifier), Cbo.Save().GetBuffer().AsIoBuffer()); } -std::vector<CbObject> +eastl::vector<CbObject> ReadAllCentralManifests(const std::filesystem::path& SystemRoot) { - std::vector<CbObject> Manifests; + eastl::vector<CbObject> Manifests; DirectoryContent Content; GetDirectoryContent(SystemRoot / "States", DirectoryContentFlags::IncludeFiles, Content); diff --git a/src/zenserver/config.h b/src/zenserver/config.h index c7781aada..8e8077e38 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -5,9 +5,11 @@ #include <zencore/logbase.h> #include <zencore/zencore.h> #include <zenhttp/httpserver.h> + +#include <EASTL/vector.h> + #include <filesystem> #include <string> -#include <vector> namespace zen { @@ -92,7 +94,7 @@ struct ZenOpenIdProviderConfig struct ZenAuthConfig { - std::vector<ZenOpenIdProviderConfig> OpenIdProviders; + eastl::vector<ZenOpenIdProviderConfig> OpenIdProviders; }; struct ZenObjectStoreConfig @@ -103,7 +105,7 @@ struct ZenObjectStoreConfig std::filesystem::path Directory; }; - std::vector<BucketConfig> Buckets; + eastl::vector<BucketConfig> Buckets; }; struct ZenStatsConfig @@ -187,6 +189,6 @@ struct ZenServerOptions void ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions); void EmitCentralManifest(const std::filesystem::path& SystemRoot, Oid Identifier, CbObject Manifest, std::filesystem::path ManifestPath); -std::vector<CbObject> ReadAllCentralManifests(const std::filesystem::path& SystemRoot); +eastl::vector<CbObject> ReadAllCentralManifests(const std::filesystem::path& SystemRoot); } // namespace zen diff --git a/src/zenserver/config/luaconfig.cpp b/src/zenserver/config/luaconfig.cpp index f742fa34a..f6748b113 100644 --- a/src/zenserver/config/luaconfig.cpp +++ b/src/zenserver/config/luaconfig.cpp @@ -163,7 +163,7 @@ LuaContainerWriter::EndContainer() ////////////////////////////////////////////////////////////////////////// -StringArrayOption::StringArrayOption(std::vector<std::string>& Value) : Value(Value) +StringArrayOption::StringArrayOption(std::vector<std::string>& InValue) : Value(InValue) { } @@ -339,23 +339,23 @@ Options::Print(zen::StringBuilderBase& SB, const cxxopts::ParseResult& CmdLineRe } } - std::vector<std::string> SortedKeys(UsedKeys.begin(), UsedKeys.end()); + eastl::vector<std::string> SortedKeys(UsedKeys.begin(), UsedKeys.end()); std::sort(SortedKeys.begin(), SortedKeys.end()); - auto GetTablePath = [](const std::string& Key) -> std::vector<std::string> { - std::vector<std::string> Path; + auto GetTablePath = [](const std::string& Key) -> eastl::vector<std::string> { + eastl::vector<std::string> Path; zen::ForEachStrTok(Key, '.', [&Path](std::string_view Part) { Path.push_back(std::string(Part)); return true; }); return Path; }; - std::vector<std::string> CurrentTablePath; - std::string Indent; - auto It = SortedKeys.begin(); + eastl::vector<std::string> CurrentTablePath; + std::string Indent; + for (const std::string& Key : SortedKeys) { - std::vector<std::string> KeyPath = GetTablePath(Key); - std::string Name = KeyPath.back(); + eastl::vector<std::string> KeyPath = GetTablePath(Key); + std::string Name = KeyPath.back(); KeyPath.pop_back(); if (CurrentTablePath != KeyPath) { diff --git a/src/zenserver/diag/diagsvcs.h b/src/zenserver/diag/diagsvcs.h index 8cc869c83..bf1e3a929 100644 --- a/src/zenserver/diag/diagsvcs.h +++ b/src/zenserver/diag/diagsvcs.h @@ -46,7 +46,7 @@ public: } else if (Uri == "1M_1k"sv) { - std::vector<IoBuffer> Buffers; + eastl::vector<IoBuffer> Buffers; Buffers.reserve(1024); for (int i = 0; i < 1024; ++i) @@ -58,7 +58,7 @@ public: } else if (Uri == "1G"sv) { - std::vector<IoBuffer> Buffers; + eastl::vector<IoBuffer> Buffers; Buffers.reserve(1024); for (int i = 0; i < 1024; ++i) @@ -70,7 +70,7 @@ public: } else if (Uri == "1G_1k"sv) { - std::vector<IoBuffer> Buffers; + eastl::vector<IoBuffer> Buffers; Buffers.reserve(1024 * 1024); for (int i = 0; i < 1024 * 1024; ++i) diff --git a/src/zenserver/objectstore/objectstore.h b/src/zenserver/objectstore/objectstore.h index dae979c4c..428f49145 100644 --- a/src/zenserver/objectstore/objectstore.h +++ b/src/zenserver/objectstore/objectstore.h @@ -19,8 +19,8 @@ struct ObjectStoreConfig std::filesystem::path Directory; }; - std::filesystem::path RootDirectory; - std::vector<BucketConfig> Buckets; + std::filesystem::path RootDirectory; + eastl::vector<BucketConfig> Buckets; }; class HttpObjectStoreService final : public zen::HttpService diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp index fbb9bc344..e9d65f2b9 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp @@ -163,7 +163,7 @@ public: return Result; } - virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Chunks) override + virtual SaveAttachmentsResult SaveAttachments(const eastl::vector<SharedBuffer>& Chunks) override { SaveAttachmentsResult Result; for (const SharedBuffer& Chunk : Chunks) @@ -332,7 +332,7 @@ public: m_BuildId); return Result; } - std::optional<std::vector<ChunkBlockDescription>> Blocks = + std::optional<eastl::vector<ChunkBlockDescription>> Blocks = ParseChunkBlockDescriptionList(LoadCompactBinaryObject(FindResult.Response)); if (!Blocks) { @@ -374,7 +374,7 @@ public: return Result; } - virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) override + virtual LoadAttachmentsResult LoadAttachments(const eastl::vector<IoHash>& RawHashes) override { LoadAttachmentsResult Result; for (const IoHash& Hash : RawHashes) diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp index 98e292d91..7455efdef 100644 --- a/src/zenserver/projectstore/fileremoteprojectstore.cpp +++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp @@ -137,7 +137,7 @@ public: return Result; } - virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Chunks) override + virtual SaveAttachmentsResult SaveAttachments(const eastl::vector<SharedBuffer>& Chunks) override { Stopwatch Timer; @@ -171,14 +171,14 @@ public: { return GetKnownBlocksResult{LoadResult}; } - Stopwatch Timer; - std::vector<IoHash> BlockHashes = GetBlockHashesFromOplog(LoadResult.ContainerObject); + Stopwatch Timer; + eastl::vector<IoHash> BlockHashes = GetBlockHashesFromOplog(LoadResult.ContainerObject); if (BlockHashes.empty()) { return GetKnownBlocksResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent), .ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}}; } - std::vector<IoHash> ExistingBlockHashes; + eastl::vector<IoHash> ExistingBlockHashes; for (const IoHash& RawHash : BlockHashes) { std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); @@ -192,8 +192,8 @@ public: return GetKnownBlocksResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent), .ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}}; } - std::vector<ThinChunkBlockDescription> KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes); - GetKnownBlocksResult Result{{.ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}}; + eastl::vector<ThinChunkBlockDescription> KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes); + GetKnownBlocksResult Result{{.ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}}; Result.Blocks = std::move(KnownBlocks); return Result; } @@ -220,7 +220,7 @@ public: return Result; } - virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) override + virtual LoadAttachmentsResult LoadAttachments(const eastl::vector<IoHash>& RawHashes) override { Stopwatch Timer; LoadAttachmentsResult Result; diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 47748dd90..80e09939a 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -192,13 +192,13 @@ namespace { Cbo.EndObject(); } - void CbWriteOplogs(CidStore& CidStore, - ProjectStore::Project& Project, - std::vector<std::string> OpLogs, - bool Details, - bool OpDetails, - bool AttachmentDetails, - CbObjectWriter& Cbo) + void CbWriteOplogs(CidStore& CidStore, + ProjectStore::Project& Project, + eastl::vector<std::string> OpLogs, + bool Details, + bool OpDetails, + bool AttachmentDetails, + CbObjectWriter& Cbo) { Cbo.BeginArray("oplogs"); { @@ -214,13 +214,13 @@ namespace { Cbo.EndArray(); } - void CbWriteProject(CidStore& CidStore, - ProjectStore::Project& Project, - std::vector<std::string> OpLogs, - bool Details, - bool OpDetails, - bool AttachmentDetails, - CbObjectWriter& Cbo) + void CbWriteProject(CidStore& CidStore, + ProjectStore::Project& Project, + eastl::vector<std::string> OpLogs, + bool Details, + bool OpDetails, + bool AttachmentDetails, + CbObjectWriter& Cbo) { Cbo.BeginObject(); { @@ -537,7 +537,7 @@ HttpProjectService::HandleChunkBatchRequest(HttpRouterRequest& Req) HttpReq.WriteResponse(HttpResponseCode::BadRequest); } - std::vector<RequestChunkEntry> RequestedChunks; + eastl::vector<RequestChunkEntry> RequestedChunks; RequestedChunks.resize(RequestHdr.ChunkCount); Reader.Read(RequestedChunks.data(), sizeof(RequestChunkEntry) * RequestHdr.ChunkCount); @@ -558,7 +558,7 @@ HttpProjectService::HandleChunkBatchRequest(HttpRouterRequest& Req) uint64_t ChunkSize; }; - std::vector<IoBuffer> OutBlobs; + eastl::vector<IoBuffer> OutBlobs; OutBlobs.emplace_back(sizeof(ResponseHeader) + RequestHdr.ChunkCount * sizeof(ResponseChunkEntry)); for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex) { @@ -607,7 +607,7 @@ HttpProjectService::HandleChunkBatchRequest(HttpRouterRequest& Req) memcpy(ResponsePtr, &ResponseChunk, sizeof(ResponseChunk)); ResponsePtr += sizeof(ResponseChunk); } - std::erase_if(OutBlobs, [](IoBuffer Buffer) -> bool { return !Buffer; }); + zen::erase_if(OutBlobs, [](IoBuffer Buffer) -> bool { return !Buffer; }); return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, OutBlobs); } @@ -983,7 +983,7 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req) IoBuffer Payload = HttpReq.ReadPayload(); CbObject RequestObject = LoadCompactBinaryObject(Payload); - std::vector<IoHash> NeedList; + eastl::vector<IoHash> NeedList; { eastl::fixed_vector<IoHash, 16> ChunkList; @@ -1063,8 +1063,8 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) // the prep step rejected the chunk. This should be fixed since there's // a performance cost associated with any file system activity - bool IsValid = true; - std::vector<IoHash> MissingChunks; + bool IsValid = true; + eastl::vector<IoHash> MissingChunks; CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer { if (m_CidStore.ContainsChunk(Hash)) @@ -1790,7 +1790,7 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req) } Project->TouchProject(); - std::vector<std::string> OpLogs = Project->ScanForOplogs(); + eastl::vector<std::string> OpLogs = Project->ScanForOplogs(); CbObjectWriter Response; Response << "id"sv << Project->Identifier; @@ -1984,7 +1984,7 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req) m_ProjectStore->DiscoverProjects(); m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) { - std::vector<std::string> OpLogs = Project.ScanForOplogs(); + eastl::vector<std::string> OpLogs = Project.ScanForOplogs(); CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); }); } @@ -2031,8 +2031,8 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req) } else { - CbObjectWriter Cbo; - std::vector<std::string> OpLogs = FoundProject->ScanForOplogs(); + CbObjectWriter Cbo; + eastl::vector<std::string> OpLogs = FoundProject->ScanForOplogs(); Cbo.BeginArray("projects"); { CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp index e5839ad3b..0125a8c86 100644 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp +++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp @@ -110,7 +110,7 @@ public: return Result; } - virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Chunks) override + virtual SaveAttachmentsResult SaveAttachments(const eastl::vector<SharedBuffer>& Chunks) override { SaveAttachmentsResult Result; for (const SharedBuffer& Chunk : Chunks) @@ -157,7 +157,7 @@ public: { return GetKnownBlocksResult{LoadResult}; } - std::vector<IoHash> BlockHashes = GetBlockHashesFromOplog(LoadResult.ContainerObject); + eastl::vector<IoHash> BlockHashes = GetBlockHashesFromOplog(LoadResult.ContainerObject); if (BlockHashes.empty()) { return GetKnownBlocksResult{ @@ -179,8 +179,8 @@ public: ExistsResult.Reason)}}; } - Stopwatch Timer; - std::vector<IoHash> ExistingBlockHashes; + Stopwatch Timer; + eastl::vector<IoHash> ExistingBlockHashes; for (const IoHash& RawHash : BlockHashes) { if (!ExistsResult.Needs.contains(RawHash)) @@ -193,7 +193,7 @@ public: return GetKnownBlocksResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent), .ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds}}; } - std::vector<ThinChunkBlockDescription> KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes); + eastl::vector<ThinChunkBlockDescription> KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes); GetKnownBlocksResult Result{ {.ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000.0}}; @@ -219,7 +219,7 @@ public: return Result; } - virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) override + virtual LoadAttachmentsResult LoadAttachments(const eastl::vector<IoHash>& RawHashes) override { LoadAttachmentsResult Result; for (const IoHash& Hash : RawHashes) diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 86791e29a..0c0c942af 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -617,7 +617,7 @@ struct ProjectStore::OplogStorage : public RefCounted try { - std::vector<OplogEntry> Ops; + eastl::vector<OplogEntry> Ops; Ops.reserve(LSNs.size()); tsl::robin_map<uint32_t, size_t> LSNToIndex; @@ -651,7 +651,7 @@ struct ProjectStore::OplogStorage : public RefCounted return Lhs.OpCoreOffset < Rhs.OpCoreOffset; }); - std::vector<uint32_t> OldLSNs; + eastl::vector<uint32_t> OldLSNs; OldLSNs.reserve(Ops.size()); uint64_t OpWriteOffset = 0; @@ -774,8 +774,8 @@ struct ProjectStore::OplogStorage : public RefCounted uint64_t OpsBlockSize = m_OpBlobs.FileSize(); - std::vector<OplogEntry> OpLogEntries; - uint64_t InvalidEntries = 0; + eastl::vector<OplogEntry> OpLogEntries; + uint64_t InvalidEntries = 0; { tsl::robin_map<Oid, size_t, Oid::Hasher> LatestKeys; @@ -993,14 +993,14 @@ struct ProjectStore::OplogStorage : public RefCounted return Entry; } - std::vector<OplogEntry> AppendOps(std::span<const AppendOpData> Ops) + eastl::vector<OplogEntry> AppendOps(std::span<const AppendOpData> Ops) { ZEN_MEMSCOPE(GetProjectstoreTag()); ZEN_TRACE_CPU("Store::OplogStorage::AppendOps"); - size_t OpCount = Ops.size(); - std::vector<std::pair<uint64_t, uint64_t>> OffsetAndSizes; - std::vector<uint32_t> OpLsns; + size_t OpCount = Ops.size(); + eastl::vector<std::pair<uint64_t, uint64_t>> OffsetAndSizes; + eastl::vector<uint32_t> OpLsns; OffsetAndSizes.resize(OpCount); OpLsns.resize(OpCount); @@ -1033,7 +1033,7 @@ struct ProjectStore::OplogStorage : public RefCounted IoBuffer WriteBuffer(WriteLength); - std::vector<OplogEntry> Entries; + eastl::vector<OplogEntry> Entries; Entries.resize(OpCount); for (size_t OpIndex = 0; OpIndex < OpCount; OpIndex++) { @@ -1157,14 +1157,14 @@ ProjectStore::Oplog::ScrubStorage(ScrubContext& Ctx) { ZEN_MEMSCOPE(GetProjectstoreTag()); - std::vector<Oid> BadEntryKeys; + eastl::vector<Oid> BadEntryKeys; using namespace std::literals; IterateOplogWithKey([&](uint32_t Lsn, const Oid& Key, CbObjectView Op) { ZEN_UNUSED(Lsn); - std::vector<IoHash> Cids; + eastl::vector<IoHash> Cids; Op.IterateAttachments([&](CbFieldView Visitor) { Cids.emplace_back(Visitor.AsAttachment()); }); { @@ -1488,10 +1488,10 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo ValidationResult Result; - std::vector<Oid> KeyHashes; - std::vector<std::string> Keys; - std::vector<std::vector<IoHash>> Attachments; - std::vector<OplogEntryMapping> Mappings; + eastl::vector<Oid> KeyHashes; + eastl::vector<std::string> Keys; + eastl::vector<eastl::vector<IoHash>> Attachments; + eastl::vector<OplogEntryMapping> Mappings; IterateOplogWithKey([&](uint32_t LSN, const Oid& Key, CbObjectView OpView) { Result.LSNLow = Min(Result.LSNLow, LSN); @@ -1499,7 +1499,7 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo KeyHashes.push_back(Key); Keys.emplace_back(std::string(OpView["key"sv].AsString())); - std::vector<IoHash> OpAttachments; + eastl::vector<IoHash> OpAttachments; OpView.IterateAttachments([&OpAttachments](CbFieldView Attachment) { OpAttachments.push_back(Attachment.AsAttachment()); }); Attachments.emplace_back(std::move(OpAttachments)); @@ -1551,7 +1551,7 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo } } } - const std::vector<IoHash>& OpAttachments = Attachments[OpIndex]; + const eastl::vector<IoHash>& OpAttachments = Attachments[OpIndex]; for (const IoHash& Attachment : OpAttachments) { if (IoBuffer Payload = m_CidStore.FindChunkByCid(Attachment); !Payload) @@ -1647,15 +1647,15 @@ ProjectStore::Oplog::WriteIndexSnapshot() } // Write the current state of the location map to a new index state - std::vector<uint32_t> LSNEntries; - std::vector<Oid> Keys; - std::vector<OplogEntryAddress> AddressMapEntries; - std::vector<uint32_t> LatestOpMapEntries; - std::vector<IoHash> ChunkMapEntries; - std::vector<IoHash> MetaMapEntries; - std::vector<uint32_t> FilePathLengths; - std::vector<std::string> FilePaths; - uint64_t IndexLogPosition = 0; + eastl::vector<uint32_t> LSNEntries; + eastl::vector<Oid> Keys; + eastl::vector<OplogEntryAddress> AddressMapEntries; + eastl::vector<uint32_t> LatestOpMapEntries; + eastl::vector<IoHash> ChunkMapEntries; + eastl::vector<IoHash> MetaMapEntries; + eastl::vector<uint32_t> FilePathLengths; + eastl::vector<std::string> FilePaths; + uint64_t IndexLogPosition = 0; { IndexLogPosition = m_Storage->LogCount(); @@ -1869,20 +1869,20 @@ ProjectStore::Oplog::ReadIndexSnapshot() return; } - std::vector<uint32_t> LSNEntries(Header.LSNCount); + eastl::vector<uint32_t> LSNEntries(Header.LSNCount); ObjectIndexFile.Read(LSNEntries.data(), LSNEntries.size() * sizeof(uint32_t), Offset); Offset += LSNEntries.size() * sizeof(uint32_t); Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); size_t LSNOffset = 0; - std::vector<Oid> Keys(Header.KeyCount); + eastl::vector<Oid> Keys(Header.KeyCount); ObjectIndexFile.Read(Keys.data(), Keys.size() * sizeof(Oid), Offset); Offset += Keys.size() * sizeof(Oid); Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); size_t KeyOffset = 0; { - std::vector<OplogEntryAddress> AddressMapEntries(Header.OpAddressMapCount); + eastl::vector<OplogEntryAddress> AddressMapEntries(Header.OpAddressMapCount); ObjectIndexFile.Read(AddressMapEntries.data(), AddressMapEntries.size() * sizeof(OplogEntryAddress), Offset); Offset += AddressMapEntries.size() * sizeof(OplogEntryAddress); Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); @@ -1897,7 +1897,7 @@ ProjectStore::Oplog::ReadIndexSnapshot() } } { - std::vector<uint32_t> LatestOpMapEntries(Header.LatestOpMapCount); + eastl::vector<uint32_t> LatestOpMapEntries(Header.LatestOpMapCount); ObjectIndexFile.Read(LatestOpMapEntries.data(), LatestOpMapEntries.size() * sizeof(uint32_t), Offset); Offset += LatestOpMapEntries.size() * sizeof(uint32_t); Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); @@ -1909,7 +1909,7 @@ ProjectStore::Oplog::ReadIndexSnapshot() } } { - std::vector<IoHash> ChunkMapEntries(Header.ChunkMapCount); + eastl::vector<IoHash> ChunkMapEntries(Header.ChunkMapCount); ObjectIndexFile.Read(ChunkMapEntries.data(), ChunkMapEntries.size() * sizeof(IoHash), Offset); Offset += ChunkMapEntries.size() * sizeof(IoHash); Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); @@ -1920,7 +1920,7 @@ ProjectStore::Oplog::ReadIndexSnapshot() } } { - std::vector<IoHash> MetaMapEntries(Header.MetaMapCount); + eastl::vector<IoHash> MetaMapEntries(Header.MetaMapCount); ObjectIndexFile.Read(MetaMapEntries.data(), MetaMapEntries.size() * sizeof(IoHash), Offset); Offset += MetaMapEntries.size() * sizeof(IoHash); Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); @@ -1933,7 +1933,7 @@ ProjectStore::Oplog::ReadIndexSnapshot() { m_FileMap.reserve(Header.FileMapCount); - std::vector<uint32_t> FilePathLengths(Header.FileMapCount * 2); + eastl::vector<uint32_t> FilePathLengths(Header.FileMapCount * 2); ObjectIndexFile.Read(FilePathLengths.data(), FilePathLengths.size() * sizeof(uint32_t), Offset); Offset += FilePathLengths.size() * sizeof(uint32_t); Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); @@ -2004,7 +2004,7 @@ ProjectStore::Oplog::GetUnusedSpacePercentLocked() const return 0; } - std::vector<OplogEntryAddress> Addresses; + eastl::vector<OplogEntryAddress> Addresses; { Addresses.reserve(m_LatestOpMap.size()); for (auto It : m_LatestOpMap) @@ -2040,7 +2040,7 @@ ProjectStore::Oplog::Compact(RwLock::ExclusiveLockScope&, bool DryRun, bool Reta Stopwatch Timer; - std::vector<uint32_t> LSNs; + eastl::vector<uint32_t> LSNs; LSNs.reserve(m_LatestOpMap.size()); for (auto It : m_LatestOpMap) { @@ -2111,10 +2111,10 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, { ZEN_MEMSCOPE(GetProjectstoreTag()); - std::vector<size_t> CidChunkIndexes; - std::vector<IoHash> CidChunkHashes; - std::vector<size_t> FileChunkIndexes; - std::vector<std::filesystem::path> FileChunkPaths; + eastl::vector<size_t> CidChunkIndexes; + eastl::vector<IoHash> CidChunkHashes; + eastl::vector<size_t> FileChunkIndexes; + eastl::vector<std::filesystem::path> FileChunkPaths; { RwLock::SharedLockScope OplogLock(m_OplogLock); for (size_t ChunkIndex = 0; ChunkIndex < ChunkIds.size(); ChunkIndex++) @@ -2288,14 +2288,14 @@ ProjectStore::Oplog::FindChunk(const Oid& ChunkId, uint64_t* OptOutModificationT return {}; } -std::vector<ProjectStore::Oplog::ChunkInfo> +eastl::vector<ProjectStore::Oplog::ChunkInfo> ProjectStore::Oplog::GetAllChunksInfo() { ZEN_MEMSCOPE(GetProjectstoreTag()); // First just capture all the chunk ids - std::vector<ChunkInfo> InfoArray; + eastl::vector<ChunkInfo> InfoArray; { RwLock::SharedLockScope _(m_OplogLock); @@ -2369,7 +2369,7 @@ ProjectStore::Oplog::IterateOplog(std::function<void(CbObjectView)>&& Handler, c template<typename ContainerElement> std::span<ContainerElement> -CreateSpanFromPaging(std::vector<ContainerElement>& Container, const ProjectStore::Oplog::Paging& Paging) +CreateSpanFromPaging(eastl::vector<ContainerElement>& Container, const ProjectStore::Oplog::Paging& Paging) { std::span<ContainerElement> Span(Container); int32_t Size = int32_t(Container.size()); @@ -2388,7 +2388,7 @@ ProjectStore::Oplog::IterateOplogLocked(std::function<void(CbObjectView)>&& Hand return; } - std::vector<OplogEntryAddress> Entries; + eastl::vector<OplogEntryAddress> Entries; Entries.reserve(m_LatestOpMap.size()); for (const auto& Kv : m_LatestOpMap) @@ -2410,7 +2410,7 @@ ProjectStore::Oplog::IterateOplogLocked(std::function<void(CbObjectView)>&& Hand static constexpr uint32_t OplogMetaDataExpectedMagic = 0x6f'74'6d'62; // 'omta'; void -ProjectStore::Oplog::GetAttachmentsLocked(std::vector<IoHash>& OutAttachments, bool StoreMetaDataOnDisk) +ProjectStore::Oplog::GetAttachmentsLocked(eastl::vector<IoHash>& OutAttachments, bool StoreMetaDataOnDisk) { ZEN_MEMSCOPE(GetProjectstoreTag()); ZEN_TRACE_CPU("Store::Oplog::GetAttachmentsLocked"); @@ -2439,9 +2439,9 @@ ProjectStore::Oplog::GetAttachmentsLocked(std::vector<IoHash>& OutAttachments, b } } - std::vector<Oid> Keys; - std::vector<uint32_t> AttachmentCounts; - size_t AttachmentOffset = OutAttachments.size(); + eastl::vector<Oid> Keys; + eastl::vector<uint32_t> AttachmentCounts; + size_t AttachmentOffset = OutAttachments.size(); IterateOplogLocked( [&](CbObjectView Op) { @@ -2513,17 +2513,17 @@ ProjectStore::Oplog::IterateOplogWithKey(std::function<void(uint32_t, const Oid& return; } - std::vector<OplogEntryAddress> SortedEntries; - std::vector<Oid> SortedKeys; - std::vector<uint32_t> SortedLSNs; + eastl::vector<OplogEntryAddress> SortedEntries; + eastl::vector<Oid> SortedKeys; + eastl::vector<uint32_t> SortedLSNs; { const auto TargetEntryCount = m_LatestOpMap.size(); - std::vector<size_t> EntryIndexes; - std::vector<OplogEntryAddress> Entries; - std::vector<Oid> Keys; - std::vector<uint32_t> LSNs; + eastl::vector<size_t> EntryIndexes; + eastl::vector<OplogEntryAddress> Entries; + eastl::vector<Oid> Keys; + eastl::vector<uint32_t> LSNs; Entries.reserve(TargetEntryCount); EntryIndexes.reserve(TargetEntryCount); @@ -2653,8 +2653,8 @@ ProjectStore::Oplog::EnableUpdateCapture() { ZEN_ASSERT(!m_CapturedLSNs); ZEN_ASSERT(!m_CapturedAttachments); - m_CapturedLSNs = std::make_unique<std::vector<uint32_t>>(); - m_CapturedAttachments = std::make_unique<std::vector<IoHash>>(); + m_CapturedLSNs = std::make_unique<eastl::vector<uint32_t>>(); + m_CapturedAttachments = std::make_unique<eastl::vector<IoHash>>(); } else { @@ -2704,7 +2704,7 @@ ProjectStore::Oplog::IterateCapturedLSNsLocked(std::function<bool(const CbObject } } -std::vector<IoHash> +eastl::vector<IoHash> ProjectStore::Oplog::GetCapturedAttachmentsLocked() { ZEN_MEMSCOPE(GetProjectstoreTag()); @@ -2716,7 +2716,7 @@ ProjectStore::Oplog::GetCapturedAttachmentsLocked() return {}; } -std::vector<IoHash> +eastl::vector<IoHash> ProjectStore::Oplog::CheckPendingChunkReferences(std::span<const IoHash> ChunkHashes, const GcClock::Duration& RetainTime) { ZEN_MEMSCOPE(GetProjectstoreTag()); @@ -2735,7 +2735,7 @@ ProjectStore::Oplog::CheckPendingChunkReferences(std::span<const IoHash> ChunkHa } }); - std::vector<IoHash> MissingChunks; + eastl::vector<IoHash> MissingChunks; MissingChunks.reserve(ChunkHashes.size()); for (const IoHash& FileHash : ChunkHashes) { @@ -2769,12 +2769,12 @@ ProjectStore::Oplog::RemovePendingChunkReferences(std::span<const IoHash> ChunkH }); } -std::vector<IoHash> +eastl::vector<IoHash> ProjectStore::Oplog::GetPendingChunkReferencesLocked() { ZEN_MEMSCOPE(GetProjectstoreTag()); - std::vector<IoHash> Result; + eastl::vector<IoHash> Result; Result.reserve(m_PendingPrepOpAttachments.size()); Result.insert(Result.end(), m_PendingPrepOpAttachments.begin(), m_PendingPrepOpAttachments.end()); GcClock::TimePoint Now = GcClock::Now(); @@ -2982,9 +2982,9 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbPackage OpPackage) if (!Attachments.empty()) { - std::vector<IoBuffer> WriteAttachmentBuffers; - std::vector<IoHash> WriteRawHashes; - std::vector<uint64_t> WriteRawSizes; + eastl::vector<IoBuffer> WriteAttachmentBuffers; + eastl::vector<IoHash> WriteRawHashes; + eastl::vector<uint64_t> WriteRawSizes; WriteAttachmentBuffers.reserve(Attachments.size()); WriteRawHashes.reserve(Attachments.size()); @@ -3002,7 +3002,7 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbPackage OpPackage) AttachmentBytes += AttachmentSize; } - std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + eastl::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); for (size_t Index = 0; Index < InsertResults.size(); Index++) { if (InsertResults[Index].New) @@ -3059,7 +3059,7 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbObjectView Core) return EntryId; } -std::vector<uint32_t> +eastl::vector<uint32_t> ProjectStore::Oplog::AppendNewOplogEntries(std::span<CbObjectView> Cores) { ZEN_MEMSCOPE(GetProjectstoreTag()); @@ -3070,12 +3070,12 @@ ProjectStore::Oplog::AppendNewOplogEntries(std::span<CbObjectView> Cores) RefPtr<OplogStorage> Storage = GetStorage(); if (!Storage) { - return std::vector<uint32_t>(Cores.size(), 0xffffffffu); + return eastl::vector<uint32_t>(Cores.size(), 0xffffffffu); } - size_t OpCount = Cores.size(); - std::vector<OplogEntryMapping> Mappings; - std::vector<OplogStorage::AppendOpData> OpDatas; + size_t OpCount = Cores.size(); + eastl::vector<OplogEntryMapping> Mappings; + eastl::vector<OplogStorage::AppendOpData> OpDatas; Mappings.resize(OpCount); OpDatas.resize(OpCount); @@ -3086,9 +3086,9 @@ ProjectStore::Oplog::AppendNewOplogEntries(std::span<CbObjectView> Cores) Mappings[OpIndex] = GetMapping(Core); } - std::vector<OplogEntry> OpEntries = Storage->AppendOps(OpDatas); + eastl::vector<OplogEntry> OpEntries = Storage->AppendOps(OpDatas); - std::vector<uint32_t> EntryIds; + eastl::vector<uint32_t> EntryIds; EntryIds.resize(OpCount); { { @@ -3227,7 +3227,7 @@ ProjectStore::Project::ReadAccessTimes() uint64_t Count = Reader["count"sv].AsUInt64(0); if (Count > 0) { - std::vector<uint64_t> Ticks; + eastl::vector<uint64_t> Ticks; Ticks.reserve(Count); CbArrayView TicksArray = Reader["ticks"sv].AsArrayView(); for (CbFieldView& TickView : TicksArray) @@ -3513,13 +3513,13 @@ ProjectStore::Project::DeleteOplog(std::string_view OplogId) return true; } -std::vector<std::string> +eastl::vector<std::string> ProjectStore::Project::ScanForOplogs() const { ZEN_MEMSCOPE(GetProjectstoreTag()); RwLock::SharedLockScope _(m_ProjectLock); - std::vector<std::string> Oplogs; + eastl::vector<std::string> Oplogs; if (Project::Exists(m_OplogStoragePath)) { DirectoryContent DirContent; @@ -3577,7 +3577,7 @@ ProjectStore::Project::ScrubStorage(ScrubContext& Ctx) { ZEN_MEMSCOPE(GetProjectstoreTag()); // Scrubbing needs to check all existing oplogs - std::vector<std::string> OpLogs = ScanForOplogs(); + eastl::vector<std::string> OpLogs = ScanForOplogs(); for (const std::string& OpLogId : OpLogs) { OpenOplog(OpLogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); @@ -3617,7 +3617,7 @@ ProjectStore::Project::TotalSize() const ZEN_MEMSCOPE(GetProjectstoreTag()); uint64_t Result = TotalSize(m_OplogStoragePath); { - std::vector<std::string> OpLogs = ScanForOplogs(); + eastl::vector<std::string> OpLogs = ScanForOplogs(); for (const std::string& OpLogId : OpLogs) { std::filesystem::path OplogBasePath = BasePathForOplog(OpLogId); @@ -3658,7 +3658,7 @@ ProjectStore::Project::EnableUpdateCapture() if (m_UpdateCaptureRefCounter == 0) { ZEN_ASSERT(!m_CapturedOplogs); - m_CapturedOplogs = std::make_unique<std::vector<std::string>>(); + m_CapturedOplogs = std::make_unique<eastl::vector<std::string>>(); } else { @@ -3683,7 +3683,7 @@ ProjectStore::Project::DisableUpdateCapture() }); } -std::vector<std::string> +eastl::vector<std::string> ProjectStore::Project::GetCapturedOplogsLocked() { ZEN_MEMSCOPE(GetProjectstoreTag()); @@ -3694,11 +3694,11 @@ ProjectStore::Project::GetCapturedOplogsLocked() return {}; } -std::vector<RwLock::SharedLockScope> +eastl::vector<RwLock::SharedLockScope> ProjectStore::Project::GetGcReferencerLocks() { ZEN_MEMSCOPE(GetProjectstoreTag()); - std::vector<RwLock::SharedLockScope> Locks; + eastl::vector<RwLock::SharedLockScope> Locks; Locks.emplace_back(RwLock::SharedLockScope(m_ProjectLock)); Locks.reserve(1 + m_Oplogs.size()); for (auto& Kv : m_Oplogs) @@ -3907,7 +3907,7 @@ ProjectStore::Flush() ZEN_TRACE_CPU("Store::Flush"); ZEN_INFO("flushing project store at '{}'", m_ProjectBasePath); - std::vector<Ref<Project>> Projects; + eastl::vector<Ref<Project>> Projects; { RwLock::SharedLockScope _(m_ProjectsLock); Projects.reserve(m_Projects.size()); @@ -3949,7 +3949,7 @@ ProjectStore::ScrubStorage(ScrubContext& Ctx) DiscoverProjects(); - std::vector<Ref<Project>> Projects; + eastl::vector<Ref<Project>> Projects; { RwLock::SharedLockScope Lock(m_ProjectsLock); Projects.reserve(m_Projects.size()); @@ -4227,11 +4227,11 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId, const bool WantsRawSizeField = WantsAllFields || WantedFieldNames.contains("rawsize"); const bool WantsSizeField = WantsAllFields || WantedFieldNames.contains("size"); - std::vector<Oid> Ids; - std::vector<std::string> ServerPaths; - std::vector<std::string> ClientPaths; - std::vector<uint64_t> Sizes; - std::vector<uint64_t> RawSizes; + eastl::vector<Oid> Ids; + eastl::vector<std::string> ServerPaths; + eastl::vector<std::string> ClientPaths; + eastl::vector<uint64_t> Sizes; + eastl::vector<uint64_t> RawSizes; size_t Count = 0; FoundLog->IterateFileMap([&](const Oid& Id, const std::string_view& ServerPath, const std::string_view& ClientPath) { @@ -4390,10 +4390,10 @@ ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId, const bool WantsRawSizeField = WantsAllFields || WantedFieldNames.contains("rawsize"); const bool WantsSizeField = WantsAllFields || WantedFieldNames.contains("size"); - std::vector<Oid> Ids; - std::vector<IoHash> Hashes; - std::vector<uint64_t> RawSizes; - std::vector<uint64_t> Sizes; + eastl::vector<Oid> Ids; + eastl::vector<IoHash> Hashes; + eastl::vector<uint64_t> RawSizes; + eastl::vector<uint64_t> Sizes; size_t Count = 0; size_t EstimatedCount = FoundLog->OplogCount(); @@ -4866,7 +4866,7 @@ ProjectStore::PutChunk(const std::string_view ProjectId, return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid payload format for chunk '{}'", Cid)}; } - FoundLog->CaptureAddedAttachments(std::vector<IoHash>{Hash}); + FoundLog->CaptureAddedAttachments(eastl::vector<IoHash>{Hash}); CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk, Hash); return {Result.New ? HttpResponseCode::Created : HttpResponseCode::OK, {}}; } @@ -4949,16 +4949,16 @@ ProjectStore::GetChunks(const std::string_view ProjectId, } Output; }; - std::vector<Request> Requests; - size_t RequestCount = ChunksArray.Num(); + eastl::vector<Request> Requests; + size_t RequestCount = ChunksArray.Num(); if (RequestCount > 0) { Requests.reserve(RequestCount); - std::vector<IoHash> ChunkRawHashes; - std::vector<size_t> ChunkRawHashesRequestIndex; - std::vector<Oid> ChunkIds; - std::vector<size_t> ChunkIdsRequestIndex; - bool DoBatch = RequestCount > 1; + eastl::vector<IoHash> ChunkRawHashes; + eastl::vector<size_t> ChunkRawHashesRequestIndex; + eastl::vector<Oid> ChunkIds; + eastl::vector<size_t> ChunkIdsRequestIndex; + bool DoBatch = RequestCount > 1; if (DoBatch) { ChunkRawHashes.reserve(RequestCount); @@ -5251,7 +5251,7 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie tsl::robin_set<IoHash, IoHash::Hasher> Attachments; auto HasAttachment = [&ChunkStore](const IoHash& RawHash) { return ChunkStore.ContainsChunk(RawHash); }; - auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector<IoHash>&& ChunkHashes) { + auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const eastl::vector<IoHash>&& ChunkHashes) { RwLock::ExclusiveLockScope _(AttachmentsLock); if (BlockHash != IoHash::Zero) { @@ -5366,7 +5366,7 @@ ProjectStore::ReadOplog(const std::string_view ProjectId, /* AllowChunking*/ false, [](CompressedBuffer&&, ChunkBlockDescription&&) {}, [](const IoHash&, TGetAttachmentBufferFunc&&) {}, - [](std::vector<std::pair<IoHash, FetchChunkFunc>>&&) {}, + [](eastl::vector<std::pair<IoHash, FetchChunkFunc>>&&) {}, /* EmbedLooseFiles*/ false); OutResponse = std::move(ContainerResult.ContainerObject); @@ -5530,8 +5530,8 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, std::span<const CbAttachment> Attachments = Package.GetAttachments(); if (!Attachments.empty()) { - std::vector<IoBuffer> WriteAttachmentBuffers; - std::vector<IoHash> WriteRawHashes; + eastl::vector<IoBuffer> WriteAttachmentBuffers; + eastl::vector<IoHash> WriteRawHashes; WriteAttachmentBuffers.reserve(Attachments.size()); WriteRawHashes.reserve(Attachments.size()); @@ -5568,7 +5568,7 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, uint64_t TotalBytes = 0; uint64_t TotalFiles = 0; - std::vector<CbObject> NewOps; + eastl::vector<CbObject> NewOps; struct AddedChunk { IoBuffer Buffer; @@ -5865,7 +5865,7 @@ ProjectStore::EnableUpdateCapture() if (m_UpdateCaptureRefCounter == 0) { ZEN_ASSERT(!m_CapturedProjects); - m_CapturedProjects = std::make_unique<std::vector<std::string>>(); + m_CapturedProjects = std::make_unique<eastl::vector<std::string>>(); } else { @@ -5889,7 +5889,7 @@ ProjectStore::DisableUpdateCapture() }); } -std::vector<std::string> +eastl::vector<std::string> ProjectStore::GetCapturedProjectsLocked() { ZEN_MEMSCOPE(GetProjectstoreTag()); @@ -5911,10 +5911,10 @@ ProjectStore::GetGcName(GcCtx&) class ProjectStoreGcStoreCompactor : public GcStoreCompactor { public: - ProjectStoreGcStoreCompactor(ProjectStore& ProjectStore, - const std::filesystem::path& BasePath, - std::vector<std::filesystem::path>&& OplogPathsToRemove, - std::vector<std::filesystem::path>&& ProjectPathsToRemove) + ProjectStoreGcStoreCompactor(ProjectStore& ProjectStore, + const std::filesystem::path& BasePath, + eastl::vector<std::filesystem::path>&& OplogPathsToRemove, + eastl::vector<std::filesystem::path>&& ProjectPathsToRemove) : m_ProjectStore(ProjectStore) , m_BasePath(BasePath) , m_OplogPathsToRemove(std::move(OplogPathsToRemove)) @@ -5982,7 +5982,7 @@ public: for (auto ProjectIt : m_ProjectStore.m_Projects) { Ref<ProjectStore::Project> Project = ProjectIt.second; - std::vector<std::string> OplogsToCompact = Project->GetOplogsToCompact(); + eastl::vector<std::string> OplogsToCompact = Project->GetOplogsToCompact(); CompactOplogCount += OplogsToCompact.size(); for (const std::string& OplogId : OplogsToCompact) { @@ -6040,10 +6040,10 @@ public: virtual std::string GetGcName(GcCtx&) override { return fmt::format("projectstore: '{}'", m_BasePath.string()); } private: - ProjectStore& m_ProjectStore; - std::filesystem::path m_BasePath; - std::vector<std::filesystem::path> m_OplogPathsToRemove; - std::vector<std::filesystem::path> m_ProjectPathsToRemove; + ProjectStore& m_ProjectStore; + std::filesystem::path m_BasePath; + eastl::vector<std::filesystem::path> m_OplogPathsToRemove; + eastl::vector<std::filesystem::path> m_ProjectPathsToRemove; }; GcStoreCompactor* @@ -6068,11 +6068,11 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); - std::vector<std::filesystem::path> OplogPathsToRemove; - std::vector<std::filesystem::path> ProjectPathsToRemove; + eastl::vector<std::filesystem::path> OplogPathsToRemove; + eastl::vector<std::filesystem::path> ProjectPathsToRemove; - std::vector<Ref<Project>> ExpiredProjects; - std::vector<Ref<Project>> Projects; + eastl::vector<Ref<Project>> ExpiredProjects; + eastl::vector<Ref<Project>> Projects; DiscoverProjects(); @@ -6098,9 +6098,9 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) break; } - std::vector<std::string> ExpiredOplogs; + eastl::vector<std::string> ExpiredOplogs; - std::vector<std::string> OpLogs = Project->ScanForOplogs(); + eastl::vector<std::string> OpLogs = Project->ScanForOplogs(); for (const std::string& OplogId : OpLogs) { Stats.CheckedCount++; @@ -6199,7 +6199,7 @@ public: Stopwatch Timer; - std::vector<ProjectStore::Oplog*> AddedOplogs; + eastl::vector<ProjectStore::Oplog*> AddedOplogs; const auto _ = MakeGuard([&] { if (!Ctx.Settings.Verbose) @@ -6213,7 +6213,7 @@ public: AddedOplogs.size()); }); - std::vector<std::string> AddedProjects = m_ProjectStore.GetCapturedProjectsLocked(); + eastl::vector<std::string> AddedProjects = m_ProjectStore.GetCapturedProjectsLocked(); for (const std::string& AddedProject : AddedProjects) { if (auto It = m_ProjectStore.m_Projects.find(AddedProject); It != m_ProjectStore.m_Projects.end()) @@ -6230,7 +6230,7 @@ public: { ProjectStore::Project& Project = *ProjectPair.second; - std::vector<std::string> AddedOplogNames(Project.GetCapturedOplogsLocked()); + eastl::vector<std::string> AddedOplogNames(Project.GetCapturedOplogsLocked()); for (const std::string& OplogName : AddedOplogNames) { if (auto It = Project.m_Oplogs.find(OplogName); It != Project.m_Oplogs.end()) @@ -6259,7 +6259,7 @@ public: }); Oplog->GetAttachmentsLocked(m_References, Ctx.Settings.StoreProjectAttachmentMetaData); - if (std::vector<IoHash> PendingChunkReferences = Oplog->GetPendingChunkReferencesLocked(); !PendingChunkReferences.empty()) + if (eastl::vector<IoHash> PendingChunkReferences = Oplog->GetPendingChunkReferencesLocked(); !PendingChunkReferences.empty()) { m_References.insert(m_References.end(), PendingChunkReferences.begin(), PendingChunkReferences.end()); } @@ -6296,8 +6296,8 @@ public: } private: - ProjectStore& m_ProjectStore; - std::vector<IoHash> m_References; + ProjectStore& m_ProjectStore; + eastl::vector<IoHash> m_References; }; class ProjectStoreOplogReferenceChecker : public GcReferenceChecker @@ -6429,9 +6429,9 @@ public: UpdateOp.IterateAttachments([&](CbFieldView Visitor) { m_AddedReferences.emplace_back(Visitor.AsAttachment()); }); return true; }); - std::vector<IoHash> AddedAttachments = Oplog->GetCapturedAttachmentsLocked(); + eastl::vector<IoHash> AddedAttachments = Oplog->GetCapturedAttachmentsLocked(); m_AddedReferences.insert(m_AddedReferences.end(), AddedAttachments.begin(), AddedAttachments.end()); - if (std::vector<IoHash> PendingChunkReferences = Oplog->GetPendingChunkReferencesLocked(); !PendingChunkReferences.empty()) + if (eastl::vector<IoHash> PendingChunkReferences = Oplog->GetPendingChunkReferencesLocked(); !PendingChunkReferences.empty()) { m_AddedReferences.insert(m_AddedReferences.end(), PendingChunkReferences.begin(), PendingChunkReferences.end()); } @@ -6487,12 +6487,12 @@ public: std::string m_OplogId; std::filesystem::path m_OplogBasePath; ProjectStore::Oplog* m_OplogWithUpdateCapture = nullptr; - std::vector<IoHash> m_References; - std::vector<IoHash> m_AddedReferences; + eastl::vector<IoHash> m_References; + eastl::vector<IoHash> m_AddedReferences; GcClock::TimePoint m_OplogAccessTime; }; -std::vector<GcReferenceChecker*> +eastl::vector<GcReferenceChecker*> ProjectStore::CreateReferenceCheckers(GcCtx& Ctx) { ZEN_TRACE_CPU("Store::CreateReferenceCheckers"); @@ -6517,8 +6517,8 @@ ProjectStore::CreateReferenceCheckers(GcCtx& Ctx) DiscoverProjects(); - std::vector<Ref<ProjectStore::Project>> Projects; - std::vector<GcReferenceChecker*> Checkers; + eastl::vector<Ref<ProjectStore::Project>> Projects; + eastl::vector<GcReferenceChecker*> Checkers; Checkers.emplace_back(new ProjectStoreReferenceChecker(*this)); { RwLock::SharedLockScope Lock(m_ProjectsLock); @@ -6534,7 +6534,7 @@ ProjectStore::CreateReferenceCheckers(GcCtx& Ctx) { for (const Ref<ProjectStore::Project>& Project : Projects) { - std::vector<std::string> OpLogs = Project->ScanForOplogs(); + eastl::vector<std::string> OpLogs = Project->ScanForOplogs(); Checkers.reserve(Checkers.size() + OpLogs.size()); for (const std::string& OpLogId : OpLogs) { @@ -6556,18 +6556,18 @@ ProjectStore::CreateReferenceCheckers(GcCtx& Ctx) return Checkers; } -std::vector<RwLock::SharedLockScope> +eastl::vector<RwLock::SharedLockScope> ProjectStore::LockState(GcCtx& Ctx) { ZEN_TRACE_CPU("Store::LockState"); auto Log = [&Ctx]() { return Ctx.Logger; }; - std::vector<RwLock::SharedLockScope> Locks; + eastl::vector<RwLock::SharedLockScope> Locks; Locks.emplace_back(RwLock::SharedLockScope(m_ProjectsLock)); for (auto& ProjectIt : m_Projects) { - std::vector<RwLock::SharedLockScope> ProjectLocks = ProjectIt.second->GetGcReferencerLocks(); + eastl::vector<RwLock::SharedLockScope> ProjectLocks = ProjectIt.second->GetGcReferencerLocks(); for (auto It = std::make_move_iterator(ProjectLocks.begin()); It != std::make_move_iterator(ProjectLocks.end()); It++) { Locks.emplace_back(std::move(*It)); @@ -6675,7 +6675,7 @@ public: std::string m_OplogId; }; -std::vector<GcReferenceValidator*> +eastl::vector<GcReferenceValidator*> ProjectStore::CreateReferenceValidators(GcCtx& Ctx) { if (Ctx.Settings.SkipCidDelete) @@ -6684,20 +6684,20 @@ ProjectStore::CreateReferenceValidators(GcCtx& Ctx) } DiscoverProjects(); - std::vector<std::pair<std::string, std::string>> Oplogs; + eastl::vector<std::pair<std::string, std::string>> Oplogs; { RwLock::SharedLockScope _(m_ProjectsLock); for (auto& ProjectPair : m_Projects) { - ProjectStore::Project& Project = *ProjectPair.second; - std::vector<std::string> OpLogs = Project.ScanForOplogs(); + ProjectStore::Project& Project = *ProjectPair.second; + eastl::vector<std::string> OpLogs = Project.ScanForOplogs(); for (const std::string& OplogName : OpLogs) { Oplogs.push_back({Project.Identifier, OplogName}); } } } - std::vector<GcReferenceValidator*> Validators; + eastl::vector<GcReferenceValidator*> Validators; Validators.reserve(Oplogs.size()); for (const std::pair<std::string, std::string>& Oplog : Oplogs) { @@ -6788,12 +6788,12 @@ namespace testutils { return Package; }; - std::vector<std::pair<Oid, CompressedBuffer>> CreateAttachments( + eastl::vector<std::pair<Oid, CompressedBuffer>> CreateAttachments( const std::span<const size_t>& Sizes, OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, uint64_t BlockSize = 0) { - std::vector<std::pair<Oid, CompressedBuffer>> Result; + eastl::vector<std::pair<Oid, CompressedBuffer>> Result; Result.reserve(Sizes.size()); for (size_t Size : Sizes) { @@ -6821,11 +6821,11 @@ namespace testutils { } template<typename ChunkType> - CbObject BuildChunksRequest(bool SkipData, - std::string_view IdName, - const std::vector<ChunkType>& Chunks, - const std::vector<std::pair<size_t, size_t>>& Ranges, - const std::vector<uint64_t>& ModTags) + CbObject BuildChunksRequest(bool SkipData, + std::string_view IdName, + const eastl::vector<ChunkType>& Chunks, + const eastl::vector<std::pair<size_t, size_t>>& Ranges, + const eastl::vector<uint64_t>& ModTags) { CbObjectWriter Request; Request.BeginObject("Request"sv); @@ -6861,18 +6861,18 @@ namespace testutils { return Request.Save(); }; - CbObject BuildChunksRequest(bool SkipData, - const std::vector<Oid>& Chunks, - const std::vector<std::pair<size_t, size_t>>& Ranges, - const std::vector<uint64_t>& ModTags) + CbObject BuildChunksRequest(bool SkipData, + const eastl::vector<Oid>& Chunks, + const eastl::vector<std::pair<size_t, size_t>>& Ranges, + const eastl::vector<uint64_t>& ModTags) { return BuildChunksRequest<Oid>(SkipData, "Oid", Chunks, Ranges, ModTags); } - CbObject BuildChunksRequest(bool SkipData, - const std::vector<IoHash>& Chunks, - const std::vector<std::pair<size_t, size_t>>& Ranges, - const std::vector<uint64_t>& ModTags) + CbObject BuildChunksRequest(bool SkipData, + const eastl::vector<IoHash>& Chunks, + const eastl::vector<std::pair<size_t, size_t>>& Ranges, + const eastl::vector<uint64_t>& ModTags) { return BuildChunksRequest<IoHash>(SkipData, "RawHash", Chunks, Ranges, ModTags); } @@ -7360,8 +7360,8 @@ TEST_CASE("project.store.gc.prep") OplogFile.Open(Project1OplogPath, BasicFile::Mode::kTruncate); } - std::vector<std::pair<Oid, CompressedBuffer>> OpAttachments = CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99}); - std::vector<IoHash> OpChunkHashes; + eastl::vector<std::pair<Oid, CompressedBuffer>> OpAttachments = CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99}); + eastl::vector<IoHash> OpChunkHashes; for (const auto& Chunk : OpAttachments) { OpChunkHashes.push_back(Chunk.second.DecodeRawHash()); @@ -7569,11 +7569,11 @@ TEST_CASE("project.store.rpc.getchunks") ProjectFile.Open(Project1FilePath, BasicFile::Mode::kTruncate); } - std::vector<Oid> OpIds; + eastl::vector<Oid> OpIds; OpIds.insert(OpIds.end(), {Oid::NewOid(), Oid::NewOid(), Oid::NewOid(), Oid::NewOid()}); - std::unordered_map<Oid, std::vector<std::pair<Oid, CompressedBuffer>>, Oid::Hasher> Attachments; - Oid FilesOpId = Oid::NewOid(); - std::vector<std::pair<Oid, std::filesystem::path>> FilesOpIdAttachments; + std::unordered_map<Oid, eastl::vector<std::pair<Oid, CompressedBuffer>>, Oid::Hasher> Attachments; + Oid FilesOpId = Oid::NewOid(); + eastl::vector<std::pair<Oid, std::filesystem::path>> FilesOpIdAttachments; { Ref<ProjectStore::Project> Project1(ProjectStore.NewProject(BasePath / "proj1"sv, "proj1"sv, @@ -7616,7 +7616,7 @@ TEST_CASE("project.store.rpc.getchunks") CbPackage Response; auto Result = ProjectStore.GetChunks("proj1"sv, "oplog1"sv, - testutils::BuildChunksRequest(/*SkipData*/ false, std::vector<IoHash>{}, {}, {}), + testutils::BuildChunksRequest(/*SkipData*/ false, eastl::vector<IoHash>{}, {}, {}), Response); CHECK_EQ(HttpResponseCode::OK, Result.first); CHECK_EQ(0, Response.GetAttachments().size()); @@ -8151,13 +8151,13 @@ TEST_CASE("project.store.rpc.getchunks") // Multi RawHash Request { - std::vector<CompressedBuffer> AttachmentBuffers{Attachments[OpIds[1]][0].second, - Attachments[OpIds[2]][0].second, - Attachments[OpIds[2]][1].second}; - std::vector<IoHash> AttachmentHashes{AttachmentBuffers[0].DecodeRawHash(), - AttachmentBuffers[1].DecodeRawHash(), - AttachmentBuffers[2].DecodeRawHash()}; - std::vector<uint64_t> ResponseModTags(3, 0); + eastl::vector<CompressedBuffer> AttachmentBuffers{Attachments[OpIds[1]][0].second, + Attachments[OpIds[2]][0].second, + Attachments[OpIds[2]][1].second}; + eastl::vector<IoHash> AttachmentHashes{AttachmentBuffers[0].DecodeRawHash(), + AttachmentBuffers[1].DecodeRawHash(), + AttachmentBuffers[2].DecodeRawHash()}; + eastl::vector<uint64_t> ResponseModTags(3, 0); { // Fresh fetch CbPackage Response; @@ -8274,7 +8274,7 @@ TEST_CASE("project.store.rpc.getchunks") } { // Modtime query with mismatching ModTags - std::vector<uint64_t> MismatchingModTags(ResponseModTags); + eastl::vector<uint64_t> MismatchingModTags(ResponseModTags); for (uint64_t& Tag : MismatchingModTags) { Tag++; @@ -8307,14 +8307,14 @@ TEST_CASE("project.store.rpc.getchunks") } // Multi Id Request { - std::vector<CompressedBuffer> AttachmentBuffers{Attachments[OpIds[1]][0].second, - Attachments[OpIds[2]][0].second, - Attachments[OpIds[2]][1].second}; - std::vector<IoHash> AttachmentHashes{AttachmentBuffers[0].DecodeRawHash(), - AttachmentBuffers[1].DecodeRawHash(), - AttachmentBuffers[2].DecodeRawHash()}; - std::vector<Oid> AttachedIds{Attachments[OpIds[1]][0].first, Attachments[OpIds[2]][0].first, Attachments[OpIds[2]][1].first}; - std::vector<uint64_t> ResponseModTags(3, 0); + eastl::vector<CompressedBuffer> AttachmentBuffers{Attachments[OpIds[1]][0].second, + Attachments[OpIds[2]][0].second, + Attachments[OpIds[2]][1].second}; + eastl::vector<IoHash> AttachmentHashes{AttachmentBuffers[0].DecodeRawHash(), + AttachmentBuffers[1].DecodeRawHash(), + AttachmentBuffers[2].DecodeRawHash()}; + eastl::vector<Oid> AttachedIds{Attachments[OpIds[1]][0].first, Attachments[OpIds[2]][0].first, Attachments[OpIds[2]][1].first}; + eastl::vector<uint64_t> ResponseModTags(3, 0); { // Fresh fetch CbPackage Response; @@ -8431,7 +8431,7 @@ TEST_CASE("project.store.rpc.getchunks") } { // Modtime query with mismatching ModTags - std::vector<uint64_t> MismatchingModTags(ResponseModTags); + eastl::vector<uint64_t> MismatchingModTags(ResponseModTags); for (uint64_t& Tag : MismatchingModTags) { Tag++; @@ -8491,9 +8491,9 @@ TEST_CASE("project.store.partial.read") ProjectFile.Open(Project1FilePath, BasicFile::Mode::kTruncate); } - std::vector<Oid> OpIds; + eastl::vector<Oid> OpIds; OpIds.insert(OpIds.end(), {Oid::NewOid(), Oid::NewOid(), Oid::NewOid(), Oid::NewOid()}); - std::unordered_map<Oid, std::vector<std::pair<Oid, CompressedBuffer>>, Oid::Hasher> Attachments; + std::unordered_map<Oid, eastl::vector<std::pair<Oid, CompressedBuffer>>, Oid::Hasher> Attachments; { Ref<ProjectStore::Project> Project1(ProjectStore.NewProject(BasePath / "proj1"sv, "proj1"sv, @@ -8629,12 +8629,12 @@ TEST_CASE("project.store.block") using namespace std::literals; using namespace testutils; - std::vector<std::size_t> AttachmentSizes({7633, 6825, 5738, 8031, 7225, 566, 3656, 6006, 24, 3466, 1093, 4269, 2257, 3685, 3489, - 7194, 6151, 5482, 6217, 3511, 6738, 5061, 7537, 2759, 1916, 8210, 2235, 4024, 1582, 5251, - 491, 5464, 4607, 8135, 3767, 4045, 4415, 5007, 8876, 6761, 3359, 8526, 4097, 4855, 8225}); + eastl::vector<std::size_t> AttachmentSizes({7633, 6825, 5738, 8031, 7225, 566, 3656, 6006, 24, 3466, 1093, 4269, 2257, 3685, 3489, + 7194, 6151, 5482, 6217, 3511, 6738, 5061, 7537, 2759, 1916, 8210, 2235, 4024, 1582, 5251, + 491, 5464, 4607, 8135, 3767, 4045, 4415, 5007, 8876, 6761, 3359, 8526, 4097, 4855, 8225}); - std::vector<std::pair<Oid, CompressedBuffer>> AttachmentsWithId = CreateAttachments(AttachmentSizes); - std::vector<std::pair<IoHash, FetchChunkFunc>> Chunks; + eastl::vector<std::pair<Oid, CompressedBuffer>> AttachmentsWithId = CreateAttachments(AttachmentSizes); + eastl::vector<std::pair<IoHash, FetchChunkFunc>> Chunks; Chunks.reserve(AttachmentSizes.size()); for (const auto& It : AttachmentsWithId) { diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index 8f2d3ce0d..b502bd6dc 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -107,8 +107,8 @@ public: int32_t Count = -1; }; - std::vector<ChunkInfo> GetAllChunksInfo(); - void IterateChunkMap(std::function<void(const Oid&, const IoHash& Hash)>&& Fn); + eastl::vector<ChunkInfo> GetAllChunksInfo(); + void IterateChunkMap(std::function<void(const Oid&, const IoHash& Hash)>&& Fn); void IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn); void IterateOplog(std::function<void(CbObjectView)>&& Fn, const Paging& EntryPaging); void IterateOplogWithKey(std::function<void(uint32_t, const Oid&, CbObjectView)>&& Fn); @@ -140,8 +140,8 @@ public: */ uint32_t AppendNewOplogEntry(CbPackage Op); - uint32_t AppendNewOplogEntry(CbObjectView Core); - std::vector<uint32_t> AppendNewOplogEntries(std::span<CbObjectView> Cores); + uint32_t AppendNewOplogEntry(CbObjectView Core); + eastl::vector<uint32_t> AppendNewOplogEntries(std::span<CbObjectView> Cores); enum UpdateType { @@ -171,20 +171,20 @@ public: void AddChunkMappings(const std::unordered_map<Oid, IoHash, Oid::Hasher>& ChunkMappings); - void EnableUpdateCapture(); - void DisableUpdateCapture(); - void CaptureAddedAttachments(std::span<const IoHash> AttachmentHashes); - std::vector<IoHash> GetCapturedAttachmentsLocked(); - std::vector<IoHash> CheckPendingChunkReferences(std::span<const IoHash> ChunkHashes, const GcClock::Duration& RetainTime); - void RemovePendingChunkReferences(std::span<const IoHash> ChunkHashes); - std::vector<IoHash> GetPendingChunkReferencesLocked(); + void EnableUpdateCapture(); + void DisableUpdateCapture(); + void CaptureAddedAttachments(std::span<const IoHash> AttachmentHashes); + eastl::vector<IoHash> GetCapturedAttachmentsLocked(); + eastl::vector<IoHash> CheckPendingChunkReferences(std::span<const IoHash> ChunkHashes, const GcClock::Duration& RetainTime); + void RemovePendingChunkReferences(std::span<const IoHash> ChunkHashes); + eastl::vector<IoHash> GetPendingChunkReferencesLocked(); RwLock::SharedLockScope GetGcReferencerLock() { return RwLock::SharedLockScope(m_OplogLock); } uint32_t GetUnusedSpacePercent() const; void Compact(bool DryRun, bool RetainLSNs, std::string_view LogPrefix); - void GetAttachmentsLocked(std::vector<IoHash>& OutAttachments, bool StoreMetaDataOnDisk); + void GetAttachmentsLocked(eastl::vector<IoHash>& OutAttachments, bool StoreMetaDataOnDisk); Project* GetOuterProject() const { return m_OuterProject; } void CompactIfUnusedExceeds(bool DryRun, uint32_t CompactUnusedThreshold, std::string_view LogPrefix); @@ -207,14 +207,14 @@ public: struct ValidationResult { - uint32_t OpCount = 0; - uint32_t LSNLow = 0; - uint32_t LSNHigh = 0; - std::vector<std::pair<Oid, FileMapping>> MissingFiles; - std::vector<std::pair<Oid, ChunkMapping>> MissingChunks; - std::vector<std::pair<Oid, ChunkMapping>> MissingMetas; - std::vector<std::pair<Oid, IoHash>> MissingAttachments; - std::vector<std::pair<Oid, std::string>> OpKeys; + uint32_t OpCount = 0; + uint32_t LSNLow = 0; + uint32_t LSNHigh = 0; + eastl::vector<std::pair<Oid, FileMapping>> MissingFiles; + eastl::vector<std::pair<Oid, ChunkMapping>> MissingChunks; + eastl::vector<std::pair<Oid, ChunkMapping>> MissingMetas; + eastl::vector<std::pair<Oid, IoHash>> MissingAttachments; + eastl::vector<std::pair<Oid, std::string>> OpKeys; bool IsEmpty() const { @@ -252,8 +252,8 @@ public: std::atomic<bool> m_MetaValid = false; uint32_t m_UpdateCaptureRefCounter = 0; - std::unique_ptr<std::vector<uint32_t>> m_CapturedLSNs; - std::unique_ptr<std::vector<IoHash>> m_CapturedAttachments; + std::unique_ptr<eastl::vector<uint32_t>> m_CapturedLSNs; + std::unique_ptr<eastl::vector<IoHash>> m_CapturedAttachments; std::unordered_set<IoHash, IoHash::Hasher> m_PendingPrepOpAttachments; GcClock::TimePoint m_PendingPrepOpAttachmentsRetainEnd; @@ -270,9 +270,9 @@ public: struct OplogEntryMapping { - std::vector<ChunkMapping> Chunks; - std::vector<ChunkMapping> Meta; - std::vector<FileMapping> Files; + eastl::vector<ChunkMapping> Chunks; + eastl::vector<ChunkMapping> Meta; + eastl::vector<FileMapping> Files; }; OplogEntryMapping GetMapping(CbObjectView Core); @@ -308,20 +308,20 @@ public: std::filesystem::path ProjectRootDir; std::filesystem::path ProjectFilePath; - Oplog* NewOplog(std::string_view OplogId, const std::filesystem::path& MarkerPath); - Oplog* OpenOplog(std::string_view OplogId, bool AllowCompact, bool VerifyPathOnDisk); - bool DeleteOplog(std::string_view OplogId); - bool RemoveOplog(std::string_view OplogId, std::filesystem::path& OutDeletePath); - void IterateOplogs(std::function<void(const RwLock::SharedLockScope&, const Oplog&)>&& Fn) const; - void IterateOplogs(std::function<void(const RwLock::SharedLockScope&, Oplog&)>&& Fn); - std::vector<std::string> ScanForOplogs() const; - bool IsExpired(const GcClock::TimePoint ExpireTime) const; - bool IsExpired(const GcClock::TimePoint ExpireTime, const ProjectStore::Oplog& Oplog) const; - bool IsExpired(const GcClock::TimePoint ExpireTime, std::string_view OplogId) const; - bool IsOplogTouchedSince(const GcClock::TimePoint TouchTime, std::string_view Oplog) const; - void TouchProject(); - void TouchOplog(std::string_view Oplog); - GcClock::TimePoint LastOplogAccessTime(std::string_view Oplog) const; + Oplog* NewOplog(std::string_view OplogId, const std::filesystem::path& MarkerPath); + Oplog* OpenOplog(std::string_view OplogId, bool AllowCompact, bool VerifyPathOnDisk); + bool DeleteOplog(std::string_view OplogId); + bool RemoveOplog(std::string_view OplogId, std::filesystem::path& OutDeletePath); + void IterateOplogs(std::function<void(const RwLock::SharedLockScope&, const Oplog&)>&& Fn) const; + void IterateOplogs(std::function<void(const RwLock::SharedLockScope&, Oplog&)>&& Fn); + eastl::vector<std::string> ScanForOplogs() const; + bool IsExpired(const GcClock::TimePoint ExpireTime) const; + bool IsExpired(const GcClock::TimePoint ExpireTime, const ProjectStore::Oplog& Oplog) const; + bool IsExpired(const GcClock::TimePoint ExpireTime, std::string_view OplogId) const; + bool IsOplogTouchedSince(const GcClock::TimePoint TouchTime, std::string_view Oplog) const; + void TouchProject(); + void TouchOplog(std::string_view Oplog); + GcClock::TimePoint LastOplogAccessTime(std::string_view Oplog) const; Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath); virtual ~Project(); @@ -336,19 +336,19 @@ public: uint64_t TotalSize() const; bool PrepareForDelete(std::filesystem::path& OutDeletePath); - void EnableUpdateCapture(); - void DisableUpdateCapture(); - std::vector<std::string> GetCapturedOplogsLocked(); + void EnableUpdateCapture(); + void DisableUpdateCapture(); + eastl::vector<std::string> GetCapturedOplogsLocked(); - std::vector<RwLock::SharedLockScope> GetGcReferencerLocks(); + eastl::vector<RwLock::SharedLockScope> GetGcReferencerLocks(); void AddOplogToCompact(std::string_view OplogId) { m_OplogsToCompactLock.WithExclusiveLock([&]() { m_OplogsToCompact.insert(std::string(OplogId)); }); } - std::vector<std::string> GetOplogsToCompact() + eastl::vector<std::string> GetOplogsToCompact() { - std::vector<std::string> Result; + eastl::vector<std::string> Result; m_OplogsToCompactLock.WithExclusiveLock([&]() { Result.reserve(m_OplogsToCompact.size()); Result.insert(Result.end(), m_OplogsToCompact.begin(), m_OplogsToCompact.end()); @@ -362,12 +362,12 @@ public: CidStore& m_CidStore; mutable RwLock m_ProjectLock; std::map<std::string, std::unique_ptr<Oplog>> m_Oplogs; - std::vector<std::unique_ptr<Oplog>> m_DeletedOplogs; + eastl::vector<std::unique_ptr<Oplog>> m_DeletedOplogs; std::filesystem::path m_OplogStoragePath; mutable RwLock m_LastAccessTimesLock; mutable tsl::robin_map<std::string, GcClock::Tick> m_LastAccessTimes; uint32_t m_UpdateCaptureRefCounter = 0; - std::unique_ptr<std::vector<std::string>> m_CapturedOplogs; + std::unique_ptr<eastl::vector<std::string>> m_CapturedOplogs; RwLock m_OplogsToCompactLock; std::unordered_set<std::string> m_OplogsToCompact; @@ -408,12 +408,12 @@ public: virtual void ScrubStorage(ScrubContext& Ctx) override; virtual GcStorageSize StorageSize() const override; - virtual std::string GetGcName(GcCtx& Ctx) override; - virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; - virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; - virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override; + virtual std::string GetGcName(GcCtx& Ctx) override; + virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; + virtual eastl::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; + virtual eastl::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override; - virtual std::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) override; + virtual eastl::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) override; CbArray GetProjectsList(); std::pair<HttpResponseCode, std::string> GetProjectFiles(const std::string_view ProjectId, @@ -492,23 +492,23 @@ public: bool AreDiskWritesAllowed() const; - void EnableUpdateCapture(); - void DisableUpdateCapture(); - std::vector<std::string> GetCapturedProjectsLocked(); + void EnableUpdateCapture(); + void DisableUpdateCapture(); + eastl::vector<std::string> GetCapturedProjectsLocked(); private: - LoggerRef m_Log; - GcManager& m_Gc; - CidStore& m_CidStore; - JobQueue& m_JobQueue; - OpenProcessCache& m_OpenProcessCache; - std::filesystem::path m_ProjectBasePath; - const Configuration m_Config; - mutable RwLock m_ProjectsLock; - std::map<std::string, Ref<Project>> m_Projects; - const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; - uint32_t m_UpdateCaptureRefCounter = 0; - std::unique_ptr<std::vector<std::string>> m_CapturedProjects; + LoggerRef m_Log; + GcManager& m_Gc; + CidStore& m_CidStore; + JobQueue& m_JobQueue; + OpenProcessCache& m_OpenProcessCache; + std::filesystem::path m_ProjectBasePath; + const Configuration m_Config; + mutable RwLock m_ProjectsLock; + std::map<std::string, Ref<Project>> m_Projects; + const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; + uint32_t m_UpdateCaptureRefCounter = 0; + std::unique_ptr<eastl::vector<std::string>> m_CapturedProjects; std::filesystem::path BasePathForProject(std::string_view ProjectId); diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp index a7263da83..41f20d760 100644 --- a/src/zenserver/projectstore/remoteprojectstore.cpp +++ b/src/zenserver/projectstore/remoteprojectstore.cpp @@ -143,7 +143,7 @@ namespace remotestore_impl { NiceBytes(Stats.m_PeakReceivedBytes)); } - size_t AddBlock(RwLock& BlocksLock, std::vector<ChunkBlockDescription>& Blocks) + size_t AddBlock(RwLock& BlocksLock, eastl::vector<ChunkBlockDescription>& Blocks) { size_t BlockIndex; { @@ -165,21 +165,21 @@ namespace remotestore_impl { ReportMessage(OptionalContext, fmt::format("Writing {} ops to oplog", OpCount)); - const size_t OpsBatchSize = 8192; - std::vector<uint8_t> OpsData; - std::vector<size_t> OpDataOffsets; - size_t OpsCompleteCount = 0; + const size_t OpsBatchSize = 8192; + eastl::vector<uint8_t> OpsData; + eastl::vector<size_t> OpDataOffsets; + size_t OpsCompleteCount = 0; OpsData.reserve(OpsBatchSize); auto AppendBatch = [&]() { - std::vector<CbObjectView> Ops; + eastl::vector<CbObjectView> Ops; Ops.reserve(OpDataOffsets.size()); for (size_t OpDataOffset : OpDataOffsets) { Ops.emplace_back(CbObjectView(&OpsData[OpDataOffset])); } - std::vector<uint32_t> OpLsns = Oplog.AppendNewOplogEntries(Ops); + eastl::vector<uint32_t> OpLsns = Oplog.AppendNewOplogEntries(Ops); OpsCompleteCount += OpLsns.size(); OpsData.clear(); OpDataOffsets.clear(); @@ -229,19 +229,19 @@ namespace remotestore_impl { std::atomic_size_t MissingAttachmentCount = 0; }; - void DownloadAndSaveBlockChunks(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - bool IgnoreMissingAttachments, - JobContext* OptionalContext, - WorkerThreadPool& NetworkWorkerPool, - WorkerThreadPool& WorkerPool, - Latch& AttachmentsDownloadLatch, - Latch& AttachmentsWriteLatch, - AsyncRemoteResult& RemoteResult, - DownloadInfo& Info, - Stopwatch& LoadAttachmentsTimer, - std::atomic_uint64_t& DownloadStartMS, - const std::vector<IoHash>& Chunks) + void DownloadAndSaveBlockChunks(CidStore& ChunkStore, + RemoteProjectStore& RemoteStore, + bool IgnoreMissingAttachments, + JobContext* OptionalContext, + WorkerThreadPool& NetworkWorkerPool, + WorkerThreadPool& WorkerPool, + Latch& AttachmentsDownloadLatch, + Latch& AttachmentsWriteLatch, + AsyncRemoteResult& RemoteResult, + DownloadInfo& Info, + Stopwatch& LoadAttachmentsTimer, + std::atomic_uint64_t& DownloadStartMS, + const eastl::vector<IoHash>& Chunks) { AttachmentsDownloadLatch.AddCount(1); NetworkWorkerPool.ScheduleWork([&RemoteStore, @@ -299,8 +299,8 @@ namespace remotestore_impl { { try { - std::vector<IoBuffer> WriteAttachmentBuffers; - std::vector<IoHash> WriteRawHashes; + eastl::vector<IoBuffer> WriteAttachmentBuffers; + eastl::vector<IoHash> WriteRawHashes; WriteAttachmentBuffers.reserve(Chunks.size()); WriteRawHashes.reserve(Chunks.size()); @@ -311,7 +311,7 @@ namespace remotestore_impl { WriteAttachmentBuffers.push_back(It.second.GetCompressed().Flatten().AsIoBuffer()); WriteRawHashes.push_back(It.first); } - std::vector<CidStore::InsertResult> InsertResults = + eastl::vector<CidStore::InsertResult> InsertResults = ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly); for (size_t Index = 0; Index < InsertResults.size(); Index++) @@ -341,21 +341,21 @@ namespace remotestore_impl { }); }; - void DownloadAndSaveBlock(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - bool IgnoreMissingAttachments, - JobContext* OptionalContext, - WorkerThreadPool& NetworkWorkerPool, - WorkerThreadPool& WorkerPool, - Latch& AttachmentsDownloadLatch, - Latch& AttachmentsWriteLatch, - AsyncRemoteResult& RemoteResult, - DownloadInfo& Info, - Stopwatch& LoadAttachmentsTimer, - std::atomic_uint64_t& DownloadStartMS, - const IoHash& BlockHash, - const std::vector<IoHash>& Chunks, - uint32_t RetriesLeft) + void DownloadAndSaveBlock(CidStore& ChunkStore, + RemoteProjectStore& RemoteStore, + bool IgnoreMissingAttachments, + JobContext* OptionalContext, + WorkerThreadPool& NetworkWorkerPool, + WorkerThreadPool& WorkerPool, + Latch& AttachmentsDownloadLatch, + Latch& AttachmentsWriteLatch, + AsyncRemoteResult& RemoteResult, + DownloadInfo& Info, + Stopwatch& LoadAttachmentsTimer, + std::atomic_uint64_t& DownloadStartMS, + const IoHash& BlockHash, + const eastl::vector<IoHash>& Chunks, + uint32_t RetriesLeft) { AttachmentsDownloadLatch.AddCount(1); NetworkWorkerPool.ScheduleWork([&AttachmentsDownloadLatch, @@ -437,8 +437,8 @@ namespace remotestore_impl { std::unordered_set<IoHash, IoHash::Hasher> WantedChunks; WantedChunks.reserve(Chunks.size()); WantedChunks.insert(Chunks.begin(), Chunks.end()); - std::vector<IoBuffer> WriteAttachmentBuffers; - std::vector<IoHash> WriteRawHashes; + eastl::vector<IoBuffer> WriteAttachmentBuffers; + eastl::vector<IoHash> WriteRawHashes; IoHash RawHash; uint64_t RawSize; @@ -685,9 +685,9 @@ namespace remotestore_impl { void CreateBlock(WorkerThreadPool& WorkerPool, Latch& OpSectionsLatch, - std::vector<std::pair<IoHash, FetchChunkFunc>>&& ChunksInBlock, + eastl::vector<std::pair<IoHash, FetchChunkFunc>>&& ChunksInBlock, RwLock& SectionsLock, - std::vector<ChunkBlockDescription>& Blocks, + eastl::vector<ChunkBlockDescription>& Blocks, size_t BlockIndex, const std::function<void(CompressedBuffer&&, ChunkBlockDescription&&)>& AsyncOnBlock, AsyncRemoteResult& RemoteResult) @@ -753,7 +753,7 @@ namespace remotestore_impl { CidStore& ChunkStore, RemoteProjectStore& RemoteStore, const std::unordered_set<IoHash, IoHash::Hasher>& LargeAttachments, - const std::vector<std::vector<std::pair<IoHash, FetchChunkFunc>>>& BlockChunks, + const eastl::vector<eastl::vector<std::pair<IoHash, FetchChunkFunc>>>& BlockChunks, const std::unordered_map<IoHash, CreatedBlock, IoHash::Hasher>& CreatedBlocks, const tsl::robin_map<IoHash, TGetAttachmentBufferFunc, IoHash::Hasher>& LooseFileAttachments, const std::unordered_set<IoHash, IoHash::Hasher>& Needs, @@ -799,7 +799,7 @@ namespace remotestore_impl { UnknownAttachments.erase(LargeAttachment); } } - for (const std::vector<std::pair<IoHash, FetchChunkFunc>>& BlockHashes : BlockChunks) + for (const eastl::vector<std::pair<IoHash, FetchChunkFunc>>& BlockHashes : BlockChunks) { for (const std::pair<IoHash, FetchChunkFunc>& Chunk : BlockHashes) { @@ -957,14 +957,14 @@ namespace remotestore_impl { if (!BulkBlockAttachmentsToUpload.empty()) { - for (const std::vector<std::pair<IoHash, FetchChunkFunc>>& Chunks : BlockChunks) + for (const eastl::vector<std::pair<IoHash, FetchChunkFunc>>& Chunks : BlockChunks) { if (RemoteResult.IsError()) { break; } - std::vector<IoHash> NeededChunks; + eastl::vector<IoHash> NeededChunks; NeededChunks.reserve(Chunks.size()); for (const std::pair<IoHash, FetchChunkFunc>& Chunk : Chunks) { @@ -996,8 +996,8 @@ namespace remotestore_impl { } try { - size_t ChunksSize = 0; - std::vector<SharedBuffer> ChunkBuffers; + size_t ChunksSize = 0; + eastl::vector<SharedBuffer> ChunkBuffers; ChunkBuffers.reserve(NeededChunks.size()); for (const IoHash& Chunk : NeededChunks) { @@ -1085,14 +1085,14 @@ namespace remotestore_impl { } // namespace remotestore_impl -std::vector<IoHash> +eastl::vector<IoHash> GetBlockHashesFromOplog(CbObjectView ContainerObject) { using namespace std::literals; - std::vector<ChunkBlockDescription> Result; - CbArrayView BlocksArray = ContainerObject["blocks"sv].AsArrayView(); + eastl::vector<ChunkBlockDescription> Result; + CbArrayView BlocksArray = ContainerObject["blocks"sv].AsArrayView(); - std::vector<IoHash> BlockHashes; + eastl::vector<IoHash> BlockHashes; BlockHashes.reserve(BlocksArray.Num()); for (CbFieldView BlockField : BlocksArray) { @@ -1103,13 +1103,13 @@ GetBlockHashesFromOplog(CbObjectView ContainerObject) return BlockHashes; } -std::vector<ThinChunkBlockDescription> +eastl::vector<ThinChunkBlockDescription> GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> IncludeBlockHashes) { using namespace std::literals; - std::vector<ThinChunkBlockDescription> Result; - CbArrayView BlocksArray = ContainerObject["blocks"sv].AsArrayView(); - tsl::robin_set<IoHash, IoHash::Hasher> IncludeSet; + eastl::vector<ThinChunkBlockDescription> Result; + CbArrayView BlocksArray = ContainerObject["blocks"sv].AsArrayView(); + tsl::robin_set<IoHash, IoHash::Hasher> IncludeSet; IncludeSet.insert(IncludeBlockHashes.begin(), IncludeBlockHashes.end()); Result.reserve(IncludeBlockHashes.size()); @@ -1119,8 +1119,8 @@ GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> Include IoHash BlockHash = BlockView["rawhash"sv].AsBinaryAttachment(); if (IncludeSet.contains(BlockHash)) { - std::vector<IoHash> ChunkHashes; - CbArrayView ChunksArray = BlockView["chunks"sv].AsArrayView(); + eastl::vector<IoHash> ChunkHashes; + CbArrayView ChunksArray = BlockView["chunks"sv].AsArrayView(); if (BlockHash == IoHash::Zero) { continue; @@ -1137,23 +1137,23 @@ GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> Include } CbObject -BuildContainer(CidStore& ChunkStore, - ProjectStore::Project& Project, - ProjectStore::Oplog& Oplog, - size_t MaxBlockSize, - size_t MaxChunkEmbedSize, - size_t ChunkFileSizeLimit, - bool BuildBlocks, - bool IgnoreMissingAttachments, - bool AllowChunking, - const std::vector<ThinChunkBlockDescription>& KnownBlocks, - WorkerThreadPool& WorkerPool, - const std::function<void(CompressedBuffer&&, ChunkBlockDescription&&)>& AsyncOnBlock, - const std::function<void(const IoHash&, TGetAttachmentBufferFunc&&)>& OnLargeAttachment, - const std::function<void(std::vector<std::pair<IoHash, FetchChunkFunc>>&&)>& OnBlockChunks, - bool EmbedLooseFiles, - JobContext* OptionalContext, - remotestore_impl::AsyncRemoteResult& RemoteResult) +BuildContainer(CidStore& ChunkStore, + ProjectStore::Project& Project, + ProjectStore::Oplog& Oplog, + size_t MaxBlockSize, + size_t MaxChunkEmbedSize, + size_t ChunkFileSizeLimit, + bool BuildBlocks, + bool IgnoreMissingAttachments, + bool AllowChunking, + const eastl::vector<ThinChunkBlockDescription>& KnownBlocks, + WorkerThreadPool& WorkerPool, + const std::function<void(CompressedBuffer&&, ChunkBlockDescription&&)>& AsyncOnBlock, + const std::function<void(const IoHash&, TGetAttachmentBufferFunc&&)>& OnLargeAttachment, + const std::function<void(eastl::vector<std::pair<IoHash, FetchChunkFunc>>&&)>& OnBlockChunks, + bool EmbedLooseFiles, + JobContext* OptionalContext, + remotestore_impl::AsyncRemoteResult& RemoteResult) { using namespace std::literals; @@ -1170,9 +1170,9 @@ BuildContainer(CidStore& ChunkStore, std::unordered_map<IoHash, FoundAttachment, IoHash::Hasher> UploadAttachments; - RwLock BlocksLock; - std::vector<ChunkBlockDescription> Blocks; - CompressedBuffer OpsBuffer; + RwLock BlocksLock; + eastl::vector<ChunkBlockDescription> Blocks; + CompressedBuffer OpsBuffer; std::filesystem::path AttachmentTempPath = Oplog.TempPath(); AttachmentTempPath.append(".pending"); @@ -1388,10 +1388,10 @@ BuildContainer(CidStore& ChunkStore, return {}; } - auto FindReuseBlocks = [](const std::vector<ThinChunkBlockDescription>& KnownBlocks, + auto FindReuseBlocks = [](const eastl::vector<ThinChunkBlockDescription>& KnownBlocks, const std::unordered_set<IoHash, IoHash::Hasher>& Attachments, - JobContext* OptionalContext) -> std::vector<size_t> { - std::vector<size_t> ReuseBlockIndexes; + JobContext* OptionalContext) -> eastl::vector<size_t> { + eastl::vector<size_t> ReuseBlockIndexes; if (!Attachments.empty() && !KnownBlocks.empty()) { remotestore_impl::ReportMessage( @@ -1445,8 +1445,8 @@ BuildContainer(CidStore& ChunkStore, FoundHashes.insert(It.first); } - size_t ReusedAttachmentCount = 0; - std::vector<size_t> ReusedBlockIndexes = FindReuseBlocks(KnownBlocks, FoundHashes, OptionalContext); + size_t ReusedAttachmentCount = 0; + eastl::vector<size_t> ReusedBlockIndexes = FindReuseBlocks(KnownBlocks, FoundHashes, OptionalContext); for (size_t KnownBlockIndex : ReusedBlockIndexes) { const ThinChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; @@ -1466,7 +1466,7 @@ BuildContainer(CidStore& ChunkStore, ChunkedInfoWithSource Chunked; tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkLoookup; }; - std::vector<ChunkedFile> ChunkedFiles; + eastl::vector<ChunkedFile> ChunkedFiles; auto ChunkFile = [](const IoHash& RawHash, IoBuffer& RawData, const IoBufferFileReference& FileRef, JobContext*) -> ChunkedFile { ChunkedFile Chunked; @@ -1783,7 +1783,7 @@ BuildContainer(CidStore& ChunkStore, UploadAttachments.erase(It); } - std::vector<size_t> ReusedBlockFromChunking = FindReuseBlocks(KnownBlocks, ChunkedHashes, OptionalContext); + eastl::vector<size_t> ReusedBlockFromChunking = FindReuseBlocks(KnownBlocks, ChunkedHashes, OptionalContext); for (size_t KnownBlockIndex : ReusedBlockIndexes) { const ThinChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; @@ -1811,7 +1811,7 @@ BuildContainer(CidStore& ChunkStore, fmt::format("Reused {} attachments from {} blocks", ReusedAttachmentCount, ReuseBlockCount)); } - std::vector<std::pair<IoHash, Oid>> SortedUploadAttachments; + eastl::vector<std::pair<IoHash, Oid>> SortedUploadAttachments; SortedUploadAttachments.reserve(UploadAttachments.size()); for (const auto& It : UploadAttachments) { @@ -1842,7 +1842,7 @@ BuildContainer(CidStore& ChunkStore, return Lhs.second < Rhs.second; }); - std::vector<size_t> ChunkedFilesOrder; + eastl::vector<size_t> ChunkedFilesOrder; ChunkedFilesOrder.reserve(ChunkedFiles.size()); for (size_t Index = 0; Index < ChunkedFiles.size(); Index++) { @@ -1885,10 +1885,10 @@ BuildContainer(CidStore& ChunkStore, remotestore_impl::ReportMessage(OptionalContext, fmt::format("Assembling {} attachments from {} ops into blocks", ChunkAssembleCount, TotalOpCount)); - Latch BlockCreateLatch(1); - size_t GeneratedBlockCount = 0; - size_t BlockSize = 0; - std::vector<std::pair<IoHash, FetchChunkFunc>> ChunksInBlock; + Latch BlockCreateLatch(1); + size_t GeneratedBlockCount = 0; + size_t BlockSize = 0; + eastl::vector<std::pair<IoHash, FetchChunkFunc>> ChunksInBlock; Oid LastOpKey = Oid::Zero; uint32_t ComposedBlocks = 0; @@ -2252,19 +2252,19 @@ BuildContainer(CidStore& ChunkStore, } RemoteProjectStore::LoadContainerResult -BuildContainer(CidStore& ChunkStore, - ProjectStore::Project& Project, - ProjectStore::Oplog& Oplog, - size_t MaxBlockSize, - size_t MaxChunkEmbedSize, - size_t ChunkFileSizeLimit, - bool BuildBlocks, - bool IgnoreMissingAttachments, - bool AllowChunking, - const std::function<void(CompressedBuffer&&, ChunkBlockDescription&&)>& AsyncOnBlock, - const std::function<void(const IoHash&, TGetAttachmentBufferFunc&&)>& OnLargeAttachment, - const std::function<void(std::vector<std::pair<IoHash, FetchChunkFunc>>&&)>& OnBlockChunks, - bool EmbedLooseFiles) +BuildContainer(CidStore& ChunkStore, + ProjectStore::Project& Project, + ProjectStore::Oplog& Oplog, + size_t MaxBlockSize, + size_t MaxChunkEmbedSize, + size_t ChunkFileSizeLimit, + bool BuildBlocks, + bool IgnoreMissingAttachments, + bool AllowChunking, + const std::function<void(CompressedBuffer&&, ChunkBlockDescription&&)>& AsyncOnBlock, + const std::function<void(const IoHash&, TGetAttachmentBufferFunc&&)>& OnLargeAttachment, + const std::function<void(eastl::vector<std::pair<IoHash, FetchChunkFunc>>&&)>& OnBlockChunks, + bool EmbedLooseFiles) { WorkerThreadPool& WorkerPool = GetLargeWorkerPool(EWorkloadType::Background); @@ -2365,8 +2365,8 @@ SaveOplog(CidStore& ChunkStore, ZEN_DEBUG("Saved block {}, {}", BlockHash, NiceBytes(CompressedBlock.GetCompressedSize())); }; - std::vector<std::vector<std::pair<IoHash, FetchChunkFunc>>> BlockChunks; - auto OnBlockChunks = [&BlockChunks](std::vector<std::pair<IoHash, FetchChunkFunc>>&& Chunks) { + eastl::vector<eastl::vector<std::pair<IoHash, FetchChunkFunc>>> BlockChunks; + auto OnBlockChunks = [&BlockChunks](eastl::vector<std::pair<IoHash, FetchChunkFunc>>&& Chunks) { BlockChunks.push_back({Chunks.begin(), Chunks.end()}); ZEN_DEBUG("Found {} block chunks", Chunks.size()); }; @@ -2391,7 +2391,7 @@ SaveOplog(CidStore& ChunkStore, OnBlock = UploadBlock; } - std::vector<ThinChunkBlockDescription> KnownBlocks; + eastl::vector<ThinChunkBlockDescription> KnownBlocks; uint64_t TransferWallTimeMS = 0; @@ -2633,14 +2633,14 @@ SaveOplog(CidStore& ChunkStore, }; RemoteProjectStore::Result -ParseOplogContainer(const CbObject& ContainerObject, - const std::function<void(std::span<IoHash> RawHashes)>& OnReferencedAttachments, - const std::function<bool(const IoHash& RawHash)>& HasAttachment, - const std::function<void(const IoHash& BlockHash, std::vector<IoHash>&& Chunks)>& OnNeedBlock, - const std::function<void(const IoHash& RawHash)>& OnNeedAttachment, - const std::function<void(const ChunkedInfo&)>& OnChunkedAttachment, - CbObject& OutOplogSection, - JobContext* OptionalContext) +ParseOplogContainer(const CbObject& ContainerObject, + const std::function<void(std::span<IoHash> RawHashes)>& OnReferencedAttachments, + const std::function<bool(const IoHash& RawHash)>& HasAttachment, + const std::function<void(const IoHash& BlockHash, eastl::vector<IoHash>&& Chunks)>& OnNeedBlock, + const std::function<void(const IoHash& RawHash)>& OnNeedAttachment, + const std::function<void(const ChunkedInfo&)>& OnChunkedAttachment, + CbObject& OutOplogSection, + JobContext* OptionalContext) { using namespace std::literals; @@ -2672,7 +2672,7 @@ ParseOplogContainer(const CbObject& ContainerObject, } } { - std::vector<IoHash> ReferencedAttachments(OpsAttachments.begin(), OpsAttachments.end()); + eastl::vector<IoHash> ReferencedAttachments(OpsAttachments.begin(), OpsAttachments.end()); OnReferencedAttachments(ReferencedAttachments); } remotestore_impl::ReportMessage(OptionalContext, fmt::format("Oplog references {} attachments", OpsAttachments.size())); @@ -2721,7 +2721,7 @@ ParseOplogContainer(const CbObject& ContainerObject, CbArrayView ChunksArray = BlockView["chunks"sv].AsArrayView(); - std::vector<IoHash> NeededChunks; + eastl::vector<IoHash> NeededChunks; NeededChunks.reserve(ChunksArray.Num()); if (BlockHash == IoHash::Zero) { @@ -2776,14 +2776,14 @@ ParseOplogContainer(const CbObject& ContainerObject, } RemoteProjectStore::Result -SaveOplogContainer(ProjectStore::Oplog& Oplog, - const CbObject& ContainerObject, - const std::function<void(std::span<IoHash> RawHashes)>& OnReferencedAttachments, - const std::function<bool(const IoHash& RawHash)>& HasAttachment, - const std::function<void(const IoHash& BlockHash, std::vector<IoHash>&& Chunks)>& OnNeedBlock, - const std::function<void(const IoHash& RawHash)>& OnNeedAttachment, - const std::function<void(const ChunkedInfo&)>& OnChunkedAttachment, - JobContext* OptionalContext) +SaveOplogContainer(ProjectStore::Oplog& Oplog, + const CbObject& ContainerObject, + const std::function<void(std::span<IoHash> RawHashes)>& OnReferencedAttachments, + const std::function<bool(const IoHash& RawHash)>& HasAttachment, + const std::function<void(const IoHash& BlockHash, eastl::vector<IoHash>&& Chunks)>& OnNeedBlock, + const std::function<void(const IoHash& RawHash)>& OnNeedAttachment, + const std::function<void(const ChunkedInfo&)>& OnChunkedAttachment, + JobContext* OptionalContext) { using namespace std::literals; @@ -2884,7 +2884,7 @@ LoadOplog(CidStore& ChunkStore, &LoadAttachmentsTimer, &DownloadStartMS, IgnoreMissingAttachments, - OptionalContext](const IoHash& BlockHash, std::vector<IoHash>&& Chunks) { + OptionalContext](const IoHash& BlockHash, eastl::vector<IoHash>&& Chunks) { if (RemoteResult.IsError()) { return; @@ -2967,12 +2967,12 @@ LoadOplog(CidStore& ChunkStore, RawHash); }; - std::vector<ChunkedInfo> FilesToDechunk; - auto OnChunkedAttachment = [&Oplog, &ChunkStore, &FilesToDechunk, ForceDownload](const ChunkedInfo& Chunked) { - if (ForceDownload || !ChunkStore.ContainsChunk(Chunked.RawHash)) - { - FilesToDechunk.push_back(Chunked); - } + eastl::vector<ChunkedInfo> FilesToDechunk; + auto OnChunkedAttachment = [&Oplog, &ChunkStore, &FilesToDechunk, ForceDownload](const ChunkedInfo& Chunked) { + if (ForceDownload || !ChunkStore.ContainsChunk(Chunked.RawHash)) + { + FilesToDechunk.push_back(Chunked); + } }; auto OnReferencedAttachments = [&Oplog](std::span<IoHash> RawHashes) { Oplog.CaptureAddedAttachments(RawHashes); }; diff --git a/src/zenserver/projectstore/remoteprojectstore.h b/src/zenserver/projectstore/remoteprojectstore.h index 1210afc7c..26788179d 100644 --- a/src/zenserver/projectstore/remoteprojectstore.h +++ b/src/zenserver/projectstore/remoteprojectstore.h @@ -61,12 +61,12 @@ public: struct LoadAttachmentsResult : public Result { - std::vector<std::pair<IoHash, CompressedBuffer>> Chunks; + eastl::vector<std::pair<IoHash, CompressedBuffer>> Chunks; }; struct GetKnownBlocksResult : public Result { - std::vector<ThinChunkBlockDescription> Blocks; + eastl::vector<ThinChunkBlockDescription> Blocks; }; struct RemoteStoreInfo @@ -99,12 +99,12 @@ public: virtual SaveResult SaveContainer(const IoBuffer& Payload) = 0; virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&& Block) = 0; virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) = 0; - virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Payloads) = 0; + virtual SaveAttachmentsResult SaveAttachments(const eastl::vector<SharedBuffer>& Payloads) = 0; - virtual LoadContainerResult LoadContainer() = 0; - virtual GetKnownBlocksResult GetKnownBlocks() = 0; - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) = 0; - virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) = 0; + virtual LoadContainerResult LoadContainer() = 0; + virtual GetKnownBlocksResult GetKnownBlocks() = 0; + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) = 0; + virtual LoadAttachmentsResult LoadAttachments(const eastl::vector<IoHash>& RawHashes) = 0; }; struct RemoteStoreOptions @@ -121,30 +121,31 @@ struct RemoteStoreOptions typedef std::function<IoBuffer(const IoHash& AttachmentHash)> TGetAttachmentBufferFunc; RemoteProjectStore::LoadContainerResult BuildContainer( - CidStore& ChunkStore, - ProjectStore::Project& Project, - ProjectStore::Oplog& Oplog, - size_t MaxBlockSize, - size_t MaxChunkEmbedSize, - size_t ChunkFileSizeLimit, - bool BuildBlocks, - bool IgnoreMissingAttachments, - bool AllowChunking, - const std::function<void(CompressedBuffer&&, ChunkBlockDescription&&)>& AsyncOnBlock, - const std::function<void(const IoHash&, TGetAttachmentBufferFunc&&)>& OnLargeAttachment, - const std::function<void(std::vector<std::pair<IoHash, FetchChunkFunc>>&&)>& OnBlockChunks, - bool EmbedLooseFiles); + CidStore& ChunkStore, + ProjectStore::Project& Project, + ProjectStore::Oplog& Oplog, + size_t MaxBlockSize, + size_t MaxChunkEmbedSize, + size_t ChunkFileSizeLimit, + bool BuildBlocks, + bool IgnoreMissingAttachments, + bool AllowChunking, + const std::function<void(CompressedBuffer&&, ChunkBlockDescription&&)>& AsyncOnBlock, + const std::function<void(const IoHash&, TGetAttachmentBufferFunc&&)>& OnLargeAttachment, + const std::function<void(eastl::vector<std::pair<IoHash, FetchChunkFunc>>&&)>& OnBlockChunks, + bool EmbedLooseFiles); class JobContext; -RemoteProjectStore::Result SaveOplogContainer(ProjectStore::Oplog& Oplog, - const CbObject& ContainerObject, - const std::function<void(std::span<IoHash> RawHashes)>& OnReferencedAttachments, - const std::function<bool(const IoHash& RawHash)>& HasAttachment, - const std::function<void(const IoHash& BlockHash, std::vector<IoHash>&& Chunks)>& OnNeedBlock, - const std::function<void(const IoHash& RawHash)>& OnNeedAttachment, - const std::function<void(const ChunkedInfo& Chunked)>& OnChunkedAttachment, - JobContext* OptionalContext); +RemoteProjectStore::Result SaveOplogContainer( + ProjectStore::Oplog& Oplog, + const CbObject& ContainerObject, + const std::function<void(std::span<IoHash> RawHashes)>& OnReferencedAttachments, + const std::function<bool(const IoHash& RawHash)>& HasAttachment, + const std::function<void(const IoHash& BlockHash, eastl::vector<IoHash>&& Chunks)>& OnNeedBlock, + const std::function<void(const IoHash& RawHash)>& OnNeedAttachment, + const std::function<void(const ChunkedInfo& Chunked)>& OnChunkedAttachment, + JobContext* OptionalContext); RemoteProjectStore::Result SaveOplog(CidStore& ChunkStore, RemoteProjectStore& RemoteStore, @@ -166,7 +167,7 @@ RemoteProjectStore::Result LoadOplog(CidStore& ChunkStore, bool CleanOplog, JobContext* OptionalContext); -std::vector<IoHash> GetBlockHashesFromOplog(CbObjectView ContainerObject); -std::vector<ThinChunkBlockDescription> GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> IncludeBlockHashes); +eastl::vector<IoHash> GetBlockHashesFromOplog(CbObjectView ContainerObject); +eastl::vector<ThinChunkBlockDescription> GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> IncludeBlockHashes); } // namespace zen diff --git a/src/zenserver/projectstore/zenremoteprojectstore.cpp b/src/zenserver/projectstore/zenremoteprojectstore.cpp index 2ebf58a5d..0f6be6d6a 100644 --- a/src/zenserver/projectstore/zenremoteprojectstore.cpp +++ b/src/zenserver/projectstore/zenremoteprojectstore.cpp @@ -111,7 +111,7 @@ public: return Result; } - virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Chunks) override + virtual SaveAttachmentsResult SaveAttachments(const eastl::vector<SharedBuffer>& Chunks) override { CbPackage RequestPackage; { @@ -149,7 +149,7 @@ public: return Result; } - virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) override + virtual LoadAttachmentsResult LoadAttachments(const eastl::vector<IoHash>& RawHashes) override { std::string LoadRequest = fmt::format("/{}/oplog/{}/rpc"sv, m_Project, m_Oplog); diff --git a/src/zenserver/stats/statsreporter.h b/src/zenserver/stats/statsreporter.h index 2f93aa8bb..de13b3591 100644 --- a/src/zenserver/stats/statsreporter.h +++ b/src/zenserver/stats/statsreporter.h @@ -28,7 +28,7 @@ public: private: RwLock m_Lock; std::unique_ptr<StatsDaemonClient> m_Statsd; - std::vector<StatsProvider*> m_Providers; + eastl::vector<StatsProvider*> m_Providers; }; } // namespace zen diff --git a/src/zenserver/upstream/upstreamcache.cpp b/src/zenserver/upstream/upstreamcache.cpp index e438a840a..5ddc141d4 100644 --- a/src/zenserver/upstream/upstreamcache.cpp +++ b/src/zenserver/upstream/upstreamcache.cpp @@ -1461,7 +1461,7 @@ namespace detail { UpstreamEndpointInfo m_Info; UpstreamStatus m_Status; UpstreamEndpointStats m_Stats; - std::vector<ZenEndpoint> m_Endpoints; + eastl::vector<ZenEndpoint> m_Endpoints; std::chrono::milliseconds m_ConnectTimeout; std::chrono::milliseconds m_Timeout; RwLock m_ClientLock; @@ -1594,7 +1594,7 @@ public: std::shared_lock<std::shared_mutex> _(m_EndpointsMutex); - std::vector<CacheKeyRequest*> RemainingKeys(Requests.begin(), Requests.end()); + eastl::vector<CacheKeyRequest*> RemainingKeys(Requests.begin(), Requests.end()); if (m_Options.ReadUpstream) { @@ -1610,9 +1610,9 @@ public: continue; } - UpstreamEndpointStats& Stats = Endpoint->Stats(); - std::vector<CacheKeyRequest*> Missing; - GetUpstreamCacheResult Result; + UpstreamEndpointStats& Stats = Endpoint->Stats(); + eastl::vector<CacheKeyRequest*> Missing; + GetUpstreamCacheResult Result; { metrics::OperationTiming::Scope Scope(Stats.CacheGetRequestTiming); @@ -1662,7 +1662,7 @@ public: std::shared_lock<std::shared_mutex> _(m_EndpointsMutex); - std::vector<CacheChunkRequest*> RemainingKeys(CacheChunkRequests.begin(), CacheChunkRequests.end()); + eastl::vector<CacheChunkRequest*> RemainingKeys(CacheChunkRequests.begin(), CacheChunkRequests.end()); if (m_Options.ReadUpstream) { @@ -1678,9 +1678,9 @@ public: continue; } - UpstreamEndpointStats& Stats = Endpoint->Stats(); - std::vector<CacheChunkRequest*> Missing; - GetUpstreamCacheResult Result; + UpstreamEndpointStats& Stats = Endpoint->Stats(); + eastl::vector<CacheChunkRequest*> Missing; + GetUpstreamCacheResult Result; { metrics::OperationTiming::Scope Scope(Endpoint->Stats().CacheGetRequestTiming); @@ -1775,7 +1775,7 @@ public: std::shared_lock<std::shared_mutex> _(m_EndpointsMutex); - std::vector<CacheValueRequest*> RemainingKeys(CacheValueRequests.begin(), CacheValueRequests.end()); + eastl::vector<CacheValueRequest*> RemainingKeys(CacheValueRequests.begin(), CacheValueRequests.end()); if (m_Options.ReadUpstream) { @@ -1791,9 +1791,9 @@ public: continue; } - UpstreamEndpointStats& Stats = Endpoint->Stats(); - std::vector<CacheValueRequest*> Missing; - GetUpstreamCacheResult Result; + UpstreamEndpointStats& Stats = Endpoint->Stats(); + eastl::vector<CacheValueRequest*> Missing; + GetUpstreamCacheResult Result; { metrics::OperationTiming::Scope Scope(Endpoint->Stats().CacheGetRequestTiming); @@ -1903,8 +1903,8 @@ private: { ZEN_TRACE_CPU("Upstream::ProcessCacheRecord"); - ZenCacheValue CacheValue; - std::vector<IoBuffer> Payloads; + ZenCacheValue CacheValue; + eastl::vector<IoBuffer> Payloads; if (!m_CacheStore.Get(CacheRecord.Context, CacheRecord.Namespace, CacheRecord.Key.Bucket, CacheRecord.Key.Hash, CacheValue)) { @@ -2009,7 +2009,7 @@ private: try { - std::vector<UpstreamEndpoint*> Endpoints; + eastl::vector<UpstreamEndpoint*> Endpoints; { std::shared_lock<std::shared_mutex> _(m_EndpointsMutex); @@ -2098,16 +2098,16 @@ private: } }; - LoggerRef m_Log; - UpstreamCacheOptions m_Options; - ZenCacheStore& m_CacheStore; - CidStore& m_CidStore; - UpstreamQueue m_UpstreamQueue; - std::shared_mutex m_EndpointsMutex; - std::vector<std::unique_ptr<UpstreamEndpoint>> m_Endpoints; - std::vector<std::thread> m_UpstreamThreads; - std::thread m_EndpointMonitorThread; - RunState m_RunState; + LoggerRef m_Log; + UpstreamCacheOptions m_Options; + ZenCacheStore& m_CacheStore; + CidStore& m_CidStore; + UpstreamQueue m_UpstreamQueue; + std::shared_mutex m_EndpointsMutex; + eastl::vector<std::unique_ptr<UpstreamEndpoint>> m_Endpoints; + eastl::vector<std::thread> m_UpstreamThreads; + std::thread m_EndpointMonitorThread; + RunState m_RunState; }; ////////////////////////////////////////////////////////////////////////// diff --git a/src/zenserver/upstream/upstreamcache.h b/src/zenserver/upstream/upstreamcache.h index 26e5decac..219f2474e 100644 --- a/src/zenserver/upstream/upstreamcache.h +++ b/src/zenserver/upstream/upstreamcache.h @@ -11,11 +11,11 @@ #include <zenstore/cache/upstreamcacheclient.h> #include <zenutil/cache/cache.h> +#include <EASTL/vector.h> #include <atomic> #include <chrono> #include <functional> #include <memory> -#include <vector> namespace zen { diff --git a/src/zenserver/vfs/vfsimpl.cpp b/src/zenserver/vfs/vfsimpl.cpp index 2bac6b756..2cdb56003 100644 --- a/src/zenserver/vfs/vfsimpl.cpp +++ b/src/zenserver/vfs/vfsimpl.cpp @@ -340,7 +340,7 @@ VfsServiceDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNo // Oplog enumeration - std::vector<std::string> Oplogs = Project->ScanForOplogs(); + eastl::vector<std::string> Oplogs = Project->ScanForOplogs(); for (auto& Oplog : Oplogs) { @@ -368,7 +368,7 @@ VfsServiceDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNo Ref<VfsOplogDataSource> DataSource = GetOplogDataSource(ProjectId, OplogId); // Get metadata for all chunks - std::vector<ProjectStore::Oplog::ChunkInfo> ChunkInfos = Oplog->GetAllChunksInfo(); + eastl::vector<ProjectStore::Oplog::ChunkInfo> ChunkInfos = Oplog->GetAllChunksInfo(); std::unordered_map<zen::Oid, uint64_t> ChunkSizes; @@ -412,7 +412,7 @@ VfsServiceDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNo { // Namespace enumeration - std::vector<std::string> Namespaces = m_VfsImpl->m_ZenCacheStore->GetNamespaces(); + eastl::vector<std::string> Namespaces = m_VfsImpl->m_ZenCacheStore->GetNamespaces(); for (auto& Namespace : Namespaces) { diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp index 8a4b977ad..2ad661b85 100644 --- a/src/zenserver/workspaces/httpworkspaces.cpp +++ b/src/zenserver/workspaces/httpworkspaces.cpp @@ -49,7 +49,7 @@ namespace { void WriteWorkspaceAndSharesConfig(CbWriter& Writer, Workspaces& Workspaces, const Workspaces::WorkspaceConfiguration& WorkspaceConfig) { WriteWorkspaceConfig(Writer, WorkspaceConfig); - if (std::optional<std::vector<Oid>> ShareIds = Workspaces.GetWorkspaceShares(WorkspaceConfig.Id); ShareIds) + if (std::optional<eastl::vector<Oid>> ShareIds = Workspaces.GetWorkspaceShares(WorkspaceConfig.Id); ShareIds) { Writer.BeginArray("shares"); { @@ -275,8 +275,8 @@ HttpWorkspacesService::WorkspacesRequest(HttpRouterRequest& Req) { HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::vector<Oid> WorkspaceIds = m_Workspaces.GetWorkspaces(); - CbObjectWriter Response; + eastl::vector<Oid> WorkspaceIds = m_Workspaces.GetWorkspaces(); + CbObjectWriter Response; Response.BeginArray("workspaces"); for (const Oid& WorkspaceId : WorkspaceIds) { @@ -758,7 +758,7 @@ HttpWorkspacesService::FilesRequest(HttpRouterRequest& Req, const Oid& Workspace const bool WantsRawSizeField = WantsAllFields || WantedFieldNames.contains("rawsize"); const bool WantsSizeField = WantsAllFields || WantedFieldNames.contains("size"); - std::optional<std::vector<Workspaces::ShareFile>> Files = + std::optional<eastl::vector<Workspaces::ShareFile>> Files = m_Workspaces.GetWorkspaceShareFiles(WorkspaceId, ShareId, Refresh, GetSmallWorkerPool(EWorkloadType::Burst)); if (!Files.has_value()) { @@ -819,16 +819,16 @@ HttpWorkspacesService::ChunkInfoRequest(HttpRouterRequest& Req, const Oid& Works void HttpWorkspacesService::BatchRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId) { - HttpServerRequest& ServerRequest = Req.ServerRequest(); - IoBuffer Payload = ServerRequest.ReadPayload(); - std::optional<std::vector<RequestChunkEntry>> ChunkRequests = ParseChunkBatchRequest(Payload); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + IoBuffer Payload = ServerRequest.ReadPayload(); + std::optional<eastl::vector<RequestChunkEntry>> ChunkRequests = ParseChunkBatchRequest(Payload); if (!ChunkRequests.has_value()) { m_WorkspacesStats.BadRequestCount++; return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "batch payload malformed"); } m_WorkspacesStats.WorkspaceShareBatchReadCount++; - std::vector<Workspaces::ChunkRequest> Requests; + eastl::vector<Workspaces::ChunkRequest> Requests; Requests.reserve(ChunkRequests.value().size()); std::transform(ChunkRequests.value().begin(), ChunkRequests.value().end(), @@ -836,7 +836,7 @@ HttpWorkspacesService::BatchRequest(HttpRouterRequest& Req, const Oid& Workspace [](const RequestChunkEntry& Entry) { return Workspaces::ChunkRequest{.ChunkId = Entry.ChunkId, .Offset = Entry.Offset, .Size = Entry.RequestBytes}; }); - std::vector<IoBuffer> Chunks = + eastl::vector<IoBuffer> Chunks = m_Workspaces.GetWorkspaceShareChunks(WorkspaceId, ShareId, Requests, GetSmallWorkerPool(EWorkloadType::Burst)); if (Chunks.empty()) { @@ -853,7 +853,7 @@ HttpWorkspacesService::BatchRequest(HttpRouterRequest& Req, const Oid& Workspace m_WorkspacesStats.WorkspaceShareChunkMissCount++; } } - std::vector<IoBuffer> Response = BuildChunkBatchResponse(ChunkRequests.value(), Chunks); + eastl::vector<IoBuffer> Response = BuildChunkBatchResponse(ChunkRequests.value(), Chunks); if (!Response.empty()) { return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Response); @@ -892,7 +892,7 @@ HttpWorkspacesService::EntriesRequest(HttpRouterRequest& Req, const Oid& Workspa } m_WorkspacesStats.WorkspaceShareEntriesReadCount++; - std::optional<std::vector<Workspaces::ShareFile>> Files = + std::optional<eastl::vector<Workspaces::ShareFile>> Files = m_Workspaces.GetWorkspaceShareFiles(WorkspaceId, ShareId, Refresh, GetSmallWorkerPool(EWorkloadType::Burst)); if (!Files.has_value()) { @@ -995,10 +995,10 @@ HttpWorkspacesService::ChunkRequest(HttpRouterRequest& Req, const Oid& Workspace } } - std::vector<IoBuffer> Response = m_Workspaces.GetWorkspaceShareChunks( + eastl::vector<IoBuffer> Response = m_Workspaces.GetWorkspaceShareChunks( WorkspaceId, ShareId, - std::vector<Workspaces::ChunkRequest>{Workspaces::ChunkRequest{.ChunkId = ChunkId, .Offset = Offset, .Size = Size}}, + eastl::vector<Workspaces::ChunkRequest>{Workspaces::ChunkRequest{.ChunkId = ChunkId, .Offset = Offset, .Size = Size}}, GetSmallWorkerPool(EWorkloadType::Burst)); if (!Response.empty() && Response[0]) { diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index f84bc0b00..db74000b1 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -74,10 +74,10 @@ namespace utils { using namespace std::literals; namespace utils { - asio::error_code ResolveHostname(asio::io_context& Ctx, - std::string_view Host, - std::string_view DefaultPort, - std::vector<std::string>& OutEndpoints) + asio::error_code ResolveHostname(asio::io_context& Ctx, + std::string_view Host, + std::string_view DefaultPort, + eastl::vector<std::string>& OutEndpoints) { std::string_view Port = DefaultPort; @@ -562,7 +562,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) { // Zen upstream { - std::vector<std::string> ZenUrls = UpstreamConfig.ZenConfig.Urls; + eastl::vector<std::string> ZenUrls{UpstreamConfig.ZenConfig.Urls.begin(), UpstreamConfig.ZenConfig.Urls.end()}; if (!UpstreamConfig.ZenConfig.Dns.empty()) { for (const std::string& Dns : UpstreamConfig.ZenConfig.Dns) @@ -578,7 +578,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) } } - std::erase_if(ZenUrls, [](const auto& Url) { return Url.empty(); }); + zen::erase_if(ZenUrls, [](const auto& Url) { return Url.empty(); }); if (!ZenUrls.empty()) { diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index e976c061d..7df9117db 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -293,8 +293,8 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max if (std::filesystem::is_directory(m_BlocksBasePath)) { - uint32_t NextBlockIndex = 0; - std::vector<std::filesystem::path> FoldersToScan; + uint32_t NextBlockIndex = 0; + eastl::vector<std::filesystem::path> FoldersToScan; FoldersToScan.push_back(m_BlocksBasePath); size_t FolderOffset = 0; while (FolderOffset < FoldersToScan.size()) @@ -399,7 +399,7 @@ BlockStore::GetBlocksToCompact(const BlockUsageMap& BlockUsage, uint32_t BlockUs const uint64_t SmallBlockLimit = m_MaxBlockSize / 2; - std::vector<uint32_t> SmallBlockIndexes; + eastl::vector<uint32_t> SmallBlockIndexes; for (const auto& It : m_ChunkBlocks) { @@ -598,9 +598,9 @@ BlockStore::WriteChunks(std::span<IoBuffer> Datas, uint32_t Alignment, const Wri LargestSize = Max(LargestSize, Size); } - const uint64_t MinSize = Max(LargestSize, 8u * 1024u * 1024u); - const uint64_t BufferSize = Min(TotalSize, MinSize); - std::vector<uint8_t> Buffer(BufferSize); + const uint64_t MinSize = Max(LargestSize, 8u * 1024u * 1024u); + const uint64_t BufferSize = Min(TotalSize, MinSize); + eastl::vector<uint8_t> Buffer(BufferSize); size_t Offset = 0; while (Offset < TotalCount) @@ -660,8 +660,8 @@ BlockStore::WriteChunks(std::span<IoBuffer> Datas, uint32_t Alignment, const Wri m_TotalSize.fetch_add(RangeSize, std::memory_order::relaxed); - uint32_t ChunkOffset = AlignedInsertOffset; - std::vector<BlockStoreLocation> Locations(Count); + uint32_t ChunkOffset = AlignedInsertOffset; + eastl::vector<BlockStoreLocation> Locations(Count); for (size_t Index = 0; Index < Count; Index++) { uint32_t ChunkSize = gsl::narrow<uint32_t>(Datas[Offset + Index].GetSize()); @@ -746,8 +746,8 @@ BlockStore::IterateBlock(std::span<const BlockStoreLocation> ChunkLocations, IterateSmallChunkWindowSize = Min((LargeSizeLimit + IterateSmallChunkMaxGapSize) * ChunkLocations.size(), IterateSmallChunkWindowSize); - uint32_t BlockIndex = ChunkLocations[InChunkIndexes[0]].BlockIndex; - std::vector<size_t> ChunkIndexes(InChunkIndexes.begin(), InChunkIndexes.end()); + uint32_t BlockIndex = ChunkLocations[InChunkIndexes[0]].BlockIndex; + eastl::vector<size_t> ChunkIndexes(InChunkIndexes.begin(), InChunkIndexes.end()); std::sort(ChunkIndexes.begin(), ChunkIndexes.end(), [&](size_t IndexA, size_t IndexB) -> bool { return ChunkLocations[IndexA].Offset < ChunkLocations[IndexB].Offset; }); @@ -894,7 +894,7 @@ BlockStore::IterateChunks(const std::span<const BlockStoreLocation>& ChunkLocati ZEN_LOG_SCOPE("iterating chunks from '{}'", m_BlocksBasePath); - std::vector<size_t> ChunkOrder(ChunkLocations.size()); + eastl::vector<size_t> ChunkOrder(ChunkLocations.size()); for (size_t ChunkIndex = 0; ChunkIndex < ChunkLocations.size(); ++ChunkIndex) { ChunkOrder[ChunkIndex] = ChunkIndex; @@ -998,11 +998,11 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, return Continue; }; - std::vector<uint32_t> RemovedBlocks; + eastl::vector<uint32_t> RemovedBlocks; - CompactState.IterateBlocks([&](uint32_t BlockIndex, - const std::vector<size_t>& KeepChunkIndexes, - const std::vector<BlockStoreLocation>& ChunkLocations) -> bool { + CompactState.IterateBlocks([&](uint32_t BlockIndex, + const eastl::vector<size_t>& KeepChunkIndexes, + const eastl::vector<BlockStoreLocation>& ChunkLocations) -> bool { ZEN_TRACE_CPU("BlockStore::CompactBlock"); Ref<BlockStoreFile> OldBlockFile; { @@ -1047,15 +1047,15 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, } else { - std::vector<size_t> SortedChunkIndexes(KeepChunkIndexes); + eastl::vector<size_t> SortedChunkIndexes(KeepChunkIndexes); std::sort(SortedChunkIndexes.begin(), SortedChunkIndexes.end(), [&ChunkLocations](size_t Lhs, size_t Rhs) { return ChunkLocations[Lhs].Offset < ChunkLocations[Rhs].Offset; }); BasicFileBuffer SourceFileBuffer(OldBlockFile->GetBasicFile(), Min(65536u, OldBlockSize)); - uint64_t WrittenBytesToBlock = 0; - uint64_t MovedFromBlock = 0; - std::vector<uint8_t> Chunk; + uint64_t WrittenBytesToBlock = 0; + uint64_t MovedFromBlock = 0; + eastl::vector<uint8_t> Chunk; for (const size_t& ChunkIndex : SortedChunkIndexes) { const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex]; @@ -1424,14 +1424,14 @@ namespace blockstore::impl { return AsString; }; - std::vector<std::filesystem::path> GetDirectoryContent(std::filesystem::path RootDir, bool Files, bool Directories) + eastl::vector<std::filesystem::path> GetDirectoryContent(std::filesystem::path RootDir, bool Files, bool Directories) { DirectoryContent DirectoryContent; GetDirectoryContent(RootDir, DirectoryContentFlags::Recursive | (Files ? DirectoryContentFlags::IncludeFiles : DirectoryContentFlags::None) | (Directories ? DirectoryContentFlags::IncludeDirs : DirectoryContentFlags::None), DirectoryContent); - std::vector<std::filesystem::path> Result; + eastl::vector<std::filesystem::path> Result; Result.insert(Result.end(), DirectoryContent.Directories.begin(), DirectoryContent.Directories.end()); Result.insert(Result.end(), DirectoryContent.Files.begin(), DirectoryContent.Files.end()); return Result; @@ -1449,8 +1449,8 @@ TEST_CASE("blockstore.multichunks") BlockStore Store; Store.Initialize(RootDirectory, 128, 1024); - std::vector<IoBuffer> MultiChunkData; - std::string FirstChunkData = "0123456789012345678901234567890123456789012345678901234567890123"; + eastl::vector<IoBuffer> MultiChunkData; + std::string FirstChunkData = "0123456789012345678901234567890123456789012345678901234567890123"; MultiChunkData.push_back(IoBuffer(IoBuffer::Wrap, FirstChunkData.data(), FirstChunkData.size())); std::string SecondChunkData = "12345678901234567890123456789012345678901234567890123456"; @@ -1588,16 +1588,16 @@ TEST_CASE("blockstore.iterate.chunks") WorkerThreadPool WorkerPool(4); - std::vector<BlockStoreLocation> Locations{FirstChunkLocation, - SecondChunkLocation, - VeryLargeChunkLocation, - BadLocationZeroSize, - BadLocationOutOfRange, - BadBlockIndex}; - Latch WorkLatch(1); + eastl::vector<BlockStoreLocation> Locations{FirstChunkLocation, + SecondChunkLocation, + VeryLargeChunkLocation, + BadLocationZeroSize, + BadLocationOutOfRange, + BadBlockIndex}; + Latch WorkLatch(1); Store.IterateChunks(Locations, [&](uint32_t, std::span<const size_t> ChunkIndexes) -> bool { WorkLatch.AddCount(1); - WorkerPool.ScheduleWork([&, ChunkIndexes = std::vector<size_t>(ChunkIndexes.begin(), ChunkIndexes.end())]() { + WorkerPool.ScheduleWork([&, ChunkIndexes = eastl::vector<size_t>(ChunkIndexes.begin(), ChunkIndexes.end())]() { auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); bool Continue = Store.IterateBlock( Locations, @@ -1687,10 +1687,10 @@ TEST_CASE("blockstore.thread.read.write") BlockStore Store; Store.Initialize(RootDirectory / "store", 1088, 1024); - constexpr size_t ChunkCount = 1000; - constexpr size_t Alignment = 8; - std::vector<IoBuffer> Chunks; - std::vector<IoHash> ChunkHashes; + constexpr size_t ChunkCount = 1000; + constexpr size_t Alignment = 8; + eastl::vector<IoBuffer> Chunks; + eastl::vector<IoHash> ChunkHashes; Chunks.reserve(ChunkCount); ChunkHashes.reserve(ChunkCount); for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex) @@ -1700,7 +1700,7 @@ TEST_CASE("blockstore.thread.read.write") ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size())); } - std::vector<BlockStoreLocation> ChunkLocations; + eastl::vector<BlockStoreLocation> ChunkLocations; ChunkLocations.resize(ChunkCount); WorkerThreadPool WorkerPool(8); @@ -1734,7 +1734,7 @@ TEST_CASE("blockstore.thread.read.write") Sleep(1); } - std::vector<BlockStoreLocation> SecondChunkLocations; + eastl::vector<BlockStoreLocation> SecondChunkLocations; SecondChunkLocations.resize(ChunkCount); WorkCompleted = 0; for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex) @@ -1770,10 +1770,10 @@ TEST_CASE("blockstore.compact.blocks") BlockStore Store; Store.Initialize(RootDirectory / "store", 1088, 1024); - constexpr size_t ChunkCount = 200; - constexpr size_t Alignment = 8; - std::vector<IoBuffer> Chunks; - std::vector<IoHash> ChunkHashes; + constexpr size_t ChunkCount = 200; + constexpr size_t Alignment = 8; + eastl::vector<IoBuffer> Chunks; + eastl::vector<IoHash> ChunkHashes; Chunks.reserve(ChunkCount); ChunkHashes.reserve(ChunkCount); for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex) @@ -1783,7 +1783,7 @@ TEST_CASE("blockstore.compact.blocks") ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size())); } - std::vector<BlockStoreLocation> ChunkLocations; + eastl::vector<BlockStoreLocation> ChunkLocations; ChunkLocations.resize(ChunkCount); for (size_t ChunkIndex = 0; ChunkIndex < ChunkCount; ++ChunkIndex) @@ -1924,8 +1924,8 @@ TEST_CASE("blockstore.compact.blocks") CHECK(!Store.IsWriting(0)); State.IncludeBlock(0); - uint64_t SkipChunkCount = 2; - std::vector<BlockStoreLocation> DroppedLocations(ChunkLocations.begin(), ChunkLocations.begin() + 2); + uint64_t SkipChunkCount = 2; + eastl::vector<BlockStoreLocation> DroppedLocations(ChunkLocations.begin(), ChunkLocations.begin() + 2); for (auto It = ChunkLocations.begin() + 2; It != ChunkLocations.end(); It++) { const BlockStoreLocation& Location = *It; @@ -1996,7 +1996,7 @@ TEST_CASE("blockstore.compact.blocks") } SkipFlag = false; - std::vector<BlockStoreLocation> DroppedLocations; + eastl::vector<BlockStoreLocation> DroppedLocations; for (const BlockStoreLocation& Location : ChunkLocations) { if (Store.IsWriting(Location.BlockIndex)) diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 61552fafc..2f648bc2d 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -262,36 +262,36 @@ public: return OutVersion == CurrentDiskBucketVersion; } - void ParseManifest(RwLock::ExclusiveLockScope& BucketLock, - ZenCacheDiskLayer::CacheBucket& Bucket, - std::filesystem::path ManifestPath, - ZenCacheDiskLayer::CacheBucket::IndexMap& Index, - std::vector<AccessTime>& AccessTimes, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads); + void ParseManifest(RwLock::ExclusiveLockScope& BucketLock, + ZenCacheDiskLayer::CacheBucket& Bucket, + std::filesystem::path ManifestPath, + ZenCacheDiskLayer::CacheBucket::IndexMap& Index, + eastl::vector<AccessTime>& AccessTimes, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads); Oid GenerateNewManifest(std::filesystem::path ManifestPath); IoBuffer MakeSidecarManifest(const Oid& BucketId, uint64_t EntryCount); uint64_t GetSidecarSize() const { return sizeof(BucketMetaHeader) + m_ManifestEntryCount * sizeof(ManifestData); } - void WriteSidecarFile(RwLock::SharedLockScope& BucketLock, - const std::filesystem::path& SidecarPath, - uint64_t SnapshotLogPosition, - const ZenCacheDiskLayer::CacheBucket::IndexMap& Index, - const std::vector<AccessTime>& AccessTimes, - const std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads, - const std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas); - bool ReadSidecarFile(RwLock::ExclusiveLockScope& BucketLock, - ZenCacheDiskLayer::CacheBucket& Bucket, - std::filesystem::path SidecarPath, - ZenCacheDiskLayer::CacheBucket::IndexMap& Index, - std::vector<AccessTime>& AccessTimes, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads); - - IoBuffer MakeManifest(const Oid& BucketId, - ZenCacheDiskLayer::CacheBucket::IndexMap&& Index, - std::vector<AccessTime>&& AccessTimes, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&& Payloads, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas); + void WriteSidecarFile(RwLock::SharedLockScope& BucketLock, + const std::filesystem::path& SidecarPath, + uint64_t SnapshotLogPosition, + const ZenCacheDiskLayer::CacheBucket::IndexMap& Index, + const eastl::vector<AccessTime>& AccessTimes, + const eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads, + const eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas); + bool ReadSidecarFile(RwLock::ExclusiveLockScope& BucketLock, + ZenCacheDiskLayer::CacheBucket& Bucket, + std::filesystem::path SidecarPath, + ZenCacheDiskLayer::CacheBucket::IndexMap& Index, + eastl::vector<AccessTime>& AccessTimes, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads); + + IoBuffer MakeManifest(const Oid& BucketId, + ZenCacheDiskLayer::CacheBucket::IndexMap&& Index, + eastl::vector<AccessTime>&& AccessTimes, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&& Payloads, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas); CbObject Manifest; @@ -329,12 +329,12 @@ private: }; void -BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& BucketLock, - ZenCacheDiskLayer::CacheBucket& Bucket, - std::filesystem::path ManifestPath, - ZenCacheDiskLayer::CacheBucket::IndexMap& Index, - std::vector<AccessTime>& AccessTimes, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads) +BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& BucketLock, + ZenCacheDiskLayer::CacheBucket& Bucket, + std::filesystem::path ManifestPath, + ZenCacheDiskLayer::CacheBucket::IndexMap& Index, + eastl::vector<AccessTime>& AccessTimes, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads) { if (Manifest["UsingMetaFile"sv].AsBool()) { @@ -348,8 +348,8 @@ BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope& Buck Stopwatch Timer; const auto _ = MakeGuard([&] { ZEN_INFO("parsed store manifest '{}' in {}", ManifestPath, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); - const uint64_t Count = Manifest["Count"sv].AsUInt64(0); - std::vector<PayloadIndex> KeysIndexes; + const uint64_t Count = Manifest["Count"sv].AsUInt64(0); + eastl::vector<PayloadIndex> KeysIndexes; KeysIndexes.reserve(Count); CbArrayView KeyArray = Manifest["Keys"sv].AsArrayView(); @@ -425,11 +425,11 @@ BucketManifestSerializer::GenerateNewManifest(std::filesystem::path ManifestPath } IoBuffer -BucketManifestSerializer::MakeManifest(const Oid& BucketId, - ZenCacheDiskLayer::CacheBucket::IndexMap&& Index, - std::vector<AccessTime>&& AccessTimes, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&& Payloads, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas) +BucketManifestSerializer::MakeManifest(const Oid& BucketId, + ZenCacheDiskLayer::CacheBucket::IndexMap&& Index, + eastl::vector<AccessTime>&& AccessTimes, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&& Payloads, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas) { using namespace std::literals; @@ -519,12 +519,12 @@ BucketManifestSerializer::MakeSidecarManifest(const Oid& BucketId, uint64_t Entr } bool -BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& BucketLock, - ZenCacheDiskLayer::CacheBucket& Bucket, - std::filesystem::path SidecarPath, - ZenCacheDiskLayer::CacheBucket::IndexMap& Index, - std::vector<AccessTime>& AccessTimes, - std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads) +BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& BucketLock, + ZenCacheDiskLayer::CacheBucket& Bucket, + std::filesystem::path SidecarPath, + ZenCacheDiskLayer::CacheBucket::IndexMap& Index, + eastl::vector<AccessTime>& AccessTimes, + eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads) { ZEN_TRACE_CPU("Z$::ReadSidecarFile"); @@ -621,12 +621,12 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& B void BucketManifestSerializer::WriteSidecarFile(RwLock::SharedLockScope&, - const std::filesystem::path& SidecarPath, - uint64_t SnapshotLogPosition, - const ZenCacheDiskLayer::CacheBucket::IndexMap& Index, - const std::vector<AccessTime>& AccessTimes, - const std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads, - const std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas) + const std::filesystem::path& SidecarPath, + uint64_t SnapshotLogPosition, + const ZenCacheDiskLayer::CacheBucket::IndexMap& Index, + const eastl::vector<AccessTime>& AccessTimes, + const eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads, + const eastl::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas) { ZEN_TRACE_CPU("Z$::WriteSidecarFile"); @@ -653,8 +653,8 @@ BucketManifestSerializer::WriteSidecarFile(RwLock::SharedLockScope&, // BasicFileWriter SidecarWriter(SidecarFile, 128 * 1024); - std::vector<ManifestData> ManifestDataBuffer; - const size_t MaxManifestDataBufferCount = Min(Index.size(), 8192u); // 512 Kb + eastl::vector<ManifestData> ManifestDataBuffer; + const size_t MaxManifestDataBufferCount = Min(Index.size(), 8192u); // 512 Kb ManifestDataBuffer.reserve(MaxManifestDataBufferCount); for (auto& Kv : Index) { @@ -1221,20 +1221,20 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, struct ZenCacheDiskLayer::CacheBucket::PutBatchHandle { - PutBatchHandle(std::vector<bool>& OutResults) : OutResults(OutResults) {} + PutBatchHandle(eastl::vector<bool>& OutResults) : OutResults(OutResults) {} struct Entry { - std::vector<IoHash> HashKeyAndReferences; + eastl::vector<IoHash> HashKeyAndReferences; }; - std::vector<IoBuffer> Buffers; - std::vector<Entry> Entries; - std::vector<size_t> EntryResultIndexes; + eastl::vector<IoBuffer> Buffers; + eastl::vector<Entry> Entries; + eastl::vector<size_t> EntryResultIndexes; - std::vector<bool>& OutResults; + eastl::vector<bool>& OutResults; }; ZenCacheDiskLayer::CacheBucket::PutBatchHandle* -ZenCacheDiskLayer::CacheBucket::BeginPutBatch(std::vector<bool>& OutResults) +ZenCacheDiskLayer::CacheBucket::BeginPutBatch(eastl::vector<bool>& OutResults) { ZEN_TRACE_CPU("Z$::Bucket::BeginPutBatch"); return new PutBatchHandle(OutResults); @@ -1250,7 +1250,7 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept ZEN_ASSERT(Batch); if (!Batch->Buffers.empty()) { - std::vector<uint8_t> EntryFlags; + eastl::vector<uint8_t> EntryFlags; for (const IoBuffer& Buffer : Batch->Buffers) { uint8_t Flags = 0; @@ -1268,13 +1268,13 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept size_t IndexOffset = 0; m_BlockStore.WriteChunks(Batch->Buffers, m_Configuration.PayloadAlignment, [&](std::span<BlockStoreLocation> Locations) { ZEN_MEMSCOPE(GetCacheDiskTag()); - std::vector<DiskIndexEntry> DiskEntries; + eastl::vector<DiskIndexEntry> DiskEntries; { RwLock::ExclusiveLockScope IndexLock(m_IndexLock); for (size_t Index = 0; Index < Locations.size(); Index++) { DiskLocation Location(Locations[Index], m_Configuration.PayloadAlignment, EntryFlags[IndexOffset + Index]); - const std::vector<IoHash>& HashKeyAndReferences = Batch->Entries[IndexOffset + Index].HashKeyAndReferences; + const eastl::vector<IoHash>& HashKeyAndReferences = Batch->Entries[IndexOffset + Index].HashKeyAndReferences; ZEN_ASSERT(HashKeyAndReferences.size() > 1); const IoHash HashKey = HashKeyAndReferences[0]; DiskEntries.push_back({.Key = HashKey, .Location = Location}); @@ -1337,8 +1337,8 @@ struct ZenCacheDiskLayer::CacheBucket::GetBatchHandle ~GetBatchHandle() {} - std::vector<IoHash> Keys; - std::vector<size_t> ResultIndexes; + eastl::vector<IoHash> Keys; + eastl::vector<size_t> ResultIndexes; ZenCacheValueVec_t& OutResults; }; @@ -1829,7 +1829,7 @@ ZenCacheDiskLayer::CacheBucket::MemCacheTrim(GcClock::TimePoint ExpireTime) uint64_t Trimmed = 0; GcClock::Tick ExpireTicks = ExpireTime.time_since_epoch().count(); - std::vector<IoBuffer> PurgedBuffers; + eastl::vector<IoBuffer> PurgedBuffers; { RwLock::ExclusiveLockScope IndexLock(m_IndexLock); @@ -1868,12 +1868,12 @@ ZenCacheDiskLayer::CacheBucket::MemCacheTrim(GcClock::TimePoint ExpireTime) } void -ZenCacheDiskLayer::CacheBucket::GetUsageByAccess(GcClock::TimePoint Now, GcClock::Duration MaxAge, std::vector<uint64_t>& InOutUsageSlots) +ZenCacheDiskLayer::CacheBucket::GetUsageByAccess(GcClock::TimePoint Now, GcClock::Duration MaxAge, eastl::vector<uint64_t>& InOutUsageSlots) { ZEN_TRACE_CPU("Z$::Bucket::GetUsageByAccess"); - std::vector<uint64_t> PayloadSizes; - std::vector<AccessTime> AccessTimes; + eastl::vector<uint64_t> PayloadSizes; + eastl::vector<AccessTime> AccessTimes; size_t SlotCount = InOutUsageSlots.capacity(); @@ -1925,7 +1925,7 @@ ZenCacheDiskLayer::CacheBucket::Drop() RwLock::ExclusiveLockScope _(m_IndexLock); - std::vector<std::unique_ptr<RwLock::ExclusiveLockScope>> ShardLocks; + eastl::vector<std::unique_ptr<RwLock::ExclusiveLockScope>> ShardLocks; ShardLocks.reserve(256); for (RwLock& Lock : m_ShardedLocks) { @@ -1989,10 +1989,10 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl if (UseLegacyScheme) { - std::vector<AccessTime> AccessTimes; - std::vector<BucketPayload> Payloads; - std::vector<BucketMetaData> MetaDatas; - IndexMap Index; + eastl::vector<AccessTime> AccessTimes; + eastl::vector<BucketPayload> Payloads; + eastl::vector<BucketMetaData> MetaDatas; + IndexMap Index; { RwLock::SharedLockScope IndexLock(m_IndexLock); @@ -2104,14 +2104,14 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) NiceRate(VerifiedChunkBytes, DurationMs)); }); - RwLock BadKeysLock; - std::vector<IoHash> BadKeys; - auto ReportBadKey = [&](const IoHash& Key) { BadKeysLock.WithExclusiveLock([&]() { BadKeys.push_back(Key); }); }; + RwLock BadKeysLock; + eastl::vector<IoHash> BadKeys; + auto ReportBadKey = [&](const IoHash& Key) { BadKeysLock.WithExclusiveLock([&]() { BadKeys.push_back(Key); }); }; try { - std::vector<BlockStoreLocation> ChunkLocations; - std::vector<IoHash> ChunkIndexToChunkHash; + eastl::vector<BlockStoreLocation> ChunkLocations; + eastl::vector<IoHash> ChunkIndexToChunkHash; RwLock::SharedLockScope _(m_IndexLock); @@ -2255,7 +2255,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) { // Deal with bad chunks by removing them from our lookup map - std::vector<DiskIndexEntry> LogEntries; + eastl::vector<DiskIndexEntry> LogEntries; LogEntries.reserve(BadKeys.size()); { @@ -2299,11 +2299,11 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) // Clean up m_AccessTimes and m_Payloads vectors { - std::vector<BucketPayload> Payloads; - std::vector<AccessTime> AccessTimes; - std::vector<BucketMetaData> MetaDatas; - std::vector<MemCacheData> MemCachedPayloads; - IndexMap Index; + eastl::vector<BucketPayload> Payloads; + eastl::vector<AccessTime> AccessTimes; + eastl::vector<BucketMetaData> MetaDatas; + eastl::vector<MemCacheData> MemCachedPayloads; + IndexMap Index; { RwLock::ExclusiveLockScope IndexLock(m_IndexLock); @@ -2348,8 +2348,8 @@ ZenCacheDiskLayer::CacheBucket::EntryCount() const CacheValueDetails::ValueDetails ZenCacheDiskLayer::CacheBucket::GetValueDetails(RwLock::SharedLockScope& IndexLock, const IoHash& Key, PayloadIndex Index) const { - std::vector<IoHash> Attachments; - const BucketPayload& Payload = m_Payloads[Index]; + eastl::vector<IoHash> Attachments; + const BucketPayload& Payload = m_Payloads[Index]; if (Payload.Location.IsFlagSet(DiskLocation::kStructured)) { IoBuffer Value = Payload.Location.IsFlagSet(DiskLocation::kStandaloneFile) ? GetStandaloneCacheValue(Payload.Location, Key) @@ -2661,7 +2661,7 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, OptionalBatchHandle->Entries.push_back({}); OptionalBatchHandle->EntryResultIndexes.push_back(OptionalBatchHandle->OutResults.size()); OptionalBatchHandle->OutResults.push_back(false); - std::vector<IoHash>& HashKeyAndReferences = OptionalBatchHandle->Entries.back().HashKeyAndReferences; + eastl::vector<IoHash>& HashKeyAndReferences = OptionalBatchHandle->Entries.back().HashKeyAndReferences; HashKeyAndReferences.reserve(1 + HashKeyAndReferences.size()); HashKeyAndReferences.push_back(HashKey); HashKeyAndReferences.insert(HashKeyAndReferences.end(), HashKeyAndReferences.begin(), HashKeyAndReferences.end()); @@ -2731,7 +2731,7 @@ class DiskBucketStoreCompactor : public GcStoreCompactor using CacheBucket = ZenCacheDiskLayer::CacheBucket; public: - DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys) + DiskBucketStoreCompactor(CacheBucket& Bucket, eastl::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys) : m_Bucket(Bucket) , m_ExpiredStandaloneKeys(std::move(ExpiredStandaloneKeys)) { @@ -2871,7 +2871,7 @@ public: { BlockStoreCompactState BlockCompactState; - std::vector<IoHash> BlockCompactStateKeys; + eastl::vector<IoHash> BlockCompactStateKeys; BlockCompactStateKeys.reserve(InlineEntryCount); BlockStore::BlockEntryCountMap BlocksToCompact = @@ -2913,7 +2913,7 @@ public: BlockCompactState, m_Bucket.m_Configuration.PayloadAlignment, [&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) { - std::vector<DiskIndexEntry> MovedEntries; + eastl::vector<DiskIndexEntry> MovedEntries; MovedEntries.reserve(MovedArray.size()); RwLock::ExclusiveLockScope _(m_Bucket.m_IndexLock); for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray) @@ -2965,8 +2965,8 @@ public: virtual std::string GetGcName(GcCtx& Ctx) override { return m_Bucket.GetGcName(Ctx); } private: - ZenCacheDiskLayer::CacheBucket& m_Bucket; - std::vector<std::pair<IoHash, uint64_t>> m_ExpiredStandaloneKeys; + ZenCacheDiskLayer::CacheBucket& m_Bucket; + eastl::vector<std::pair<IoHash, uint64_t>> m_ExpiredStandaloneKeys; }; GcStoreCompactor* @@ -3012,9 +3012,9 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count(); - std::vector<DiskIndexEntry> ExpiredEntries; - std::vector<std::pair<IoHash, uint64_t>> ExpiredStandaloneKeys; - uint64_t RemovedStandaloneSize = 0; + eastl::vector<DiskIndexEntry> ExpiredEntries; + eastl::vector<std::pair<IoHash, uint64_t>> ExpiredStandaloneKeys; + uint64_t RemovedStandaloneSize = 0; { RwLock::ExclusiveLockScope IndexLock(m_IndexLock); if (Ctx.IsCancelledFlag.load()) @@ -3078,11 +3078,11 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) if (Ctx.Settings.IsDeleteMode && !ExpiredEntries.empty()) { - std::vector<BucketPayload> Payloads; - std::vector<AccessTime> AccessTimes; - std::vector<BucketMetaData> MetaDatas; - std::vector<MemCacheData> MemCachedPayloads; - IndexMap Index; + eastl::vector<BucketPayload> Payloads; + eastl::vector<AccessTime> AccessTimes; + eastl::vector<BucketMetaData> MetaDatas; + eastl::vector<MemCacheData> MemCachedPayloads; + IndexMap Index; { RwLock::ExclusiveLockScope IndexLock(m_IndexLock); CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, Index); @@ -3101,7 +3101,7 @@ bool ZenCacheDiskLayer::CacheBucket::ReadAttachmentsFromMetaData(uint32_t BlockIndex, std::span<const IoHash> InlineKeys, std::span<const std::size_t> ChunkIndexes, - std::vector<IoHash>& OutReferences) const + eastl::vector<IoHash>& OutReferences) const { ZEN_TRACE_CPU("Z$::Bucket::ReadAttachmentsFromMetaData"); IoBuffer MetaDataPayload = m_BlockStore.GetMetaData(BlockIndex); @@ -3146,13 +3146,13 @@ ZenCacheDiskLayer::CacheBucket::ReadAttachmentsFromMetaData(uint32_t BlockI } bool -ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger, - std::atomic_bool& IsCancelledFlag, - bool StateIsAlreadyLocked, - bool ReadCacheAttachmentMetaData, - bool WriteCacheAttachmentMetaData, - std::vector<IoHash>& OutReferences, - ReferencesStats* OptionalOutReferencesStats) +ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger, + std::atomic_bool& IsCancelledFlag, + bool StateIsAlreadyLocked, + bool ReadCacheAttachmentMetaData, + bool WriteCacheAttachmentMetaData, + eastl::vector<IoHash>& OutReferences, + ReferencesStats* OptionalOutReferencesStats) { ZEN_TRACE_CPU("Z$::Bucket::GetReferencesLocked"); @@ -3168,11 +3168,11 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger, return false; }; - std::vector<std::pair<IoHash, DiskLocation>> StandaloneKeys; + eastl::vector<std::pair<IoHash, DiskLocation>> StandaloneKeys; { - std::vector<IoHash> InlineKeys; - std::vector<BlockStoreLocation> InlineLocations; - std::vector<std::vector<std::size_t>> InlineBlockChunkIndexes; + eastl::vector<IoHash> InlineKeys; + eastl::vector<BlockStoreLocation> InlineLocations; + eastl::vector<eastl::vector<std::size_t>> InlineBlockChunkIndexes; { std::unordered_map<uint32_t, std::size_t> BlockIndexToChunkIndexes; @@ -3229,7 +3229,7 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger, else { BlockIndexToChunkIndexes.insert_or_assign(ChunkLocation.BlockIndex, InlineBlockChunkIndexes.size()); - InlineBlockChunkIndexes.emplace_back(std::vector<size_t>{ChunkIndex}); + InlineBlockChunkIndexes.emplace_back(eastl::vector<size_t>{ChunkIndex}); } } } @@ -3237,17 +3237,17 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger, OutReferences.reserve(OutReferences.size() + InlineKeys.size() + StandaloneKeys.size()); // Make space for at least one attachment per record - for (const std::vector<std::size_t>& ChunkIndexes : InlineBlockChunkIndexes) + for (const eastl::vector<std::size_t>& ChunkIndexes : InlineBlockChunkIndexes) { ZEN_ASSERT(!ChunkIndexes.empty()); uint32_t BlockIndex = InlineLocations[ChunkIndexes[0]].BlockIndex; if (!ReadCacheAttachmentMetaData || !ReadAttachmentsFromMetaData(BlockIndex, InlineKeys, ChunkIndexes, OutReferences)) { - std::vector<IoHash> Keys; - std::vector<uint32_t> AttachmentCounts; - size_t PrecachedReferencesStart = OutReferences.size(); - size_t NextPrecachedReferencesStart = PrecachedReferencesStart; + eastl::vector<IoHash> Keys; + eastl::vector<uint32_t> AttachmentCounts; + size_t PrecachedReferencesStart = OutReferences.size(); + size_t NextPrecachedReferencesStart = PrecachedReferencesStart; bool WriteMetaData = WriteCacheAttachmentMetaData && !m_BlockStore.IsWriting(BlockIndex); if (WriteMetaData) @@ -3372,7 +3372,8 @@ public: NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); - m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_TrackedReferences = std::make_unique<std::vector<IoHash>>(); }); + m_CacheBucket.m_IndexLock.WithExclusiveLock( + [&]() { m_CacheBucket.m_TrackedReferences = std::make_unique<eastl::vector<IoHash>>(); }); bool Continue = m_CacheBucket.GetReferences(Ctx.Logger, Ctx.IsCancelledFlag, @@ -3448,12 +3449,12 @@ public: return UnusedReferences; } - CacheBucket& m_CacheBucket; - std::vector<IoHash> m_PrecachedReferences; - std::vector<IoHash> m_AddedReferences; + CacheBucket& m_CacheBucket; + eastl::vector<IoHash> m_PrecachedReferences; + eastl::vector<IoHash> m_AddedReferences; }; -std::vector<GcReferenceChecker*> +eastl::vector<GcReferenceChecker*> ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx) { ZEN_TRACE_CPU("Z$::Bucket::CreateReferenceCheckers"); @@ -3480,7 +3481,7 @@ ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx) return {new DiskBucketReferenceChecker(*this)}; } -std::vector<GcReferenceValidator*> +eastl::vector<GcReferenceValidator*> ZenCacheDiskLayer::CacheBucket::CreateReferenceValidators(GcCtx& /*Ctx*/) { return {}; @@ -3488,11 +3489,11 @@ ZenCacheDiskLayer::CacheBucket::CreateReferenceValidators(GcCtx& /*Ctx*/) void ZenCacheDiskLayer::CacheBucket::CompactState(RwLock::ExclusiveLockScope&, - std::vector<BucketPayload>& Payloads, - std::vector<AccessTime>& AccessTimes, - std::vector<BucketMetaData>& MetaDatas, - std::vector<MemCacheData>& MemCachedPayloads, - IndexMap& Index) + eastl::vector<BucketPayload>& Payloads, + eastl::vector<AccessTime>& AccessTimes, + eastl::vector<BucketMetaData>& MetaDatas, + eastl::vector<MemCacheData>& MemCachedPayloads, + IndexMap& Index) { ZEN_TRACE_CPU("Z$::Bucket::CompactState"); @@ -3650,7 +3651,7 @@ ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket) struct ZenCacheDiskLayer::PutBatchHandle { - PutBatchHandle(std::vector<bool>& OutResults) : OutResults(OutResults) {} + PutBatchHandle(eastl::vector<bool>& OutResults) : OutResults(OutResults) {} struct BucketHandle { CacheBucket* Bucket; @@ -3709,13 +3710,13 @@ struct ZenCacheDiskLayer::PutBatchHandle return NewBucketHandle; } - RwLock Lock; - std::vector<BucketHandle> BucketHandles; - std::vector<bool>& OutResults; + RwLock Lock; + eastl::vector<BucketHandle> BucketHandles; + eastl::vector<bool>& OutResults; }; ZenCacheDiskLayer::PutBatchHandle* -ZenCacheDiskLayer::BeginPutBatch(std::vector<bool>& OutResults) +ZenCacheDiskLayer::BeginPutBatch(eastl::vector<bool>& OutResults) { return new PutBatchHandle(OutResults); } @@ -3879,8 +3880,8 @@ ZenCacheDiskLayer::DiscoverBuckets() // Initialize buckets - std::vector<std::filesystem::path> BadBucketDirectories; - std::vector<std::filesystem::path> FoundBucketDirectories; + eastl::vector<std::filesystem::path> BadBucketDirectories; + eastl::vector<std::filesystem::path> FoundBucketDirectories; RwLock::ExclusiveLockScope _(m_Lock); @@ -4003,7 +4004,7 @@ ZenCacheDiskLayer::Drop() RwLock::ExclusiveLockScope _(m_Lock); - std::vector<std::unique_ptr<CacheBucket>> Buckets; + eastl::vector<std::unique_ptr<CacheBucket>> Buckets; Buckets.reserve(m_Buckets.size()); while (!m_Buckets.empty()) { @@ -4025,15 +4026,15 @@ ZenCacheDiskLayer::Flush() ZEN_MEMSCOPE(GetCacheDiskTag()); ZEN_TRACE_CPU("Z$::Flush"); - std::vector<CacheBucket*> Buckets; - Stopwatch Timer; - const auto _ = MakeGuard([&] { - if (Buckets.empty()) - { - return; - } - ZEN_INFO("Flushed {} buckets at '{}' in {}", Buckets.size(), m_RootDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }); + eastl::vector<CacheBucket*> Buckets; + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (Buckets.empty()) + { + return; + } + ZEN_INFO("Flushed {} buckets at '{}' in {}", Buckets.size(), m_RootDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); { RwLock::SharedLockScope __(m_Lock); @@ -4086,7 +4087,7 @@ ZenCacheDiskLayer::ScrubStorage(ScrubContext& Ctx) RwLock::SharedLockScope _(m_Lock); { - std::vector<std::future<void>> Results; + eastl::vector<std::future<void>> Results; Results.reserve(m_Buckets.size()); for (auto& Kv : m_Buckets) @@ -4211,10 +4212,10 @@ ZenCacheDiskLayer::GetValueDetails(const std::string_view BucketFilter, const st return Details; } -std::vector<RwLock::SharedLockScope> +eastl::vector<RwLock::SharedLockScope> ZenCacheDiskLayer::GetGcReferencerLocks() { - std::vector<RwLock::SharedLockScope> Locks; + eastl::vector<RwLock::SharedLockScope> Locks; Locks.emplace_back(RwLock::SharedLockScope(m_Lock)); for (auto& Kv : m_Buckets) { @@ -4230,7 +4231,7 @@ ZenCacheDiskLayer::EnableUpdateCapture() if (m_UpdateCaptureRefCounter == 0) { ZEN_ASSERT(!m_CapturedBuckets); - m_CapturedBuckets = std::make_unique<std::vector<std::string>>(); + m_CapturedBuckets = std::make_unique<eastl::vector<std::string>>(); } else { @@ -4254,7 +4255,7 @@ ZenCacheDiskLayer::DisableUpdateCapture() }); } -std::vector<std::string> +eastl::vector<std::string> ZenCacheDiskLayer::GetCapturedBucketsLocked() { if (m_CapturedBuckets) @@ -4332,11 +4333,11 @@ ZenCacheDiskLayer::MemCacheTrim() const std::chrono::seconds MaxAge = std::chrono::seconds(m_Configuration.MemCacheMaxAgeSeconds); - static const size_t UsageSlotCount = 2048; - std::vector<uint64_t> UsageSlots; + static const size_t UsageSlotCount = 2048; + eastl::vector<uint64_t> UsageSlots; UsageSlots.reserve(UsageSlotCount); - std::vector<CacheBucket*> Buckets; + eastl::vector<CacheBucket*> Buckets; { RwLock::SharedLockScope __(m_Lock); Buckets.reserve(m_Buckets.size()); @@ -4371,7 +4372,7 @@ ZenCacheDiskLayer::MemCacheTrim() } uint64_t -ZenCacheDiskLayer::MemCacheTrim(std::vector<CacheBucket*>& Buckets, GcClock::TimePoint ExpireTime) +ZenCacheDiskLayer::MemCacheTrim(eastl::vector<CacheBucket*>& Buckets, GcClock::TimePoint ExpireTime) { if (m_Configuration.MemCacheTargetFootprintBytes == 0) { diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index 97e26a38d..481a8906e 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -107,7 +107,7 @@ namespace cache::detail { struct RecordBody { IoBuffer CacheValue; - std::vector<RecordValue> Values; + eastl::vector<RecordValue> Values; const UpstreamEndpointInfo* Source = nullptr; CachePolicy DownstreamPolicy; bool Exists = false; @@ -362,12 +362,12 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag uint64_t RecordObjectSize = Record.GetSize(); uint64_t TransferredSize = RecordObjectSize; - AttachmentCount Count; - size_t NumAttachments = Package->GetAttachments().size(); - std::vector<IoHash> ValidAttachments; - std::vector<IoHash> ReferencedAttachments; - std::vector<IoBuffer> WriteAttachmentBuffers; - std::vector<IoHash> WriteRawHashes; + AttachmentCount Count; + size_t NumAttachments = Package->GetAttachments().size(); + eastl::vector<IoHash> ValidAttachments; + eastl::vector<IoHash> ReferencedAttachments; + eastl::vector<IoBuffer> WriteAttachmentBuffers; + eastl::vector<IoHash> WriteRawHashes; ValidAttachments.reserve(NumAttachments); WriteAttachmentBuffers.reserve(NumAttachments); WriteRawHashes.reserve(NumAttachments); @@ -429,7 +429,7 @@ CacheRpcHandler::PutCacheRecord(PutRequestData& Request, const CbPackage* Packag if (!WriteAttachmentBuffers.empty()) { - std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + eastl::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); for (size_t Index = 0; Index < InsertResults.size(); Index++) { if (InsertResults[Index].New) @@ -563,11 +563,11 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb Request.RecordObject = CbObjectView(Request.RecordCacheValue.GetData()); ParseValues(Request); - Request.Complete = true; - size_t ValueCount = Request.Values.size(); - std::vector<IoHash> CidHashes; - std::vector<size_t> RequestValueIndexes; - const bool DoBatch = ValueCount > 7; + Request.Complete = true; + size_t ValueCount = Request.Values.size(); + eastl::vector<IoHash> CidHashes; + eastl::vector<size_t> RequestValueIndexes; + const bool DoBatch = ValueCount > 7; if (DoBatch) { CidHashes.reserve(ValueCount); @@ -703,7 +703,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb if (!UpstreamIndexes.empty()) { - std::vector<CacheKeyRequest*> UpstreamRequests; + eastl::vector<CacheKeyRequest*> UpstreamRequests; UpstreamRequests.reserve(UpstreamIndexes.size()); for (size_t Index : UpstreamIndexes) { @@ -753,7 +753,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::StoreLocal) && AreDiskWritesAllowed(); if (StoreLocal) { - std::vector<IoHash> ReferencedAttachments; + eastl::vector<IoHash> ReferencedAttachments; ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) { const IoHash ValueHash = HashView.AsHash(); ReferencedAttachments.push_back(ValueHash); @@ -912,7 +912,7 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con const bool HasUpstream = m_UpstreamCache.IsActive(); CbArrayView RequestsArray = Params["Requests"sv].AsArrayView(); - std::vector<bool> BatchResults; + eastl::vector<bool> BatchResults; eastl::fixed_vector<size_t, 32> BatchResultIndexes; eastl::fixed_vector<bool, 32> Results; eastl::fixed_vector<CacheKey, 32> UpstreamCacheKeys; @@ -1200,8 +1200,8 @@ CacheRpcHandler::HandleRpcGetCacheValues(const CacheRequestContext& Context, CbO if (!RemoteRequestIndexes.empty()) { - std::vector<CacheValueRequest> RequestedRecordsData; - std::vector<CacheValueRequest*> CacheValueRequests; + eastl::vector<CacheValueRequest> RequestedRecordsData; + eastl::vector<CacheValueRequest*> CacheValueRequests; RequestedRecordsData.reserve(RemoteRequestIndexes.size()); CacheValueRequests.reserve(RemoteRequestIndexes.size()); for (size_t Index : RemoteRequestIndexes) @@ -1320,14 +1320,14 @@ CacheRpcHandler::HandleRpcGetCacheChunks(const CacheRequestContext& Context, Rpc ZEN_TRACE_CPU("Z$::RpcGetCacheChunks"); using namespace cache::detail; - std::string Namespace; - std::vector<CacheKeyRequest> RecordKeys; // Data about a Record necessary to identify it to the upstream - std::vector<RecordBody> Records; // Scratch-space data about a Record when fulfilling RecordRequests - std::vector<CacheChunkRequest> RequestKeys; // Data about a ChunkRequest necessary to identify it to the upstream - std::vector<ChunkRequest> Requests; // Intermediate and result data about a ChunkRequest - std::vector<ChunkRequest*> RecordRequests; // The ChunkRequests that are requesting a subvalue from a Record Key - std::vector<ChunkRequest*> ValueRequests; // The ChunkRequests that are requesting a Value Key - std::vector<CacheChunkRequest*> UpstreamChunks; // ChunkRequests that we need to send to the upstream + std::string Namespace; + eastl::vector<CacheKeyRequest> RecordKeys; // Data about a Record necessary to identify it to the upstream + eastl::vector<RecordBody> Records; // Scratch-space data about a Record when fulfilling RecordRequests + eastl::vector<CacheChunkRequest> RequestKeys; // Data about a ChunkRequest necessary to identify it to the upstream + eastl::vector<ChunkRequest> Requests; // Intermediate and result data about a ChunkRequest + eastl::vector<ChunkRequest*> RecordRequests; // The ChunkRequests that are requesting a subvalue from a Record Key + eastl::vector<ChunkRequest*> ValueRequests; // The ChunkRequests that are requesting a Value Key + eastl::vector<CacheChunkRequest*> UpstreamChunks; // ChunkRequests that we need to send to the upstream // Parse requests from the CompactBinary body of the RpcRequest and divide it into RecordRequests and ValueRequests if (!ParseGetCacheChunksRequest(Namespace, RecordKeys, Records, RequestKeys, Requests, RecordRequests, ValueRequests, RpcRequest)) @@ -1350,14 +1350,14 @@ CacheRpcHandler::HandleRpcGetCacheChunks(const CacheRequestContext& Context, Rpc } bool -CacheRpcHandler::ParseGetCacheChunksRequest(std::string& Namespace, - std::vector<CacheKeyRequest>& RecordKeys, - std::vector<cache::detail::RecordBody>& Records, - std::vector<CacheChunkRequest>& RequestKeys, - std::vector<cache::detail::ChunkRequest>& Requests, - std::vector<cache::detail::ChunkRequest*>& RecordRequests, - std::vector<cache::detail::ChunkRequest*>& ValueRequests, - CbObjectView RpcRequest) +CacheRpcHandler::ParseGetCacheChunksRequest(std::string& Namespace, + eastl::vector<CacheKeyRequest>& RecordKeys, + eastl::vector<cache::detail::RecordBody>& Records, + eastl::vector<CacheChunkRequest>& RequestKeys, + eastl::vector<cache::detail::ChunkRequest>& Requests, + eastl::vector<cache::detail::ChunkRequest*>& RecordRequests, + eastl::vector<cache::detail::ChunkRequest*>& ValueRequests, + CbObjectView RpcRequest) { ZEN_TRACE_CPU("Z$::ParseGetCacheChunksRequest"); @@ -1470,12 +1470,12 @@ CacheRpcHandler::ParseGetCacheChunksRequest(std::string& Namespace, } void -CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, - std::string_view Namespace, - std::vector<CacheKeyRequest>& RecordKeys, - std::vector<cache::detail::RecordBody>& Records, - std::vector<cache::detail::ChunkRequest*>& RecordRequests, - std::vector<CacheChunkRequest*>& OutUpstreamChunks) +CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, + std::string_view Namespace, + eastl::vector<CacheKeyRequest>& RecordKeys, + eastl::vector<cache::detail::RecordBody>& Records, + eastl::vector<cache::detail::ChunkRequest*>& RecordRequests, + eastl::vector<CacheChunkRequest*>& OutUpstreamChunks) { ZEN_TRACE_CPU("Z$::GetLocalCacheRecords"); @@ -1483,7 +1483,7 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, const bool HasUpstream = m_UpstreamCache.IsActive(); // TODO: BatchGet records? - std::vector<CacheKeyRequest*> UpstreamRecordRequests; + eastl::vector<CacheKeyRequest*> UpstreamRecordRequests; for (size_t RecordIndex = 0; RecordIndex < Records.size(); ++RecordIndex) { ZEN_TRACE_CPU("Z$::GetLocalCacheRecords::Record"); @@ -1536,7 +1536,7 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, bool StoreLocal = EnumHasAllFlags(Record.DownstreamPolicy, CachePolicy::StoreLocal) && AreDiskWritesAllowed(); if (StoreLocal) { - std::vector<IoHash> ReferencedAttachments; + eastl::vector<IoHash> ReferencedAttachments; ObjectBuffer.IterateAttachments([&ReferencedAttachments](CbFieldView HashView) { const IoHash ValueHash = HashView.AsHash(); ReferencedAttachments.push_back(ValueHash); @@ -1637,10 +1637,10 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, } void -CacheRpcHandler::GetLocalCacheValues(const CacheRequestContext& Context, - std::string_view Namespace, - std::vector<cache::detail::ChunkRequest*>& ValueRequests, - std::vector<CacheChunkRequest*>& OutUpstreamChunks) +CacheRpcHandler::GetLocalCacheValues(const CacheRequestContext& Context, + std::string_view Namespace, + eastl::vector<cache::detail::ChunkRequest*>& ValueRequests, + eastl::vector<CacheChunkRequest*>& OutUpstreamChunks) { ZEN_TRACE_CPU("Z$::GetLocalCacheValues"); @@ -1717,11 +1717,11 @@ CacheRpcHandler::GetLocalCacheValues(const CacheRequestContext& Context, } void -CacheRpcHandler::GetUpstreamCacheChunks(const CacheRequestContext& Context, - std::string_view Namespace, - std::vector<CacheChunkRequest*>& UpstreamChunks, - std::vector<CacheChunkRequest>& RequestKeys, - std::vector<cache::detail::ChunkRequest>& Requests) +CacheRpcHandler::GetUpstreamCacheChunks(const CacheRequestContext& Context, + std::string_view Namespace, + eastl::vector<CacheChunkRequest*>& UpstreamChunks, + eastl::vector<CacheChunkRequest>& RequestKeys, + eastl::vector<cache::detail::ChunkRequest>& Requests) { if (UpstreamChunks.empty()) { @@ -1790,7 +1790,7 @@ CbPackage CacheRpcHandler::WriteGetCacheChunksResponse([[maybe_unused]] const CacheRequestContext& Context, std::string_view Namespace, RpcAcceptOptions AcceptOptions, - std::vector<cache::detail::ChunkRequest>& Requests) + eastl::vector<cache::detail::ChunkRequest>& Requests) { ZEN_TRACE_CPU("Z$::WriteGetCacheChunksResponse"); diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 7d277329e..87c95f8a2 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -154,7 +154,7 @@ struct ZenCacheNamespace::PutBatchHandle }; ZenCacheNamespace::PutBatchHandle* -ZenCacheNamespace::BeginPutBatch(std::vector<bool>& OutResult) +ZenCacheNamespace::BeginPutBatch(eastl::vector<bool>& OutResult) { ZenCacheNamespace::PutBatchHandle* Handle = new ZenCacheNamespace::PutBatchHandle; Handle->DiskLayerHandle = m_DiskLayer.BeginPutBatch(OutResult); @@ -366,7 +366,7 @@ ZenCacheNamespace::GetValueDetails(const std::string_view BucketFilter, const st return m_DiskLayer.GetValueDetails(BucketFilter, ValueFilter); } -std::vector<RwLock::SharedLockScope> +eastl::vector<RwLock::SharedLockScope> ZenCacheNamespace::GetGcReferencerLocks() { return m_DiskLayer.GetGcReferencerLocks(); @@ -426,7 +426,7 @@ ZenCacheStore::ZenCacheStore(GcManager& Gc, DirectoryContent DirContent; GetDirectoryContent(m_BasePath, DirectoryContentFlags::IncludeDirs, DirContent); - std::vector<std::string> Namespaces; + eastl::vector<std::string> Namespaces; for (const std::filesystem::path& DirPath : DirContent.Directories) { std::string DirName = PathToUtf8(DirPath.filename()); @@ -480,7 +480,7 @@ ZenCacheStore::LogWorker() auto Log = [&ZCacheLog]() -> LoggerRef { return ZCacheLog; }; - std::vector<AccessLogItem> Items; + eastl::vector<AccessLogItem> Items; while (true) { try @@ -553,7 +553,7 @@ ZenCacheStore::LogWorker() } } -ZenCacheStore::PutBatch::PutBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, std::vector<bool>& OutResult) +ZenCacheStore::PutBatch::PutBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, eastl::vector<bool>& OutResult) : m_CacheStore(CacheStore) { ZEN_MEMSCOPE(GetCacheStoreTag()); @@ -904,10 +904,10 @@ ZenCacheStore::FindNamespace(std::string_view Namespace) const return nullptr; } -std::vector<std::string> +eastl::vector<std::string> ZenCacheStore::GetNamespaces() { - std::vector<std::string> Namespaces; + eastl::vector<std::string> Namespaces; IterateNamespaces([&](std::string_view Namespace, ZenCacheNamespace&) { Namespaces.push_back(std::string(Namespace)); }); @@ -917,7 +917,7 @@ ZenCacheStore::GetNamespaces() void ZenCacheStore::IterateNamespaces(const std::function<void(std::string_view Namespace, ZenCacheNamespace& Store)>& Callback) const { - std::vector<std::pair<std::string, ZenCacheNamespace&>> Namespaces; + eastl::vector<std::pair<std::string, ZenCacheNamespace&>> Namespaces; { RwLock::SharedLockScope _(m_NamespacesLock); Namespaces.reserve(m_Namespaces.size()); @@ -1042,18 +1042,18 @@ ZenCacheStore::GetBucketInfo(std::string_view NamespaceName, std::string_view Bu return {}; } -std::vector<RwLock::SharedLockScope> +eastl::vector<RwLock::SharedLockScope> ZenCacheStore::LockState(GcCtx& Ctx) { ZEN_TRACE_CPU("CacheStore::LockState"); auto Log = [&Ctx]() { return Ctx.Logger; }; - std::vector<RwLock::SharedLockScope> Locks; + eastl::vector<RwLock::SharedLockScope> Locks; Locks.emplace_back(RwLock::SharedLockScope(m_NamespacesLock)); for (auto& NamespaceIt : m_Namespaces) { - std::vector<RwLock::SharedLockScope> NamespaceLocks = NamespaceIt.second->GetGcReferencerLocks(); + eastl::vector<RwLock::SharedLockScope> NamespaceLocks = NamespaceIt.second->GetGcReferencerLocks(); for (auto It = std::make_move_iterator(NamespaceLocks.begin()); It != std::make_move_iterator(NamespaceLocks.end()); It++) { Locks.emplace_back(std::move(*It)); @@ -1065,12 +1065,12 @@ ZenCacheStore::LockState(GcCtx& Ctx) void ZenCacheStore::EnableUpdateCapture() { - std::vector<ZenCacheNamespace*> Namespaces; + eastl::vector<ZenCacheNamespace*> Namespaces; m_NamespacesLock.WithExclusiveLock([&]() { if (m_UpdateCaptureRefCounter == 0) { ZEN_ASSERT(!m_CapturedNamespaces); - m_CapturedNamespaces = std::make_unique<std::vector<std::string>>(); + m_CapturedNamespaces = std::make_unique<eastl::vector<std::string>>(); } else { @@ -1093,7 +1093,7 @@ ZenCacheStore::EnableUpdateCapture() void ZenCacheStore::DisableUpdateCapture() { - std::vector<ZenCacheNamespace*> Namespaces; + eastl::vector<ZenCacheNamespace*> Namespaces; m_NamespacesLock.WithExclusiveLock([&]() { ZEN_ASSERT(m_CapturedNamespaces); ZEN_ASSERT(m_UpdateCaptureRefCounter > 0); @@ -1114,7 +1114,7 @@ ZenCacheStore::DisableUpdateCapture() } } -std::vector<std::string> +eastl::vector<std::string> ZenCacheStore::GetCapturedNamespacesLocked() { if (m_CapturedNamespaces) @@ -1173,7 +1173,7 @@ public: Stopwatch Timer; - std::vector<ZenCacheDiskLayer::CacheBucket*> AddedBuckets; + eastl::vector<ZenCacheDiskLayer::CacheBucket*> AddedBuckets; const auto _ = MakeGuard([&] { if (!Ctx.Settings.Verbose) @@ -1187,7 +1187,7 @@ public: AddedBuckets.size()); }); - std::vector<std::string> AddedNamespaces = m_CacheStore.GetCapturedNamespacesLocked(); + eastl::vector<std::string> AddedNamespaces = m_CacheStore.GetCapturedNamespacesLocked(); for (const std::string& AddedNamespace : AddedNamespaces) { @@ -1202,8 +1202,8 @@ public: } for (auto& NamepaceKV : m_CacheStore.m_Namespaces) { - ZenCacheNamespace& Namespace = *NamepaceKV.second; - std::vector<std::string> NamespaceAddedBuckets = Namespace.m_DiskLayer.GetCapturedBucketsLocked(); + ZenCacheNamespace& Namespace = *NamepaceKV.second; + eastl::vector<std::string> NamespaceAddedBuckets = Namespace.m_DiskLayer.GetCapturedBucketsLocked(); for (const std::string& AddedBucketName : NamespaceAddedBuckets) { if (auto It = Namespace.m_DiskLayer.m_Buckets.find(AddedBucketName); It != Namespace.m_DiskLayer.m_Buckets.end()) @@ -1258,31 +1258,31 @@ public: } private: - ZenCacheStore& m_CacheStore; - std::vector<IoHash> m_References; + ZenCacheStore& m_CacheStore; + eastl::vector<IoHash> m_References; }; -std::vector<GcReferenceChecker*> +eastl::vector<GcReferenceChecker*> ZenCacheStore::CreateReferenceCheckers(GcCtx& Ctx) { ZEN_TRACE_CPU("CacheStore::CreateReferenceCheckers"); auto Log = [&Ctx]() { return Ctx.Logger; }; - Stopwatch Timer; - const auto _ = MakeGuard([&] { - if (!Ctx.Settings.Verbose) - { - return; - } - ZEN_INFO("GCV2: cachestore [CREATE CHECKERS] '{}': completed in {}", m_BasePath, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }); - std::vector<GcReferenceChecker*> Checkers; + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: cachestore [CREATE CHECKERS] '{}': completed in {}", m_BasePath, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + eastl::vector<GcReferenceChecker*> Checkers; Checkers.emplace_back(new CacheStoreReferenceChecker(*this)); return Checkers; } -std::vector<GcReferenceValidator*> +eastl::vector<GcReferenceValidator*> ZenCacheStore::CreateReferenceValidators(GcCtx& /*Ctx*/) { return {}; @@ -1307,9 +1307,9 @@ namespace testutils { std::pair<Oid, IoBuffer> CreateBinaryBlob(size_t Size) { return {Oid::NewOid(), CreateRandomBlob(Size)}; } - std::vector<std::pair<Oid, CompressedBuffer>> CreateCompressedAttachment(CidStore& Store, const std::span<const size_t>& Sizes) + eastl::vector<std::pair<Oid, CompressedBuffer>> CreateCompressedAttachment(CidStore& Store, const std::span<const size_t>& Sizes) { - std::vector<std::pair<Oid, CompressedBuffer>> Result; + eastl::vector<std::pair<Oid, CompressedBuffer>> Result; Result.reserve(Sizes.size()); for (size_t Size : Sizes) { @@ -1386,7 +1386,7 @@ TEST_CASE("cachestore.size") auto JobQueue = MakeJobQueue(1, "testqueue"); const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size); CbObjectWriter Writer; @@ -1410,7 +1410,7 @@ TEST_CASE("cachestore.size") IoBuffer Buffer = CacheValue.GetBuffer().AsIoBuffer(); Buffer.SetContentType(ZenContentType::kCbObject); - std::vector<std::pair<std::string, IoHash>> Keys; + eastl::vector<std::pair<std::string, IoHash>> Keys; for (size_t Key = 0; Key < Count; ++Key) { @@ -1710,7 +1710,7 @@ TEST_CASE("cachestore.namespaces") using namespace testutils; const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size); CbObjectWriter Writer; @@ -1778,7 +1778,7 @@ TEST_CASE("cachestore.drop.bucket") using namespace testutils; const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size); CbObjectWriter Writer; @@ -1855,7 +1855,7 @@ TEST_CASE("cachestore.drop.namespace") const CacheRequestContext Context; const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size); CbObjectWriter Writer; @@ -1945,7 +1945,7 @@ TEST_CASE("cachestore.blocked.disklayer.put") GcStorageSize CacheSize; const auto CreateCacheValue = [](size_t Size) -> CbObject { - std::vector<uint8_t> Buf; + eastl::vector<uint8_t> Buf; Buf.resize(Size, Size & 0xff); CbObjectWriter Writer; @@ -1999,11 +1999,11 @@ TEST_CASE("cachestore.scrub") struct CacheRecord { - IoBuffer Record; - std::vector<CompressedBuffer> Attachments; + IoBuffer Record; + eastl::vector<CompressedBuffer> Attachments; }; - auto CreateCacheRecord = [](bool Structured, std::string_view Bucket, const IoHash& Key, const std::vector<size_t>& AttachmentSizes) { + auto CreateCacheRecord = [](bool Structured, std::string_view Bucket, const IoHash& Key, const eastl::vector<size_t>& AttachmentSizes) { CacheRecord Result; if (Structured) { @@ -2054,27 +2054,29 @@ TEST_CASE("cachestore.scrub") CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; CidStore.Initialize(CidConfig); - auto CreateRecords = - [&](bool IsStructured, std::string_view BucketName, const std::vector<IoHash>& Cids, const std::vector<size_t>& AttachmentSizes) { - for (const IoHash& Cid : Cids) + auto CreateRecords = [&](bool IsStructured, + std::string_view BucketName, + const eastl::vector<IoHash>& Cids, + const eastl::vector<size_t>& AttachmentSizes) { + for (const IoHash& Cid : Cids) + { + CacheRecord Record = CreateCacheRecord(IsStructured, BucketName, Cid, AttachmentSizes); + eastl::vector<IoHash> AttachmentHashes; + for (const CompressedBuffer& Attachment : Record.Attachments) { - CacheRecord Record = CreateCacheRecord(IsStructured, BucketName, Cid, AttachmentSizes); - std::vector<IoHash> AttachmentHashes; - for (const CompressedBuffer& Attachment : Record.Attachments) - { - AttachmentHashes.push_back(Attachment.DecodeRawHash()); - CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), AttachmentHashes.back()); - } - Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes); + AttachmentHashes.push_back(Attachment.DecodeRawHash()); + CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), AttachmentHashes.back()); } - }; + Zcs.Put("mybucket", Cid, {.Value = Record.Record}, AttachmentHashes); + } + }; - std::vector<size_t> AttachmentSizes = {16, 1000, 2000, 4000, 8000, 64000, 80000}; + eastl::vector<size_t> AttachmentSizes = {16, 1000, 2000, 4000, 8000, 64000, 80000}; - std::vector<IoHash> UnstructuredCids{CreateKey(4), CreateKey(5), CreateKey(6)}; + eastl::vector<IoHash> UnstructuredCids{CreateKey(4), CreateKey(5), CreateKey(6)}; CreateRecords(false, "mybucket"sv, UnstructuredCids, AttachmentSizes); - std::vector<IoHash> StructuredCids{CreateKey(1), CreateKey(2), CreateKey(3)}; + eastl::vector<IoHash> StructuredCids{CreateKey(1), CreateKey(2), CreateKey(3)}; CreateRecords(true, "mybucket"sv, StructuredCids, AttachmentSizes); WorkerThreadPool ThreadPool{1}; @@ -2095,15 +2097,15 @@ TEST_CASE("cachestore.newgc.basics") struct CacheEntry { - IoBuffer Data; - std::vector<std::pair<Oid, CompressedBuffer>> Attachments; + IoBuffer Data; + eastl::vector<std::pair<Oid, CompressedBuffer>> Attachments; }; std::unordered_map<IoHash, CacheEntry> CacheEntries; auto CreateCacheRecord = [&](ZenCacheNamespace& Zcs, CidStore& CidStore, std::string_view Bucket, std::span<std::pair<Oid, CompressedBuffer>> Attachments) { - std::vector<IoHash> AttachmentKeys; + eastl::vector<IoHash> AttachmentKeys; for (const auto& Attachment : Attachments) { AttachmentKeys.push_back(Attachment.second.DecodeRawHash()); @@ -2194,8 +2196,8 @@ TEST_CASE("cachestore.newgc.basics") return true; }; - std::vector<IoHash> CacheRecords; - std::vector<IoHash> UnstructuredCacheValues; + eastl::vector<IoHash> CacheRecords; + eastl::vector<IoHash> UnstructuredCacheValues; const auto TearDrinkerBucket = "teardrinker"sv; { @@ -2207,11 +2209,11 @@ TEST_CASE("cachestore.newgc.basics") // Create some basic data { // Structured record with attachments - auto Attachments1 = CreateCompressedAttachment(CidStore, std::vector<size_t>{77, 1024 * 1024 * 2, 99, 1024 * 1024 * 2 + 87}); + auto Attachments1 = CreateCompressedAttachment(CidStore, eastl::vector<size_t>{77, 1024 * 1024 * 2, 99, 1024 * 1024 * 2 + 87}); CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments1)); // Structured record with reuse of attachments - auto Attachments2 = CreateCompressedAttachment(CidStore, std::vector<size_t>{971}); + auto Attachments2 = CreateCompressedAttachment(CidStore, eastl::vector<size_t>{971}); Attachments2.push_back(Attachments1[0]); Attachments2.push_back(Attachments1[1]); CacheRecords.emplace_back(CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments2)); @@ -2625,7 +2627,7 @@ TEST_CASE("cachestore.newgc.basics") CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount); auto Attachments = - CreateCompressedAttachment(CidStore, std::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187}); + CreateCompressedAttachment(CidStore, eastl::vector<size_t>{177, 1024 * 1024 * 2 + 31, 8999, 1024 * 1024 * 2 + 187}); IoHash CacheRecord = CreateCacheRecord(Zcs, CidStore, TearDrinkerBucket, Attachments); { // Do get so it ends up in memcache diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index 73c10a6db..e85383a2a 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -60,21 +60,21 @@ public: CasImpl(GcManager& Gc); virtual ~CasImpl(); - virtual void Initialize(const CidStoreConfiguration& InConfig) override; - virtual CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, InsertMode Mode) override; - virtual std::vector<InsertResult> InsertChunks(std::span<IoBuffer> Data, - std::span<IoHash> ChunkHashes, - InsertMode Mode = InsertMode::kMayBeMovedInPlace) override; - virtual IoBuffer FindChunk(const IoHash& ChunkHash) override; - virtual bool ContainsChunk(const IoHash& ChunkHash) override; - virtual void FilterChunks(HashKeySet& InOutChunks) override; - virtual bool IterateChunks(std::span<IoHash> DecompressedIds, - const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool, - uint64_t LargeSizeLimit) override; - virtual void Flush() override; - virtual void ScrubStorage(ScrubContext& Ctx) override; - virtual CidStoreSize TotalSize() const override; + virtual void Initialize(const CidStoreConfiguration& InConfig) override; + virtual CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, InsertMode Mode) override; + virtual eastl::vector<InsertResult> InsertChunks(std::span<IoBuffer> Data, + std::span<IoHash> ChunkHashes, + InsertMode Mode = InsertMode::kMayBeMovedInPlace) override; + virtual IoBuffer FindChunk(const IoHash& ChunkHash) override; + virtual bool ContainsChunk(const IoHash& ChunkHash) override; + virtual void FilterChunks(HashKeySet& InOutChunks) override; + virtual bool IterateChunks(std::span<IoHash> DecompressedIds, + const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit) override; + virtual void Flush() override; + virtual void ScrubStorage(ScrubContext& Ctx) override; + virtual CidStoreSize TotalSize() const override; private: CasContainerStrategy m_TinyStrategy; @@ -126,8 +126,8 @@ CasImpl::Initialize(const CidStoreConfiguration& InConfig) // Initialize payload storage { - WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Burst); - std::vector<std::future<void>> Work; + WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Burst); + eastl::vector<std::future<void>> Work; Work.emplace_back( WorkerPool.EnqueueTask(std::packaged_task<void()>{[&]() { m_LargeStrategy.Initialize(m_Config.RootDirectory, IsNewStore); }})); Work.emplace_back(WorkerPool.EnqueueTask(std::packaged_task<void()>{[&]() { @@ -259,11 +259,11 @@ CasImpl::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, InsertMode Mode) } static void -GetCompactCasResults(CasContainerStrategy& Strategy, - std::span<IoBuffer> Data, - std::span<IoHash> ChunkHashes, - std::span<size_t> Indexes, - std::vector<CasStore::InsertResult> Results) +GetCompactCasResults(CasContainerStrategy& Strategy, + std::span<IoBuffer> Data, + std::span<IoHash> ChunkHashes, + std::span<size_t> Indexes, + eastl::vector<CasStore::InsertResult> Results) { const size_t Count = Indexes.size(); if (Count == 1) @@ -272,8 +272,8 @@ GetCompactCasResults(CasContainerStrategy& Strategy, Results[Index] = Strategy.InsertChunk(Data[Index], ChunkHashes[Index]); return; } - std::vector<IoBuffer> Chunks; - std::vector<IoHash> Hashes; + eastl::vector<IoBuffer> Chunks; + eastl::vector<IoHash> Hashes; Chunks.reserve(Count); Hashes.reserve(Count); for (size_t Index : Indexes) @@ -292,12 +292,12 @@ GetCompactCasResults(CasContainerStrategy& Strategy, }; static void -GetFileCasResults(FileCasStrategy& Strategy, - CasStore::InsertMode Mode, - std::span<IoBuffer> Data, - std::span<IoHash> ChunkHashes, - std::span<size_t> Indexes, - std::vector<CasStore::InsertResult> Results) +GetFileCasResults(FileCasStrategy& Strategy, + CasStore::InsertMode Mode, + std::span<IoBuffer> Data, + std::span<IoHash> ChunkHashes, + std::span<size_t> Indexes, + eastl::vector<CasStore::InsertResult> Results) { for (size_t Index : Indexes) { @@ -305,7 +305,7 @@ GetFileCasResults(FileCasStrategy& Strategy, } }; -std::vector<CasStore::InsertResult> +eastl::vector<CasStore::InsertResult> CasImpl::InsertChunks(std::span<IoBuffer> Data, std::span<IoHash> ChunkHashes, CasStore::InsertMode Mode) { ZEN_MEMSCOPE(GetCasTag()); @@ -314,14 +314,14 @@ CasImpl::InsertChunks(std::span<IoBuffer> Data, std::span<IoHash> ChunkHashes, C if (Data.size() == 1) { - std::vector<CasStore::InsertResult> Result(1); + eastl::vector<CasStore::InsertResult> Result(1); Result[0] = InsertChunk(Data[0], ChunkHashes[0], Mode); return Result; } - std::vector<size_t> TinyIndexes; - std::vector<size_t> SmallIndexes; - std::vector<size_t> LargeIndexes; + eastl::vector<size_t> TinyIndexes; + eastl::vector<size_t> SmallIndexes; + eastl::vector<size_t> LargeIndexes; for (size_t Index = 0; Index < Data.size(); Index++) { @@ -341,7 +341,7 @@ CasImpl::InsertChunks(std::span<IoBuffer> Data, std::span<IoHash> ChunkHashes, C } } - std::vector<CasStore::InsertResult> Result(Data.size()); + eastl::vector<CasStore::InsertResult> Result(Data.size()); if (!TinyIndexes.empty()) { diff --git a/src/zenstore/cas.h b/src/zenstore/cas.h index e279dd2cc..5e7085d9b 100644 --- a/src/zenstore/cas.h +++ b/src/zenstore/cas.h @@ -39,19 +39,19 @@ public: virtual void Initialize(const CidStoreConfiguration& Config) = 0; virtual InsertResult InsertChunk(IoBuffer Data, const IoHash& ChunkHash, InsertMode Mode = InsertMode::kMayBeMovedInPlace) = 0; - virtual std::vector<InsertResult> InsertChunks(std::span<IoBuffer> Data, - std::span<IoHash> ChunkHashes, - InsertMode Mode = InsertMode::kMayBeMovedInPlace) = 0; - virtual IoBuffer FindChunk(const IoHash& ChunkHash) = 0; - virtual bool ContainsChunk(const IoHash& ChunkHash) = 0; - virtual void FilterChunks(HashKeySet& InOutChunks) = 0; - virtual bool IterateChunks(std::span<IoHash> DecompressedIds, - const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool, - uint64_t LargeSizeLimit) = 0; - virtual void Flush() = 0; - virtual void ScrubStorage(ScrubContext& Ctx) = 0; - virtual CidStoreSize TotalSize() const = 0; + virtual eastl::vector<InsertResult> InsertChunks(std::span<IoBuffer> Data, + std::span<IoHash> ChunkHashes, + InsertMode Mode = InsertMode::kMayBeMovedInPlace) = 0; + virtual IoBuffer FindChunk(const IoHash& ChunkHash) = 0; + virtual bool ContainsChunk(const IoHash& ChunkHash) = 0; + virtual void FilterChunks(HashKeySet& InOutChunks) = 0; + virtual bool IterateChunks(std::span<IoHash> DecompressedIds, + const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit) = 0; + virtual void Flush() = 0; + virtual void ScrubStorage(ScrubContext& Ctx) = 0; + virtual CidStoreSize TotalSize() const = 0; protected: CidStoreConfiguration m_Config; diff --git a/src/zenstore/caslog.cpp b/src/zenstore/caslog.cpp index 6c7b1b297..fe3970a02 100644 --- a/src/zenstore/caslog.cpp +++ b/src/zenstore/caslog.cpp @@ -193,7 +193,7 @@ CasLogFile::Replay(std::function<void(const void*)>&& Handler, uint64_t SkipEntr const uint64_t MaxBufferSize = 1024 * 1024; - std::vector<uint8_t> ReadBuffer; + eastl::vector<uint8_t> ReadBuffer; ReadBuffer.resize((Min(LogDataSize, MaxBufferSize) / m_RecordSize) * m_RecordSize); uint64_t ReadOffset = 0; diff --git a/src/zenstore/cidstore.cpp b/src/zenstore/cidstore.cpp index 2ab769d04..ca505cc7a 100644 --- a/src/zenstore/cidstore.cpp +++ b/src/zenstore/cidstore.cpp @@ -45,16 +45,16 @@ struct CidStore::Impl return {.New = Result.New}; } - std::vector<CidStore::InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas, std::span<IoHash> RawHashes, CidStore::InsertMode Mode) + eastl::vector<CidStore::InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas, std::span<IoHash> RawHashes, CidStore::InsertMode Mode) { if (ChunkDatas.size() == 1) { - std::vector<CidStore::InsertResult> Result(1); + eastl::vector<CidStore::InsertResult> Result(1); Result[0] = AddChunk(ChunkDatas[0], RawHashes[0], Mode); return Result; } ZEN_ASSERT(ChunkDatas.size() == RawHashes.size()); - std::vector<IoBuffer> Chunks; + eastl::vector<IoBuffer> Chunks; Chunks.reserve(ChunkDatas.size()); #if ZEN_BUILD_DEBUG size_t Offset = 0; @@ -74,10 +74,10 @@ struct CidStore::Impl metrics::RequestStats::Scope $(m_AddChunkOps, TotalSize); - std::vector<CasStore::InsertResult> CasResults = + eastl::vector<CasStore::InsertResult> CasResults = m_CasStore.InsertChunks(Chunks, RawHashes, static_cast<CasStore::InsertMode>(Mode)); ZEN_ASSERT(CasResults.size() == ChunkDatas.size()); - std::vector<CidStore::InsertResult> Result; + eastl::vector<CidStore::InsertResult> Result; for (const CasStore::InsertResult& CasResult : CasResults) { if (CasResult.New) @@ -197,7 +197,7 @@ CidStore::AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode return m_Impl->AddChunk(ChunkData, RawHash, Mode); } -std::vector<CidStore::InsertResult> +eastl::vector<CidStore::InsertResult> CidStore::AddChunks(std::span<IoBuffer> ChunkDatas, std::span<IoHash> RawHashes, InsertMode Mode) { return m_Impl->AddChunks(ChunkDatas, RawHashes, Mode); diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 2be0542db..2e331ab34 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -225,14 +225,14 @@ CasContainerStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) return InsertChunk(Chunk.Data(), Chunk.Size(), ChunkHash); } -std::vector<CasStore::InsertResult> +eastl::vector<CasStore::InsertResult> CasContainerStrategy::InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash> ChunkHashes) { ZEN_MEMSCOPE(GetCasContainerTag()); ZEN_ASSERT(Chunks.size() == ChunkHashes.size()); - std::vector<CasStore::InsertResult> Result(Chunks.size()); - std::vector<size_t> NewChunkIndexes; + eastl::vector<CasStore::InsertResult> Result(Chunks.size()); + eastl::vector<size_t> NewChunkIndexes; { RwLock::SharedLockScope _(m_LocationMapLock); for (size_t ChunkIndex = 0; ChunkIndex < ChunkHashes.size(); ChunkIndex++) @@ -252,7 +252,7 @@ CasContainerStrategy::InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash> return Result; } - std::vector<IoBuffer> Datas; + eastl::vector<IoBuffer> Datas; for (size_t ChunkIndex : NewChunkIndexes) { const IoBuffer& Chunk = Chunks[ChunkIndex]; @@ -265,7 +265,7 @@ CasContainerStrategy::InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash> size_t ChunkOffset = 0; m_BlockStore.WriteChunks(Datas, m_PayloadAlignment, [&](std::span<BlockStoreLocation> Locations) { ZEN_MEMSCOPE(GetCasContainerTag()); - std::vector<CasDiskIndexEntry> IndexEntries; + eastl::vector<CasDiskIndexEntry> IndexEntries; for (const BlockStoreLocation& Location : Locations) { size_t ChunkIndex = NewChunkIndexes[ChunkOffset++]; @@ -330,9 +330,9 @@ CasContainerStrategy::IterateChunks(std::span<IoHash> ChunkHashes, { ZEN_MEMSCOPE(GetCasContainerTag()); - const size_t ChunkCount = ChunkHashes.size(); - std::vector<size_t> FoundChunkIndexes; - std::vector<BlockStoreLocation> FoundChunkLocations; + const size_t ChunkCount = ChunkHashes.size(); + eastl::vector<size_t> FoundChunkIndexes; + eastl::vector<BlockStoreLocation> FoundChunkLocations; FoundChunkIndexes.reserve(ChunkCount); FoundChunkLocations.reserve(ChunkCount); { @@ -395,7 +395,7 @@ CasContainerStrategy::IterateChunks(std::span<IoHash> ChunkHashes, if (OptionalWorkerPool && (ChunkIndexes.size() > 3)) { WorkLatch.AddCount(1); - OptionalWorkerPool->ScheduleWork([&, ChunkIndexes = std::vector<size_t>(ChunkIndexes.begin(), ChunkIndexes.end())]() { + OptionalWorkerPool->ScheduleWork([&, ChunkIndexes = eastl::vector<size_t>(ChunkIndexes.begin(), ChunkIndexes.end())]() { auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); if (!AsyncContinue) { @@ -455,11 +455,11 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx) ZEN_INFO("scrubbing '{}'", m_BlocksBasePath); - RwLock BadKeysLock; - std::vector<IoHash> BadKeys; - std::atomic_uint64_t ChunkCount{0}, ChunkBytes{0}; - std::vector<BlockStoreLocation> ChunkLocations; - std::vector<IoHash> ChunkIndexToChunkHash; + RwLock BadKeysLock; + eastl::vector<IoHash> BadKeys; + std::atomic_uint64_t ChunkCount{0}, ChunkBytes{0}; + eastl::vector<BlockStoreLocation> ChunkLocations; + eastl::vector<IoHash> ChunkIndexToChunkHash; try { @@ -550,7 +550,7 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx) { // Deal with bad chunks by removing them from our lookup map - std::vector<CasDiskIndexEntry> LogEntries; + eastl::vector<CasDiskIndexEntry> LogEntries; LogEntries.reserve(BadKeys.size()); { RwLock::ExclusiveLockScope IndexLock(m_LocationMapLock); @@ -642,7 +642,7 @@ public: { BlockStoreCompactState BlockCompactState; - std::vector<IoHash> BlockCompactStateKeys; + eastl::vector<IoHash> BlockCompactStateKeys; BlockStore::BlockEntryCountMap BlocksToCompact = m_CasContainerStrategy.m_BlockStore.GetBlocksToCompact(BlockUsage, Ctx.Settings.CompactBlockUsageThresholdPercent); @@ -678,8 +678,8 @@ public: BlockCompactState, m_CasContainerStrategy.m_PayloadAlignment, [&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) { - std::vector<CasDiskIndexEntry> MovedEntries; - RwLock::ExclusiveLockScope _(m_CasContainerStrategy.m_LocationMapLock); + eastl::vector<CasDiskIndexEntry> MovedEntries; + RwLock::ExclusiveLockScope _(m_CasContainerStrategy.m_LocationMapLock); for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray) { size_t ChunkIndex = Moved.first; @@ -736,7 +736,7 @@ public: class CasContainerReferencePruner : public GcReferencePruner { public: - CasContainerReferencePruner(CasContainerStrategy& Owner, std::vector<IoHash>&& Cids) + CasContainerReferencePruner(CasContainerStrategy& Owner, eastl::vector<IoHash>&& Cids) : m_CasContainerStrategy(Owner) , m_Cids(std::move(Cids)) { @@ -780,7 +780,7 @@ public: { if (Ctx.Settings.IsDeleteMode) { - std::vector<CasDiskIndexEntry> ExpiredEntries; + eastl::vector<CasDiskIndexEntry> ExpiredEntries; ExpiredEntries.reserve(UnusedCids.size()); { @@ -819,7 +819,7 @@ public: private: CasContainerStrategy& m_CasContainerStrategy; - std::vector<IoHash> m_Cids; + eastl::vector<IoHash> m_Cids; }; std::string @@ -847,7 +847,7 @@ CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats&) NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); - std::vector<IoHash> CidsToCheck; + eastl::vector<IoHash> CidsToCheck; { RwLock::SharedLockScope __(m_LocationMapLock); if (m_LocationMap.empty()) @@ -878,9 +878,9 @@ CasContainerStrategy::CompactIndex(RwLock::ExclusiveLockScope&) ZEN_MEMSCOPE(GetCasContainerTag()); ZEN_TRACE_CPU("CasContainer::CompactIndex"); - size_t EntryCount = m_LocationMap.size(); - LocationMap_t LocationMap; - std::vector<BlockStoreDiskLocation> Locations; + size_t EntryCount = m_LocationMap.size(); + LocationMap_t LocationMap; + eastl::vector<BlockStoreDiskLocation> Locations; Locations.reserve(EntryCount); LocationMap.reserve(EntryCount); for (auto It : m_LocationMap) @@ -945,8 +945,8 @@ CasContainerStrategy::MakeIndexSnapshot() } // Write the current state of the location map to a new index state - std::vector<CasDiskIndexEntry> Entries; - uint64_t IndexLogPosition = 0; + eastl::vector<CasDiskIndexEntry> Entries; + uint64_t IndexLogPosition = 0; { RwLock::SharedLockScope ___(m_LocationMapLock); @@ -1042,7 +1042,7 @@ CasContainerStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint m_Locations.reserve(ExpectedEntryCount); m_LocationMap.reserve(ExpectedEntryCount); - std::vector<CasDiskIndexEntry> Entries; + eastl::vector<CasDiskIndexEntry> Entries; Entries.resize(128 * 1024 / sizeof(CasDiskIndexEntry)); uint64_t RemainingEntries = Header.EntryCount; @@ -1258,7 +1258,7 @@ TEST_CASE("compactcas.compact.gc") const int kIterationCount = 1000; - std::vector<IoHash> Keys(kIterationCount); + eastl::vector<IoHash> Keys(kIterationCount); { GcManager Gc; @@ -1478,7 +1478,7 @@ TEST_CASE("compactcas.threadedinsert") } tsl::robin_set<IoHash, IoHash::Hasher> ChunksToDelete; - std::vector<IoHash> KeepHashes(GcChunkHashes.begin(), GcChunkHashes.end()); + eastl::vector<IoHash> KeepHashes(GcChunkHashes.begin(), GcChunkHashes.end()); size_t C = 0; while (C < KeepHashes.size()) @@ -1503,7 +1503,7 @@ TEST_CASE("compactcas.threadedinsert") auto DoGC = [](CasContainerStrategy& Cas, const tsl::robin_set<IoHash, IoHash::Hasher>& ChunksToDelete, - const std::vector<IoHash>& KeepHashes, + const eastl::vector<IoHash>& KeepHashes, tsl::robin_set<IoHash, IoHash::Hasher>& GcChunkHashes) { std::atomic_bool IsCancelledFlag = false; GcCtx Ctx = {.Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), @@ -1519,8 +1519,8 @@ TEST_CASE("compactcas.threadedinsert") GcStats Stats; GcStoreCompactor* Compactor = Pruner->RemoveUnreferencedData(Ctx, Stats, [&](std::span<IoHash> References) -> std::span<IoHash> { - std::vector<IoHash> Unreferenced; - HashKeySet Retain; + eastl::vector<IoHash> Unreferenced; + HashKeySet Retain; Retain.AddHashesToSet(KeepHashes); for (const IoHash& ChunkHash : References) { diff --git a/src/zenstore/compactcas.h b/src/zenstore/compactcas.h index 07e620086..f72a707ca 100644 --- a/src/zenstore/compactcas.h +++ b/src/zenstore/compactcas.h @@ -51,21 +51,21 @@ struct CasContainerStrategy final : public GcStorage, public GcReferenceStore CasContainerStrategy(GcManager& Gc); ~CasContainerStrategy(); - CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash); - std::vector<CasStore::InsertResult> InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash> ChunkHashes); - IoBuffer FindChunk(const IoHash& ChunkHash); - bool HaveChunk(const IoHash& ChunkHash); - void FilterChunks(HashKeySet& InOutChunks); - bool IterateChunks(std::span<IoHash> ChunkHashes, - const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool, - uint64_t LargeSizeLimit); - void Initialize(const std::filesystem::path& RootDirectory, - const std::string_view ContainerBaseName, - uint32_t MaxBlockSize, - uint32_t Alignment, - bool IsNewStore); - void Flush(); + CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash); + eastl::vector<CasStore::InsertResult> InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash> ChunkHashes); + IoBuffer FindChunk(const IoHash& ChunkHash); + bool HaveChunk(const IoHash& ChunkHash); + void FilterChunks(HashKeySet& InOutChunks); + bool IterateChunks(std::span<IoHash> ChunkHashes, + const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit); + void Initialize(const std::filesystem::path& RootDirectory, + const std::string_view ContainerBaseName, + uint32_t MaxBlockSize, + uint32_t Alignment, + bool IsNewStore); + void Flush(); // GcStorage @@ -100,7 +100,7 @@ private: RwLock m_LocationMapLock; typedef tsl::robin_map<IoHash, size_t, IoHash::Hasher> LocationMap_t; LocationMap_t m_LocationMap; - std::vector<BlockStoreDiskLocation> m_Locations; + eastl::vector<BlockStoreDiskLocation> m_Locations; friend class CasContainerReferencePruner; friend class CasContainerStoreCompactor; diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 34db51aa9..36942b515 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -204,7 +204,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN } return false; } - std::vector<std::filesystem::path> ShardedRoots; + eastl::vector<std::filesystem::path> ShardedRoots; } CasVisitor; FileSystemTraversal Traversal; @@ -619,8 +619,8 @@ FileCasStrategy::IterateChunks(std::span<IoHash> ChunkHashes, const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, WorkerThreadPool* OptionalWorkerPool) { - std::vector<size_t> FoundChunkIndexes; - std::vector<uint64_t> FoundChunkExpectedSizes; + eastl::vector<size_t> FoundChunkIndexes; + eastl::vector<uint64_t> FoundChunkExpectedSizes; { RwLock::SharedLockScope _(m_Lock); for (size_t ChunkIndex = 0; ChunkIndex < ChunkHashes.size(); ChunkIndex++) @@ -697,8 +697,8 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, IoBuffer&& ZEN_ASSERT(m_IsInitialized); - std::vector<IoHash> RawHashes; - std::vector<uint64_t> ExpectedSizes; + eastl::vector<IoHash> RawHashes; + eastl::vector<uint64_t> ExpectedSizes; { RwLock::SharedLockScope _(m_Lock); @@ -748,14 +748,14 @@ FileCasStrategy::ScrubStorage(ScrubContext& Ctx) ZEN_ASSERT(m_IsInitialized); - std::vector<IoHash> BadHashes; - uint64_t ChunkCount{0}, ChunkBytes{0}; + eastl::vector<IoHash> BadHashes; + uint64_t ChunkCount{0}, ChunkBytes{0}; int DiscoveredFilesNotInIndex = 0; { - std::vector<FileCasStrategy::FileCasIndexEntry> ScannedEntries = FileCasStrategy::ScanFolderForCasFiles(m_RootDirectory); - RwLock::ExclusiveLockScope _(m_Lock); + eastl::vector<FileCasStrategy::FileCasIndexEntry> ScannedEntries = FileCasStrategy::ScanFolderForCasFiles(m_RootDirectory); + RwLock::ExclusiveLockScope _(m_Lock); for (const FileCasStrategy::FileCasIndexEntry& Entry : ScannedEntries) { if (m_Index.insert_or_assign(Entry.Key, IndexEntry{.Size = Entry.Size}).second) @@ -959,8 +959,8 @@ FileCasStrategy::MakeIndexSnapshot() } // Write the current state of the location map to a new index state - std::vector<FileCasIndexEntry> Entries; - uint64_t IndexLogPosition = 0; + eastl::vector<FileCasIndexEntry> Entries; + uint64_t IndexLogPosition = 0; { RwLock::SharedLockScope __(m_Lock); @@ -1031,7 +1031,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& using namespace filecas::impl; - std::vector<FileCasIndexEntry> Entries; + eastl::vector<FileCasIndexEntry> Entries; if (std::filesystem::is_regular_file(IndexPath)) { Stopwatch Timer; @@ -1093,7 +1093,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& std::filesystem::path LogPath = GetLogPath(m_RootDirectory); - std::vector<FileCasStrategy::FileCasIndexEntry> ScannedEntries = FileCasStrategy::ScanFolderForCasFiles(m_RootDirectory); + eastl::vector<FileCasStrategy::FileCasIndexEntry> ScannedEntries = FileCasStrategy::ScanFolderForCasFiles(m_RootDirectory); CasLog.Open(LogPath, CasLogFile::Mode::kTruncate); std::string InvalidEntryReason; for (const FileCasStrategy::FileCasIndexEntry& Entry : ScannedEntries) @@ -1163,17 +1163,19 @@ FileCasStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t SkipEntr return 0; } -std::vector<FileCasStrategy::FileCasIndexEntry> +eastl::vector<FileCasStrategy::FileCasIndexEntry> FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir) { ZEN_TRACE_CPU("FileCas::ScanFolderForCasFiles"); using namespace filecas::impl; - std::vector<FileCasIndexEntry> Entries; + eastl::vector<FileCasIndexEntry> Entries; struct Visitor : public FileSystemTraversal::TreeVisitor { - Visitor(const std::filesystem::path& RootDir, std::vector<FileCasIndexEntry>& Entries) : RootDirectory(RootDir), Entries(Entries) {} + Visitor(const std::filesystem::path& RootDir, eastl::vector<FileCasIndexEntry>& Entries) : RootDirectory(RootDir), Entries(Entries) + { + } virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override { std::filesystem::path RelPath = std::filesystem::relative(Parent, RootDirectory); @@ -1202,8 +1204,8 @@ FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir) virtual bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return true; } - const std::filesystem::path& RootDirectory; - std::vector<FileCasIndexEntry>& Entries; + const std::filesystem::path& RootDirectory; + eastl::vector<FileCasIndexEntry>& Entries; } CasVisitor{RootDir, Entries}; FileSystemTraversal Traversal; @@ -1214,7 +1216,7 @@ FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir) class FileCasStoreCompactor : public GcStoreCompactor { public: - FileCasStoreCompactor(FileCasStrategy& Owner, std::vector<IoHash>&& ReferencesToClean) + FileCasStoreCompactor(FileCasStrategy& Owner, eastl::vector<IoHash>&& ReferencesToClean) : m_FileCasStrategy(Owner) , m_ReferencesToClean(std::move(ReferencesToClean)) { @@ -1352,14 +1354,14 @@ public: virtual std::string GetGcName(GcCtx& Ctx) override { return m_FileCasStrategy.GetGcName(Ctx); } private: - FileCasStrategy& m_FileCasStrategy; - std::vector<IoHash> m_ReferencesToClean; + FileCasStrategy& m_FileCasStrategy; + eastl::vector<IoHash> m_ReferencesToClean; }; class FileCasReferencePruner : public GcReferencePruner { public: - FileCasReferencePruner(FileCasStrategy& Owner, std::vector<IoHash>&& Cids) : m_FileCasStrategy(Owner), m_Cids(std::move(Cids)) {} + FileCasReferencePruner(FileCasStrategy& Owner, eastl::vector<IoHash>&& Cids) : m_FileCasStrategy(Owner), m_Cids(std::move(Cids)) {} virtual std::string GetGcName(GcCtx& Ctx) override { return m_FileCasStrategy.GetGcName(Ctx); } @@ -1397,7 +1399,7 @@ public: if (Ctx.Settings.IsDeleteMode) { - std::vector<FileCasStrategy::FileCasIndexEntry> ExpiredEntries; + eastl::vector<FileCasStrategy::FileCasIndexEntry> ExpiredEntries; ExpiredEntries.reserve(UnusedCids.size()); { RwLock::ExclusiveLockScope __(m_FileCasStrategy.m_Lock); @@ -1432,12 +1434,12 @@ public: } } - return new FileCasStoreCompactor(m_FileCasStrategy, std::vector<IoHash>(UnusedCids.begin(), UnusedCids.end())); + return new FileCasStoreCompactor(m_FileCasStrategy, eastl::vector<IoHash>(UnusedCids.begin(), UnusedCids.end())); } private: - FileCasStrategy& m_FileCasStrategy; - std::vector<IoHash> m_Cids; + FileCasStrategy& m_FileCasStrategy; + eastl::vector<IoHash> m_Cids; }; std::string @@ -1455,15 +1457,15 @@ FileCasStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats&) auto Log = [&Ctx]() { return Ctx.Logger; }; - Stopwatch Timer; - const auto _ = MakeGuard([&] { - if (!Ctx.Settings.Verbose) - { - return; - } - ZEN_INFO("GCV2: filecas [CREATE PRUNER] '{}' in {}", m_RootDirectory, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }); - std::vector<IoHash> CidsToCheck; + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: filecas [CREATE PRUNER] '{}' in {}", m_RootDirectory, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + eastl::vector<IoHash> CidsToCheck; { RwLock::SharedLockScope __(m_Lock); if (m_Index.empty()) @@ -1740,7 +1742,7 @@ TEST_CASE("cas.file.move") # if 0 SUBCASE("stresstest") { - std::vector<IoHash> PayloadHashes; + eastl::vector<IoHash> PayloadHashes; const int kWorkers = 64; const int kItemCount = 128; @@ -1761,7 +1763,7 @@ TEST_CASE("cas.file.move") std::barrier Sync{kWorkers}; auto PopulateAll = [&](int w) { - std::vector<IoBuffer> Buffers; + eastl::vector<IoBuffer> Buffers; for (int i = 0; i < kItemCount; ++i) { @@ -1773,7 +1775,7 @@ TEST_CASE("cas.file.move") } }; - std::vector<std::jthread> Threads; + eastl::vector<std::jthread> Threads; for (int i = 0; i < kWorkers; ++i) { diff --git a/src/zenstore/filecas.h b/src/zenstore/filecas.h index 21d8c3b9e..c3adf125c 100644 --- a/src/zenstore/filecas.h +++ b/src/zenstore/filecas.h @@ -80,8 +80,8 @@ private: uint32_t Flags = 0; uint64_t Size = 0; }; - static bool ValidateEntry(const FileCasIndexEntry& Entry, std::string& OutReason); - static std::vector<FileCasStrategy::FileCasIndexEntry> ScanFolderForCasFiles(const std::filesystem::path& RootDir); + static bool ValidateEntry(const FileCasIndexEntry& Entry, std::string& OutReason); + static eastl::vector<FileCasStrategy::FileCasIndexEntry> ScanFolderForCasFiles(const std::filesystem::path& RootDir); static_assert(sizeof(FileCasIndexEntry) == 32); diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index 7ac10d613..258ec80ca 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -536,7 +536,7 @@ CompareForGC(const IoHash& Lhs, const IoHash& Rhs) } bool -FilterReferences(GcCtx& Ctx, std::string_view Context, std::vector<IoHash>& InOutReferences) +FilterReferences(GcCtx& Ctx, std::string_view Context, eastl::vector<IoHash>& InOutReferences) { if (InOutReferences.empty()) { @@ -564,7 +564,7 @@ FilterReferences(GcCtx& Ctx, std::string_view Context, std::vector<IoHash>& InOu if (Filter) { - std::erase_if(InOutReferences, [&Ctx](const IoHash& Key) { + zen::erase_if(InOutReferences, [&Ctx](const IoHash& Key) { return ((Ctx.Settings.AttachmentRangeMax < Key) || (Key < Ctx.Settings.AttachmentRangeMin)); }); RemovedCount = TotalCount - InOutReferences.size(); @@ -651,7 +651,7 @@ void GcManager::RemoveGcReferencer(GcReferencer& Referencer) { RwLock::ExclusiveLockScope _(m_Lock); - std::erase_if(m_GcReferencers, [&](GcReferencer* $) { return $ == &Referencer; }); + zen::erase_if(m_GcReferencers, [&](GcReferencer* $) { return $ == &Referencer; }); } void @@ -666,7 +666,7 @@ GcManager::RemoveGcReferenceLocker(GcReferenceLocker& ReferenceLocker) { ZEN_MEMSCOPE(GetGcTag()); RwLock::ExclusiveLockScope _(m_Lock); - std::erase_if(m_GcReferencerLockers, [&](GcReferenceLocker* $) { return $ == &ReferenceLocker; }); + zen::erase_if(m_GcReferencerLockers, [&](GcReferenceLocker* $) { return $ == &ReferenceLocker; }); } void @@ -680,7 +680,7 @@ void GcManager::RemoveGcReferenceStore(GcReferenceStore& ReferenceStore) { RwLock::ExclusiveLockScope _(m_Lock); - std::erase_if(m_GcReferenceStores, [&](GcReferenceStore* $) { return $ == &ReferenceStore; }); + zen::erase_if(m_GcReferenceStores, [&](GcReferenceStore* $) { return $ == &ReferenceStore; }); } #define SCOPED_TIMER(closure) \ @@ -892,7 +892,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); // The Referencer will create a reference checker that guarantees that the references do not change // as long as it lives - std::vector<GcReferenceChecker*> Checkers; + eastl::vector<GcReferenceChecker*> Checkers; try { { @@ -973,8 +973,8 @@ GcManager::CollectGarbage(const GcSettings& Settings) &ReferenceValidators]() { ZEN_MEMSCOPE(GetGcTag()); - auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); - std::vector<GcReferenceValidator*> Validators; + auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); + eastl::vector<GcReferenceValidator*> Validators; try { { @@ -1069,7 +1069,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) WorkerThreadPool& LockedPhaseThreadPool = Settings.SingleThread ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Background); - std::vector<RwLock::SharedLockScope> LockerScopes; + eastl::vector<RwLock::SharedLockScope> LockerScopes; SCOPED_TIMER(uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); Result.WriteBlockMS = std::chrono::milliseconds(ElapsedMS);); { if (!ReferenceCheckers.empty()) @@ -1093,7 +1093,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) }); for (GcReferenceLocker* ReferenceLocker : m_GcReferencerLockers) { - std::vector<RwLock::SharedLockScope> LockScopes = ReferenceLocker->LockState(Ctx); + eastl::vector<RwLock::SharedLockScope> LockScopes = ReferenceLocker->LockState(Ctx); for (auto It = std::make_move_iterator(LockScopes.begin()); It != std::make_move_iterator(LockScopes.end()); It++) @@ -1373,7 +1373,7 @@ void GcManager::RemoveGcStorage(GcStorage* Storage) { RwLock::ExclusiveLockScope _(m_Lock); - std::erase_if(m_GcStorage, [&](GcStorage* $) { return $ == Storage; }); + zen::erase_if(m_GcStorage, [&](GcStorage* $) { return $ == Storage; }); } void @@ -1442,13 +1442,13 @@ DiskUsageWindow::KeepRange(GcClock::Tick StartTick, GcClock::Tick EndTick) } } -std::vector<uint64_t> +eastl::vector<uint64_t> DiskUsageWindow::GetDiskDeltas(GcClock::Tick StartTick, GcClock::Tick EndTick, GcClock::Tick DeltaWidth, uint64_t& OutMaxDelta) const { ZEN_ASSERT(StartTick != -1); ZEN_ASSERT(DeltaWidth > 0); - std::vector<uint64_t> Result; + eastl::vector<uint64_t> Result; Result.reserve((EndTick - StartTick + DeltaWidth - 1) / DeltaWidth); size_t WindowSize = m_LogWindow.size(); @@ -1729,7 +1729,7 @@ GcScheduler::AppendGCLog(std::string_view Id, GcClock::TimePoint StartTime, cons ZEN_MEMSCOPE(GetGcTag()); try { - std::vector<uint8_t> Blob; + eastl::vector<uint8_t> Blob; { CbObjectWriter Writer; Writer.BeginObject(Id); @@ -2055,7 +2055,7 @@ GcScheduler::SchedulerThread() const int64_t PressureGraphLength = 30; const std::chrono::duration LoadGraphTime = PressureGraphLength * m_Config.MonitorInterval; - std::vector<uint64_t> DiskDeltas; + eastl::vector<uint64_t> DiskDeltas; uint64_t MaxLoad = 0; { @@ -2721,7 +2721,7 @@ TEST_CASE("gc.diskusagewindow") { uint64_t MaxDelta = 0; // 0-10, 10-20, 20-30, 30-40, 40-50, 50-60, 60-70, 70-80 - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(0, 80, 10, MaxDelta); + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(0, 80, 10, MaxDelta); CHECK(DiskDeltas.size() == 8); CHECK(MaxDelta == 15); CHECK(DiskDeltas[0] == 0); @@ -2736,8 +2736,8 @@ TEST_CASE("gc.diskusagewindow") SUBCASE("Sub range") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(20, 40, 10, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(20, 40, 10, MaxDelta); CHECK(DiskDeltas.size() == 2); CHECK(MaxDelta == 10); CHECK(DiskDeltas[0] == 10); // [20:30] @@ -2745,8 +2745,8 @@ TEST_CASE("gc.diskusagewindow") } SUBCASE("Unaligned sub range 1") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(21, 51, 10, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(21, 51, 10, MaxDelta); CHECK(DiskDeltas.size() == 3); CHECK(MaxDelta == 10); CHECK(DiskDeltas[0] == 0); // [21:31] @@ -2755,8 +2755,8 @@ TEST_CASE("gc.diskusagewindow") } SUBCASE("Unaligned end range") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(29, 79, 10, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(29, 79, 10, MaxDelta); CHECK(DiskDeltas.size() == 5); CHECK(MaxDelta == 15); CHECK(DiskDeltas[0] == 0); // [29:39] @@ -2767,8 +2767,8 @@ TEST_CASE("gc.diskusagewindow") } SUBCASE("Ahead of window") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(-40, 0, 10, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(-40, 0, 10, MaxDelta); CHECK(DiskDeltas.size() == 4); CHECK(MaxDelta == 0); CHECK(DiskDeltas[0] == 0); // [-40:-30] @@ -2778,8 +2778,8 @@ TEST_CASE("gc.diskusagewindow") } SUBCASE("After of window") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(90, 120, 10, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(90, 120, 10, MaxDelta); CHECK(DiskDeltas.size() == 3); CHECK(MaxDelta == 0); CHECK(DiskDeltas[0] == 0); // [90:100] @@ -2788,8 +2788,8 @@ TEST_CASE("gc.diskusagewindow") } SUBCASE("Encapsulating window") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(-20, 100, 10, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(-20, 100, 10, MaxDelta); CHECK(DiskDeltas.size() == 12); CHECK(MaxDelta == 15); CHECK(DiskDeltas[0] == 0); // [-20:-10] @@ -2808,8 +2808,8 @@ TEST_CASE("gc.diskusagewindow") SUBCASE("Full range half stride") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(0, 80, 20, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(0, 80, 20, MaxDelta); CHECK(DiskDeltas.size() == 4); CHECK(MaxDelta == 20); CHECK(DiskDeltas[0] == 10); // [0:20] @@ -2820,8 +2820,8 @@ TEST_CASE("gc.diskusagewindow") SUBCASE("Partial odd stride") { - uint64_t MaxDelta = 0; - std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(13, 67, 18, MaxDelta); + uint64_t MaxDelta = 0; + eastl::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(13, 67, 18, MaxDelta); CHECK(DiskDeltas.size() == 3); CHECK(MaxDelta == 15); CHECK(DiskDeltas[0] == 10); // [13:31] @@ -2850,72 +2850,72 @@ TEST_CASE("gc.keepunusedreferences") IoHash::FromHexString("ab3917854bfef7e7af2c372d795bb907a15cab15"), IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")}; { - std::vector<IoHash> UsedReferences; - std::vector<IoHash> References; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences; + eastl::vector<IoHash> References; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.empty()); } { - std::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::vector<IoHash> References; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + eastl::vector<IoHash> References; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.empty()); } { - std::vector<IoHash> UsedReferences{}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 5); } { - std::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.empty()); } { - std::vector<IoHash> UsedReferences{Hashes[0], Hashes[2], Hashes[4]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[0], Hashes[2], Hashes[4]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 2); CHECK(UnusedReferences[0] == Hashes[1]); CHECK(UnusedReferences[1] == Hashes[3]); } { - std::vector<IoHash> UsedReferences{Hashes[2], Hashes[3], Hashes[4]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[2], Hashes[3], Hashes[4]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 2); CHECK(UnusedReferences[0] == Hashes[0]); CHECK(UnusedReferences[1] == Hashes[1]); } { - std::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 2); CHECK(UnusedReferences[0] == Hashes[3]); CHECK(UnusedReferences[1] == Hashes[4]); } { - std::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2], Hashes[4]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[0], Hashes[1], Hashes[2], Hashes[4]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 1); CHECK(UnusedReferences[0] == Hashes[3]); } { - std::vector<IoHash> UsedReferences{Hashes[1], Hashes[3]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[1], Hashes[3]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 3); CHECK(UnusedReferences[0] == Hashes[0]); CHECK(UnusedReferences[1] == Hashes[2]); @@ -2923,9 +2923,9 @@ TEST_CASE("gc.keepunusedreferences") } { - std::vector<IoHash> UsedReferences{Hashes[0]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[0]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 4); CHECK(UnusedReferences[0] == Hashes[1]); CHECK(UnusedReferences[1] == Hashes[2]); @@ -2934,9 +2934,9 @@ TEST_CASE("gc.keepunusedreferences") } { - std::vector<IoHash> UsedReferences{Hashes[1]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[1]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 4); CHECK(UnusedReferences[0] == Hashes[0]); CHECK(UnusedReferences[1] == Hashes[2]); @@ -2945,9 +2945,9 @@ TEST_CASE("gc.keepunusedreferences") } { - std::vector<IoHash> UsedReferences{Hashes[3]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[3]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 4); CHECK(UnusedReferences[0] == Hashes[0]); CHECK(UnusedReferences[1] == Hashes[1]); @@ -2956,9 +2956,9 @@ TEST_CASE("gc.keepunusedreferences") } { - std::vector<IoHash> UsedReferences{Hashes[4]}; - std::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; - std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); + eastl::vector<IoHash> UsedReferences{Hashes[4]}; + eastl::vector<IoHash> References{Hashes[0], Hashes[1], Hashes[2], Hashes[3], Hashes[4]}; + std::span<IoHash> UnusedReferences = KeepUnusedReferences(UsedReferences, References); CHECK(UnusedReferences.size() == 4); CHECK(UnusedReferences[0] == Hashes[0]); CHECK(UnusedReferences[1] == Hashes[1]); diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h index 97357e5cb..fe38d40ae 100644 --- a/src/zenstore/include/zenstore/blockstore.h +++ b/src/zenstore/include/zenstore/blockstore.h @@ -124,8 +124,8 @@ public: std::unordered_set<uint32_t> m_BlockIndexes; }; - typedef std::vector<std::pair<size_t, BlockStoreLocation>> MovedChunksArray; - typedef std::vector<size_t> ChunkIndexArray; + typedef eastl::vector<std::pair<size_t, BlockStoreLocation>> MovedChunksArray; + typedef eastl::vector<size_t> ChunkIndexArray; typedef std::function<bool(const MovedChunksArray& MovedChunks, uint64_t FreedDiskSpace)> CompactCallback; typedef std::function<uint64_t()> ClaimDiskReserveCallback; @@ -190,11 +190,11 @@ private: tsl::robin_map<uint32_t, Ref<BlockStoreFile>> m_ChunkBlocks; - mutable RwLock m_InsertLock; // used to serialize inserts - Ref<BlockStoreFile> m_WriteBlock; - std::uint32_t m_CurrentInsertOffset = 0; - std::atomic_uint32_t m_WriteBlockIndex{}; - std::vector<uint32_t> m_ActiveWriteBlocks; + mutable RwLock m_InsertLock; // used to serialize inserts + Ref<BlockStoreFile> m_WriteBlock; + std::uint32_t m_CurrentInsertOffset = 0; + std::atomic_uint32_t m_WriteBlockIndex{}; + eastl::vector<uint32_t> m_ActiveWriteBlocks; uint64_t m_MaxBlockSize = 1u << 28; uint64_t m_MaxBlockCount = BlockStoreDiskLocation::MaxBlockIndex + 1; @@ -216,7 +216,7 @@ public: uint32_t BlockIndex = BlockUsageIt.first; ZEN_ASSERT(m_BlockIndexToChunkMapIndex.find(BlockIndex) == m_BlockIndexToChunkMapIndex.end()); - m_KeepChunks.emplace_back(std::vector<size_t>()); + m_KeepChunks.emplace_back(eastl::vector<size_t>()); m_KeepChunks.back().reserve(BlockUsageIt.second); m_BlockIndexToChunkMapIndex.insert_or_assign(BlockIndex, m_KeepChunks.size() - 1); EntryCountTotal += BlockUsageIt.second; @@ -228,7 +228,7 @@ public: { if (m_BlockIndexToChunkMapIndex.find(BlockIndex) == m_BlockIndexToChunkMapIndex.end()) { - m_KeepChunks.emplace_back(std::vector<size_t>()); + m_KeepChunks.emplace_back(eastl::vector<size_t>()); m_BlockIndexToChunkMapIndex.insert_or_assign(BlockIndex, m_KeepChunks.size() - 1); } } @@ -241,8 +241,8 @@ public: return false; } - std::vector<size_t>& KeepChunks = m_KeepChunks[It->second]; - size_t Index = m_ChunkLocations.size(); + eastl::vector<size_t>& KeepChunks = m_KeepChunks[It->second]; + size_t Index = m_ChunkLocations.size(); KeepChunks.push_back(Index); m_ChunkLocations.push_back(Location); return true; @@ -250,11 +250,11 @@ public: const BlockStoreLocation& GetLocation(size_t Index) const { return m_ChunkLocations[Index]; } - void IterateBlocks(std::function<bool(uint32_t BlockIndex, - const std::vector<size_t>& KeepChunkIndexes, - const std::vector<BlockStoreLocation>& ChunkLocations)> Callback) const + void IterateBlocks(std::function<bool(uint32_t BlockIndex, + const eastl::vector<size_t>& KeepChunkIndexes, + const eastl::vector<BlockStoreLocation>& ChunkLocations)> Callback) const { - std::vector<uint32_t> BlockOrder; + eastl::vector<uint32_t> BlockOrder; BlockOrder.reserve(m_BlockIndexToChunkMapIndex.size()); for (auto It : m_BlockIndexToChunkMapIndex) { @@ -275,9 +275,9 @@ public: } private: - tsl::robin_map<uint32_t, size_t> m_BlockIndexToChunkMapIndex; // Maps to which vector in BlockKeepChunks to use for a block - std::vector<std::vector<size_t>> m_KeepChunks; // One vector per block index with index into ChunkLocations - std::vector<BlockStoreLocation> m_ChunkLocations; + tsl::robin_map<uint32_t, size_t> m_BlockIndexToChunkMapIndex; // Maps to which vector in BlockKeepChunks to use for a block + eastl::vector<eastl::vector<size_t>> m_KeepChunks; // One vector per block index with index into ChunkLocations + eastl::vector<BlockStoreLocation> m_ChunkLocations; }; void blockstore_forcelink(); diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h index 05400c784..c5c0ae36e 100644 --- a/src/zenstore/include/zenstore/cache/cachedisklayer.h +++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h @@ -132,11 +132,11 @@ public: struct Info { - std::filesystem::path RootDir; - Configuration Config; - std::vector<std::string> BucketNames; - uint64_t EntryCount = 0; - GcStorageSize StorageSize; + std::filesystem::path RootDir; + Configuration Config; + eastl::vector<std::string> BucketNames; + uint64_t EntryCount = 0; + GcStorageSize StorageSize; }; struct BucketStats @@ -161,9 +161,9 @@ public: struct DiskStats { - std::vector<NamedBucketStats> BucketStats; - uint64_t DiskSize; - uint64_t MemorySize; + eastl::vector<NamedBucketStats> BucketStats; + uint64_t DiskSize; + uint64_t MemorySize; }; explicit ZenCacheDiskLayer(GcManager& Gc, JobQueue& JobQueue, const std::filesystem::path& RootDir, const Configuration& Config); @@ -176,7 +176,7 @@ public: void Get(std::string_view Bucket, const IoHash& HashKey, GetBatchHandle& BatchHandle); struct PutBatchHandle; - PutBatchHandle* BeginPutBatch(std::vector<bool>& OutResult); + PutBatchHandle* BeginPutBatch(eastl::vector<bool>& OutResult); void EndPutBatch(PutBatchHandle* Batch) noexcept; void Put(std::string_view Bucket, @@ -200,11 +200,11 @@ public: CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const; - std::vector<RwLock::SharedLockScope> GetGcReferencerLocks(); + eastl::vector<RwLock::SharedLockScope> GetGcReferencerLocks(); - void EnableUpdateCapture(); - void DisableUpdateCapture(); - std::vector<std::string> GetCapturedBucketsLocked(); + void EnableUpdateCapture(); + void DisableUpdateCapture(); + eastl::vector<std::string> GetCapturedBucketsLocked(); #if ZEN_WITH_TESTS void SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time); @@ -232,7 +232,7 @@ public: void Get(const IoHash& HashKey, GetBatchHandle& BatchHandle); struct PutBatchHandle; - PutBatchHandle* BeginPutBatch(std::vector<bool>& OutResult); + PutBatchHandle* BeginPutBatch(eastl::vector<bool>& OutResult); void EndPutBatch(PutBatchHandle* Batch) noexcept; void Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References, PutBatchHandle* OptionalBatchHandle); uint64_t MemCacheTrim(GcClock::TimePoint ExpireTime); @@ -243,23 +243,23 @@ public: struct ReferencesStats { - std::vector<uint64_t> ValueSizes; - uint64_t StructuredValuesCount = 0; - uint64_t StandaloneValuesCount = 0; + eastl::vector<uint64_t> ValueSizes; + uint64_t StructuredValuesCount = 0; + uint64_t StandaloneValuesCount = 0; }; - bool GetReferences(const LoggerRef& Logger, - std::atomic_bool& IsCancelledFlag, - bool StateIsAlreadyLocked, - bool ReadCacheAttachmentMetaData, - bool WriteCacheAttachmentMetaData, - std::vector<IoHash>& OutReferences, - ReferencesStats* OptionalOutReferencesStats); + bool GetReferences(const LoggerRef& Logger, + std::atomic_bool& IsCancelledFlag, + bool StateIsAlreadyLocked, + bool ReadCacheAttachmentMetaData, + bool WriteCacheAttachmentMetaData, + eastl::vector<IoHash>& OutReferences, + ReferencesStats* OptionalOutReferencesStats); bool ReadAttachmentsFromMetaData(uint32_t BlockIndex, std::span<const IoHash> InlineKeys, std::span<const std::size_t> ChunkIndexes, - std::vector<IoHash>& OutReferences) const; + eastl::vector<IoHash>& OutReferences) const; inline GcStorageSize StorageSize() const { @@ -272,7 +272,7 @@ public: CacheValueDetails::BucketDetails GetValueDetails(RwLock::SharedLockScope& IndexLock, const std::string_view ValueFilter) const; void EnumerateBucketContents(std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const; - void GetUsageByAccess(GcClock::TimePoint Now, GcClock::Duration MaxAge, std::vector<uint64_t>& InOutUsageSlots); + void GetUsageByAccess(GcClock::TimePoint Now, GcClock::Duration MaxAge, eastl::vector<uint64_t>& InOutUsageSlots); #if ZEN_WITH_TESTS void SetAccessTime(const IoHash& HashKey, GcClock::TimePoint Time); #endif // ZEN_WITH_TESTS @@ -362,23 +362,23 @@ public: metrics::RequestStats m_PutOps; metrics::RequestStats m_GetOps; - mutable RwLock m_IndexLock; - IndexMap m_Index; - std::vector<AccessTime> m_AccessTimes; - std::vector<BucketPayload> m_Payloads; - std::vector<BucketMetaData> m_MetaDatas; - std::vector<MetaDataIndex> m_FreeMetaDatas; - std::vector<MemCacheData> m_MemCachedPayloads; - std::vector<MemCachedIndex> m_FreeMemCachedPayloads; - std::unique_ptr<HashSet> m_TrackedCacheKeys; - std::unique_ptr<std::vector<IoHash>> m_TrackedReferences; - std::atomic_uint64_t m_StandaloneSize{}; - std::atomic_uint64_t m_MemCachedSize{}; - - virtual std::string GetGcName(GcCtx& Ctx) override; - virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; - virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; - virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override; + mutable RwLock m_IndexLock; + IndexMap m_Index; + eastl::vector<AccessTime> m_AccessTimes; + eastl::vector<BucketPayload> m_Payloads; + eastl::vector<BucketMetaData> m_MetaDatas; + eastl::vector<MetaDataIndex> m_FreeMetaDatas; + eastl::vector<MemCacheData> m_MemCachedPayloads; + eastl::vector<MemCachedIndex> m_FreeMemCachedPayloads; + std::unique_ptr<HashSet> m_TrackedCacheKeys; + std::unique_ptr<eastl::vector<IoHash>> m_TrackedReferences; + std::atomic_uint64_t m_StandaloneSize{}; + std::atomic_uint64_t m_MemCachedSize{}; + + virtual std::string GetGcName(GcCtx& Ctx) override; + virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; + virtual eastl::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; + virtual eastl::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override; void BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const; void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References); @@ -421,12 +421,12 @@ public: bool FlushLockPosition, const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; }); - void CompactState(RwLock::ExclusiveLockScope& IndexLock, - std::vector<BucketPayload>& Payloads, - std::vector<AccessTime>& AccessTimes, - std::vector<BucketMetaData>& MetaDatas, - std::vector<MemCacheData>& MemCachedPayloads, - IndexMap& Index); + void CompactState(RwLock::ExclusiveLockScope& IndexLock, + eastl::vector<BucketPayload>& Payloads, + eastl::vector<AccessTime>& AccessTimes, + eastl::vector<BucketMetaData>& MetaDatas, + eastl::vector<MemCacheData>& MemCachedPayloads, + IndexMap& Index); void AddMemCacheUsage(uint64_t ValueSize) { @@ -486,24 +486,24 @@ private: StartAsyncMemCacheTrim(); } - uint64_t MemCacheTrim(std::vector<CacheBucket*>& Buckets, GcClock::TimePoint ExpireTime); + uint64_t MemCacheTrim(eastl::vector<CacheBucket*>& Buckets, GcClock::TimePoint ExpireTime); bool StartAsyncMemCacheTrim(); void MemCacheTrim(); typedef eastl::unordered_map<std::string, std::unique_ptr<CacheBucket>, std::hash<std::string>, std::equal_to<std::string>> BucketMap_t; - GcManager& m_Gc; - JobQueue& m_JobQueue; - std::filesystem::path m_RootDir; - Configuration m_Configuration; - std::atomic_uint64_t m_TotalMemCachedSize{}; - std::atomic_bool m_IsMemCacheTrimming = false; - std::atomic<GcClock::Tick> m_NextAllowedTrimTick; - mutable RwLock m_Lock; - BucketMap_t m_Buckets; - std::vector<std::unique_ptr<CacheBucket>> m_DroppedBuckets; - uint32_t m_UpdateCaptureRefCounter = 0; - std::unique_ptr<std::vector<std::string>> m_CapturedBuckets; + GcManager& m_Gc; + JobQueue& m_JobQueue; + std::filesystem::path m_RootDir; + Configuration m_Configuration; + std::atomic_uint64_t m_TotalMemCachedSize{}; + std::atomic_bool m_IsMemCacheTrimming = false; + std::atomic<GcClock::Tick> m_NextAllowedTrimTick; + mutable RwLock m_Lock; + BucketMap_t m_Buckets; + eastl::vector<std::unique_ptr<CacheBucket>> m_DroppedBuckets; + uint32_t m_UpdateCaptureRefCounter = 0; + std::unique_ptr<eastl::vector<std::string>> m_CapturedBuckets; ZenCacheDiskLayer(const ZenCacheDiskLayer&) = delete; ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete; diff --git a/src/zenstore/include/zenstore/cache/cacherpc.h b/src/zenstore/include/zenstore/cache/cacherpc.h index da8cf69fe..f180c80d1 100644 --- a/src/zenstore/include/zenstore/cache/cacherpc.h +++ b/src/zenstore/include/zenstore/cache/cacherpc.h @@ -6,9 +6,9 @@ #include <zencore/logging.h> #include <zenutil/cache/cache.h> +#include <EASTL/vector.h> #include <atomic> #include <string_view> -#include <vector> namespace zen { @@ -110,38 +110,38 @@ private: PutResult PutCacheRecord(PutRequestData& Request, const CbPackage* Package); /** HandleRpcGetCacheChunks Helper: Parse the Body object into RecordValue Requests and Value Requests. */ - bool ParseGetCacheChunksRequest(std::string& Namespace, - std::vector<CacheKeyRequest>& RecordKeys, - std::vector<cache::detail::RecordBody>& Records, - std::vector<CacheChunkRequest>& RequestKeys, - std::vector<cache::detail::ChunkRequest>& Requests, - std::vector<cache::detail::ChunkRequest*>& RecordRequests, - std::vector<cache::detail::ChunkRequest*>& ValueRequests, - CbObjectView RpcRequest); + bool ParseGetCacheChunksRequest(std::string& Namespace, + eastl::vector<CacheKeyRequest>& RecordKeys, + eastl::vector<cache::detail::RecordBody>& Records, + eastl::vector<CacheChunkRequest>& RequestKeys, + eastl::vector<cache::detail::ChunkRequest>& Requests, + eastl::vector<cache::detail::ChunkRequest*>& RecordRequests, + eastl::vector<cache::detail::ChunkRequest*>& ValueRequests, + CbObjectView RpcRequest); /** HandleRpcGetCacheChunks Helper: Load records to get ContentId for RecordRequests, and load their payloads if they exist locally. */ - void GetLocalCacheRecords(const CacheRequestContext& Context, - std::string_view Namespace, - std::vector<CacheKeyRequest>& RecordKeys, - std::vector<cache::detail::RecordBody>& Records, - std::vector<cache::detail::ChunkRequest*>& RecordRequests, - std::vector<CacheChunkRequest*>& OutUpstreamChunks); + void GetLocalCacheRecords(const CacheRequestContext& Context, + std::string_view Namespace, + eastl::vector<CacheKeyRequest>& RecordKeys, + eastl::vector<cache::detail::RecordBody>& Records, + eastl::vector<cache::detail::ChunkRequest*>& RecordRequests, + eastl::vector<CacheChunkRequest*>& OutUpstreamChunks); /** HandleRpcGetCacheChunks Helper: For ValueRequests, load their payloads if they exist locally. */ - void GetLocalCacheValues(const CacheRequestContext& Context, - std::string_view Namespace, - std::vector<cache::detail::ChunkRequest*>& ValueRequests, - std::vector<CacheChunkRequest*>& OutUpstreamChunks); + void GetLocalCacheValues(const CacheRequestContext& Context, + std::string_view Namespace, + eastl::vector<cache::detail::ChunkRequest*>& ValueRequests, + eastl::vector<CacheChunkRequest*>& OutUpstreamChunks); /** HandleRpcGetCacheChunks Helper: Load payloads from upstream that did not exist locally. */ - void GetUpstreamCacheChunks(const CacheRequestContext& Context, - std::string_view Namespace, - std::vector<CacheChunkRequest*>& UpstreamChunks, - std::vector<CacheChunkRequest>& RequestKeys, - std::vector<cache::detail::ChunkRequest>& Requests); + void GetUpstreamCacheChunks(const CacheRequestContext& Context, + std::string_view Namespace, + eastl::vector<CacheChunkRequest*>& UpstreamChunks, + eastl::vector<CacheChunkRequest>& RequestKeys, + eastl::vector<cache::detail::ChunkRequest>& Requests); /** HandleRpcGetCacheChunks Helper: Send response message containing all chunk results. */ - CbPackage WriteGetCacheChunksResponse(const CacheRequestContext& Context, - std::string_view Namespace, - RpcAcceptOptions AcceptOptions, - std::vector<cache::detail::ChunkRequest>& Requests); + CbPackage WriteGetCacheChunksResponse(const CacheRequestContext& Context, + std::string_view Namespace, + RpcAcceptOptions AcceptOptions, + eastl::vector<cache::detail::ChunkRequest>& Requests); LoggerRef Log() { return m_Log; } LoggerRef m_Log; diff --git a/src/zenstore/include/zenstore/cache/cacheshared.h b/src/zenstore/include/zenstore/cache/cacheshared.h index 521c78bb1..bafa3d09e 100644 --- a/src/zenstore/include/zenstore/cache/cacheshared.h +++ b/src/zenstore/include/zenstore/cache/cacheshared.h @@ -23,7 +23,7 @@ namespace access_tracking { struct AccessTimes { - std::unordered_map<std::string, std::vector<KeyAccessTime>> Buckets; + std::unordered_map<std::string, eastl::vector<KeyAccessTime>> Buckets; }; }; // namespace access_tracking @@ -40,12 +40,12 @@ struct CacheValueDetails { struct ValueDetails { - uint64_t Size; - uint64_t RawSize; - IoHash RawHash; - GcClock::Tick LastAccess{}; - std::vector<IoHash> Attachments; - ZenContentType ContentType; + uint64_t Size; + uint64_t RawSize; + IoHash RawHash; + GcClock::Tick LastAccess{}; + eastl::vector<IoHash> Attachments; + ZenContentType ContentType; }; struct BucketDetails @@ -63,10 +63,10 @@ struct CacheValueDetails struct CacheContentStats { - std::vector<uint64_t> ValueSizes; - uint64_t StructuredValuesCount = 0; - uint64_t StandaloneValuesCount = 0; - std::vector<IoHash> Attachments; + eastl::vector<uint64_t> ValueSizes; + uint64_t StructuredValuesCount = 0; + uint64_t StandaloneValuesCount = 0; + eastl::vector<IoHash> Attachments; }; bool IsKnownBadBucketName(std::string_view BucketName); diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h index 5e056cf2d..faaba407f 100644 --- a/src/zenstore/include/zenstore/cache/structuredcachestore.h +++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h @@ -62,10 +62,10 @@ public: }; struct Info { - std::filesystem::path RootDir; - Configuration Config; - std::vector<std::string> BucketNames; - ZenCacheDiskLayer::Info DiskLayerInfo; + std::filesystem::path RootDir; + Configuration Config; + eastl::vector<std::string> BucketNames; + ZenCacheDiskLayer::Info DiskLayerInfo; }; struct NamespaceStats @@ -82,7 +82,7 @@ public: ~ZenCacheNamespace(); struct PutBatchHandle; - PutBatchHandle* BeginPutBatch(std::vector<bool>& OutResults); + PutBatchHandle* BeginPutBatch(eastl::vector<bool>& OutResults); void EndPutBatch(PutBatchHandle* Batch) noexcept; struct GetBatchHandle; @@ -115,7 +115,7 @@ public: CacheValueDetails::NamespaceDetails GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const; - std::vector<RwLock::SharedLockScope> GetGcReferencerLocks(); + eastl::vector<RwLock::SharedLockScope> GetGcReferencerLocks(); void EnableUpdateCapture(); void DisableUpdateCapture(); @@ -171,11 +171,11 @@ public: struct Info { - std::filesystem::path BasePath; - Configuration Config; - std::vector<std::string> NamespaceNames; - uint64_t DiskEntryCount = 0; - GcStorageSize StorageSize; + std::filesystem::path BasePath; + Configuration Config; + eastl::vector<std::string> NamespaceNames; + uint64_t DiskEntryCount = 0; + GcStorageSize StorageSize; }; struct NamedNamespaceStats @@ -186,14 +186,14 @@ public: struct CacheStoreStats { - uint64_t HitCount = 0; - uint64_t MissCount = 0; - uint64_t WriteCount = 0; - uint64_t RejectedWriteCount = 0; - uint64_t RejectedReadCount = 0; - metrics::RequestStatsSnapshot PutOps; - metrics::RequestStatsSnapshot GetOps; - std::vector<NamedNamespaceStats> NamespaceStats; + uint64_t HitCount = 0; + uint64_t MissCount = 0; + uint64_t WriteCount = 0; + uint64_t RejectedWriteCount = 0; + uint64_t RejectedReadCount = 0; + metrics::RequestStatsSnapshot PutOps; + metrics::RequestStatsSnapshot GetOps; + eastl::vector<NamedNamespaceStats> NamespaceStats; }; ZenCacheStore(GcManager& Gc, @@ -206,7 +206,7 @@ public: class PutBatch { public: - PutBatch(ZenCacheStore& CacheStore, std::string_view Namespace, std::vector<bool>& OutResult); + PutBatch(ZenCacheStore& CacheStore, std::string_view Namespace, eastl::vector<bool>& OutResult); ~PutBatch(); private: @@ -268,7 +268,7 @@ public: Info GetInfo() const; std::optional<ZenCacheNamespace::Info> GetNamespaceInfo(std::string_view Namespace); std::optional<ZenCacheNamespace::BucketInfo> GetBucketInfo(std::string_view Namespace, std::string_view Bucket); - std::vector<std::string> GetNamespaces(); + eastl::vector<std::string> GetNamespaces(); void EnumerateBucketContents(std::string_view Namespace, std::string_view Bucket, @@ -277,16 +277,16 @@ public: // StatsProvider virtual void ReportMetrics(StatsMetrics& Statsd) override; - virtual std::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) override; + virtual eastl::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) override; - virtual std::string GetGcName(GcCtx& Ctx) override; - virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; - virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; - virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override; + virtual std::string GetGcName(GcCtx& Ctx) override; + virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; + virtual eastl::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; + virtual eastl::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override; - void EnableUpdateCapture(); - void DisableUpdateCapture(); - std::vector<std::string> GetCapturedNamespacesLocked(); + void EnableUpdateCapture(); + void DisableUpdateCapture(); + eastl::vector<std::string> GetCapturedNamespacesLocked(); bool GetContentStats(std::string_view Namespace, std::string_view BucketName, CacheContentStats& OutContentStats) const; @@ -297,13 +297,13 @@ private: typedef std::unordered_map<std::string, std::unique_ptr<ZenCacheNamespace>> NamespaceMap; - CacheStoreStats m_LastReportedMetrics; - const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; - mutable RwLock m_NamespacesLock; - NamespaceMap m_Namespaces; - std::vector<std::unique_ptr<ZenCacheNamespace>> m_DroppedNamespaces; - uint32_t m_UpdateCaptureRefCounter = 0; - std::unique_ptr<std::vector<std::string>> m_CapturedNamespaces; + CacheStoreStats m_LastReportedMetrics; + const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; + mutable RwLock m_NamespacesLock; + NamespaceMap m_Namespaces; + eastl::vector<std::unique_ptr<ZenCacheNamespace>> m_DroppedNamespaces; + uint32_t m_UpdateCaptureRefCounter = 0; + std::unique_ptr<eastl::vector<std::string>> m_CapturedNamespaces; GcManager& m_Gc; JobQueue& m_JobQueue; @@ -327,14 +327,14 @@ private: ZenCacheValue Value; }; - void LogWorker(); - RwLock m_LogQueueLock; - std::vector<AccessLogItem> m_LogQueue; - std::atomic_bool m_ExitLogging; - Event m_LogEvent; - std::thread m_AsyncLoggingThread; - std::atomic_bool m_WriteLogEnabled; - std::atomic_bool m_AccessLogEnabled; + void LogWorker(); + RwLock m_LogQueueLock; + eastl::vector<AccessLogItem> m_LogQueue; + std::atomic_bool m_ExitLogging; + Event m_LogEvent; + std::thread m_AsyncLoggingThread; + std::atomic_bool m_WriteLogEnabled; + std::atomic_bool m_AccessLogEnabled; friend class CacheStoreReferenceChecker; }; diff --git a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h index 152031c3a..ee5f93019 100644 --- a/src/zenstore/include/zenstore/cache/upstreamcacheclient.h +++ b/src/zenstore/include/zenstore/cache/upstreamcacheclient.h @@ -10,10 +10,10 @@ #include <zencore/zencore.h> #include <zenutil/cache/cache.h> +#include <EASTL/vector.h> #include <functional> #include <memory> #include <string> -#include <vector> namespace zen { @@ -22,11 +22,11 @@ class CbPackage; struct UpstreamCacheRecord { - ZenContentType Type = ZenContentType::kBinary; - std::string Namespace; - CacheKey Key; - std::vector<IoHash> ValueContentIds; - CacheRequestContext Context; + ZenContentType Type = ZenContentType::kBinary; + std::string Namespace; + CacheKey Key; + eastl::vector<IoHash> ValueContentIds; + CacheRequestContext Context; }; struct UpstreamError diff --git a/src/zenstore/include/zenstore/cidstore.h b/src/zenstore/include/zenstore/cidstore.h index b3d00fec0..ac73d71ea 100644 --- a/src/zenstore/include/zenstore/cidstore.h +++ b/src/zenstore/include/zenstore/cidstore.h @@ -74,22 +74,22 @@ public: kMayBeMovedInPlace }; - void Initialize(const CidStoreConfiguration& Config); - InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode Mode = InsertMode::kMayBeMovedInPlace); - std::vector<InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas, - std::span<IoHash> RawHashes, - InsertMode Mode = InsertMode::kMayBeMovedInPlace); - virtual IoBuffer FindChunkByCid(const IoHash& DecompressedId) override; - bool IterateChunks(std::span<IoHash> DecompressedIds, - const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, - WorkerThreadPool* OptionalWorkerPool, - uint64_t LargeSizeLimit); - bool ContainsChunk(const IoHash& DecompressedId); - void FilterChunks(HashKeySet& InOutChunks); - void Flush(); - void ScrubStorage(ScrubContext& Ctx); - CidStoreSize TotalSize() const; - CidStoreStats Stats() const; + void Initialize(const CidStoreConfiguration& Config); + InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode Mode = InsertMode::kMayBeMovedInPlace); + eastl::vector<InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas, + std::span<IoHash> RawHashes, + InsertMode Mode = InsertMode::kMayBeMovedInPlace); + virtual IoBuffer FindChunkByCid(const IoHash& DecompressedId) override; + bool IterateChunks(std::span<IoHash> DecompressedIds, + const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, + WorkerThreadPool* OptionalWorkerPool, + uint64_t LargeSizeLimit); + bool ContainsChunk(const IoHash& DecompressedId); + void FilterChunks(HashKeySet& InOutChunks); + void Flush(); + void ScrubStorage(ScrubContext& Ctx); + CidStoreSize TotalSize() const; + CidStoreStats Stats() const; virtual void ReportMetrics(StatsMetrics& Statsd) override; diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h index 3daae0a93..f98b0a537 100644 --- a/src/zenstore/include/zenstore/gc.h +++ b/src/zenstore/include/zenstore/gc.h @@ -121,9 +121,9 @@ struct GcReferenceStoreStats struct GcResult { - std::vector<std::pair<std::string, GcReferencerStats>> ReferencerStats; - std::vector<std::pair<std::string, GcReferenceStoreStats>> ReferenceStoreStats; - std::vector<std::pair<std::string, GcReferenceValidatorStats>> ReferenceValidatorStats; + eastl::vector<std::pair<std::string, GcReferencerStats>> ReferencerStats; + eastl::vector<std::pair<std::string, GcReferenceStoreStats>> ReferenceStoreStats; + eastl::vector<std::pair<std::string, GcReferenceValidatorStats>> ReferenceValidatorStats; GcReferencerStats ReferencerStatSum; GcReferenceStoreStats ReferenceStoreStatSum; @@ -232,7 +232,7 @@ public: }; std::span<IoHash> KeepUnusedReferences(std::span<const IoHash> SortedUsedReferences, std::span<IoHash> SortedReferences); -bool FilterReferences(GcCtx& Ctx, std::string_view Context, std::vector<IoHash>& InOutReferences); +bool FilterReferences(GcCtx& Ctx, std::string_view Context, eastl::vector<IoHash>& InOutReferences); /** * @brief An interface to implement a lock for Stop The World (from writing new data) @@ -247,7 +247,7 @@ public: // Take all the locks needed to execute UpdateLockedState for the all the GcReferenceChecker in your domain // Once all the GcReferenceChecker has executed UpdateLockedState and GetUnusedReferences for all // domains has completed, the locks will be disposed and writes are allowed once again - virtual std::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) = 0; + virtual eastl::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) = 0; }; /** @@ -268,11 +268,11 @@ public: // Create 0-n GcReferenceChecker for this GcReferencer. Caller will manage lifetime of // returned instances - virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) = 0; + virtual eastl::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) = 0; // Create 0-n GcReferenceValidator for this GcReferencer. Caller will manage lifetime of // returned instances - virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) = 0; + virtual eastl::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) = 0; }; /** @@ -377,17 +377,17 @@ public: void SetDiskWriteBlocker(const DiskWriteBlocker* Monitor) { m_DiskWriteBlocker = Monitor; } private: - bool CheckGCCancel() { return m_CancelGC.load(); } - LoggerRef Log() { return m_Log; } - LoggerRef m_Log; - mutable RwLock m_Lock; - std::vector<GcStorage*> m_GcStorage; - CidStore* m_CidStore = nullptr; - const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; - - std::vector<GcReferencer*> m_GcReferencers; - std::vector<GcReferenceLocker*> m_GcReferencerLockers; - std::vector<GcReferenceStore*> m_GcReferenceStores; + bool CheckGCCancel() { return m_CancelGC.load(); } + LoggerRef Log() { return m_Log; } + LoggerRef m_Log; + mutable RwLock m_Lock; + eastl::vector<GcStorage*> m_GcStorage; + CidStore* m_CidStore = nullptr; + const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; + + eastl::vector<GcReferencer*> m_GcReferencers; + eastl::vector<GcReferenceLocker*> m_GcReferencerLockers; + eastl::vector<GcReferenceStore*> m_GcReferenceStores; std::atomic_bool m_CancelGC{false}; }; @@ -466,15 +466,15 @@ public: uint64_t DiskUsage; }; - std::vector<DiskUsageEntry> m_LogWindow; - inline void Append(const DiskUsageEntry& Entry) { m_LogWindow.push_back(Entry); } - inline void Append(DiskUsageEntry&& Entry) { m_LogWindow.emplace_back(std::move(Entry)); } - void KeepRange(GcClock::Tick StartTick, GcClock::Tick EndTick); - std::vector<uint64_t> GetDiskDeltas(GcClock::Tick StartTick, - GcClock::Tick EndTick, - GcClock::Tick DeltaWidth, - uint64_t& OutMaxDelta) const; - GcClock::Tick FindTimepointThatRemoves(uint64_t Amount, GcClock::Tick EndTick) const; + eastl::vector<DiskUsageEntry> m_LogWindow; + inline void Append(const DiskUsageEntry& Entry) { m_LogWindow.push_back(Entry); } + inline void Append(DiskUsageEntry&& Entry) { m_LogWindow.emplace_back(std::move(Entry)); } + void KeepRange(GcClock::Tick StartTick, GcClock::Tick EndTick); + eastl::vector<uint64_t> GetDiskDeltas(GcClock::Tick StartTick, + GcClock::Tick EndTick, + GcClock::Tick DeltaWidth, + uint64_t& OutMaxDelta) const; + GcClock::Tick FindTimepointThatRemoves(uint64_t Amount, GcClock::Tick EndTick) const; }; /** diff --git a/src/zenstore/include/zenstore/workspaces.h b/src/zenstore/include/zenstore/workspaces.h index 3e9edf9f9..f2eb119a9 100644 --- a/src/zenstore/include/zenstore/workspaces.h +++ b/src/zenstore/include/zenstore/workspaces.h @@ -64,21 +64,21 @@ public: Workspaces(); ~Workspaces(); - std::optional<std::vector<ShareFile>> GetWorkspaceShareFiles(const Oid& WorkspaceId, - const Oid& ShareId, - bool ForceRefresh, - WorkerThreadPool& WorkerPool); + std::optional<eastl::vector<ShareFile>> GetWorkspaceShareFiles(const Oid& WorkspaceId, + const Oid& ShareId, + bool ForceRefresh, + WorkerThreadPool& WorkerPool); ShareFile GetWorkspaceShareChunkInfo(const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId, WorkerThreadPool& WorkerPool); - std::vector<IoBuffer> GetWorkspaceShareChunks(const Oid& WorkspaceId, - const Oid& ShareId, - const std::span<const ChunkRequest> ChunkRequests, - WorkerThreadPool& WorkerPool); + eastl::vector<IoBuffer> GetWorkspaceShareChunks(const Oid& WorkspaceId, + const Oid& ShareId, + const std::span<const ChunkRequest> ChunkRequests, + WorkerThreadPool& WorkerPool); - std::vector<Oid> GetWorkspaces() const; + eastl::vector<Oid> GetWorkspaces() const; std::optional<WorkspaceConfiguration> GetWorkspaceConfiguration(const Oid& WorkspaceId) const; - std::optional<std::vector<Oid>> GetWorkspaceShares(const Oid& WorkspaceId) const; + std::optional<eastl::vector<Oid>> GetWorkspaceShares(const Oid& WorkspaceId) const; std::optional<WorkspaceShareConfiguration> GetWorkspaceShareConfiguration(const Oid& WorkspaceId, const Oid& ShareId) const; void RefreshState(const std::filesystem::path& WorkspaceStatePath); @@ -106,37 +106,37 @@ public: const std::filesystem::path& WorkspaceStatePath, const std::filesystem::path& WorkspaceRoot); - static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& Log, - const std::filesystem::path& WorkspaceStatePath, - std::string_view ShareAlias, - WorkspaceConfiguration& OutWorkspace); - static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& InLog, - const std::filesystem::path& WorkspaceStatePath, - const Oid& WorkspaceId, - const Oid& WorkspaceShareId); - static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& Log, - const std::filesystem::path& WorkspaceRoot, - const Oid& WorkspaceShareId); - static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& Log, - const std::filesystem::path& WorkspaceRoot, - const std::filesystem::path& SharePath); - static std::vector<WorkspaceConfiguration> ReadConfig(const LoggerRef& Log, - const std::filesystem::path& WorkspaceStatePath, - std::string& OutError); - static std::vector<WorkspaceShareConfiguration> ReadWorkspaceConfig(const LoggerRef& Log, - const std::filesystem::path& WorkspaceRoot, - std::string& OutError); + static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& Log, + const std::filesystem::path& WorkspaceStatePath, + std::string_view ShareAlias, + WorkspaceConfiguration& OutWorkspace); + static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& InLog, + const std::filesystem::path& WorkspaceStatePath, + const Oid& WorkspaceId, + const Oid& WorkspaceShareId); + static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& Log, + const std::filesystem::path& WorkspaceRoot, + const Oid& WorkspaceShareId); + static WorkspaceShareConfiguration FindWorkspaceShare(const LoggerRef& Log, + const std::filesystem::path& WorkspaceRoot, + const std::filesystem::path& SharePath); + static eastl::vector<WorkspaceConfiguration> ReadConfig(const LoggerRef& Log, + const std::filesystem::path& WorkspaceStatePath, + std::string& OutError); + static eastl::vector<WorkspaceShareConfiguration> ReadWorkspaceConfig(const LoggerRef& Log, + const std::filesystem::path& WorkspaceRoot, + std::string& OutError); static Oid PathToId(const std::filesystem::path& Path); private: - static void WriteConfig(const LoggerRef& Log, - const std::filesystem::path& WorkspaceStatePath, - const std::vector<WorkspaceConfiguration>& WorkspaceConfigurations); + static void WriteConfig(const LoggerRef& Log, + const std::filesystem::path& WorkspaceStatePath, + const eastl::vector<WorkspaceConfiguration>& WorkspaceConfigurations); - static void WriteWorkspaceConfig(const LoggerRef& Log, - const std::filesystem::path& WorkspaceRoot, - const std::vector<WorkspaceShareConfiguration>& WorkspaceShareConfigurations); + static void WriteWorkspaceConfig(const LoggerRef& Log, + const std::filesystem::path& WorkspaceRoot, + const eastl::vector<WorkspaceShareConfiguration>& WorkspaceShareConfigurations); void RefreshWorkspaceShares(const Oid& WorkspaceId); bool RemoveWorkspace(RwLock::ExclusiveLockScope& Lock, const Oid& WorkspaceId); diff --git a/src/zenstore/workspaces.cpp b/src/zenstore/workspaces.cpp index 02a83d2a6..c33997d13 100644 --- a/src/zenstore/workspaces.cpp +++ b/src/zenstore/workspaces.cpp @@ -51,7 +51,7 @@ namespace { return Json.ToString(); } - std::vector<Workspaces::WorkspaceConfiguration> WorkspacesFromJson(const IoBuffer& WorkspaceJson, std::string& OutError) + eastl::vector<Workspaces::WorkspaceConfiguration> WorkspacesFromJson(const IoBuffer& WorkspaceJson, std::string& OutError) { using namespace std::literals; @@ -59,7 +59,7 @@ namespace { LoadCompactBinaryFromJson(std::string_view((const char*)(WorkspaceJson.Data()), WorkspaceJson.GetSize()), OutError); if (OutError.empty()) { - std::vector<Workspaces::WorkspaceConfiguration> Workspaces; + eastl::vector<Workspaces::WorkspaceConfiguration> Workspaces; if (CbObjectView RootObject = RootField.AsObjectView(); RootObject) { for (CbFieldView WorkspaceField : RootObject["workspaces"].AsArrayView()) @@ -111,7 +111,7 @@ namespace { return Json.ToString(); } - std::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceSharesFromJson(const IoBuffer& WorkspaceJson, std::string& OutError) + eastl::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceSharesFromJson(const IoBuffer& WorkspaceJson, std::string& OutError) { using namespace std::literals; @@ -119,7 +119,7 @@ namespace { LoadCompactBinaryFromJson(std::string_view((const char*)(WorkspaceJson.Data()), WorkspaceJson.GetSize()), OutError); if (OutError.empty()) { - std::vector<Workspaces::WorkspaceShareConfiguration> Shares; + eastl::vector<Workspaces::WorkspaceShareConfiguration> Shares; if (CbObjectView RootObject = RootField.AsObjectView(); RootObject) { for (CbFieldView ShareField : RootObject["shares"].AsArrayView()) @@ -168,7 +168,7 @@ public: }; FolderStructure() {} - FolderStructure(std::vector<FileEntry>&& InEntries, std::vector<Oid>&& Ids); + FolderStructure(eastl::vector<FileEntry>&& InEntries, eastl::vector<Oid>&& Ids); const FileEntry* FindEntry(const Oid& Id) const { @@ -190,7 +190,7 @@ public: } private: - const std::vector<FileEntry> Entries; + const eastl::vector<FileEntry> Entries; tsl::robin_map<Oid, size_t, Oid::Hasher> IdLookup; }; @@ -226,7 +226,7 @@ public: Workspace(const LoggerRef& Log, const Workspaces::WorkspaceConfiguration& Config); const Workspaces::WorkspaceConfiguration& GetConfig() const; - std::vector<Ref<WorkspaceShare>> GetShares() const; + eastl::vector<Ref<WorkspaceShare>> GetShares() const; Ref<WorkspaceShare> GetShare(const Oid& ShareId) const; void SetShare(const Oid& ShareId, Ref<WorkspaceShare>&& Share); @@ -241,7 +241,7 @@ private: ////////////////////////////////////////////////////////////////////////// -FolderStructure::FolderStructure(std::vector<FileEntry>&& InEntries, std::vector<Oid>&& Ids) : Entries(std::move(InEntries)) +FolderStructure::FolderStructure(eastl::vector<FileEntry>&& InEntries, eastl::vector<Oid>&& Ids) : Entries(std::move(InEntries)) { IdLookup.reserve(Entries.size()); for (size_t Index = 0; Index < Entries.size(); Index++) @@ -265,9 +265,9 @@ namespace { virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override { - std::vector<FolderStructure::FileEntry> FileEntries; - std::vector<Oid> PathIds; - const size_t FileCount = Content.FileNames.size(); + eastl::vector<FolderStructure::FileEntry> FileEntries; + eastl::vector<Oid> PathIds; + const size_t FileCount = Content.FileNames.size(); FileEntries.reserve(FileCount); PathIds.reserve(FileCount); @@ -290,13 +290,13 @@ namespace { }); } - LoggerRef& Log() { return m_Log; } - LoggerRef& m_Log; - const std::filesystem::path Path; - RwLock WorkLock; - std::vector<FolderStructure::FileEntry> FoundFiles; - std::vector<Oid> FoundFileIds; - WorkerThreadPool& WorkerPool; + LoggerRef& Log() { return m_Log; } + LoggerRef& m_Log; + const std::filesystem::path Path; + RwLock WorkLock; + eastl::vector<FolderStructure::FileEntry> FoundFiles; + eastl::vector<Oid> FoundFileIds; + WorkerThreadPool& WorkerPool; }; void FolderScanner::Traverse() @@ -376,10 +376,10 @@ Workspace::GetConfig() const { return m_Config; } -std::vector<Ref<WorkspaceShare>> +eastl::vector<Ref<WorkspaceShare>> Workspace::GetShares() const { - std::vector<Ref<WorkspaceShare>> Shares; + eastl::vector<Ref<WorkspaceShare>> Shares; Shares.reserve(m_Shares.size()); for (auto It : m_Shares) { @@ -446,8 +446,8 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) std::filesystem::path ConfigPath = RootPath / WorkspaceConfigName; if (std::filesystem::exists(ConfigPath)) { - std::string Error; - std::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceShares = ReadWorkspaceConfig(m_Log, RootPath, Error); + std::string Error; + eastl::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceShares = ReadWorkspaceConfig(m_Log, RootPath, Error); if (!Error.empty()) { ZEN_WARN("Failed to read workspace state from {}. Reason: '{}'", ConfigPath, Error); @@ -529,7 +529,7 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) } } -std::optional<std::vector<Workspaces::ShareFile>> +std::optional<eastl::vector<Workspaces::ShareFile>> Workspaces::GetWorkspaceShareFiles(const Oid& WorkspaceId, const Oid& ShareId, bool ForceRefresh, WorkerThreadPool& WorkerPool) { std::pair<Ref<Workspace>, Ref<WorkspaceShare>> WorkspaceAndShare = FindWorkspaceShare(WorkspaceId, ShareId, ForceRefresh, WorkerPool); @@ -538,8 +538,8 @@ Workspaces::GetWorkspaceShareFiles(const Oid& WorkspaceId, const Oid& ShareId, b return {}; } - const FolderStructure& Structure = WorkspaceAndShare.second->GetStructure(); - std::vector<Workspaces::ShareFile> Files; + const FolderStructure& Structure = WorkspaceAndShare.second->GetStructure(); + eastl::vector<Workspaces::ShareFile> Files; Files.reserve(Structure.EntryCount()); Structure.IterateEntries([&Files](const Oid& Id, const FolderStructure::FileEntry& Entry) { std::string GenericPath(reinterpret_cast<const char*>(Entry.RelativePath.generic_u8string().c_str())); @@ -568,7 +568,7 @@ Workspaces::GetWorkspaceShareChunkInfo(const Oid& WorkspaceId, const Oid& ShareI return {}; } -std::vector<IoBuffer> +eastl::vector<IoBuffer> Workspaces::GetWorkspaceShareChunks(const Oid& WorkspaceId, const Oid& ShareId, const std::span<const ChunkRequest> ChunkRequests, @@ -612,10 +612,10 @@ Workspaces::GetWorkspaceShareChunks(const Oid& WorkspaceId, if (ChunkRequests.size() == 1) { - return std::vector<IoBuffer>({GetOne(RootPath, *WorkspaceAndShare.second, ChunkRequests[0])}); + return eastl::vector<IoBuffer>({GetOne(RootPath, *WorkspaceAndShare.second, ChunkRequests[0])}); } - std::vector<IoBuffer> Chunks; + eastl::vector<IoBuffer> Chunks; Chunks.resize(ChunkRequests.size()); Latch WorkLatch(1); @@ -640,10 +640,10 @@ Workspaces::GetWorkspaceShareChunks(const Oid& WorkspaceId, return Chunks; } -std::vector<Oid> +eastl::vector<Oid> Workspaces::GetWorkspaces() const { - std::vector<Oid> Workspaces; + eastl::vector<Oid> Workspaces; RwLock::SharedLockScope Lock(m_Lock); for (auto It : m_Workspaces) { @@ -667,14 +667,14 @@ Workspaces::GetWorkspaceConfiguration(const Oid& WorkspaceId) const return {}; } -std::optional<std::vector<Oid>> +std::optional<eastl::vector<Oid>> Workspaces::GetWorkspaceShares(const Oid& WorkspaceId) const { RwLock::SharedLockScope Lock(m_Lock); Ref<Workspace> Workspace = FindWorkspace(Lock, WorkspaceId); if (Workspace) { - std::vector<Oid> Shares; + eastl::vector<Oid> Shares; for (auto Share : Workspace->GetShares()) { Shares.push_back(Share->GetConfig().Id); @@ -705,8 +705,8 @@ Workspaces::RefreshState(const std::filesystem::path& WorkspaceStatePath) { using namespace std::literals; - std::string Error; - std::vector<Workspaces::WorkspaceConfiguration> Workspaces = ReadConfig(Log(), WorkspaceStatePath, Error); + std::string Error; + eastl::vector<Workspaces::WorkspaceConfiguration> Workspaces = ReadConfig(Log(), WorkspaceStatePath, Error); if (!Error.empty()) { @@ -798,7 +798,7 @@ Workspaces::GetShareAlias(std::string_view Alias) const return {}; } -std::vector<Workspaces::WorkspaceConfiguration> +eastl::vector<Workspaces::WorkspaceConfiguration> Workspaces::ReadConfig(const LoggerRef& InLog, const std::filesystem::path& WorkspaceStatePath, std::string& OutError) { auto Log = [&InLog]() { return InLog; }; @@ -810,7 +810,7 @@ Workspaces::ReadConfig(const LoggerRef& InLog, const std::filesystem::path& Work const std::filesystem::path ConfigPath = WorkspaceStatePath / WorkspacesConfigName; if (std::filesystem::exists(ConfigPath)) { - std::vector<Workspaces::WorkspaceConfiguration> Workspaces = + eastl::vector<Workspaces::WorkspaceConfiguration> Workspaces = WorkspacesFromJson(IoBufferBuilder::MakeFromFile(ConfigPath), OutError); if (OutError.empty()) { @@ -821,9 +821,9 @@ Workspaces::ReadConfig(const LoggerRef& InLog, const std::filesystem::path& Work } void -Workspaces::WriteConfig(const LoggerRef& InLog, - const std::filesystem::path& WorkspaceStatePath, - const std::vector<WorkspaceConfiguration>& WorkspaceConfigurations) +Workspaces::WriteConfig(const LoggerRef& InLog, + const std::filesystem::path& WorkspaceStatePath, + const eastl::vector<WorkspaceConfiguration>& WorkspaceConfigurations) { auto Log = [&InLog]() { return InLog; }; @@ -837,7 +837,7 @@ Workspaces::WriteConfig(const LoggerRef& InLog, TemporaryFile::SafeWriteFile(WorkspaceStatePath / WorkspacesConfigName, MemoryView(ConfigJson.data(), ConfigJson.size())); } -std::vector<Workspaces::WorkspaceShareConfiguration> +eastl::vector<Workspaces::WorkspaceShareConfiguration> Workspaces::ReadWorkspaceConfig(const LoggerRef& InLog, const std::filesystem::path& WorkspaceRoot, std::string& OutError) { auto Log = [&InLog]() { return InLog; }; @@ -849,7 +849,7 @@ Workspaces::ReadWorkspaceConfig(const LoggerRef& InLog, const std::filesystem::p std::filesystem::path ConfigPath = WorkspaceRoot / WorkspaceConfigName; if (std::filesystem::exists(ConfigPath)) { - std::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceShares = + eastl::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceShares = WorkspaceSharesFromJson(IoBufferBuilder::MakeFromFile(ConfigPath), OutError); if (OutError.empty()) { @@ -860,9 +860,9 @@ Workspaces::ReadWorkspaceConfig(const LoggerRef& InLog, const std::filesystem::p } void -Workspaces::WriteWorkspaceConfig(const LoggerRef& InLog, - const std::filesystem::path& WorkspaceRoot, - const std::vector<WorkspaceShareConfiguration>& WorkspaceShareConfigurations) +Workspaces::WriteWorkspaceConfig(const LoggerRef& InLog, + const std::filesystem::path& WorkspaceRoot, + const eastl::vector<WorkspaceShareConfiguration>& WorkspaceShareConfigurations) { auto Log = [&InLog]() { return InLog; }; @@ -891,8 +891,8 @@ Workspaces::AddWorkspace(const LoggerRef& Log, const std::filesystem::path& Work throw std::invalid_argument( fmt::format("workspace root path '{}' does not exist for workspace '{}'", Configuration.RootPath, Configuration.Id)); } - std::string Error; - std::vector<WorkspaceConfiguration> WorkspaceConfigurations = ReadConfig(Log, WorkspaceStatePath, Error); + std::string Error; + eastl::vector<WorkspaceConfiguration> WorkspaceConfigurations = ReadConfig(Log, WorkspaceStatePath, Error); if (!Error.empty()) { throw std::invalid_argument( @@ -931,8 +931,8 @@ Workspaces::AddWorkspace(const LoggerRef& Log, const std::filesystem::path& Work bool Workspaces::RemoveWorkspace(const LoggerRef& Log, const std::filesystem::path& WorkspaceStatePath, const Oid& WorkspaceId) { - std::string Error; - std::vector<WorkspaceConfiguration> WorkspaceConfigurations = ReadConfig(Log, WorkspaceStatePath, Error); + std::string Error; + eastl::vector<WorkspaceConfiguration> WorkspaceConfigurations = ReadConfig(Log, WorkspaceStatePath, Error); if (!Error.empty()) { throw std::invalid_argument( @@ -976,8 +976,8 @@ Workspaces::AddWorkspaceShare(const LoggerRef& Log, fmt::format("invalid workspace share alias '{}' for workspace share {}", Configuration.Alias, Configuration.Id)); } - std::string Error; - std::vector<WorkspaceShareConfiguration> WorkspaceShareConfigurations = ReadWorkspaceConfig(Log, WorkspaceRoot, Error); + std::string Error; + eastl::vector<WorkspaceShareConfiguration> WorkspaceShareConfigurations = ReadWorkspaceConfig(Log, WorkspaceRoot, Error); if (!Error.empty()) { throw std::invalid_argument(fmt::format("failed to read workspace configuration from '{}'. Reason: '{}'", WorkspaceRoot, Error)); @@ -1018,8 +1018,8 @@ Workspaces::AddWorkspaceShare(const LoggerRef& Log, bool Workspaces::RemoveWorkspaceShare(const LoggerRef& Log, const std::filesystem::path& WorkspaceRoot, const Oid& WorkspaceShareId) { - std::string Error; - std::vector<WorkspaceShareConfiguration> WorkspaceShareConfigurations = ReadWorkspaceConfig(Log, WorkspaceRoot, Error); + std::string Error; + eastl::vector<WorkspaceShareConfiguration> WorkspaceShareConfigurations = ReadWorkspaceConfig(Log, WorkspaceRoot, Error); if (!Error.empty()) { throw std::invalid_argument(fmt::format("failed to read workspace configuration from '{}'. Reason: '{}'", WorkspaceRoot, Error)); @@ -1041,8 +1041,8 @@ Workspaces::FindWorkspace(const LoggerRef& InLog, const std::filesystem::path& W { auto Log = [&InLog]() { return InLog; }; - std::string Error; - std::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); + std::string Error; + eastl::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); if (!Error.empty()) { throw std::invalid_argument( @@ -1066,8 +1066,8 @@ Workspaces::FindWorkspace(const LoggerRef& InLog, { auto Log = [&InLog]() { return InLog; }; - std::string Error; - std::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); + std::string Error; + eastl::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); if (!Error.empty()) { throw std::invalid_argument( @@ -1092,8 +1092,8 @@ Workspaces::FindWorkspaceShare(const LoggerRef& InLog, { auto Log = [&InLog]() { return InLog; }; - std::string Error; - std::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); + std::string Error; + eastl::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); if (!Error.empty()) { throw std::invalid_argument( @@ -1102,7 +1102,7 @@ Workspaces::FindWorkspaceShare(const LoggerRef& InLog, for (const WorkspaceConfiguration& WorkspaceConfig : Workspaces) { - std::vector<WorkspaceShareConfiguration> Shares = ReadWorkspaceConfig(InLog, WorkspaceConfig.RootPath, Error); + eastl::vector<WorkspaceShareConfiguration> Shares = ReadWorkspaceConfig(InLog, WorkspaceConfig.RootPath, Error); if (!Error.empty()) { ZEN_WARN("Invalid workspace config in workspace root '{}'", WorkspaceConfig.RootPath); @@ -1139,9 +1139,9 @@ Workspaces::FindWorkspaceShare(const LoggerRef& InLog, Workspaces::WorkspaceShareConfiguration Workspaces::FindWorkspaceShare(const LoggerRef& InLog, const std::filesystem::path& WorkspaceRoot, const Oid& WorkspaceShareId) { - auto Log = [&InLog]() { return InLog; }; - std::string Error; - std::vector<WorkspaceShareConfiguration> Shares = ReadWorkspaceConfig(InLog, WorkspaceRoot, Error); + auto Log = [&InLog]() { return InLog; }; + std::string Error; + eastl::vector<WorkspaceShareConfiguration> Shares = ReadWorkspaceConfig(InLog, WorkspaceRoot, Error); if (!Error.empty()) { ZEN_WARN("Invalid workspace config in workspace root '{}'", WorkspaceRoot); @@ -1162,9 +1162,9 @@ Workspaces::FindWorkspaceShare(const LoggerRef& InLog, const std::filesystem::pa Workspaces::WorkspaceShareConfiguration Workspaces::FindWorkspaceShare(const LoggerRef& InLog, const std::filesystem::path& WorkspaceRoot, const std::filesystem::path& SharePath) { - auto Log = [&InLog]() { return InLog; }; - std::string Error; - std::vector<WorkspaceShareConfiguration> Shares = ReadWorkspaceConfig(InLog, WorkspaceRoot, Error); + auto Log = [&InLog]() { return InLog; }; + std::string Error; + eastl::vector<WorkspaceShareConfiguration> Shares = ReadWorkspaceConfig(InLog, WorkspaceRoot, Error); if (!Error.empty()) { ZEN_WARN("Invalid workspace config in workspace root '{}'", WorkspaceRoot); @@ -1200,7 +1200,7 @@ Workspaces::RemoveWorkspace(RwLock::ExclusiveLockScope&, const Oid& WorkspaceId) { if (auto It = m_Workspaces.find(WorkspaceId); It != m_Workspaces.end()) { - std::vector<std::string> Aliases; + eastl::vector<std::string> Aliases; for (const auto& AliasIt : m_ShareAliases) { if (AliasIt.second.WorkspaceId == WorkspaceId) @@ -1296,10 +1296,10 @@ Workspaces::FindWorkspace(const RwLock::SharedLockScope&, const Oid& WorkspaceId #if ZEN_WITH_TESTS namespace { - std::vector<std::pair<std::filesystem::path, IoBuffer>> GenerateFolderContent(const std::filesystem::path& RootPath) + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> GenerateFolderContent(const std::filesystem::path& RootPath) { CreateDirectories(RootPath); - std::vector<std::pair<std::filesystem::path, IoBuffer>> Result; + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> Result; Result.push_back(std::make_pair(RootPath / "root_blob_1.bin", CreateRandomBlob(4122))); Result.push_back(std::make_pair(RootPath / "root_blob_2.bin", CreateRandomBlob(2122))); @@ -1328,10 +1328,10 @@ namespace { return Result; } - std::vector<std::pair<std::filesystem::path, IoBuffer>> GenerateFolderContent2(const std::filesystem::path& RootPath) + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> GenerateFolderContent2(const std::filesystem::path& RootPath) { CreateDirectories(RootPath); - std::vector<std::pair<std::filesystem::path, IoBuffer>> Result; + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> Result; Result.push_back(std::make_pair(RootPath / "root_blob_3.bin", CreateRandomBlob(312))); std::filesystem::path FirstFolder(RootPath / "first_folder"); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob3.bin", CreateRandomBlob(722))); @@ -1381,9 +1381,9 @@ TEST_CASE("workspace.share.paths") WorkerThreadPool WorkerPool(std::thread::hardware_concurrency()); - ScopedTemporaryDirectory TempDir; - std::filesystem::path RootPath = TempDir.Path() / "workspace"; - std::vector<std::pair<std::filesystem::path, IoBuffer>> Content = GenerateFolderContent(RootPath); + ScopedTemporaryDirectory TempDir; + std::filesystem::path RootPath = TempDir.Path() / "workspace"; + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> Content = GenerateFolderContent(RootPath); CHECK(Workspaces::AddWorkspace(Log(), TempDir.Path(), Workspaces::WorkspaceConfiguration{.Id = Workspaces::PathToId(RootPath), .RootPath = RootPath})); @@ -1409,9 +1409,9 @@ TEST_CASE("workspace.share.basic") WorkerThreadPool WorkerPool(std::thread::hardware_concurrency()); - ScopedTemporaryDirectory TempDir; - std::filesystem::path RootPath = TempDir.Path() / "workspace"; - std::vector<std::pair<std::filesystem::path, IoBuffer>> Content = GenerateFolderContent(RootPath); + ScopedTemporaryDirectory TempDir; + std::filesystem::path RootPath = TempDir.Path() / "workspace"; + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> Content = GenerateFolderContent(RootPath); Workspaces::AddWorkspace(Log(), TempDir.Path(), @@ -1421,18 +1421,18 @@ TEST_CASE("workspace.share.basic") Workspaces WS; WS.RefreshState(TempDir.Path()); - std::filesystem::path SharePath = RootPath / "second_folder"; - std::vector<std::filesystem::path> Paths = {{std::filesystem::relative(Content[4].first, SharePath)}, - {std::filesystem::relative(Content[6].first, SharePath)}, - {std::filesystem::relative(Content[7].first, SharePath)}, - {"the_file_that_is_not_there.txt"}}; - std::vector<IoBuffer> Chunks = + std::filesystem::path SharePath = RootPath / "second_folder"; + eastl::vector<std::filesystem::path> Paths = {{std::filesystem::relative(Content[4].first, SharePath)}, + {std::filesystem::relative(Content[6].first, SharePath)}, + {std::filesystem::relative(Content[7].first, SharePath)}, + {"the_file_that_is_not_there.txt"}}; + eastl::vector<IoBuffer> Chunks = WS.GetWorkspaceShareChunks(Workspaces::PathToId(RootPath), Workspaces::PathToId("second_folder"), - std::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths[0])}, - {.ChunkId = Workspaces::PathToId(Paths[1])}, - {.ChunkId = Workspaces::PathToId(Paths[2])}, - {.ChunkId = Workspaces::PathToId(Paths[3])}}, + eastl::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths[0])}, + {.ChunkId = Workspaces::PathToId(Paths[1])}, + {.ChunkId = Workspaces::PathToId(Paths[2])}, + {.ChunkId = Workspaces::PathToId(Paths[3])}}, WorkerPool); CHECK(Chunks.size() == 4); CHECK(Chunks[0].GetView().EqualBytes(Content[4].second.GetView())); @@ -1440,20 +1440,20 @@ TEST_CASE("workspace.share.basic") CHECK(Chunks[2].GetView().EqualBytes(Content[7].second.GetView())); CHECK(Chunks[3].GetSize() == 0); - std::vector<std::pair<std::filesystem::path, IoBuffer>> Content2 = GenerateFolderContent2(RootPath); - std::vector<std::filesystem::path> Paths2 = {{std::filesystem::relative(Content2[2].first, SharePath)}, - {std::filesystem::relative(Content2[3].first, SharePath)}}; + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> Content2 = GenerateFolderContent2(RootPath); + eastl::vector<std::filesystem::path> Paths2 = {{std::filesystem::relative(Content2[2].first, SharePath)}, + {std::filesystem::relative(Content2[3].first, SharePath)}}; - std::vector<IoBuffer> Chunks2 = WS.GetWorkspaceShareChunks( + eastl::vector<IoBuffer> Chunks2 = WS.GetWorkspaceShareChunks( Workspaces::PathToId(RootPath), Workspaces::PathToId("second_folder"), - std::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths2[0])}, {.ChunkId = Workspaces::PathToId(Paths2[1])}}, + eastl::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths2[0])}, {.ChunkId = Workspaces::PathToId(Paths2[1])}}, WorkerPool); CHECK(Chunks2.size() == 2); CHECK(Chunks2[0].GetSize() == 0); CHECK(Chunks2[1].GetSize() == 0); - std::optional<std::vector<Workspaces::ShareFile>> Files = + std::optional<eastl::vector<Workspaces::ShareFile>> Files = WS.GetWorkspaceShareFiles(Workspaces::PathToId(RootPath), Workspaces::PathToId("second_folder"), true, WorkerPool); CHECK(Files.has_value()); CHECK(Files.value().size() == 6); @@ -1461,7 +1461,7 @@ TEST_CASE("workspace.share.basic") Chunks2 = WS.GetWorkspaceShareChunks( Workspaces::PathToId(RootPath), Workspaces::PathToId("second_folder"), - std::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths2[0])}, {.ChunkId = Workspaces::PathToId(Paths2[1])}}, + eastl::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths2[0])}, {.ChunkId = Workspaces::PathToId(Paths2[1])}}, WorkerPool); CHECK(Chunks2.size() == 2); CHECK(Chunks2[0].GetView().EqualBytes(Content2[2].second.GetView())); @@ -1489,7 +1489,7 @@ TEST_CASE("workspace.share.basic") Chunks2 = WS.GetWorkspaceShareChunks( Workspaces::PathToId(RootPath), Workspaces::PathToId("second_folder"), - std::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths2[0])}, {.ChunkId = Workspaces::PathToId(Paths2[1])}}, + eastl::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths2[0])}, {.ChunkId = Workspaces::PathToId(Paths2[1])}}, WorkerPool); CHECK(Chunks2.empty()); @@ -1503,9 +1503,9 @@ TEST_CASE("workspace.share.alias") WorkerThreadPool WorkerPool(std::thread::hardware_concurrency()); - ScopedTemporaryDirectory TempDir; - std::filesystem::path RootPath = TempDir.Path() / "workspace"; - std::vector<std::pair<std::filesystem::path, IoBuffer>> Content = GenerateFolderContent(RootPath); + ScopedTemporaryDirectory TempDir; + std::filesystem::path RootPath = TempDir.Path() / "workspace"; + eastl::vector<std::pair<std::filesystem::path, IoBuffer>> Content = GenerateFolderContent(RootPath); CHECK(Workspaces::AddWorkspace(Log(), TempDir.Path(), {Workspaces::PathToId(RootPath), RootPath})); CHECK(Workspaces::AddWorkspaceShare(Log(), RootPath, {Workspaces::PathToId("second_folder"), "second_folder", "my_share"})); @@ -1517,18 +1517,18 @@ TEST_CASE("workspace.share.alias") CHECK(Alias.has_value()); CHECK(!WS.GetShareAlias("my_share2").has_value()); - std::filesystem::path SharePath = RootPath / "second_folder"; - std::vector<std::filesystem::path> Paths = {{std::filesystem::relative(Content[4].first, SharePath)}, - {std::filesystem::relative(Content[6].first, SharePath)}, - {std::filesystem::relative(Content[7].first, SharePath)}, - {"the_file_that_is_not_there.txt"}}; - std::vector<IoBuffer> Chunks = + std::filesystem::path SharePath = RootPath / "second_folder"; + eastl::vector<std::filesystem::path> Paths = {{std::filesystem::relative(Content[4].first, SharePath)}, + {std::filesystem::relative(Content[6].first, SharePath)}, + {std::filesystem::relative(Content[7].first, SharePath)}, + {"the_file_that_is_not_there.txt"}}; + eastl::vector<IoBuffer> Chunks = WS.GetWorkspaceShareChunks(Alias->WorkspaceId, Alias->ShareId, - std::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths[0])}, - {.ChunkId = Workspaces::PathToId(Paths[1])}, - {.ChunkId = Workspaces::PathToId(Paths[2])}, - {.ChunkId = Workspaces::PathToId(Paths[3])}}, + eastl::vector<Workspaces::ChunkRequest>{{.ChunkId = Workspaces::PathToId(Paths[0])}, + {.ChunkId = Workspaces::PathToId(Paths[1])}, + {.ChunkId = Workspaces::PathToId(Paths[2])}, + {.ChunkId = Workspaces::PathToId(Paths[3])}}, WorkerPool); CHECK(Chunks.size() == 4); CHECK(Chunks[0].GetView().EqualBytes(Content[4].second.GetView())); diff --git a/src/zenutil/cache/cachepolicy.cpp b/src/zenutil/cache/cachepolicy.cpp index 0bdfd87ce..8bfb11a30 100644 --- a/src/zenutil/cache/cachepolicy.cpp +++ b/src/zenutil/cache/cachepolicy.cpp @@ -166,7 +166,7 @@ public: inline std::span<const CacheValuePolicy> GetValuePolicies() const final { return Values; } private: - std::vector<CacheValuePolicy> Values; + eastl::vector<CacheValuePolicy> Values; }; CachePolicy diff --git a/src/zenutil/cache/cacherequests.cpp b/src/zenutil/cache/cacherequests.cpp index 7c6f493f2..9b999304e 100644 --- a/src/zenutil/cache/cacherequests.cpp +++ b/src/zenutil/cache/cacherequests.cpp @@ -928,11 +928,11 @@ namespace cacherequests { bool HttpRequestParseRelativeUri(std::string_view Key, std::string_view DefaultNamespace, HttpRequestData& Data) { - std::vector<std::string_view> Tokens; - uint32_t TokenCount = ForEachStrTok(Key, '/', [&](const std::string_view& Token) { - Tokens.push_back(Token); - return true; - }); + eastl::vector<std::string_view> Tokens; + uint32_t TokenCount = ForEachStrTok(Key, '/', [&](const std::string_view& Token) { + Tokens.push_back(Token); + return true; + }); switch (TokenCount) { diff --git a/src/zenutil/cache/rpcrecording.cpp b/src/zenutil/cache/rpcrecording.cpp index 1f951167d..398351fbf 100644 --- a/src/zenutil/cache/rpcrecording.cpp +++ b/src/zenutil/cache/rpcrecording.cpp @@ -136,12 +136,12 @@ struct RecordedRequestsWriter } } - std::filesystem::path m_BasePath; - mutable RwLock m_Lock; - std::vector<RecordedRequest> m_Entries; - std::vector<std::unique_ptr<BasicFile>> m_BlockFiles; - uint64_t m_ChunkOffset; - zen::DateTime m_StartTime = DateTime::Now(); + std::filesystem::path m_BasePath; + mutable RwLock m_Lock; + eastl::vector<RecordedRequest> m_Entries; + eastl::vector<std::unique_ptr<BasicFile>> m_BlockFiles; + uint64_t m_ChunkOffset; + zen::DateTime m_StartTime = DateTime::Now(); }; struct RecordedRequestsReader @@ -205,9 +205,9 @@ struct RecordedRequestsReader return {.ContentType = Entry.ContentType, .AcceptType = Entry.AcceptType, .SessionId = Oid::Zero}; } - std::filesystem::path m_BasePath; - std::vector<RecordedRequest> m_Entries; - std::vector<IoBuffer> m_BlockFiles; + std::filesystem::path m_BasePath; + eastl::vector<RecordedRequest> m_Entries; + eastl::vector<IoBuffer> m_BlockFiles; }; class DiskRequestRecorder : public IRpcRequestRecorder @@ -320,18 +320,18 @@ struct RecordedRequestsSegmentWriter inline DateTime GetEndTime() const { return m_EndTime; } private: - std::filesystem::path m_BasePath; - uint64_t m_SegmentIndex = 0; - uint64_t m_RequestBaseIndex = 0; - uint64_t m_RequestCount = 0; - std::atomic_uint64_t m_FileCount{}; - std::atomic_uint64_t m_RequestsByteCount{}; - mutable RwLock m_Lock; - std::vector<RecordedRequest> m_Entries; - std::vector<std::unique_ptr<BasicFile>> m_BlockFiles; - uint64_t m_ChunkOffset; - DateTime m_StartTime = DateTime::Now(); - DateTime m_EndTime = DateTime::Now(); + std::filesystem::path m_BasePath; + uint64_t m_SegmentIndex = 0; + uint64_t m_RequestBaseIndex = 0; + uint64_t m_RequestCount = 0; + std::atomic_uint64_t m_FileCount{}; + std::atomic_uint64_t m_RequestsByteCount{}; + mutable RwLock m_Lock; + eastl::vector<RecordedRequest> m_Entries; + eastl::vector<std::unique_ptr<BasicFile>> m_BlockFiles; + uint64_t m_ChunkOffset; + DateTime m_StartTime = DateTime::Now(); + DateTime m_EndTime = DateTime::Now(); }; struct RecordedRequestsWriter @@ -347,11 +347,11 @@ private: std::filesystem::path m_BasePath; zen::DateTime m_StartTime = DateTime::Now(); - mutable RwLock m_Lock; - std::unique_ptr<RecordedRequestsSegmentWriter> m_CurrentWriter; - std::vector<std::unique_ptr<RecordedRequestsSegmentWriter>> m_FinishedSegments; - std::vector<uint64_t> m_SegmentBaseIndex; - uint64_t m_NextSegmentBaseIndex = 0; + mutable RwLock m_Lock; + std::unique_ptr<RecordedRequestsSegmentWriter> m_CurrentWriter; + eastl::vector<std::unique_ptr<RecordedRequestsSegmentWriter>> m_FinishedSegments; + eastl::vector<uint64_t> m_SegmentBaseIndex; + uint64_t m_NextSegmentBaseIndex = 0; RecordedRequestsSegmentWriter& EnsureCurrentSegment(); void CommitCurrentSegment(RwLock::ExclusiveLockScope&); @@ -388,9 +388,9 @@ struct RecordedRequestsSegmentReader RecordedRequestInfo ReadRequest(uint64_t RequestIndex, IoBuffer& OutBuffer) const; private: - std::filesystem::path m_BasePath; - std::vector<RecordedRequest> m_Entries; - std::vector<IoBuffer> m_BlockFiles; + std::filesystem::path m_BasePath; + eastl::vector<RecordedRequest> m_Entries; + eastl::vector<IoBuffer> m_BlockFiles; }; struct RecordedRequestsReader @@ -410,12 +410,12 @@ private: DateTime EndTime{0}; }; - bool m_InMemory = false; - std::filesystem::path m_BasePath; - std::vector<SegmentInfo> m_KnownSegments; + bool m_InMemory = false; + std::filesystem::path m_BasePath; + eastl::vector<SegmentInfo> m_KnownSegments; - mutable RwLock m_SegmentLock; - mutable std::vector<std::unique_ptr<RecordedRequestsSegmentReader>> m_SegmentReaders; + mutable RwLock m_SegmentLock; + mutable eastl::vector<std::unique_ptr<RecordedRequestsSegmentReader>> m_SegmentReaders; }; ////////////////////////////////////////////////////////////////////////// @@ -471,7 +471,7 @@ RecordedRequestsSegmentWriter::EndWrite() // note that simply calling `.reset()` here will *not* release backing memory // and it's important that we do because on high traffic servers this will use // a lot of memory otherwise - std::vector<RecordedRequest> EmptyEntries; + eastl::vector<RecordedRequest> EmptyEntries; swap(m_Entries, EmptyEntries); } diff --git a/src/zenutil/chunkblock.cpp b/src/zenutil/chunkblock.cpp index a19cf5c1b..6c8ed43cc 100644 --- a/src/zenutil/chunkblock.cpp +++ b/src/zenutil/chunkblock.cpp @@ -6,7 +6,7 @@ #include <zencore/fmtutils.h> #include <zencore/logging.h> -#include <vector> +#include <EASTL/vector.h> namespace zen { @@ -44,15 +44,15 @@ ParseChunkBlockDescription(const CbObjectView& BlockObject) return Result; } -std::vector<ChunkBlockDescription> +eastl::vector<ChunkBlockDescription> ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject) { if (!BlocksObject) { return {}; } - std::vector<ChunkBlockDescription> Result; - CbArrayView Blocks = BlocksObject["blocks"].AsArrayView(); + eastl::vector<ChunkBlockDescription> Result; + CbArrayView Blocks = BlocksObject["blocks"].AsArrayView(); Result.reserve(Blocks.Num()); for (CbFieldView BlockView : Blocks) { @@ -144,11 +144,11 @@ GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash } CompressedBuffer -GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, ChunkBlockDescription& OutBlock) +GenerateChunkBlock(eastl::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, ChunkBlockDescription& OutBlock) { const size_t ChunkCount = FetchChunks.size(); - std::vector<SharedBuffer> ChunkSegments; + eastl::vector<SharedBuffer> ChunkSegments; ChunkSegments.resize(1); ChunkSegments.reserve(1 + ChunkCount); OutBlock.ChunkRawHashes.reserve(ChunkCount); @@ -203,7 +203,7 @@ IterateChunkBlock(const SharedBuffer& BlockPayload, uint32_t NumberSize; uint64_t ChunkCount = ReadVarUInt(ReadPtr, NumberSize); ReadPtr += NumberSize; - std::vector<uint64_t> ChunkSizes; + eastl::vector<uint64_t> ChunkSizes; ChunkSizes.reserve(ChunkCount); while (ChunkCount--) { diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index 4ca89d996..13b8dd0d2 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -115,7 +115,7 @@ namespace { { RawHashToSequenceRawHashIndex.insert( {Chunked.Info.RawHash, gsl::narrow<uint32_t>(OutChunkedContent.ChunkedContent.SequenceRawHashes.size())}); - std::vector<uint64_t> ChunkSizes; + eastl::vector<uint64_t> ChunkSizes; ChunkSizes.reserve(Chunked.ChunkSources.size()); for (const ChunkSource& Source : Chunked.ChunkSources) { @@ -257,7 +257,7 @@ FolderContent::AreKnownFilesEqual(const FolderContent& Rhs) const } void -FolderContent::UpdateState(const FolderContent& Rhs, std::vector<uint32_t>& OutPathIndexesOufOfDate) +FolderContent::UpdateState(const FolderContent& Rhs, eastl::vector<uint32_t>& OutPathIndexesOufOfDate) { tsl::robin_map<std::string, uint32_t> RhsPathToIndex; const uint32_t RhsPathCount = gsl::narrow<uint32_t>(Rhs.Paths.size()); @@ -295,7 +295,7 @@ FolderContent::UpdateState(const FolderContent& Rhs, std::vector<uint32_t>& OutP } FolderContent -GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector<std::filesystem::path>& OutDeletedPathIndexes) +GetUpdatedContent(const FolderContent& Old, const FolderContent& New, eastl::vector<std::filesystem::path>& OutDeletedPathIndexes) { FolderContent Result = {.Platform = Old.Platform}; tsl::robin_map<std::string, uint32_t> NewPathToIndex; @@ -393,10 +393,10 @@ GetFolderContent(GetFolderContentStatistics& Stats, std::string RelativeDirectoryPath = RelativeRoot.generic_string(); if (m_AcceptDirectory(RelativeDirectoryPath)) { - std::vector<std::filesystem::path> Paths; - std::vector<uint64_t> RawSizes; - std::vector<uint32_t> Attributes; - std::vector<uint64_t> ModificatonTicks; + eastl::vector<std::filesystem::path> Paths; + eastl::vector<uint64_t> RawSizes; + eastl::vector<uint32_t> Attributes; + eastl::vector<uint64_t> ModificatonTicks; Paths.reserve(Content.FileNames.size()); RawSizes.reserve(Content.FileNames.size()); Attributes.reserve(Content.FileNames.size()); @@ -450,12 +450,12 @@ GetFolderContent(GetFolderContentStatistics& Stats, { UpdateCallback(AbortFlag.load(), PendingWork.Remaining()); } - std::vector<size_t> Order; - size_t PathCount = Content.Paths.size(); + eastl::vector<size_t> Order; + size_t PathCount = Content.Paths.size(); Order.resize(Content.Paths.size()); - std::vector<std::string> Parents; + eastl::vector<std::string> Parents; Parents.reserve(PathCount); - std::vector<std::string> Filenames; + eastl::vector<std::string> Filenames; Filenames.reserve(PathCount); for (size_t OrderIndex = 0; OrderIndex < PathCount; OrderIndex++) { @@ -727,7 +727,7 @@ ChunkFolderContent(ChunkingStatistics& Stats, RawHashToChunkSequenceIndex.reserve(ItemCount); ChunkHashToChunkIndex.reserve(ItemCount); { - std::vector<uint32_t> Order; + eastl::vector<uint32_t> Order; Order.resize(ItemCount); for (uint32_t I = 0; I < ItemCount; I++) { @@ -807,7 +807,7 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content) } } - std::vector<ChunkLocationReference> Locations; + eastl::vector<ChunkLocationReference> Locations; Locations.reserve(Content.ChunkedContent.ChunkOrders.size()); for (uint32_t SequenceIndex = 0; SequenceIndex < Content.ChunkedContent.SequenceRawHashes.size(); SequenceIndex++) { diff --git a/src/zenutil/chunkedfile.cpp b/src/zenutil/chunkedfile.cpp index 4f9344039..4613ecb51 100644 --- a/src/zenutil/chunkedfile.cpp +++ b/src/zenutil/chunkedfile.cpp @@ -210,17 +210,17 @@ TEST_CASE("chunkedfile.findparams") # if 1 DirectoryContent SourceContent1; GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208", DirectoryContentFlags::IncludeFiles, SourceContent1); - const std::vector<std::filesystem::path>& SourceFiles1 = SourceContent1.Files; + const eastl::vector<std::filesystem::path>& SourceFiles1 = SourceContent1.Files; DirectoryContent SourceContent2; GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208_2", DirectoryContentFlags::IncludeFiles, SourceContent2); - const std::vector<std::filesystem::path>& SourceFiles2 = SourceContent2.Files; + const eastl::vector<std::filesystem::path>& SourceFiles2 = SourceContent2.Files; # else std::filesystem::path SourcePath1 = "E:\\Temp\\ChunkingTestData\\31375996\\ShaderArchive-FortniteGame_Chunk10-PCD3D_SM6-PCD3D_SM6.ushaderbytecode"; std::filesystem::path SourcePath2 = "E:\\Temp\\ChunkingTestData\\31379208\\ShaderArchive-FortniteGame_Chunk10-PCD3D_SM6-PCD3D_SM6.ushaderbytecode"; - const std::vector<std::filesystem::path>& SourceFiles1 = {SourcePath1}; - const std::vector<std::filesystem::path>& SourceFiles2 = {SourcePath2}; + const eastl::vector<std::filesystem::path>& SourceFiles1 = {SourcePath1}; + const eastl::vector<std::filesystem::path>& SourceFiles2 = {SourcePath2}; # endif ChunkedParams Params[] = {ChunkedParams{.UseThreshold = false, .MinSize = 17280, .MaxSize = 139264, .AvgSize = 36340}, ChunkedParams{.UseThreshold = false, .MinSize = 15456, .MaxSize = 122880, .AvgSize = 35598}, @@ -348,8 +348,8 @@ TEST_CASE("chunkedfile.findparams") ChunkedParams{.UseThreshold = false, .MinSize = 19440, .MaxSize = 155648, .AvgSize = 36464}}; static const size_t ParamsCount = sizeof(Params) / sizeof(ChunkedParams); - std::vector<ChunkedInfoWithSource> Infos1(SourceFiles1.size()); - std::vector<ChunkedInfoWithSource> Infos2(SourceFiles2.size()); + eastl::vector<ChunkedInfoWithSource> Infos1(SourceFiles1.size()); + eastl::vector<ChunkedInfoWithSource> Infos2(SourceFiles2.size()); WorkerThreadPool WorkerPool(32); diff --git a/src/zenutil/chunkingcontroller.cpp b/src/zenutil/chunkingcontroller.cpp index 017d12433..bdc030f2c 100644 --- a/src/zenutil/chunkingcontroller.cpp +++ b/src/zenutil/chunkingcontroller.cpp @@ -13,9 +13,9 @@ namespace zen { using namespace std::literals; namespace { - std::vector<std::string> ReadStringArray(CbArrayView StringArray) + eastl::vector<std::string> ReadStringArray(CbArrayView StringArray) { - std::vector<std::string> Result; + eastl::vector<std::string> Result; Result.reserve(StringArray.Num()); for (CbFieldView FieldView : StringArray) { @@ -103,9 +103,9 @@ public: static constexpr std::string_view Name = "BasicChunkingController"sv; protected: - const std::vector<std::string> m_ChunkExcludeExtensions; - const uint64_t m_ChunkFileSizeLimit; - const ChunkedParams m_ChunkingParams; + const eastl::vector<std::string> m_ChunkExcludeExtensions; + const uint64_t m_ChunkFileSizeLimit; + const ChunkedParams m_ChunkingParams; }; class ChunkingControllerWithFixedChunking : public ChunkingController @@ -218,10 +218,10 @@ public: static constexpr std::string_view Name = "ChunkingControllerWithFixedChunking"sv; protected: - const std::vector<std::string> m_FixedChunkingExtensions; - const uint64_t m_ChunkFileSizeLimit; - const ChunkedParams m_ChunkingParams; - const uint32_t m_FixedChunkingChunkSize; + const eastl::vector<std::string> m_FixedChunkingExtensions; + const uint64_t m_ChunkFileSizeLimit; + const ChunkedParams m_ChunkingParams; + const uint32_t m_FixedChunkingChunkSize; }; std::unique_ptr<ChunkingController> diff --git a/src/zenutil/chunkrequests.cpp b/src/zenutil/chunkrequests.cpp index 745363668..49ac9e2f5 100644 --- a/src/zenutil/chunkrequests.cpp +++ b/src/zenutil/chunkrequests.cpp @@ -42,7 +42,7 @@ namespace { } // namespace IoBuffer -BuildChunkBatchRequest(const std::vector<RequestChunkEntry>& Entries) +BuildChunkBatchRequest(const eastl::vector<RequestChunkEntry>& Entries) { RequestHeader RequestHdr; RequestHdr.Magic = (uint32_t)RequestHeader::kMagic; @@ -54,7 +54,7 @@ BuildChunkBatchRequest(const std::vector<RequestChunkEntry>& Entries) return Buffer.MoveToShared().AsIoBuffer(); } -std::optional<std::vector<RequestChunkEntry>> +std::optional<eastl::vector<RequestChunkEntry>> ParseChunkBatchRequest(const IoBuffer& Payload) { if (Payload.Size() <= sizeof(RequestHeader)) @@ -72,19 +72,19 @@ ParseChunkBatchRequest(const IoBuffer& Payload) return {}; } - std::vector<RequestChunkEntry> RequestedChunks; + eastl::vector<RequestChunkEntry> RequestedChunks; RequestedChunks.resize(RequestHdr.ChunkCount); Reader.Read(RequestedChunks.data(), sizeof(RequestChunkEntry) * RequestHdr.ChunkCount); return RequestedChunks; } -std::vector<IoBuffer> -BuildChunkBatchResponse(const std::vector<RequestChunkEntry>& Requests, std::span<IoBuffer> Chunks) +eastl::vector<IoBuffer> +BuildChunkBatchResponse(const eastl::vector<RequestChunkEntry>& Requests, std::span<IoBuffer> Chunks) { ZEN_ASSERT(Requests.size() == Chunks.size()); size_t ChunkCount = Requests.size(); - std::vector<IoBuffer> OutBlobs; + eastl::vector<IoBuffer> OutBlobs; OutBlobs.reserve(1 + ChunkCount); OutBlobs.emplace_back(sizeof(ResponseHeader) + ChunkCount * sizeof(ResponseChunkEntry)); @@ -115,7 +115,7 @@ BuildChunkBatchResponse(const std::vector<RequestChunkEntry>& Requests, std::spa return OutBlobs; } -std::vector<IoBuffer> +eastl::vector<IoBuffer> ParseChunkBatchResponse(const IoBuffer& Buffer) { MemoryView View = Buffer.GetView(); @@ -127,7 +127,7 @@ ParseChunkBatchResponse(const IoBuffer& Buffer) View.MidInline(sizeof(ResponseHeader)); const ResponseChunkEntry* Entries = (const ResponseChunkEntry*)View.GetData(); View.MidInline(sizeof(ResponseChunkEntry) * Header->ChunkCount); - std::vector<IoBuffer> Result(Header->ChunkCount); + eastl::vector<IoBuffer> Result(Header->ChunkCount); for (uint32_t Index = 0; Index < Header->ChunkCount; Index++) { const ResponseChunkEntry& Entry = Entries[Index]; diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index a4bb759e7..37bf82afe 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -114,10 +114,10 @@ public: SimulateLatency(0, 0); } - virtual std::pair<IoHash, std::vector<IoHash>> PutBuildPart(const Oid& BuildId, - const Oid& BuildPartId, - std::string_view PartName, - const CbObject& MetaData) override + virtual std::pair<IoHash, eastl::vector<IoHash>> PutBuildPart(const Oid& BuildId, + const Oid& BuildPartId, + std::string_view PartName, + const CbObject& MetaData) override { SimulateLatency(MetaData.GetSize(), 0); Stopwatch ExecutionTimer; @@ -155,7 +155,7 @@ public: } WriteBuild(BuildId, Writer.Save()); - std::vector<IoHash> NeededAttachments = GetNeededAttachments(MetaData); + eastl::vector<IoHash> NeededAttachments = GetNeededAttachments(MetaData); SimulateLatency(0, sizeof(IoHash) * NeededAttachments.size()); @@ -184,7 +184,7 @@ public: return BuildPartObject; } - virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override + virtual eastl::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override { SimulateLatency(0, 0); @@ -202,8 +202,8 @@ public: fmt::format("Failed finalizing build part {}: Expected hash {}, got {}", BuildPartId, PartHash, RawHash)); } - CbObject BuildPartObject = CbObject(SharedBuffer(Payload)); - std::vector<IoHash> NeededAttachments(GetNeededAttachments(BuildPartObject)); + CbObject BuildPartObject = CbObject(SharedBuffer(Payload)); + eastl::vector<IoHash> NeededAttachments(GetNeededAttachments(BuildPartObject)); SimulateLatency(0, NeededAttachments.size() * sizeof(IoHash)); @@ -235,12 +235,12 @@ public: SimulateLatency(0, 0); } - virtual std::vector<std::function<void()>> PutLargeBuildBlob(const Oid& BuildId, - const IoHash& RawHash, - ZenContentType ContentType, - uint64_t PayloadSize, - std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter, - std::function<void(uint64_t, bool)>&& OnSentBytes) override + virtual eastl::vector<std::function<void()>> PutLargeBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + ZenContentType ContentType, + uint64_t PayloadSize, + std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter, + std::function<void(uint64_t, bool)>&& OnSentBytes) override { ZEN_UNUSED(BuildId); ZEN_UNUSED(ContentType); @@ -274,8 +274,8 @@ public: fmt::format("Failed opening temporary file '{}': {} ({})", Workload->TempFile.GetPath(), Ec.message(), Ec.value())); } - std::vector<std::function<void()>> WorkItems; - uint64_t Offset = 0; + eastl::vector<std::function<void()>> WorkItems; + uint64_t Offset = 0; while (Offset < PayloadSize) { uint64_t Size = Min(32u * 1024u * 1024u, PayloadSize - Offset); @@ -348,7 +348,7 @@ public: return IoBuffer{}; } - virtual std::vector<std::function<void()>> GetLargeBuildBlob( + virtual eastl::vector<std::function<void()>> GetLargeBuildBlob( const Oid& BuildId, const IoHash& RawHash, uint64_t ChunkSize, @@ -377,8 +377,8 @@ public: Workload->Receiver = std::move(Receiver); Workload->BytesRemaining = BlobSize; - std::vector<std::function<void()>> WorkItems; - uint64_t Offset = 0; + eastl::vector<std::function<void()>> WorkItems; + uint64_t Offset = 0; while (Offset < BlobSize) { uint64_t Size = Min(ChunkSize, BlobSize - Offset); @@ -418,7 +418,7 @@ public: SimulateLatency(0, 0); } - virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override + virtual eastl::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override { ZEN_UNUSED(BuildId); SimulateLatency(0, 0); @@ -428,7 +428,7 @@ public: DirectoryContent Content; GetDirectoryContent(GetBlobsMetadataFolder(), DirectoryContentFlags::IncludeFiles, Content); - std::vector<ChunkBlockDescription> Result; + eastl::vector<ChunkBlockDescription> Result; for (const std::filesystem::path& MetaDataFile : Content.Files) { IoHash ChunkHash; @@ -450,7 +450,7 @@ public: return Result; } - virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override + virtual eastl::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override { ZEN_UNUSED(BuildId); SimulateLatency(0, 0); @@ -458,7 +458,7 @@ public: auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); m_Stats.TotalRequestCount++; - std::vector<ChunkBlockDescription> Result; + eastl::vector<ChunkBlockDescription> Result; for (const IoHash& BlockHash : BlockHashes) { std::filesystem::path MetaDataFile = GetBlobMetadataPath(BlockHash); @@ -555,9 +555,9 @@ protected: return BuildObject; } - std::vector<IoHash> GetNeededAttachments(CbObjectView BuildPartObject) + eastl::vector<IoHash> GetNeededAttachments(CbObjectView BuildPartObject) { - std::vector<IoHash> NeededAttachments; + eastl::vector<IoHash> NeededAttachments; BuildPartObject.IterateAttachments([&](CbFieldView FieldView) { const IoHash AttachmentHash = FieldView.AsBinaryAttachment(); const std::filesystem::path BlockPath = GetBlobPayloadPath(AttachmentHash); diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 9c236310f..5eddd2da7 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -26,30 +26,30 @@ public: virtual CbObject GetBuild(const Oid& BuildId) = 0; virtual void FinalizeBuild(const Oid& BuildId) = 0; - virtual std::pair<IoHash, std::vector<IoHash>> PutBuildPart(const Oid& BuildId, - const Oid& BuildPartId, - std::string_view PartName, - const CbObject& MetaData) = 0; - virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) = 0; - virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) = 0; + virtual std::pair<IoHash, eastl::vector<IoHash>> PutBuildPart(const Oid& BuildId, + const Oid& BuildPartId, + std::string_view PartName, + const CbObject& MetaData) = 0; + virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) = 0; + virtual eastl::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) = 0; virtual void PutBuildBlob(const Oid& BuildId, const IoHash& RawHash, ZenContentType ContentType, const CompositeBuffer& Payload) = 0; - virtual std::vector<std::function<void()>> PutLargeBuildBlob(const Oid& BuildId, - const IoHash& RawHash, - ZenContentType ContentType, - uint64_t PayloadSize, - std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter, - std::function<void(uint64_t, bool)>&& OnSentBytes) = 0; - - virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) = 0; - virtual std::vector<std::function<void()>> GetLargeBuildBlob( + virtual eastl::vector<std::function<void()>> PutLargeBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + ZenContentType ContentType, + uint64_t PayloadSize, + std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter, + std::function<void(uint64_t, bool)>&& OnSentBytes) = 0; + + virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) = 0; + virtual eastl::vector<std::function<void()>> GetLargeBuildBlob( const Oid& BuildId, const IoHash& RawHash, uint64_t ChunkSize, std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver) = 0; - virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; - virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) = 0; - virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) = 0; + virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; + virtual eastl::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) = 0; + virtual eastl::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) = 0; }; } // namespace zen diff --git a/src/zenutil/include/zenutil/cache/cacherequests.h b/src/zenutil/include/zenutil/cache/cacherequests.h index fbf3e08cc..351245d75 100644 --- a/src/zenutil/include/zenutil/cache/cacherequests.h +++ b/src/zenutil/include/zenutil/cache/cacherequests.h @@ -41,7 +41,7 @@ namespace cacherequests { // struct GetCacheRecordRequest // { // CacheKey Key = CacheKey::Empty; - // std::vector<GetCacheRecordValueRequest> ValueRequests; + // eastl::vector<GetCacheRecordValueRequest> ValueRequests; // std::optional<CachePolicy> Policy; // }; // @@ -67,17 +67,17 @@ namespace cacherequests { struct PutCacheRecordRequest { - CacheKey Key = CacheKey::Empty; - std::vector<PutCacheRecordRequestValue> Values; - std::optional<CacheRecordPolicy> Policy; + CacheKey Key = CacheKey::Empty; + eastl::vector<PutCacheRecordRequestValue> Values; + std::optional<CacheRecordPolicy> Policy; }; struct PutCacheRecordsRequest { - uint32_t AcceptMagic = 0; - CachePolicy DefaultPolicy = CachePolicy::Default; - std::string Namespace; - std::vector<PutCacheRecordRequest> Requests; + uint32_t AcceptMagic = 0; + CachePolicy DefaultPolicy = CachePolicy::Default; + std::string Namespace; + eastl::vector<PutCacheRecordRequest> Requests; bool Parse(const CbPackage& Package); bool Format(CbPackage& OutPackage) const; @@ -85,7 +85,7 @@ namespace cacherequests { struct PutCacheRecordsResult { - std::vector<bool> Success; + eastl::vector<bool> Success; bool Parse(const CbPackage& Package); bool Format(CbPackage& OutPackage) const; @@ -106,12 +106,12 @@ namespace cacherequests { struct GetCacheRecordsRequest { - uint32_t AcceptMagic = 0; - uint16_t AcceptOptions = 0; - int32_t ProcessPid = 0; - CachePolicy DefaultPolicy = CachePolicy::Default; - std::string Namespace; - std::vector<GetCacheRecordRequest> Requests; + uint32_t AcceptMagic = 0; + uint16_t AcceptOptions = 0; + int32_t ProcessPid = 0; + CachePolicy DefaultPolicy = CachePolicy::Default; + std::string Namespace; + eastl::vector<GetCacheRecordRequest> Requests; bool Parse(const CbPackage& RpcRequest); bool Parse(const CbObjectView& RpcRequest); @@ -129,13 +129,13 @@ namespace cacherequests { struct GetCacheRecordResult { - CacheKey Key = CacheKey::Empty; - std::vector<GetCacheRecordResultValue> Values; + CacheKey Key = CacheKey::Empty; + eastl::vector<GetCacheRecordResultValue> Values; }; struct GetCacheRecordsResult { - std::vector<std::optional<GetCacheRecordResult>> Results; + eastl::vector<std::optional<GetCacheRecordResult>> Results; bool Parse(const CbPackage& Package, const std::span<const size_t> OptionalRecordResultIndexes = {}); bool Format(CbPackage& OutPackage) const; @@ -154,10 +154,10 @@ namespace cacherequests { struct PutCacheValuesRequest { - uint32_t AcceptMagic = 0; - CachePolicy DefaultPolicy = CachePolicy::Default; - std::string Namespace; - std::vector<PutCacheValueRequest> Requests; + uint32_t AcceptMagic = 0; + CachePolicy DefaultPolicy = CachePolicy::Default; + std::string Namespace; + eastl::vector<PutCacheValueRequest> Requests; bool Parse(const CbPackage& Package); bool Format(CbPackage& OutPackage) const; @@ -165,7 +165,7 @@ namespace cacherequests { struct PutCacheValuesResult { - std::vector<bool> Success; + eastl::vector<bool> Success; bool Parse(const CbPackage& Package); bool Format(CbPackage& OutPackage) const; @@ -182,12 +182,12 @@ namespace cacherequests { struct GetCacheValuesRequest { - uint32_t AcceptMagic = 0; - uint16_t AcceptOptions = 0; - int32_t ProcessPid = 0; - CachePolicy DefaultPolicy = CachePolicy::Default; - std::string Namespace; - std::vector<GetCacheValueRequest> Requests; + uint32_t AcceptMagic = 0; + uint16_t AcceptOptions = 0; + int32_t ProcessPid = 0; + CachePolicy DefaultPolicy = CachePolicy::Default; + std::string Namespace; + eastl::vector<GetCacheValueRequest> Requests; bool Parse(const CbObjectView& BatchObject); bool Format(CbPackage& OutPackage, const std::span<const size_t> OptionalValueFilter = {}) const; @@ -204,7 +204,7 @@ namespace cacherequests { struct CacheValuesResult { - std::vector<CacheValueResult> Results; + eastl::vector<CacheValueResult> Results; bool Parse(const CbPackage& Package, const std::span<const size_t> OptionalValueResultIndexes = {}); bool Format(CbPackage& OutPackage) const; @@ -227,12 +227,12 @@ namespace cacherequests { struct GetCacheChunksRequest { - uint32_t AcceptMagic = 0; - uint16_t AcceptOptions = 0; - int32_t ProcessPid = 0; - CachePolicy DefaultPolicy = CachePolicy::Default; - std::string Namespace; - std::vector<GetCacheChunkRequest> Requests; + uint32_t AcceptMagic = 0; + uint16_t AcceptOptions = 0; + int32_t ProcessPid = 0; + CachePolicy DefaultPolicy = CachePolicy::Default; + std::string Namespace; + eastl::vector<GetCacheChunkRequest> Requests; bool Parse(const CbObjectView& BatchObject); bool Format(CbPackage& OutPackage) const; @@ -272,7 +272,7 @@ namespace cacherequests { // struct CacheRecord // { // CacheKey Key = CacheKey::Empty; - // std::vector<CacheRecordValue> Values; + // eastl::vector<CacheRecordValue> Values; // // bool Parse(CbObjectView& Reader); // bool Format(CbObjectWriter& Writer) const; diff --git a/src/zenutil/include/zenutil/chunkblock.h b/src/zenutil/include/zenutil/chunkblock.h index 21107fb7c..9cf0b3a3e 100644 --- a/src/zenutil/include/zenutil/chunkblock.h +++ b/src/zenutil/include/zenutil/chunkblock.h @@ -7,31 +7,31 @@ #include <zencore/compactbinary.h> #include <zencore/compress.h> +#include <EASTL/vector.h> #include <optional> -#include <vector> namespace zen { struct ThinChunkBlockDescription { - IoHash BlockHash; - std::vector<IoHash> ChunkRawHashes; + IoHash BlockHash; + eastl::vector<IoHash> ChunkRawHashes; }; struct ChunkBlockDescription : public ThinChunkBlockDescription { - uint64_t HeaderSize; - std::vector<uint32_t> ChunkRawLengths; - std::vector<uint32_t> ChunkCompressedLengths; + uint64_t HeaderSize; + eastl::vector<uint32_t> ChunkRawLengths; + eastl::vector<uint32_t> ChunkCompressedLengths; }; -std::vector<ChunkBlockDescription> ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject); -ChunkBlockDescription ParseChunkBlockDescription(const CbObjectView& BlockObject); -CbObject BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData); -ChunkBlockDescription GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash); +eastl::vector<ChunkBlockDescription> ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject); +ChunkBlockDescription ParseChunkBlockDescription(const CbObjectView& BlockObject); +CbObject BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData); +ChunkBlockDescription GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash); typedef std::function<std::pair<uint64_t, CompressedBuffer>(const IoHash& RawHash)> FetchChunkFunc; -CompressedBuffer GenerateChunkBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, ChunkBlockDescription& OutBlock); +CompressedBuffer GenerateChunkBlock(eastl::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, ChunkBlockDescription& OutBlock); bool IterateChunkBlock(const SharedBuffer& BlockPayload, std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor, uint64_t& OutHeaderSize); diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h index 309341550..931127c6d 100644 --- a/src/zenutil/include/zenutil/chunkedcontent.h +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -6,8 +6,8 @@ #include <zencore/compactbinarybuilder.h> #include <zencore/iohash.h> +#include <EASTL/vector.h> #include <filesystem> -#include <vector> ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_map.h> @@ -33,22 +33,22 @@ SourcePlatform GetSourceCurrentPlatform(); struct FolderContent { - SourcePlatform Platform = GetSourceCurrentPlatform(); - std::vector<std::filesystem::path> Paths; - std::vector<uint64_t> RawSizes; - std::vector<uint32_t> Attributes; - std::vector<uint64_t> ModificationTicks; + SourcePlatform Platform = GetSourceCurrentPlatform(); + eastl::vector<std::filesystem::path> Paths; + eastl::vector<uint64_t> RawSizes; + eastl::vector<uint32_t> Attributes; + eastl::vector<uint64_t> ModificationTicks; bool operator==(const FolderContent& Rhs) const; bool AreKnownFilesEqual(const FolderContent& Rhs) const; - void UpdateState(const FolderContent& Rhs, std::vector<uint32_t>& PathIndexesOufOfDate); + void UpdateState(const FolderContent& Rhs, eastl::vector<uint32_t>& PathIndexesOufOfDate); static bool AreFileAttributesEqual(const uint32_t Lhs, const uint32_t Rhs); }; -FolderContent GetUpdatedContent(const FolderContent& Old, - const FolderContent& New, - std::vector<std::filesystem::path>& OutDeletedPathIndexes); +FolderContent GetUpdatedContent(const FolderContent& Old, + const FolderContent& New, + eastl::vector<std::filesystem::path>& OutDeletedPathIndexes); void SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output); FolderContent LoadFolderContentToCompactBinary(CbObjectView Input); @@ -77,21 +77,21 @@ struct ChunkedContentData // ChunkCounts for that index will be the number of indexes in ChunkOrders that describe // the sequence of chunks required to reconstruct the asset. // Offset into ChunkOrders is based on how many entries in ChunkOrders the previous [n - 1] SequenceRawHashes uses - std::vector<IoHash> SequenceRawHashes; // Raw hash for Chunk sequence - std::vector<uint32_t> ChunkCounts; // Chunk count of ChunkOrder for SequenceRawHashes[n] - std::vector<uint32_t> ChunkOrders; // Chunk sequence indexed into ChunkHashes, ChunkCounts[n] indexes per SequenceRawHashes[n] - std::vector<IoHash> ChunkHashes; // Unique chunk hashes - std::vector<uint64_t> ChunkRawSizes; // Unique chunk raw size for ChunkHash[n] + eastl::vector<IoHash> SequenceRawHashes; // Raw hash for Chunk sequence + eastl::vector<uint32_t> ChunkCounts; // Chunk count of ChunkOrder for SequenceRawHashes[n] + eastl::vector<uint32_t> ChunkOrders; // Chunk sequence indexed into ChunkHashes, ChunkCounts[n] indexes per SequenceRawHashes[n] + eastl::vector<IoHash> ChunkHashes; // Unique chunk hashes + eastl::vector<uint64_t> ChunkRawSizes; // Unique chunk raw size for ChunkHash[n] }; struct ChunkedFolderContent { - SourcePlatform Platform = GetSourceCurrentPlatform(); - std::vector<std::filesystem::path> Paths; - std::vector<uint64_t> RawSizes; - std::vector<uint32_t> Attributes; - std::vector<IoHash> RawHashes; - ChunkedContentData ChunkedContent; + SourcePlatform Platform = GetSourceCurrentPlatform(); + eastl::vector<std::filesystem::path> Paths; + eastl::vector<uint64_t> RawSizes; + eastl::vector<uint32_t> Attributes; + eastl::vector<IoHash> RawHashes; + ChunkedContentData ChunkedContent; }; void SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output); @@ -129,12 +129,13 @@ struct ChunkedContentLookup }; tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex; tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceIndex; - std::vector<uint32_t> SequenceIndexChunkOrderOffset; - std::vector<ChunkSequenceLocation> ChunkSequenceLocations; - std::vector<size_t> + eastl::vector<uint32_t> SequenceIndexChunkOrderOffset; + eastl::vector<ChunkSequenceLocation> ChunkSequenceLocations; + eastl::vector<size_t> ChunkSequenceLocationOffset; // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex - std::vector<uint32_t> ChunkSequenceLocationCounts; // ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex - std::vector<uint32_t> SequenceIndexFirstPathIndex; // SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash + eastl::vector<uint32_t> ChunkSequenceLocationCounts; // ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex + eastl::vector<uint32_t> + SequenceIndexFirstPathIndex; // SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash }; ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content); @@ -190,7 +191,7 @@ namespace compactbinary_helpers { } template<typename Type> - void WriteArray(const std::vector<Type>& Values, std::string_view ArrayName, CbWriter& Output) + void WriteArray(const eastl::vector<Type>& Values, std::string_view ArrayName, CbWriter& Output) { WriteArray(std::span<const Type>(Values), ArrayName, Output); } @@ -207,7 +208,7 @@ namespace compactbinary_helpers { } template<> - inline void WriteArray(const std::vector<std::filesystem::path>& Values, std::string_view ArrayName, CbWriter& Output) + inline void WriteArray(const eastl::vector<std::filesystem::path>& Values, std::string_view ArrayName, CbWriter& Output) { WriteArray(std::span<const std::filesystem::path>(Values), ArrayName, Output); } @@ -222,12 +223,12 @@ namespace compactbinary_helpers { Output.EndArray(); } - inline void WriteBinaryAttachmentArray(const std::vector<IoHash>& Values, std::string_view ArrayName, CbWriter& Output) + inline void WriteBinaryAttachmentArray(const eastl::vector<IoHash>& Values, std::string_view ArrayName, CbWriter& Output) { WriteArray(std::span<const IoHash>(Values), ArrayName, Output); } - inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<uint32_t>& Result) + inline void ReadArray(std::string_view ArrayName, CbObjectView Input, eastl::vector<uint32_t>& Result) { CbArrayView Array = Input[ArrayName].AsArrayView(); Result.reserve(Array.Num()); @@ -237,7 +238,7 @@ namespace compactbinary_helpers { } } - inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<uint64_t>& Result) + inline void ReadArray(std::string_view ArrayName, CbObjectView Input, eastl::vector<uint64_t>& Result) { CbArrayView Array = Input[ArrayName].AsArrayView(); Result.reserve(Array.Num()); @@ -247,7 +248,7 @@ namespace compactbinary_helpers { } } - inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<std::filesystem::path>& Result) + inline void ReadArray(std::string_view ArrayName, CbObjectView Input, eastl::vector<std::filesystem::path>& Result) { CbArrayView Array = Input[ArrayName].AsArrayView(); Result.reserve(Array.Num()); @@ -258,7 +259,7 @@ namespace compactbinary_helpers { } } - inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<IoHash>& Result) + inline void ReadArray(std::string_view ArrayName, CbObjectView Input, eastl::vector<IoHash>& Result) { CbArrayView Array = Input[ArrayName].AsArrayView(); Result.reserve(Array.Num()); @@ -268,7 +269,7 @@ namespace compactbinary_helpers { } } - inline void ReadBinaryAttachmentArray(std::string_view ArrayName, CbObjectView Input, std::vector<IoHash>& Result) + inline void ReadBinaryAttachmentArray(std::string_view ArrayName, CbObjectView Input, eastl::vector<IoHash>& Result) { CbArrayView Array = Input[ArrayName].AsArrayView(); Result.reserve(Array.Num()); diff --git a/src/zenutil/include/zenutil/chunkedfile.h b/src/zenutil/include/zenutil/chunkedfile.h index 4cec80fdb..33dacd432 100644 --- a/src/zenutil/include/zenutil/chunkedfile.h +++ b/src/zenutil/include/zenutil/chunkedfile.h @@ -6,8 +6,8 @@ #include <zencore/iohash.h> #include <zencore/zencore.h> +#include <EASTL/vector.h> #include <functional> -#include <vector> namespace zen { @@ -15,10 +15,10 @@ class BasicFile; struct ChunkedInfo { - uint64_t RawSize = 0; - IoHash RawHash; - std::vector<uint32_t> ChunkSequence; - std::vector<IoHash> ChunkHashes; + uint64_t RawSize = 0; + IoHash RawHash; + eastl::vector<uint32_t> ChunkSequence; + eastl::vector<IoHash> ChunkHashes; }; struct ChunkSource @@ -29,8 +29,8 @@ struct ChunkSource struct ChunkedInfoWithSource { - ChunkedInfo Info; - std::vector<ChunkSource> ChunkSources; + ChunkedInfo Info; + eastl::vector<ChunkSource> ChunkSources; }; struct ChunkedParams diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h index ebc80e207..ffdf453e5 100644 --- a/src/zenutil/include/zenutil/chunkingcontroller.h +++ b/src/zenutil/include/zenutil/chunkingcontroller.h @@ -11,7 +11,7 @@ namespace zen { -const std::vector<std::string_view> DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self"}; +const eastl::vector<std::string_view> DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self"}; const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u, .MaxSize = 128u * 1024u, diff --git a/src/zenutil/include/zenutil/chunkrequests.h b/src/zenutil/include/zenutil/chunkrequests.h index 2d6c222bb..91e23ec07 100644 --- a/src/zenutil/include/zenutil/chunkrequests.h +++ b/src/zenutil/include/zenutil/chunkrequests.h @@ -4,9 +4,9 @@ #include <zencore/uid.h> +#include <EASTL/vector.h> #include <optional> #include <span> -#include <vector> namespace zen { class IoBuffer; @@ -19,9 +19,9 @@ struct RequestChunkEntry uint64_t RequestBytes; }; -std::vector<IoBuffer> ParseChunkBatchResponse(const IoBuffer& Buffer); -IoBuffer BuildChunkBatchRequest(const std::vector<RequestChunkEntry>& Entries); -std::optional<std::vector<RequestChunkEntry>> ParseChunkBatchRequest(const IoBuffer& Payload); -std::vector<IoBuffer> BuildChunkBatchResponse(const std::vector<RequestChunkEntry>& Requests, std::span<IoBuffer> Chunks); +eastl::vector<IoBuffer> ParseChunkBatchResponse(const IoBuffer& Buffer); +IoBuffer BuildChunkBatchRequest(const eastl::vector<RequestChunkEntry>& Entries); +std::optional<eastl::vector<RequestChunkEntry>> ParseChunkBatchRequest(const IoBuffer& Payload); +eastl::vector<IoBuffer> BuildChunkBatchResponse(const eastl::vector<RequestChunkEntry>& Requests, std::span<IoBuffer> Chunks); } // namespace zen diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index 852271868..f67c95d92 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -25,13 +25,13 @@ struct JupiterResult struct PutRefResult : JupiterResult { - std::vector<IoHash> Needs; - IoHash RawHash; + eastl::vector<IoHash> Needs; + IoHash RawHash; }; struct FinalizeRefResult : JupiterResult { - std::vector<IoHash> Needs; + eastl::vector<IoHash> Needs; }; struct JupiterExistsResult : JupiterResult @@ -46,13 +46,13 @@ struct GetObjectReferencesResult : JupiterResult struct PutBuildPartResult : JupiterResult { - std::vector<IoHash> Needs; - IoHash RawHash; + eastl::vector<IoHash> Needs; + IoHash RawHash; }; struct FinalizeBuildPartResult : JupiterResult { - std::vector<IoHash> Needs; + eastl::vector<IoHash> Needs; }; /** @@ -100,7 +100,7 @@ public: JupiterExistsResult CompressedBlobExists(std::string_view Namespace, const std::set<IoHash>& Keys); JupiterExistsResult ObjectExists(std::string_view Namespace, const std::set<IoHash>& Keys); - std::vector<IoHash> Filter(std::string_view Namespace, std::string_view BucketId, const std::vector<IoHash>& ChunkHashes); + eastl::vector<IoHash> Filter(std::string_view Namespace, std::string_view BucketId, const eastl::vector<IoHash>& ChunkHashes); JupiterResult ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload); JupiterResult PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload); @@ -125,21 +125,21 @@ public: const IoHash& Hash, std::filesystem::path TempFolderPath); - JupiterResult PutMultipartBuildBlob(std::string_view Namespace, - std::string_view BucketId, - const Oid& BuildId, - const IoHash& Hash, - ZenContentType ContentType, - uint64_t PayloadSize, - std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter, - std::vector<std::function<JupiterResult(bool& OutIsComplete)>>& OutWorkItems); + JupiterResult PutMultipartBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const IoHash& Hash, + ZenContentType ContentType, + uint64_t PayloadSize, + std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter, + eastl::vector<std::function<JupiterResult(bool& OutIsComplete)>>& OutWorkItems); JupiterResult GetMultipartBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoHash& Hash, uint64_t ChunkSize, std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver, - std::vector<std::function<JupiterResult()>>& OutWorkItems); + eastl::vector<std::function<JupiterResult()>>& OutWorkItems); JupiterResult PutBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, diff --git a/src/zenutil/include/zenutil/parallellwork.h b/src/zenutil/include/zenutil/parallellwork.h index 79798fc8d..ffa50e64c 100644 --- a/src/zenutil/include/zenutil/parallellwork.h +++ b/src/zenutil/include/zenutil/parallellwork.h @@ -110,8 +110,8 @@ private: std::atomic<bool>& m_AbortFlag; Latch m_PendingWork; - RwLock m_ErrorLock; - std::vector<std::string> m_Errors; + RwLock m_ErrorLock; + eastl::vector<std::string> m_Errors; }; } // namespace zen diff --git a/src/zenutil/include/zenutil/referencemetadata.h b/src/zenutil/include/zenutil/referencemetadata.h index 5160bfb8b..f10eb1453 100644 --- a/src/zenutil/include/zenutil/referencemetadata.h +++ b/src/zenutil/include/zenutil/referencemetadata.h @@ -64,7 +64,7 @@ BuildReferenceMetaData(uint32_t HeaderMagic, uint32_t KeyCount = gsl::narrow<uint32_t>(Keys.size()); ReferenceMetaDataHeader Header(HeaderMagic, KeyCount, Attachments.size()); - return CompositeBuffer(std::vector<SharedBuffer>{ + return CompositeBuffer(eastl::vector<SharedBuffer>{ SharedBuffer(IoBuffer(IoBuffer::Clone, &Header, sizeof(ReferenceMetaDataHeader))), SharedBuffer(IoBuffer(IoBuffer::Wrap, Keys.data(), sizeof(KeyType) * Keys.size())), SharedBuffer(IoBuffer(IoBuffer::Wrap, AttachmentCounts.data(), sizeof(uint32_t) * AttachmentCounts.size())), diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index 309885b05..bedbbd0a0 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -99,10 +99,10 @@ public: } } - virtual std::pair<IoHash, std::vector<IoHash>> PutBuildPart(const Oid& BuildId, - const Oid& BuildPartId, - std::string_view PartName, - const CbObject& MetaData) override + virtual std::pair<IoHash, eastl::vector<IoHash>> PutBuildPart(const Oid& BuildId, + const Oid& BuildPartId, + std::string_view PartName, + const CbObject& MetaData) override { ZEN_TRACE_CPU("Jupiter::PutBuildPart"); @@ -137,7 +137,7 @@ public: return PayloadToJson(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response); } - virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override + virtual eastl::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override { ZEN_TRACE_CPU("Jupiter::FinalizeBuildPart"); @@ -170,26 +170,26 @@ public: } } - virtual std::vector<std::function<void()>> PutLargeBuildBlob(const Oid& BuildId, - const IoHash& RawHash, - ZenContentType ContentType, - uint64_t PayloadSize, - std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter, - std::function<void(uint64_t, bool)>&& OnSentBytes) override + virtual eastl::vector<std::function<void()>> PutLargeBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + ZenContentType ContentType, + uint64_t PayloadSize, + std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter, + std::function<void(uint64_t, bool)>&& OnSentBytes) override { ZEN_TRACE_CPU("Jupiter::PutLargeBuildBlob"); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - std::vector<std::function<JupiterResult(bool&)>> WorkItems; - JupiterResult PutMultipartBlobResult = m_Session.PutMultipartBuildBlob(m_Namespace, - m_Bucket, - BuildId, - RawHash, - ContentType, - PayloadSize, - std::move(Transmitter), - WorkItems); + eastl::vector<std::function<JupiterResult(bool&)>> WorkItems; + JupiterResult PutMultipartBlobResult = m_Session.PutMultipartBuildBlob(m_Namespace, + m_Bucket, + BuildId, + RawHash, + ContentType, + PayloadSize, + std::move(Transmitter), + WorkItems); AddStatistic(PutMultipartBlobResult); if (!PutMultipartBlobResult.Success) { @@ -198,7 +198,7 @@ public: } OnSentBytes(PutMultipartBlobResult.SentBytes, WorkItems.empty()); - std::vector<std::function<void()>> WorkList; + eastl::vector<std::function<void()>> WorkList; for (auto& WorkItem : WorkItems) { WorkList.emplace_back([this, WorkItem = std::move(WorkItem), OnSentBytes]() { @@ -233,7 +233,7 @@ public: return std::move(GetBuildBlobResult.Response); } - virtual std::vector<std::function<void()>> GetLargeBuildBlob( + virtual eastl::vector<std::function<void()>> GetLargeBuildBlob( const Oid& BuildId, const IoHash& RawHash, uint64_t ChunkSize, @@ -243,8 +243,8 @@ public: Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - std::vector<std::function<JupiterResult()>> WorkItems; - JupiterResult GetMultipartBlobResult = + eastl::vector<std::function<JupiterResult()>> WorkItems; + JupiterResult GetMultipartBlobResult = m_Session.GetMultipartBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, ChunkSize, std::move(Receiver), WorkItems); AddStatistic(GetMultipartBlobResult); @@ -253,7 +253,7 @@ public: throw std::runtime_error( fmt::format("Failed getting build part: {} ({})", GetMultipartBlobResult.Reason, GetMultipartBlobResult.ErrorCode)); } - std::vector<std::function<void()>> WorkList; + eastl::vector<std::function<void()>> WorkList; for (auto& WorkItem : WorkItems) { WorkList.emplace_back([this, WorkItem = std::move(WorkItem)]() { @@ -287,7 +287,7 @@ public: } } - virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override + virtual eastl::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override { ZEN_TRACE_CPU("Jupiter::FindBlocks"); @@ -302,7 +302,7 @@ public: return ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching known blocks"sv, FindResult.Response)); } - virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override + virtual eastl::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override { ZEN_TRACE_CPU("Jupiter::GetBlockMetadata"); @@ -326,7 +326,7 @@ public: throw std::runtime_error( fmt::format("Failed fetching block metadatas: {} ({})", GetBlockMetadataResult.Reason, GetBlockMetadataResult.ErrorCode)); } - std::vector<ChunkBlockDescription> UnorderedList = + eastl::vector<ChunkBlockDescription> UnorderedList = ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching block metadatas", GetBlockMetadataResult.Response)); tsl::robin_map<IoHash, size_t, IoHash::Hasher> BlockDescriptionLookup; for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++) @@ -334,7 +334,7 @@ public: const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex]; BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex); } - std::vector<ChunkBlockDescription> SortedBlockDescriptions; + eastl::vector<ChunkBlockDescription> SortedBlockDescriptions; SortedBlockDescriptions.reserve(BlockDescriptionLookup.size()); for (const IoHash& BlockHash : BlockHashes) { diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp index 06ac6ae36..3b2571503 100644 --- a/src/zenutil/jupiter/jupitersession.cpp +++ b/src/zenutil/jupiter/jupitersession.cpp @@ -303,8 +303,8 @@ JupiterSession::ObjectExists(std::string_view Namespace, const std::set<IoHash>& return CacheTypeExists(Namespace, "objects"sv, Keys); } -std::vector<IoHash> -JupiterSession::Filter(std::string_view Namespace, std::string_view BucketId, const std::vector<IoHash>& ChunkHashes) +eastl::vector<IoHash> +JupiterSession::Filter(std::string_view Namespace, std::string_view BucketId, const eastl::vector<IoHash>& ChunkHashes) { // ExtendableStringBuilder<256> Uri; // Uri << m_CacheClient->ServiceUrl(); @@ -448,14 +448,14 @@ JupiterSession::PutBuildBlob(std::string_view Namespace, } JupiterResult -JupiterSession::PutMultipartBuildBlob(std::string_view Namespace, - std::string_view BucketId, - const Oid& BuildId, - const IoHash& Hash, - ZenContentType ContentType, - uint64_t PayloadSize, - std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter, - std::vector<std::function<JupiterResult(bool& OutIsComplete)>>& OutWorkItems) +JupiterSession::PutMultipartBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const IoHash& Hash, + ZenContentType ContentType, + uint64_t PayloadSize, + std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter, + eastl::vector<std::function<JupiterResult(bool& OutIsComplete)>>& OutWorkItems) { struct MultipartUploadResponse { @@ -467,9 +467,9 @@ JupiterSession::PutMultipartBuildBlob(std::string_view Namespace, std::string QueryString; }; - std::string UploadId; - std::string BlobName; - std::vector<Part> Parts; + std::string UploadId; + std::string BlobName; + eastl::vector<Part> Parts; static MultipartUploadResponse Parse(CbObject& Payload) { @@ -632,7 +632,7 @@ JupiterSession::GetMultipartBuildBlob(std::string_view Namespa const IoHash& Hash, uint64_t ChunkSize, std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver, - std::vector<std::function<JupiterResult()>>& OutWorkItems) + eastl::vector<std::function<JupiterResult()>>& OutWorkItems) { std::string RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()); HttpClient::Response Response = diff --git a/src/zenutil/openprocesscache.cpp b/src/zenutil/openprocesscache.cpp index 88bbce8a6..f7f29d623 100644 --- a/src/zenutil/openprocesscache.cpp +++ b/src/zenutil/openprocesscache.cpp @@ -142,9 +142,9 @@ OpenProcessCache::GetProcessHandle(Oid SessionId, int ProcessPid) void OpenProcessCache::GCHandles() { - std::vector<void*> DeadHandles; + eastl::vector<void*> DeadHandles; { - std::vector<Oid> DeadSessions; + eastl::vector<Oid> DeadSessions; RwLock::ExclusiveLockScope Lock(m_SessionsLock); for (auto& It : m_Sessions) { diff --git a/src/zenvfs/include/zenvfs/vfs.h b/src/zenvfs/include/zenvfs/vfs.h index 6da9697a1..65d433bee 100644 --- a/src/zenvfs/include/zenvfs/vfs.h +++ b/src/zenvfs/include/zenvfs/vfs.h @@ -12,7 +12,7 @@ # include <string_view> # include <memory> -# include <vector> +# include <EASTL/vector.h> namespace zen { @@ -49,8 +49,8 @@ struct VfsTreeNode bool IsDirectory() const; void PopulateVirtualNode(); - const std::vector<std::unique_ptr<VfsTreeNode>>& GetChildren() const { return ChildArray; } - std::vector<std::unique_ptr<VfsTreeNode>>& GetChildren() { return ChildArray; } + const eastl::vector<std::unique_ptr<VfsTreeNode>>& GetChildren() const { return ChildArray; } + eastl::vector<std::unique_ptr<VfsTreeNode>>& GetChildren() { return ChildArray; } private: std::string Name; @@ -58,9 +58,9 @@ private: Oid ChunkId = Oid::Zero; Ref<VfsTreeDataSource> DataSource; - inline static const uint64_t DirectoryMarker = ~uint64_t(0); - std::vector<std::unique_ptr<VfsTreeNode>> ChildArray; - VfsTreeNode* ParentNode = nullptr; + inline static const uint64_t DirectoryMarker = ~uint64_t(0); + eastl::vector<std::unique_ptr<VfsTreeNode>> ChildArray; + VfsTreeNode* ParentNode = nullptr; VfsTreeNode* AddNewChildNode() { return ChildArray.emplace_back(std::make_unique<VfsTreeNode>(this)).get(); } diff --git a/src/zenvfs/vfsprovider.cpp b/src/zenvfs/vfsprovider.cpp index a3cfe9d15..7b4443100 100644 --- a/src/zenvfs/vfsprovider.cpp +++ b/src/zenvfs/vfsprovider.cpp @@ -10,6 +10,8 @@ # include <system_error> +# include <EASTL/sort.h> + namespace zen { bool @@ -194,7 +196,7 @@ VfsTreeNode::NormalizeTree() // don't do this then there will be spurious duplicates and things will // behave strangely - std::sort(begin(ChildArray), end(ChildArray), FileNameLessThan); + eastl::sort(ChildArray.begin(), ChildArray.end(), FileNameLessThan); for (auto& Child : GetChildren()) { diff --git a/src/zenvfs/vfsprovider.h b/src/zenvfs/vfsprovider.h index 8e6896956..87c661a57 100644 --- a/src/zenvfs/vfsprovider.h +++ b/src/zenvfs/vfsprovider.h @@ -19,7 +19,7 @@ ZEN_THIRD_PARTY_INCLUDES_START ZEN_THIRD_PARTY_INCLUDES_END # include <unordered_map> -# include <vector> +# include <EASTL/vector.h> namespace zen { @@ -47,10 +47,10 @@ struct VfsProvider : public ProjFsProviderInterface { struct EnumerationState { - std::string Path; - const VfsTreeNode* Node = nullptr; - std::vector<std::unique_ptr<VfsTreeNode>>::const_iterator Iterator; - std::vector<std::unique_ptr<VfsTreeNode>>::const_iterator EndIterator; + std::string Path; + const VfsTreeNode* Node = nullptr; + eastl::vector<std::unique_ptr<VfsTreeNode>>::const_iterator Iterator; + eastl::vector<std::unique_ptr<VfsTreeNode>>::const_iterator EndIterator; void RestartScan(); inline bool IsCurrentValid() { return Iterator != EndIterator; } |