diff options
| author | Dan Engelbrecht <[email protected]> | 2022-05-02 11:03:04 +0200 |
|---|---|---|
| committer | Dan Engelbrecht <[email protected]> | 2022-05-02 11:11:54 +0200 |
| commit | 10690e805f45a590094a659c8e1f6482d12aaf8e (patch) | |
| tree | 42aac794a1fcd723530aa0f2bc4940f095ece673 /zenstore | |
| parent | add back gc space reclaim call (diff) | |
| download | zen-10690e805f45a590094a659c8e1f6482d12aaf8e.tar.xz zen-10690e805f45a590094a659c8e1f6482d12aaf8e.zip | |
cleanup
Diffstat (limited to 'zenstore')
| -rw-r--r-- | zenstore/blockstore.cpp | 50 | ||||
| -rw-r--r-- | zenstore/compactcas.cpp | 13 | ||||
| -rw-r--r-- | zenstore/include/zenstore/blockstore.h | 33 |
3 files changed, 48 insertions, 48 deletions
diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp index b3608687b..fd54e7291 100644 --- a/zenstore/blockstore.cpp +++ b/zenstore/blockstore.cpp @@ -276,7 +276,7 @@ BlockStore::Flush() void BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot, const std::vector<BlockStoreLocation>& ChunkLocations, - const std::vector<size_t>& KeepChunkIndexes, + const ChunkIndexArray& KeepChunkIndexes, uint64_t PayloadAlignment, bool DryRun, const ReclaimCallback& ChangeCallback, @@ -338,8 +338,8 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot, } std::unordered_map<uint32_t, size_t> BlockIndexToChunkMapIndex; - std::vector<std::vector<size_t>> BlockKeepChunks; - std::vector<std::vector<size_t>> BlockDeleteChunks; + std::vector<ChunkIndexArray> BlockKeepChunks; + std::vector<ChunkIndexArray> BlockDeleteChunks; BlockIndexToChunkMapIndex.reserve(BlockCount); BlockKeepChunks.reserve(BlockCount); @@ -374,12 +374,12 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot, if (KeepChunkMap.contains(Index)) { - std::vector<size_t>& IndexMap = BlockKeepChunks[ChunkMapIndex]; + ChunkIndexArray& IndexMap = BlockKeepChunks[ChunkMapIndex]; IndexMap.push_back(Index); NewTotalSize += Location.Size; continue; } - std::vector<size_t>& IndexMap = BlockDeleteChunks[ChunkMapIndex]; + ChunkIndexArray& IndexMap = BlockDeleteChunks[ChunkMapIndex]; IndexMap.push_back(Index); DeleteCount++; } @@ -388,9 +388,9 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot, BlocksToReWrite.reserve(BlockIndexToChunkMapIndex.size()); for (const auto& Entry : BlockIndexToChunkMapIndex) { - uint32_t BlockIndex = Entry.first; - size_t ChunkMapIndex = Entry.second; - const std::vector<size_t>& ChunkMap = BlockDeleteChunks[ChunkMapIndex]; + uint32_t BlockIndex = Entry.first; + size_t ChunkMapIndex = Entry.second; + const ChunkIndexArray& ChunkMap = BlockDeleteChunks[ChunkMapIndex]; if (ChunkMap.empty()) { continue; @@ -430,10 +430,10 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot, ZEN_ASSERT(OldBlockFile); } - const std::vector<size_t>& KeepMap = BlockKeepChunks[ChunkMapIndex]; + const ChunkIndexArray& KeepMap = BlockKeepChunks[ChunkMapIndex]; if (KeepMap.empty()) { - const std::vector<size_t>& DeleteMap = BlockDeleteChunks[ChunkMapIndex]; + const ChunkIndexArray& DeleteMap = BlockDeleteChunks[ChunkMapIndex]; for (size_t DeleteIndex : DeleteMap) { DeletedSize += ChunkLocations[DeleteIndex].Size; @@ -460,8 +460,8 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot, continue; } - std::vector<std::pair<size_t, BlockStoreLocation>> MovedChunks; - std::vector<uint8_t> Chunk; + MovedChunksArray MovedChunks; + std::vector<uint8_t> Chunk; for (const size_t& ChunkIndex : KeepMap) { const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex]; @@ -553,7 +553,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot, NewBlockFile = {}; } - const std::vector<size_t>& DeleteMap = BlockDeleteChunks[ChunkMapIndex]; + const ChunkIndexArray& DeleteMap = BlockDeleteChunks[ChunkMapIndex]; for (size_t DeleteIndex : DeleteMap) { DeletedSize += ChunkLocations[DeleteIndex].Size; @@ -595,10 +595,10 @@ BlockStore::IterateChunks(const std::vector<BlockStoreLocation>& ChunkLocations, // pass. An alternative strategy would be to use memory mapping. { - std::vector<size_t> BigChunks; - const uint64_t WindowSize = 4 * 1024 * 1024; - IoBuffer ReadBuffer{WindowSize}; - void* BufferBase = ReadBuffer.MutableData(); + ChunkIndexArray BigChunks; + const uint64_t WindowSize = 4 * 1024 * 1024; + IoBuffer ReadBuffer{WindowSize}; + void* BufferBase = ReadBuffer.MutableData(); RwLock::SharedLockScope _(m_InsertLock); @@ -740,7 +740,7 @@ BlockStore::Split(const std::vector<BlockStoreLocation>& ChunkLocations, if (CleanSource && (MaxRequiredBlockCount < 2)) { - std::vector<std::pair<size_t, BlockStoreLocation>> Chunks; + MovedChunksArray Chunks; Chunks.reserve(ChunkCount); for (size_t Index = 0; Index < ChunkCount; ++Index) { @@ -755,7 +755,7 @@ BlockStore::Split(const std::vector<BlockStoreLocation>& ChunkLocations, return true; } - std::vector<size_t> ChunkIndexes; + ChunkIndexArray ChunkIndexes; ChunkIndexes.reserve(ChunkCount); for (size_t Index = 0; Index < ChunkCount; ++Index) { @@ -773,14 +773,14 @@ BlockStore::Split(const std::vector<BlockStoreLocation>& ChunkLocations, std::vector<BlockStoreLocation> NewLocations; struct BlockData { - std::vector<std::pair<size_t, BlockStoreLocation>> Chunks; - uint64_t BlockOffset; - uint64_t BlockSize; - uint32_t BlockIndex; + MovedChunksArray Chunks; + uint64_t BlockOffset; + uint64_t BlockSize; + uint32_t BlockIndex; }; - std::vector<BlockData> BlockRanges; - std::vector<std::pair<size_t, BlockStoreLocation>> Chunks; + std::vector<BlockData> BlockRanges; + MovedChunksArray Chunks; BlockRanges.reserve(MaxRequiredBlockCount); for (const size_t& ChunkIndex : ChunkIndexes) { diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp index 5734a16b6..ade6a7daf 100644 --- a/zenstore/compactcas.cpp +++ b/zenstore/compactcas.cpp @@ -345,11 +345,6 @@ CasContainerStrategy::Scrub(ScrubContext& Ctx) std::vector<IoHash> BadKeys; - // We do a read sweep through the payloads file and validate - // any entries that are contained within each segment, with - // the assumption that most entries will be checked in this - // pass. An alternative strategy would be to use memory mapping. - m_BlockStore.IterateChunks( ChunkLocations, [&ChunkIndexToChunkHash, &BadKeys](size_t ChunkIndex, const void* Data, uint64_t Size) { @@ -443,7 +438,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx) } std::vector<BlockStoreLocation> ChunkLocations; - std::vector<size_t> KeepChunkIndexes; + BlockStore::ChunkIndexArray KeepChunkIndexes; std::vector<IoHash> ChunkIndexToChunkHash; ChunkLocations.reserve(TotalChunkCount); ChunkIndexToChunkHash.reserve(TotalChunkCount); @@ -477,8 +472,8 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx) m_PayloadAlignment, false, [this, &DeletedChunks, &ChunkIndexToChunkHash, &LocationMap, &ReadBlockTimeUs, &ReadBlockLongestTimeUs]( - const std::vector<std::pair<size_t, BlockStoreLocation>>& MovedChunks, - const std::vector<size_t>& RemovedChunks) { + const BlockStore::MovedChunksArray& MovedChunks, + const BlockStore::ChunkIndexArray& RemovedChunks) { std::vector<CasDiskIndexEntry> LogEntries; LogEntries.reserve(MovedChunks.size() + RemovedChunks.size()); for (const auto& Entry : MovedChunks) @@ -849,7 +844,7 @@ CasContainerStrategy::MigrateLegacyData(bool CleanSource) m_PayloadAlignment, CleanSource, [this, &LegacyDiskIndex, &ChunkIndexToChunkHash, &LegacyCasLog, &CasLog, CleanSource, &MigratedBlockCount, &MigratedChunkCount]( - const std::vector<std::pair<size_t, BlockStoreLocation>>& MovedChunks) { + const BlockStore::MovedChunksArray& MovedChunks) { std::vector<CasDiskIndexEntry> LogEntries; LogEntries.reserve(MovedChunks.size()); for (const auto& Entry : MovedChunks) diff --git a/zenstore/include/zenstore/blockstore.h b/zenstore/include/zenstore/blockstore.h index 28f0a5541..53bcf72db 100644 --- a/zenstore/include/zenstore/blockstore.h +++ b/zenstore/include/zenstore/blockstore.h @@ -117,24 +117,28 @@ public: std::unordered_set<uint32_t> ExcludeBlockIndexes; size_t BlockCount; }; - typedef std::function<void(const std::vector<std::pair<size_t, BlockStoreLocation>>& MovedChunks, - const std::vector<size_t>& RemovedChunks)> - ReclaimCallback; - typedef std::function<uint64_t()> ClaimGCReserveCallback; - typedef std::function<void(const BlockStoreLocation& Location)> WriteChunkCallback; - typedef std::function<void(size_t ChunkIndex, const void* Data, uint64_t Size)> IterateChunksSmallSizeCallback; - typedef std::function<void(size_t ChunkIndex, BasicFile& BlockFile, uint64_t Offset, uint64_t Size)> IterateChunksLargeSizeCallback; - typedef std::function<void(const std::vector<std::pair<size_t, BlockStoreLocation>>& MovedChunks)> SplitCallback; + + typedef std::vector<std::pair<size_t, BlockStoreLocation>> MovedChunksArray; + typedef std::vector<size_t> ChunkIndexArray; + + typedef std::function<void(const MovedChunksArray& MovedChunks, const ChunkIndexArray& RemovedChunks)> ReclaimCallback; + typedef std::function<uint64_t()> ClaimGCReserveCallback; + typedef std::function<void(const BlockStoreLocation& Location)> WriteChunkCallback; + typedef std::function<void(size_t ChunkIndex, const void* Data, uint64_t Size)> IterateChunksSmallSizeCallback; + typedef std::function<void(size_t ChunkIndex, BasicFile& BlockFile, uint64_t Offset, uint64_t Size)> IterateChunksLargeSizeCallback; + typedef std::function<void(const MovedChunksArray& MovedChunks)> SplitCallback; void Initialize(const std::filesystem::path& BlocksBasePath, uint64_t MaxBlockSize, uint64_t MaxBlockCount, const std::vector<BlockStoreLocation>& KnownLocations); + void WriteChunk( const void* Data, uint64_t Size, uint64_t Alignment, WriteChunkCallback Callback = [](const BlockStoreLocation&) {}); + Ref<BlockStoreFile> GetChunkBlock(const BlockStoreLocation& Location); void Flush(); @@ -142,15 +146,16 @@ public: void ReclaimSpace( const ReclaimSnapshotState& Snapshot, const std::vector<BlockStoreLocation>& ChunkLocations, - const std::vector<size_t>& KeepChunkIndexes, + const ChunkIndexArray& KeepChunkIndexes, uint64_t PayloadAlignment, bool DryRun, - const ReclaimCallback& ChangeCallback = [](const std::vector<std::pair<size_t, BlockStoreLocation>>&, - const std::vector<size_t>&) {}, + const ReclaimCallback& ChangeCallback = [](const MovedChunksArray&, const ChunkIndexArray&) {}, const ClaimGCReserveCallback& GcReserveCallback = []() { return 0; }); - void IterateChunks(const std::vector<BlockStoreLocation>& ChunkLocations, - IterateChunksSmallSizeCallback SmallSizeCallback, - IterateChunksLargeSizeCallback LargeSizeCallback); + + void IterateChunks(const std::vector<BlockStoreLocation>& ChunkLocations, + IterateChunksSmallSizeCallback SmallSizeCallback, + IterateChunksLargeSizeCallback LargeSizeCallback); + static bool Split(const std::vector<BlockStoreLocation>& ChunkLocations, const std::filesystem::path& SourceBlockFilePath, const std::filesystem::path& BlocksBasePath, |