aboutsummaryrefslogtreecommitdiff
path: root/zenstore
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2022-05-02 11:03:04 +0200
committerDan Engelbrecht <[email protected]>2022-05-02 11:11:54 +0200
commit10690e805f45a590094a659c8e1f6482d12aaf8e (patch)
tree42aac794a1fcd723530aa0f2bc4940f095ece673 /zenstore
parentadd back gc space reclaim call (diff)
downloadzen-10690e805f45a590094a659c8e1f6482d12aaf8e.tar.xz
zen-10690e805f45a590094a659c8e1f6482d12aaf8e.zip
cleanup
Diffstat (limited to 'zenstore')
-rw-r--r--zenstore/blockstore.cpp50
-rw-r--r--zenstore/compactcas.cpp13
-rw-r--r--zenstore/include/zenstore/blockstore.h33
3 files changed, 48 insertions, 48 deletions
diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp
index b3608687b..fd54e7291 100644
--- a/zenstore/blockstore.cpp
+++ b/zenstore/blockstore.cpp
@@ -276,7 +276,7 @@ BlockStore::Flush()
void
BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot,
const std::vector<BlockStoreLocation>& ChunkLocations,
- const std::vector<size_t>& KeepChunkIndexes,
+ const ChunkIndexArray& KeepChunkIndexes,
uint64_t PayloadAlignment,
bool DryRun,
const ReclaimCallback& ChangeCallback,
@@ -338,8 +338,8 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot,
}
std::unordered_map<uint32_t, size_t> BlockIndexToChunkMapIndex;
- std::vector<std::vector<size_t>> BlockKeepChunks;
- std::vector<std::vector<size_t>> BlockDeleteChunks;
+ std::vector<ChunkIndexArray> BlockKeepChunks;
+ std::vector<ChunkIndexArray> BlockDeleteChunks;
BlockIndexToChunkMapIndex.reserve(BlockCount);
BlockKeepChunks.reserve(BlockCount);
@@ -374,12 +374,12 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot,
if (KeepChunkMap.contains(Index))
{
- std::vector<size_t>& IndexMap = BlockKeepChunks[ChunkMapIndex];
+ ChunkIndexArray& IndexMap = BlockKeepChunks[ChunkMapIndex];
IndexMap.push_back(Index);
NewTotalSize += Location.Size;
continue;
}
- std::vector<size_t>& IndexMap = BlockDeleteChunks[ChunkMapIndex];
+ ChunkIndexArray& IndexMap = BlockDeleteChunks[ChunkMapIndex];
IndexMap.push_back(Index);
DeleteCount++;
}
@@ -388,9 +388,9 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot,
BlocksToReWrite.reserve(BlockIndexToChunkMapIndex.size());
for (const auto& Entry : BlockIndexToChunkMapIndex)
{
- uint32_t BlockIndex = Entry.first;
- size_t ChunkMapIndex = Entry.second;
- const std::vector<size_t>& ChunkMap = BlockDeleteChunks[ChunkMapIndex];
+ uint32_t BlockIndex = Entry.first;
+ size_t ChunkMapIndex = Entry.second;
+ const ChunkIndexArray& ChunkMap = BlockDeleteChunks[ChunkMapIndex];
if (ChunkMap.empty())
{
continue;
@@ -430,10 +430,10 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot,
ZEN_ASSERT(OldBlockFile);
}
- const std::vector<size_t>& KeepMap = BlockKeepChunks[ChunkMapIndex];
+ const ChunkIndexArray& KeepMap = BlockKeepChunks[ChunkMapIndex];
if (KeepMap.empty())
{
- const std::vector<size_t>& DeleteMap = BlockDeleteChunks[ChunkMapIndex];
+ const ChunkIndexArray& DeleteMap = BlockDeleteChunks[ChunkMapIndex];
for (size_t DeleteIndex : DeleteMap)
{
DeletedSize += ChunkLocations[DeleteIndex].Size;
@@ -460,8 +460,8 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot,
continue;
}
- std::vector<std::pair<size_t, BlockStoreLocation>> MovedChunks;
- std::vector<uint8_t> Chunk;
+ MovedChunksArray MovedChunks;
+ std::vector<uint8_t> Chunk;
for (const size_t& ChunkIndex : KeepMap)
{
const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex];
@@ -553,7 +553,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot,
NewBlockFile = {};
}
- const std::vector<size_t>& DeleteMap = BlockDeleteChunks[ChunkMapIndex];
+ const ChunkIndexArray& DeleteMap = BlockDeleteChunks[ChunkMapIndex];
for (size_t DeleteIndex : DeleteMap)
{
DeletedSize += ChunkLocations[DeleteIndex].Size;
@@ -595,10 +595,10 @@ BlockStore::IterateChunks(const std::vector<BlockStoreLocation>& ChunkLocations,
// pass. An alternative strategy would be to use memory mapping.
{
- std::vector<size_t> BigChunks;
- const uint64_t WindowSize = 4 * 1024 * 1024;
- IoBuffer ReadBuffer{WindowSize};
- void* BufferBase = ReadBuffer.MutableData();
+ ChunkIndexArray BigChunks;
+ const uint64_t WindowSize = 4 * 1024 * 1024;
+ IoBuffer ReadBuffer{WindowSize};
+ void* BufferBase = ReadBuffer.MutableData();
RwLock::SharedLockScope _(m_InsertLock);
@@ -740,7 +740,7 @@ BlockStore::Split(const std::vector<BlockStoreLocation>& ChunkLocations,
if (CleanSource && (MaxRequiredBlockCount < 2))
{
- std::vector<std::pair<size_t, BlockStoreLocation>> Chunks;
+ MovedChunksArray Chunks;
Chunks.reserve(ChunkCount);
for (size_t Index = 0; Index < ChunkCount; ++Index)
{
@@ -755,7 +755,7 @@ BlockStore::Split(const std::vector<BlockStoreLocation>& ChunkLocations,
return true;
}
- std::vector<size_t> ChunkIndexes;
+ ChunkIndexArray ChunkIndexes;
ChunkIndexes.reserve(ChunkCount);
for (size_t Index = 0; Index < ChunkCount; ++Index)
{
@@ -773,14 +773,14 @@ BlockStore::Split(const std::vector<BlockStoreLocation>& ChunkLocations,
std::vector<BlockStoreLocation> NewLocations;
struct BlockData
{
- std::vector<std::pair<size_t, BlockStoreLocation>> Chunks;
- uint64_t BlockOffset;
- uint64_t BlockSize;
- uint32_t BlockIndex;
+ MovedChunksArray Chunks;
+ uint64_t BlockOffset;
+ uint64_t BlockSize;
+ uint32_t BlockIndex;
};
- std::vector<BlockData> BlockRanges;
- std::vector<std::pair<size_t, BlockStoreLocation>> Chunks;
+ std::vector<BlockData> BlockRanges;
+ MovedChunksArray Chunks;
BlockRanges.reserve(MaxRequiredBlockCount);
for (const size_t& ChunkIndex : ChunkIndexes)
{
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index 5734a16b6..ade6a7daf 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -345,11 +345,6 @@ CasContainerStrategy::Scrub(ScrubContext& Ctx)
std::vector<IoHash> BadKeys;
- // We do a read sweep through the payloads file and validate
- // any entries that are contained within each segment, with
- // the assumption that most entries will be checked in this
- // pass. An alternative strategy would be to use memory mapping.
-
m_BlockStore.IterateChunks(
ChunkLocations,
[&ChunkIndexToChunkHash, &BadKeys](size_t ChunkIndex, const void* Data, uint64_t Size) {
@@ -443,7 +438,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
}
std::vector<BlockStoreLocation> ChunkLocations;
- std::vector<size_t> KeepChunkIndexes;
+ BlockStore::ChunkIndexArray KeepChunkIndexes;
std::vector<IoHash> ChunkIndexToChunkHash;
ChunkLocations.reserve(TotalChunkCount);
ChunkIndexToChunkHash.reserve(TotalChunkCount);
@@ -477,8 +472,8 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
m_PayloadAlignment,
false,
[this, &DeletedChunks, &ChunkIndexToChunkHash, &LocationMap, &ReadBlockTimeUs, &ReadBlockLongestTimeUs](
- const std::vector<std::pair<size_t, BlockStoreLocation>>& MovedChunks,
- const std::vector<size_t>& RemovedChunks) {
+ const BlockStore::MovedChunksArray& MovedChunks,
+ const BlockStore::ChunkIndexArray& RemovedChunks) {
std::vector<CasDiskIndexEntry> LogEntries;
LogEntries.reserve(MovedChunks.size() + RemovedChunks.size());
for (const auto& Entry : MovedChunks)
@@ -849,7 +844,7 @@ CasContainerStrategy::MigrateLegacyData(bool CleanSource)
m_PayloadAlignment,
CleanSource,
[this, &LegacyDiskIndex, &ChunkIndexToChunkHash, &LegacyCasLog, &CasLog, CleanSource, &MigratedBlockCount, &MigratedChunkCount](
- const std::vector<std::pair<size_t, BlockStoreLocation>>& MovedChunks) {
+ const BlockStore::MovedChunksArray& MovedChunks) {
std::vector<CasDiskIndexEntry> LogEntries;
LogEntries.reserve(MovedChunks.size());
for (const auto& Entry : MovedChunks)
diff --git a/zenstore/include/zenstore/blockstore.h b/zenstore/include/zenstore/blockstore.h
index 28f0a5541..53bcf72db 100644
--- a/zenstore/include/zenstore/blockstore.h
+++ b/zenstore/include/zenstore/blockstore.h
@@ -117,24 +117,28 @@ public:
std::unordered_set<uint32_t> ExcludeBlockIndexes;
size_t BlockCount;
};
- typedef std::function<void(const std::vector<std::pair<size_t, BlockStoreLocation>>& MovedChunks,
- const std::vector<size_t>& RemovedChunks)>
- ReclaimCallback;
- typedef std::function<uint64_t()> ClaimGCReserveCallback;
- typedef std::function<void(const BlockStoreLocation& Location)> WriteChunkCallback;
- typedef std::function<void(size_t ChunkIndex, const void* Data, uint64_t Size)> IterateChunksSmallSizeCallback;
- typedef std::function<void(size_t ChunkIndex, BasicFile& BlockFile, uint64_t Offset, uint64_t Size)> IterateChunksLargeSizeCallback;
- typedef std::function<void(const std::vector<std::pair<size_t, BlockStoreLocation>>& MovedChunks)> SplitCallback;
+
+ typedef std::vector<std::pair<size_t, BlockStoreLocation>> MovedChunksArray;
+ typedef std::vector<size_t> ChunkIndexArray;
+
+ typedef std::function<void(const MovedChunksArray& MovedChunks, const ChunkIndexArray& RemovedChunks)> ReclaimCallback;
+ typedef std::function<uint64_t()> ClaimGCReserveCallback;
+ typedef std::function<void(const BlockStoreLocation& Location)> WriteChunkCallback;
+ typedef std::function<void(size_t ChunkIndex, const void* Data, uint64_t Size)> IterateChunksSmallSizeCallback;
+ typedef std::function<void(size_t ChunkIndex, BasicFile& BlockFile, uint64_t Offset, uint64_t Size)> IterateChunksLargeSizeCallback;
+ typedef std::function<void(const MovedChunksArray& MovedChunks)> SplitCallback;
void Initialize(const std::filesystem::path& BlocksBasePath,
uint64_t MaxBlockSize,
uint64_t MaxBlockCount,
const std::vector<BlockStoreLocation>& KnownLocations);
+
void WriteChunk(
const void* Data,
uint64_t Size,
uint64_t Alignment,
WriteChunkCallback Callback = [](const BlockStoreLocation&) {});
+
Ref<BlockStoreFile> GetChunkBlock(const BlockStoreLocation& Location);
void Flush();
@@ -142,15 +146,16 @@ public:
void ReclaimSpace(
const ReclaimSnapshotState& Snapshot,
const std::vector<BlockStoreLocation>& ChunkLocations,
- const std::vector<size_t>& KeepChunkIndexes,
+ const ChunkIndexArray& KeepChunkIndexes,
uint64_t PayloadAlignment,
bool DryRun,
- const ReclaimCallback& ChangeCallback = [](const std::vector<std::pair<size_t, BlockStoreLocation>>&,
- const std::vector<size_t>&) {},
+ const ReclaimCallback& ChangeCallback = [](const MovedChunksArray&, const ChunkIndexArray&) {},
const ClaimGCReserveCallback& GcReserveCallback = []() { return 0; });
- void IterateChunks(const std::vector<BlockStoreLocation>& ChunkLocations,
- IterateChunksSmallSizeCallback SmallSizeCallback,
- IterateChunksLargeSizeCallback LargeSizeCallback);
+
+ void IterateChunks(const std::vector<BlockStoreLocation>& ChunkLocations,
+ IterateChunksSmallSizeCallback SmallSizeCallback,
+ IterateChunksLargeSizeCallback LargeSizeCallback);
+
static bool Split(const std::vector<BlockStoreLocation>& ChunkLocations,
const std::filesystem::path& SourceBlockFilePath,
const std::filesystem::path& BlocksBasePath,