aboutsummaryrefslogtreecommitdiff
path: root/src/zenstore/blockstore.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenstore/blockstore.cpp')
-rw-r--r--src/zenstore/blockstore.cpp99
1 files changed, 61 insertions, 38 deletions
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index e19712c40..378d9fd52 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -118,28 +118,19 @@ BlockStoreFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::functio
constexpr uint64_t ScrubSmallChunkWindowSize = 4 * 1024 * 1024;
-void
-BlockStore::Initialize(const std::filesystem::path& BlocksBasePath,
- uint64_t MaxBlockSize,
- uint64_t MaxBlockCount,
- const std::vector<BlockStoreLocation>& KnownLocations)
+std::unordered_map<uint32_t, uint64_t>
+BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t MaxBlockSize, uint64_t MaxBlockCount)
{
ZEN_ASSERT(MaxBlockSize > 0);
ZEN_ASSERT(MaxBlockCount > 0);
ZEN_ASSERT(IsPow2(MaxBlockCount));
+ std::unordered_map<uint32_t, uint64_t> FoundBlocks;
+
m_TotalSize = 0;
m_BlocksBasePath = BlocksBasePath;
m_MaxBlockSize = MaxBlockSize;
- m_ChunkBlocks.clear();
-
- std::unordered_set<uint32_t> KnownBlocks;
- for (const auto& Entry : KnownLocations)
- {
- KnownBlocks.insert(Entry.BlockIndex);
- }
-
if (std::filesystem::is_directory(m_BlocksBasePath))
{
std::vector<std::filesystem::path> FoldersToScan;
@@ -168,23 +159,11 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath,
{
continue;
}
- if (!KnownBlocks.contains(BlockIndex))
- {
- // Log removing unreferenced block
- // Clear out unused blocks
- ZEN_DEBUG("removing unused block at '{}'", Path);
- std::error_code Ec;
- std::filesystem::remove(Path, Ec);
- if (Ec)
- {
- ZEN_WARN("Failed to delete file '{}' reason: '{}'", Path, Ec.message());
- }
- continue;
- }
Ref<BlockStoreFile> BlockFile{new BlockStoreFile(Path)};
BlockFile->Open();
m_TotalSize.fetch_add(BlockFile->FileSize(), std::memory_order::relaxed);
m_ChunkBlocks[BlockIndex] = BlockFile;
+ FoundBlocks[BlockIndex] = BlockFile->FileSize();
}
}
++FolderOffset;
@@ -194,6 +173,39 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath,
{
CreateDirectories(m_BlocksBasePath);
}
+ return FoundBlocks;
+}
+
+void
+BlockStore::Prune(const std::vector<BlockStoreLocation>& KnownLocations)
+{
+ RwLock::ExclusiveLockScope InsertLock(m_InsertLock);
+
+ std::unordered_set<uint32_t> KnownBlocks;
+ for (const auto& Entry : KnownLocations)
+ {
+ KnownBlocks.insert(Entry.BlockIndex);
+ }
+ std::vector<uint32_t> BlocksToDelete;
+ for (auto It = m_ChunkBlocks.begin(); It != m_ChunkBlocks.end(); ++It)
+ {
+ uint32_t BlockIndex = It->first;
+ if (!KnownBlocks.contains(BlockIndex))
+ {
+ Ref<BlockStoreFile> BlockFile = m_ChunkBlocks[BlockIndex];
+ m_TotalSize.fetch_add(BlockFile->FileSize(), std::memory_order::relaxed);
+ BlocksToDelete.push_back(BlockIndex);
+ }
+ }
+
+ for (uint32_t BlockIndex : BlocksToDelete)
+ {
+ // Clear out unused blocks
+ Ref<BlockStoreFile> BlockFile = m_ChunkBlocks[BlockIndex];
+ m_ChunkBlocks.erase(BlockIndex);
+ ZEN_DEBUG("marking block store file '{}' for delete, block #{}", BlockFile->GetPath(), BlockIndex);
+ BlockFile->MarkAsDeleteOnClose();
+ }
}
void
@@ -460,7 +472,10 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot,
WriteBlockTimeUs += ElapsedUs;
WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
});
- OldBlockFile = m_ChunkBlocks[BlockIndex];
+ if (auto It = m_ChunkBlocks.find(BlockIndex); It != m_ChunkBlocks.end())
+ {
+ OldBlockFile = It->second;
+ }
}
if (!OldBlockFile)
@@ -492,8 +507,9 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot,
});
if (OldBlockFile)
{
- m_ChunkBlocks[BlockIndex] = nullptr;
ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
+ ZEN_ASSERT(m_ChunkBlocks[BlockIndex] == OldBlockFile);
+ m_ChunkBlocks.erase(BlockIndex);
m_TotalSize.fetch_sub(OldBlockFile->FileSize(), std::memory_order::relaxed);
OldBlockFile->MarkAsDeleteOnClose();
}
@@ -570,7 +586,9 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot,
ReadBlockTimeUs += ElapsedUs;
ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
});
+ ZEN_ASSERT(m_ChunkBlocks[NextBlockIndex] == NewBlockFile);
m_ChunkBlocks.erase(NextBlockIndex);
+ NewBlockFile->MarkAsDeleteOnClose();
return;
}
@@ -615,8 +633,9 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot,
ReadBlockTimeUs += ElapsedUs;
ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs);
});
- m_ChunkBlocks[BlockIndex] = nullptr;
ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
+ ZEN_ASSERT(m_ChunkBlocks[BlockIndex] == OldBlockFile);
+ m_ChunkBlocks.erase(BlockIndex);
m_TotalSize.fetch_sub(OldBlockFile->FileSize(), std::memory_order::relaxed);
OldBlockFile->MarkAsDeleteOnClose();
}
@@ -692,8 +711,8 @@ BlockStore::IterateChunks(const std::vector<BlockStoreLocation>& ChunkLocations,
size_t ChunkIndex = LocationIndexes[LocationIndexOffset];
const BlockStoreLocation& FirstLocation = ChunkLocations[ChunkIndex];
- const Ref<BlockStoreFile>& BlockFile = m_ChunkBlocks[FirstLocation.BlockIndex];
- if (!BlockFile)
+ auto FindBlockIt = m_ChunkBlocks.find(FirstLocation.BlockIndex);
+ if (FindBlockIt == m_ChunkBlocks.end())
{
while (ChunkLocations[ChunkIndex].BlockIndex == FirstLocation.BlockIndex)
{
@@ -707,6 +726,9 @@ BlockStore::IterateChunks(const std::vector<BlockStoreLocation>& ChunkLocations,
}
continue;
}
+ const Ref<BlockStoreFile>& BlockFile = FindBlockIt->second;
+ ZEN_ASSERT(BlockFile);
+
size_t BlockSize = BlockFile->FileSize();
size_t RangeCount = GetNextRange(LocationIndexOffset);
if (RangeCount > 0)
@@ -939,7 +961,7 @@ TEST_CASE("blockstore.chunks")
auto RootDirectory = TempDir.Path();
BlockStore Store;
- Store.Initialize(RootDirectory, 128, 1024, {});
+ Store.Initialize(RootDirectory, 128, 1024);
IoBuffer BadChunk = Store.TryGetChunk({.BlockIndex = 0, .Offset = 0, .Size = 512});
CHECK(!BadChunk);
@@ -969,7 +991,7 @@ TEST_CASE("blockstore.clean.stray.blocks")
auto RootDirectory = TempDir.Path();
BlockStore Store;
- Store.Initialize(RootDirectory / "store", 128, 1024, {});
+ Store.Initialize(RootDirectory / "store", 128, 1024);
std::string FirstChunkData = "This is the data of the first chunk that we will write";
BlockStoreLocation FirstChunkLocation = WriteStringAsChunk(Store, FirstChunkData, 4);
@@ -982,7 +1004,8 @@ TEST_CASE("blockstore.clean.stray.blocks")
Store.Close();
// Not referencing the second block means that we should be deleted
- Store.Initialize(RootDirectory / "store", 128, 1024, {FirstChunkLocation, SecondChunkLocation});
+ Store.Initialize(RootDirectory / "store", 128, 1024);
+ Store.Prune({FirstChunkLocation, SecondChunkLocation});
CHECK(GetDirectoryContent(RootDirectory / "store", true, false).size() == 1);
}
@@ -995,7 +1018,7 @@ TEST_CASE("blockstore.flush.forces.new.block")
auto RootDirectory = TempDir.Path();
BlockStore Store;
- Store.Initialize(RootDirectory / "store", 128, 1024, {});
+ Store.Initialize(RootDirectory / "store", 128, 1024);
std::string FirstChunkData = "This is the data of the first chunk that we will write";
WriteStringAsChunk(Store, FirstChunkData, 4);
@@ -1018,7 +1041,7 @@ TEST_CASE("blockstore.iterate.chunks")
auto RootDirectory = TempDir.Path();
BlockStore Store;
- Store.Initialize(RootDirectory / "store", ScrubSmallChunkWindowSize * 2, 1024, {});
+ Store.Initialize(RootDirectory / "store", ScrubSmallChunkWindowSize * 2, 1024);
IoBuffer BadChunk = Store.TryGetChunk({.BlockIndex = 0, .Offset = 0, .Size = 512});
CHECK(!BadChunk);
@@ -1114,7 +1137,7 @@ TEST_CASE("blockstore.reclaim.space")
auto RootDirectory = TempDir.Path();
BlockStore Store;
- Store.Initialize(RootDirectory / "store", 512, 1024, {});
+ Store.Initialize(RootDirectory / "store", 512, 1024);
constexpr size_t ChunkCount = 200;
constexpr size_t Alignment = 8;
@@ -1231,7 +1254,7 @@ TEST_CASE("blockstore.thread.read.write")
auto RootDirectory = TempDir.Path();
BlockStore Store;
- Store.Initialize(RootDirectory / "store", 1088, 1024, {});
+ Store.Initialize(RootDirectory / "store", 1088, 1024);
constexpr size_t ChunkCount = 1000;
constexpr size_t Alignment = 8;
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-17 03:00:55 +0000