aboutsummaryrefslogtreecommitdiff
path: root/zenstore/chunkbundler.cpp
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2022-03-15 13:07:37 +0100
committerDan Engelbrecht <[email protected]>2022-03-31 11:28:32 +0200
commit0902650e15c474b6e507b7da8e1e722b189b903e (patch)
tree93f675a11af46457e975c05e9449cd99e6216709 /zenstore/chunkbundler.cpp
parentRework GC for blocks (diff)
downloadzen-0902650e15c474b6e507b7da8e1e722b189b903e.tar.xz
zen-0902650e15c474b6e507b7da8e1e722b189b903e.zip
WIP
Diffstat (limited to 'zenstore/chunkbundler.cpp')
-rw-r--r--zenstore/chunkbundler.cpp50
1 files changed, 26 insertions, 24 deletions
diff --git a/zenstore/chunkbundler.cpp b/zenstore/chunkbundler.cpp
index 5b9242eec..0e81dc8dc 100644
--- a/zenstore/chunkbundler.cpp
+++ b/zenstore/chunkbundler.cpp
@@ -106,15 +106,15 @@ ChunkBundler::Initialize(const std::string_view ContainerBaseName, uint64_t MaxB
{
try
{
- std::string FileName = Entry.path().stem().string();
- uint16_t BlockIndex = static_cast<uint16_t>(std::stoi(FileName));
+ std::string FileName = Entry.path().stem().string();
+ uint16_t BlockIndex = static_cast<uint16_t>(std::stoi(FileName));
if (!ReferencedBlockIndexes.contains(BlockIndex))
{
// Clear out unused blocks
std::filesystem::remove(Entry.path());
continue;
}
- auto SmallObjectFile = std::make_shared<BasicFile>();
+ auto SmallObjectFile = std::make_shared<BasicFile>();
SmallObjectFile->Open(Entry.path(), false);
m_OpenBlocks[BlockIndex] = SmallObjectFile;
if (SmallObjectFile->FileSize() < SmallestBlockSize)
@@ -138,7 +138,6 @@ ChunkBundler::Initialize(const std::string_view ContainerBaseName, uint64_t MaxB
m_OpenBlocks[m_CurrentBlockIndex] = SmallObjectFile;
m_CurrentBlock = SmallObjectFile;
m_CurrentInsertOffset = 0;
-
}
else
{
@@ -504,9 +503,9 @@ ChunkBundler::CollectGarbage(GcContext& GcCtx)
// Move all chunks in blocks that have chunks removed to new blocks
- std::shared_ptr<BasicFile> NewBlockFile;
- uint64_t WriteOffset = {};
- uint16_t NewBlockIndex = {};
+ std::shared_ptr<BasicFile> NewBlockFile;
+ uint64_t WriteOffset = {};
+ uint16_t NewBlockIndex = {};
std::unordered_map<IoHash, CompactDiskLocation> MovedBlocks;
for (auto BlockIndex : BlocksToReWrite)
@@ -517,19 +516,19 @@ ChunkBundler::CollectGarbage(GcContext& GcCtx)
// The block has no references to it, it should be removed as soon as no references is held on the file
// TODO: We currently don't know if someone is holding a IoBuffer for this block at this point!
- //std::filesystem::path BlockPath = m_RootDirectory / "ucas" / (std::to_string(BlockIndex) + ".ucas");
- //RwLock::ExclusiveLockScope _i(m_InsertLock);
- //auto BlockFile = m_OpenBlocks[BlockIndex];
- //m_OpenBlocks.erase(BlockIndex);
- //BlockFile->Close();
- //fs::remove(BlockPath);
+ // std::filesystem::path BlockPath = m_RootDirectory / "ucas" / (std::to_string(BlockIndex) + ".ucas");
+ // RwLock::ExclusiveLockScope _i(m_InsertLock);
+ // auto BlockFile = m_OpenBlocks[BlockIndex];
+ // m_OpenBlocks.erase(BlockIndex);
+ // BlockFile->Close();
+ // fs::remove(BlockPath);
continue;
}
std::shared_ptr<BasicFile> BlockFile;
{
RwLock::ExclusiveLockScope _i(m_InsertLock);
- BlockFile = m_OpenBlocks[BlockIndex];
+ BlockFile = m_OpenBlocks[BlockIndex];
}
{
@@ -560,11 +559,13 @@ ChunkBundler::CollectGarbage(GcContext& GcCtx)
NewBlockIndex++;
if (NewBlockIndex == m_CurrentBlockIndex)
{
- ZEN_ERROR("unable to allocate a new block in {}, count limit {} exeeded", m_ContainerBaseName, std::numeric_limits<uint16_t>::max()+1);
+ ZEN_ERROR("unable to allocate a new block in {}, count limit {} exeeded",
+ m_ContainerBaseName,
+ std::numeric_limits<uint16_t>::max() + 1);
return;
}
}
- m_OpenBlocks[NewBlockIndex] = std::shared_ptr<BasicFile>(); // Make sure nobody steals this slot
+ m_OpenBlocks[NewBlockIndex] = std::shared_ptr<BasicFile>(); // Make sure nobody steals this slot
}
std::error_code Error;
@@ -595,13 +596,13 @@ ChunkBundler::CollectGarbage(GcContext& GcCtx)
NewBlockFile->Write(Chunk.data(), Chunk.size(), WriteOffset);
CompactDiskLocation NewChunkLocation(NewBlockIndex,
- gsl::narrow<uint32_t>(WriteOffset),
- gsl::narrow<uint32_t>(Chunk.size()));
- Entry.second = {.BlockIndex = NewBlockIndex,
+ gsl::narrow<uint32_t>(WriteOffset),
+ gsl::narrow<uint32_t>(Chunk.size()));
+ Entry.second = {.BlockIndex = NewBlockIndex,
.Offset = gsl::narrow<uint32_t>(WriteOffset),
.Size = gsl::narrow<uint32_t>(Chunk.size())};
MovedBlocks[Entry.first] = Entry.second;
- WriteOffset = AlignPositon(WriteOffset + Chunk.size(), m_PayloadAlignment);
+ WriteOffset = AlignPositon(WriteOffset + Chunk.size(), m_PayloadAlignment);
}
Chunk.clear();
@@ -908,7 +909,7 @@ TEST_CASE("chunkbundler.gc.compact")
ChunkBundler ChunkBundlerStore(TempDir.Path(), nullptr);
ChunkBundlerStore.Initialize("cb", 2048, 1 << 4, true);
- uint64_t ChunkSizes[9] = {128, 541, 1023, 781, 218, 37, 4, 997, 5};
+ uint64_t ChunkSizes[9] = {128, 541, 1023, 781, 218, 37, 4, 997, 5};
std::vector<IoBuffer> Chunks;
Chunks.reserve(9);
for (const auto& Size : ChunkSizes)
@@ -1133,7 +1134,7 @@ TEST_CASE("chunkbundler.gc.deleteblockonopen")
{
ScopedTemporaryDirectory TempDir;
- uint64_t ChunkSizes[20] = {128, 541, 311, 181, 218, 37, 4, 397, 5, 92, 551, 721, 31, 92, 16, 99, 131, 41, 541, 84};
+ uint64_t ChunkSizes[20] = {128, 541, 311, 181, 218, 37, 4, 397, 5, 92, 551, 721, 31, 92, 16, 99, 131, 41, 541, 84};
std::vector<IoBuffer> Chunks;
Chunks.reserve(20);
for (const auto& Size : ChunkSizes)
@@ -1173,7 +1174,7 @@ TEST_CASE("chunkbundler.gc.deleteblockonopen")
for (size_t i = 0; i < 20; i += 2)
{
CHECK(ChunkBundlerStore.HaveChunk(ChunkHashes[i]));
- CHECK(!ChunkBundlerStore.HaveChunk(ChunkHashes[i+1]));
+ CHECK(!ChunkBundlerStore.HaveChunk(ChunkHashes[i + 1]));
CHECK(ChunkHashes[i] == IoHash::HashBuffer(ChunkBundlerStore.FindChunk(ChunkHashes[i])));
}
}
@@ -1233,7 +1234,8 @@ TEST_CASE("chunkbundler.gc.handleopeniobuffer")
CHECK(ChunkHashes[5] == IoHash::HashBuffer(RetainChunk));
}
-void chunkbundler_forcelink()
+void
+chunkbundler_forcelink()
{
}