diff options
| author | Dan Engelbrecht <[email protected]> | 2022-03-15 13:07:37 +0100 |
|---|---|---|
| committer | Dan Engelbrecht <[email protected]> | 2022-03-31 11:28:32 +0200 |
| commit | 0902650e15c474b6e507b7da8e1e722b189b903e (patch) | |
| tree | 93f675a11af46457e975c05e9449cd99e6216709 /zenstore/chunkbundler.cpp | |
| parent | Rework GC for blocks (diff) | |
| download | zen-0902650e15c474b6e507b7da8e1e722b189b903e.tar.xz zen-0902650e15c474b6e507b7da8e1e722b189b903e.zip | |
WIP
Diffstat (limited to 'zenstore/chunkbundler.cpp')
| -rw-r--r-- | zenstore/chunkbundler.cpp | 50 |
1 files changed, 26 insertions, 24 deletions
diff --git a/zenstore/chunkbundler.cpp b/zenstore/chunkbundler.cpp index 5b9242eec..0e81dc8dc 100644 --- a/zenstore/chunkbundler.cpp +++ b/zenstore/chunkbundler.cpp @@ -106,15 +106,15 @@ ChunkBundler::Initialize(const std::string_view ContainerBaseName, uint64_t MaxB { try { - std::string FileName = Entry.path().stem().string(); - uint16_t BlockIndex = static_cast<uint16_t>(std::stoi(FileName)); + std::string FileName = Entry.path().stem().string(); + uint16_t BlockIndex = static_cast<uint16_t>(std::stoi(FileName)); if (!ReferencedBlockIndexes.contains(BlockIndex)) { // Clear out unused blocks std::filesystem::remove(Entry.path()); continue; } - auto SmallObjectFile = std::make_shared<BasicFile>(); + auto SmallObjectFile = std::make_shared<BasicFile>(); SmallObjectFile->Open(Entry.path(), false); m_OpenBlocks[BlockIndex] = SmallObjectFile; if (SmallObjectFile->FileSize() < SmallestBlockSize) @@ -138,7 +138,6 @@ ChunkBundler::Initialize(const std::string_view ContainerBaseName, uint64_t MaxB m_OpenBlocks[m_CurrentBlockIndex] = SmallObjectFile; m_CurrentBlock = SmallObjectFile; m_CurrentInsertOffset = 0; - } else { @@ -504,9 +503,9 @@ ChunkBundler::CollectGarbage(GcContext& GcCtx) // Move all chunks in blocks that have chunks removed to new blocks - std::shared_ptr<BasicFile> NewBlockFile; - uint64_t WriteOffset = {}; - uint16_t NewBlockIndex = {}; + std::shared_ptr<BasicFile> NewBlockFile; + uint64_t WriteOffset = {}; + uint16_t NewBlockIndex = {}; std::unordered_map<IoHash, CompactDiskLocation> MovedBlocks; for (auto BlockIndex : BlocksToReWrite) @@ -517,19 +516,19 @@ ChunkBundler::CollectGarbage(GcContext& GcCtx) // The block has no references to it, it should be removed as soon as no references is held on the file // TODO: We currently don't know if someone is holding a IoBuffer for this block at this point! - //std::filesystem::path BlockPath = m_RootDirectory / "ucas" / (std::to_string(BlockIndex) + ".ucas"); - //RwLock::ExclusiveLockScope _i(m_InsertLock); - //auto BlockFile = m_OpenBlocks[BlockIndex]; - //m_OpenBlocks.erase(BlockIndex); - //BlockFile->Close(); - //fs::remove(BlockPath); + // std::filesystem::path BlockPath = m_RootDirectory / "ucas" / (std::to_string(BlockIndex) + ".ucas"); + // RwLock::ExclusiveLockScope _i(m_InsertLock); + // auto BlockFile = m_OpenBlocks[BlockIndex]; + // m_OpenBlocks.erase(BlockIndex); + // BlockFile->Close(); + // fs::remove(BlockPath); continue; } std::shared_ptr<BasicFile> BlockFile; { RwLock::ExclusiveLockScope _i(m_InsertLock); - BlockFile = m_OpenBlocks[BlockIndex]; + BlockFile = m_OpenBlocks[BlockIndex]; } { @@ -560,11 +559,13 @@ ChunkBundler::CollectGarbage(GcContext& GcCtx) NewBlockIndex++; if (NewBlockIndex == m_CurrentBlockIndex) { - ZEN_ERROR("unable to allocate a new block in {}, count limit {} exeeded", m_ContainerBaseName, std::numeric_limits<uint16_t>::max()+1); + ZEN_ERROR("unable to allocate a new block in {}, count limit {} exeeded", + m_ContainerBaseName, + std::numeric_limits<uint16_t>::max() + 1); return; } } - m_OpenBlocks[NewBlockIndex] = std::shared_ptr<BasicFile>(); // Make sure nobody steals this slot + m_OpenBlocks[NewBlockIndex] = std::shared_ptr<BasicFile>(); // Make sure nobody steals this slot } std::error_code Error; @@ -595,13 +596,13 @@ ChunkBundler::CollectGarbage(GcContext& GcCtx) NewBlockFile->Write(Chunk.data(), Chunk.size(), WriteOffset); CompactDiskLocation NewChunkLocation(NewBlockIndex, - gsl::narrow<uint32_t>(WriteOffset), - gsl::narrow<uint32_t>(Chunk.size())); - Entry.second = {.BlockIndex = NewBlockIndex, + gsl::narrow<uint32_t>(WriteOffset), + gsl::narrow<uint32_t>(Chunk.size())); + Entry.second = {.BlockIndex = NewBlockIndex, .Offset = gsl::narrow<uint32_t>(WriteOffset), .Size = gsl::narrow<uint32_t>(Chunk.size())}; MovedBlocks[Entry.first] = Entry.second; - WriteOffset = AlignPositon(WriteOffset + Chunk.size(), m_PayloadAlignment); + WriteOffset = AlignPositon(WriteOffset + Chunk.size(), m_PayloadAlignment); } Chunk.clear(); @@ -908,7 +909,7 @@ TEST_CASE("chunkbundler.gc.compact") ChunkBundler ChunkBundlerStore(TempDir.Path(), nullptr); ChunkBundlerStore.Initialize("cb", 2048, 1 << 4, true); - uint64_t ChunkSizes[9] = {128, 541, 1023, 781, 218, 37, 4, 997, 5}; + uint64_t ChunkSizes[9] = {128, 541, 1023, 781, 218, 37, 4, 997, 5}; std::vector<IoBuffer> Chunks; Chunks.reserve(9); for (const auto& Size : ChunkSizes) @@ -1133,7 +1134,7 @@ TEST_CASE("chunkbundler.gc.deleteblockonopen") { ScopedTemporaryDirectory TempDir; - uint64_t ChunkSizes[20] = {128, 541, 311, 181, 218, 37, 4, 397, 5, 92, 551, 721, 31, 92, 16, 99, 131, 41, 541, 84}; + uint64_t ChunkSizes[20] = {128, 541, 311, 181, 218, 37, 4, 397, 5, 92, 551, 721, 31, 92, 16, 99, 131, 41, 541, 84}; std::vector<IoBuffer> Chunks; Chunks.reserve(20); for (const auto& Size : ChunkSizes) @@ -1173,7 +1174,7 @@ TEST_CASE("chunkbundler.gc.deleteblockonopen") for (size_t i = 0; i < 20; i += 2) { CHECK(ChunkBundlerStore.HaveChunk(ChunkHashes[i])); - CHECK(!ChunkBundlerStore.HaveChunk(ChunkHashes[i+1])); + CHECK(!ChunkBundlerStore.HaveChunk(ChunkHashes[i + 1])); CHECK(ChunkHashes[i] == IoHash::HashBuffer(ChunkBundlerStore.FindChunk(ChunkHashes[i]))); } } @@ -1233,7 +1234,8 @@ TEST_CASE("chunkbundler.gc.handleopeniobuffer") CHECK(ChunkHashes[5] == IoHash::HashBuffer(RetainChunk)); } -void chunkbundler_forcelink() +void +chunkbundler_forcelink() { } |