diff options
| author | Dan Engelbrecht <[email protected]> | 2022-09-08 10:54:31 +0200 |
|---|---|---|
| committer | GitHub <[email protected]> | 2022-09-08 01:54:31 -0700 |
| commit | 033e742c5102df67c000e2b281152fd37f59afb3 (patch) | |
| tree | a395e3b22d94b890148745eeff796550f8e3a411 /zenstore/blockstore.cpp | |
| parent | 0.1.4-pre27 (diff) | |
| download | zen-033e742c5102df67c000e2b281152fd37f59afb3.tar.xz zen-033e742c5102df67c000e2b281152fd37f59afb3.zip | |
Remove legacy code (#161)
* changelog
* remove obsolete legacy code
Diffstat (limited to 'zenstore/blockstore.cpp')
| -rw-r--r-- | zenstore/blockstore.cpp | 211 |
1 files changed, 0 insertions, 211 deletions
diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp index 88592d785..38371124d 100644 --- a/zenstore/blockstore.cpp +++ b/zenstore/blockstore.cpp @@ -718,217 +718,6 @@ BlockStore::IterateChunks(const std::vector<BlockStoreLocation>& ChunkLocations, } } -bool -BlockStore::Split(const std::vector<BlockStoreLocation>& ChunkLocations, - const std::filesystem::path& SourceBlockFilePath, - const std::filesystem::path& BlocksBasePath, - uint64_t MaxBlockSize, - uint64_t MaxBlockCount, - size_t PayloadAlignment, - bool CleanSource, - const SplitCallback& Callback) -{ - std::error_code Error; - DiskSpace Space = DiskSpaceInfo(BlocksBasePath.parent_path(), Error); - if (Error) - { - ZEN_ERROR("get disk space in {} FAILED, reason: '{}'", BlocksBasePath, Error.message()); - return false; - } - - if (Space.Free < MaxBlockSize) - { - ZEN_ERROR("legacy store migration from '{}' FAILED, required disk space {}, free {}", - BlocksBasePath, - MaxBlockSize, - NiceBytes(Space.Free)); - return false; - } - - size_t TotalSize = 0; - for (const BlockStoreLocation& Location : ChunkLocations) - { - TotalSize += Location.Size; - } - size_t ChunkCount = ChunkLocations.size(); - uint64_t RequiredDiskSpace = TotalSize + ((PayloadAlignment - 1) * ChunkCount); - uint64_t MaxRequiredBlockCount = RoundUp(RequiredDiskSpace, MaxBlockSize) / MaxBlockSize; - if (MaxRequiredBlockCount > MaxBlockCount) - { - ZEN_ERROR("legacy store migration from '{}' FAILED, required block count {}, possible {}", - BlocksBasePath, - MaxRequiredBlockCount, - MaxBlockCount); - return false; - } - - constexpr const uint64_t DiskReserve = 1ul << 28; - - if (CleanSource) - { - if (Space.Free < (MaxBlockSize + DiskReserve)) - { - ZEN_INFO("legacy store migration from '{}' aborted, not enough disk space available {} ({})", - BlocksBasePath, - NiceBytes(MaxBlockSize + DiskReserve), - NiceBytes(Space.Free)); - return false; - } - } - else - { - if (Space.Free < (RequiredDiskSpace + DiskReserve)) - { - ZEN_INFO("legacy store migration from '{}' aborted, not enough disk space available {} ({})", - BlocksBasePath, - NiceBytes(RequiredDiskSpace + DiskReserve), - NiceBytes(Space.Free)); - return false; - } - } - - uint32_t WriteBlockIndex = 0; - while (std::filesystem::exists(BlockStore::GetBlockPath(BlocksBasePath, WriteBlockIndex))) - { - ++WriteBlockIndex; - } - - BasicFile BlockFile; - BlockFile.Open(SourceBlockFilePath, CleanSource ? BasicFile::Mode::kWrite : BasicFile::Mode::kRead); - - if (CleanSource && (MaxRequiredBlockCount < 2)) - { - MovedChunksArray Chunks; - Chunks.reserve(ChunkCount); - for (size_t Index = 0; Index < ChunkCount; ++Index) - { - const BlockStoreLocation& ChunkLocation = ChunkLocations[Index]; - Chunks.push_back({Index, {.BlockIndex = WriteBlockIndex, .Offset = ChunkLocation.Offset, .Size = ChunkLocation.Size}}); - } - std::filesystem::path BlockPath = BlockStore::GetBlockPath(BlocksBasePath, WriteBlockIndex); - CreateDirectories(BlockPath.parent_path()); - BlockFile.Close(); - std::filesystem::rename(SourceBlockFilePath, BlockPath); - Callback(Chunks); - return true; - } - - ChunkIndexArray ChunkIndexes; - ChunkIndexes.reserve(ChunkCount); - for (size_t Index = 0; Index < ChunkCount; ++Index) - { - ChunkIndexes.push_back(Index); - } - - std::sort(begin(ChunkIndexes), end(ChunkIndexes), [&](size_t Lhs, size_t Rhs) { - const BlockStoreLocation& LhsLocation = ChunkLocations[Lhs]; - const BlockStoreLocation& RhsLocation = ChunkLocations[Rhs]; - return LhsLocation.Offset < RhsLocation.Offset; - }); - - uint64_t BlockSize = 0; - uint64_t BlockOffset = 0; - std::vector<BlockStoreLocation> NewLocations; - struct BlockData - { - MovedChunksArray Chunks; - uint64_t BlockOffset; - uint64_t BlockSize; - uint32_t BlockIndex; - }; - - std::vector<BlockData> BlockRanges; - MovedChunksArray Chunks; - BlockRanges.reserve(MaxRequiredBlockCount); - for (const size_t& ChunkIndex : ChunkIndexes) - { - const BlockStoreLocation& LegacyChunkLocation = ChunkLocations[ChunkIndex]; - - uint64_t ChunkOffset = LegacyChunkLocation.Offset; - uint64_t ChunkSize = LegacyChunkLocation.Size; - uint64_t ChunkEnd = ChunkOffset + ChunkSize; - - if (BlockSize == 0) - { - BlockOffset = ChunkOffset; - } - if ((ChunkEnd - BlockOffset) > MaxBlockSize) - { - BlockData BlockRange{.BlockOffset = BlockOffset, .BlockSize = BlockSize, .BlockIndex = WriteBlockIndex}; - BlockRange.Chunks.swap(Chunks); - BlockRanges.push_back(BlockRange); - - WriteBlockIndex++; - while (std::filesystem::exists(BlockStore::GetBlockPath(BlocksBasePath, WriteBlockIndex))) - { - ++WriteBlockIndex; - } - BlockOffset = ChunkOffset; - BlockSize = 0; - } - BlockSize = RoundUp(BlockSize, PayloadAlignment); - BlockStoreLocation ChunkLocation = {.BlockIndex = WriteBlockIndex, .Offset = ChunkOffset - BlockOffset, .Size = ChunkSize}; - Chunks.push_back({ChunkIndex, ChunkLocation}); - BlockSize = ChunkEnd - BlockOffset; - } - if (BlockSize > 0) - { - BlockRanges.push_back( - {.Chunks = std::move(Chunks), .BlockOffset = BlockOffset, .BlockSize = BlockSize, .BlockIndex = WriteBlockIndex}); - } - - Stopwatch WriteBlockTimer; - - std::reverse(BlockRanges.begin(), BlockRanges.end()); - std::vector<std::uint8_t> Buffer(1 << 28); - for (size_t Idx = 0; Idx < BlockRanges.size(); ++Idx) - { - const BlockData& BlockRange = BlockRanges[Idx]; - if (Idx > 0) - { - uint64_t Remaining = BlockRange.BlockOffset + BlockRange.BlockSize; - uint64_t Completed = BlockOffset + BlockSize - Remaining; - uint64_t ETA = (WriteBlockTimer.GetElapsedTimeMs() * Remaining) / Completed; - - ZEN_INFO("migrating store '{}' {}/{} blocks, remaining {} ({}) ETA: {}", - BlocksBasePath, - Idx, - BlockRanges.size(), - NiceBytes(BlockRange.BlockOffset + BlockRange.BlockSize), - NiceBytes(BlockOffset + BlockSize), - NiceTimeSpanMs(ETA)); - } - - std::filesystem::path BlockPath = BlockStore::GetBlockPath(BlocksBasePath, BlockRange.BlockIndex); - BlockStoreFile ChunkBlock(BlockPath); - ChunkBlock.Create(BlockRange.BlockSize); - uint64_t Offset = 0; - while (Offset < BlockRange.BlockSize) - { - uint64_t Size = BlockRange.BlockSize - Offset; - if (Size > Buffer.size()) - { - Size = Buffer.size(); - } - BlockFile.Read(Buffer.data(), Size, BlockRange.BlockOffset + Offset); - ChunkBlock.Write(Buffer.data(), Size, Offset); - Offset += Size; - } - ChunkBlock.Truncate(Offset); - ChunkBlock.Flush(); - - Callback(BlockRange.Chunks); - - if (CleanSource) - { - BlockFile.SetFileSize(BlockRange.BlockOffset); - } - } - BlockFile.Close(); - - return true; -} - const char* BlockStore::GetBlockFileExtension() { |