aboutsummaryrefslogtreecommitdiff
path: root/zenstore/blockstore.cpp
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2022-09-08 10:54:31 +0200
committerGitHub <[email protected]>2022-09-08 01:54:31 -0700
commit033e742c5102df67c000e2b281152fd37f59afb3 (patch)
treea395e3b22d94b890148745eeff796550f8e3a411 /zenstore/blockstore.cpp
parent0.1.4-pre27 (diff)
downloadzen-033e742c5102df67c000e2b281152fd37f59afb3.tar.xz
zen-033e742c5102df67c000e2b281152fd37f59afb3.zip
Remove legacy code (#161)
* changelog * remove obsolete legacy code
Diffstat (limited to 'zenstore/blockstore.cpp')
-rw-r--r--zenstore/blockstore.cpp211
1 files changed, 0 insertions, 211 deletions
diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp
index 88592d785..38371124d 100644
--- a/zenstore/blockstore.cpp
+++ b/zenstore/blockstore.cpp
@@ -718,217 +718,6 @@ BlockStore::IterateChunks(const std::vector<BlockStoreLocation>& ChunkLocations,
}
}
-bool
-BlockStore::Split(const std::vector<BlockStoreLocation>& ChunkLocations,
- const std::filesystem::path& SourceBlockFilePath,
- const std::filesystem::path& BlocksBasePath,
- uint64_t MaxBlockSize,
- uint64_t MaxBlockCount,
- size_t PayloadAlignment,
- bool CleanSource,
- const SplitCallback& Callback)
-{
- std::error_code Error;
- DiskSpace Space = DiskSpaceInfo(BlocksBasePath.parent_path(), Error);
- if (Error)
- {
- ZEN_ERROR("get disk space in {} FAILED, reason: '{}'", BlocksBasePath, Error.message());
- return false;
- }
-
- if (Space.Free < MaxBlockSize)
- {
- ZEN_ERROR("legacy store migration from '{}' FAILED, required disk space {}, free {}",
- BlocksBasePath,
- MaxBlockSize,
- NiceBytes(Space.Free));
- return false;
- }
-
- size_t TotalSize = 0;
- for (const BlockStoreLocation& Location : ChunkLocations)
- {
- TotalSize += Location.Size;
- }
- size_t ChunkCount = ChunkLocations.size();
- uint64_t RequiredDiskSpace = TotalSize + ((PayloadAlignment - 1) * ChunkCount);
- uint64_t MaxRequiredBlockCount = RoundUp(RequiredDiskSpace, MaxBlockSize) / MaxBlockSize;
- if (MaxRequiredBlockCount > MaxBlockCount)
- {
- ZEN_ERROR("legacy store migration from '{}' FAILED, required block count {}, possible {}",
- BlocksBasePath,
- MaxRequiredBlockCount,
- MaxBlockCount);
- return false;
- }
-
- constexpr const uint64_t DiskReserve = 1ul << 28;
-
- if (CleanSource)
- {
- if (Space.Free < (MaxBlockSize + DiskReserve))
- {
- ZEN_INFO("legacy store migration from '{}' aborted, not enough disk space available {} ({})",
- BlocksBasePath,
- NiceBytes(MaxBlockSize + DiskReserve),
- NiceBytes(Space.Free));
- return false;
- }
- }
- else
- {
- if (Space.Free < (RequiredDiskSpace + DiskReserve))
- {
- ZEN_INFO("legacy store migration from '{}' aborted, not enough disk space available {} ({})",
- BlocksBasePath,
- NiceBytes(RequiredDiskSpace + DiskReserve),
- NiceBytes(Space.Free));
- return false;
- }
- }
-
- uint32_t WriteBlockIndex = 0;
- while (std::filesystem::exists(BlockStore::GetBlockPath(BlocksBasePath, WriteBlockIndex)))
- {
- ++WriteBlockIndex;
- }
-
- BasicFile BlockFile;
- BlockFile.Open(SourceBlockFilePath, CleanSource ? BasicFile::Mode::kWrite : BasicFile::Mode::kRead);
-
- if (CleanSource && (MaxRequiredBlockCount < 2))
- {
- MovedChunksArray Chunks;
- Chunks.reserve(ChunkCount);
- for (size_t Index = 0; Index < ChunkCount; ++Index)
- {
- const BlockStoreLocation& ChunkLocation = ChunkLocations[Index];
- Chunks.push_back({Index, {.BlockIndex = WriteBlockIndex, .Offset = ChunkLocation.Offset, .Size = ChunkLocation.Size}});
- }
- std::filesystem::path BlockPath = BlockStore::GetBlockPath(BlocksBasePath, WriteBlockIndex);
- CreateDirectories(BlockPath.parent_path());
- BlockFile.Close();
- std::filesystem::rename(SourceBlockFilePath, BlockPath);
- Callback(Chunks);
- return true;
- }
-
- ChunkIndexArray ChunkIndexes;
- ChunkIndexes.reserve(ChunkCount);
- for (size_t Index = 0; Index < ChunkCount; ++Index)
- {
- ChunkIndexes.push_back(Index);
- }
-
- std::sort(begin(ChunkIndexes), end(ChunkIndexes), [&](size_t Lhs, size_t Rhs) {
- const BlockStoreLocation& LhsLocation = ChunkLocations[Lhs];
- const BlockStoreLocation& RhsLocation = ChunkLocations[Rhs];
- return LhsLocation.Offset < RhsLocation.Offset;
- });
-
- uint64_t BlockSize = 0;
- uint64_t BlockOffset = 0;
- std::vector<BlockStoreLocation> NewLocations;
- struct BlockData
- {
- MovedChunksArray Chunks;
- uint64_t BlockOffset;
- uint64_t BlockSize;
- uint32_t BlockIndex;
- };
-
- std::vector<BlockData> BlockRanges;
- MovedChunksArray Chunks;
- BlockRanges.reserve(MaxRequiredBlockCount);
- for (const size_t& ChunkIndex : ChunkIndexes)
- {
- const BlockStoreLocation& LegacyChunkLocation = ChunkLocations[ChunkIndex];
-
- uint64_t ChunkOffset = LegacyChunkLocation.Offset;
- uint64_t ChunkSize = LegacyChunkLocation.Size;
- uint64_t ChunkEnd = ChunkOffset + ChunkSize;
-
- if (BlockSize == 0)
- {
- BlockOffset = ChunkOffset;
- }
- if ((ChunkEnd - BlockOffset) > MaxBlockSize)
- {
- BlockData BlockRange{.BlockOffset = BlockOffset, .BlockSize = BlockSize, .BlockIndex = WriteBlockIndex};
- BlockRange.Chunks.swap(Chunks);
- BlockRanges.push_back(BlockRange);
-
- WriteBlockIndex++;
- while (std::filesystem::exists(BlockStore::GetBlockPath(BlocksBasePath, WriteBlockIndex)))
- {
- ++WriteBlockIndex;
- }
- BlockOffset = ChunkOffset;
- BlockSize = 0;
- }
- BlockSize = RoundUp(BlockSize, PayloadAlignment);
- BlockStoreLocation ChunkLocation = {.BlockIndex = WriteBlockIndex, .Offset = ChunkOffset - BlockOffset, .Size = ChunkSize};
- Chunks.push_back({ChunkIndex, ChunkLocation});
- BlockSize = ChunkEnd - BlockOffset;
- }
- if (BlockSize > 0)
- {
- BlockRanges.push_back(
- {.Chunks = std::move(Chunks), .BlockOffset = BlockOffset, .BlockSize = BlockSize, .BlockIndex = WriteBlockIndex});
- }
-
- Stopwatch WriteBlockTimer;
-
- std::reverse(BlockRanges.begin(), BlockRanges.end());
- std::vector<std::uint8_t> Buffer(1 << 28);
- for (size_t Idx = 0; Idx < BlockRanges.size(); ++Idx)
- {
- const BlockData& BlockRange = BlockRanges[Idx];
- if (Idx > 0)
- {
- uint64_t Remaining = BlockRange.BlockOffset + BlockRange.BlockSize;
- uint64_t Completed = BlockOffset + BlockSize - Remaining;
- uint64_t ETA = (WriteBlockTimer.GetElapsedTimeMs() * Remaining) / Completed;
-
- ZEN_INFO("migrating store '{}' {}/{} blocks, remaining {} ({}) ETA: {}",
- BlocksBasePath,
- Idx,
- BlockRanges.size(),
- NiceBytes(BlockRange.BlockOffset + BlockRange.BlockSize),
- NiceBytes(BlockOffset + BlockSize),
- NiceTimeSpanMs(ETA));
- }
-
- std::filesystem::path BlockPath = BlockStore::GetBlockPath(BlocksBasePath, BlockRange.BlockIndex);
- BlockStoreFile ChunkBlock(BlockPath);
- ChunkBlock.Create(BlockRange.BlockSize);
- uint64_t Offset = 0;
- while (Offset < BlockRange.BlockSize)
- {
- uint64_t Size = BlockRange.BlockSize - Offset;
- if (Size > Buffer.size())
- {
- Size = Buffer.size();
- }
- BlockFile.Read(Buffer.data(), Size, BlockRange.BlockOffset + Offset);
- ChunkBlock.Write(Buffer.data(), Size, Offset);
- Offset += Size;
- }
- ChunkBlock.Truncate(Offset);
- ChunkBlock.Flush();
-
- Callback(BlockRange.Chunks);
-
- if (CleanSource)
- {
- BlockFile.SetFileSize(BlockRange.BlockOffset);
- }
- }
- BlockFile.Close();
-
- return true;
-}
-
const char*
BlockStore::GetBlockFileExtension()
{