From 16fd9ea89c7560216b654843400ab3d852b04e16 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 13 Dec 2023 09:25:05 -0500 Subject: improve trace (#606) * Adding some more trace scopes for better visiblity * Removed spammy trace scope when replaying oplogs * Remove "::Disk" from trace scopes - redundant now that we have merge disk and memory layers --- src/zenstore/blockstore.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/zenstore/blockstore.cpp') diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index 71e306eca..a11b4a323 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -51,6 +51,7 @@ BlockStoreFile::GetPath() const void BlockStoreFile::Open() { + ZEN_TRACE_CPU("BlockStoreFile::Open"); uint32_t RetriesLeft = 3; m_File.Open(m_Path, BasicFile::Mode::kDelete, [&](std::error_code& Ec) { if (RetriesLeft == 0) @@ -285,6 +286,7 @@ BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownLocations) BlockStore::BlockEntryCountMap BlockStore::GetBlocksToCompact(const BlockUsageMap& BlockUsage, uint32_t BlockUsageThresholdPercent) { + ZEN_TRACE_CPU("BlockStoreFile::GetBlocksToCompact"); BlockEntryCountMap Result; { RwLock::SharedLockScope InsertLock(m_InsertLock); @@ -345,6 +347,7 @@ BlockStore::GetBlocksToCompact(const BlockUsageMap& BlockUsage, uint32_t BlockUs void BlockStore::Close() { + ZEN_TRACE_CPU("BlockStore::Close"); RwLock::ExclusiveLockScope InsertLock(m_InsertLock); m_WriteBlock = nullptr; m_CurrentInsertOffset = 0; @@ -1044,6 +1047,8 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, const CompactCallback& ChangeCallback, const ClaimDiskReserveCallback& DiskReserveCallback) { + ZEN_TRACE_CPU("BlockStore::CompactBlocks"); + uint64_t DeletedSize = 0; uint64_t MovedCount = 0; uint64_t MovedSize = 0; @@ -1100,6 +1105,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, CompactState.IterateBlocks([&](uint32_t BlockIndex, const std::vector& KeepChunkIndexes, const std::vector& ChunkLocations) -> bool { + ZEN_TRACE_CPU("BlockStore::CompactBlock"); Ref OldBlockFile; { RwLock::SharedLockScope _(m_InsertLock); -- cgit v1.2.3 From 54f9ad56c45e51ba0ce8a695ae30756d1b34d3ab Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 13 Dec 2023 10:17:21 -0500 Subject: skip invalid chunks when reclaiming space in block store (#607) * skip invalid chunks when reclaiming space in block store --- src/zenstore/blockstore.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'src/zenstore/blockstore.cpp') diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index a11b4a323..01d7e043c 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -757,10 +757,23 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot, if (OldBlockFile) { ZEN_TRACE_CPU("BlockStore::ReclaimSpace::MoveBlock"); + uint64_t OldBlockSize = OldBlockFile->FileSize(); std::vector Chunk; for (const size_t& ChunkIndex : KeepMap) { const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex]; + if (ChunkLocation.Offset + ChunkLocation.Size > OldBlockSize) + { + ZEN_WARN( + "ReclaimSpace skipping chunk outside of block range in '{}', Chunk start {}, Chunk size {} in Block {}, Block " + "size {}", + m_BlocksBasePath, + ChunkLocation.Offset, + ChunkLocation.Size, + OldBlockFile->GetPath(), + OldBlockSize); + continue; + } Chunk.resize(ChunkLocation.Size); OldBlockFile->Read(Chunk.data(), ChunkLocation.Size, ChunkLocation.Offset); -- cgit v1.2.3 From 9b2925a1a3d3e67042883cfe8ccca98d3791c450 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 15 Dec 2023 08:31:10 -0500 Subject: log each block that is compacted (#613) --- src/zenstore/blockstore.cpp | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/zenstore/blockstore.cpp') diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index 01d7e043c..7a09d349a 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -757,6 +757,9 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot, if (OldBlockFile) { ZEN_TRACE_CPU("BlockStore::ReclaimSpace::MoveBlock"); + + ZEN_INFO("Moving {} chunks from '{}' to new block", KeepMap.size(), GetBlockPath(m_BlocksBasePath, BlockIndex)); + uint64_t OldBlockSize = OldBlockFile->FileSize(); std::vector Chunk; for (const size_t& ChunkIndex : KeepMap) @@ -1144,6 +1147,8 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, } ZEN_ASSERT(OldBlockFile); + ZEN_INFO("Moving {} chunks from '{}' to new block", KeepChunkIndexes.size(), GetBlockPath(m_BlocksBasePath, BlockIndex)); + uint64_t OldBlockSize = OldBlockFile->FileSize(); std::vector Chunk; -- cgit v1.2.3 From cc8b464751e243d80d54588bd2cec79d8009dc54 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 18 Dec 2023 10:07:32 -0500 Subject: Fix crash bug when trying to inspect non-open block file in GC (#614) --- src/zenstore/blockstore.cpp | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) (limited to 'src/zenstore/blockstore.cpp') diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index 7a09d349a..73a8ad538 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -150,6 +150,11 @@ BlockStoreFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::functio { m_File.StreamByteRange(FileOffset, Size, std::move(ChunkFun)); } +bool +BlockStoreFile::IsOpen() const +{ + return !!m_IoBuffer; +} constexpr uint64_t ScrubSmallChunkWindowSize = 4 * 1024 * 1024; @@ -669,7 +674,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot, ZEN_TRACE_CPU("BlockStore::ReclaimSpace::Compact"); Ref NewBlockFile; auto NewBlockFileGuard = MakeGuard([&]() { - if (NewBlockFile) + if (NewBlockFile && NewBlockFile->IsOpen()) { ZEN_DEBUG("dropping incomplete cas block store file '{}'", NewBlockFile->GetPath()); m_TotalSize.fetch_sub(NewBlockFile->FileSize(), std::memory_order::relaxed); @@ -786,6 +791,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot, if (NewBlockFile) { + ZEN_ASSERT_SLOW(NewBlockFile->IsOpen()); NewBlockFile->Flush(); NewBlockFile = nullptr; } @@ -839,8 +845,8 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot, ReadBlockLongestTimeUs = std::max(ElapsedUs, ReadBlockLongestTimeUs); }); ZEN_ASSERT(m_ChunkBlocks[NextBlockIndex] == NewBlockFile); + ZEN_ASSERT_SLOW(!NewBlockFile->IsOpen()); m_ChunkBlocks.erase(NextBlockIndex); - NewBlockFile->MarkAsDeleteOnClose(); return; } @@ -865,6 +871,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot, Chunk.clear(); if (NewBlockFile) { + ZEN_ASSERT_SLOW(NewBlockFile->IsOpen()); NewBlockFile->Flush(); } } @@ -899,6 +906,7 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState& Snapshot, } if (NewBlockFile) { + ZEN_ASSERT_SLOW(NewBlockFile->IsOpen()); NewBlockFile->Flush(); NewBlockFile = nullptr; } @@ -1090,7 +1098,6 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, auto NewBlockFileGuard = MakeGuard([&]() { if (NewBlockFile) { - ZEN_DEBUG("Dropping incomplete cas block store file '{}'", NewBlockFile->GetPath()); { RwLock::ExclusiveLockScope _l(m_InsertLock); if (m_ChunkBlocks[NewBlockIndex] == NewBlockFile) @@ -1098,7 +1105,11 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, m_ChunkBlocks.erase(NewBlockIndex); } } - NewBlockFile->MarkAsDeleteOnClose(); + if (NewBlockFile->IsOpen()) + { + ZEN_DEBUG("Dropping incomplete cas block store file '{}'", NewBlockFile->GetPath()); + NewBlockFile->MarkAsDeleteOnClose(); + } } }); @@ -1175,6 +1186,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, { if (NewBlockFile) { + ZEN_ASSERT_SLOW(NewBlockFile->IsOpen()); NewBlockFile->Flush(); MovedSize += NewBlockFile->FileSize(); NewBlockFile = nullptr; @@ -1203,7 +1215,6 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, NewBlockFile = new BlockStoreFile(NewBlockPath); m_ChunkBlocks[NextBlockIndex] = NewBlockFile; } - ZEN_ASSERT(NewBlockFile); std::error_code Error; DiskSpace Space = DiskSpaceInfo(m_BlocksBasePath, Error); @@ -1215,7 +1226,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, ZEN_ASSERT(m_ChunkBlocks[NextBlockIndex] == NewBlockFile); m_ChunkBlocks.erase(NextBlockIndex); } - NewBlockFile->MarkAsDeleteOnClose(); + ZEN_ASSERT_SLOW(!NewBlockFile->IsOpen()); NewBlockFile = nullptr; return false; } @@ -1234,7 +1245,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, ZEN_ASSERT(m_ChunkBlocks[NextBlockIndex] == NewBlockFile); m_ChunkBlocks.erase(NextBlockIndex); } - NewBlockFile->MarkAsDeleteOnClose(); + ZEN_ASSERT_SLOW(!NewBlockFile->IsOpen()); NewBlockFile = nullptr; return false; } @@ -1275,6 +1286,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, if (NewBlockFile) { + ZEN_ASSERT_SLOW(NewBlockFile->IsOpen()); NewBlockFile->Flush(); MovedSize += NewBlockFile->FileSize(); NewBlockFile = nullptr; -- cgit v1.2.3