aboutsummaryrefslogtreecommitdiff
path: root/src/zenremotestore/chunking/chunkblock.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenremotestore/chunking/chunkblock.cpp')
-rw-r--r--src/zenremotestore/chunking/chunkblock.cpp274
1 files changed, 123 insertions, 151 deletions
diff --git a/src/zenremotestore/chunking/chunkblock.cpp b/src/zenremotestore/chunking/chunkblock.cpp
index 0fe3c09ce..f29112f53 100644
--- a/src/zenremotestore/chunking/chunkblock.cpp
+++ b/src/zenremotestore/chunking/chunkblock.cpp
@@ -7,7 +7,6 @@
#include <zencore/logging.h>
#include <zencore/timer.h>
#include <zencore/trace.h>
-#include <zenremotestore/operationlogoutput.h>
#include <numeric>
@@ -445,7 +444,7 @@ IterateChunkBlock(const SharedBuffer& BlockPayload,
};
std::vector<size_t>
-FindReuseBlocks(OperationLogOutput& Output,
+FindReuseBlocks(LoggerRef InLog,
const uint8_t BlockReuseMinPercentLimit,
const bool IsVerbose,
ReuseBlocksStatistics& Stats,
@@ -455,6 +454,7 @@ FindReuseBlocks(OperationLogOutput& Output,
std::vector<uint32_t>& OutUnusedChunkIndexes)
{
ZEN_TRACE_CPU("FindReuseBlocks");
+ ZEN_SCOPED_LOG(InLog);
// Find all blocks with a usage level higher than MinPercentLimit
// Pick out the blocks with usage higher or equal to MinPercentLimit
@@ -521,11 +521,10 @@ FindReuseBlocks(OperationLogOutput& Output,
{
if (IsVerbose)
{
- ZEN_OPERATION_LOG_INFO(Output,
- "Reusing block {}. {} attachments found, usage level: {}%",
- KnownBlock.BlockHash,
- FoundAttachmentCount,
- ReusePercent);
+ ZEN_INFO("Reusing block {}. {} attachments found, usage level: {}%",
+ KnownBlock.BlockHash,
+ FoundAttachmentCount,
+ ReusePercent);
}
ReuseBlockIndexes.push_back(KnownBlockIndex);
@@ -534,12 +533,13 @@ FindReuseBlocks(OperationLogOutput& Output,
}
else if (FoundAttachmentCount > 0)
{
- // if (IsVerbose)
- //{
- // ZEN_OPERATION_LOG_INFO(Output, "Skipping block {}. {} attachments found, usage level: {}%",
- // KnownBlock.BlockHash,
- // FoundAttachmentCount, ReusePercent);
- //}
+ if (IsVerbose)
+ {
+ ZEN_INFO("Skipping block {}. {} attachments found, usage level: {}%",
+ KnownBlock.BlockHash,
+ FoundAttachmentCount,
+ ReusePercent);
+ }
Stats.RejectedBlockCount++;
Stats.RejectedChunkCount += FoundAttachmentCount;
Stats.RejectedByteCount += ReuseSize;
@@ -583,11 +583,10 @@ FindReuseBlocks(OperationLogOutput& Output,
{
if (IsVerbose)
{
- ZEN_OPERATION_LOG_INFO(Output,
- "Reusing block {}. {} attachments found, usage level: {}%",
- KnownBlock.BlockHash,
- FoundChunkIndexes.size(),
- ReusePercent);
+ ZEN_INFO("Reusing block {}. {} attachments found, usage level: {}%",
+ KnownBlock.BlockHash,
+ FoundChunkIndexes.size(),
+ ReusePercent);
}
FilteredReuseBlockIndexes.push_back(KnownBlockIndex);
@@ -604,11 +603,10 @@ FindReuseBlocks(OperationLogOutput& Output,
}
else
{
- // if (IsVerbose)
- //{
- // ZEN_OPERATION_LOG_INFO(Output, "Skipping block {}. filtered usage level: {}%", KnownBlock.BlockHash,
- // ReusePercent);
- //}
+ if (IsVerbose)
+ {
+ ZEN_INFO("Skipping block {}. filtered usage level: {}%", KnownBlock.BlockHash, ReusePercent);
+ }
Stats.RejectedBlockCount++;
Stats.RejectedChunkCount += FoundChunkIndexes.size();
Stats.RejectedByteCount += AdjustedReuseSize;
@@ -629,10 +627,8 @@ FindReuseBlocks(OperationLogOutput& Output,
return FilteredReuseBlockIndexes;
}
-ChunkBlockAnalyser::ChunkBlockAnalyser(OperationLogOutput& LogOutput,
- std::span<const ChunkBlockDescription> BlockDescriptions,
- const Options& Options)
-: m_LogOutput(LogOutput)
+ChunkBlockAnalyser::ChunkBlockAnalyser(LoggerRef Log, std::span<const ChunkBlockDescription> BlockDescriptions, const Options& Options)
+: m_Log(Log)
, m_BlockDescriptions(BlockDescriptions)
, m_Options(Options)
{
@@ -899,20 +895,20 @@ ChunkBlockAnalyser::CalculatePartialBlockDownloads(std::span<const NeededBlock>
if (Ideal < FullDownloadTotalSize && !m_Options.IsQuiet)
{
const double AchievedPercent = Ideal == 0 ? 100.0 : (100.0 * Actual) / Ideal;
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Block Partial Analysis: Blocks: {}, Full: {}, Ideal: {}, Actual: {}. Skipping {} ({:.1f}%) out of "
- "possible {} using {} extra ranges "
- "via {} extra requests. Completed in {}",
- NeededBlocks.size(),
- NiceBytes(FullDownloadTotalSize),
- NiceBytes(IdealDownloadTotalSize),
- NiceBytes(ActualDownloadTotalSize),
- NiceBytes(FullDownloadTotalSize - ActualDownloadTotalSize),
- AchievedPercent,
- NiceBytes(Ideal),
- RangeCount - MinRequestCount,
- RequestCount - MinRequestCount,
- NiceTimeSpanMs(PartialAnalisysTimer.GetElapsedTimeMs()));
+ ZEN_INFO(
+ "Block Partial Analysis: Blocks: {}, Full: {}, Ideal: {}, Actual: {}. Skipping {} ({:.1f}%) out of "
+ "possible {} using {} extra ranges "
+ "via {} extra requests. Completed in {}",
+ NeededBlocks.size(),
+ NiceBytes(FullDownloadTotalSize),
+ NiceBytes(IdealDownloadTotalSize),
+ NiceBytes(ActualDownloadTotalSize),
+ NiceBytes(FullDownloadTotalSize - ActualDownloadTotalSize),
+ AchievedPercent,
+ NiceBytes(Ideal),
+ RangeCount - MinRequestCount,
+ RequestCount - MinRequestCount,
+ NiceTimeSpanMs(PartialAnalisysTimer.GetElapsedTimeMs()));
}
}
@@ -1001,8 +997,7 @@ TEST_CASE("chunkblock.reuseblocks")
BlockDescriptions.emplace_back(std::move(Block));
}
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
{
// We use just about all the chunks - should result in use of both blocks
@@ -1019,14 +1014,8 @@ TEST_CASE("chunkblock.reuseblocks")
std::iota(ManyChunkIndexes.begin(), ManyChunkIndexes.end(), 0);
std::vector<uint32_t> UnusedChunkIndexes;
- std::vector<size_t> ReusedBlocks = FindReuseBlocks(*LogOutput,
- 80,
- false,
- ReuseBlocksStats,
- BlockDescriptions,
- ManyChunkHashes,
- ManyChunkIndexes,
- UnusedChunkIndexes);
+ std::vector<size_t> ReusedBlocks =
+ FindReuseBlocks(LogRef, 80, false, ReuseBlocksStats, BlockDescriptions, ManyChunkHashes, ManyChunkIndexes, UnusedChunkIndexes);
CHECK_EQ(2u, ReusedBlocks.size());
CHECK_EQ(0u, UnusedChunkIndexes.size());
@@ -1047,7 +1036,7 @@ TEST_CASE("chunkblock.reuseblocks")
std::iota(ManyChunkIndexes.begin(), ManyChunkIndexes.end(), 0);
std::vector<uint32_t> UnusedChunkIndexes;
- std::vector<size_t> ReusedBlocks = FindReuseBlocks(*LogOutput,
+ std::vector<size_t> ReusedBlocks = FindReuseBlocks(LogRef,
80,
false,
ReuseBlocksStats,
@@ -1076,7 +1065,7 @@ TEST_CASE("chunkblock.reuseblocks")
// We use half the chunks - should result in no use of blocks due to 80% limit
std::vector<uint32_t> UnusedChunkIndexes80Percent;
ReuseBlocksStatistics ReuseBlocksStats;
- std::vector<size_t> ReusedBlocks80Percent = FindReuseBlocks(*LogOutput,
+ std::vector<size_t> ReusedBlocks80Percent = FindReuseBlocks(LogRef,
80,
false,
ReuseBlocksStats,
@@ -1092,7 +1081,7 @@ TEST_CASE("chunkblock.reuseblocks")
// We use half the chunks - should result in use of both blocks due to 40% limit
std::vector<uint32_t> UnusedChunkIndexes40Percent;
ReuseBlocksStatistics ReuseBlocksStats;
- std::vector<size_t> ReusedBlocks40Percent = FindReuseBlocks(*LogOutput,
+ std::vector<size_t> ReusedBlocks40Percent = FindReuseBlocks(LogRef,
40,
false,
ReuseBlocksStats,
@@ -1122,7 +1111,7 @@ TEST_CASE("chunkblock.reuseblocks")
// We use half the chunks for first block - should result in use of one blocks due to 80% limit
ReuseBlocksStatistics ReuseBlocksStats;
std::vector<uint32_t> UnusedChunkIndexes80Percent;
- std::vector<size_t> ReusedBlocks80Percent = FindReuseBlocks(*LogOutput,
+ std::vector<size_t> ReusedBlocks80Percent = FindReuseBlocks(LogRef,
80,
false,
ReuseBlocksStats,
@@ -1139,7 +1128,7 @@ TEST_CASE("chunkblock.reuseblocks")
// We use half the chunks - should result in use of both blocks due to 40% limit
ReuseBlocksStatistics ReuseBlocksStats;
std::vector<uint32_t> UnusedChunkIndexes40Percent;
- std::vector<size_t> ReusedBlocks40Percent = FindReuseBlocks(*LogOutput,
+ std::vector<size_t> ReusedBlocks40Percent = FindReuseBlocks(LogRef,
40,
false,
ReuseBlocksStats,
@@ -1178,7 +1167,7 @@ TEST_CASE("chunkblock.reuseblocks")
// We use half the chunks for first block - should result in use of one blocks due to 80% limit
ReuseBlocksStatistics ReuseBlocksStats;
std::vector<uint32_t> UnusedChunkIndexes80Percent;
- std::vector<size_t> ReusedBlocks80Percent = FindReuseBlocks(*LogOutput,
+ std::vector<size_t> ReusedBlocks80Percent = FindReuseBlocks(LogRef,
80,
false,
ReuseBlocksStats,
@@ -1195,7 +1184,7 @@ TEST_CASE("chunkblock.reuseblocks")
// We use half the chunks - should result in use of both blocks due to 40% limit
ReuseBlocksStatistics ReuseBlocksStats;
std::vector<uint32_t> UnusedChunkIndexes40Percent;
- std::vector<size_t> ReusedBlocks40Percent = FindReuseBlocks(*LogOutput,
+ std::vector<size_t> ReusedBlocks40Percent = FindReuseBlocks(LogRef,
40,
false,
ReuseBlocksStats,
@@ -1214,7 +1203,7 @@ namespace chunkblock_analyser_testutils {
// Build a ChunkBlockDescription without any real payload.
// Hashes are derived deterministically from (BlockSeed XOR ChunkIndex) so that the same
- // seed produces the same hashes — useful for deduplication tests.
+ // seed produces the same hashes - useful for deduplication tests.
static ChunkBlockDescription MakeBlockDesc(uint64_t HeaderSize,
std::initializer_list<uint32_t> CompressedLengths,
uint32_t BlockSeed = 0)
@@ -1257,7 +1246,7 @@ namespace chunkblock_analyser_testutils {
TEST_CASE("chunkblock.mergecheapestrange.picks_smallest_gap")
{
using RD = chunkblock_impl::RangeDescriptor;
- // Gap between ranges 0-1 is 50, gap between 1-2 is 150 → pair 0-1 gets merged
+ // Gap between ranges 0-1 is 50, gap between 1-2 is 150 -> pair 0-1 gets merged
std::vector<RD> Ranges = {
{.RangeStart = 0, .RangeLength = 100, .ChunkBlockIndexStart = 0, .ChunkBlockIndexCount = 1},
{.RangeStart = 150, .RangeLength = 100, .ChunkBlockIndexStart = 1, .ChunkBlockIndexCount = 1},
@@ -1279,7 +1268,7 @@ TEST_CASE("chunkblock.mergecheapestrange.picks_smallest_gap")
TEST_CASE("chunkblock.mergecheapestrange.tiebreak_smaller_merged")
{
using RD = chunkblock_impl::RangeDescriptor;
- // Gap 0-1 == gap 1-2 == 100; merged size 0-1 (250) < merged size 1-2 (350) → pair 0-1 wins
+ // Gap 0-1 == gap 1-2 == 100; merged size 0-1 (250) < merged size 1-2 (350) -> pair 0-1 wins
std::vector<RD> Ranges = {
{.RangeStart = 0, .RangeLength = 100, .ChunkBlockIndexStart = 0, .ChunkBlockIndexCount = 1},
{.RangeStart = 200, .RangeLength = 50, .ChunkBlockIndexStart = 1, .ChunkBlockIndexCount = 1},
@@ -1304,7 +1293,7 @@ TEST_CASE("chunkblock.optimizeranges.preserves_ranges_low_latency")
{
using RD = chunkblock_impl::RangeDescriptor;
// With MaxRangeCountPerRequest unlimited, RequestCount=1
- // RequestTimeAsBytes = 100000 * 1 * 0.001 = 100 << slack=7000 → all ranges preserved
+ // RequestTimeAsBytes = 100000 * 1 * 0.001 = 100 << slack=7000 -> all ranges preserved
std::vector<RD> ExactRanges = {
{.RangeStart = 0, .RangeLength = 1000, .ChunkBlockIndexStart = 0, .ChunkBlockIndexCount = 1},
{.RangeStart = 2000, .RangeLength = 1000, .ChunkBlockIndexStart = 1, .ChunkBlockIndexCount = 1},
@@ -1325,7 +1314,7 @@ TEST_CASE("chunkblock.optimizeranges.preserves_ranges_low_latency")
TEST_CASE("chunkblock.optimizeranges.falls_back_to_full_block")
{
using RD = chunkblock_impl::RangeDescriptor;
- // 1 range already; slack=100 < SpeedBytesPerSec*LatencySec=200 → full block (empty result)
+ // 1 range already; slack=100 < SpeedBytesPerSec*LatencySec=200 -> full block (empty result)
std::vector<RD> ExactRanges = {
{.RangeStart = 100, .RangeLength = 900, .ChunkBlockIndexStart = 0, .ChunkBlockIndexCount = 3},
};
@@ -1344,7 +1333,7 @@ TEST_CASE("chunkblock.optimizeranges.falls_back_to_full_block")
TEST_CASE("chunkblock.optimizeranges.maxrangesperblock_clamp")
{
using RD = chunkblock_impl::RangeDescriptor;
- // 5 input ranges; MaxRangesPerBlock=2 clamps to ≤2 before the cost model runs
+ // 5 input ranges; MaxRangesPerBlock=2 clamps to <=2 before the cost model runs
std::vector<RD> ExactRanges = {
{.RangeStart = 0, .RangeLength = 100, .ChunkBlockIndexStart = 0, .ChunkBlockIndexCount = 1},
{.RangeStart = 300, .RangeLength = 100, .ChunkBlockIndexStart = 1, .ChunkBlockIndexCount = 1},
@@ -1378,8 +1367,8 @@ TEST_CASE("chunkblock.optimizeranges.low_maxrangecountperrequest_drives_merge")
uint64_t TotalBlockSize = 1000;
double LatencySec = 1.0;
uint64_t SpeedBytesPerSec = 500;
- // With MaxRangeCountPerRequest=-1: RequestCount=1, RequestTimeAsBytes=500 < slack=700 → preserved
- // With MaxRangeCountPerRequest=1: RequestCount=3, RequestTimeAsBytes=1500 > slack=700 → merged
+ // With MaxRangeCountPerRequest=-1: RequestCount=1, RequestTimeAsBytes=500 < slack=700 -> preserved
+ // With MaxRangeCountPerRequest=1: RequestCount=3, RequestTimeAsBytes=1500 > slack=700 -> merged
uint64_t MaxRangesPerBlock = 1024;
auto Unlimited =
@@ -1394,7 +1383,7 @@ TEST_CASE("chunkblock.optimizeranges.low_maxrangecountperrequest_drives_merge")
TEST_CASE("chunkblock.optimizeranges.unlimited_rangecountperrequest_no_extra_cost")
{
using RD = chunkblock_impl::RangeDescriptor;
- // MaxRangeCountPerRequest=-1 → RequestCount always 1, even with many ranges and high latency
+ // MaxRangeCountPerRequest=-1 -> RequestCount always 1, even with many ranges and high latency
std::vector<RD> ExactRanges = {
{.RangeStart = 0, .RangeLength = 50, .ChunkBlockIndexStart = 0, .ChunkBlockIndexCount = 1},
{.RangeStart = 200, .RangeLength = 50, .ChunkBlockIndexStart = 1, .ChunkBlockIndexCount = 1},
@@ -1418,7 +1407,7 @@ TEST_CASE("chunkblock.optimizeranges.two_range_direct_merge_path")
{
using RD = chunkblock_impl::RangeDescriptor;
// Exactly 2 ranges; cost model demands merge; exercises the RangeCount==2 direct-merge branch
- // After direct merge → 1 range with small slack → full block (empty)
+ // After direct merge -> 1 range with small slack -> full block (empty)
std::vector<RD> ExactRanges = {
{.RangeStart = 0, .RangeLength = 100, .ChunkBlockIndexStart = 0, .ChunkBlockIndexCount = 2},
{.RangeStart = 400, .RangeLength = 100, .ChunkBlockIndexStart = 2, .ChunkBlockIndexCount = 2},
@@ -1429,8 +1418,8 @@ TEST_CASE("chunkblock.optimizeranges.two_range_direct_merge_path")
uint64_t MaxRangeCountPerReq = (uint64_t)-1;
uint64_t MaxRangesPerBlock = 1024;
- // Iteration 1: RangeCount=2, RequestCount=1, RequestTimeAsBytes=500 > slack=400 → direct merge
- // After merge: 1 range [{0,500,0,4}], slack=100 < Speed*Lat=500 → full block
+ // Iteration 1: RangeCount=2, RequestCount=1, RequestTimeAsBytes=500 > slack=400 -> direct merge
+ // After merge: 1 range [{0,500,0,4}], slack=100 < Speed*Lat=500 -> full block
auto Result =
chunkblock_impl::OptimizeRanges(TotalBlockSize, ExactRanges, LatencySec, SpeedBytesPerSec, MaxRangeCountPerReq, MaxRangesPerBlock);
@@ -1441,12 +1430,11 @@ TEST_CASE("chunkblock.getneeded.all_chunks")
{
using namespace chunkblock_analyser_testutils;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
auto Block = MakeBlockDesc(50, {100, 100, 100, 100});
ChunkBlockAnalyser::Options Options;
- ChunkBlockAnalyser Analyser(*LogOutput, std::span<const ChunkBlockDescription>(&Block, 1), Options);
+ ChunkBlockAnalyser Analyser(LogRef, std::span<const ChunkBlockDescription>(&Block, 1), Options);
auto HashMap = MakeHashMap({Block});
auto NeededBlocks = Analyser.GetNeeded(HashMap, [](uint32_t) { return true; });
@@ -1464,12 +1452,11 @@ TEST_CASE("chunkblock.getneeded.no_chunks")
{
using namespace chunkblock_analyser_testutils;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
auto Block = MakeBlockDesc(50, {100, 100, 100, 100});
ChunkBlockAnalyser::Options Options;
- ChunkBlockAnalyser Analyser(*LogOutput, std::span<const ChunkBlockDescription>(&Block, 1), Options);
+ ChunkBlockAnalyser Analyser(LogRef, std::span<const ChunkBlockDescription>(&Block, 1), Options);
auto HashMap = MakeHashMap({Block});
auto NeededBlocks = Analyser.GetNeeded(HashMap, [](uint32_t) { return false; });
@@ -1481,12 +1468,11 @@ TEST_CASE("chunkblock.getneeded.subset_within_block")
{
using namespace chunkblock_analyser_testutils;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
auto Block = MakeBlockDesc(50, {100, 100, 100, 100});
ChunkBlockAnalyser::Options Options;
- ChunkBlockAnalyser Analyser(*LogOutput, std::span<const ChunkBlockDescription>(&Block, 1), Options);
+ ChunkBlockAnalyser Analyser(LogRef, std::span<const ChunkBlockDescription>(&Block, 1), Options);
auto HashMap = MakeHashMap({Block});
// Indices 0 and 2 are needed; 1 and 3 are not
@@ -1503,12 +1489,11 @@ TEST_CASE("chunkblock.getneeded.dedup_low_slack_wins")
{
using namespace chunkblock_analyser_testutils;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
- // Block 0: {H0, H1, SharedH, H3} — 3 of 4 needed (H3 not needed); slack = 100
- // Block 1: {H4, H5, SharedH, H6} — only SharedH needed; slack = 300
- // Block 0 has less slack → processed first → SharedH assigned to block 0
+ // Block 0: {H0, H1, SharedH, H3} - 3 of 4 needed (H3 not needed); slack = 100
+ // Block 1: {H4, H5, SharedH, H6} - only SharedH needed; slack = 300
+ // Block 0 has less slack -> processed first -> SharedH assigned to block 0
IoHash SharedH = IoHash::HashBuffer(MemoryView("shared_chunk_dedup", 18));
IoHash H0 = IoHash::HashBuffer(MemoryView("block0_chunk0", 13));
IoHash H1 = IoHash::HashBuffer(MemoryView("block0_chunk1", 13));
@@ -1531,9 +1516,9 @@ TEST_CASE("chunkblock.getneeded.dedup_low_slack_wins")
std::vector<ChunkBlockDescription> Blocks = {Block0, Block1};
ChunkBlockAnalyser::Options Options;
- ChunkBlockAnalyser Analyser(*LogOutput, Blocks, Options);
+ ChunkBlockAnalyser Analyser(LogRef, Blocks, Options);
- // Map: H0→0, H1→1, SharedH→2, H3→3, H4→4, H5→5, H6→6
+ // Map: H0->0, H1->1, SharedH->2, H3->3, H4->4, H5->5, H6->6
auto HashMap = MakeHashMap(Blocks);
// Need H0(0), H1(1), SharedH(2) from block 0; SharedH from block 1 (already index 2)
// H3(3) not needed; H4,H5,H6 not needed
@@ -1541,7 +1526,7 @@ TEST_CASE("chunkblock.getneeded.dedup_low_slack_wins")
// Block 0 slack=100 (H3 unused), block 1 slack=300 (H4,H5,H6 unused)
// Block 0 processed first; picks up H0, H1, SharedH
- // Block 1 tries SharedH but it's already picked up → empty → not added
+ // Block 1 tries SharedH but it's already picked up -> empty -> not added
REQUIRE_EQ(1u, NeededBlocks.size());
CHECK_EQ(0u, NeededBlocks[0].BlockIndex);
REQUIRE_EQ(3u, NeededBlocks[0].ChunkIndexes.size());
@@ -1554,8 +1539,7 @@ TEST_CASE("chunkblock.getneeded.dedup_no_double_pickup")
{
using namespace chunkblock_analyser_testutils;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
// SharedH appears in both blocks; should appear in the result exactly once
IoHash SharedH = IoHash::HashBuffer(MemoryView("shared_chunk_nodup", 18));
@@ -1578,16 +1562,16 @@ TEST_CASE("chunkblock.getneeded.dedup_no_double_pickup")
std::vector<ChunkBlockDescription> Blocks = {Block0, Block1};
ChunkBlockAnalyser::Options Options;
- ChunkBlockAnalyser Analyser(*LogOutput, Blocks, Options);
+ ChunkBlockAnalyser Analyser(LogRef, Blocks, Options);
- // Map: SharedH→0, H0→1, H1→2, H2→3, H3→4
+ // Map: SharedH->0, H0->1, H1->2, H2->3, H3->4
// Only SharedH (index 0) needed; no other chunks
auto HashMap = MakeHashMap(Blocks);
auto NeededBlocks = Analyser.GetNeeded(HashMap, [](uint32_t ChunkIndex) { return ChunkIndex == 0; });
- // Block 0: SharedH needed, H0 not needed → slack=100
- // Block 1: SharedH needed, H1/H2/H3 not needed → slack=300
- // Block 0 processed first → picks up SharedH; Block 1 skips it
+ // Block 0: SharedH needed, H0 not needed -> slack=100
+ // Block 1: SharedH needed, H1/H2/H3 not needed -> slack=300
+ // Block 0 processed first -> picks up SharedH; Block 1 skips it
// Count total occurrences of SharedH across all NeededBlocks
uint32_t SharedOccurrences = 0;
@@ -1609,13 +1593,12 @@ TEST_CASE("chunkblock.getneeded.skips_unrequested_chunks")
{
using namespace chunkblock_analyser_testutils;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
- // Block has 4 chunks but only 2 appear in the hash map → ChunkIndexes has exactly those 2
+ // Block has 4 chunks but only 2 appear in the hash map -> ChunkIndexes has exactly those 2
auto Block = MakeBlockDesc(50, {100, 100, 100, 100});
ChunkBlockAnalyser::Options Options;
- ChunkBlockAnalyser Analyser(*LogOutput, std::span<const ChunkBlockDescription>(&Block, 1), Options);
+ ChunkBlockAnalyser Analyser(LogRef, std::span<const ChunkBlockDescription>(&Block, 1), Options);
// Only put chunks at positions 0 and 2 in the map
tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> HashMap;
@@ -1635,19 +1618,18 @@ TEST_CASE("chunkblock.getneeded.two_blocks_both_contribute")
{
using namespace chunkblock_analyser_testutils;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
// Block 0: all 4 needed (slack=0); block 1: 3 of 4 needed (slack=100)
- // Both blocks contribute chunks → 2 NeededBlocks in result
+ // Both blocks contribute chunks -> 2 NeededBlocks in result
auto Block0 = MakeBlockDesc(50, {100, 100, 100, 100}, /*BlockSeed=*/0);
auto Block1 = MakeBlockDesc(50, {100, 100, 100, 100}, /*BlockSeed=*/200);
std::vector<ChunkBlockDescription> Blocks = {Block0, Block1};
ChunkBlockAnalyser::Options Options;
- ChunkBlockAnalyser Analyser(*LogOutput, Blocks, Options);
+ ChunkBlockAnalyser Analyser(LogRef, Blocks, Options);
- // HashMap: Block0 hashes → indices 0-3, Block1 hashes → indices 4-7
+ // HashMap: Block0 hashes -> indices 0-3, Block1 hashes -> indices 4-7
auto HashMap = MakeHashMap(Blocks);
// Need all Block0 chunks (0-3) and Block1 chunks 0-2 (indices 4-6); not chunk index 7 (Block1 chunk 3)
auto NeededBlocks = Analyser.GetNeeded(HashMap, [](uint32_t ChunkIndex) { return ChunkIndex <= 6; });
@@ -1666,15 +1648,14 @@ TEST_CASE("chunkblock.calc.off_mode")
using namespace chunkblock_analyser_testutils;
using Mode = ChunkBlockAnalyser::EPartialBlockDownloadMode;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
- // HeaderSize > 0, chunks size matches → CanDoPartialBlockDownload = true
+ // HeaderSize > 0, chunks size matches -> CanDoPartialBlockDownload = true
// But mode Off forces full block regardless
auto Block = MakeBlockDesc(50, {100, 200, 300, 400});
ChunkBlockAnalyser::Options Options;
Options.IsQuiet = true;
- ChunkBlockAnalyser Analyser(*LogOutput, std::span<const ChunkBlockDescription>(&Block, 1), Options);
+ ChunkBlockAnalyser Analyser(LogRef, std::span<const ChunkBlockDescription>(&Block, 1), Options);
std::vector<ChunkBlockAnalyser::NeededBlock> NeededBlocks = {{.BlockIndex = 0, .ChunkIndexes = {0, 2}}};
std::vector<Mode> Modes = {Mode::Off};
@@ -1691,17 +1672,16 @@ TEST_CASE("chunkblock.calc.exact_mode")
using namespace chunkblock_analyser_testutils;
using Mode = ChunkBlockAnalyser::EPartialBlockDownloadMode;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
auto Block = MakeBlockDesc(50, {100, 200, 300, 400});
ChunkBlockAnalyser::Options Options;
Options.IsQuiet = true;
- ChunkBlockAnalyser Analyser(*LogOutput, std::span<const ChunkBlockDescription>(&Block, 1), Options);
+ ChunkBlockAnalyser Analyser(LogRef, std::span<const ChunkBlockDescription>(&Block, 1), Options);
uint64_t ChunkStartOffset = CompressedBuffer::GetHeaderSizeForNoneEncoder() + Block.HeaderSize;
- // Need chunks 0 and 2 → 2 non-contiguous ranges; Exact mode passes them straight through
+ // Need chunks 0 and 2 -> 2 non-contiguous ranges; Exact mode passes them straight through
std::vector<ChunkBlockAnalyser::NeededBlock> NeededBlocks = {{.BlockIndex = 0, .ChunkIndexes = {0, 2}}};
std::vector<Mode> Modes = {Mode::Exact};
@@ -1728,18 +1708,17 @@ TEST_CASE("chunkblock.calc.singlerange_mode")
using namespace chunkblock_analyser_testutils;
using Mode = ChunkBlockAnalyser::EPartialBlockDownloadMode;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
auto Block = MakeBlockDesc(50, {100, 200, 300, 400});
- // Default HostLatencySec=-1 → OptimizeRanges not called after SingleRange collapse
+ // Default HostLatencySec=-1 -> OptimizeRanges not called after SingleRange collapse
ChunkBlockAnalyser::Options Options;
Options.IsQuiet = true;
- ChunkBlockAnalyser Analyser(*LogOutput, std::span<const ChunkBlockDescription>(&Block, 1), Options);
+ ChunkBlockAnalyser Analyser(LogRef, std::span<const ChunkBlockDescription>(&Block, 1), Options);
uint64_t ChunkStartOffset = CompressedBuffer::GetHeaderSizeForNoneEncoder() + Block.HeaderSize;
- // Need chunks 0 and 2 → 2 ranges that get collapsed to 1
+ // Need chunks 0 and 2 -> 2 ranges that get collapsed to 1
std::vector<ChunkBlockAnalyser::NeededBlock> NeededBlocks = {{.BlockIndex = 0, .ChunkIndexes = {0, 2}}};
std::vector<Mode> Modes = {Mode::SingleRange};
@@ -1761,16 +1740,15 @@ TEST_CASE("chunkblock.calc.multirange_mode")
using namespace chunkblock_analyser_testutils;
using Mode = ChunkBlockAnalyser::EPartialBlockDownloadMode;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
auto Block = MakeBlockDesc(50, {100, 200, 300, 400});
- // Low latency: RequestTimeAsBytes=100 << slack → OptimizeRanges preserves ranges
+ // Low latency: RequestTimeAsBytes=100 << slack -> OptimizeRanges preserves ranges
ChunkBlockAnalyser::Options Options;
Options.IsQuiet = true;
Options.HostLatencySec = 0.001;
Options.HostSpeedBytesPerSec = 100000;
- ChunkBlockAnalyser Analyser(*LogOutput, std::span<const ChunkBlockDescription>(&Block, 1), Options);
+ ChunkBlockAnalyser Analyser(LogRef, std::span<const ChunkBlockDescription>(&Block, 1), Options);
uint64_t ChunkStartOffset = CompressedBuffer::GetHeaderSizeForNoneEncoder() + Block.HeaderSize;
@@ -1792,17 +1770,16 @@ TEST_CASE("chunkblock.calc.multirangehighspeed_mode")
using namespace chunkblock_analyser_testutils;
using Mode = ChunkBlockAnalyser::EPartialBlockDownloadMode;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
auto Block = MakeBlockDesc(50, {100, 200, 300, 400});
- // Block slack ≈ 714 bytes (TotalBlockSize≈1114, RangeTotalSize=400 for chunks 0+2)
- // RequestTimeAsBytes = 400000 * 1 * 0.001 = 400 < 714 → ranges preserved
+ // Block slack ~= 714 bytes (TotalBlockSize~=1114, RangeTotalSize=400 for chunks 0+2)
+ // RequestTimeAsBytes = 400000 * 1 * 0.001 = 400 < 714 -> ranges preserved
ChunkBlockAnalyser::Options Options;
Options.IsQuiet = true;
Options.HostHighSpeedLatencySec = 0.001;
Options.HostHighSpeedBytesPerSec = 400000;
- ChunkBlockAnalyser Analyser(*LogOutput, std::span<const ChunkBlockDescription>(&Block, 1), Options);
+ ChunkBlockAnalyser Analyser(LogRef, std::span<const ChunkBlockDescription>(&Block, 1), Options);
uint64_t ChunkStartOffset = CompressedBuffer::GetHeaderSizeForNoneEncoder() + Block.HeaderSize;
@@ -1824,17 +1801,16 @@ TEST_CASE("chunkblock.calc.all_chunks_needed_full_block")
using namespace chunkblock_analyser_testutils;
using Mode = ChunkBlockAnalyser::EPartialBlockDownloadMode;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
auto Block = MakeBlockDesc(50, {100, 200, 300, 400});
ChunkBlockAnalyser::Options Options;
Options.IsQuiet = true;
Options.HostLatencySec = 0.001;
Options.HostSpeedBytesPerSec = 100000;
- ChunkBlockAnalyser Analyser(*LogOutput, std::span<const ChunkBlockDescription>(&Block, 1), Options);
+ ChunkBlockAnalyser Analyser(LogRef, std::span<const ChunkBlockDescription>(&Block, 1), Options);
- // All 4 chunks needed → short-circuit to full block regardless of mode
+ // All 4 chunks needed -> short-circuit to full block regardless of mode
std::vector<ChunkBlockAnalyser::NeededBlock> NeededBlocks = {{.BlockIndex = 0, .ChunkIndexes = {0, 1, 2, 3}}};
std::vector<Mode> Modes = {Mode::Exact};
@@ -1850,14 +1826,13 @@ TEST_CASE("chunkblock.calc.headersize_zero_forces_full_block")
using namespace chunkblock_analyser_testutils;
using Mode = ChunkBlockAnalyser::EPartialBlockDownloadMode;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
- // HeaderSize=0 → CanDoPartialBlockDownload=false → full block even in Exact mode
+ // HeaderSize=0 -> CanDoPartialBlockDownload=false -> full block even in Exact mode
auto Block = MakeBlockDesc(0, {100, 200, 300, 400});
ChunkBlockAnalyser::Options Options;
Options.IsQuiet = true;
- ChunkBlockAnalyser Analyser(*LogOutput, std::span<const ChunkBlockDescription>(&Block, 1), Options);
+ ChunkBlockAnalyser Analyser(LogRef, std::span<const ChunkBlockDescription>(&Block, 1), Options);
std::vector<ChunkBlockAnalyser::NeededBlock> NeededBlocks = {{.BlockIndex = 0, .ChunkIndexes = {0, 2}}};
std::vector<Mode> Modes = {Mode::Exact};
@@ -1874,25 +1849,24 @@ TEST_CASE("chunkblock.calc.low_maxrangecountperrequest")
using namespace chunkblock_analyser_testutils;
using Mode = ChunkBlockAnalyser::EPartialBlockDownloadMode;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
- // 5 chunks of 100 bytes each; need chunks 0, 2, 4 → 3 non-contiguous ranges
- // With MaxRangeCountPerRequest=1 and high latency, cost model merges aggressively → full block
+ // 5 chunks of 100 bytes each; need chunks 0, 2, 4 -> 3 non-contiguous ranges
+ // With MaxRangeCountPerRequest=1 and high latency, cost model merges aggressively -> full block
auto Block = MakeBlockDesc(10, {100, 100, 100, 100, 100});
ChunkBlockAnalyser::Options Options;
Options.IsQuiet = true;
Options.HostLatencySec = 0.1;
Options.HostSpeedBytesPerSec = 1000;
Options.HostMaxRangeCountPerRequest = 1;
- ChunkBlockAnalyser Analyser(*LogOutput, std::span<const ChunkBlockDescription>(&Block, 1), Options);
+ ChunkBlockAnalyser Analyser(LogRef, std::span<const ChunkBlockDescription>(&Block, 1), Options);
std::vector<ChunkBlockAnalyser::NeededBlock> NeededBlocks = {{.BlockIndex = 0, .ChunkIndexes = {0, 2, 4}}};
std::vector<Mode> Modes = {Mode::MultiRange};
auto Result = Analyser.CalculatePartialBlockDownloads(NeededBlocks, Modes);
- // Cost model drives merging: 3 requests × 1000 × 0.1 = 300 > slack ≈ 210+headersize
+ // Cost model drives merging: 3 requests x 1000 x 0.1 = 300 > slack ~= 210+headersize
// After merges converges to full block
REQUIRE_EQ(1u, Result.FullBlockIndexes.size());
CHECK_EQ(0u, Result.FullBlockIndexes[0]);
@@ -1904,14 +1878,13 @@ TEST_CASE("chunkblock.calc.no_latency_skips_optimize")
using namespace chunkblock_analyser_testutils;
using Mode = ChunkBlockAnalyser::EPartialBlockDownloadMode;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
auto Block = MakeBlockDesc(50, {100, 200, 300, 400});
- // Default HostLatencySec=-1 → OptimizeRanges not called; raw GetBlockRanges result used
+ // Default HostLatencySec=-1 -> OptimizeRanges not called; raw GetBlockRanges result used
ChunkBlockAnalyser::Options Options;
Options.IsQuiet = true;
- ChunkBlockAnalyser Analyser(*LogOutput, std::span<const ChunkBlockDescription>(&Block, 1), Options);
+ ChunkBlockAnalyser Analyser(LogRef, std::span<const ChunkBlockDescription>(&Block, 1), Options);
uint64_t ChunkStartOffset = CompressedBuffer::GetHeaderSizeForNoneEncoder() + Block.HeaderSize;
@@ -1920,7 +1893,7 @@ TEST_CASE("chunkblock.calc.no_latency_skips_optimize")
auto Result = Analyser.CalculatePartialBlockDownloads(NeededBlocks, Modes);
- // No optimize pass → exact ranges from GetBlockRanges
+ // No optimize pass -> exact ranges from GetBlockRanges
CHECK(Result.FullBlockIndexes.empty());
REQUIRE_EQ(2u, Result.BlockRanges.size());
CHECK_EQ(ChunkStartOffset, Result.BlockRanges[0].RangeStart);
@@ -1934,8 +1907,7 @@ TEST_CASE("chunkblock.calc.multiple_blocks_different_modes")
using namespace chunkblock_analyser_testutils;
using Mode = ChunkBlockAnalyser::EPartialBlockDownloadMode;
- LoggerRef LogRef = Log();
- std::unique_ptr<OperationLogOutput> LogOutput(CreateStandardLogOutput(LogRef));
+ LoggerRef LogRef = Log();
// 3 blocks with different modes: Off, Exact, MultiRange
auto Block0 = MakeBlockDesc(50, {100, 200, 300, 400}, /*BlockSeed=*/0);
@@ -1948,7 +1920,7 @@ TEST_CASE("chunkblock.calc.multiple_blocks_different_modes")
Options.HostSpeedBytesPerSec = 100000;
std::vector<ChunkBlockDescription> Blocks = {Block0, Block1, Block2};
- ChunkBlockAnalyser Analyser(*LogOutput, Blocks, Options);
+ ChunkBlockAnalyser Analyser(LogRef, Blocks, Options);
uint64_t ChunkStartOffset = CompressedBuffer::GetHeaderSizeForNoneEncoder() + 50;
@@ -1961,11 +1933,11 @@ TEST_CASE("chunkblock.calc.multiple_blocks_different_modes")
auto Result = Analyser.CalculatePartialBlockDownloads(NeededBlocks, Modes);
- // Block 0: Off → FullBlockIndexes
+ // Block 0: Off -> FullBlockIndexes
REQUIRE_EQ(1u, Result.FullBlockIndexes.size());
CHECK_EQ(0u, Result.FullBlockIndexes[0]);
- // Block 1: Exact → 2 ranges; Block 2: MultiRange (low latency) → 2 ranges
+ // Block 1: Exact -> 2 ranges; Block 2: MultiRange (low latency) -> 2 ranges
// Total: 4 ranges
REQUIRE_EQ(4u, Result.BlockRanges.size());
@@ -2058,7 +2030,7 @@ TEST_CASE("chunkblock.getblockranges.non_contiguous")
{
using namespace chunkblock_analyser_testutils;
- // Chunks 0 and 2 needed, chunk 1 skipped → two separate ranges
+ // Chunks 0 and 2 needed, chunk 1 skipped -> two separate ranges
auto Block = MakeBlockDesc(50, {100, 200, 300});
uint64_t ChunkStartOffset = CompressedBuffer::GetHeaderSizeForNoneEncoder() + Block.HeaderSize;
@@ -2082,7 +2054,7 @@ TEST_CASE("chunkblock.getblockranges.contiguous_run")
{
using namespace chunkblock_analyser_testutils;
- // Chunks 1, 2, 3 needed (consecutive) → one merged range
+ // Chunks 1, 2, 3 needed (consecutive) -> one merged range
auto Block = MakeBlockDesc(50, {50, 100, 150, 200, 250});
uint64_t ChunkStartOffset = CompressedBuffer::GetHeaderSizeForNoneEncoder() + Block.HeaderSize;
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-07 00:18:54 +0000