aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2022-03-23 17:20:39 +0100
committerDan Engelbrecht <[email protected]>2022-03-31 11:29:27 +0200
commita24ebc13bd3321eaef5757f6b42fd7fa2ebdfcc0 (patch)
tree5292d47e59ca5821c13328670fd0952082506b6c
parenttypedef for LocationMap (diff)
downloadzen-a24ebc13bd3321eaef5757f6b42fd7fa2ebdfcc0.tar.xz
zen-a24ebc13bd3321eaef5757f6b42fd7fa2ebdfcc0.zip
review feedback
-rw-r--r--zencore/iobuffer.cpp2
-rw-r--r--zenstore/blockstore.cpp25
-rw-r--r--zenstore/compactcas.cpp90
-rw-r--r--zenstore/compactcas.h12
-rw-r--r--zenstore/gc.cpp2
-rw-r--r--zenstore/include/zenstore/blockstore.h8
6 files changed, 69 insertions, 70 deletions
diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp
index aa782bb5c..8a3ab8427 100644
--- a/zencore/iobuffer.cpp
+++ b/zencore/iobuffer.cpp
@@ -219,7 +219,7 @@ IoBufferExtendedCore::~IoBufferExtendedCore()
#endif
if (!Success)
{
- ZEN_WARN("Error reported on file handle close, reason {}", GetLastErrorAsString());
+ ZEN_WARN("Error reported on file handle close, reason '{}'", GetLastErrorAsString());
}
}
diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp
index e865ede8a..441b5e926 100644
--- a/zenstore/blockstore.cpp
+++ b/zenstore/blockstore.cpp
@@ -189,12 +189,12 @@ TEST_CASE("blockstore.blockfile")
File1.Create(16384);
CHECK(File1.FileSize() == 16384);
File1.Write("data", 5, 0);
- auto DataChunk = File1.GetChunk(0, 5);
+ IoBuffer DataChunk = File1.GetChunk(0, 5);
File1.Write("boop", 5, 5);
- auto BoopChunk = File1.GetChunk(5, 5);
- auto Data = static_cast<const char*>(DataChunk.GetData());
+ IoBuffer BoopChunk = File1.GetChunk(5, 5);
+ const char* Data = static_cast<const char*>(DataChunk.GetData());
CHECK(std::string(Data) == "data");
- auto Boop = static_cast<const char*>(BoopChunk.GetData());
+ const char* Boop = static_cast<const char*>(BoopChunk.GetData());
CHECK(std::string(Boop) == "boop");
File1.Flush();
}
@@ -205,16 +205,16 @@ TEST_CASE("blockstore.blockfile")
char DataRaw[5];
File1.Read(DataRaw, 5, 0);
CHECK(std::string(DataRaw) == "data");
- auto DataChunk = File1.GetChunk(0, 5);
+ IoBuffer DataChunk = File1.GetChunk(0, 5);
char BoopRaw[5];
File1.Read(BoopRaw, 5, 5);
CHECK(std::string(BoopRaw) == "boop");
- auto BoopChunk = File1.GetChunk(5, 5);
- auto Data = static_cast<const char*>(DataChunk.GetData());
+ IoBuffer BoopChunk = File1.GetChunk(5, 5);
+ const char* Data = static_cast<const char*>(DataChunk.GetData());
CHECK(std::string(Data) == "data");
- auto Boop = static_cast<const char*>(BoopChunk.GetData());
+ const char* Boop = static_cast<const char*>(BoopChunk.GetData());
CHECK(std::string(Boop) == "boop");
}
@@ -231,9 +231,9 @@ TEST_CASE("blockstore.blockfile")
CHECK(std::filesystem::exists(RootDirectory / "1"));
- auto Data = static_cast<const char*>(DataChunk.GetData());
+ const char* Data = static_cast<const char*>(DataChunk.GetData());
CHECK(std::string(Data) == "data");
- auto Boop = static_cast<const char*>(BoopChunk.GetData());
+ const char* Boop = static_cast<const char*>(BoopChunk.GetData());
CHECK(std::string(Boop) == "boop");
}
CHECK(std::filesystem::exists(RootDirectory / "1"));
@@ -247,15 +247,14 @@ TEST_CASE("blockstore.blockfile")
File1.Open();
std::error_code Ec;
File1.MarkAsDeleteOnClose(Ec);
- auto M = Ec.message();
CHECK(!Ec);
DataChunk = File1.GetChunk(0, 5);
BoopChunk = File1.GetChunk(5, 5);
}
- auto Data = static_cast<const char*>(DataChunk.GetData());
+ const char* Data = static_cast<const char*>(DataChunk.GetData());
CHECK(std::string(Data) == "data");
- auto Boop = static_cast<const char*>(BoopChunk.GetData());
+ const char* Boop = static_cast<const char*>(BoopChunk.GetData());
CHECK(std::string(Boop) == "boop");
}
CHECK(!std::filesystem::exists(RootDirectory / "1"));
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index 4de5635a8..a2656fb0a 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -66,7 +66,7 @@ namespace {
{
result.push_back({.Key = MovedEntry.first, .Location = MovedEntry.second});
}
- for (const auto& ChunkHash : DeletedChunks)
+ for (const IoHash& ChunkHash : DeletedChunks)
{
result.push_back({.Key = ChunkHash, .Flags = CasDiskIndexEntry::kTombstone});
}
@@ -127,12 +127,12 @@ CasContainerStrategy::InsertChunk(const void* ChunkData, size_t ChunkSize, const
// New entry
WriteBlockIndex = m_WriteBlockIndex.load(std::memory_order_acquire);
- WriteBlock = m_WriteBlock.lock();
- if (!WriteBlock || (m_CurrentInsertOffset + ChunkSize) > m_MaxBlockSize)
+ bool IsWriting = m_WriteBlock != nullptr;
+ if (!IsWriting || (m_CurrentInsertOffset + ChunkSize) > m_MaxBlockSize)
{
- if (WriteBlock)
+ if (m_WriteBlock)
{
- WriteBlock->Flush();
+ m_WriteBlock.reset();
}
{
RwLock::ExclusiveLockScope __(m_LocationMapLock);
@@ -140,26 +140,26 @@ CasContainerStrategy::InsertChunk(const void* ChunkData, size_t ChunkSize, const
{
throw std::runtime_error(fmt::format("unable to allocate a new block in {}", m_ContainerBaseName));
}
- WriteBlockIndex += WriteBlock ? 1 : 0;
+ WriteBlockIndex += IsWriting ? 1 : 0;
while (m_ChunkBlocks.contains(WriteBlockIndex))
{
WriteBlockIndex = (WriteBlockIndex + 1) & BlockStoreDiskLocation::MaxBlockIndex;
}
auto BlockPath = BuildUcasPath(m_BlocksBasePath, WriteBlockIndex);
- WriteBlock = std::make_shared<BlockStoreFile>(BlockPath);
- m_ChunkBlocks[WriteBlockIndex] = WriteBlock;
+ m_WriteBlock = std::make_shared<BlockStoreFile>(BlockPath);
+ m_ChunkBlocks[WriteBlockIndex] = m_WriteBlock;
m_WriteBlockIndex.store(WriteBlockIndex, std::memory_order_release);
}
- m_WriteBlock = WriteBlock;
m_CurrentInsertOffset = 0;
- WriteBlock->Create(m_MaxBlockSize);
+ m_WriteBlock->Create(m_MaxBlockSize);
}
else
{
- WriteBlock->Open();
+ m_WriteBlock->Open();
}
InsertOffset = m_CurrentInsertOffset;
m_CurrentInsertOffset = RoundUp(InsertOffset + ChunkSize, m_PayloadAlignment);
+ WriteBlock = m_WriteBlock;
}
WriteBlock->Write(ChunkData, ChunkSize, InsertOffset);
@@ -234,7 +234,7 @@ CasContainerStrategy::Flush()
if (m_CurrentInsertOffset > 0)
{
uint32_t WriteBlockIndex = m_WriteBlockIndex.load(std::memory_order_acquire);
- auto WriteBlock = m_WriteBlock.lock();
+ auto WriteBlock = m_WriteBlock;
WriteBlockIndex++;
while (m_ChunkBlocks.contains(WriteBlockIndex))
{
@@ -363,7 +363,7 @@ CasContainerStrategy::Scrub(ScrubContext& Ctx)
void
CasContainerStrategy::UpdateLocations(const std::span<CasDiskIndexEntry>& Entries)
{
- for (const auto& Entry : Entries)
+ for (const CasDiskIndexEntry& Entry : Entries)
{
if (Entry.Flags & CasDiskIndexEntry::kTombstone)
{
@@ -414,7 +414,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
Stopwatch TotalTimer;
uint64_t WriteBlockTimeUs = 0;
uint64_t ReadBlockTimeUs = 0;
- uint64_t TotalChunkCount = {};
+ uint64_t TotalChunkCount = 0;
uint64_t DeletedSize = 0;
uint64_t OldTotalSize = m_TotalSize.load(std::memory_order::relaxed);
@@ -452,7 +452,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
RwLock::SharedLockScope _l(m_LocationMapLock);
Stopwatch Timer;
const auto TimerGuard = MakeGuard([&Timer, &WriteBlockTimeUs] { WriteBlockTimeUs += Timer.GetElapsedTimeUs(); });
- if (!m_WriteBlock.expired())
+ if (m_WriteBlock)
{
ExcludeBlockIndex = m_WriteBlockIndex.load(std::memory_order_acquire);
}
@@ -500,7 +500,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
DeleteChunks.back().reserve(GuesstimateCountPerBlock);
}
- uint64_t DeleteCount = {};
+ uint64_t DeleteCount = 0;
uint64_t NewTotalSize = 0;
GcCtx.FilterCas(TotalChunkHashes, [&](const IoHash& ChunkHash, bool Keep) {
@@ -538,12 +538,12 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
// Move all chunks in blocks that have chunks removed to new blocks
std::shared_ptr<BlockStoreFile> NewBlockFile;
- uint64_t WriteOffset = {};
- uint32_t NewBlockIndex = {};
+ uint64_t WriteOffset = 0;
+ uint32_t NewBlockIndex = 0;
DeletedChunks.reserve(DeleteCount);
std::unordered_map<IoHash, BlockStoreDiskLocation> MovedBlockChunks;
- for (auto BlockIndex : BlocksToReWrite)
+ for (uint32_t BlockIndex : BlocksToReWrite)
{
const size_t ChunkMapIndex = BlockIndexToChunkMapIndex[BlockIndex];
const auto& KeepMap = KeepChunks[ChunkMapIndex];
@@ -580,7 +580,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
MovedChunks.reserve(MovedChunks.size() + KeepMap.size());
std::vector<uint8_t> Chunk;
- for (const auto& ChunkHash : KeepMap)
+ for (const IoHash& ChunkHash : KeepMap)
{
auto KeyIt = LocationMap.find(ChunkHash);
const BlockStoreLocation ChunkLocation = KeyIt->second.Get(m_PayloadAlignment);
@@ -692,7 +692,7 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
OldBlockFile.reset();
}
- for (const auto& ChunkHash : DeletedChunks)
+ for (const IoHash& ChunkHash : DeletedChunks)
{
DeletedSize += LocationMap[ChunkHash].GetSize();
}
@@ -921,7 +921,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
{
if (std::filesystem::is_regular_file(LegacySobsPath))
{
- uint32_t NewBlockIndex = {};
+ uint32_t NewBlockIndex = 0;
Stopwatch MigrationTimer;
uint64_t TotalSize = 0;
const auto Guard = MakeGuard([this, &MigrationTimer, &NewBlockIndex, &TotalSize] {
@@ -1009,10 +1009,10 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
m_CasLog.Open(SlogPath, true);
std::unique_ptr<BlockStoreFile> NewBlockFile;
- uint64_t WriteOffset = {};
+ uint64_t WriteOffset = 0;
std::vector<uint8_t> Chunk;
- for (const auto& ChunkHash : ChunkHashes)
+ for (const IoHash& ChunkHash : ChunkHashes)
{
const auto& Entry = LegacyDiskIndex[ChunkHash];
const LegacyCasDiskLocation& ChunkLocation = Entry.Location;
@@ -1088,7 +1088,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
std::unordered_set<uint32_t> BlockUsage;
for (const auto& Entry : m_LocationMap)
{
- const auto& Location = Entry.second;
+ const BlockStoreDiskLocation& Location = Entry.second;
m_TotalSize.fetch_add(Location.GetSize());
BlockUsage.insert(Location.GetBlockIndex());
}
@@ -1436,14 +1436,14 @@ TEST_CASE("compactcas.gc.compact")
uint64_t ChunkSizes[9] = {128, 541, 1023, 781, 218, 37, 4, 997, 5};
std::vector<IoBuffer> Chunks;
Chunks.reserve(9);
- for (const auto& Size : ChunkSizes)
+ for (uint64_t Size : ChunkSizes)
{
Chunks.push_back(CreateChunk(Size));
}
std::vector<IoHash> ChunkHashes;
ChunkHashes.reserve(9);
- for (const auto& Chunk : Chunks)
+ for (const IoBuffer& Chunk : Chunks)
{
ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size()));
}
@@ -1468,7 +1468,7 @@ TEST_CASE("compactcas.gc.compact")
CHECK(Cas.HaveChunk(ChunkHashes[7]));
CHECK(Cas.HaveChunk(ChunkHashes[8]));
- auto InitialSize = Cas.StorageSize().DiskSize;
+ uint64_t InitialSize = Cas.StorageSize().DiskSize;
// Keep first and last
{
@@ -1655,7 +1655,7 @@ TEST_CASE("compactcas.gc.compact")
CHECK(ChunkHashes[7] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[7])));
CHECK(ChunkHashes[8] == IoHash::HashBuffer(Cas.FindChunk(ChunkHashes[8])));
- auto FinalSize = Cas.StorageSize().DiskSize;
+ uint64_t FinalSize = Cas.StorageSize().DiskSize;
CHECK(InitialSize == FinalSize);
}
@@ -1666,14 +1666,14 @@ TEST_CASE("compactcas.gc.deleteblockonopen")
uint64_t ChunkSizes[20] = {128, 541, 311, 181, 218, 37, 4, 397, 5, 92, 551, 721, 31, 92, 16, 99, 131, 41, 541, 84};
std::vector<IoBuffer> Chunks;
Chunks.reserve(20);
- for (const auto& Size : ChunkSizes)
+ for (uint64_t Size : ChunkSizes)
{
Chunks.push_back(CreateChunk(Size));
}
std::vector<IoHash> ChunkHashes;
ChunkHashes.reserve(20);
- for (const auto& Chunk : Chunks)
+ for (const IoBuffer& Chunk : Chunks)
{
ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size()));
}
@@ -1735,14 +1735,14 @@ TEST_CASE("compactcas.gc.handleopeniobuffer")
uint64_t ChunkSizes[20] = {128, 541, 311, 181, 218, 37, 4, 397, 5, 92, 551, 721, 31, 92, 16, 99, 131, 41, 541, 84};
std::vector<IoBuffer> Chunks;
Chunks.reserve(20);
- for (const auto& Size : ChunkSizes)
+ for (const uint64_t& Size : ChunkSizes)
{
Chunks.push_back(CreateChunk(Size));
}
std::vector<IoHash> ChunkHashes;
ChunkHashes.reserve(20);
- for (const auto& Chunk : Chunks)
+ for (const IoBuffer& Chunk : Chunks)
{
ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size()));
}
@@ -1760,7 +1760,7 @@ TEST_CASE("compactcas.gc.handleopeniobuffer")
CHECK(Cas.InsertChunk(Chunks[i], ChunkHashes[i]).New);
}
- auto RetainChunk = Cas.FindChunk(ChunkHashes[5]);
+ IoBuffer RetainChunk = Cas.FindChunk(ChunkHashes[5]);
Cas.Flush();
// GC everything
@@ -1785,7 +1785,7 @@ TEST_CASE("compactcas.legacyconversion")
size_t SingleBlockSize = 0;
std::vector<IoBuffer> Chunks;
Chunks.reserve(ChunkCount);
- for (const auto& Size : ChunkSizes)
+ for (uint64_t Size : ChunkSizes)
{
Chunks.push_back(CreateChunk(Size));
SingleBlockSize += Size;
@@ -1793,7 +1793,7 @@ TEST_CASE("compactcas.legacyconversion")
std::vector<IoHash> ChunkHashes;
ChunkHashes.reserve(ChunkCount);
- for (const auto& Chunk : Chunks)
+ for (const IoBuffer& Chunk : Chunks)
{
ChunkHashes.push_back(IoHash::HashBuffer(Chunk.Data(), Chunk.Size()));
}
@@ -1859,7 +1859,7 @@ TEST_CASE("compactcas.legacyconversion")
}
TCasLogFile<LegacyCasDiskIndexEntry> LegacyCasLog;
LegacyCasLog.Open(SlogPath, true);
- for (const auto& Entry : LogEntries)
+ for (const CasDiskIndexEntry& Entry : LogEntries)
{
BlockStoreLocation Location = Entry.Location.Get(16);
LegacyCasDiskLocation LegacyLocation(Location.Offset, Location.Size);
@@ -1937,9 +1937,9 @@ TEST_CASE("compactcas.threadedinsert") // * doctest::skip(true))
for (int32_t Idx = 0; Idx < kChunkCount; ++Idx)
{
ThreadPool.ScheduleWork([&Cas, &OldChunkHashes, Idx]() {
- auto ChunkHash = OldChunkHashes[Idx];
- auto Chunk = Cas.FindChunk(ChunkHash);
- auto Hash = IoHash::HashBuffer(Chunk);
+ IoHash ChunkHash = OldChunkHashes[Idx];
+ IoBuffer Chunk = Cas.FindChunk(ChunkHash);
+ IoHash Hash = IoHash::HashBuffer(Chunk);
CHECK(ChunkHash == Hash);
});
}
@@ -1975,8 +1975,8 @@ TEST_CASE("compactcas.threadedinsert") // * doctest::skip(true))
AddedChunkCount.fetch_add(1);
});
ThreadPool.ScheduleWork([&Cas, &ChunkHashesLock, &OldChunkHashes, Idx]() {
- IoHash ChunkHash = OldChunkHashes[Idx];
- auto Chunk = Cas.FindChunk(OldChunkHashes[Idx]);
+ IoHash ChunkHash = OldChunkHashes[Idx];
+ IoBuffer Chunk = Cas.FindChunk(OldChunkHashes[Idx]);
if (Chunk)
{
CHECK(ChunkHash == IoHash::HashBuffer(Chunk));
@@ -1992,7 +1992,7 @@ TEST_CASE("compactcas.threadedinsert") // * doctest::skip(true))
AddedHashes.swap(NewChunkHashes);
}
// Need to be careful since we might GC blocks we don't know outside of RwLock::ExclusiveLockScope
- for (auto ChunkHash : AddedHashes)
+ for (const IoHash& ChunkHash : AddedHashes)
{
if (Cas.HaveChunk(ChunkHash))
{
@@ -2036,7 +2036,7 @@ TEST_CASE("compactcas.threadedinsert") // * doctest::skip(true))
AddedHashes.swap(NewChunkHashes);
}
// Need to be careful since we might GC blocks we don't know outside of RwLock::ExclusiveLockScope
- for (auto ChunkHash : AddedHashes)
+ for (const IoHash& ChunkHash : AddedHashes)
{
if (Cas.HaveChunk(ChunkHash))
{
@@ -2064,7 +2064,7 @@ TEST_CASE("compactcas.threadedinsert") // * doctest::skip(true))
}
}
{
- for (const auto& ChunkHash : GcChunkHashes)
+ for (const IoHash& ChunkHash : GcChunkHashes)
{
ThreadPool.ScheduleWork([&Cas, ChunkHash]() {
CHECK(Cas.HaveChunk(ChunkHash));
diff --git a/zenstore/compactcas.h b/zenstore/compactcas.h
index 7a200eeed..b5871d700 100644
--- a/zenstore/compactcas.h
+++ b/zenstore/compactcas.h
@@ -68,9 +68,9 @@ private:
const CasStoreConfiguration& m_Config;
spdlog::logger& m_Log;
- uint64_t m_PayloadAlignment;
- uint64_t m_MaxBlockSize;
- bool m_IsInitialized = false;
+ uint64_t m_PayloadAlignment = 1u << 4;
+ uint64_t m_MaxBlockSize = 1u << 28;
+ bool m_IsInitialized = false;
TCasLogFile<CasDiskIndexEntry> m_CasLog;
std::string m_ContainerBaseName;
std::filesystem::path m_BlocksBasePath;
@@ -80,9 +80,9 @@ private:
LocationMap_t m_LocationMap;
std::unordered_map<uint32_t, std::shared_ptr<BlockStoreFile>> m_ChunkBlocks;
- RwLock m_InsertLock; // used to serialize inserts
- std::weak_ptr<BlockStoreFile> m_WriteBlock;
- std::uint64_t m_CurrentInsertOffset{};
+ RwLock m_InsertLock; // used to serialize inserts
+ std::shared_ptr<BlockStoreFile> m_WriteBlock;
+ std::uint64_t m_CurrentInsertOffset = 0;
std::atomic_uint32_t m_WriteBlockIndex{};
std::atomic_uint64_t m_TotalSize{};
diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp
index a2bd4ce15..0f430f61c 100644
--- a/zenstore/gc.cpp
+++ b/zenstore/gc.cpp
@@ -662,7 +662,7 @@ TEST_CASE("gc.full")
CasStore->InsertChunk(Chunks[7], ChunkHashes[7]);
CasStore->InsertChunk(Chunks[8], ChunkHashes[8]);
- auto InitialSize = CasStore->TotalSize();
+ CasStoreSize InitialSize = CasStore->TotalSize();
// Keep first and last
{
diff --git a/zenstore/include/zenstore/blockstore.h b/zenstore/include/zenstore/blockstore.h
index 69ad38d87..306282665 100644
--- a/zenstore/include/zenstore/blockstore.h
+++ b/zenstore/include/zenstore/blockstore.h
@@ -27,19 +27,19 @@ struct BlockStoreDiskLocation
constexpr static uint32_t MaxBlockIndex = (1ul << BlockStoreDiskLocation::MaxBlockIndexBits) - 1ul;
constexpr static uint32_t MaxOffset = (1ul << BlockStoreDiskLocation::MaxOffsetBits) - 1ul;
- BlockStoreDiskLocation(const BlockStoreLocation& Location, uint64_t OffsetAlignement)
+ BlockStoreDiskLocation(const BlockStoreLocation& Location, uint64_t OffsetAlignment)
{
- Init(Location.BlockIndex, Location.Offset / OffsetAlignement, Location.Size);
+ Init(Location.BlockIndex, Location.Offset / OffsetAlignment, Location.Size);
}
BlockStoreDiskLocation() = default;
- inline BlockStoreLocation Get(uint64_t OffsetAlignement) const
+ inline BlockStoreLocation Get(uint64_t OffsetAlignment) const
{
uint64_t PackedOffset = 0;
memcpy(&PackedOffset, &m_Offset, sizeof m_Offset);
return {.BlockIndex = static_cast<std::uint32_t>(PackedOffset >> MaxOffsetBits),
- .Offset = (PackedOffset & MaxOffset) * OffsetAlignement,
+ .Offset = (PackedOffset & MaxOffset) * OffsetAlignment,
.Size = GetSize()};
}