From 769dc100be423f43df52e24b78b210a969520478 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 11 Mar 2022 12:17:46 +0100 Subject: WIP --- zenstore/gc.cpp | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) (limited to 'zenstore/gc.cpp') diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index 3b090cae9..0f15eef2c 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -618,6 +618,59 @@ TEST_CASE("gc.basic") CHECK(!CidStore.ContainsChunk(InsertResult.DecompressedId)); } +TEST_CASE("gc.compact") +{ + ScopedTemporaryDirectory TempDir; + + CasStoreConfiguration CasConfig; + CasConfig.RootDirectory = TempDir.Path() / "cas"; + + CasGc Gc; + std::unique_ptr CasStore = CreateCasStore(Gc); + CidStore CidStore{*CasStore, TempDir.Path() / "cid"}; + + CasStore->Initialize(CasConfig); + Gc.SetCidStore(&CidStore); + + uint64_t ChunkSizes[9] = {128, 541, 5123, 1293, 218, 37, 4, 4095, 5}; + CompressedBuffer CompressedChunks[9] = {Compress(CreateChunk(ChunkSizes[0])), + Compress(CreateChunk(ChunkSizes[1])), + Compress(CreateChunk(ChunkSizes[2])), + Compress(CreateChunk(ChunkSizes[3])), + Compress(CreateChunk(ChunkSizes[4])), + Compress(CreateChunk(ChunkSizes[5])), + Compress(CreateChunk(ChunkSizes[6])), + Compress(CreateChunk(ChunkSizes[7])), + Compress(CreateChunk(ChunkSizes[8]))}; + + CidStore::InsertResult InsertResults[9] = { + CidStore.AddChunk(CompressedChunks[0]), + CidStore.AddChunk(CompressedChunks[1]), + CidStore.AddChunk(CompressedChunks[2]), + CidStore.AddChunk(CompressedChunks[3]), + CidStore.AddChunk(CompressedChunks[4]), + CidStore.AddChunk(CompressedChunks[5]), + CidStore.AddChunk(CompressedChunks[6]), + CidStore.AddChunk(CompressedChunks[7]), + CidStore.AddChunk(CompressedChunks[8]), + }; + + GcContext GcCtx; + GcCtx.CollectSmallObjects(true); + + Gc.CollectGarbage(GcCtx); + + CHECK(!CidStore.ContainsChunk(InsertResults[0].DecompressedId)); + CHECK(!CidStore.ContainsChunk(InsertResults[1].DecompressedId)); + CHECK(!CidStore.ContainsChunk(InsertResults[2].DecompressedId)); + CHECK(!CidStore.ContainsChunk(InsertResults[3].DecompressedId)); + CHECK(!CidStore.ContainsChunk(InsertResults[4].DecompressedId)); + CHECK(!CidStore.ContainsChunk(InsertResults[5].DecompressedId)); + CHECK(!CidStore.ContainsChunk(InsertResults[6].DecompressedId)); + CHECK(!CidStore.ContainsChunk(InsertResults[7].DecompressedId)); + CHECK(!CidStore.ContainsChunk(InsertResults[8].DecompressedId)); +} + #endif void -- cgit v1.2.3 From 632d221363979c7f34b90cb44eee971da9cca5b3 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 11 Mar 2022 16:11:52 +0100 Subject: More tests --- zenstore/gc.cpp | 208 ++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 165 insertions(+), 43 deletions(-) (limited to 'zenstore/gc.cpp') diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index 0f15eef2c..b685200bc 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -573,16 +573,15 @@ namespace { static std::random_device rd; static std::mt19937 g(rd()); - const size_t Count = static_cast(Size / sizeof(uint32_t)); - std::vector Values; - Values.resize(Count); - for (size_t Idx = 0; Idx < Count; ++Idx) + std::vector Values; + Values.resize(Size); + for (size_t Idx = 0; Idx < Size; ++Idx) { - Values[Idx] = static_cast(Idx); + Values[Idx] = static_cast(Idx); } std::shuffle(Values.begin(), Values.end(), g); - return IoBufferBuilder::MakeCloneFromMemory(Values.data(), Values.size() * sizeof(uint32_t)); + return IoBufferBuilder::MakeCloneFromMemory(Values.data(), Values.size()); } static CompressedBuffer Compress(IoBuffer Buffer) @@ -627,50 +626,173 @@ TEST_CASE("gc.compact") CasGc Gc; std::unique_ptr CasStore = CreateCasStore(Gc); - CidStore CidStore{*CasStore, TempDir.Path() / "cid"}; CasStore->Initialize(CasConfig); - Gc.SetCidStore(&CidStore); - uint64_t ChunkSizes[9] = {128, 541, 5123, 1293, 218, 37, 4, 4095, 5}; - CompressedBuffer CompressedChunks[9] = {Compress(CreateChunk(ChunkSizes[0])), - Compress(CreateChunk(ChunkSizes[1])), - Compress(CreateChunk(ChunkSizes[2])), - Compress(CreateChunk(ChunkSizes[3])), - Compress(CreateChunk(ChunkSizes[4])), - Compress(CreateChunk(ChunkSizes[5])), - Compress(CreateChunk(ChunkSizes[6])), - Compress(CreateChunk(ChunkSizes[7])), - Compress(CreateChunk(ChunkSizes[8]))}; - - CidStore::InsertResult InsertResults[9] = { - CidStore.AddChunk(CompressedChunks[0]), - CidStore.AddChunk(CompressedChunks[1]), - CidStore.AddChunk(CompressedChunks[2]), - CidStore.AddChunk(CompressedChunks[3]), - CidStore.AddChunk(CompressedChunks[4]), - CidStore.AddChunk(CompressedChunks[5]), - CidStore.AddChunk(CompressedChunks[6]), - CidStore.AddChunk(CompressedChunks[7]), - CidStore.AddChunk(CompressedChunks[8]), - }; + uint64_t ChunkSizes[9] = {128, 541, 1023, 781, 218, 37, 4, 997, 5}; + IoBuffer Chunks[9] = {CreateChunk(ChunkSizes[0]), + CreateChunk(ChunkSizes[1]), + CreateChunk(ChunkSizes[2]), + CreateChunk(ChunkSizes[3]), + CreateChunk(ChunkSizes[4]), + CreateChunk(ChunkSizes[5]), + CreateChunk(ChunkSizes[6]), + CreateChunk(ChunkSizes[7]), + CreateChunk(ChunkSizes[8])}; + IoHash ChunkHashes[9] = { + IoHash::HashBuffer(Chunks[0].Data(), Chunks[0].Size()), + IoHash::HashBuffer(Chunks[1].Data(), Chunks[1].Size()), + IoHash::HashBuffer(Chunks[2].Data(), Chunks[2].Size()), + IoHash::HashBuffer(Chunks[3].Data(), Chunks[3].Size()), + IoHash::HashBuffer(Chunks[4].Data(), Chunks[4].Size()), + IoHash::HashBuffer(Chunks[5].Data(), Chunks[5].Size()), + IoHash::HashBuffer(Chunks[6].Data(), Chunks[6].Size()), + IoHash::HashBuffer(Chunks[7].Data(), Chunks[7].Size()), + IoHash::HashBuffer(Chunks[8].Data(), Chunks[8].Size()), + }; + + CasStore->InsertChunk(Chunks[0], ChunkHashes[0]); + CasStore->InsertChunk(Chunks[1], ChunkHashes[1]); + CasStore->InsertChunk(Chunks[2], ChunkHashes[2]); + CasStore->InsertChunk(Chunks[3], ChunkHashes[3]); + CasStore->InsertChunk(Chunks[4], ChunkHashes[4]); + CasStore->InsertChunk(Chunks[5], ChunkHashes[5]); + CasStore->InsertChunk(Chunks[6], ChunkHashes[6]); + CasStore->InsertChunk(Chunks[7], ChunkHashes[7]); + CasStore->InsertChunk(Chunks[8], ChunkHashes[8]); + + // Keep first and last + { + GcContext GcCtx; + GcCtx.CollectSmallObjects(true); + + std::vector KeepChunks; + KeepChunks.push_back(ChunkHashes[0]); + KeepChunks.push_back(ChunkHashes[8]); + GcCtx.ContributeCas(KeepChunks); + + Gc.CollectGarbage(GcCtx); + + CHECK(CasStore->ContainsChunk(ChunkHashes[0])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[1])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[2])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[3])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[4])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[5])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[6])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[7])); + CHECK(CasStore->ContainsChunk(ChunkHashes[8])); + + CHECK(ChunkHashes[0] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[0]))); + CHECK(ChunkHashes[8] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[8]))); + } - GcContext GcCtx; - GcCtx.CollectSmallObjects(true); + CasStore->InsertChunk(Chunks[1], ChunkHashes[1]); + CasStore->InsertChunk(Chunks[2], ChunkHashes[2]); + CasStore->InsertChunk(Chunks[3], ChunkHashes[3]); + CasStore->InsertChunk(Chunks[4], ChunkHashes[4]); + CasStore->InsertChunk(Chunks[5], ChunkHashes[5]); + CasStore->InsertChunk(Chunks[6], ChunkHashes[6]); + CasStore->InsertChunk(Chunks[7], ChunkHashes[7]); - Gc.CollectGarbage(GcCtx); + // Keep last + { + GcContext GcCtx; + GcCtx.CollectSmallObjects(true); + std::vector KeepChunks; + KeepChunks.push_back(ChunkHashes[8]); + GcCtx.ContributeCas(KeepChunks); + + Gc.CollectGarbage(GcCtx); + + CHECK(!CasStore->ContainsChunk(ChunkHashes[0])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[1])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[2])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[3])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[4])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[5])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[6])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[7])); + CHECK(CasStore->ContainsChunk(ChunkHashes[8])); + + CHECK(ChunkHashes[8] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[8]))); + + CasStore->InsertChunk(Chunks[1], ChunkHashes[1]); + CasStore->InsertChunk(Chunks[2], ChunkHashes[2]); + CasStore->InsertChunk(Chunks[3], ChunkHashes[3]); + CasStore->InsertChunk(Chunks[4], ChunkHashes[4]); + CasStore->InsertChunk(Chunks[5], ChunkHashes[5]); + CasStore->InsertChunk(Chunks[6], ChunkHashes[6]); + CasStore->InsertChunk(Chunks[7], ChunkHashes[7]); + } - CHECK(!CidStore.ContainsChunk(InsertResults[0].DecompressedId)); - CHECK(!CidStore.ContainsChunk(InsertResults[1].DecompressedId)); - CHECK(!CidStore.ContainsChunk(InsertResults[2].DecompressedId)); - CHECK(!CidStore.ContainsChunk(InsertResults[3].DecompressedId)); - CHECK(!CidStore.ContainsChunk(InsertResults[4].DecompressedId)); - CHECK(!CidStore.ContainsChunk(InsertResults[5].DecompressedId)); - CHECK(!CidStore.ContainsChunk(InsertResults[6].DecompressedId)); - CHECK(!CidStore.ContainsChunk(InsertResults[7].DecompressedId)); - CHECK(!CidStore.ContainsChunk(InsertResults[8].DecompressedId)); -} + // Keep mixed + { + GcContext GcCtx; + GcCtx.CollectSmallObjects(true); + std::vector KeepChunks; + KeepChunks.push_back(ChunkHashes[1]); + KeepChunks.push_back(ChunkHashes[4]); + KeepChunks.push_back(ChunkHashes[7]); + GcCtx.ContributeCas(KeepChunks); + + Gc.CollectGarbage(GcCtx); + + CHECK(!CasStore->ContainsChunk(ChunkHashes[0])); + CHECK(CasStore->ContainsChunk(ChunkHashes[1])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[2])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[3])); + CHECK(CasStore->ContainsChunk(ChunkHashes[4])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[5])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[6])); + CHECK(CasStore->ContainsChunk(ChunkHashes[7])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[8])); + + CHECK(ChunkHashes[1] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[1]))); + CHECK(ChunkHashes[4] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[4]))); + CHECK(ChunkHashes[7] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[7]))); + + CasStore->InsertChunk(Chunks[0], ChunkHashes[0]); + CasStore->InsertChunk(Chunks[2], ChunkHashes[2]); + CasStore->InsertChunk(Chunks[3], ChunkHashes[3]); + CasStore->InsertChunk(Chunks[5], ChunkHashes[5]); + CasStore->InsertChunk(Chunks[6], ChunkHashes[6]); + CasStore->InsertChunk(Chunks[8], ChunkHashes[8]); + } + // Keep multiple at end + { + GcContext GcCtx; + GcCtx.CollectSmallObjects(true); + std::vector KeepChunks; + KeepChunks.push_back(ChunkHashes[6]); + KeepChunks.push_back(ChunkHashes[7]); + KeepChunks.push_back(ChunkHashes[8]); + GcCtx.ContributeCas(KeepChunks); + + Gc.CollectGarbage(GcCtx); + + CHECK(!CasStore->ContainsChunk(ChunkHashes[0])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[1])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[2])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[3])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[4])); + CHECK(!CasStore->ContainsChunk(ChunkHashes[5])); + CHECK(CasStore->ContainsChunk(ChunkHashes[6])); + CHECK(CasStore->ContainsChunk(ChunkHashes[7])); + CHECK(CasStore->ContainsChunk(ChunkHashes[8])); + + CHECK(ChunkHashes[6] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[6]))); + CHECK(ChunkHashes[7] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[7]))); + CHECK(ChunkHashes[8] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[8]))); + + CasStore->InsertChunk(Chunks[0], ChunkHashes[0]); + CasStore->InsertChunk(Chunks[1], ChunkHashes[1]); + CasStore->InsertChunk(Chunks[2], ChunkHashes[2]); + CasStore->InsertChunk(Chunks[3], ChunkHashes[3]); + CasStore->InsertChunk(Chunks[5], ChunkHashes[5]); + } +} #endif void -- cgit v1.2.3 From f2074aa777decc5104b548b3a1dbe44434bc177f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 11 Mar 2022 23:34:43 +0100 Subject: Fix insert offset when moving last chunk --- zenstore/gc.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'zenstore/gc.cpp') diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index b685200bc..8a9fceb65 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -661,6 +661,8 @@ TEST_CASE("gc.compact") CasStore->InsertChunk(Chunks[7], ChunkHashes[7]); CasStore->InsertChunk(Chunks[8], ChunkHashes[8]); + auto InitialSize = CasStore->TotalSize(); + // Keep first and last { GcContext GcCtx; @@ -790,8 +792,23 @@ TEST_CASE("gc.compact") CasStore->InsertChunk(Chunks[1], ChunkHashes[1]); CasStore->InsertChunk(Chunks[2], ChunkHashes[2]); CasStore->InsertChunk(Chunks[3], ChunkHashes[3]); + CasStore->InsertChunk(Chunks[4], ChunkHashes[4]); CasStore->InsertChunk(Chunks[5], ChunkHashes[5]); } + + // Verify that we nicely appended blocks even after all GC operations + CHECK(ChunkHashes[0] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[0]))); + CHECK(ChunkHashes[1] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[1]))); + CHECK(ChunkHashes[2] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[2]))); + CHECK(ChunkHashes[3] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[3]))); + CHECK(ChunkHashes[4] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[4]))); + CHECK(ChunkHashes[5] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[5]))); + CHECK(ChunkHashes[6] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[6]))); + CHECK(ChunkHashes[7] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[7]))); + CHECK(ChunkHashes[8] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[8]))); + + auto FinalSize = CasStore->TotalSize(); + CHECK(InitialSize.TinySize == FinalSize.TinySize); } #endif -- cgit v1.2.3 From 9082df7cc729af9d0c75f1ed4241b687ac9b9cf3 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 15 Mar 2022 13:15:27 +0100 Subject: cleanup --- zenstore/gc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenstore/gc.cpp') diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index 8a9fceb65..853f2a3db 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -617,7 +617,7 @@ TEST_CASE("gc.basic") CHECK(!CidStore.ContainsChunk(InsertResult.DecompressedId)); } -TEST_CASE("gc.compact") +TEST_CASE("gc.full") { ScopedTemporaryDirectory TempDir; -- cgit v1.2.3 From f4d040e2c0f602c34af47614907fecd8f6229316 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 22 Mar 2022 17:34:11 +0100 Subject: WIP --- zenstore/gc.cpp | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'zenstore/gc.cpp') diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index 853f2a3db..a2bd4ce15 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -612,6 +612,7 @@ TEST_CASE("gc.basic") GcContext GcCtx; GcCtx.CollectSmallObjects(true); + CasStore->Flush(); Gc.CollectGarbage(GcCtx); CHECK(!CidStore.ContainsChunk(InsertResult.DecompressedId)); @@ -673,6 +674,7 @@ TEST_CASE("gc.full") KeepChunks.push_back(ChunkHashes[8]); GcCtx.ContributeCas(KeepChunks); + CasStore->Flush(); Gc.CollectGarbage(GcCtx); CHECK(CasStore->ContainsChunk(ChunkHashes[0])); @@ -705,6 +707,7 @@ TEST_CASE("gc.full") KeepChunks.push_back(ChunkHashes[8]); GcCtx.ContributeCas(KeepChunks); + CasStore->Flush(); Gc.CollectGarbage(GcCtx); CHECK(!CasStore->ContainsChunk(ChunkHashes[0])); @@ -738,6 +741,7 @@ TEST_CASE("gc.full") KeepChunks.push_back(ChunkHashes[7]); GcCtx.ContributeCas(KeepChunks); + CasStore->Flush(); Gc.CollectGarbage(GcCtx); CHECK(!CasStore->ContainsChunk(ChunkHashes[0])); @@ -772,6 +776,7 @@ TEST_CASE("gc.full") KeepChunks.push_back(ChunkHashes[8]); GcCtx.ContributeCas(KeepChunks); + CasStore->Flush(); Gc.CollectGarbage(GcCtx); CHECK(!CasStore->ContainsChunk(ChunkHashes[0])); -- cgit v1.2.3 From a24ebc13bd3321eaef5757f6b42fd7fa2ebdfcc0 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 23 Mar 2022 17:20:39 +0100 Subject: review feedback --- zenstore/gc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenstore/gc.cpp') diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index a2bd4ce15..0f430f61c 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -662,7 +662,7 @@ TEST_CASE("gc.full") CasStore->InsertChunk(Chunks[7], ChunkHashes[7]); CasStore->InsertChunk(Chunks[8], ChunkHashes[8]); - auto InitialSize = CasStore->TotalSize(); + CasStoreSize InitialSize = CasStore->TotalSize(); // Keep first and last { -- cgit v1.2.3 From d7331a809702551446496e1a0e86b9fcea42c0e3 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 31 Mar 2022 11:18:33 +0200 Subject: Improved GC logging --- zenstore/gc.cpp | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) (limited to 'zenstore/gc.cpp') diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index 0f430f61c..dee8c209f 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -262,10 +263,13 @@ CasGc::CollectGarbage(GcContext& GcCtx) RwLock::SharedLockScope _(m_Lock); // First gather reference set - - for (GcContributor* Contributor : m_GcContribs) { - Contributor->GatherReferences(GcCtx); + Stopwatch Timer; + const auto Guard = MakeGuard([this, &Timer] { ZEN_INFO("gathered references in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); + for (GcContributor* Contributor : m_GcContribs) + { + Contributor->GatherReferences(GcCtx); + } } // Cache records reference CAS chunks with the uncompressed @@ -300,15 +304,22 @@ CasGc::CollectGarbage(GcContext& GcCtx) // Then trim storage - for (GcStorage* Storage : m_GcStorage) { - Storage->CollectGarbage(GcCtx); + Stopwatch Timer; + const auto Guard = MakeGuard([this, &Timer] { ZEN_INFO("collected garbage in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); + for (GcStorage* Storage : m_GcStorage) + { + Storage->CollectGarbage(GcCtx); + } } // Remove Cid to CAS hash mappings. Scrub? if (CidStore* CidStore = m_CidStore) { + Stopwatch Timer; + const auto Guard = + MakeGuard([this, &Timer] { ZEN_INFO("clean up deleted content ids in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); CidStore->RemoveCids(GcCtx.DeletedCas()); } } -- cgit v1.2.3 From ef8cb5f11168183be77b1c121a9171b2ea9fbaab Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 1 Apr 2022 15:20:42 +0200 Subject: Make gc reserve a global resource --- zenstore/gc.cpp | 177 +++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 163 insertions(+), 14 deletions(-) (limited to 'zenstore/gc.cpp') diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index dee8c209f..ab7eece20 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,15 @@ #include #include +#if ZEN_PLATFORM_WINDOWS +# include +#else +# include +# include +# include +# include +#endif + #if ZEN_WITH_TESTS # include # include @@ -32,6 +42,102 @@ namespace fs = std::filesystem; ////////////////////////////////////////////////////////////////////////// +namespace { + std::error_code CreateGCReserve(const std::filesystem::path& Path, uint64_t Size) + { + if (Size == 0) + { + std::filesystem::remove(Path); + return std::error_code{}; + } + CreateDirectories(Path.parent_path()); + if (std::filesystem::is_regular_file(Path) && std::filesystem::file_size(Path) == Size) + { + return std::error_code(); + } +#if ZEN_PLATFORM_WINDOWS + DWORD dwCreationDisposition = CREATE_ALWAYS; + DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; + + const DWORD dwShareMode = 0; + const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; + HANDLE hTemplateFile = nullptr; + + HANDLE FileHandle = CreateFile(Path.c_str(), + dwDesiredAccess, + dwShareMode, + /* lpSecurityAttributes */ nullptr, + dwCreationDisposition, + dwFlagsAndAttributes, + hTemplateFile); + + if (FileHandle == INVALID_HANDLE_VALUE) + { + return zen::MakeErrorCodeFromLastError(); + } + bool Keep = true; + auto _ = MakeGuard([FileHandle, &Keep, Path]() { + ::CloseHandle(FileHandle); + if (!Keep) + { + ::DeleteFile(Path.c_str()); + } + }); + LARGE_INTEGER liFileSize; + liFileSize.QuadPart = Size; + BOOL OK = ::SetFilePointerEx(FileHandle, liFileSize, 0, FILE_BEGIN); + if (!OK) + { + return zen::MakeErrorCodeFromLastError(); + } + OK = ::SetEndOfFile(FileHandle); + if (!OK) + { + return zen::MakeErrorCodeFromLastError(); + } + Keep = true; +#else + int OpenFlags = O_CLOEXEC | O_RDWR | O_CREAT; + int Fd = open(Path.c_str(), OpenFlags, 0666); + if (Fd < 0) + { + return zen::MakeErrorCodeFromLastError(); + } + + bool Keep = true; + auto _ = MakeGuard([Fd, &Keep, Path]() { + close(Fd); + if (!Keep) + { + unlink(Path.c_str()); + } + }); + + if (fchmod(Fd, 0666) < 0) + { + return zen::MakeErrorCodeFromLastError(); + } + +# if ZEN_PLATFORM_MAC + if (ftruncate(Fd, (off_t)FileSize) < 0) + { + return zen::MakeErrorCodeFromLastError(); + } +# else + if (ftruncate64(Fd, (off64_t)FileSize) < 0) + { + return zen::MakeErrorCodeFromLastError(); + } +# endif + Keep = true; +#endif + return std::error_code{}; + } + +} // namespace + +////////////////////////////////////////////////////////////////////////// + CbObject LoadCompactBinaryObject(const fs::path& Path) { @@ -75,6 +181,8 @@ struct GcContext::GcState GcClock::Duration m_MaxCacheDuration = std::chrono::hours(24); bool m_DeletionMode = true; bool m_CollectSmallObjects = false; + + std::filesystem::path DiskReservePath; }; GcContext::GcContext(GcClock::TimePoint Time) : m_State(std::make_unique()) @@ -195,6 +303,27 @@ GcContext::MaxCacheDuration(GcClock::Duration Duration) m_State->m_MaxCacheDuration = Duration; } +void +GcContext::DiskReservePath(const std::filesystem::path& Path) +{ + m_State->DiskReservePath = Path; +} + +uint64_t +GcContext::ClaimGCReserve() +{ + if (!std::filesystem::is_regular_file(m_State->DiskReservePath)) + { + return 0; + } + uint64_t ReclaimedSize = std::filesystem::file_size(m_State->DiskReservePath); + if (std::filesystem::remove(m_State->DiskReservePath)) + { + return ReclaimedSize; + } + return 0; +} + ////////////////////////////////////////////////////////////////////////// GcContributor::GcContributor(CasGc& Gc) : m_Gc(Gc) @@ -390,6 +519,15 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config) std::filesystem::create_directories(Config.RootDirectory); + std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize); + if (Ec) + { + ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason '{}'", + m_Config.RootDirectory / "reserve.gc", + NiceBytes(m_Config.DiskReserveSize), + Ec.message()); + } + m_LastGcTime = GcClock::Now(); if (CbObject SchedulerState = LoadCompactBinaryObject(Config.RootDirectory / "gc_state")) @@ -517,6 +655,7 @@ GcScheduler::SchedulerThread() GcCtx.SetDeletionMode(true); GcCtx.CollectSmallObjects(m_Config.CollectSmallObjects); GcCtx.MaxCacheDuration(m_Config.MaxCacheDuration); + GcCtx.DiskReservePath(m_Config.RootDirectory / "reserve.gc"); if (m_TriggerParams) { @@ -530,27 +669,37 @@ GcScheduler::SchedulerThread() } } - Stopwatch Timer; - ZEN_INFO("garbage collection STARTING, small objects gc {}, max cache duration {}", GcCtx.CollectSmallObjects() ? "ENABLED"sv : "DISABLED"sv, NiceTimeSpanMs(uint64_t(std::chrono::duration_cast(GcCtx.MaxCacheDuration()).count()))); + { + Stopwatch Timer; + const auto __ = + MakeGuard([this, &Timer] { ZEN_INFO("garbage collection DONE after {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); - m_CasGc.CollectGarbage(GcCtx); + m_CasGc.CollectGarbage(GcCtx); - m_LastGcTime = GcClock::Now(); - m_NextGcTime = NextGcTime(m_LastGcTime); - WaitTime = m_Config.MonitorInterval; + m_LastGcTime = GcClock::Now(); + m_NextGcTime = NextGcTime(m_LastGcTime); + WaitTime = m_Config.MonitorInterval; - { - const fs::path Path = m_Config.RootDirectory / "gc_state"; - ZEN_DEBUG("saving scheduler state to '{}'", Path); - CbObjectWriter SchedulderState; - SchedulderState << "LastGcTime"sv << static_cast(m_LastGcTime.time_since_epoch().count()); - SaveCompactBinaryObject(Path, SchedulderState.Save()); - } + { + const fs::path Path = m_Config.RootDirectory / "gc_state"; + ZEN_DEBUG("saving scheduler state to '{}'", Path); + CbObjectWriter SchedulderState; + SchedulderState << "LastGcTime"sv << static_cast(m_LastGcTime.time_since_epoch().count()); + SaveCompactBinaryObject(Path, SchedulderState.Save()); + } - ZEN_INFO("garbage collection DONE after {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize); + if (Ec) + { + ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason '{}'", + m_Config.RootDirectory / "reserve.gc", + NiceBytes(m_Config.DiskReserveSize), + Ec.message()); + } + } uint32_t RunningState = static_cast(GcSchedulerStatus::kRunning); if (!m_Status.compare_exchange_strong(RunningState, static_cast(GcSchedulerStatus::kIdle))) -- cgit v1.2.3 From 98c7fd153f4d25f13b8e4bc81648b02f80698c93 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 1 Apr 2022 15:39:49 +0200 Subject: make sure we allocate disk space when we set file size --- zenstore/gc.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'zenstore/gc.cpp') diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index ab7eece20..a37fc2f5b 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -123,11 +123,19 @@ namespace { { return zen::MakeErrorCodeFromLastError(); } + if (posix_fallocate(Fd, 0, (off_t)FileSize) < 0) + { + return zen::MakeErrorCodeFromLastError(); + } # else if (ftruncate64(Fd, (off64_t)FileSize) < 0) { return zen::MakeErrorCodeFromLastError(); } + if (posix_fallocate64(Fd, 0, (off64_t)FileSize) < 0) + { + return zen::MakeErrorCodeFromLastError(); + } # endif Keep = true; #endif -- cgit v1.2.3 From e51efe1674fa3d9d4af715d85a4045e1ee7450f3 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 1 Apr 2022 22:47:46 +0200 Subject: linux compilation fixes --- zenstore/gc.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'zenstore/gc.cpp') diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index a37fc2f5b..a86bae143 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -119,20 +119,20 @@ namespace { } # if ZEN_PLATFORM_MAC - if (ftruncate(Fd, (off_t)FileSize) < 0) + if (ftruncate(Fd, (off_t)Size) < 0) { return zen::MakeErrorCodeFromLastError(); } - if (posix_fallocate(Fd, 0, (off_t)FileSize) < 0) + if (posix_fallocate(Fd, 0, (off_t)Size) < 0) { return zen::MakeErrorCodeFromLastError(); } # else - if (ftruncate64(Fd, (off64_t)FileSize) < 0) + if (ftruncate64(Fd, (off64_t)Size) < 0) { return zen::MakeErrorCodeFromLastError(); } - if (posix_fallocate64(Fd, 0, (off64_t)FileSize) < 0) + if (posix_fallocate64(Fd, 0, (off64_t)Size) < 0) { return zen::MakeErrorCodeFromLastError(); } -- cgit v1.2.3 From 6e9f6df8e4d51852327d05b254813b7b792f40a8 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 1 Apr 2022 23:46:04 +0200 Subject: error handling in BasicFile::SetFileSize --- zenstore/gc.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'zenstore/gc.cpp') diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index a86bae143..117fe3b49 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -123,18 +123,20 @@ namespace { { return zen::MakeErrorCodeFromLastError(); } - if (posix_fallocate(Fd, 0, (off_t)Size) < 0) + std::error_code Ec = posix_fallocate(Fd, 0, (off_t)Size); + if (Ec) { - return zen::MakeErrorCodeFromLastError(); + return Ec; } # else if (ftruncate64(Fd, (off64_t)Size) < 0) { return zen::MakeErrorCodeFromLastError(); } - if (posix_fallocate64(Fd, 0, (off64_t)Size) < 0) + std::error_code Ec = posix_fallocate64(Fd, 0, (off64_t)Size); + if (Ec) { - return zen::MakeErrorCodeFromLastError(); + return Ec; } # endif Keep = true; -- cgit v1.2.3 From c2603f19ae2ac57b68eb8d6187cda668c70f2b2b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Sat, 2 Apr 2022 00:08:26 +0200 Subject: proper error handling when setting file size --- zenstore/gc.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'zenstore/gc.cpp') diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index 117fe3b49..311f7ced4 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -73,7 +73,7 @@ namespace { if (FileHandle == INVALID_HANDLE_VALUE) { - return zen::MakeErrorCodeFromLastError(); + return MakeErrorCodeFromLastError(); } bool Keep = true; auto _ = MakeGuard([FileHandle, &Keep, Path]() { @@ -88,12 +88,12 @@ namespace { BOOL OK = ::SetFilePointerEx(FileHandle, liFileSize, 0, FILE_BEGIN); if (!OK) { - return zen::MakeErrorCodeFromLastError(); + return MakeErrorCodeFromLastError(); } OK = ::SetEndOfFile(FileHandle); if (!OK) { - return zen::MakeErrorCodeFromLastError(); + return MakeErrorCodeFromLastError(); } Keep = true; #else @@ -101,7 +101,7 @@ namespace { int Fd = open(Path.c_str(), OpenFlags, 0666); if (Fd < 0) { - return zen::MakeErrorCodeFromLastError(); + return MakeErrorCodeFromLastError(); } bool Keep = true; @@ -115,13 +115,13 @@ namespace { if (fchmod(Fd, 0666) < 0) { - return zen::MakeErrorCodeFromLastError(); + return MakeErrorCodeFromLastError(); } # if ZEN_PLATFORM_MAC if (ftruncate(Fd, (off_t)Size) < 0) { - return zen::MakeErrorCodeFromLastError(); + return MakeErrorCodeFromLastError(); } std::error_code Ec = posix_fallocate(Fd, 0, (off_t)Size); if (Ec) @@ -131,7 +131,7 @@ namespace { # else if (ftruncate64(Fd, (off64_t)Size) < 0) { - return zen::MakeErrorCodeFromLastError(); + return MakeErrorCodeFromLastError(); } std::error_code Ec = posix_fallocate64(Fd, 0, (off64_t)Size); if (Ec) -- cgit v1.2.3 From 35ab521ce481d25215582c9fea2dacb98e0820c1 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Sat, 2 Apr 2022 00:12:03 +0200 Subject: more linux fixes --- zenstore/gc.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'zenstore/gc.cpp') diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index 311f7ced4..ac9beac63 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -123,20 +123,20 @@ namespace { { return MakeErrorCodeFromLastError(); } - std::error_code Ec = posix_fallocate(Fd, 0, (off_t)Size); - if (Ec) + int Error = posix_fallocate(Fd, 0, (off_t)Size); + if (Error) { - return Ec; + return MakeErrorCode(Error); } # else if (ftruncate64(Fd, (off64_t)Size) < 0) { return MakeErrorCodeFromLastError(); } - std::error_code Ec = posix_fallocate64(Fd, 0, (off64_t)Size); - if (Ec) + int Error = posix_fallocate64(Fd, 0, (off64_t)Size); + if (Error) { - return Ec; + return MakeErrorCode(Error); } # endif Keep = true; -- cgit v1.2.3 From f4901e91cc43346b7904438901adb7d9b8f950ef Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 4 Apr 2022 10:31:15 +0200 Subject: logging cleanup --- zenstore/gc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'zenstore/gc.cpp') diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp index ac9beac63..287dfb48a 100644 --- a/zenstore/gc.cpp +++ b/zenstore/gc.cpp @@ -634,7 +634,7 @@ GcScheduler::SchedulerThread() if (Ec) { - ZEN_WARN("get disk space info FAILED, reason '{}'", Ec.message()); + ZEN_WARN("get disk space info FAILED, reason: '{}'", Ec.message()); } ZEN_INFO("{} in use, {} of total {} free disk space, {}", @@ -704,7 +704,7 @@ GcScheduler::SchedulerThread() std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize); if (Ec) { - ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason '{}'", + ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason: '{}'", m_Config.RootDirectory / "reserve.gc", NiceBytes(m_Config.DiskReserveSize), Ec.message()); -- cgit v1.2.3