aboutsummaryrefslogtreecommitdiff
path: root/zenstore/gc.cpp
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2023-05-02 10:01:47 +0200
committerGitHub <[email protected]>2023-05-02 10:01:47 +0200
commit075d17f8ada47e990fe94606c3d21df409223465 (patch)
treee50549b766a2f3c354798a54ff73404217b4c9af /zenstore/gc.cpp
parentfix: bundle shouldn't append content zip to zen (diff)
downloadzen-075d17f8ada47e990fe94606c3d21df409223465.tar.xz
zen-075d17f8ada47e990fe94606c3d21df409223465.zip
moved source directories into `/src` (#264)
* moved source directories into `/src` * updated bundle.lua for new `src` path * moved some docs, icon * removed old test trees
Diffstat (limited to 'zenstore/gc.cpp')
-rw-r--r--zenstore/gc.cpp1312
1 files changed, 0 insertions, 1312 deletions
diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp
deleted file mode 100644
index 370c3c965..000000000
--- a/zenstore/gc.cpp
+++ /dev/null
@@ -1,1312 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include <zenstore/gc.h>
-
-#include <zencore/compactbinary.h>
-#include <zencore/compactbinarybuilder.h>
-#include <zencore/compactbinaryvalidation.h>
-#include <zencore/except.h>
-#include <zencore/filesystem.h>
-#include <zencore/fmtutils.h>
-#include <zencore/logging.h>
-#include <zencore/scopeguard.h>
-#include <zencore/string.h>
-#include <zencore/testing.h>
-#include <zencore/testutils.h>
-#include <zencore/timer.h>
-#include <zenstore/cidstore.h>
-
-#include "cas.h"
-
-#include <fmt/format.h>
-#include <filesystem>
-
-#if ZEN_PLATFORM_WINDOWS
-# include <zencore/windows.h>
-#else
-# include <fcntl.h>
-# include <sys/file.h>
-# include <sys/stat.h>
-# include <unistd.h>
-#endif
-
-#if ZEN_WITH_TESTS
-# include <zencore/compress.h>
-# include <algorithm>
-# include <random>
-#endif
-
-template<>
-struct fmt::formatter<zen::GcClock::TimePoint> : formatter<string_view>
-{
- template<typename FormatContext>
- auto format(const zen::GcClock::TimePoint& TimePoint, FormatContext& ctx)
- {
- std::time_t Time = std::chrono::system_clock::to_time_t(TimePoint);
- zen::ExtendableStringBuilder<32> String;
- String << std::ctime(&Time);
- return formatter<string_view>::format(String.ToView(), ctx);
- }
-};
-
-namespace zen {
-
-using namespace std::literals;
-namespace fs = std::filesystem;
-
-//////////////////////////////////////////////////////////////////////////
-
-namespace {
- std::error_code CreateGCReserve(const std::filesystem::path& Path, uint64_t Size)
- {
- if (Size == 0)
- {
- std::filesystem::remove(Path);
- return std::error_code{};
- }
- CreateDirectories(Path.parent_path());
- if (std::filesystem::is_regular_file(Path) && std::filesystem::file_size(Path) == Size)
- {
- return std::error_code();
- }
-#if ZEN_PLATFORM_WINDOWS
- DWORD dwCreationDisposition = CREATE_ALWAYS;
- DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE;
-
- const DWORD dwShareMode = 0;
- const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL;
- HANDLE hTemplateFile = nullptr;
-
- HANDLE FileHandle = CreateFile(Path.c_str(),
- dwDesiredAccess,
- dwShareMode,
- /* lpSecurityAttributes */ nullptr,
- dwCreationDisposition,
- dwFlagsAndAttributes,
- hTemplateFile);
-
- if (FileHandle == INVALID_HANDLE_VALUE)
- {
- return MakeErrorCodeFromLastError();
- }
- bool Keep = true;
- auto _ = MakeGuard([&]() {
- ::CloseHandle(FileHandle);
- if (!Keep)
- {
- ::DeleteFile(Path.c_str());
- }
- });
- LARGE_INTEGER liFileSize;
- liFileSize.QuadPart = Size;
- BOOL OK = ::SetFilePointerEx(FileHandle, liFileSize, 0, FILE_BEGIN);
- if (!OK)
- {
- return MakeErrorCodeFromLastError();
- }
- OK = ::SetEndOfFile(FileHandle);
- if (!OK)
- {
- return MakeErrorCodeFromLastError();
- }
- Keep = true;
-#else
- int OpenFlags = O_CLOEXEC | O_RDWR | O_CREAT;
- int Fd = open(Path.c_str(), OpenFlags, 0666);
- if (Fd < 0)
- {
- return MakeErrorCodeFromLastError();
- }
-
- bool Keep = true;
- auto _ = MakeGuard([&]() {
- close(Fd);
- if (!Keep)
- {
- unlink(Path.c_str());
- }
- });
-
- if (fchmod(Fd, 0666) < 0)
- {
- return MakeErrorCodeFromLastError();
- }
-
-# if ZEN_PLATFORM_MAC
- if (ftruncate(Fd, (off_t)Size) < 0)
- {
- return MakeErrorCodeFromLastError();
- }
-# else
- if (ftruncate64(Fd, (off64_t)Size) < 0)
- {
- return MakeErrorCodeFromLastError();
- }
- int Error = posix_fallocate64(Fd, 0, (off64_t)Size);
- if (Error)
- {
- return MakeErrorCode(Error);
- }
-# endif
- Keep = true;
-#endif
- return std::error_code{};
- }
-
-} // namespace
-
-//////////////////////////////////////////////////////////////////////////
-
-CbObject
-LoadCompactBinaryObject(const fs::path& Path)
-{
- FileContents Result = ReadFile(Path);
-
- if (!Result.ErrorCode)
- {
- IoBuffer Buffer = Result.Flatten();
- if (CbValidateError Error = ValidateCompactBinary(Buffer, CbValidateMode::All); Error == CbValidateError::None)
- {
- return LoadCompactBinaryObject(Buffer);
- }
- }
-
- return CbObject();
-}
-
-void
-SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object)
-{
- WriteFile(Path, Object.GetBuffer().AsIoBuffer());
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-struct GcContext::GcState
-{
- using CacheKeyContexts = std::unordered_map<std::string, std::vector<IoHash>>;
-
- CacheKeyContexts m_ExpiredCacheKeys;
- HashKeySet m_RetainedCids;
- HashKeySet m_DeletedCids;
- GcClock::TimePoint m_ExpireTime;
- bool m_DeletionMode = true;
- bool m_CollectSmallObjects = false;
-
- std::filesystem::path DiskReservePath;
-};
-
-GcContext::GcContext(const GcClock::TimePoint& ExpireTime) : m_State(std::make_unique<GcState>())
-{
- m_State->m_ExpireTime = ExpireTime;
-}
-
-GcContext::~GcContext()
-{
-}
-
-void
-GcContext::AddRetainedCids(std::span<const IoHash> Cids)
-{
- m_State->m_RetainedCids.AddHashesToSet(Cids);
-}
-
-void
-GcContext::SetExpiredCacheKeys(const std::string& CacheKeyContext, std::vector<IoHash>&& ExpiredKeys)
-{
- m_State->m_ExpiredCacheKeys[CacheKeyContext] = std::move(ExpiredKeys);
-}
-
-void
-GcContext::IterateCids(std::function<void(const IoHash&)> Callback)
-{
- m_State->m_RetainedCids.IterateHashes([&](const IoHash& Hash) { Callback(Hash); });
-}
-
-void
-GcContext::FilterCids(std::span<const IoHash> Cid, std::function<void(const IoHash&)> KeepFunc)
-{
- m_State->m_RetainedCids.FilterHashes(Cid, [&](const IoHash& Hash) { KeepFunc(Hash); });
-}
-
-void
-GcContext::FilterCids(std::span<const IoHash> Cid, std::function<void(const IoHash&, bool)>&& FilterFunc)
-{
- m_State->m_RetainedCids.FilterHashes(Cid, std::move(FilterFunc));
-}
-
-void
-GcContext::AddDeletedCids(std::span<const IoHash> Cas)
-{
- m_State->m_DeletedCids.AddHashesToSet(Cas);
-}
-
-const HashKeySet&
-GcContext::DeletedCids()
-{
- return m_State->m_DeletedCids;
-}
-
-std::span<const IoHash>
-GcContext::ExpiredCacheKeys(const std::string& CacheKeyContext) const
-{
- return m_State->m_ExpiredCacheKeys[CacheKeyContext];
-}
-
-bool
-GcContext::IsDeletionMode() const
-{
- return m_State->m_DeletionMode;
-}
-
-void
-GcContext::SetDeletionMode(bool NewState)
-{
- m_State->m_DeletionMode = NewState;
-}
-
-bool
-GcContext::CollectSmallObjects() const
-{
- return m_State->m_CollectSmallObjects;
-}
-
-void
-GcContext::CollectSmallObjects(bool NewState)
-{
- m_State->m_CollectSmallObjects = NewState;
-}
-
-GcClock::TimePoint
-GcContext::ExpireTime() const
-{
- return m_State->m_ExpireTime;
-}
-
-void
-GcContext::DiskReservePath(const std::filesystem::path& Path)
-{
- m_State->DiskReservePath = Path;
-}
-
-uint64_t
-GcContext::ClaimGCReserve()
-{
- if (!std::filesystem::is_regular_file(m_State->DiskReservePath))
- {
- return 0;
- }
- uint64_t ReclaimedSize = std::filesystem::file_size(m_State->DiskReservePath);
- if (std::filesystem::remove(m_State->DiskReservePath))
- {
- return ReclaimedSize;
- }
- return 0;
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-GcContributor::GcContributor(GcManager& Gc) : m_Gc(Gc)
-{
- m_Gc.AddGcContributor(this);
-}
-
-GcContributor::~GcContributor()
-{
- m_Gc.RemoveGcContributor(this);
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-GcStorage::GcStorage(GcManager& Gc) : m_Gc(Gc)
-{
- m_Gc.AddGcStorage(this);
-}
-
-GcStorage::~GcStorage()
-{
- m_Gc.RemoveGcStorage(this);
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-GcManager::GcManager() : m_Log(logging::Get("gc"))
-{
-}
-
-GcManager::~GcManager()
-{
-}
-
-void
-GcManager::AddGcContributor(GcContributor* Contributor)
-{
- RwLock::ExclusiveLockScope _(m_Lock);
- m_GcContribs.push_back(Contributor);
-}
-
-void
-GcManager::RemoveGcContributor(GcContributor* Contributor)
-{
- RwLock::ExclusiveLockScope _(m_Lock);
- std::erase_if(m_GcContribs, [&](GcContributor* $) { return $ == Contributor; });
-}
-
-void
-GcManager::AddGcStorage(GcStorage* Storage)
-{
- ZEN_ASSERT(Storage != nullptr);
- RwLock::ExclusiveLockScope _(m_Lock);
- m_GcStorage.push_back(Storage);
-}
-
-void
-GcManager::RemoveGcStorage(GcStorage* Storage)
-{
- RwLock::ExclusiveLockScope _(m_Lock);
- std::erase_if(m_GcStorage, [&](GcStorage* $) { return $ == Storage; });
-}
-
-void
-GcManager::CollectGarbage(GcContext& GcCtx)
-{
- RwLock::SharedLockScope _(m_Lock);
-
- // First gather reference set
- {
- Stopwatch Timer;
- const auto Guard = MakeGuard([&] { ZEN_INFO("gathered references in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
- for (GcContributor* Contributor : m_GcContribs)
- {
- Contributor->GatherReferences(GcCtx);
- }
- }
-
- // Then trim storage
- {
- GcStorageSize GCTotalSizeDiff;
- Stopwatch Timer;
- const auto Guard = MakeGuard([&] {
- ZEN_INFO("collected garbage in {}. Removed {} disk space, {} memory",
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
- NiceBytes(GCTotalSizeDiff.DiskSize),
- NiceBytes(GCTotalSizeDiff.MemorySize));
- });
- for (GcStorage* Storage : m_GcStorage)
- {
- const auto PreSize = Storage->StorageSize();
- Storage->CollectGarbage(GcCtx);
- const auto PostSize = Storage->StorageSize();
- GCTotalSizeDiff.DiskSize += PreSize.DiskSize > PostSize.DiskSize ? PreSize.DiskSize - PostSize.DiskSize : 0;
- GCTotalSizeDiff.MemorySize += PreSize.MemorySize > PostSize.MemorySize ? PreSize.MemorySize - PostSize.MemorySize : 0;
- }
- }
-}
-
-GcStorageSize
-GcManager::TotalStorageSize() const
-{
- RwLock::SharedLockScope _(m_Lock);
-
- GcStorageSize TotalSize;
-
- for (GcStorage* Storage : m_GcStorage)
- {
- const auto Size = Storage->StorageSize();
- TotalSize.DiskSize += Size.DiskSize;
- TotalSize.MemorySize += Size.MemorySize;
- }
-
- return TotalSize;
-}
-
-#if ZEN_USE_REF_TRACKING
-void
-GcManager::OnNewCidReferences(std::span<IoHash> Hashes)
-{
- ZEN_UNUSED(Hashes);
-}
-
-void
-GcManager::OnCommittedCidReferences(std::span<IoHash> Hashes)
-{
- ZEN_UNUSED(Hashes);
-}
-
-void
-GcManager::OnDroppedCidReferences(std::span<IoHash> Hashes)
-{
- ZEN_UNUSED(Hashes);
-}
-#endif
-
-//////////////////////////////////////////////////////////////////////////
-void
-DiskUsageWindow::KeepRange(GcClock::Tick StartTick, GcClock::Tick EndTick)
-{
- auto It = m_LogWindow.begin();
- if (It == m_LogWindow.end())
- {
- return;
- }
- while (It->SampleTime < StartTick)
- {
- ++It;
- if (It == m_LogWindow.end())
- {
- m_LogWindow.clear();
- return;
- }
- }
- m_LogWindow.erase(m_LogWindow.begin(), It);
-
- It = m_LogWindow.begin();
- while (It != m_LogWindow.end())
- {
- if (It->SampleTime >= EndTick)
- {
- m_LogWindow.erase(It, m_LogWindow.end());
- return;
- }
- It++;
- }
-}
-
-std::vector<uint64_t>
-DiskUsageWindow::GetDiskDeltas(GcClock::Tick StartTick, GcClock::Tick EndTick, GcClock::Tick DeltaWidth, uint64_t& OutMaxDelta) const
-{
- ZEN_ASSERT(StartTick != -1);
- ZEN_ASSERT(DeltaWidth > 0);
-
- std::vector<uint64_t> Result;
- Result.reserve((EndTick - StartTick + DeltaWidth - 1) / DeltaWidth);
-
- size_t WindowSize = m_LogWindow.size();
- GcClock::Tick FirstWindowTick = WindowSize < 2 ? EndTick : m_LogWindow[1].SampleTime;
-
- GcClock::Tick RangeStart = StartTick;
- while (FirstWindowTick >= RangeStart + DeltaWidth && RangeStart < EndTick)
- {
- Result.push_back(0);
- RangeStart += DeltaWidth;
- }
-
- uint64_t DeltaSum = 0;
- size_t WindowIndex = 1;
- while (WindowIndex < WindowSize && RangeStart < EndTick)
- {
- const DiskUsageEntry& Entry = m_LogWindow[WindowIndex];
- if (Entry.SampleTime < RangeStart)
- {
- ++WindowIndex;
- continue;
- }
- GcClock::Tick RangeEnd = Min(EndTick, RangeStart + DeltaWidth);
- ZEN_ASSERT(Entry.SampleTime >= RangeStart);
- if (Entry.SampleTime >= RangeEnd)
- {
- Result.push_back(DeltaSum);
- OutMaxDelta = Max(DeltaSum, OutMaxDelta);
- DeltaSum = 0;
- RangeStart = RangeEnd;
- continue;
- }
- const DiskUsageEntry& PrevEntry = m_LogWindow[WindowIndex - 1];
- if (Entry.DiskUsage > PrevEntry.DiskUsage)
- {
- uint64_t Delta = Entry.DiskUsage - PrevEntry.DiskUsage;
- DeltaSum += Delta;
- }
- WindowIndex++;
- }
-
- while (RangeStart < EndTick)
- {
- Result.push_back(DeltaSum);
- OutMaxDelta = Max(DeltaSum, OutMaxDelta);
- DeltaSum = 0;
- RangeStart += DeltaWidth;
- }
- return Result;
-}
-
-GcClock::Tick
-DiskUsageWindow::FindTimepointThatRemoves(uint64_t Amount, GcClock::Tick EndTick) const
-{
- ZEN_ASSERT(Amount > 0);
- uint64_t RemainingToFind = Amount;
- size_t Offset = 1;
- while (Offset < m_LogWindow.size())
- {
- const DiskUsageEntry& Entry = m_LogWindow[Offset];
- if (Entry.SampleTime >= EndTick)
- {
- return EndTick;
- }
- const DiskUsageEntry& PreviousEntry = m_LogWindow[Offset - 1];
- uint64_t Delta = Entry.DiskUsage > PreviousEntry.DiskUsage ? Entry.DiskUsage - PreviousEntry.DiskUsage : 0;
- if (Delta >= RemainingToFind)
- {
- return m_LogWindow[Offset].SampleTime + 1;
- }
- RemainingToFind -= Delta;
- Offset++;
- }
- return EndTick;
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-GcScheduler::GcScheduler(GcManager& GcManager) : m_Log(logging::Get("gc")), m_GcManager(GcManager)
-{
-}
-
-GcScheduler::~GcScheduler()
-{
- Shutdown();
-}
-
-void
-GcScheduler::Initialize(const GcSchedulerConfig& Config)
-{
- using namespace std::chrono;
-
- m_Config = Config;
-
- if (m_Config.Interval.count() && m_Config.Interval < m_Config.MonitorInterval)
- {
- m_Config.Interval = m_Config.MonitorInterval;
- }
-
- std::filesystem::create_directories(Config.RootDirectory);
-
- std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
- if (Ec)
- {
- ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason '{}'",
- m_Config.RootDirectory / "reserve.gc",
- NiceBytes(m_Config.DiskReserveSize),
- Ec.message());
- }
-
- m_LastGcTime = GcClock::Now();
- m_LastGcExpireTime = GcClock::TimePoint::min();
-
- if (CbObject SchedulerState = LoadCompactBinaryObject(Config.RootDirectory / "gc_state"))
- {
- m_LastGcTime = GcClock::TimePoint(GcClock::Duration(SchedulerState["LastGcTime"sv].AsInt64()));
- m_LastGcExpireTime =
- GcClock::TimePoint(GcClock::Duration(SchedulerState["LastGcExpireTime"].AsInt64(GcClock::Duration::min().count())));
- if (m_LastGcTime + m_Config.Interval < GcClock::Now())
- {
- // TODO: Trigger GC?
- m_LastGcTime = GcClock::Now();
- }
- }
-
- m_DiskUsageLog.Open(m_Config.RootDirectory / "gc.dlog", CasLogFile::Mode::kWrite);
- m_DiskUsageLog.Initialize();
- const GcClock::Tick LastGCTick = m_LastGcTime.time_since_epoch().count();
- m_DiskUsageLog.Replay(
- [this, LastGCTick](const DiskUsageWindow::DiskUsageEntry& Entry) {
- if (Entry.SampleTime >= m_LastGcExpireTime.time_since_epoch().count())
- {
- m_DiskUsageWindow.Append(Entry);
- }
- },
- 0);
-
- m_NextGcTime = NextGcTime(m_LastGcTime);
- m_GcThread = std::thread(&GcScheduler::SchedulerThread, this);
-}
-
-void
-GcScheduler::Shutdown()
-{
- if (static_cast<uint32_t>(GcSchedulerStatus::kStopped) != m_Status)
- {
- bool GcIsRunning = m_Status == static_cast<uint32_t>(GcSchedulerStatus::kRunning);
- m_Status = static_cast<uint32_t>(GcSchedulerStatus::kStopped);
- m_GcSignal.notify_one();
-
- if (m_GcThread.joinable())
- {
- if (GcIsRunning)
- {
- ZEN_INFO("Waiting for garbage collection to complete");
- }
- m_GcThread.join();
- }
- }
- m_DiskUsageLog.Flush();
- m_DiskUsageLog.Close();
-}
-
-bool
-GcScheduler::Trigger(const GcScheduler::TriggerParams& Params)
-{
- if (m_Config.Enabled)
- {
- std::unique_lock Lock(m_GcMutex);
- if (static_cast<uint32_t>(GcSchedulerStatus::kIdle) == m_Status)
- {
- m_TriggerParams = Params;
- uint32_t IdleState = static_cast<uint32_t>(GcSchedulerStatus::kIdle);
- if (m_Status.compare_exchange_strong(IdleState, static_cast<uint32_t>(GcSchedulerStatus::kRunning)))
- {
- m_GcSignal.notify_one();
- return true;
- }
- }
- }
-
- return false;
-}
-
-void
-GcScheduler::SchedulerThread()
-{
- std::chrono::seconds WaitTime{0};
-
- for (;;)
- {
- bool Timeout = false;
- {
- ZEN_ASSERT(WaitTime.count() >= 0);
- std::unique_lock Lock(m_GcMutex);
- Timeout = std::cv_status::timeout == m_GcSignal.wait_for(Lock, WaitTime);
- }
-
- if (Status() == GcSchedulerStatus::kStopped)
- {
- break;
- }
-
- if (!m_Config.Enabled)
- {
- WaitTime = std::chrono::seconds::max();
- continue;
- }
-
- if (!Timeout && Status() == GcSchedulerStatus::kIdle)
- {
- continue;
- }
-
- bool Delete = true;
- bool CollectSmallObjects = m_Config.CollectSmallObjects;
- std::chrono::seconds MaxCacheDuration = m_Config.MaxCacheDuration;
- uint64_t DiskSizeSoftLimit = m_Config.DiskSizeSoftLimit;
- GcClock::TimePoint Now = GcClock::Now();
- if (m_TriggerParams)
- {
- const auto TriggerParams = m_TriggerParams.value();
- m_TriggerParams.reset();
-
- CollectSmallObjects = TriggerParams.CollectSmallObjects;
- if (TriggerParams.MaxCacheDuration != std::chrono::seconds::max())
- {
- MaxCacheDuration = TriggerParams.MaxCacheDuration;
- }
- if (TriggerParams.DiskSizeSoftLimit != 0)
- {
- DiskSizeSoftLimit = TriggerParams.DiskSizeSoftLimit;
- }
- }
-
- GcClock::TimePoint ExpireTime = MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxCacheDuration;
-
- std::error_code Ec;
- const GcStorageSize TotalSize = m_GcManager.TotalStorageSize();
-
- if (Timeout && Status() == GcSchedulerStatus::kIdle)
- {
- DiskSpace Space = DiskSpaceInfo(m_Config.RootDirectory, Ec);
- if (Ec)
- {
- ZEN_WARN("get disk space info FAILED, reason: '{}'", Ec.message());
- }
-
- const int64_t PressureGraphLength = 30;
- const std::chrono::duration LoadGraphTime = PressureGraphLength * m_Config.MonitorInterval;
- std::vector<uint64_t> DiskDeltas;
- uint64_t MaxLoad = 0;
- {
- const GcClock::Tick EpochTickCount = GcClock::Now().time_since_epoch().count();
- std::unique_lock Lock(m_GcMutex);
- m_DiskUsageWindow.Append({.SampleTime = EpochTickCount, .DiskUsage = TotalSize.DiskSize});
- m_DiskUsageLog.Append({.SampleTime = EpochTickCount, .DiskUsage = TotalSize.DiskSize});
- const GcClock::TimePoint LoadGraphStartTime = Now - LoadGraphTime;
- GcClock::Tick Start = LoadGraphStartTime.time_since_epoch().count();
- GcClock::Tick End = Now.time_since_epoch().count();
- DiskDeltas = m_DiskUsageWindow.GetDiskDeltas(Start,
- End,
- Max(1, (End - Start + PressureGraphLength - 1) / PressureGraphLength),
- MaxLoad);
- }
-
- std::string LoadGraph;
- LoadGraph.resize(DiskDeltas.size(), '0');
- if (DiskDeltas.size() > 0 && MaxLoad > 0)
- {
- char LoadIndicator[11] = "0123456789";
- for (size_t Index = 0; Index < DiskDeltas.size(); ++Index)
- {
- size_t LoadIndex = (9 * DiskDeltas[Index] + MaxLoad - 1) / MaxLoad;
- LoadGraph[Index] = LoadIndicator[LoadIndex];
- }
- }
-
- uint64_t GcDiskSpaceGoal = 0;
- if (DiskSizeSoftLimit != 0 && TotalSize.DiskSize > DiskSizeSoftLimit)
- {
- GcDiskSpaceGoal = TotalSize.DiskSize - DiskSizeSoftLimit;
- std::unique_lock Lock(m_GcMutex);
- GcClock::Tick AgeTick = m_DiskUsageWindow.FindTimepointThatRemoves(GcDiskSpaceGoal, Now.time_since_epoch().count());
- GcClock::TimePoint SizeBasedExpireTime = GcClock::TimePointFromTick(AgeTick);
- if (SizeBasedExpireTime > ExpireTime)
- {
- ExpireTime = SizeBasedExpireTime;
- }
- }
-
- bool DiskSpaceGCTriggered = GcDiskSpaceGoal > 0;
-
- std::chrono::seconds RemaingTime = std::chrono::duration_cast<std::chrono::seconds>(m_NextGcTime - GcClock::Now());
-
- if (RemaingTime < std::chrono::seconds::zero())
- {
- RemaingTime = std::chrono::seconds::zero();
- }
-
- bool TimeBasedGCTriggered = !DiskSpaceGCTriggered && RemaingTime.count() == 0;
- ZEN_INFO(
- "{} in use,{} {} of total {} free disk space, disk writes last {} per {} [{}], peak {}/s. {}",
- NiceBytes(TotalSize.DiskSize),
- DiskSizeSoftLimit == 0 ? "" : fmt::format(" {} soft limit,", NiceBytes(DiskSizeSoftLimit)),
- NiceBytes(Space.Free),
- NiceBytes(Space.Total),
- NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(LoadGraphTime).count())),
- NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(LoadGraphTime).count() / PressureGraphLength)),
- LoadGraph,
- NiceBytes(MaxLoad * uint64_t(std::chrono::seconds(1).count()) / uint64_t(std::chrono::seconds(LoadGraphTime).count())),
- DiskSpaceGCTriggered ? fmt::format("Disk use threshold triggered, trying to reclaim {}. ", NiceBytes(GcDiskSpaceGoal))
- : TimeBasedGCTriggered ? "GC schedule triggered."
- : m_NextGcTime == GcClock::TimePoint::max()
- ? ""
- : fmt::format("{} until next scheduled GC.", NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(RemaingTime).count()))));
-
- if (!DiskSpaceGCTriggered && !TimeBasedGCTriggered)
- {
- WaitTime = m_Config.MonitorInterval < RemaingTime ? m_Config.MonitorInterval : RemaingTime;
- continue;
- }
-
- WaitTime = m_Config.MonitorInterval;
- uint32_t IdleState = static_cast<uint32_t>(GcSchedulerStatus::kIdle);
- if (!m_Status.compare_exchange_strong(IdleState, static_cast<uint32_t>(GcSchedulerStatus::kRunning)))
- {
- continue;
- }
- }
-
- CollectGarbage(ExpireTime, Delete, CollectSmallObjects);
-
- uint32_t RunningState = static_cast<uint32_t>(GcSchedulerStatus::kRunning);
- if (!m_Status.compare_exchange_strong(RunningState, static_cast<uint32_t>(GcSchedulerStatus::kIdle)))
- {
- ZEN_ASSERT(m_Status == static_cast<uint32_t>(GcSchedulerStatus::kStopped));
- break;
- }
-
- WaitTime = m_Config.MonitorInterval;
- }
-}
-
-GcClock::TimePoint
-GcScheduler::NextGcTime(GcClock::TimePoint CurrentTime)
-{
- if (m_Config.Interval.count())
- {
- return CurrentTime + m_Config.Interval;
- }
- else
- {
- return GcClock::TimePoint::max();
- }
-}
-
-void
-GcScheduler::CollectGarbage(const GcClock::TimePoint& ExpireTime, bool Delete, bool CollectSmallObjects)
-{
- GcContext GcCtx(ExpireTime);
- GcCtx.SetDeletionMode(Delete);
- GcCtx.CollectSmallObjects(CollectSmallObjects);
- // GcCtx.MaxCacheDuration(MaxCacheDuration);
- GcCtx.DiskReservePath(m_Config.RootDirectory / "reserve.gc");
-
- ZEN_INFO("garbage collection STARTING, small objects gc {}, cutoff time {}",
- GcCtx.CollectSmallObjects() ? "ENABLED"sv : "DISABLED"sv,
- ExpireTime);
- {
- Stopwatch Timer;
- const auto __ = MakeGuard([&] { ZEN_INFO("garbage collection DONE in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
-
- m_GcManager.CollectGarbage(GcCtx);
-
- if (Delete)
- {
- m_LastGcExpireTime = ExpireTime;
- std::unique_lock Lock(m_GcMutex);
- m_DiskUsageWindow.KeepRange(ExpireTime.time_since_epoch().count(), GcClock::Duration::max().count());
- }
-
- m_LastGcTime = GcClock::Now();
- m_NextGcTime = NextGcTime(m_LastGcTime);
-
- {
- const fs::path Path = m_Config.RootDirectory / "gc_state";
- ZEN_DEBUG("saving scheduler state to '{}'", Path);
- CbObjectWriter SchedulerState;
- SchedulerState << "LastGcTime"sv << static_cast<int64_t>(m_LastGcTime.time_since_epoch().count());
- SchedulerState << "LastGcExpireTime"sv << static_cast<int64_t>(m_LastGcExpireTime.time_since_epoch().count());
- SaveCompactBinaryObject(Path, SchedulerState.Save());
- }
-
- std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize);
- if (Ec)
- {
- ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason: '{}'",
- m_Config.RootDirectory / "reserve.gc",
- NiceBytes(m_Config.DiskReserveSize),
- Ec.message());
- }
- }
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-#if ZEN_WITH_TESTS
-
-namespace gc::impl {
- static IoBuffer CreateChunk(uint64_t Size)
- {
- static std::random_device rd;
- static std::mt19937 g(rd());
-
- std::vector<uint8_t> Values;
- Values.resize(Size);
- for (size_t Idx = 0; Idx < Size; ++Idx)
- {
- Values[Idx] = static_cast<uint8_t>(Idx);
- }
- std::shuffle(Values.begin(), Values.end(), g);
-
- return IoBufferBuilder::MakeCloneFromMemory(Values.data(), Values.size());
- }
-
- static CompressedBuffer Compress(IoBuffer Buffer)
- {
- return CompressedBuffer::Compress(SharedBuffer::MakeView(Buffer.GetData(), Buffer.GetSize()));
- }
-} // namespace gc::impl
-
-TEST_CASE("gc.basic")
-{
- using namespace gc::impl;
-
- ScopedTemporaryDirectory TempDir;
-
- CidStoreConfiguration CasConfig;
- CasConfig.RootDirectory = TempDir.Path() / "cas";
-
- GcManager Gc;
- CidStore CidStore(Gc);
-
- CidStore.Initialize(CasConfig);
-
- IoBuffer Chunk = CreateChunk(128);
- auto CompressedChunk = Compress(Chunk);
-
- const auto InsertResult = CidStore.AddChunk(CompressedChunk.GetCompressed().Flatten().AsIoBuffer(), CompressedChunk.DecodeRawHash());
- CHECK(InsertResult.New);
-
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
-
- CidStore.Flush();
- Gc.CollectGarbage(GcCtx);
-
- CHECK(!CidStore.ContainsChunk(CompressedChunk.DecodeRawHash()));
-}
-
-TEST_CASE("gc.full")
-{
- using namespace gc::impl;
-
- ScopedTemporaryDirectory TempDir;
-
- CidStoreConfiguration CasConfig;
- CasConfig.RootDirectory = TempDir.Path() / "cas";
-
- GcManager Gc;
- std::unique_ptr<CasStore> CasStore = CreateCasStore(Gc);
-
- CasStore->Initialize(CasConfig);
-
- uint64_t ChunkSizes[9] = {128, 541, 1023, 781, 218, 37, 4, 997, 5};
- IoBuffer Chunks[9] = {CreateChunk(ChunkSizes[0]),
- CreateChunk(ChunkSizes[1]),
- CreateChunk(ChunkSizes[2]),
- CreateChunk(ChunkSizes[3]),
- CreateChunk(ChunkSizes[4]),
- CreateChunk(ChunkSizes[5]),
- CreateChunk(ChunkSizes[6]),
- CreateChunk(ChunkSizes[7]),
- CreateChunk(ChunkSizes[8])};
- IoHash ChunkHashes[9] = {
- IoHash::HashBuffer(Chunks[0].Data(), Chunks[0].Size()),
- IoHash::HashBuffer(Chunks[1].Data(), Chunks[1].Size()),
- IoHash::HashBuffer(Chunks[2].Data(), Chunks[2].Size()),
- IoHash::HashBuffer(Chunks[3].Data(), Chunks[3].Size()),
- IoHash::HashBuffer(Chunks[4].Data(), Chunks[4].Size()),
- IoHash::HashBuffer(Chunks[5].Data(), Chunks[5].Size()),
- IoHash::HashBuffer(Chunks[6].Data(), Chunks[6].Size()),
- IoHash::HashBuffer(Chunks[7].Data(), Chunks[7].Size()),
- IoHash::HashBuffer(Chunks[8].Data(), Chunks[8].Size()),
- };
-
- CasStore->InsertChunk(Chunks[0], ChunkHashes[0]);
- CasStore->InsertChunk(Chunks[1], ChunkHashes[1]);
- CasStore->InsertChunk(Chunks[2], ChunkHashes[2]);
- CasStore->InsertChunk(Chunks[3], ChunkHashes[3]);
- CasStore->InsertChunk(Chunks[4], ChunkHashes[4]);
- CasStore->InsertChunk(Chunks[5], ChunkHashes[5]);
- CasStore->InsertChunk(Chunks[6], ChunkHashes[6]);
- CasStore->InsertChunk(Chunks[7], ChunkHashes[7]);
- CasStore->InsertChunk(Chunks[8], ChunkHashes[8]);
-
- CidStoreSize InitialSize = CasStore->TotalSize();
-
- // Keep first and last
- {
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
-
- std::vector<IoHash> KeepChunks;
- KeepChunks.push_back(ChunkHashes[0]);
- KeepChunks.push_back(ChunkHashes[8]);
- GcCtx.AddRetainedCids(KeepChunks);
-
- CasStore->Flush();
- Gc.CollectGarbage(GcCtx);
-
- CHECK(CasStore->ContainsChunk(ChunkHashes[0]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[1]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[2]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[3]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[4]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[5]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[6]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[7]));
- CHECK(CasStore->ContainsChunk(ChunkHashes[8]));
-
- CHECK(ChunkHashes[0] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[0])));
- CHECK(ChunkHashes[8] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[8])));
- }
-
- CasStore->InsertChunk(Chunks[1], ChunkHashes[1]);
- CasStore->InsertChunk(Chunks[2], ChunkHashes[2]);
- CasStore->InsertChunk(Chunks[3], ChunkHashes[3]);
- CasStore->InsertChunk(Chunks[4], ChunkHashes[4]);
- CasStore->InsertChunk(Chunks[5], ChunkHashes[5]);
- CasStore->InsertChunk(Chunks[6], ChunkHashes[6]);
- CasStore->InsertChunk(Chunks[7], ChunkHashes[7]);
-
- // Keep last
- {
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
- std::vector<IoHash> KeepChunks;
- KeepChunks.push_back(ChunkHashes[8]);
- GcCtx.AddRetainedCids(KeepChunks);
-
- CasStore->Flush();
- Gc.CollectGarbage(GcCtx);
-
- CHECK(!CasStore->ContainsChunk(ChunkHashes[0]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[1]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[2]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[3]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[4]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[5]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[6]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[7]));
- CHECK(CasStore->ContainsChunk(ChunkHashes[8]));
-
- CHECK(ChunkHashes[8] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[8])));
-
- CasStore->InsertChunk(Chunks[1], ChunkHashes[1]);
- CasStore->InsertChunk(Chunks[2], ChunkHashes[2]);
- CasStore->InsertChunk(Chunks[3], ChunkHashes[3]);
- CasStore->InsertChunk(Chunks[4], ChunkHashes[4]);
- CasStore->InsertChunk(Chunks[5], ChunkHashes[5]);
- CasStore->InsertChunk(Chunks[6], ChunkHashes[6]);
- CasStore->InsertChunk(Chunks[7], ChunkHashes[7]);
- }
-
- // Keep mixed
- {
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
- std::vector<IoHash> KeepChunks;
- KeepChunks.push_back(ChunkHashes[1]);
- KeepChunks.push_back(ChunkHashes[4]);
- KeepChunks.push_back(ChunkHashes[7]);
- GcCtx.AddRetainedCids(KeepChunks);
-
- CasStore->Flush();
- Gc.CollectGarbage(GcCtx);
-
- CHECK(!CasStore->ContainsChunk(ChunkHashes[0]));
- CHECK(CasStore->ContainsChunk(ChunkHashes[1]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[2]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[3]));
- CHECK(CasStore->ContainsChunk(ChunkHashes[4]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[5]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[6]));
- CHECK(CasStore->ContainsChunk(ChunkHashes[7]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[8]));
-
- CHECK(ChunkHashes[1] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[1])));
- CHECK(ChunkHashes[4] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[4])));
- CHECK(ChunkHashes[7] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[7])));
-
- CasStore->InsertChunk(Chunks[0], ChunkHashes[0]);
- CasStore->InsertChunk(Chunks[2], ChunkHashes[2]);
- CasStore->InsertChunk(Chunks[3], ChunkHashes[3]);
- CasStore->InsertChunk(Chunks[5], ChunkHashes[5]);
- CasStore->InsertChunk(Chunks[6], ChunkHashes[6]);
- CasStore->InsertChunk(Chunks[8], ChunkHashes[8]);
- }
-
- // Keep multiple at end
- {
- GcContext GcCtx(GcClock::Now() - std::chrono::hours(24));
- GcCtx.CollectSmallObjects(true);
- std::vector<IoHash> KeepChunks;
- KeepChunks.push_back(ChunkHashes[6]);
- KeepChunks.push_back(ChunkHashes[7]);
- KeepChunks.push_back(ChunkHashes[8]);
- GcCtx.AddRetainedCids(KeepChunks);
-
- CasStore->Flush();
- Gc.CollectGarbage(GcCtx);
-
- CHECK(!CasStore->ContainsChunk(ChunkHashes[0]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[1]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[2]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[3]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[4]));
- CHECK(!CasStore->ContainsChunk(ChunkHashes[5]));
- CHECK(CasStore->ContainsChunk(ChunkHashes[6]));
- CHECK(CasStore->ContainsChunk(ChunkHashes[7]));
- CHECK(CasStore->ContainsChunk(ChunkHashes[8]));
-
- CHECK(ChunkHashes[6] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[6])));
- CHECK(ChunkHashes[7] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[7])));
- CHECK(ChunkHashes[8] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[8])));
-
- CasStore->InsertChunk(Chunks[0], ChunkHashes[0]);
- CasStore->InsertChunk(Chunks[1], ChunkHashes[1]);
- CasStore->InsertChunk(Chunks[2], ChunkHashes[2]);
- CasStore->InsertChunk(Chunks[3], ChunkHashes[3]);
- CasStore->InsertChunk(Chunks[4], ChunkHashes[4]);
- CasStore->InsertChunk(Chunks[5], ChunkHashes[5]);
- }
-
- // Verify that we nicely appended blocks even after all GC operations
- CHECK(ChunkHashes[0] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[0])));
- CHECK(ChunkHashes[1] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[1])));
- CHECK(ChunkHashes[2] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[2])));
- CHECK(ChunkHashes[3] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[3])));
- CHECK(ChunkHashes[4] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[4])));
- CHECK(ChunkHashes[5] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[5])));
- CHECK(ChunkHashes[6] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[6])));
- CHECK(ChunkHashes[7] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[7])));
- CHECK(ChunkHashes[8] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[8])));
-
- auto FinalSize = CasStore->TotalSize();
-
- CHECK_LE(InitialSize.TinySize, FinalSize.TinySize);
- CHECK_GE(InitialSize.TinySize + (1u << 28), FinalSize.TinySize);
-}
-
-TEST_CASE("gc.diskusagewindow")
-{
- using namespace gc::impl;
-
- DiskUsageWindow Stats;
- Stats.Append({.SampleTime = 0, .DiskUsage = 0}); // 0 0
- Stats.Append({.SampleTime = 10, .DiskUsage = 10}); // 1 10
- Stats.Append({.SampleTime = 20, .DiskUsage = 20}); // 2 10
- Stats.Append({.SampleTime = 30, .DiskUsage = 20}); // 3 0
- Stats.Append({.SampleTime = 40, .DiskUsage = 15}); // 4 0
- Stats.Append({.SampleTime = 50, .DiskUsage = 25}); // 5 10
- Stats.Append({.SampleTime = 60, .DiskUsage = 30}); // 6 5
- Stats.Append({.SampleTime = 70, .DiskUsage = 45}); // 7 15
-
- SUBCASE("Truncate start")
- {
- Stats.KeepRange(-15, 31);
- CHECK(Stats.m_LogWindow.size() == 4);
- CHECK(Stats.m_LogWindow[0].SampleTime == 0);
- CHECK(Stats.m_LogWindow[3].SampleTime == 30);
- }
-
- SUBCASE("Truncate end")
- {
- Stats.KeepRange(70, 71);
- CHECK(Stats.m_LogWindow.size() == 1);
- CHECK(Stats.m_LogWindow[0].SampleTime == 70);
- }
-
- SUBCASE("Truncate middle")
- {
- Stats.KeepRange(29, 69);
- CHECK(Stats.m_LogWindow.size() == 4);
- CHECK(Stats.m_LogWindow[0].SampleTime == 30);
- CHECK(Stats.m_LogWindow[3].SampleTime == 60);
- }
-
- SUBCASE("Full range")
- {
- uint64_t MaxDelta = 0;
- // 0-10, 10-20, 20-30, 30-40, 40-50, 50-60, 60-70, 70-80
- std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(0, 80, 10, MaxDelta);
- CHECK(DiskDeltas.size() == 8);
- CHECK(MaxDelta == 15);
- CHECK(DiskDeltas[0] == 0);
- CHECK(DiskDeltas[1] == 10);
- CHECK(DiskDeltas[2] == 10);
- CHECK(DiskDeltas[3] == 0);
- CHECK(DiskDeltas[4] == 0);
- CHECK(DiskDeltas[5] == 10);
- CHECK(DiskDeltas[6] == 5);
- CHECK(DiskDeltas[7] == 15);
- }
-
- SUBCASE("Sub range")
- {
- uint64_t MaxDelta = 0;
- std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(20, 40, 10, MaxDelta);
- CHECK(DiskDeltas.size() == 2);
- CHECK(MaxDelta == 10);
- CHECK(DiskDeltas[0] == 10); // [20:30]
- CHECK(DiskDeltas[1] == 0); // [30:40]
- }
- SUBCASE("Unaligned sub range 1")
- {
- uint64_t MaxDelta = 0;
- std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(21, 51, 10, MaxDelta);
- CHECK(DiskDeltas.size() == 3);
- CHECK(MaxDelta == 10);
- CHECK(DiskDeltas[0] == 0); // [21:31]
- CHECK(DiskDeltas[1] == 0); // [31:41]
- CHECK(DiskDeltas[2] == 10); // [41:51]
- }
- SUBCASE("Unaligned end range")
- {
- uint64_t MaxDelta = 0;
- std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(29, 79, 10, MaxDelta);
- CHECK(DiskDeltas.size() == 5);
- CHECK(MaxDelta == 15);
- CHECK(DiskDeltas[0] == 0); // [29:39]
- CHECK(DiskDeltas[1] == 0); // [39:49]
- CHECK(DiskDeltas[2] == 10); // [49:59]
- CHECK(DiskDeltas[3] == 5); // [59:69]
- CHECK(DiskDeltas[4] == 15); // [69:79]
- }
- SUBCASE("Ahead of window")
- {
- uint64_t MaxDelta = 0;
- std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(-40, 0, 10, MaxDelta);
- CHECK(DiskDeltas.size() == 4);
- CHECK(MaxDelta == 0);
- CHECK(DiskDeltas[0] == 0); // [-40:-30]
- CHECK(DiskDeltas[1] == 0); // [-30:-20]
- CHECK(DiskDeltas[2] == 0); // [-20:-10]
- CHECK(DiskDeltas[3] == 0); // [-10:0]
- }
- SUBCASE("After of window")
- {
- uint64_t MaxDelta = 0;
- std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(90, 120, 10, MaxDelta);
- CHECK(DiskDeltas.size() == 3);
- CHECK(MaxDelta == 0);
- CHECK(DiskDeltas[0] == 0); // [90:100]
- CHECK(DiskDeltas[1] == 0); // [100:110]
- CHECK(DiskDeltas[2] == 0); // [110:120]
- }
- SUBCASE("Encapsulating window")
- {
- uint64_t MaxDelta = 0;
- std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(-20, 100, 10, MaxDelta);
- CHECK(DiskDeltas.size() == 12);
- CHECK(MaxDelta == 15);
- CHECK(DiskDeltas[0] == 0); // [-20:-10]
- CHECK(DiskDeltas[1] == 0); // [ -10:0]
- CHECK(DiskDeltas[2] == 0); // [0:10]
- CHECK(DiskDeltas[3] == 10); // [10:20]
- CHECK(DiskDeltas[4] == 10); // [20:30]
- CHECK(DiskDeltas[5] == 0); // [30:40]
- CHECK(DiskDeltas[6] == 0); // [40:50]
- CHECK(DiskDeltas[7] == 10); // [50:60]
- CHECK(DiskDeltas[8] == 5); // [60:70]
- CHECK(DiskDeltas[9] == 15); // [70:80]
- CHECK(DiskDeltas[10] == 0); // [80:90]
- CHECK(DiskDeltas[11] == 0); // [90:100]
- }
-
- SUBCASE("Full range half stride")
- {
- uint64_t MaxDelta = 0;
- std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(0, 80, 20, MaxDelta);
- CHECK(DiskDeltas.size() == 4);
- CHECK(MaxDelta == 20);
- CHECK(DiskDeltas[0] == 10); // [0:20]
- CHECK(DiskDeltas[1] == 10); // [20:40]
- CHECK(DiskDeltas[2] == 10); // [40:60]
- CHECK(DiskDeltas[3] == 20); // [60:80]
- }
-
- SUBCASE("Partial odd stride")
- {
- uint64_t MaxDelta = 0;
- std::vector<uint64_t> DiskDeltas = Stats.GetDiskDeltas(13, 67, 18, MaxDelta);
- CHECK(DiskDeltas.size() == 3);
- CHECK(MaxDelta == 15);
- CHECK(DiskDeltas[0] == 10); // [13:31]
- CHECK(DiskDeltas[1] == 0); // [31:49]
- CHECK(DiskDeltas[2] == 15); // [49:67]
- }
-
- SUBCASE("Find size window")
- {
- DiskUsageWindow Empty;
- CHECK(Empty.FindTimepointThatRemoves(15u, 10000) == 10000);
-
- CHECK(Stats.FindTimepointThatRemoves(15u, 40) == 21);
- CHECK(Stats.FindTimepointThatRemoves(15u, 20) == 20);
- CHECK(Stats.FindTimepointThatRemoves(100000u, 50) == 50);
- CHECK(Stats.FindTimepointThatRemoves(100000u, 1000));
- }
-}
-#endif
-
-void
-gc_forcelink()
-{
-}
-
-} // namespace zen