// Copyright Epic Games, Inc. All Rights Reserved. #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "cas.h" #include #include #if ZEN_PLATFORM_WINDOWS # include #else # include # include # include # include #endif #if ZEN_WITH_TESTS # include # include # include #endif namespace zen { using namespace std::literals; namespace fs = std::filesystem; ////////////////////////////////////////////////////////////////////////// namespace { std::error_code CreateGCReserve(const std::filesystem::path& Path, uint64_t Size) { if (Size == 0) { std::filesystem::remove(Path); return std::error_code{}; } CreateDirectories(Path.parent_path()); if (std::filesystem::is_regular_file(Path) && std::filesystem::file_size(Path) == Size) { return std::error_code(); } #if ZEN_PLATFORM_WINDOWS DWORD dwCreationDisposition = CREATE_ALWAYS; DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; const DWORD dwShareMode = 0; const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; HANDLE hTemplateFile = nullptr; HANDLE FileHandle = CreateFile(Path.c_str(), dwDesiredAccess, dwShareMode, /* lpSecurityAttributes */ nullptr, dwCreationDisposition, dwFlagsAndAttributes, hTemplateFile); if (FileHandle == INVALID_HANDLE_VALUE) { return MakeErrorCodeFromLastError(); } bool Keep = true; auto _ = MakeGuard([&]() { ::CloseHandle(FileHandle); if (!Keep) { ::DeleteFile(Path.c_str()); } }); LARGE_INTEGER liFileSize; liFileSize.QuadPart = Size; BOOL OK = ::SetFilePointerEx(FileHandle, liFileSize, 0, FILE_BEGIN); if (!OK) { return MakeErrorCodeFromLastError(); } OK = ::SetEndOfFile(FileHandle); if (!OK) { return MakeErrorCodeFromLastError(); } Keep = true; #else int OpenFlags = O_CLOEXEC | O_RDWR | O_CREAT; int Fd = open(Path.c_str(), OpenFlags, 0666); if (Fd < 0) { return MakeErrorCodeFromLastError(); } bool Keep = true; auto _ = MakeGuard([&]() { close(Fd); if (!Keep) { unlink(Path.c_str()); } }); if (fchmod(Fd, 0666) < 0) { return MakeErrorCodeFromLastError(); } # if ZEN_PLATFORM_MAC if (ftruncate(Fd, (off_t)Size) < 0) { return MakeErrorCodeFromLastError(); } # else if (ftruncate64(Fd, (off64_t)Size) < 0) { return MakeErrorCodeFromLastError(); } int Error = posix_fallocate64(Fd, 0, (off64_t)Size); if (Error) { return MakeErrorCode(Error); } # endif Keep = true; #endif return std::error_code{}; } } // namespace ////////////////////////////////////////////////////////////////////////// CbObject LoadCompactBinaryObject(const fs::path& Path) { FileContents Result = ReadFile(Path); if (!Result.ErrorCode) { IoBuffer Buffer = Result.Flatten(); if (CbValidateError Error = ValidateCompactBinary(Buffer, CbValidateMode::All); Error == CbValidateError::None) { return LoadCompactBinaryObject(Buffer); } } return CbObject(); } void SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object) { WriteFile(Path, Object.GetBuffer().AsIoBuffer()); } ////////////////////////////////////////////////////////////////////////// struct GcContext::GcState { using CacheKeyContexts = std::unordered_map>; CacheKeyContexts m_ExpiredCacheKeys; HashKeySet m_RetainedCids; HashKeySet m_DeletedCids; GcClock::TimePoint m_GcTime; GcClock::Duration m_MaxCacheDuration = std::chrono::hours(24); bool m_DeletionMode = true; bool m_CollectSmallObjects = false; std::filesystem::path DiskReservePath; }; GcContext::GcContext(GcClock::TimePoint Time) : m_State(std::make_unique()) { m_State->m_GcTime = Time; } GcContext::~GcContext() { } void GcContext::AddRetainedCids(std::span Cids) { m_State->m_RetainedCids.AddHashesToSet(Cids); } void GcContext::SetExpiredCacheKeys(const std::string& CacheKeyContext, std::vector&& ExpiredKeys) { m_State->m_ExpiredCacheKeys[CacheKeyContext] = std::move(ExpiredKeys); } void GcContext::IterateCids(std::function Callback) { m_State->m_RetainedCids.IterateHashes([&](const IoHash& Hash) { Callback(Hash); }); } void GcContext::FilterCids(std::span Cid, std::function KeepFunc) { m_State->m_RetainedCids.FilterHashes(Cid, [&](const IoHash& Hash) { KeepFunc(Hash); }); } void GcContext::FilterCids(std::span Cid, std::function&& FilterFunc) { m_State->m_RetainedCids.FilterHashes(Cid, std::move(FilterFunc)); } void GcContext::AddDeletedCids(std::span Cas) { m_State->m_DeletedCids.AddHashesToSet(Cas); } const HashKeySet& GcContext::DeletedCids() { return m_State->m_DeletedCids; } std::span GcContext::ExpiredCacheKeys(const std::string& CacheKeyContext) const { return m_State->m_ExpiredCacheKeys[CacheKeyContext]; } bool GcContext::IsDeletionMode() const { return m_State->m_DeletionMode; } void GcContext::SetDeletionMode(bool NewState) { m_State->m_DeletionMode = NewState; } bool GcContext::CollectSmallObjects() const { return m_State->m_CollectSmallObjects; } void GcContext::CollectSmallObjects(bool NewState) { m_State->m_CollectSmallObjects = NewState; } GcClock::TimePoint GcContext::Time() const { return m_State->m_GcTime; } GcClock::Duration GcContext::MaxCacheDuration() const { return m_State->m_MaxCacheDuration; } void GcContext::MaxCacheDuration(GcClock::Duration Duration) { m_State->m_MaxCacheDuration = Duration; } void GcContext::DiskReservePath(const std::filesystem::path& Path) { m_State->DiskReservePath = Path; } uint64_t GcContext::ClaimGCReserve() { if (!std::filesystem::is_regular_file(m_State->DiskReservePath)) { return 0; } uint64_t ReclaimedSize = std::filesystem::file_size(m_State->DiskReservePath); if (std::filesystem::remove(m_State->DiskReservePath)) { return ReclaimedSize; } return 0; } ////////////////////////////////////////////////////////////////////////// GcContributor::GcContributor(GcManager& Gc) : m_Gc(Gc) { m_Gc.AddGcContributor(this); } GcContributor::~GcContributor() { m_Gc.RemoveGcContributor(this); } ////////////////////////////////////////////////////////////////////////// GcStorage::GcStorage(GcManager& Gc) : m_Gc(Gc) { m_Gc.AddGcStorage(this); } GcStorage::~GcStorage() { m_Gc.RemoveGcStorage(this); } ////////////////////////////////////////////////////////////////////////// GcManager::GcManager() { } GcManager::~GcManager() { } void GcManager::AddGcContributor(GcContributor* Contributor) { RwLock::ExclusiveLockScope _(m_Lock); m_GcContribs.push_back(Contributor); } void GcManager::RemoveGcContributor(GcContributor* Contributor) { RwLock::ExclusiveLockScope _(m_Lock); std::erase_if(m_GcContribs, [&](GcContributor* $) { return $ == Contributor; }); } void GcManager::AddGcStorage(GcStorage* Storage) { ZEN_ASSERT(Storage != nullptr); RwLock::ExclusiveLockScope _(m_Lock); m_GcStorage.push_back(Storage); } void GcManager::RemoveGcStorage(GcStorage* Storage) { RwLock::ExclusiveLockScope _(m_Lock); std::erase_if(m_GcStorage, [&](GcStorage* $) { return $ == Storage; }); } void GcManager::CollectGarbage(GcContext& GcCtx) { RwLock::SharedLockScope _(m_Lock); // First gather reference set { Stopwatch Timer; const auto Guard = MakeGuard([&] { ZEN_INFO("gathered references in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); for (GcContributor* Contributor : m_GcContribs) { Contributor->GatherReferences(GcCtx); } } // Then trim storage { Stopwatch Timer; const auto Guard = MakeGuard([&] { ZEN_INFO("collected garbage in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); for (GcStorage* Storage : m_GcStorage) { Storage->CollectGarbage(GcCtx); } } } GcStorageSize GcManager::TotalStorageSize() const { RwLock::SharedLockScope _(m_Lock); GcStorageSize TotalSize; for (GcStorage* Storage : m_GcStorage) { const auto Size = Storage->StorageSize(); TotalSize.DiskSize += Size.DiskSize; TotalSize.MemorySize += Size.MemorySize; } return TotalSize; } #if ZEN_USE_REF_TRACKING void GcManager::OnNewCidReferences(std::span Hashes) { ZEN_UNUSED(Hashes); } void GcManager::OnCommittedCidReferences(std::span Hashes) { ZEN_UNUSED(Hashes); } void GcManager::OnDroppedCidReferences(std::span Hashes) { ZEN_UNUSED(Hashes); } #endif ////////////////////////////////////////////////////////////////////////// GcScheduler::GcScheduler(GcManager& GcManager) : m_Log(logging::Get("gc")), m_GcManager(GcManager) { } GcScheduler::~GcScheduler() { Shutdown(); } void GcScheduler::Initialize(const GcSchedulerConfig& Config) { using namespace std::chrono; m_Config = Config; if (m_Config.Interval.count() && m_Config.Interval < m_Config.MonitorInterval) { m_Config.Interval = m_Config.MonitorInterval; } std::filesystem::create_directories(Config.RootDirectory); std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize); if (Ec) { ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason '{}'", m_Config.RootDirectory / "reserve.gc", NiceBytes(m_Config.DiskReserveSize), Ec.message()); } m_LastGcTime = GcClock::Now(); if (CbObject SchedulerState = LoadCompactBinaryObject(Config.RootDirectory / "gc_state")) { m_LastGcTime = GcClock::TimePoint(GcClock::Duration(SchedulerState["LastGcTime"sv].AsInt64())); if (m_LastGcTime + m_Config.Interval < GcClock::Now()) { // TODO: Trigger GC? m_LastGcTime = GcClock::Now(); } } m_NextGcTime = NextGcTime(m_LastGcTime); m_GcThread = std::thread(&GcScheduler::SchedulerThread, this); } void GcScheduler::Shutdown() { if (static_cast(GcSchedulerStatus::kStopped) != m_Status) { bool GcIsRunning = m_Status == static_cast(GcSchedulerStatus::kRunning); m_Status = static_cast(GcSchedulerStatus::kStopped); m_GcSignal.notify_one(); if (m_GcThread.joinable()) { if (GcIsRunning) { ZEN_INFO("Waiting for garbage collection to complete"); } m_GcThread.join(); } } } bool GcScheduler::Trigger(const GcScheduler::TriggerParams& Params) { if (m_Config.Enabled) { std::unique_lock Lock(m_GcMutex); if (static_cast(GcSchedulerStatus::kIdle) == m_Status) { m_TriggerParams = Params; uint32_t IdleState = static_cast(GcSchedulerStatus::kIdle); if (m_Status.compare_exchange_strong(IdleState, static_cast(GcSchedulerStatus::kRunning))) { m_GcSignal.notify_one(); return true; } } } return false; } void GcScheduler::SchedulerThread() { std::chrono::seconds WaitTime = m_Config.MonitorInterval; for (;;) { bool Timeout = false; { ZEN_ASSERT(WaitTime.count() >= 0); std::unique_lock Lock(m_GcMutex); Timeout = std::cv_status::timeout == m_GcSignal.wait_for(Lock, WaitTime); } if (Status() == GcSchedulerStatus::kStopped) { break; } if (!m_Config.Enabled || (!Timeout && Status() == GcSchedulerStatus::kIdle)) { continue; } if (Timeout && Status() == GcSchedulerStatus::kIdle) { std::error_code Ec; DiskSpace Space = DiskSpaceInfo(m_Config.RootDirectory, Ec); GcStorageSize TotalSize = m_GcManager.TotalStorageSize(); std::chrono::seconds RemaingTime = std::chrono::duration_cast(m_NextGcTime - GcClock::Now()); if (RemaingTime < std::chrono::seconds::zero()) { RemaingTime = std::chrono::seconds::zero(); } if (Ec) { ZEN_WARN("get disk space info FAILED, reason: '{}'", Ec.message()); } ZEN_INFO("{} in use, {} of total {} free disk space, {}", NiceBytes(TotalSize.DiskSize), NiceBytes(Space.Free), NiceBytes(Space.Total), m_Config.Interval.count() ? fmt::format("{} until next GC", NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(RemaingTime).count()))) : std::string("no GC scheduled")); // TODO: Trigger GC if max disk usage water mark is reached if (RemaingTime.count() > 0) { WaitTime = m_Config.MonitorInterval < RemaingTime ? m_Config.MonitorInterval : RemaingTime; continue; } WaitTime = m_Config.MonitorInterval; uint32_t IdleState = static_cast(GcSchedulerStatus::kIdle); if (!m_Status.compare_exchange_strong(IdleState, static_cast(GcSchedulerStatus::kRunning))) { continue; } } GcContext GcCtx; GcCtx.SetDeletionMode(true); GcCtx.CollectSmallObjects(m_Config.CollectSmallObjects); GcCtx.MaxCacheDuration(m_Config.MaxCacheDuration); GcCtx.DiskReservePath(m_Config.RootDirectory / "reserve.gc"); if (m_TriggerParams) { const auto TriggerParams = m_TriggerParams.value(); m_TriggerParams.reset(); GcCtx.CollectSmallObjects(TriggerParams.CollectSmallObjects); if (TriggerParams.MaxCacheDuration != std::chrono::seconds::max()) { GcCtx.MaxCacheDuration(TriggerParams.MaxCacheDuration); } } ZEN_INFO("garbage collection STARTING, small objects gc {}, max cache duration {}", GcCtx.CollectSmallObjects() ? "ENABLED"sv : "DISABLED"sv, NiceTimeSpanMs(uint64_t(std::chrono::duration_cast(GcCtx.MaxCacheDuration()).count()))); { Stopwatch Timer; const auto __ = MakeGuard([&] { ZEN_INFO("garbage collection DONE after {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); m_GcManager.CollectGarbage(GcCtx); m_LastGcTime = GcClock::Now(); m_NextGcTime = NextGcTime(m_LastGcTime); WaitTime = m_Config.MonitorInterval; { const fs::path Path = m_Config.RootDirectory / "gc_state"; ZEN_DEBUG("saving scheduler state to '{}'", Path); CbObjectWriter SchedulderState; SchedulderState << "LastGcTime"sv << static_cast(m_LastGcTime.time_since_epoch().count()); SaveCompactBinaryObject(Path, SchedulderState.Save()); } std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize); if (Ec) { ZEN_WARN("unable to create GC reserve at '{}' with size {}, reason: '{}'", m_Config.RootDirectory / "reserve.gc", NiceBytes(m_Config.DiskReserveSize), Ec.message()); } } uint32_t RunningState = static_cast(GcSchedulerStatus::kRunning); if (!m_Status.compare_exchange_strong(RunningState, static_cast(GcSchedulerStatus::kIdle))) { ZEN_ASSERT(m_Status == static_cast(GcSchedulerStatus::kStopped)); break; } } } GcClock::TimePoint GcScheduler::NextGcTime(GcClock::TimePoint CurrentTime) { if (m_Config.Interval.count()) { return CurrentTime + m_Config.Interval; } else { return GcClock::TimePoint::max(); } } ////////////////////////////////////////////////////////////////////////// #if ZEN_WITH_TESTS namespace { static IoBuffer CreateChunk(uint64_t Size) { static std::random_device rd; static std::mt19937 g(rd()); std::vector Values; Values.resize(Size); for (size_t Idx = 0; Idx < Size; ++Idx) { Values[Idx] = static_cast(Idx); } std::shuffle(Values.begin(), Values.end(), g); return IoBufferBuilder::MakeCloneFromMemory(Values.data(), Values.size()); } static CompressedBuffer Compress(IoBuffer Buffer) { return CompressedBuffer::Compress(SharedBuffer::MakeView(Buffer.GetData(), Buffer.GetSize())); } } // namespace TEST_CASE("gc.basic") { ScopedTemporaryDirectory TempDir; CidStoreConfiguration CasConfig; CasConfig.RootDirectory = TempDir.Path() / "cas"; GcManager Gc; CidStore CidStore(Gc); CidStore.Initialize(CasConfig); IoBuffer Chunk = CreateChunk(128); auto CompressedChunk = Compress(Chunk); const auto InsertResult = CidStore.AddChunk(CompressedChunk); CHECK(InsertResult.New); GcContext GcCtx; GcCtx.CollectSmallObjects(true); CidStore.Flush(); Gc.CollectGarbage(GcCtx); CHECK(!CidStore.ContainsChunk(IoHash::FromBLAKE3(CompressedChunk.GetRawHash()))); } TEST_CASE("gc.full") { ScopedTemporaryDirectory TempDir; CidStoreConfiguration CasConfig; CasConfig.RootDirectory = TempDir.Path() / "cas"; GcManager Gc; std::unique_ptr CasStore = CreateCasStore(Gc); CasStore->Initialize(CasConfig); uint64_t ChunkSizes[9] = {128, 541, 1023, 781, 218, 37, 4, 997, 5}; IoBuffer Chunks[9] = {CreateChunk(ChunkSizes[0]), CreateChunk(ChunkSizes[1]), CreateChunk(ChunkSizes[2]), CreateChunk(ChunkSizes[3]), CreateChunk(ChunkSizes[4]), CreateChunk(ChunkSizes[5]), CreateChunk(ChunkSizes[6]), CreateChunk(ChunkSizes[7]), CreateChunk(ChunkSizes[8])}; IoHash ChunkHashes[9] = { IoHash::HashBuffer(Chunks[0].Data(), Chunks[0].Size()), IoHash::HashBuffer(Chunks[1].Data(), Chunks[1].Size()), IoHash::HashBuffer(Chunks[2].Data(), Chunks[2].Size()), IoHash::HashBuffer(Chunks[3].Data(), Chunks[3].Size()), IoHash::HashBuffer(Chunks[4].Data(), Chunks[4].Size()), IoHash::HashBuffer(Chunks[5].Data(), Chunks[5].Size()), IoHash::HashBuffer(Chunks[6].Data(), Chunks[6].Size()), IoHash::HashBuffer(Chunks[7].Data(), Chunks[7].Size()), IoHash::HashBuffer(Chunks[8].Data(), Chunks[8].Size()), }; CasStore->InsertChunk(Chunks[0], ChunkHashes[0]); CasStore->InsertChunk(Chunks[1], ChunkHashes[1]); CasStore->InsertChunk(Chunks[2], ChunkHashes[2]); CasStore->InsertChunk(Chunks[3], ChunkHashes[3]); CasStore->InsertChunk(Chunks[4], ChunkHashes[4]); CasStore->InsertChunk(Chunks[5], ChunkHashes[5]); CasStore->InsertChunk(Chunks[6], ChunkHashes[6]); CasStore->InsertChunk(Chunks[7], ChunkHashes[7]); CasStore->InsertChunk(Chunks[8], ChunkHashes[8]); CidStoreSize InitialSize = CasStore->TotalSize(); // Keep first and last { GcContext GcCtx; GcCtx.CollectSmallObjects(true); std::vector KeepChunks; KeepChunks.push_back(ChunkHashes[0]); KeepChunks.push_back(ChunkHashes[8]); GcCtx.AddRetainedCids(KeepChunks); CasStore->Flush(); Gc.CollectGarbage(GcCtx); CHECK(CasStore->ContainsChunk(ChunkHashes[0])); CHECK(!CasStore->ContainsChunk(ChunkHashes[1])); CHECK(!CasStore->ContainsChunk(ChunkHashes[2])); CHECK(!CasStore->ContainsChunk(ChunkHashes[3])); CHECK(!CasStore->ContainsChunk(ChunkHashes[4])); CHECK(!CasStore->ContainsChunk(ChunkHashes[5])); CHECK(!CasStore->ContainsChunk(ChunkHashes[6])); CHECK(!CasStore->ContainsChunk(ChunkHashes[7])); CHECK(CasStore->ContainsChunk(ChunkHashes[8])); CHECK(ChunkHashes[0] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[0]))); CHECK(ChunkHashes[8] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[8]))); } CasStore->InsertChunk(Chunks[1], ChunkHashes[1]); CasStore->InsertChunk(Chunks[2], ChunkHashes[2]); CasStore->InsertChunk(Chunks[3], ChunkHashes[3]); CasStore->InsertChunk(Chunks[4], ChunkHashes[4]); CasStore->InsertChunk(Chunks[5], ChunkHashes[5]); CasStore->InsertChunk(Chunks[6], ChunkHashes[6]); CasStore->InsertChunk(Chunks[7], ChunkHashes[7]); // Keep last { GcContext GcCtx; GcCtx.CollectSmallObjects(true); std::vector KeepChunks; KeepChunks.push_back(ChunkHashes[8]); GcCtx.AddRetainedCids(KeepChunks); CasStore->Flush(); Gc.CollectGarbage(GcCtx); CHECK(!CasStore->ContainsChunk(ChunkHashes[0])); CHECK(!CasStore->ContainsChunk(ChunkHashes[1])); CHECK(!CasStore->ContainsChunk(ChunkHashes[2])); CHECK(!CasStore->ContainsChunk(ChunkHashes[3])); CHECK(!CasStore->ContainsChunk(ChunkHashes[4])); CHECK(!CasStore->ContainsChunk(ChunkHashes[5])); CHECK(!CasStore->ContainsChunk(ChunkHashes[6])); CHECK(!CasStore->ContainsChunk(ChunkHashes[7])); CHECK(CasStore->ContainsChunk(ChunkHashes[8])); CHECK(ChunkHashes[8] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[8]))); CasStore->InsertChunk(Chunks[1], ChunkHashes[1]); CasStore->InsertChunk(Chunks[2], ChunkHashes[2]); CasStore->InsertChunk(Chunks[3], ChunkHashes[3]); CasStore->InsertChunk(Chunks[4], ChunkHashes[4]); CasStore->InsertChunk(Chunks[5], ChunkHashes[5]); CasStore->InsertChunk(Chunks[6], ChunkHashes[6]); CasStore->InsertChunk(Chunks[7], ChunkHashes[7]); } // Keep mixed { GcContext GcCtx; GcCtx.CollectSmallObjects(true); std::vector KeepChunks; KeepChunks.push_back(ChunkHashes[1]); KeepChunks.push_back(ChunkHashes[4]); KeepChunks.push_back(ChunkHashes[7]); GcCtx.AddRetainedCids(KeepChunks); CasStore->Flush(); Gc.CollectGarbage(GcCtx); CHECK(!CasStore->ContainsChunk(ChunkHashes[0])); CHECK(CasStore->ContainsChunk(ChunkHashes[1])); CHECK(!CasStore->ContainsChunk(ChunkHashes[2])); CHECK(!CasStore->ContainsChunk(ChunkHashes[3])); CHECK(CasStore->ContainsChunk(ChunkHashes[4])); CHECK(!CasStore->ContainsChunk(ChunkHashes[5])); CHECK(!CasStore->ContainsChunk(ChunkHashes[6])); CHECK(CasStore->ContainsChunk(ChunkHashes[7])); CHECK(!CasStore->ContainsChunk(ChunkHashes[8])); CHECK(ChunkHashes[1] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[1]))); CHECK(ChunkHashes[4] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[4]))); CHECK(ChunkHashes[7] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[7]))); CasStore->InsertChunk(Chunks[0], ChunkHashes[0]); CasStore->InsertChunk(Chunks[2], ChunkHashes[2]); CasStore->InsertChunk(Chunks[3], ChunkHashes[3]); CasStore->InsertChunk(Chunks[5], ChunkHashes[5]); CasStore->InsertChunk(Chunks[6], ChunkHashes[6]); CasStore->InsertChunk(Chunks[8], ChunkHashes[8]); } // Keep multiple at end { GcContext GcCtx; GcCtx.CollectSmallObjects(true); std::vector KeepChunks; KeepChunks.push_back(ChunkHashes[6]); KeepChunks.push_back(ChunkHashes[7]); KeepChunks.push_back(ChunkHashes[8]); GcCtx.AddRetainedCids(KeepChunks); CasStore->Flush(); Gc.CollectGarbage(GcCtx); CHECK(!CasStore->ContainsChunk(ChunkHashes[0])); CHECK(!CasStore->ContainsChunk(ChunkHashes[1])); CHECK(!CasStore->ContainsChunk(ChunkHashes[2])); CHECK(!CasStore->ContainsChunk(ChunkHashes[3])); CHECK(!CasStore->ContainsChunk(ChunkHashes[4])); CHECK(!CasStore->ContainsChunk(ChunkHashes[5])); CHECK(CasStore->ContainsChunk(ChunkHashes[6])); CHECK(CasStore->ContainsChunk(ChunkHashes[7])); CHECK(CasStore->ContainsChunk(ChunkHashes[8])); CHECK(ChunkHashes[6] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[6]))); CHECK(ChunkHashes[7] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[7]))); CHECK(ChunkHashes[8] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[8]))); CasStore->InsertChunk(Chunks[0], ChunkHashes[0]); CasStore->InsertChunk(Chunks[1], ChunkHashes[1]); CasStore->InsertChunk(Chunks[2], ChunkHashes[2]); CasStore->InsertChunk(Chunks[3], ChunkHashes[3]); CasStore->InsertChunk(Chunks[4], ChunkHashes[4]); CasStore->InsertChunk(Chunks[5], ChunkHashes[5]); } // Verify that we nicely appended blocks even after all GC operations CHECK(ChunkHashes[0] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[0]))); CHECK(ChunkHashes[1] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[1]))); CHECK(ChunkHashes[2] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[2]))); CHECK(ChunkHashes[3] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[3]))); CHECK(ChunkHashes[4] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[4]))); CHECK(ChunkHashes[5] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[5]))); CHECK(ChunkHashes[6] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[6]))); CHECK(ChunkHashes[7] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[7]))); CHECK(ChunkHashes[8] == IoHash::HashBuffer(CasStore->FindChunk(ChunkHashes[8]))); auto FinalSize = CasStore->TotalSize(); CHECK(InitialSize.TinySize == FinalSize.TinySize); } #endif void gc_forcelink() { } } // namespace zen