// Copyright Epic Games, Inc. All Rights Reserved. #pragma once #include #include #include #include #include #include #include #include "cas.h" #include #include ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END namespace zen { class BasicFile; /** CAS storage strategy using a file-per-chunk storage strategy */ struct FileCasStrategy final : public GcStorage, public GcReferenceStore { FileCasStrategy(GcManager& Gc); ~FileCasStrategy(); void Initialize(const std::filesystem::path& RootDirectory, bool IsNewStore); CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore::InsertMode Mode = CasStore::InsertMode::kMayBeMovedInPlace); IoBuffer FindChunk(const IoHash& ChunkHash); bool HaveChunk(const IoHash& ChunkHash); void FilterChunks(HashKeySet& InOutChunks); bool IterateChunks(std::span ChunkHashes, const std::function& AsyncCallback, WorkerThreadPool* OptionalWorkerPool); void Flush(); virtual void ScrubStorage(ScrubContext& ScrubCtx) override; virtual GcStorageSize StorageSize() const override; virtual std::string GetGcName(GcCtx& Ctx) override; virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats) override; private: void MakeIndexSnapshot(bool ResetLog); uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion); uint64_t ReadLog(const std::filesystem::path& LogPath, uint64_t LogPosition); LoggerRef Log() { return m_Log; } struct IndexEntry { uint64_t Size = 0; }; using IndexMap = tsl::robin_map; LoggerRef m_Log; GcManager& m_Gc; std::filesystem::path m_RootDirectory; RwLock m_Lock; IndexMap m_Index; RwLock m_ShardLocks[256]; // TODO: these should be spaced out so they don't share cache lines std::atomic_uint64_t m_TotalSize{}; bool m_IsInitialized = false; struct FileCasIndexEntry { static const uint32_t kTombStone = 0x0000'0001; bool IsFlagSet(const uint32_t Flag) const { return (Flags & kTombStone) == Flag; } IoHash Key; uint32_t Flags = 0; uint64_t Size = 0; }; static bool ValidateEntry(const FileCasIndexEntry& Entry, std::string& OutReason); static std::vector ScanFolderForCasFiles(const std::filesystem::path& RootDir); static_assert(sizeof(FileCasIndexEntry) == 32); TCasLogFile m_CasLog; uint64_t m_LogFlushPosition = 0; inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardLocks[Hash.Hash[19]]; } void IterateChunks(std::function&& Callback); void DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec); IoBuffer SafeOpenChunk(const IoHash& ChunkHash, uint64_t ExpectedSize); struct ShardingHelper { ShardingHelper(const std::filesystem::path& RootPath, const IoHash& ChunkHash); size_t Shard2len = 0; ExtendablePathBuilder<128> ShardedPath; }; bool UpdateIndex(const IoHash& ChunkHash, uint64_t ChunkSize); friend class FileCasReferencePruner; friend class FileCasStoreCompactor; }; void filecas_forcelink(); } // namespace zen