Merge branch 'main' into zs/put-overwrite-policy

8 files changed, 352 insertions, 153 deletions

diff --git a/src/zenstore/include/zenstore/accesstime.h b/src/zenstore/include/zenstore/accesstime.h
new file mode 100644
index 000000000..e53937b52
--- /dev/null
+++ b/src/zenstore/include/zenstore/accesstime.h
@@ -0,0 +1,53 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenstore/gc.h>
+
+#include <gsl/gsl-lite.hpp>
+
+namespace zen {
+
+// This store the access time as seconds since epoch internally in a 32-bit value giving is a range of 136 years since epoch
+struct AccessTime
+{
+	explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSecondsSinceEpoch(Tick)) {}
+	AccessTime& operator=(GcClock::Tick Tick) noexcept
+	{
+		SecondsSinceEpoch.store(ToSecondsSinceEpoch(Tick), std::memory_order_relaxed);
+		return *this;
+	}
+	operator GcClock::Tick() const noexcept
+	{
+		return std::chrono::duration_cast<GcClock::Duration>(std::chrono::seconds(SecondsSinceEpoch.load(std::memory_order_relaxed)))
+			.count();
+	}
+
+	AccessTime(AccessTime&& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {}
+	AccessTime(const AccessTime& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {}
+	AccessTime& operator=(AccessTime&& Rhs) noexcept
+	{
+		SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed);
+		return *this;
+	}
+	AccessTime& operator=(const AccessTime& Rhs) noexcept
+	{
+		SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed);
+		return *this;
+	}
+
+	void SetSecondsSinceEpoch(uint32_t InSecondsSinceEpoch) { SecondsSinceEpoch.store(InSecondsSinceEpoch); }
+
+	uint32_t GetSecondsSinceEpoch() const { return SecondsSinceEpoch.load(); }
+
+private:
+	AccessTime(uint32_t InSecondsSinceEpoch) noexcept : SecondsSinceEpoch(InSecondsSinceEpoch) {}
+
+	static uint32_t ToSecondsSinceEpoch(GcClock::Tick Tick)
+	{
+		return gsl::narrow<uint32_t>(std::chrono::duration_cast<std::chrono::seconds>(GcClock::Duration(Tick)).count());
+	}
+	std::atomic_uint32_t SecondsSinceEpoch;
+};
+
+}  // namespace zen
diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h
index 97357e5cb..fce05766f 100644
--- a/src/zenstore/include/zenstore/blockstore.h
+++ b/src/zenstore/include/zenstore/blockstore.h
@@ -94,7 +94,7 @@ struct BlockStoreFile : public RefCounted
 	IoBuffer					 GetChunk(uint64_t Offset, uint64_t Size);
 	void						 Read(void* Data, uint64_t Size, uint64_t FileOffset);
 	void						 Write(const void* Data, uint64_t Size, uint64_t FileOffset);
-	void						 Flush();
+	void						 Flush(uint64_t FinalSize = (uint64_t)-1);
 	BasicFile&					 GetBasicFile();
 	void	 StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
 	bool	 IsOpen() const;
@@ -107,7 +107,7 @@ private:
 	const std::filesystem::path m_Path;
 	IoBuffer					m_IoBuffer;
 	BasicFile					m_File;
-	uint64_t					m_CachedFileSize = 0;
+	std::atomic<uint64_t>		m_CachedFileSize = 0;
 };
 
 class BlockStoreCompactState;
@@ -127,7 +127,8 @@ public:
 	typedef std::vector<std::pair<size_t, BlockStoreLocation>> MovedChunksArray;
 	typedef std::vector<size_t>								   ChunkIndexArray;
 
-	typedef std::function<bool(const MovedChunksArray& MovedChunks, uint64_t FreedDiskSpace)>			 CompactCallback;
+	typedef std::function<bool(const MovedChunksArray& MovedChunks, const ChunkIndexArray& ScrubbedChunks, uint64_t FreedDiskSpace)>
+																										 CompactCallback;
 	typedef std::function<uint64_t()>																	 ClaimDiskReserveCallback;
 	typedef std::function<bool(size_t ChunkIndex, const void* Data, uint64_t Size)>						 IterateChunksSmallSizeCallback;
 	typedef std::function<bool(size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size)> IterateChunksLargeSizeCallback;
@@ -146,18 +147,19 @@ public:
 
 	typedef tsl::robin_set<uint32_t> BlockIndexSet;
 
-	// Ask the store to create empty blocks for all locations that does not have a block
 	// Remove any block that is not referenced
-	void			   SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks);
-	BlockEntryCountMap GetBlocksToCompact(const BlockUsageMap& BlockUsage, uint32_t BlockUsageThresholdPercent);
+	// Return a list of blocks that are not present
+	[[nodiscard]] BlockIndexSet SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks);
+	BlockEntryCountMap			GetBlocksToCompact(const BlockUsageMap& BlockUsage, uint32_t BlockUsageThresholdPercent);
 
 	void Close();
 
 	void WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, const WriteChunkCallback& Callback);
 
 	typedef std::function<void(std::span<BlockStoreLocation> Locations)> WriteChunksCallback;
-	void WriteChunks(std::span<IoBuffer> Datas, uint32_t Alignment, const WriteChunksCallback& Callback);
+	void WriteChunks(std::span<const IoBuffer> Datas, uint32_t Alignment, const WriteChunksCallback& Callback);
 
+	bool	 HasChunk(const BlockStoreLocation& Location) const;
 	IoBuffer TryGetChunk(const BlockStoreLocation& Location) const;
 	void	 Flush(bool ForceNewBlock);
 
@@ -172,7 +174,7 @@ public:
 	void CompactBlocks(
 		const BlockStoreCompactState&	CompactState,
 		uint32_t						PayloadAlignment,
-		const CompactCallback&			ChangeCallback		= [](const MovedChunksArray&, uint64_t) { return true; },
+		const CompactCallback&			ChangeCallback = [](const MovedChunksArray&, const ChunkIndexArray&, uint64_t) { return true; },
 		const ClaimDiskReserveCallback& DiskReserveCallback = []() { return 0; },
 		std::string_view				LogPrefix			= {});
 
diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h
new file mode 100644
index 000000000..adf48dc26
--- /dev/null
+++ b/src/zenstore/include/zenstore/buildstore/buildstore.h
@@ -0,0 +1,228 @@
+
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenstore/blockstore.h>
+
+#include <zencore/iohash.h>
+#include <zenstore/accesstime.h>
+#include <zenstore/caslog.h>
+#include <zenstore/gc.h>
+#include "../compactcas.h"
+#include "../filecas.h"
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+struct BuildStoreConfig
+{
+	std::filesystem::path RootDirectory;
+	uint32_t			  SmallBlobBlockStoreMaxBlockSize	   = 256 * 1024 * 1024;
+	uint64_t			  SmallBlobBlockStoreMaxBlockEmbedSize = 1 * 1024 * 1024;
+	uint32_t			  SmallBlobBlockStoreAlignement		   = 16;
+	uint32_t			  MetadataBlockStoreMaxBlockSize	   = 64 * 1024 * 1024;
+	uint32_t			  MetadataBlockStoreAlignement		   = 8;
+	uint64_t			  MaxDiskSpaceLimit					   = 1u * 1024u * 1024u * 1024u * 1024u;  // 1TB
+};
+
+class BuildStore : public GcReferencer, public GcReferenceLocker, public GcStorage
+{
+public:
+	explicit BuildStore(const BuildStoreConfig& Config, GcManager& Gc);
+	virtual ~BuildStore();
+
+	void	 PutBlob(const IoHash& BlobHashes, const IoBuffer& Payload);
+	IoBuffer GetBlob(const IoHash& BlobHashes);
+
+	struct BlobExistsResult
+	{
+		bool HasBody	 = 0;
+		bool HasMetadata = 0;
+	};
+
+	std::vector<BlobExistsResult> BlobsExists(std::span<const IoHash> BlobHashes);
+
+	void				  PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoBuffer> MetaDatas);
+	std::vector<IoBuffer> GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* OptionalWorkerPool);
+
+	void Flush();
+
+	struct StorageStats
+	{
+		uint64_t EntryCount		   = 0;
+		uint64_t LargeBlobCount	   = 0;
+		uint64_t LargeBlobBytes	   = 0;
+		uint64_t SmallBlobCount	   = 0;
+		uint64_t SmallBlobBytes	   = 0;
+		uint64_t MetadataCount	   = 0;
+		uint64_t MetadataByteCount = 0;
+	};
+
+	StorageStats GetStorageStats() const;
+
+#if ZEN_WITH_TESTS
+	std::optional<AccessTime> GetLastAccessTime(const IoHash& Key) const;
+	bool					  SetLastAccessTime(const IoHash& Key, const AccessTime& Time);
+#endif	// ZEN_WITH_TESTS
+
+private:
+	LoggerRef Log() { return m_Log; }
+
+	void CompactState();
+
+	uint64_t ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount);
+	uint64_t ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount);
+	void	 WriteAccessTimes(const RwLock::ExclusiveLockScope&, const std::filesystem::path& AccessTimesPath);
+	void	 ReadAccessTimes(const RwLock::ExclusiveLockScope&, const std::filesystem::path& AccessTimesPath);
+
+	//////// GcReferencer
+	virtual std::string						   GetGcName(GcCtx& Ctx) override;
+	virtual GcStoreCompactor*				   RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override;
+	virtual std::vector<GcReferenceChecker*>   CreateReferenceCheckers(GcCtx& Ctx) override;
+	virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override;
+
+	//////// GcReferenceLocker
+	virtual std::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) override;
+
+	//////// GcStorage
+	virtual void		  ScrubStorage(ScrubContext& ScrubCtx) override;
+	virtual GcStorageSize StorageSize() const override;
+
+#pragma pack(push)
+#pragma pack(1)
+	struct PayloadEntry
+	{
+		PayloadEntry() {}
+		PayloadEntry(uint64_t Flags, uint64_t Size)
+		{
+			ZEN_ASSERT((Size & 0x00ffffffffffffffu) == Size);
+			ZEN_ASSERT((Flags & (kTombStone | kStandalone)) == Flags);
+			FlagsAndSize = (Size << 8) | Flags;
+		}
+		static const uint8_t kTombStone	 = 0x10u;  // Represents a deleted key/value
+		static const uint8_t kStandalone = 0x20u;  // This payload is stored as a standalone value
+
+		uint64_t FlagsAndSize = 0;
+		uint64_t GetSize() const { return FlagsAndSize >> 8; }
+		uint8_t	 GetFlags() const { return uint8_t(FlagsAndSize & 0xff); }
+		void	 AddFlag(uint8_t Flag) { FlagsAndSize |= Flag; }
+		void	 SetSize(uint64_t Size)
+		{
+			ZEN_ASSERT((Size & 0x00ffffffffffffffu) == Size);
+			FlagsAndSize = (Size << 8) | (FlagsAndSize & 0xff);
+		}
+		void SetFlags(uint8_t Flags) { FlagsAndSize = (FlagsAndSize & 0xffffffffffffff00u) | Flags; }
+	};
+	static_assert(sizeof(PayloadEntry) == 8);
+
+	struct PayloadDiskEntry
+	{
+		PayloadEntry Entry;		// 8 bytes
+		IoHash		 BlobHash;	// 20 bytes
+	};
+	static_assert(sizeof(PayloadDiskEntry) == 28);
+
+	struct MetadataEntry
+	{
+		BlockStoreLocation Location;  // 12 bytes
+
+		ZenContentType		 ContentType = ZenContentType::kCOUNT;	// 1 byte
+		static const uint8_t kTombStone	 = 0x10u;					// Represents a deleted key/value
+		uint8_t				 Flags		 = 0;						// 1 byte
+
+		uint8_t Reserved1 = 0;
+		uint8_t Reserved2 = 0;
+	};
+	static_assert(sizeof(MetadataEntry) == 16);
+
+	struct MetadataDiskEntry
+	{
+		MetadataEntry Entry;	 // 16 bytes
+		IoHash		  BlobHash;	 // 20 bytes
+		uint8_t		  Reserved1 = 0;
+		uint8_t		  Reserved2 = 0;
+		uint8_t		  Reserved3 = 0;
+		uint8_t		  Reserved4 = 0;
+	};
+	static_assert(sizeof(MetadataDiskEntry) == 40);
+
+#pragma pack(pop)
+
+	static bool ValidatePayloadDiskEntry(const PayloadDiskEntry& Entry, std::string& OutReason);
+	static bool ValidateMetadataDiskEntry(const MetadataDiskEntry& Entry, std::string& OutReason);
+
+	struct PayloadIndex
+	{
+		uint32_t Index = std::numeric_limits<uint32_t>::max();
+
+		operator bool() const { return Index != std::numeric_limits<uint32_t>::max(); };
+		PayloadIndex() = default;
+		explicit PayloadIndex(size_t InIndex) : Index(uint32_t(InIndex)) {}
+					operator size_t() const { return Index; };
+		inline auto operator<=>(const PayloadIndex& Other) const = default;
+	};
+
+	struct MetadataIndex
+	{
+		uint32_t Index = std::numeric_limits<uint32_t>::max();
+
+		operator bool() const { return Index != std::numeric_limits<uint32_t>::max(); };
+		MetadataIndex() = default;
+		explicit MetadataIndex(size_t InIndex) : Index(uint32_t(InIndex)) {}
+					operator size_t() const { return Index; };
+		inline auto operator<=>(const MetadataIndex& Other) const = default;
+	};
+
+	struct BlobIndex
+	{
+		uint32_t Index = std::numeric_limits<uint32_t>::max();
+
+		operator bool() const { return Index != std::numeric_limits<uint32_t>::max(); };
+		BlobIndex() = default;
+		explicit BlobIndex(size_t InIndex) : Index(uint32_t(InIndex)) {}
+					operator size_t() const { return Index; };
+		inline auto operator<=>(const BlobIndex& Other) const = default;
+	};
+
+	struct BlobEntry
+	{
+		PayloadIndex  Payload;
+		MetadataIndex Metadata;
+		AccessTime	  LastAccessTime;
+	};
+	static_assert(sizeof(BlobEntry) == 12);
+
+	LoggerRef			   m_Log;
+	const BuildStoreConfig m_Config;
+	GcManager&			   m_Gc;
+
+	mutable RwLock m_Lock;
+
+	std::vector<PayloadEntry>  m_PayloadEntries;
+	std::vector<MetadataEntry> m_MetadataEntries;
+
+	std::vector<BlobEntry>							  m_BlobEntries;
+	tsl::robin_map<IoHash, BlobIndex, IoHash::Hasher> m_BlobLookup;
+
+	FileCasStrategy		 m_LargeBlobStore;
+	CasContainerStrategy m_SmallBlobStore;
+	BlockStore			 m_MetadataBlockStore;
+
+	TCasLogFile<PayloadDiskEntry>  m_PayloadlogFile;
+	TCasLogFile<MetadataDiskEntry> m_MetadatalogFile;
+	uint64_t					   m_BlobLogFlushPosition = 0;
+	uint64_t					   m_MetaLogFlushPosition = 0;
+
+	std::unique_ptr<HashSet> m_TrackedCacheKeys;
+	std::atomic<uint64_t>	 m_LastAccessTimeUpdateCount;
+
+	friend class BuildStoreGcReferenceChecker;
+	friend class BuildStoreGcReferencePruner;
+	friend class BuildStoreGcCompator;
+};
+
+void buildstore_forcelink();
+
+}  // namespace zen
diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h
index 4f5c905ee..11d13bede 100644
--- a/src/zenstore/include/zenstore/cache/cachedisklayer.h
+++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h
@@ -5,6 +5,7 @@
 #include "cacheshared.h"
 
 #include <zencore/stats.h>
+#include <zenstore/accesstime.h>
 #include <zenstore/blockstore.h>
 #include <zenstore/caslog.h>
 
@@ -12,8 +13,9 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #include <tsl/robin_map.h>
 ZEN_THIRD_PARTY_INCLUDES_END
 
+#include <EASTL/string.h>
+#include <EASTL/unordered_map.h>
 #include <filesystem>
-#include <unordered_map>
 
 namespace zen {
 
@@ -118,6 +120,9 @@ public:
 
 	struct Configuration
 	{
+		typedef eastl::unordered_map<std::string, BucketConfiguration, std::hash<std::string>, std::equal_to<std::string>>
+							BucketConfigMap_t;
+		BucketConfigMap_t	BucketConfigMap;
 		BucketConfiguration BucketConfig;
 		uint64_t			MemCacheTargetFootprintBytes = 512 * 1024 * 1024;
 		uint64_t			MemCacheTrimIntervalSeconds	 = 60;
@@ -176,7 +181,7 @@ public:
 	~ZenCacheDiskLayer();
 
 	struct GetBatchHandle;
-	GetBatchHandle* BeginGetBatch(std::vector<ZenCacheValue>& OutResult);
+	GetBatchHandle* BeginGetBatch(ZenCacheValueVec_t& OutResult);
 	void			EndGetBatch(GetBatchHandle* Batch) noexcept;
 	bool			Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
 	void			Get(std::string_view Bucket, const IoHash& HashKey, GetBatchHandle& BatchHandle);
@@ -185,16 +190,16 @@ public:
 	PutBatchHandle* BeginPutBatch(std::vector<PutResult>& OutResult);
 	void			EndPutBatch(PutBatchHandle* Batch) noexcept;
 
-	PutResult Put(std::string_view	   Bucket,
-				  const IoHash&		   HashKey,
-				  const ZenCacheValue& Value,
-				  std::span<IoHash>	   References,
-				  bool				   Overwrite,
-				  PutBatchHandle*	   OptionalBatchHandle);
-	bool	  Drop();
-	bool	  DropBucket(std::string_view Bucket);
-	void	  Flush();
-	void	  ScrubStorage(ScrubContext& Ctx);
+	PutResult			  Put(std::string_view	   Bucket,
+							  const IoHash&		   HashKey,
+							  const ZenCacheValue& Value,
+							  std::span<IoHash>	   References,
+							  bool				   Overwrite,
+							  PutBatchHandle*	   OptionalBatchHandle);
+	std::function<void()> Drop();
+	std::function<void()> DropBucket(std::string_view Bucket);
+	void				  Flush();
+	void				  ScrubStorage(ScrubContext& Ctx);
 
 	void		  DiscoverBuckets();
 	GcStorageSize StorageSize() const;
@@ -224,27 +229,30 @@ public:
 	  */
 	struct CacheBucket : public GcReferencer
 	{
-		CacheBucket(GcManager& Gc, std::atomic_uint64_t& OuterCacheMemoryUsage, std::string BucketName, const BucketConfiguration& Config);
+		CacheBucket(GcManager&				   Gc,
+					std::atomic_uint64_t&	   OuterCacheMemoryUsage,
+					std::string_view		   BucketName,
+					const BucketConfiguration& Config);
 		~CacheBucket();
 
 		bool OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
 
 		struct GetBatchHandle;
-		GetBatchHandle* BeginGetBatch(std::vector<ZenCacheValue>& OutResult);
+		GetBatchHandle* BeginGetBatch(ZenCacheValueVec_t& OutResult);
 		void			EndGetBatch(GetBatchHandle* Batch) noexcept;
 		bool			Get(const IoHash& HashKey, ZenCacheValue& OutValue);
 		void			Get(const IoHash& HashKey, GetBatchHandle& BatchHandle);
 
 		struct PutBatchHandle;
-		PutBatchHandle*			BeginPutBatch(std::vector<ZenCacheDiskLayer::PutResult>& OutResult);
-		void					EndPutBatch(PutBatchHandle* Batch) noexcept;
+		PutBatchHandle*	BeginPutBatch(std::vector<ZenCacheDiskLayer::PutResult>& OutResult);
+		void			EndPutBatch(PutBatchHandle* Batch) noexcept;
 		PutResult				Put(const IoHash&		 HashKey,
 									const ZenCacheValue& Value,
 									std::span<IoHash>	 References,
 									bool				 Overwrite,
 									PutBatchHandle*		 OptionalBatchHandle);
-		uint64_t				MemCacheTrim(GcClock::TimePoint ExpireTime);
-		bool					Drop();
+		uint64_t MemCacheTrim(GcClock::TimePoint ExpireTime);
+		std::function<void()>	Drop();
 		void					Flush();
 		void					ScrubStorage(ScrubContext& Ctx);
 		RwLock::SharedLockScope GetGcReferencerLock();
@@ -413,20 +421,23 @@ public:
 		void SaveSnapshot(const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; });
 		void WriteIndexSnapshot(
 			RwLock::ExclusiveLockScope&,
-			bool							 FlushLockPosition,
+			uint64_t						 LogPosition,
+			bool							 ResetLog,
 			const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; })
 		{
-			WriteIndexSnapshotLocked(FlushLockPosition, ClaimDiskReserveFunc);
+			WriteIndexSnapshotLocked(LogPosition, ResetLog, ClaimDiskReserveFunc);
 		}
 		void WriteIndexSnapshot(
 			RwLock::SharedLockScope&,
-			bool							 FlushLockPosition,
+			uint64_t						 LogPosition,
+			bool							 ResetLog,
 			const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; })
 		{
-			WriteIndexSnapshotLocked(FlushLockPosition, ClaimDiskReserveFunc);
+			WriteIndexSnapshotLocked(LogPosition, ResetLog, ClaimDiskReserveFunc);
 		}
 		void WriteIndexSnapshotLocked(
-			bool							 FlushLockPosition,
+			uint64_t						 LogPosition,
+			bool							 ResetLog,
 			const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; });
 
 		void CompactState(RwLock::ExclusiveLockScope&  IndexLock,
@@ -498,18 +509,20 @@ private:
 	bool	 StartAsyncMemCacheTrim();
 	void	 MemCacheTrim();
 
-	GcManager&													  m_Gc;
-	JobQueue&													  m_JobQueue;
-	std::filesystem::path										  m_RootDir;
-	Configuration												  m_Configuration;
-	std::atomic_uint64_t										  m_TotalMemCachedSize{};
-	std::atomic_bool											  m_IsMemCacheTrimming = false;
-	std::atomic<GcClock::Tick>									  m_NextAllowedTrimTick;
-	mutable RwLock												  m_Lock;
-	std::unordered_map<std::string, std::unique_ptr<CacheBucket>> m_Buckets;
-	std::vector<std::unique_ptr<CacheBucket>>					  m_DroppedBuckets;
-	uint32_t													  m_UpdateCaptureRefCounter = 0;
-	std::unique_ptr<std::vector<std::string>>					  m_CapturedBuckets;
+	typedef eastl::unordered_map<std::string, std::unique_ptr<CacheBucket>, std::hash<std::string>, std::equal_to<std::string>> BucketMap_t;
+
+	GcManager&								  m_Gc;
+	JobQueue&								  m_JobQueue;
+	std::filesystem::path					  m_RootDir;
+	Configuration							  m_Configuration;
+	std::atomic_uint64_t					  m_TotalMemCachedSize{};
+	std::atomic_bool						  m_IsMemCacheTrimming = false;
+	std::atomic<GcClock::Tick>				  m_NextAllowedTrimTick;
+	mutable RwLock							  m_Lock;
+	BucketMap_t								  m_Buckets;
+	std::vector<std::unique_ptr<CacheBucket>> m_DroppedBuckets;
+	uint32_t								  m_UpdateCaptureRefCounter = 0;
+	std::unique_ptr<std::vector<std::string>> m_CapturedBuckets;
 
 	ZenCacheDiskLayer(const ZenCacheDiskLayer&) = delete;
 	ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete;
diff --git a/src/zenstore/include/zenstore/cache/cacheshared.h b/src/zenstore/include/zenstore/cache/cacheshared.h
index dc0c341d0..8f40ae727 100644
--- a/src/zenstore/include/zenstore/cache/cacheshared.h
+++ b/src/zenstore/include/zenstore/cache/cacheshared.h
@@ -6,6 +6,8 @@
 #include <zencore/iohash.h>
 #include <zenstore/gc.h>
 
+#include <EASTL/fixed_vector.h>
+
 #include <gsl/gsl-lite.hpp>
 #include <unordered_map>
 
@@ -32,6 +34,8 @@ struct ZenCacheValue
 	IoHash	 RawHash = IoHash::Zero;
 };
 
+typedef eastl::fixed_vector<ZenCacheValue, 16> ZenCacheValueVec_t;
+
 struct CacheValueDetails
 {
 	struct ValueDetails
@@ -76,42 +80,4 @@ enum class PutStatus
 bool IsKnownBadBucketName(std::string_view BucketName);
 bool ValidateIoBuffer(ZenContentType ContentType, IoBuffer Buffer);
 
-//////////////////////////////////////////////////////////////////////////
-
-// This store the access time as seconds since epoch internally in a 32-bit value giving is a range of 136 years since epoch
-struct AccessTime
-{
-	explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSeconds(Tick)) {}
-	AccessTime& operator=(GcClock::Tick Tick) noexcept
-	{
-		SecondsSinceEpoch.store(ToSeconds(Tick), std::memory_order_relaxed);
-		return *this;
-	}
-	operator GcClock::Tick() const noexcept
-	{
-		return std::chrono::duration_cast<GcClock::Duration>(std::chrono::seconds(SecondsSinceEpoch.load(std::memory_order_relaxed)))
-			.count();
-	}
-
-	AccessTime(AccessTime&& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {}
-	AccessTime(const AccessTime& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {}
-	AccessTime& operator=(AccessTime&& Rhs) noexcept
-	{
-		SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed);
-		return *this;
-	}
-	AccessTime& operator=(const AccessTime& Rhs) noexcept
-	{
-		SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed);
-		return *this;
-	}
-
-private:
-	static uint32_t ToSeconds(GcClock::Tick Tick)
-	{
-		return gsl::narrow<uint32_t>(std::chrono::duration_cast<std::chrono::seconds>(GcClock::Duration(Tick)).count());
-	}
-	std::atomic_uint32_t SecondsSinceEpoch;
-};
-
 }  // namespace zen
diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h
index 581f7861b..b6e8e7565 100644
--- a/src/zenstore/include/zenstore/cache/structuredcachestore.h
+++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h
@@ -88,7 +88,7 @@ public:
 	void			EndPutBatch(PutBatchHandle* Batch) noexcept;
 
 	struct GetBatchHandle;
-	GetBatchHandle* BeginGetBatch(std::vector<ZenCacheValue>& OutResults);
+	GetBatchHandle* BeginGetBatch(ZenCacheValueVec_t& OutResults);
 	void			EndGetBatch(GetBatchHandle* Batch) noexcept;
 
 	bool	  Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
@@ -104,8 +104,8 @@ public:
 	void EnumerateBucketContents(std::string_view																		 Bucket,
 								 std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const;
 
-	bool Drop();
-	void Flush();
+	std::function<void()> Drop();
+	void				  Flush();
 
 	// GcStorage
 	virtual void		  ScrubStorage(ScrubContext& ScrubCtx) override;
@@ -225,14 +225,14 @@ public:
 	class GetBatch
 	{
 	public:
-		GetBatch(ZenCacheStore& CacheStore, std::string_view Namespace, std::vector<ZenCacheValue>& OutResult);
+		GetBatch(ZenCacheStore& CacheStore, std::string_view Namespace, ZenCacheValueVec_t& OutResult);
 		~GetBatch();
 
 	private:
 		ZenCacheStore&					   m_CacheStore;
 		ZenCacheNamespace*				   m_Store				  = nullptr;
 		ZenCacheNamespace::GetBatchHandle* m_NamespaceBatchHandle = nullptr;
-		std::vector<ZenCacheValue>&		   Results;
+		ZenCacheValueVec_t&				   Results;
 		friend class ZenCacheStore;
 	};
 
diff --git a/src/zenstore/include/zenstore/chunkedfile.h b/src/zenstore/include/zenstore/chunkedfile.h
deleted file mode 100644
index c6330bdbd..000000000
--- a/src/zenstore/include/zenstore/chunkedfile.h
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zencore/iobuffer.h>
-#include <zencore/iohash.h>
-#include <zencore/zencore.h>
-
-#include <functional>
-#include <vector>
-
-namespace zen {
-
-class BasicFile;
-
-struct ChunkedInfo
-{
-	uint64_t			  RawSize = 0;
-	IoHash				  RawHash;
-	std::vector<uint32_t> ChunkSequence;
-	std::vector<IoHash>	  ChunkHashes;
-};
-
-struct ChunkSource
-{
-	uint64_t Offset;  // 8
-	uint32_t Size;	  // 4
-};
-
-struct ChunkedInfoWithSource
-{
-	ChunkedInfo				 Info;
-	std::vector<ChunkSource> ChunkSources;
-};
-
-struct ChunkedParams
-{
-	bool   UseThreshold = true;
-	size_t MinSize		= (2u * 1024u) - 128u;
-	size_t MaxSize		= (16u * 1024u);
-	size_t AvgSize		= (3u * 1024u);
-};
-
-static const ChunkedParams UShaderByteCodeParams = {.UseThreshold = true, .MinSize = 17280, .MaxSize = 139264, .AvgSize = 36340};
-
-ChunkedInfoWithSource ChunkData(BasicFile& RawData, uint64_t Offset, uint64_t Size, ChunkedParams Params = {});
-void				  Reconstruct(const ChunkedInfo&							   Info,
-								  const std::filesystem::path&					   TargetPath,
-								  std::function<IoBuffer(const IoHash& ChunkHash)> GetChunk);
-IoBuffer			  SerializeChunkedInfo(const ChunkedInfo& Info);
-ChunkedInfo			  DeserializeChunkedInfo(IoBuffer& Buffer);
-
-void chunkedfile_forcelink();
-}  // namespace zen
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index 3daae0a93..3223fba39 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -55,6 +55,7 @@ struct GcSettings
 {
 	GcClock::TimePoint CacheExpireTime		  = GcClock::Now();
 	GcClock::TimePoint ProjectStoreExpireTime = GcClock::Now();
+	GcClock::TimePoint BuildStoreExpireTime	  = GcClock::Now();
 	bool			   CollectSmallObjects	  = false;
 	bool			   IsDeleteMode			  = false;
 	bool			   SkipCidDelete		  = false;
@@ -412,6 +413,7 @@ struct GcSchedulerConfig
 	std::chrono::seconds  Interval{};
 	std::chrono::seconds  MaxCacheDuration{86400};
 	std::chrono::seconds  MaxProjectStoreDuration{604800};
+	std::chrono::seconds  MaxBuildStoreDuration{604800};
 	bool				  CollectSmallObjects				= true;
 	bool				  Enabled							= true;
 	uint64_t			  DiskReserveSize					= 1ul << 28;
@@ -496,6 +498,7 @@ public:
 		bool					 CollectSmallObjects	 = false;
 		std::chrono::seconds	 MaxCacheDuration		 = std::chrono::seconds::max();
 		std::chrono::seconds	 MaxProjectStoreDuration = std::chrono::seconds::max();
+		std::chrono::seconds	 MaxBuildStoreDuration	 = std::chrono::seconds::max();
 		uint64_t				 DiskSizeSoftLimit		 = 0;
 		bool					 SkipCid				 = false;
 		bool					 SkipDelete				 = false;
@@ -528,6 +531,7 @@ private:
 	void		 SchedulerThread();
 	bool		 CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 								const GcClock::TimePoint& ProjectStoreExpireTime,
+								const GcClock::TimePoint& BuildStoreExpireTime,
 								bool					  Delete,
 								bool					  CollectSmallObjects,
 								bool					  SkipCid,
@@ -582,16 +586,3 @@ private:
 void gc_forcelink();
 
 }  // namespace zen
-
-template<>
-struct fmt::formatter<zen::GcClock::TimePoint> : formatter<string_view>
-{
-	template<typename FormatContext>
-	auto format(const zen::GcClock::TimePoint& TimePoint, FormatContext& ctx) const
-	{
-		std::time_t Time = std::chrono::system_clock::to_time_t(TimePoint);
-		char		TimeString[std::size("yyyy-mm-ddThh:mm:ss")];
-		std::strftime(std::data(TimeString), std::size(TimeString), "%FT%T", std::localtime(&Time));
-		return fmt::format_to(ctx.out(), "{}", TimeString);
-	}
-};