Merge branch 'main' into zs/web-ui-blank-import-name-fix

10 files changed, 391 insertions, 38 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4e1ce07f6..cabfbb652 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,10 @@
 ##
 - Feature: `zen oplog-export`, `zen oplog-import` for `--url` (cloud) and `--builds` (builds) option now has `--oidctoken-exe-path` to let zen run the OidcToken executable to get and refresh authentication token
+- Feature: zenserver option `--buildstore-disksizelimit` to set an soft upper limit for build storage data. Defaults to 1TB.
+- Bugfix: Restore Mac minver back to 12.5
 - Bugfix: Fixing bug where some imports were shown as a blank string in the web UI
+- Improvement: Hide progress ETA number until at least 5% of work is done to reduce misleading processing times. UE-256121
+- Improvement: Dynamically adjust console progress output based on console width. UE-256126
 
 ## 5.6.2
 - Bugfix: Changed Mac minver from 12.5 to 14.0, and removed `_LIBCPP_DISABLE_AVAILABILITY` as a define on Mac due to executable incompatibility issues
diff --git a/VERSION.txt b/VERSION.txt
index d6a86bf43..b9ae870cd 100644
--- a/VERSION.txt
+++ b/VERSION.txt
@@ -1 +1 @@
-5.6.2
+5.6.3-pre1
diff --git a/scripts/bundle.lua b/scripts/bundle.lua
index 3ca9b7ae4..1a81c9b7f 100644
--- a/scripts/bundle.lua
+++ b/scripts/bundle.lua
@@ -214,8 +214,8 @@ end
 --------------------------------------------------------------------------------
 local function main_mac(signidentity)
     -- Build and universalify
-    _build("x86_64", false, "--target_minver=14.0")
-    _build("arm64", false, "--target_minver=14.0")
+    _build("x86_64", false, "--target_minver=12.5")
+    _build("arm64", false, "--target_minver=12.5")
 
     os.mkdir("build/macosx/universal/release/")
 	local ret = _exec(
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index 6f831349b..50ee43341 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -58,7 +58,8 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 #include <zencore/memory/newdelete.h>
 
-#ifndef ZEN_PLATFORM_WINDOWS
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+#	include <sys/ioctl.h>
 #	include <unistd.h>
 #endif
 
@@ -329,29 +330,92 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak)
 	{
 		size_t ProgressBarSize = 20;
 
-		size_t		ProgressBarCount = (ProgressBarSize * PercentDone) / 100;
-		uint64_t	Completed		 = NewState.TotalCount - NewState.RemainingCount;
-		uint64_t	ETAMS			 = (Completed > 0) ? (ElapsedTimeMS * NewState.RemainingCount) / Completed : 0;
-		std::string ETA				 = (ETAMS > 0) ? fmt::format(" ETA {}", NiceTimeSpanMs(ETAMS)) : "";
-
-		std::string			   Output	   = fmt::format("\r{} {:#3}%: |{}{}|: {}{}{}",
-										 NewState.Task,
-										 PercentDone,
-										 std::string(ProgressBarCount, '#'),
-										 std::string(ProgressBarSize - ProgressBarCount, ' '),
-										 NiceTimeSpanMs(ElapsedTimeMS),
-										 ETA,
-										 NewState.Details.empty() ? "" : fmt::format(". {}", NewState.Details));
+		size_t	 ProgressBarCount = (ProgressBarSize * PercentDone) / 100;
+		uint64_t Completed		  = NewState.TotalCount - NewState.RemainingCount;
+		uint64_t ETAMS			  = (PercentDone > 5) ? (ElapsedTimeMS * NewState.RemainingCount) / Completed : 0;
+
+#if ZEN_PLATFORM_WINDOWS
+		static HANDLE			   hStdOut = GetStdHandle(STD_OUTPUT_HANDLE);
+		CONSOLE_SCREEN_BUFFER_INFO csbi;
+		GetConsoleScreenBufferInfo(hStdOut, &csbi);
+		uint32_t ConsoleColumns = (uint32_t)(csbi.srWindow.Right - csbi.srWindow.Left + 1);
+#else
+		struct winsize w;
+		ioctl(STDOUT_FILENO, TIOCGWINSZ, &w);
+		uint32_t ConsoleColumns = (uint32_t)w.ws_col;
+#endif
+
+		std::string_view TaskString = NewState.Task;
+
+		const std::string PercentString = fmt::format("{:#3}%", PercentDone);
+
+		const std::string ProgressBarString =
+			fmt::format(": |{}{}|", std::string(ProgressBarCount, '#'), std::string(ProgressBarSize - ProgressBarCount, ' '));
+
+		const std::string ElapsedString = fmt::format(": {}", NiceTimeSpanMs(ElapsedTimeMS));
+
+		const std::string ETAString = (ETAMS > 0) ? fmt::format(" ETA {}", NiceTimeSpanMs(ETAMS)) : "";
+
+		const std::string DetailsString = (!NewState.Details.empty()) ? fmt::format(". {}", NewState.Details) : "";
+
+		ExtendableStringBuilder<256> OutputBuilder;
+
+		OutputBuilder << "\r" << TaskString << PercentString;
+		if (OutputBuilder.Size() + 1 < ConsoleColumns)
+		{
+			size_t RemainingSpace = ConsoleColumns - (OutputBuilder.Size() + 1);
+			bool   ElapsedFits	  = RemainingSpace >= ElapsedString.length();
+			RemainingSpace -= ElapsedString.length();
+			bool ETAFits = ElapsedFits && RemainingSpace >= ETAString.length();
+			RemainingSpace -= ETAString.length();
+			bool DetailsFits = ETAFits && RemainingSpace >= DetailsString.length();
+			RemainingSpace -= DetailsString.length();
+			bool ProgressBarFits = DetailsFits && RemainingSpace >= ProgressBarString.length();
+			RemainingSpace -= ProgressBarString.length();
+
+			if (ProgressBarFits)
+			{
+				OutputBuilder << ProgressBarString;
+			}
+			if (ElapsedFits)
+			{
+				OutputBuilder << ElapsedString;
+			}
+			if (ETAFits)
+			{
+				OutputBuilder << ETAString;
+			}
+			if (DetailsFits)
+			{
+				OutputBuilder << DetailsString;
+			}
+		}
+
+		std::string_view	   Output	   = OutputBuilder.ToView();
 		std::string::size_type EraseLength = m_LastOutputLength > Output.length() ? (m_LastOutputLength - Output.length()) : 0;
 
-		ExtendableStringBuilder<128> LineToPrint;
-		LineToPrint << Output << std::string(EraseLength, ' ');
+		ExtendableStringBuilder<256> LineToPrint;
+
+		if (Output.length() + EraseLength >= ConsoleColumns)
+		{
+			if (m_LastOutputLength > 0)
+			{
+				LineToPrint << "\n";
+			}
+			LineToPrint << Output.substr(1);
+			DoLinebreak = true;
+		}
+		else
+		{
+			LineToPrint << Output << std::string(EraseLength, ' ');
+		}
+
 		if (DoLinebreak)
+		{
 			LineToPrint << "\n";
+		}
 
 #if ZEN_PLATFORM_WINDOWS
-		static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE);
-
 		if (m_StdoutIsTty)
 		{
 			WriteConsoleA(hStdOut, LineToPrint.c_str(), (DWORD)LineToPrint.Size(), 0, 0);
diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp
index c918f5683..75a333687 100644
--- a/src/zenserver/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/buildstore/httpbuildstore.cpp
@@ -524,6 +524,40 @@ HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request)
 	}
 	Cbo.EndObject();
 
+	Cbo.BeginObject("size");
+	{
+		BuildStore::StorageStats StorageStats = m_BuildStore.GetStorageStats();
+
+		Cbo << "count" << StorageStats.EntryCount;
+		Cbo << "bytes" << StorageStats.LargeBlobBytes + StorageStats.SmallBlobBytes + StorageStats.MetadataByteCount;
+		Cbo.BeginObject("blobs");
+		{
+			Cbo << "count" << (StorageStats.LargeBlobCount + StorageStats.SmallBlobCount);
+			Cbo << "bytes" << (StorageStats.LargeBlobBytes + StorageStats.SmallBlobBytes);
+			Cbo.BeginObject("large");
+			{
+				Cbo << "count" << StorageStats.LargeBlobCount;
+				Cbo << "bytes" << StorageStats.LargeBlobBytes;
+			}
+			Cbo.EndObject();  // large
+			Cbo.BeginObject("small");
+			{
+				Cbo << "count" << StorageStats.SmallBlobCount;
+				Cbo << "bytes" << StorageStats.SmallBlobBytes;
+			}
+			Cbo.EndObject();  // small
+		}
+		Cbo.EndObject();  // blobs
+
+		Cbo.BeginObject("metadata");
+		{
+			Cbo << "count" << StorageStats.MetadataCount;
+			Cbo << "bytes" << StorageStats.MetadataByteCount;
+		}
+		Cbo.EndObject();  // metadata
+	}
+	Cbo.EndObject();  // size
+
 	return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
 }
 
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 52f539dcd..31c104110 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -379,6 +379,7 @@ ParseConfigFile(const std::filesystem::path& Path,
 
 	////// buildsstore
 	LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv);
+	LuaOptions.AddOption("buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit");
 
 	////// network
 	LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv);
@@ -1050,6 +1051,12 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 					   "Whether the builds store is enabled or not.",
 					   cxxopts::value<bool>(ServerOptions.BuildStoreConfig.Enabled)->default_value("false"),
 					   "");
+	options.add_option("buildstore",
+					   "",
+					   "buildstore-disksizelimit",
+					   "Max number of bytes before build store entries get evicted. Default set to 1099511627776 (1TB week)",
+					   cxxopts::value<uint64_t>(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit)->default_value("1099511627776"),
+					   "");
 
 	options.add_option("stats",
 					   "",
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index a87b6f8b3..6bd7aa357 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -138,7 +138,8 @@ struct ZenProjectStoreConfig
 
 struct ZenBuildStoreConfig
 {
-	bool Enabled = false;
+	bool	 Enabled		   = false;
+	uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u;  // 1TB
 };
 
 struct ZenWorkspacesConfig
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index c7cb2ba6e..3f2f01d5a 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -266,9 +266,10 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 
 	if (ServerOptions.BuildStoreConfig.Enabled)
 	{
-		BuildStoreConfig ObjCfg;
-		ObjCfg.RootDirectory = m_DataRoot / "builds";
-		m_BuildStore		 = std::make_unique<BuildStore>(std::move(ObjCfg), m_GcManager);
+		BuildStoreConfig BuildsCfg;
+		BuildsCfg.RootDirectory		= m_DataRoot / "builds";
+		BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit;
+		m_BuildStore				= std::make_unique<BuildStore>(std::move(BuildsCfg), m_GcManager);
 	}
 
 	if (ServerOptions.StructuredCacheConfig.Enabled)
diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp
index d6d727aa9..cf518c06f 100644
--- a/src/zenstore/buildstore/buildstore.cpp
+++ b/src/zenstore/buildstore/buildstore.cpp
@@ -193,10 +193,12 @@ BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc)
 
 		m_Gc.AddGcReferencer(*this);
 		m_Gc.AddGcReferenceLocker(*this);
+		m_Gc.AddGcStorage(this);
 	}
 	catch (const std::exception& Ex)
 	{
 		ZEN_ERROR("Failed to initialize build store. Reason: '{}'", Ex.what());
+		m_Gc.RemoveGcStorage(this);
 		m_Gc.RemoveGcReferenceLocker(*this);
 		m_Gc.RemoveGcReferencer(*this);
 	}
@@ -207,6 +209,7 @@ BuildStore::~BuildStore()
 	try
 	{
 		ZEN_TRACE_CPU("BuildStore::~BuildStore");
+		m_Gc.RemoveGcStorage(this);
 		m_Gc.RemoveGcReferenceLocker(*this);
 		m_Gc.RemoveGcReferencer(*this);
 		Flush();
@@ -552,6 +555,44 @@ BuildStore::Flush()
 	}
 }
 
+BuildStore::StorageStats
+BuildStore::GetStorageStats() const
+{
+	StorageStats Result;
+	{
+		RwLock::SharedLockScope _(m_Lock);
+		Result.EntryCount = m_BlobLookup.size();
+
+		for (auto LookupIt : m_BlobLookup)
+		{
+			const BlobIndex	 ReadBlobIndex = LookupIt.second;
+			const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex];
+			if (ReadBlobEntry.Payload)
+			{
+				const PayloadEntry& Payload = m_PayloadEntries[ReadBlobEntry.Payload];
+				uint64_t			Size	= Payload.GetSize();
+				if ((Payload.GetFlags() & PayloadEntry::kStandalone) != 0)
+				{
+					Result.LargeBlobCount++;
+					Result.LargeBlobBytes += Size;
+				}
+				else
+				{
+					Result.SmallBlobCount++;
+					Result.SmallBlobBytes += Size;
+				}
+			}
+			if (ReadBlobEntry.Metadata)
+			{
+				const MetadataEntry& Metadata = m_MetadataEntries[ReadBlobEntry.Metadata];
+				Result.MetadataCount++;
+				Result.MetadataByteCount += Metadata.Location.Size;
+			}
+		}
+	}
+	return Result;
+}
+
 #if ZEN_WITH_TESTS
 std::optional<AccessTime>
 BuildStore::GetLastAccessTime(const IoHash& Key) const
@@ -1273,24 +1314,103 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 		}
 	});
 
-	const GcClock::Tick ExpireTicks = Ctx.Settings.BuildStoreExpireTime.time_since_epoch().count();
+	const GcClock::Tick					   ExpireTicks = Ctx.Settings.BuildStoreExpireTime.time_since_epoch().count();
+	std::vector<IoHash>					   ExpiredBlobs;
+	tsl::robin_set<IoHash, IoHash::Hasher> SizeDroppedBlobs;
 
-	std::vector<IoHash> ExpiredBlobs;
 	{
-		RwLock::SharedLockScope __(m_Lock);
-		for (const auto& It : m_BlobLookup)
+		struct SizeInfo
 		{
-			const BlobIndex	 ReadBlobIndex = It.second;
-			const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex];
+			const IoHash Key;
+			uint32_t	 SecondsSinceEpoch = 0;
+			uint64_t	 BlobSize		   = 0;
+		};
+
+		bool		   DiskSizeExceeded = false;
+		const uint64_t CurrentDiskSize =
+			m_LargeBlobStore.StorageSize().DiskSize + m_SmallBlobStore.StorageSize().DiskSize + m_MetadataBlockStore.TotalSize();
+		if (CurrentDiskSize > m_Config.MaxDiskSpaceLimit)
+		{
+			DiskSizeExceeded = true;
+		}
+
+		uint64_t ExpiredDataSize = 0;
+
+		std::vector<SizeInfo> NonExpiredBlobSizeInfos;
+
+		{
+			RwLock::SharedLockScope __(m_Lock);
+			if (DiskSizeExceeded)
+			{
+				NonExpiredBlobSizeInfos.reserve(m_BlobLookup.size());
+			}
+			for (const auto& It : m_BlobLookup)
+			{
+				const BlobIndex	 ReadBlobIndex = It.second;
+				const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex];
+				uint64_t		 Size		   = 0;
+				if (ReadBlobEntry.Payload)
+				{
+					const PayloadEntry& Payload = m_PayloadEntries[ReadBlobEntry.Payload];
+					Size += Payload.GetSize();
+				}
+				if (ReadBlobEntry.Metadata)
+				{
+					const MetadataEntry& Metadata = m_MetadataEntries[ReadBlobEntry.Metadata];
+					Size += Metadata.Location.Size;
+				}
+
+				const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime;
+				if (AccessTick < ExpireTicks)
+				{
+					ExpiredBlobs.push_back(It.first);
+					ExpiredDataSize += ExpiredDataSize;
+				}
+				else if (DiskSizeExceeded)
+				{
+					NonExpiredBlobSizeInfos.emplace_back(SizeInfo{.Key				 = It.first,
+																  .SecondsSinceEpoch = ReadBlobEntry.LastAccessTime.GetSecondsSinceEpoch(),
+																  .BlobSize			 = Size});
+				}
+			}
+			Stats.CheckedCount += m_BlobLookup.size();
+			Stats.FoundCount += ExpiredBlobs.size();
+		}
 
-			const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime;
-			if (AccessTick < ExpireTicks)
+		if (DiskSizeExceeded)
+		{
+			const uint64_t NewSizeLimit =
+				m_Config.MaxDiskSpaceLimit -
+				(m_Config.MaxDiskSpaceLimit >> 4);	// Remove a bit more than just below the limit so we have some space to grow
+			if ((CurrentDiskSize - ExpiredDataSize) > NewSizeLimit)
 			{
-				ExpiredBlobs.push_back(It.first);
+				std::vector<size_t> NonExpiredOrder;
+				NonExpiredOrder.resize(NonExpiredBlobSizeInfos.size());
+				for (size_t Index = 0; Index < NonExpiredOrder.size(); Index++)
+				{
+					NonExpiredOrder[Index] = Index;
+				}
+				std::sort(NonExpiredOrder.begin(), NonExpiredOrder.end(), [&NonExpiredBlobSizeInfos](const size_t Lhs, const size_t Rhs) {
+					const SizeInfo& LhsInfo = NonExpiredBlobSizeInfos[Lhs];
+					const SizeInfo& RhsInfo = NonExpiredBlobSizeInfos[Rhs];
+					return LhsInfo.SecondsSinceEpoch < RhsInfo.SecondsSinceEpoch;
+				});
+
+				auto It = NonExpiredOrder.begin();
+				while (It != NonExpiredOrder.end())
+				{
+					const SizeInfo& Info = NonExpiredBlobSizeInfos[*It];
+					if ((CurrentDiskSize - ExpiredDataSize) < NewSizeLimit)
+					{
+						break;
+					}
+					ExpiredDataSize += Info.BlobSize;
+					ExpiredBlobs.push_back(Info.Key);
+					SizeDroppedBlobs.insert(Info.Key);
+					It++;
+				}
 			}
 		}
-		Stats.CheckedCount += m_BlobLookup.size();
-		Stats.FoundCount += ExpiredBlobs.size();
 	}
 
 	std::vector<IoHash> RemovedBlobs;
@@ -1318,7 +1438,7 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 
 					const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime;
 
-					if (AccessTick < ExpireTicks)
+					if (SizeDroppedBlobs.contains(ExpiredBlob) || (AccessTick < ExpireTicks))
 					{
 						if (ReadBlobEntry.Payload)
 						{
@@ -1388,6 +1508,21 @@ BuildStore::LockState(GcCtx& Ctx)
 	return Locks;
 }
 
+void
+BuildStore::ScrubStorage(ScrubContext& ScrubCtx)
+{
+	ZEN_UNUSED(ScrubCtx);
+	// TODO
+}
+
+GcStorageSize
+BuildStore::StorageSize() const
+{
+	GcStorageSize Result;
+	Result.DiskSize = m_MetadataBlockStore.TotalSize();
+	return Result;
+}
+
 /*
 		___________              __
 		\__    ___/___   _______/  |_  ______
@@ -1731,6 +1866,94 @@ TEST_CASE("BuildStore.GC")
 	}
 }
 
+TEST_CASE("BuildStore.SizeLimit")
+{
+	using namespace blockstore::testing;
+
+	ScopedTemporaryDirectory _;
+
+	BuildStoreConfig Config = {.MaxDiskSpaceLimit = 1024u * 1024u};
+	Config.RootDirectory	= _.Path() / "build_store";
+
+	std::vector<IoHash>	  CompressedBlobsHashes;
+	std::vector<IoBuffer> BlobMetaPayloads;
+	{
+		GcManager  Gc;
+		BuildStore Store(Config, Gc);
+		for (size_t I = 0; I < 64; I++)
+		{
+			IoBuffer		 Blob = CreateSemiRandomBlob(65537 + I * 7);
+			CompressedBuffer CompressedBlob =
+				CompressedBuffer::Compress(SharedBuffer(std::move(Blob)), OodleCompressor::Mermaid, OodleCompressionLevel::None);
+			CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash());
+			IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer();
+			Payload.SetContentType(ZenContentType::kCompressedBinary);
+			Store.PutBlob(CompressedBlobsHashes.back(), Payload);
+		}
+		for (const IoHash& BlobHash : CompressedBlobsHashes)
+		{
+			BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}}));
+			BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject);
+		}
+		Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads);
+
+		{
+			for (size_t I = 0; I < 64; I++)
+			{
+				const IoHash& Key		 = CompressedBlobsHashes[I];
+				GcClock::Tick AccessTick = (GcClock::Now() + std::chrono::minutes(I)).time_since_epoch().count();
+
+				Store.SetLastAccessTime(Key, AccessTime(AccessTick));
+			}
+		}
+	}
+	{
+		GcManager  Gc;
+		BuildStore Store(Config, Gc);
+
+		{
+			GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() - std::chrono::hours(1),
+														   .CollectSmallObjects	 = true,
+														   .IsDeleteMode		 = true,
+														   .Verbose				 = true});
+
+			uint32_t DeletedBlobs = 0;
+
+			CHECK(!Result.WasCancelled);
+			for (const IoHash& BlobHash : CompressedBlobsHashes)
+			{
+				IoBuffer Blob = Store.GetBlob(BlobHash);
+				if (!Blob)
+				{
+					DeletedBlobs++;
+				}
+				else
+				{
+					IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer();
+					CHECK(DecompressedBlob);
+					CHECK(IoHash::HashBuffer(DecompressedBlob) == BlobHash);
+				}
+			}
+			CHECK(DeletedBlobs == 50);
+
+			std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr);
+			CHECK(MetadataPayloads.size() == BlobMetaPayloads.size());
+			for (size_t I = 0; I < MetadataPayloads.size(); I++)
+			{
+				const IoBuffer& MetadataPayload = MetadataPayloads[I];
+				if (I < DeletedBlobs)
+				{
+					CHECK(!MetadataPayload);
+				}
+				else
+				{
+					CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I]));
+				}
+			}
+		}
+	}
+}
+
 void
 buildstore_forcelink()
 {
diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h
index d88e682de..adf48dc26 100644
--- a/src/zenstore/include/zenstore/buildstore/buildstore.h
+++ b/src/zenstore/include/zenstore/buildstore/buildstore.h
@@ -24,9 +24,10 @@ struct BuildStoreConfig
 	uint32_t			  SmallBlobBlockStoreAlignement		   = 16;
 	uint32_t			  MetadataBlockStoreMaxBlockSize	   = 64 * 1024 * 1024;
 	uint32_t			  MetadataBlockStoreAlignement		   = 8;
+	uint64_t			  MaxDiskSpaceLimit					   = 1u * 1024u * 1024u * 1024u * 1024u;  // 1TB
 };
 
-class BuildStore : public GcReferencer, public GcReferenceLocker  //, public GcStorage
+class BuildStore : public GcReferencer, public GcReferenceLocker, public GcStorage
 {
 public:
 	explicit BuildStore(const BuildStoreConfig& Config, GcManager& Gc);
@@ -48,10 +49,24 @@ public:
 
 	void Flush();
 
+	struct StorageStats
+	{
+		uint64_t EntryCount		   = 0;
+		uint64_t LargeBlobCount	   = 0;
+		uint64_t LargeBlobBytes	   = 0;
+		uint64_t SmallBlobCount	   = 0;
+		uint64_t SmallBlobBytes	   = 0;
+		uint64_t MetadataCount	   = 0;
+		uint64_t MetadataByteCount = 0;
+	};
+
+	StorageStats GetStorageStats() const;
+
 #if ZEN_WITH_TESTS
 	std::optional<AccessTime> GetLastAccessTime(const IoHash& Key) const;
 	bool					  SetLastAccessTime(const IoHash& Key, const AccessTime& Time);
 #endif	// ZEN_WITH_TESTS
+
 private:
 	LoggerRef Log() { return m_Log; }
 
@@ -71,6 +86,10 @@ private:
 	//////// GcReferenceLocker
 	virtual std::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) override;
 
+	//////// GcStorage
+	virtual void		  ScrubStorage(ScrubContext& ScrubCtx) override;
+	virtual GcStorageSize StorageSize() const override;
+
 #pragma pack(push)
 #pragma pack(1)
 	struct PayloadEntry