zen build cache service (#318)

- **EXPERIMENTAL** `zen builds`
  - Feature: `--zen-cache-host` option for `upload` and `download` operations to use a zenserver host `/builds` endpoint for storing build blob and blob metadata
  - Feature: New `/builds` endpoint for caching build blobs and blob metadata
    - `/builds/{namespace}/{bucket}/{buildid}/blobs/{hash}` `GET` and `PUT` method for storing and fetching blobs
    - `/builds/{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata` `POST` method for storing metadata about blobs
    - `/builds/{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata` `POST` method for fetching metadata about blobs
    - `/builds/{namespace}/{bucket}/{buildid}/blobs/exists` `POST` method for checking existance of blobs

1 files changed, 19 insertions, 2 deletions

diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 7ac10d613..fe5ae284b 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -1081,7 +1081,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 						ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size());
 						{
 							ZEN_TRACE_CPU("GcV2::LockReferencers");
-							// From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until
+							// From this point we have blocked all writes to all References (DiskBucket/ProjectStore/BuildStore) until
 							// we delete the ReferenceLockers
 							Latch WorkLeft(1);
 							{
@@ -1108,7 +1108,7 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 							ZEN_TRACE_CPU("GcV2::UpdateLockedState");
 
 							// Locking all references checkers so we have a steady state of which references are used
-							// From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until
+							// From this point we have blocked all writes to all References (DiskBucket/ProjectStore/BuildStore) until
 							// we delete the ReferenceCheckers
 							Latch WorkLeft(1);
 
@@ -1739,6 +1739,7 @@ GcScheduler::AppendGCLog(std::string_view Id, GcClock::TimePoint StartTime, cons
 				{
 					Writer << "CacheExpireTime"sv << ToDateTime(Settings.CacheExpireTime);
 					Writer << "ProjectStoreExpireTime"sv << ToDateTime(Settings.ProjectStoreExpireTime);
+					Writer << "BuildStoreExpireTime"sv << ToDateTime(Settings.BuildStoreExpireTime);
 					Writer << "CollectSmallObjects"sv << Settings.CollectSmallObjects;
 					Writer << "IsDeleteMode"sv << Settings.IsDeleteMode;
 					Writer << "SkipCidDelete"sv << Settings.SkipCidDelete;
@@ -1940,6 +1941,7 @@ GcScheduler::SchedulerThread()
 			std::chrono::seconds LightweightGcInterval			   = m_Config.LightweightInterval;
 			std::chrono::seconds MaxCacheDuration				   = m_Config.MaxCacheDuration;
 			std::chrono::seconds MaxProjectStoreDuration		   = m_Config.MaxProjectStoreDuration;
+			std::chrono::seconds MaxBuildStoreDuration			   = m_Config.MaxBuildStoreDuration;
 			uint64_t			 DiskSizeSoftLimit				   = m_Config.DiskSizeSoftLimit;
 			bool				 SkipCid						   = false;
 			GcVersion			 UseGCVersion					   = m_Config.UseGCVersion;
@@ -1975,6 +1977,10 @@ GcScheduler::SchedulerThread()
 				{
 					MaxProjectStoreDuration = TriggerParams.MaxProjectStoreDuration;
 				}
+				if (TriggerParams.MaxBuildStoreDuration != std::chrono::seconds::max())
+				{
+					MaxBuildStoreDuration = TriggerParams.MaxBuildStoreDuration;
+				}
 				if (TriggerParams.DiskSizeSoftLimit != 0)
 				{
 					DiskSizeSoftLimit = TriggerParams.DiskSizeSoftLimit;
@@ -2046,6 +2052,8 @@ GcScheduler::SchedulerThread()
 				MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxCacheDuration;
 			GcClock::TimePoint ProjectStoreExpireTime =
 				MaxProjectStoreDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxProjectStoreDuration;
+			GcClock::TimePoint BuildStoreExpireTime =
+				MaxBuildStoreDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxBuildStoreDuration;
 
 			const GcStorageSize TotalSize = m_GcManager.TotalStorageSize();
 
@@ -2102,6 +2110,10 @@ GcScheduler::SchedulerThread()
 					{
 						ProjectStoreExpireTime = SizeBasedExpireTime;
 					}
+					if (SizeBasedExpireTime > BuildStoreExpireTime)
+					{
+						BuildStoreExpireTime = SizeBasedExpireTime;
+					}
 				}
 
 				std::chrono::seconds RemainingTimeUntilGc =
@@ -2227,6 +2239,7 @@ GcScheduler::SchedulerThread()
 
 			bool GcSuccess = CollectGarbage(CacheExpireTime,
 											ProjectStoreExpireTime,
+											BuildStoreExpireTime,
 											DoDelete,
 											CollectSmallObjects,
 											SkipCid,
@@ -2333,6 +2346,7 @@ GcScheduler::ScrubStorage(bool DoDelete, bool SkipCid, std::chrono::seconds Time
 bool
 GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 							const GcClock::TimePoint& ProjectStoreExpireTime,
+							const GcClock::TimePoint& BuildStoreExpireTime,
 							bool					  Delete,
 							bool					  CollectSmallObjects,
 							bool					  SkipCid,
@@ -2416,6 +2430,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 
 						const GcSettings Settings = {.CacheExpireTime					= CacheExpireTime,
 													 .ProjectStoreExpireTime			= ProjectStoreExpireTime,
+													 .BuildStoreExpireTime				= BuildStoreExpireTime,
 													 .CollectSmallObjects				= CollectSmallObjects,
 													 .IsDeleteMode						= Delete,
 													 .SkipCidDelete						= SkipCid,
@@ -2447,6 +2462,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 							}
 							SB.Append(fmt::format("      Cache cutoff time:           {}\n", Settings.CacheExpireTime));
 							SB.Append(fmt::format("      Project store cutoff time:   {}\n", Settings.ProjectStoreExpireTime));
+							SB.Append(fmt::format("      Build store cutoff time:     {}\n", Settings.BuildStoreExpireTime));
 						};
 
 						{
@@ -2552,6 +2568,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 				if (Delete)
 				{
 					GcClock::TimePoint KeepRangeStart = Min(CacheExpireTime, ProjectStoreExpireTime);
+					KeepRangeStart					  = Min(KeepRangeStart, BuildStoreExpireTime);
 					m_LastGcExpireTime				  = KeepRangeStart;
 					std::unique_lock Lock(m_GcMutex);
 					m_DiskUsageWindow.KeepRange(KeepRangeStart.time_since_epoch().count(), GcClock::Duration::max().count());