gc command attachment options (#176)

* zen command - add options to control meta data cache when triggering gc

4 files changed, 48 insertions, 17 deletions

diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 3fc4eca03..b67e8a570 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -3676,20 +3676,21 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(GcCtx& Ctx, bool StateIsAlreadyLoc
 		OutReferences.reserve(OutReferences.size() + InlineKeys.size() +
 							  StandaloneKeys.size());  // Make space for at least one attachment per record
 
+		bool UseMetaData = Ctx.Settings.StoreCacheAttachmentMetaData;
+
 		for (const std::vector<std::size_t>& ChunkIndexes : InlineBlockChunkIndexes)
 		{
 			ZEN_ASSERT(!ChunkIndexes.empty());
 
 			uint32_t BlockIndex = InlineLocations[ChunkIndexes[0]].BlockIndex;
-			if (!m_Configuration.StoreAttachmentMetaData ||
-				!ReadAttachmentsFromMetaData(BlockIndex, InlineKeys, ChunkIndexes, OutReferences))
+			if (!UseMetaData || !ReadAttachmentsFromMetaData(BlockIndex, InlineKeys, ChunkIndexes, OutReferences))
 			{
 				std::vector<IoHash>	  Keys;
 				std::vector<uint32_t> AttachmentCounts;
 				size_t				  PrecachedReferencesStart	   = OutReferences.size();
 				size_t				  NextPrecachedReferencesStart = PrecachedReferencesStart;
 
-				bool WriteMetaData = m_Configuration.StoreAttachmentMetaData && !m_BlockStore.IsWriting(BlockIndex);
+				bool WriteMetaData = UseMetaData && !m_BlockStore.IsWriting(BlockIndex);
 				if (WriteMetaData)
 				{
 					Keys.reserve(InlineLocations.size());
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index cde89421e..159b13af0 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -1705,6 +1705,8 @@ GcScheduler::AppendGCLog(std::string_view Id, GcClock::TimePoint StartTime, cons
 					Writer << "CompactBlockUsageThresholdPercent"sv << Settings.CompactBlockUsageThresholdPercent;
 					Writer << "AttachmentRangeMin"sv << Settings.AttachmentRangeMin;
 					Writer << "AttachmentRangeMax"sv << Settings.AttachmentRangeMin;
+					Writer << "ForceStoreCacheAttachmentMetaData"sv << Settings.StoreCacheAttachmentMetaData;
+					Writer << "ForceStoreProjectAttachmentMetaData"sv << Settings.StoreProjectAttachmentMetaData;
 				}
 				Writer.EndObject();
 
@@ -1901,7 +1903,10 @@ GcScheduler::SchedulerThread()
 			bool				 SingleThreaded					   = m_Config.SingleThreaded;
 			IoHash				 AttachmentRangeMin				   = IoHash::Zero;
 			IoHash				 AttachmentRangeMax				   = IoHash::Max;
-			uint8_t				 NextAttachmentPassIndex =
+			bool				 StoreCacheAttachmentMetaData	   = m_Config.StoreCacheAttachmentMetaData;
+			bool				 StoreProjectAttachmentMetaData	   = m_Config.StoreProjectAttachmentMetaData;
+
+			uint8_t NextAttachmentPassIndex =
 				ComputeAttachmentRange(m_AttachmentPassIndex, m_Config.AttachmentPassCount, AttachmentRangeMin, AttachmentRangeMax);
 
 			bool DiskSpaceGCTriggered = false;
@@ -1947,6 +1952,14 @@ GcScheduler::SchedulerThread()
 				{
 					NextAttachmentPassIndex = m_AttachmentPassIndex;
 				}
+				if (TriggerParams.StoreCacheAttachmentMetaData.has_value())
+				{
+					StoreCacheAttachmentMetaData = TriggerParams.StoreCacheAttachmentMetaData.value();
+				}
+				if (TriggerParams.StoreProjectAttachmentMetaData.has_value())
+				{
+					StoreProjectAttachmentMetaData = TriggerParams.StoreProjectAttachmentMetaData.value();
+				}
 				DoGc = true;
 			}
 
@@ -2173,6 +2186,8 @@ GcScheduler::SchedulerThread()
 											SingleThreaded,
 											AttachmentRangeMin,
 											AttachmentRangeMax,
+											StoreCacheAttachmentMetaData,
+											StoreProjectAttachmentMetaData,
 											SilenceErrors);
 			if (!GcSuccess)
 			{
@@ -2274,6 +2289,8 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 							bool					  SingleThreaded,
 							const IoHash&			  AttachmentRangeMin,
 							const IoHash&			  AttachmentRangeMax,
+							bool					  StoreCacheAttachmentMetaData,
+							bool					  StoreProjectAttachmentMetaData,
 							bool					  SilenceErrors)
 {
 	ZEN_TRACE_CPU("GcScheduler::CollectGarbage");
@@ -2357,18 +2374,24 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 													 .CompactBlockUsageThresholdPercent = CompactBlockUsageThresholdPercent,
 													 .DiskReservePath					= m_Config.RootDirectory / "reserve.gc",
 													 .AttachmentRangeMin				= AttachmentRangeMin,
-													 .AttachmentRangeMax				= AttachmentRangeMax};
+													 .AttachmentRangeMax				= AttachmentRangeMax,
+													 .StoreCacheAttachmentMetaData		= StoreCacheAttachmentMetaData,
+													 .StoreProjectAttachmentMetaData	= StoreProjectAttachmentMetaData};
 
 						auto AppendSettings = [](StringBuilderBase& SB, const GcSettings& Settings) {
 							SB.Append(
 								fmt::format("      GC small objects:          {}\n", Settings.CollectSmallObjects ? "yes"sv : "no"sv));
 							SB.Append(fmt::format("      GC Cid store:              {}\n", Settings.SkipCidDelete ? "no"sv : "yes"sv));
-							if (!Settings.SkipCidDelete &&
-								(Settings.AttachmentRangeMin != IoHash::Zero || Settings.AttachmentRangeMax != IoHash::Max))
+							if (!Settings.SkipCidDelete)
 							{
-								SB.Append(fmt::format("      Attachment range:          {}-{}\n",
-													  Settings.AttachmentRangeMin,
-													  Settings.AttachmentRangeMax));
+								if (Settings.AttachmentRangeMin != IoHash::Zero || Settings.AttachmentRangeMax != IoHash::Max)
+								{
+									SB.Append(fmt::format("      Attachment range:          {}-{}\n",
+														  Settings.AttachmentRangeMin,
+														  Settings.AttachmentRangeMax));
+								}
+								SB.Append(fmt::format("      Cache attachment meta:     {}\n", Settings.StoreCacheAttachmentMetaData));
+								SB.Append(fmt::format("      Porject attachment meta:   {}\n", Settings.StoreProjectAttachmentMetaData));
 							}
 							SB.Append(fmt::format("      Cache cutoff time:         {}\n", Settings.CacheExpireTime));
 							SB.Append(fmt::format("      Project store cutoff time: {}", Settings.ProjectStoreExpireTime));
diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h
index 6d85e8039..f38776b6e 100644
--- a/src/zenstore/include/zenstore/cache/cachedisklayer.h
+++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h
@@ -105,11 +105,10 @@ class ZenCacheDiskLayer
 public:
 	struct BucketConfiguration
 	{
-		uint64_t MaxBlockSize			 = 1ull << 30;
-		uint32_t PayloadAlignment		 = 1u << 4;
-		uint64_t MemCacheSizeThreshold	 = 1 * 1024;
-		uint64_t LargeObjectThreshold	 = 128 * 1024;
-		bool	 StoreAttachmentMetaData = false;
+		uint64_t MaxBlockSize		   = 1ull << 30;
+		uint32_t PayloadAlignment	   = 1u << 4;
+		uint64_t MemCacheSizeThreshold = 1 * 1024;
+		uint64_t LargeObjectThreshold  = 128 * 1024;
 	};
 
 	struct Configuration
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index 56965e3e3..b79f1b9df 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -64,8 +64,10 @@ struct GcSettings
 		90;	 // 0 = compact only empty eligible blocks, 100 = compact all non-full eligible blocks, 1-99 = compact eligible blocks with less
 			 // usage than CompactBlockUsageThresholdPercent
 	std::filesystem::path DiskReservePath;
-	IoHash				  AttachmentRangeMin = IoHash::Zero;
-	IoHash				  AttachmentRangeMax = IoHash::Max;
+	IoHash				  AttachmentRangeMin			 = IoHash::Zero;
+	IoHash				  AttachmentRangeMax			 = IoHash::Max;
+	bool				  StoreCacheAttachmentMetaData	 = false;
+	bool				  StoreProjectAttachmentMetaData = false;
 };
 
 struct GcCompactStoreStats
@@ -452,6 +454,8 @@ struct GcSchedulerConfig
 	bool				  Verbose							= false;
 	bool				  SingleThreaded					= false;
 	uint16_t			  AttachmentPassCount				= 1;
+	bool				  StoreCacheAttachmentMetaData		= false;
+	bool				  StoreProjectAttachmentMetaData	= false;
 };
 
 struct GcSchedulerState
@@ -531,6 +535,8 @@ public:
 		std::optional<bool>		 SingleThreaded;
 		std::optional<IoHash>	 AttachmentRangeMin;
 		std::optional<IoHash>	 AttachmentRangeMax;
+		std::optional<bool>		 StoreCacheAttachmentMetaData	= false;
+		std::optional<bool>		 StoreProjectAttachmentMetaData = false;
 	};
 
 	bool TriggerGc(const TriggerGcParams& Params);
@@ -560,6 +566,8 @@ private:
 								bool					  SingleThreaded,
 								const IoHash&			  AttachmentRangeMin,
 								const IoHash&			  AttachmentRangeMax,
+								bool					  StoreCacheAttachmentMetaData,
+								bool					  StoreProjectAttachmentMetaData,
 								bool					  SilenceErrors);
 	void		 ScrubStorage(bool DoDelete, bool SkipCid, std::chrono::seconds TimeSlice);
 	LoggerRef	 Log() { return m_Log; }