Merge branch 'main' into fd-fix-binary-json

1 files changed, 32 insertions, 44 deletions

diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index 61552fafc..e4d962b56 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -200,21 +200,21 @@ namespace cache::impl {
 		int DropIndex = 0;
 		do
 		{
-			if (!std::filesystem::exists(Dir))
+			if (!IsDir(Dir))
 			{
 				return false;
 			}
 
 			std::string			  DroppedName		= fmt::format("[dropped]{}({})", Dir.filename().string(), DropIndex);
 			std::filesystem::path DroppedBucketPath = Dir.parent_path() / DroppedName;
-			if (std::filesystem::exists(DroppedBucketPath))
+			if (IsDir(DroppedBucketPath))
 			{
 				DropIndex++;
 				continue;
 			}
 
 			std::error_code Ec;
-			std::filesystem::rename(Dir, DroppedBucketPath, Ec);
+			RenameDirectory(Dir, DroppedBucketPath, Ec);
 			if (!Ec)
 			{
 				DeleteDirectories(DroppedBucketPath);
@@ -909,16 +909,16 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool FlushLockPosition,
 			{
 				std::filesystem::path LogPath = cache::impl::GetLogPath(m_BucketDir, m_BucketName);
 
-				if (std::filesystem::is_regular_file(LogPath))
+				if (IsFile(LogPath))
 				{
-					if (!std::filesystem::remove(LogPath, Ec) || Ec)
+					if (!RemoveFile(LogPath, Ec) || Ec)
 					{
 						ZEN_WARN("snapshot failed to clean log file '{}', removing index at '{}', reason: '{}'",
 								 LogPath,
 								 IndexPath,
 								 Ec.message());
 						std::error_code RemoveIndexEc;
-						std::filesystem::remove(IndexPath, RemoveIndexEc);
+						RemoveFile(IndexPath, RemoveIndexEc);
 					}
 				}
 			}
@@ -939,7 +939,7 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const
 {
 	ZEN_TRACE_CPU("Z$::Bucket::ReadIndexFile");
 
-	if (!std::filesystem::is_regular_file(IndexPath))
+	if (!IsFile(IndexPath))
 	{
 		return 0;
 	}
@@ -1023,7 +1023,7 @@ ZenCacheDiskLayer::CacheBucket::ReadLog(RwLock::ExclusiveLockScope&, const std::
 {
 	ZEN_TRACE_CPU("Z$::Bucket::ReadLog");
 
-	if (!std::filesystem::is_regular_file(LogPath))
+	if (!IsFile(LogPath))
 	{
 		return 0;
 	}
@@ -1103,37 +1103,37 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockSco
 
 	if (IsNew)
 	{
-		fs::remove(LogPath);
-		fs::remove(IndexPath);
-		fs::remove_all(m_BlocksBasePath);
+		RemoveFile(LogPath);
+		RemoveFile(IndexPath);
+		DeleteDirectories(m_BlocksBasePath);
 	}
 
 	CreateDirectories(m_BucketDir);
 
 	m_BlockStore.Initialize(m_BlocksBasePath, m_Configuration.MaxBlockSize, BlockStoreDiskLocation::MaxBlockIndex + 1);
 
-	if (std::filesystem::is_regular_file(IndexPath))
+	if (IsFile(IndexPath))
 	{
 		uint32_t IndexVersion = 0;
 		m_LogFlushPosition	  = ReadIndexFile(IndexLock, IndexPath, IndexVersion);
 		if (IndexVersion == 0)
 		{
 			ZEN_WARN("removing invalid index file at '{}'", IndexPath);
-			std::filesystem::remove(IndexPath);
+			RemoveFile(IndexPath);
 		}
 	}
 
 	uint64_t LogEntryCount = 0;
-	if (std::filesystem::is_regular_file(LogPath))
+	if (IsFile(LogPath))
 	{
 		if (TCasLogFile<DiskIndexEntry>::IsValid(LogPath))
 		{
 			LogEntryCount = ReadLog(IndexLock, LogPath, m_LogFlushPosition);
 		}
-		else if (fs::is_regular_file(LogPath))
+		else if (IsFile(LogPath))
 		{
 			ZEN_WARN("removing invalid log at '{}'", LogPath);
-			std::filesystem::remove(LogPath);
+			RemoveFile(LogPath);
 		}
 	}
 
@@ -2146,7 +2146,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 					RwLock::SharedLockScope ValueLock(LockForHash(HashKey));
 
 					std::error_code Ec;
-					uintmax_t		size = std::filesystem::file_size(DataFilePath.ToPath(), Ec);
+					uintmax_t		size = FileSizeFromPath(DataFilePath.ToPath(), Ec);
 					if (Ec)
 					{
 						ReportBadKey(HashKey);
@@ -2287,11 +2287,11 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 					BuildPath(Path, Entry.Key);
 					fs::path				   FilePath = Path.ToPath();
 					RwLock::ExclusiveLockScope ValueLock(LockForHash(Entry.Key));
-					if (fs::is_regular_file(FilePath))
+					if (IsFile(FilePath))
 					{
 						ZEN_DEBUG("deleting bad standalone cache file '{}'", Path.ToUtf8());
 						std::error_code Ec;
-						fs::remove(FilePath, Ec);  // We don't care if we fail, we are no longer tracking this file...
+						RemoveFile(FilePath, Ec);  // We don't care if we fail, we are no longer tracking this file...
 					}
 				}
 			}
@@ -2424,7 +2424,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 		 if (CleanUpTempFile)
 		 {
 			 std::error_code Ec;
-			 std::filesystem::remove(DataFile.GetPath(), Ec);
+			 RemoveFile(DataFile.GetPath(), Ec);
 			 if (Ec)
 			 {
 				 ZEN_WARN("Failed to clean up temporary file '{}' for put in '{}', reason '{}'",
@@ -2452,7 +2452,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 	RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey));
 
 	// We do a speculative remove of the file instead of probing with a exists call and check the error code instead
-	std::filesystem::remove(FsPath, Ec);
+	RemoveFile(FsPath, Ec);
 	if (Ec)
 	{
 		if (Ec.value() != ENOENT)
@@ -2460,7 +2460,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 			ZEN_WARN("Failed to remove file '{}' for put in '{}', reason: '{}', retrying.", FsPath, m_BucketDir, Ec.message());
 			Sleep(100);
 			Ec.clear();
-			std::filesystem::remove(FsPath, Ec);
+			RemoveFile(FsPath, Ec);
 			if (Ec && Ec.value() != ENOENT)
 			{
 				throw std::system_error(Ec, fmt::format("Failed to remove file '{}' for put in '{}'", FsPath, m_BucketDir));
@@ -2731,9 +2731,10 @@ class DiskBucketStoreCompactor : public GcStoreCompactor
 	using CacheBucket	= ZenCacheDiskLayer::CacheBucket;
 
 public:
-	DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys)
+	DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys, bool FlushBucket)
 	: m_Bucket(Bucket)
 	, m_ExpiredStandaloneKeys(std::move(ExpiredStandaloneKeys))
+	, m_FlushBucket(FlushBucket)
 	{
 		m_ExpiredStandaloneKeys.shrink_to_fit();
 	}
@@ -2791,7 +2792,7 @@ public:
 					ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': deleting standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8());
 
 					std::error_code Ec;
-					if (!fs::remove(FilePath, Ec))
+					if (!RemoveFile(FilePath, Ec))
 					{
 						continue;
 					}
@@ -2812,7 +2813,7 @@ public:
 					ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': checking standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8());
 
 					std::error_code Ec;
-					bool			Existed = std::filesystem::is_regular_file(FilePath, Ec);
+					bool			Existed = IsFile(FilePath, Ec);
 					if (Ec)
 					{
 						ZEN_WARN("GCV2: cachebucket [COMPACT] '{}': failed checking cache payload file '{}'. Reason '{}'",
@@ -2960,6 +2961,10 @@ public:
 				}
 			}
 		}
+		if (m_FlushBucket)
+		{
+			m_Bucket.Flush();
+		}
 	}
 
 	virtual std::string GetGcName(GcCtx& Ctx) override { return m_Bucket.GetGcName(Ctx); }
@@ -2967,6 +2972,7 @@ public:
 private:
 	ZenCacheDiskLayer::CacheBucket&			 m_Bucket;
 	std::vector<std::pair<IoHash, uint64_t>> m_ExpiredStandaloneKeys;
+	bool									 m_FlushBucket = false;
 };
 
 GcStoreCompactor*
@@ -2990,24 +2996,6 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 					 NiceBytes(Stats.FreedMemory),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		}
-		if (Stats.DeletedCount > 0)
-		{
-			bool Expected = false;
-			if (m_IsFlushing || !m_IsFlushing.compare_exchange_strong(Expected, true))
-			{
-				return;
-			}
-			auto FlushingGuard = MakeGuard([&] { m_IsFlushing.store(false); });
-
-			try
-			{
-				SaveSnapshot([]() { return 0; });
-			}
-			catch (const std::exception& Ex)
-			{
-				ZEN_WARN("Failed to write index and manifest after RemoveExpiredData in '{}'. Reason: '{}'", m_BucketDir, Ex.what());
-			}
-		}
 	});
 
 	const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count();
@@ -3094,7 +3082,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 		return nullptr;
 	}
 
-	return new DiskBucketStoreCompactor(*this, std::move(ExpiredStandaloneKeys));
+	return new DiskBucketStoreCompactor(*this, std::move(ExpiredStandaloneKeys), /*FlushBucket*/ Stats.DeletedCount > 0);
 }
 
 bool