Cache request record/replay (#198)

This adds recording and playback of cache request with full data - both get and put operations can be replayed.

Invoke via web request.

`<host>/z$/exec$/start-recording?<disk-storage-path>`
`<host>/z$/exec$/stop-recording`
`<host>/z$/exec$/replay-recording?<thread-count>&<disk-storage-path>`

1 files changed, 25 insertions, 8 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index c20e40655..1f48aaebe 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -1827,7 +1827,7 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 
 			if (!Bucket->OpenOrCreate(BucketPath))
 			{
-				m_Buckets.erase(BucketName);
+				m_Buckets.erase(InsertResult.first);
 				return false;
 			}
 		}
@@ -1872,9 +1872,18 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 			std::filesystem::path BucketPath = m_RootDir;
 			BucketPath /= BucketName;
 
-			if (!Bucket->OpenOrCreate(BucketPath))
+			try
+			{
+				if (!Bucket->OpenOrCreate(BucketPath))
+				{
+					ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir);
+					m_Buckets.erase(InsertResult.first);
+					return;
+				}
+			}
+			catch (const std::exception& Err)
 			{
-				m_Buckets.erase(BucketName);
+				ZEN_ERROR("creating bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what());
 				return;
 			}
 		}
@@ -1897,7 +1906,7 @@ ZenCacheDiskLayer::DiscoverBuckets()
 
 	for (const std::filesystem::path& BucketPath : DirContent.Directories)
 	{
-		std::string BucketName = PathToUtf8(BucketPath.stem());
+		const std::string BucketName = PathToUtf8(BucketPath.stem());
 		// New bucket needs to be created
 		if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end())
 		{
@@ -1907,12 +1916,20 @@ ZenCacheDiskLayer::DiscoverBuckets()
 		auto		 InsertResult = m_Buckets.emplace(BucketName, std::make_unique<CacheBucket>(BucketName));
 		CacheBucket& Bucket		  = *InsertResult.first->second;
 
-		if (!Bucket.OpenOrCreate(BucketPath, /* AllowCreate */ false))
+		try
 		{
-			ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir);
+			if (!Bucket.OpenOrCreate(BucketPath, /* AllowCreate */ false))
+			{
+				ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir);
 
-			m_Buckets.erase(InsertResult.first);
-			continue;
+				m_Buckets.erase(InsertResult.first);
+				continue;
+			}
+		}
+		catch (const std::exception& Err)
+		{
+			ZEN_ERROR("creating bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what());
+			return;
 		}
 		ZEN_INFO("Discovered bucket '{}'", BucketName);
 	}