Implemented support for dropping z$ buckets while online

7 files changed, 127 insertions, 3 deletions

diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 7f6f49e06..f744ae131 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -94,6 +94,15 @@ HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request)
 
 	if (!ValidateUri(Request, /* out */ Ref))
 	{
+		std::string_view Key = Request.RelativeUri();
+
+		if (std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); }))
+		{
+			// Bucket reference
+
+			return HandleCacheBucketRequest(Request, Key);
+		}
+
 		return Request.WriteResponse(zen::HttpResponse::BadRequest);  // invalid URL
 	}
 
@@ -110,6 +119,30 @@ HttpStructuredCacheService::HandleRequest(zen::HttpServerRequest& Request)
 }
 
 void
+HttpStructuredCacheService::HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket)
+{
+	ZEN_UNUSED(Request, Bucket);
+	switch (auto Verb = Request.RequestVerb())
+	{
+		using enum zen::HttpVerb;
+
+		case kHead:
+		case kGet:
+			{
+			}
+			break;
+		case kDelete:
+			// Drop bucket
+
+			if (m_CacheStore.DropBucket(Bucket))
+			{
+				return Request.WriteResponse(zen::HttpResponse::OK);  // invalid URL
+			}
+			break;
+	}
+}
+
+void
 HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Request, CacheRef& Ref)
 {
 	switch (auto Verb = Request.RequestVerb())
@@ -278,7 +311,12 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req
 
 					m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, Value);
 
-					// This is currently synchronous for simplicity and debuggability but should
+					m_Log.debug("PUT - '{}/{}' ({} bytes, {})",
+								Ref.BucketSegment,
+								Ref.HashKey,
+								Value.Value.Size(),
+								Value.Value.GetContentType());
+
 					// absolutely be made asynchronous. By default these should be deferred
 					// because the client should not care if the data has propagated upstream or
 					// not
@@ -428,6 +466,14 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re
 
 						m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash);
 
+						m_Log.debug("PUT ({}) - '{}/{}/{}' ({} bytes, {})",
+									Result.New ? "NEW" : "OLD",
+									Ref.BucketSegment,
+									Ref.HashKey,
+									Ref.PayloadId,
+									Body.Size(),
+									Body.GetContentType());
+
 						if (Result.New)
 						{
 							return Request.WriteResponse(zen::HttpResponse::Created);
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index f42b5bfb7..e9796999b 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -67,6 +67,7 @@ private:
 	[[nodiscard]] bool ValidateUri(zen::HttpServerRequest& Request, CacheRef& OutRef);
 	void			   HandleCacheRecordRequest(zen::HttpServerRequest& Request, CacheRef& Ref);
 	void			   HandleCachePayloadRequest(zen::HttpServerRequest& Request, CacheRef& Ref);
+	void			   HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket);
 
 	spdlog::logger					 m_Log;
 	zen::CasStore&					 m_CasStore;
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 9ed3cf53e..e6eda05aa 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -81,6 +81,14 @@ ZenCacheStore::Put(std::string_view InBucket, const zen::IoHash& HashKey, const
 	}
 }
 
+bool
+ZenCacheStore::DropBucket(std::string_view Bucket)
+{
+	// TODO: should ensure this is done atomically across all layers
+
+	return m_MemLayer.DropBucket(Bucket) && m_DiskLayer.DropBucket(Bucket);
+}
+
 void
 ZenCacheStore::Flush()
 {
@@ -154,6 +162,16 @@ ZenCacheMemoryLayer::Put(std::string_view InBucket, const zen::IoHash& HashKey,
 }
 
 bool
+ZenCacheMemoryLayer::DropBucket(std::string_view Bucket)
+{
+	RwLock::ExclusiveLockScope _(m_Lock);
+
+	m_Buckets.erase(std::string(Bucket));
+
+	return true;
+}
+
+bool
 ZenCacheMemoryLayer::CacheBucket::Get(const zen::IoHash& HashKey, ZenCacheValue& OutValue)
 {
 	RwLock::SharedLockScope _(m_bucketLock);
@@ -216,9 +234,11 @@ struct ZenCacheDiskLayer::CacheBucket
 	~CacheBucket();
 
 	void OpenOrCreate(std::filesystem::path BucketDir);
+	static bool Delete(std::filesystem::path BucketDir);
 
 	bool Get(const zen::IoHash& HashKey, ZenCacheValue& OutValue);
 	void Put(const zen::IoHash& HashKey, const ZenCacheValue& Value);
+	void Drop();
 	void Flush();
 
 	inline bool IsOk() const { return m_Ok; }
@@ -249,6 +269,19 @@ ZenCacheDiskLayer::CacheBucket::~CacheBucket()
 {
 }
 
+bool
+ZenCacheDiskLayer::CacheBucket::Delete(std::filesystem::path BucketDir)
+{
+	if (std::filesystem::exists(BucketDir))
+	{
+		zen::DeleteDirectories(BucketDir);
+
+		return true;
+	}
+
+	return false;
+}
+
 void
 ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir)
 {
@@ -437,6 +470,16 @@ ZenCacheDiskLayer::CacheBucket::Put(const zen::IoHash& HashKey, const ZenCacheVa
 }
 
 void
+ZenCacheDiskLayer::CacheBucket::Drop()
+{
+	// TODO: add error handling
+
+	m_SobsFile.Close();
+	m_SlogFile.Close();
+	zen::DeleteDirectories(m_BucketDir);
+}
+
+void
 ZenCacheDiskLayer::CacheBucket::Flush()
 {
 	m_SobsFile.Flush();
@@ -584,6 +627,30 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const zen::IoHash& HashKey, co
 	}
 }
 
+bool
+ZenCacheDiskLayer::DropBucket(std::string_view InBucket)
+{
+	zen::RwLock::ExclusiveLockScope _(m_Lock);
+
+	auto it = m_Buckets.find(std::string(InBucket));
+
+	if (it != m_Buckets.end())
+	{
+		CacheBucket* Bucket = &it->second;
+
+		Bucket->Drop();
+
+		m_Buckets.erase(it);
+
+		return true;
+	}
+
+	std::filesystem::path BucketPath = m_RootDir;
+	BucketPath /= std::string(InBucket);
+
+	return CacheBucket::Delete(BucketPath);
+}
+
 void
 ZenCacheDiskLayer::Flush()
 {
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index 781a6e636..48c3cfde9 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -56,6 +56,7 @@ public:
 
 	bool Get(std::string_view Bucket, const zen::IoHash& HashKey, ZenCacheValue& OutValue);
 	void Put(std::string_view Bucket, const zen::IoHash& HashKey, const ZenCacheValue& Value);
+	bool DropBucket(std::string_view Bucket);
 
 private:
 	struct CacheBucket
@@ -79,7 +80,7 @@ public:
 
 	bool Get(std::string_view Bucket, const zen::IoHash& HashKey, ZenCacheValue& OutValue);
 	void Put(std::string_view Bucket, const zen::IoHash& HashKey, const ZenCacheValue& Value);
-
+	bool DropBucket(std::string_view Bucket);
 	void Flush();
 
 private:
@@ -102,6 +103,7 @@ public:
 
 	bool Get(std::string_view Bucket, const zen::IoHash& HashKey, ZenCacheValue& OutValue);
 	void Put(std::string_view Bucket, const zen::IoHash& HashKey, const ZenCacheValue& Value);
+	bool DropBucket(std::string_view Bucket);
 	void Flush();
 
 private:
diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp
index 7f10fc5e6..35ccdd042 100644
--- a/zenstore/basicfile.cpp
+++ b/zenstore/basicfile.cpp
@@ -83,4 +83,10 @@ BasicFile::FileSize()
 	return uint64_t(Sz);
 }
 
+void
+BasicFile::Close()
+{
+	m_File.Close();
+}
+
 }  // namespace zen
diff --git a/zenstore/caslog.cpp b/zenstore/caslog.cpp
index 0ef3ed1bd..169629053 100644
--- a/zenstore/caslog.cpp
+++ b/zenstore/caslog.cpp
@@ -98,8 +98,9 @@ void
 CasLogFile::Close()
 {
 	// TODO: update header and maybe add trailer
-
 	Flush();
+
+	m_File.Close();
 }
 
 void
diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h
index b38feb3da..c6f61d466 100644
--- a/zenstore/include/zenstore/basicfile.h
+++ b/zenstore/include/zenstore/basicfile.h
@@ -25,6 +25,7 @@ public:
 	void	 Flush();
 	uint64_t FileSize();
 	void*	 Handle() { return m_File; }
+	void	 Close();
 	IoBuffer ReadAll();
 
 private: