Added more scrub stubs in higher level services

4 files changed, 43 insertions, 4 deletions

diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 17f8e04ef..7f1fe7b44 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -9,6 +9,7 @@
 #include <zencore/stream.h>
 #include <zencore/timer.h>
 #include <zenhttp/httpserver.h>
+#include <zenstore/CAS.h>
 
 #include "structuredcache.h"
 #include "structuredcachestore.h"
@@ -172,6 +173,12 @@ HttpStructuredCacheService::Flush()
 {
 }
 
+void 
+HttpStructuredCacheService::Scrub(ScrubContext& Ctx)
+{
+	ZEN_UNUSED(Ctx);
+}
+
 void
 HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
 {
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index 8871094b3..bd163dd1d 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -60,6 +60,7 @@ public:
 	virtual void HandleRequest(zen::HttpServerRequest& Request) override;
 
 	void Flush();
+	void Scrub(ScrubContext& Ctx);
 
 private:
 	struct CacheRef
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 14f4531ab..502ca6605 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -105,6 +105,12 @@ ZenCacheStore::Flush()
 	m_DiskLayer.Flush();
 }
 
+void
+ZenCacheStore::Scrub(ScrubContext& Ctx)
+{
+	m_DiskLayer.Scrub(Ctx);
+	m_MemLayer.Scrub(Ctx);
+}
 //////////////////////////////////////////////////////////////////////////
 
 ZenCacheMemoryLayer::ZenCacheMemoryLayer()
@@ -179,6 +185,12 @@ ZenCacheMemoryLayer::DropBucket(std::string_view Bucket)
 	return !!m_Buckets.erase(std::string(Bucket));
 }
 
+void
+ZenCacheMemoryLayer::Scrub(ScrubContext& Ctx)
+{
+	ZEN_UNUSED(Ctx);
+}
+
 bool
 ZenCacheMemoryLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutValue)
 {
@@ -375,12 +387,16 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir)
 void
 ZenCacheDiskLayer::CacheBucket::BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey)
 {
-	char hex[sizeof(HashKey.Hash) * 2];
-	ToHexBytes(HashKey.Hash, sizeof HashKey.Hash, hex);
+	char HexString[sizeof(HashKey.Hash) * 2];
+	ToHexBytes(HashKey.Hash, sizeof HashKey.Hash, HexString);
 
 	Path.Append(m_BucketDir.c_str());
+	Path.Append(L"/blob/");
+	Path.AppendAsciiRange(HexString, HexString + 3);
 	Path.Append(L"/");
-	Path.AppendAsciiRange(hex, hex + sizeof(hex));
+	Path.AppendAsciiRange(HexString + 3, HexString + 5);
+	Path.Append(L"/");
+	Path.AppendAsciiRange(HexString + 5, HexString + sizeof(HexString));
 }
 
 bool
@@ -495,12 +511,22 @@ ZenCacheDiskLayer::CacheBucket::Flush()
 }
 
 void
+ZenCacheDiskLayer::Scrub(ScrubContext& Ctx)
+{
+	ZEN_UNUSED(Ctx);
+}
+
+void
 ZenCacheDiskLayer::CacheBucket::PutLargeObject(const IoHash& HashKey, const ZenCacheValue& Value)
 {
 	WideStringBuilder<128> DataFilePath;
 	BuildPath(DataFilePath, HashKey);
 
-	// TODO: replace this with a more efficient implementation with proper atomic rename
+	// TODO: replace this process with a more efficient implementation with proper atomic rename
+	// and also avoid creating directories if we can
+
+	std::filesystem::path ParentPath = std::filesystem::path(DataFilePath.c_str()).parent_path();
+	CreateDirectories(ParentPath);
 
 	CAtlTemporaryFile DataFile;
 
@@ -518,6 +544,8 @@ ZenCacheDiskLayer::CacheBucket::PutLargeObject(const IoHash& HashKey, const ZenC
 		ThrowSystemException(hRes, "Failed to write payload ({} bytes) to file"_format(NiceBytes(Value.Value.Size())));
 	}
 
+	// Move file into place (note: not fully atomic!)
+
 	hRes = DataFile.Close(DataFilePath.c_str());
 
 	if (FAILED(hRes))
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index c4fa20958..fdf4a8cfe 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -55,6 +55,7 @@ public:
 	bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
 	void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
 	bool DropBucket(std::string_view Bucket);
+	void Scrub(ScrubContext& Ctx);
 
 private:
 	struct CacheBucket
@@ -80,6 +81,7 @@ public:
 	void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
 	bool DropBucket(std::string_view Bucket);
 	void Flush();
+	void Scrub(ScrubContext& Ctx);
 
 private:
 	/** A cache bucket manages a single directory containing
@@ -103,6 +105,7 @@ public:
 	void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
 	bool DropBucket(std::string_view Bucket);
 	void Flush();
+	void Scrub(ScrubContext& Ctx);
 
 private:
 	std::filesystem::path m_RootDir;