Split out structured cache store code into dedicated cpp/h pair

1 files changed, 10 insertions, 558 deletions

diff --git a/zenserver/cache/cachestore.cpp b/zenserver/cache/cachestore.cpp
index f99caa24f..1977d5f01 100644
--- a/zenserver/cache/cachestore.cpp
+++ b/zenserver/cache/cachestore.cpp
@@ -337,7 +337,7 @@ struct CorruptionTrailer
 	}
 };
 
-std::wstring
+std::filesystem::path
 GenerateDdcPath(std::string_view Key, std::filesystem::path& rootDir)
 {
 	std::filesystem::path FilePath = rootDir;
@@ -413,22 +413,22 @@ FileCacheStore::Get(std::string_view Key, CacheValue& OutValue)
 {
 	CAtlFile File;
 
-	std::wstring nativePath;
+	std::filesystem::path NativePath;
 
 	HRESULT hRes = E_FAIL;
 
 	if (m_ReadRootDir.empty() == false)
 	{
-		nativePath = UE::GenerateDdcPath(Key, m_ReadRootDir);
+		NativePath = UE::GenerateDdcPath(Key, m_ReadRootDir);
 
-		hRes = File.Create(nativePath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);
+		hRes = File.Create(NativePath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);
 	}
 
 	if (FAILED(hRes))
 	{
-		nativePath = UE::GenerateDdcPath(Key, m_RootDir);
+		NativePath = UE::GenerateDdcPath(Key, m_RootDir);
 
-		hRes = File.Create(nativePath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);
+		hRes = File.Create(NativePath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);
 	}
 
 	if (FAILED(hRes))
@@ -448,28 +448,7 @@ FileCacheStore::Get(std::string_view Key, CacheValue& OutValue)
 
 	FileSize -= 16;	 // CorruptionWrapper trailer
 
-	IoBuffer Value(FileSize);
-
-	uint8_t* ReadPointer = (uint8_t*)Value.Data();
-
-	while (FileSize)
-	{
-		const int MaxChunkSize = 16 * 1024 * 1024;
-		const int ChunkSize	   = gsl::narrow_cast<int>((FileSize > MaxChunkSize) ? MaxChunkSize : FileSize);
-
-		DWORD BytesRead = 0;
-		hRes			= File.Read(ReadPointer, ChunkSize, BytesRead);
-
-		if (FAILED(hRes))
-		{
-			return false;
-		}
-
-		ReadPointer += BytesRead;
-		FileSize -= BytesRead;
-	}
-
-	OutValue.Value = std::move(Value);
+	OutValue.Value = IoBuffer(IoBuffer::File, File.Detach(), 0, FileSize);
 
 	spdlog::debug("GET HIT {}", Key);
 
@@ -485,7 +464,7 @@ FileCacheStore::Put(std::string_view Key, const CacheValue& Value)
 	UE::CorruptionTrailer Trailer;
 	Trailer.Initialize(Data, Size);
 
-	std::wstring nativePath = UE::GenerateDdcPath(Key, m_RootDir);
+	std::filesystem::path NativePath = UE::GenerateDdcPath(Key, m_RootDir);
 
 	CAtlTemporaryFile File;
 
@@ -495,7 +474,7 @@ FileCacheStore::Put(std::string_view Key, const CacheValue& Value)
 
 	if (SUCCEEDED(hRes))
 	{
-		const uint8_t* WritePointer = (const uint8_t*)Data;
+		const uint8_t* WritePointer = reinterpret_cast<const uint8_t*>(Data);
 
 		while (Size)
 		{
@@ -510,7 +489,7 @@ FileCacheStore::Put(std::string_view Key, const CacheValue& Value)
 		}
 
 		File.Write(&Trailer, sizeof Trailer);
-		hRes = File.Close(nativePath.c_str());	// This renames the file to its final name
+		hRes = File.Close(NativePath.c_str());	// This renames the file to its final name
 
 		if (FAILED(hRes))
 		{
@@ -559,530 +538,3 @@ MemoryCacheStore::Put(std::string_view Key, const CacheValue& Value)
 	RwLock::ExclusiveLockScope _(m_Lock);
 	m_CacheMap[std::string(Key)] = Value.Value;
 }
-
-//////////////////////////////////////////////////////////////////////////
-
-ZenCacheStore::ZenCacheStore(zen::CasStore& Cas, const std::filesystem::path& RootDir) : m_DiskLayer{Cas, RootDir}
-{
-	zen::CreateDirectories(RootDir);
-}
-
-ZenCacheStore::~ZenCacheStore()
-{
-}
-
-bool
-ZenCacheStore::Get(std::string_view InBucket, const zen::IoHash& HashKey, ZenCacheValue& OutValue)
-{
-	bool Ok = m_MemLayer.Get(InBucket, HashKey, OutValue);
-
-	if (!Ok)
-	{
-		Ok = m_DiskLayer.Get(InBucket, HashKey, OutValue);
-
-#if 0  // This would keep file handles open
-		if (ok)
-		{
-			m_memLayer.Put(InBucket, HashKey, OutValue);
-		}
-#endif
-	}
-
-	return Ok;
-}
-
-void
-ZenCacheStore::Put(std::string_view InBucket, const zen::IoHash& HashKey, const ZenCacheValue& Value)
-{
-	m_MemLayer.Put(InBucket, HashKey, Value);
-	m_DiskLayer.Put(InBucket, HashKey, Value);
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-ZenCacheMemoryLayer::ZenCacheMemoryLayer()
-{
-}
-
-ZenCacheMemoryLayer::~ZenCacheMemoryLayer()
-{
-}
-
-bool
-ZenCacheMemoryLayer::Get(std::string_view InBucket, const zen::IoHash& HashKey, ZenCacheValue& OutValue)
-{
-	CacheBucket* Bucket = nullptr;
-
-	{
-		RwLock::SharedLockScope _(m_Lock);
-
-		auto it = m_Buckets.find(std::string(InBucket));
-
-		if (it != m_Buckets.end())
-		{
-			Bucket = &it->second;
-		}
-	}
-
-	if (Bucket == nullptr)
-		return false;
-
-	ZEN_ASSERT(Bucket != nullptr);
-
-	return Bucket->Get(HashKey, OutValue);
-}
-
-void
-ZenCacheMemoryLayer::Put(std::string_view InBucket, const zen::IoHash& HashKey, const ZenCacheValue& Value)
-{
-	CacheBucket* Bucket = nullptr;
-
-	{
-		RwLock::SharedLockScope _(m_Lock);
-
-		auto it = m_Buckets.find(std::string(InBucket));
-
-		if (it != m_Buckets.end())
-		{
-			Bucket = &it->second;
-		}
-	}
-
-	if (Bucket == nullptr)
-	{
-		// New bucket
-
-		RwLock::ExclusiveLockScope _(m_Lock);
-
-		Bucket = &m_Buckets[std::string(InBucket)];
-	}
-
-	ZEN_ASSERT(Bucket != nullptr);
-
-	Bucket->Put(HashKey, Value);
-}
-
-bool
-ZenCacheMemoryLayer::CacheBucket::Get(const zen::IoHash& HashKey, ZenCacheValue& OutValue)
-{
-	RwLock::SharedLockScope _(m_bucketLock);
-
-	auto bucketIt = m_cacheMap.find(HashKey);
-
-	if (bucketIt == m_cacheMap.end())
-	{
-		return false;
-	}
-
-	OutValue.Value = bucketIt->second;
-
-	return true;
-}
-
-void
-ZenCacheMemoryLayer::CacheBucket::Put(const zen::IoHash& HashKey, const ZenCacheValue& Value)
-{
-	RwLock::ExclusiveLockScope _(m_bucketLock);
-
-	m_cacheMap[HashKey] = Value.Value;
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-#pragma pack(push)
-#pragma pack(1)
-
-struct DiskLocation
-{
-	uint64_t Offset;
-	uint32_t Size;
-};
-
-struct DiskIndexEntry
-{
-	zen::IoHash	 Key;
-	DiskLocation Location;
-};
-
-#pragma pack(pop)
-
-static_assert(sizeof(DiskIndexEntry) == 32);
-
-struct ZenCacheDiskLayer::CacheBucket
-{
-	CacheBucket(CasStore& Cas);
-	~CacheBucket();
-
-	void OpenOrCreate(std::filesystem::path BucketDir);
-
-	bool Get(const zen::IoHash& HashKey, ZenCacheValue& OutValue);
-	void Put(const zen::IoHash& HashKey, const ZenCacheValue& Value);
-	void Flush();
-
-	inline bool IsOk() const { return m_Ok; }
-
-private:
-	CasStore&			  m_CasStore;
-	std::filesystem::path m_BucketDir;
-	Oid					  m_BucketId;
-	bool				  m_Ok					 = false;
-	uint64_t			  m_LargeObjectThreshold = 1024;
-
-	BasicFile					m_SobsFile;
-	TCasLogFile<DiskIndexEntry> m_SlogFile;
-	BasicFile					m_SidxFile;
-
-	void BuildPath(zen::WideStringBuilderBase& Path, const zen::IoHash& HashKey);
-	void PutLargeObject(const zen::IoHash& HashKey, const ZenCacheValue& Value);
-
-	RwLock															   m_IndexLock;
-	std::unordered_map<zen::IoHash, DiskLocation, zen::IoHash::Hasher> m_Index;
-	uint64_t														   m_WriteCursor = 0;
-};
-
-ZenCacheDiskLayer::CacheBucket::CacheBucket(CasStore& Cas) : m_CasStore(Cas)
-{
-}
-
-ZenCacheDiskLayer::CacheBucket::~CacheBucket()
-{
-}
-
-void
-ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir)
-{
-	std::filesystem::create_directories(BucketDir);
-
-	m_BucketDir = BucketDir;
-
-	std::filesystem::path ManifestPath{m_BucketDir / "zen_manifest"};
-	std::filesystem::path SobsPath{m_BucketDir / "zen.sobs"};
-	std::filesystem::path SlogPath{m_BucketDir / "zen.slog"};
-	std::filesystem::path SidxPath{m_BucketDir / "zen.sidx"};
-
-	CAtlFile ManifestFile;
-
-	// Try opening existing file first
-
-	bool IsNew = false;
-
-	HRESULT hRes = ManifestFile.Create(ManifestPath.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, OPEN_EXISTING);
-
-	if (SUCCEEDED(hRes))
-	{
-		ULONGLONG FileSize;
-		ManifestFile.GetSize(FileSize);
-
-		if (FileSize == sizeof(Oid))
-		{
-			hRes = ManifestFile.Read(&m_BucketId, sizeof(Oid));
-
-			if (SUCCEEDED(hRes))
-			{
-				m_Ok = true;
-			}
-		}
-
-		if (!m_Ok)
-			ManifestFile.Close();
-	}
-
-	if (!m_Ok)
-	{
-		// This is a new bucket
-
-		hRes = ManifestFile.Create(ManifestPath.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS);
-
-		if (FAILED(hRes))
-		{
-			ThrowLastError("Failed to create bucket manifest '{}'"_format(ManifestPath));
-		}
-
-		m_BucketId.Generate();
-
-		hRes = ManifestFile.Write(&m_BucketId, sizeof(Oid));
-
-		IsNew = true;
-	}
-
-	// Initialize small object storage related files
-
-	m_SobsFile.Open(SobsPath, IsNew);
-	m_SidxFile.Open(SidxPath, IsNew);
-
-	// Open and replay log
-
-	m_SlogFile.Open(SlogPath, IsNew);
-
-	uint64_t MaxFileOffset = 0;
-
-	{
-		// This is not technically necessary but may help future static analysis
-		zen::RwLock::ExclusiveLockScope _(m_IndexLock);
-
-		m_SlogFile.Replay([&](const DiskIndexEntry& Record) {
-			m_Index[Record.Key] = Record.Location;
-
-			MaxFileOffset = std::max<uint64_t>(MaxFileOffset, Record.Location.Offset + Record.Location.Size);
-		});
-	}
-
-	m_WriteCursor = (MaxFileOffset + 15) & ~15;
-
-	m_Ok = true;
-}
-
-void
-ZenCacheDiskLayer::CacheBucket::BuildPath(zen::WideStringBuilderBase& Path, const zen::IoHash& HashKey)
-{
-	char hex[sizeof(HashKey.Hash) * 2];
-	ToHexBytes(HashKey.Hash, sizeof HashKey.Hash, hex);
-
-	Path.Append(m_BucketDir.c_str());
-	Path.Append(L"/");
-	Path.AppendAsciiRange(hex, hex + sizeof(hex));
-}
-
-bool
-ZenCacheDiskLayer::CacheBucket::Get(const zen::IoHash& HashKey, ZenCacheValue& OutValue)
-{
-	if (!m_Ok)
-		return false;
-
-	{
-		zen::RwLock::SharedLockScope _(m_IndexLock);
-
-		if (auto it = m_Index.find(HashKey); it != m_Index.end())
-		{
-			OutValue.Value = IoBufferBuilder::MakeFromFileHandle(m_SobsFile.Handle(), it->second.Offset, it->second.Size);
-
-			return true;
-		}
-	}
-
-	WideStringBuilder<128> DataFilePath;
-	BuildPath(DataFilePath, HashKey);
-
-	zen::IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.c_str());
-
-	if (!Data)
-	{
-		return false;
-	}
-
-	OutValue.Value = Data;
-
-	// TODO: should populate index?
-
-	return true;
-}
-
-void
-ZenCacheDiskLayer::CacheBucket::Put(const zen::IoHash& HashKey, const ZenCacheValue& Value)
-{
-	if (!m_Ok)
-	{
-		return;
-	}
-
-	if (Value.Value.Size() >= m_LargeObjectThreshold)
-	{
-		return PutLargeObject(HashKey, Value);
-	}
-
-	// Small object put
-
-	zen::RwLock::ExclusiveLockScope _(m_IndexLock);
-
-	auto it = m_Index.find(HashKey);
-
-	DiskLocation Loc{.Offset = m_WriteCursor, .Size = gsl::narrow<uint32_t>(Value.Value.Size())};
-
-	m_WriteCursor = (m_WriteCursor + Loc.Size + 15) & ~15;
-
-	if (it == m_Index.end())
-	{
-		m_Index.insert({HashKey, Loc});
-	}
-	else
-	{
-		// TODO: should check if write is idempotent and bail out if it is?
-
-		it->second = Loc;
-	}
-
-	DiskIndexEntry IndexEntry{.Key = HashKey, .Location = Loc};
-
-	m_SlogFile.Append(IndexEntry);
-
-	m_SobsFile.Write(Value.Value.Data(), Loc.Size, Loc.Offset);
-
-	return;
-}
-
-void
-ZenCacheDiskLayer::CacheBucket::Flush()
-{
-	m_SobsFile.Flush();
-	m_SidxFile.Flush();
-	m_SlogFile.Flush();
-}
-
-void
-ZenCacheDiskLayer::CacheBucket::PutLargeObject(const zen::IoHash& HashKey, const ZenCacheValue& Value)
-{
-	zen::WideStringBuilder<128> DataFilePath;
-	BuildPath(DataFilePath, HashKey);
-
-	// TODO: replace this with a more efficient implementation with proper atomic rename
-
-	CAtlTemporaryFile DataFile;
-
-	HRESULT hRes = DataFile.Create(m_BucketDir.c_str());
-
-	if (FAILED(hRes))
-	{
-		zen::ThrowSystemException(hRes, "Failed to open temporary file for put at '{}'"_format(m_BucketDir));
-	}
-
-	hRes = DataFile.Write(Value.Value.Data(), gsl::narrow<DWORD>(Value.Value.Size()));
-
-	if (FAILED(hRes))
-	{
-		zen::ThrowSystemException(hRes, "Failed to write payload ({} bytes) to file"_format(NiceBytes(Value.Value.Size())));
-	}
-
-	hRes = DataFile.Close(DataFilePath.c_str());
-
-	if (FAILED(hRes))
-	{
-		zen::ThrowSystemException(hRes, "Failed to finalize file '{}'"_format(zen::WideToUtf8(DataFilePath)));
-	}
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-ZenCacheDiskLayer::ZenCacheDiskLayer(CasStore& Cas, const std::filesystem::path& RootDir) : m_RootDir(RootDir), m_CasStore(Cas)
-{
-}
-
-ZenCacheDiskLayer::~ZenCacheDiskLayer() = default;
-
-bool
-ZenCacheDiskLayer::Get(std::string_view InBucket, const zen::IoHash& HashKey, ZenCacheValue& OutValue)
-{
-	CacheBucket* Bucket = nullptr;
-
-	{
-		zen::RwLock::SharedLockScope _(m_Lock);
-
-		auto it = m_Buckets.find(std::string(InBucket));
-
-		if (it != m_Buckets.end())
-		{
-			Bucket = &it->second;
-		}
-	}
-
-	if (Bucket == nullptr)
-	{
-		// Bucket needs to be opened/created
-
-		zen::RwLock::ExclusiveLockScope _(m_Lock);
-
-		auto It = m_Buckets.try_emplace(std::string(InBucket), m_CasStore);
-		Bucket	= &It.first->second;
-
-		std::filesystem::path BucketPath = m_RootDir;
-		BucketPath /= std::string(InBucket);
-
-		Bucket->OpenOrCreate(BucketPath.c_str());
-	}
-
-	ZEN_ASSERT(Bucket != nullptr);
-
-	return Bucket->Get(HashKey, OutValue);
-}
-
-void
-ZenCacheDiskLayer::Put(std::string_view InBucket, const zen::IoHash& HashKey, const ZenCacheValue& Value)
-{
-	CacheBucket* Bucket = nullptr;
-
-	{
-		zen::RwLock::SharedLockScope _(m_Lock);
-
-		auto it = m_Buckets.find(std::string(InBucket));
-
-		if (it != m_Buckets.end())
-		{
-			Bucket = &it->second;
-		}
-	}
-
-	if (Bucket == nullptr)
-	{
-		// New bucket needs to be created
-
-		zen::RwLock::ExclusiveLockScope _(m_Lock);
-
-		auto It = m_Buckets.try_emplace(std::string(InBucket), m_CasStore);
-		Bucket	= &It.first->second;
-
-		std::filesystem::path bucketPath = m_RootDir;
-		bucketPath /= std::string(InBucket);
-
-		Bucket->OpenOrCreate(bucketPath.c_str());
-	}
-
-	ZEN_ASSERT(Bucket != nullptr);
-
-	if (Bucket->IsOk())
-	{
-		Bucket->Put(HashKey, Value);
-	}
-}
-
-void
-ZenCacheDiskLayer::Flush()
-{
-	std::vector<CacheBucket*> Buckets;
-	Buckets.reserve(m_Buckets.size());
-
-	{
-		zen::RwLock::SharedLockScope _(m_Lock);
-
-		for (auto& Kv : m_Buckets)
-		{
-			Buckets.push_back(&Kv.second);
-		}
-	}
-
-	for (auto& Bucket : Buckets)
-	{
-		Bucket->Flush();
-	}
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-ZenCacheTracker::ZenCacheTracker(ZenCacheStore& CacheStore)
-{
-	ZEN_UNUSED(CacheStore);
-}
-
-ZenCacheTracker::~ZenCacheTracker()
-{
-}
-
-void
-ZenCacheTracker::TrackAccess(std::string_view Bucket, const zen::IoHash& HashKey)
-{
-	ZEN_UNUSED(Bucket);
-	ZEN_UNUSED(HashKey);
-}
-
-void
-ZenCacheTracker::Flush()
-{
-}