Merged from origin/main

1 files changed, 63 insertions, 113 deletions

diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 0d7ad0f16..c94a8fe35 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -11,13 +11,17 @@
 #include <zencore/string.h>
 #include <zencore/timer.h>
 #include <zencore/windows.h>
+#include <zenstore/basicfile.h>
 #include <zenstore/cas.h>
 #include <zenstore/caslog.h>
 
-#pragma comment(lib, "Rpcrt4.lib")	// RocksDB made me do this
-#include <rocksdb/db.h>
+#define USE_ROCKSDB 0
+
+#if USE_ROCKSDB
+#	pragma comment(lib, "Rpcrt4.lib")	// RocksDB made me do this
+#	include <rocksdb/db.h>
+#endif
 
-#include <lmdb.h>
 #include <ppl.h>
 #include <spdlog/spdlog.h>
 #include <xxh3.h>
@@ -27,9 +31,12 @@
 
 namespace zen {
 
-namespace rocksdb = ROCKSDB_NAMESPACE;
 using namespace fmt::literals;
 
+#if USE_ROCKSDB
+namespace rocksdb = ROCKSDB_NAMESPACE;
+#endif
+
 //////////////////////////////////////////////////////////////////////////
 
 struct ProjectStore::OplogStorage : public RefCounted
@@ -43,12 +50,7 @@ struct ProjectStore::OplogStorage : public RefCounted
 		Log().info("closing oplog storage at {}", m_OplogStoragePath);
 		Flush();
 
-		if (m_LmdbEnv)
-		{
-			mdb_env_close(m_LmdbEnv);
-			m_LmdbEnv = nullptr;
-		}
-
+#if USE_ROCKSDB
 		if (m_RocksDb)
 		{
 			// Column families must be torn down before database is closed
@@ -64,6 +66,7 @@ struct ProjectStore::OplogStorage : public RefCounted
 				Log().warn("db close error reported for '{}' : '{}'", m_OplogStoragePath, Status.getState());
 			}
 		}
+#endif
 	}
 
 	[[nodiscard]] bool		  Exists() { return Exists(m_OplogStoragePath); }
@@ -93,15 +96,7 @@ struct ProjectStore::OplogStorage : public RefCounted
 		ZEN_ASSERT(IsPow2(m_OpsAlign));
 		ZEN_ASSERT(!(m_NextOpsOffset & (m_OpsAlign - 1)));
 
-		{
-			std::string LmdbPath = WideToUtf8((m_OplogStoragePath / "ops.zdb").native().c_str());
-
-			int rc = mdb_env_create(&m_LmdbEnv);
-			rc	   = mdb_env_set_mapsize(m_LmdbEnv, 8 * 1024 * 1024);
-			rc	   = mdb_env_set_maxreaders(m_LmdbEnv, 256);
-			rc	   = mdb_env_open(m_LmdbEnv, LmdbPath.c_str(), MDB_NOSUBDIR | MDB_WRITEMAP | MDB_NOMETASYNC | MDB_NOSYNC, 0666);
-		}
-
+#if USE_ROCKSDB
 		{
 			std::string RocksdbPath = WideToUtf8((m_OplogStoragePath / "ops.rdb").native().c_str());
 
@@ -143,6 +138,7 @@ struct ProjectStore::OplogStorage : public RefCounted
 
 			m_RocksDb.reset(Db);
 		}
+#endif
 	}
 
 	void ReplayLog(std::function<void(CbObject, const OplogEntry&)>&& Handler)
@@ -228,13 +224,15 @@ private:
 	std::filesystem::path					  m_OplogStoragePath;
 	RwLock									  m_RwLock;
 	TCasLogFile<OplogEntry>					  m_Oplog;
-	CasBlobFile								  m_OpBlobs;
+	BasicFile								  m_OpBlobs;
 	std::atomic<uint64_t>					  m_NextOpsOffset{0};
 	uint64_t								  m_OpsAlign = 32;
 	std::atomic<uint32_t>					  m_MaxLsn{0};
-	MDB_env*								  m_LmdbEnv = nullptr;
+
+#if USE_ROCKSDB
 	std::unique_ptr<rocksdb::DB>			  m_RocksDb;
 	std::vector<rocksdb::ColumnFamilyHandle*> m_RocksDbColumnHandles;
+#endif
 };
 
 //////////////////////////////////////////////////////////////////////////
@@ -276,9 +274,9 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId)
 		return m_CasStore.FindChunk(ChunkIt->second);
 	}
 
-	if (auto FileIt = m_ServerFileMap.find(ChunkId); FileIt != m_ServerFileMap.end())
+	if (auto FileIt = m_FileMap.find(ChunkId); FileIt != m_FileMap.end())
 	{
-		std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second;
+		std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second.ServerPath;
 
 		return IoBufferBuilder::MakeFromFile(FilePath.native().c_str());
 	}
@@ -296,20 +294,35 @@ ProjectStore::Oplog::IterateFileMap(std::function<void(const Oid&, const std::st
 {
 	for (const auto& Kv : m_FileMap)
 	{
-		Fn(Kv.first, Kv.second);
+		Fn(Kv.first, Kv.second.ClientPath);
 	}
 }
 
-void
-ProjectStore::Oplog::AddFileMapping(Oid FileId, std::string_view Path)
+bool
+ProjectStore::Oplog::AddFileMapping(Oid FileId, IoHash Hash, std::string_view ServerPath, std::string_view ClientPath)
 {
-	m_FileMap.emplace(FileId, Path);
-}
+	if (ServerPath.empty() || ClientPath.empty())
+	{
+		return false;
+	}
 
-void
-ProjectStore::Oplog::AddServerFileMapping(Oid FileId, std::string_view Path)
-{
-	m_ServerFileMap.emplace(FileId, Path);
+	if (ServerPath[0] == '/')
+	{
+		ServerPath = ServerPath.substr(1);
+	}
+
+	FileMapEntry Entry;
+	Entry.ServerPath = ServerPath;
+	Entry.ClientPath = ClientPath;
+
+	m_FileMap.emplace(FileId, std::move(Entry));
+
+	if (Hash != IoHash::Zero)
+	{
+		m_ChunkMap.emplace(FileId, Hash);
+	}
+
+	return true;
 }
 
 void
@@ -356,93 +369,30 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry
 		Log().debug("bulkdata {} -> {}", BulkDataId, BulkDataHash);
 	}
 
-	if (CbFieldView FilesArray = Core["files"sv])
+	if (Core["files"sv])
 	{
-		int FileCount		= 0;
-		int ServerFileCount = 0;
-
-		std::atomic<bool> InvalidOp{false};
-
 		Stopwatch Timer;
+		int32_t	  FileCount = 0;
 
-		std::future<void> f0 = std::async(std::launch::async, [&] {
-			for (CbFieldView& Entry : FilesArray)
-			{
-				CbObjectView FileObj = Entry.AsObjectView();
-				const Oid	 FileId	 = FileObj["id"sv].AsObjectId();
-
-				if (auto PathField = FileObj["path"sv])
-				{
-					AddFileMapping(FileId, PathField.AsString());
-
-					// Log().debug("file {} -> {}", FileId, PathString);
-
-					++FileCount;
-				}
-				else
-				{
-					// Every file entry needs to specify a path
-					InvalidOp = true;
-					break;
-				}
-
-				if (InvalidOp.load(std::memory_order::relaxed))
-				{
-					break;
-				}
-			}
-		});
-
-		std::future<void> f1 = std::async(std::launch::async, [&] {
-			CbArrayView ServerFilesArray = Core["serverfiles"sv].AsArrayView();
+		for (CbFieldView& Entry : Core["files"sv])
+		{
+			CbObjectView	 FileObj	  = Entry.AsObjectView();
+			const Oid		 FileId		  = FileObj["id"sv].AsObjectId();
+			IoHash			 FileDataHash = FileObj["data"sv].AsBinaryAttachment();
+			std::string_view ServerPath	  = FileObj["serverpath"sv].AsString();
+			std::string_view ClientPath	  = FileObj["clientpath"sv].AsString();
 
-			for (CbFieldView& Entry : ServerFilesArray)
+			if (AddFileMapping(FileId, FileDataHash, ServerPath, ClientPath))
 			{
-				CbObjectView FileObj = Entry.AsObjectView();
-				const Oid	 FileId	 = FileObj["id"sv].AsObjectId();
-
-				if (auto PathField = FileObj["path"sv])
-				{
-					AddServerFileMapping(FileId, PathField.AsString());
-
-					// m_log.debug("file {} -> {}", FileId, PathString);
-
-					++ServerFileCount;
-				}
-				else
-				{
-					// Every file entry needs to specify a path
-					InvalidOp = true;
-					break;
-				}
-
-				if (InvalidOp.load(std::memory_order::relaxed))
-				{
-					break;
-				}
+				++FileCount;
 			}
-		});
-
-		f0.wait();
-		f1.wait();
-
-		if (InvalidOp)
-		{
-			return kInvalidOp;
-		}
-
-		if (FileCount || ServerFileCount)
-		{
-			Log().debug("{} files registered, {} server files (took {})",
-						FileCount,
-						ServerFileCount,
-						NiceTimeSpanMs(Timer.getElapsedTimeMs()));
-
-			if (FileCount != ServerFileCount)
+			else
 			{
-				Log().warn("client/server file list mismatch: {} vs {}", FileCount, ServerFileCount);
+				Log().warn("invalid file");
 			}
 		}
+
+		Log().debug("added {} file(s) in {}", FileCount, NiceTimeSpanMs(Timer.getElapsedTimeMs()));
 	}
 
 	for (CbFieldView& Entry : Core["meta"sv])
@@ -510,7 +460,7 @@ ProjectStore::Project::Read()
 
 	spdlog::info("reading config for project '{}' from {}", Identifier, ProjectStateFilePath);
 
-	CasBlobFile Blob;
+	BasicFile Blob;
 	Blob.Open(ProjectStateFilePath, false);
 
 	IoBuffer		Obj				= Blob.ReadAll();
@@ -551,7 +501,7 @@ ProjectStore::Project::Write()
 
 	spdlog::info("persisting config for project '{}' to {}", Identifier, ProjectStateFilePath);
 
-	CasBlobFile Blob;
+	BasicFile Blob;
 	Blob.Open(ProjectStateFilePath, true);
 	Blob.Write(Mem.Data(), Mem.Size(), 0);
 	Blob.Flush();
@@ -1114,7 +1064,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
 			// the prep step rejected the chunk. This should be fixed since there's
 			// a performance cost associated with any file system activity
 
-			bool IsValid = true;
+			bool				IsValid = true;
 			std::vector<IoHash> MissingChunks;
 
 			CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {