aboutsummaryrefslogtreecommitdiff
path: root/zenserver/projectstore.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'zenserver/projectstore.cpp')
-rw-r--r--zenserver/projectstore.cpp176
1 files changed, 63 insertions, 113 deletions
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 0d7ad0f16..c94a8fe35 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -11,13 +11,17 @@
#include <zencore/string.h>
#include <zencore/timer.h>
#include <zencore/windows.h>
+#include <zenstore/basicfile.h>
#include <zenstore/cas.h>
#include <zenstore/caslog.h>
-#pragma comment(lib, "Rpcrt4.lib") // RocksDB made me do this
-#include <rocksdb/db.h>
+#define USE_ROCKSDB 0
+
+#if USE_ROCKSDB
+# pragma comment(lib, "Rpcrt4.lib") // RocksDB made me do this
+# include <rocksdb/db.h>
+#endif
-#include <lmdb.h>
#include <ppl.h>
#include <spdlog/spdlog.h>
#include <xxh3.h>
@@ -27,9 +31,12 @@
namespace zen {
-namespace rocksdb = ROCKSDB_NAMESPACE;
using namespace fmt::literals;
+#if USE_ROCKSDB
+namespace rocksdb = ROCKSDB_NAMESPACE;
+#endif
+
//////////////////////////////////////////////////////////////////////////
struct ProjectStore::OplogStorage : public RefCounted
@@ -43,12 +50,7 @@ struct ProjectStore::OplogStorage : public RefCounted
Log().info("closing oplog storage at {}", m_OplogStoragePath);
Flush();
- if (m_LmdbEnv)
- {
- mdb_env_close(m_LmdbEnv);
- m_LmdbEnv = nullptr;
- }
-
+#if USE_ROCKSDB
if (m_RocksDb)
{
// Column families must be torn down before database is closed
@@ -64,6 +66,7 @@ struct ProjectStore::OplogStorage : public RefCounted
Log().warn("db close error reported for '{}' : '{}'", m_OplogStoragePath, Status.getState());
}
}
+#endif
}
[[nodiscard]] bool Exists() { return Exists(m_OplogStoragePath); }
@@ -93,15 +96,7 @@ struct ProjectStore::OplogStorage : public RefCounted
ZEN_ASSERT(IsPow2(m_OpsAlign));
ZEN_ASSERT(!(m_NextOpsOffset & (m_OpsAlign - 1)));
- {
- std::string LmdbPath = WideToUtf8((m_OplogStoragePath / "ops.zdb").native().c_str());
-
- int rc = mdb_env_create(&m_LmdbEnv);
- rc = mdb_env_set_mapsize(m_LmdbEnv, 8 * 1024 * 1024);
- rc = mdb_env_set_maxreaders(m_LmdbEnv, 256);
- rc = mdb_env_open(m_LmdbEnv, LmdbPath.c_str(), MDB_NOSUBDIR | MDB_WRITEMAP | MDB_NOMETASYNC | MDB_NOSYNC, 0666);
- }
-
+#if USE_ROCKSDB
{
std::string RocksdbPath = WideToUtf8((m_OplogStoragePath / "ops.rdb").native().c_str());
@@ -143,6 +138,7 @@ struct ProjectStore::OplogStorage : public RefCounted
m_RocksDb.reset(Db);
}
+#endif
}
void ReplayLog(std::function<void(CbObject, const OplogEntry&)>&& Handler)
@@ -228,13 +224,15 @@ private:
std::filesystem::path m_OplogStoragePath;
RwLock m_RwLock;
TCasLogFile<OplogEntry> m_Oplog;
- CasBlobFile m_OpBlobs;
+ BasicFile m_OpBlobs;
std::atomic<uint64_t> m_NextOpsOffset{0};
uint64_t m_OpsAlign = 32;
std::atomic<uint32_t> m_MaxLsn{0};
- MDB_env* m_LmdbEnv = nullptr;
+
+#if USE_ROCKSDB
std::unique_ptr<rocksdb::DB> m_RocksDb;
std::vector<rocksdb::ColumnFamilyHandle*> m_RocksDbColumnHandles;
+#endif
};
//////////////////////////////////////////////////////////////////////////
@@ -276,9 +274,9 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId)
return m_CasStore.FindChunk(ChunkIt->second);
}
- if (auto FileIt = m_ServerFileMap.find(ChunkId); FileIt != m_ServerFileMap.end())
+ if (auto FileIt = m_FileMap.find(ChunkId); FileIt != m_FileMap.end())
{
- std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second;
+ std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second.ServerPath;
return IoBufferBuilder::MakeFromFile(FilePath.native().c_str());
}
@@ -296,20 +294,35 @@ ProjectStore::Oplog::IterateFileMap(std::function<void(const Oid&, const std::st
{
for (const auto& Kv : m_FileMap)
{
- Fn(Kv.first, Kv.second);
+ Fn(Kv.first, Kv.second.ClientPath);
}
}
-void
-ProjectStore::Oplog::AddFileMapping(Oid FileId, std::string_view Path)
+bool
+ProjectStore::Oplog::AddFileMapping(Oid FileId, IoHash Hash, std::string_view ServerPath, std::string_view ClientPath)
{
- m_FileMap.emplace(FileId, Path);
-}
+ if (ServerPath.empty() || ClientPath.empty())
+ {
+ return false;
+ }
-void
-ProjectStore::Oplog::AddServerFileMapping(Oid FileId, std::string_view Path)
-{
- m_ServerFileMap.emplace(FileId, Path);
+ if (ServerPath[0] == '/')
+ {
+ ServerPath = ServerPath.substr(1);
+ }
+
+ FileMapEntry Entry;
+ Entry.ServerPath = ServerPath;
+ Entry.ClientPath = ClientPath;
+
+ m_FileMap.emplace(FileId, std::move(Entry));
+
+ if (Hash != IoHash::Zero)
+ {
+ m_ChunkMap.emplace(FileId, Hash);
+ }
+
+ return true;
}
void
@@ -356,93 +369,30 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry
Log().debug("bulkdata {} -> {}", BulkDataId, BulkDataHash);
}
- if (CbFieldView FilesArray = Core["files"sv])
+ if (Core["files"sv])
{
- int FileCount = 0;
- int ServerFileCount = 0;
-
- std::atomic<bool> InvalidOp{false};
-
Stopwatch Timer;
+ int32_t FileCount = 0;
- std::future<void> f0 = std::async(std::launch::async, [&] {
- for (CbFieldView& Entry : FilesArray)
- {
- CbObjectView FileObj = Entry.AsObjectView();
- const Oid FileId = FileObj["id"sv].AsObjectId();
-
- if (auto PathField = FileObj["path"sv])
- {
- AddFileMapping(FileId, PathField.AsString());
-
- // Log().debug("file {} -> {}", FileId, PathString);
-
- ++FileCount;
- }
- else
- {
- // Every file entry needs to specify a path
- InvalidOp = true;
- break;
- }
-
- if (InvalidOp.load(std::memory_order::relaxed))
- {
- break;
- }
- }
- });
-
- std::future<void> f1 = std::async(std::launch::async, [&] {
- CbArrayView ServerFilesArray = Core["serverfiles"sv].AsArrayView();
+ for (CbFieldView& Entry : Core["files"sv])
+ {
+ CbObjectView FileObj = Entry.AsObjectView();
+ const Oid FileId = FileObj["id"sv].AsObjectId();
+ IoHash FileDataHash = FileObj["data"sv].AsBinaryAttachment();
+ std::string_view ServerPath = FileObj["serverpath"sv].AsString();
+ std::string_view ClientPath = FileObj["clientpath"sv].AsString();
- for (CbFieldView& Entry : ServerFilesArray)
+ if (AddFileMapping(FileId, FileDataHash, ServerPath, ClientPath))
{
- CbObjectView FileObj = Entry.AsObjectView();
- const Oid FileId = FileObj["id"sv].AsObjectId();
-
- if (auto PathField = FileObj["path"sv])
- {
- AddServerFileMapping(FileId, PathField.AsString());
-
- // m_log.debug("file {} -> {}", FileId, PathString);
-
- ++ServerFileCount;
- }
- else
- {
- // Every file entry needs to specify a path
- InvalidOp = true;
- break;
- }
-
- if (InvalidOp.load(std::memory_order::relaxed))
- {
- break;
- }
+ ++FileCount;
}
- });
-
- f0.wait();
- f1.wait();
-
- if (InvalidOp)
- {
- return kInvalidOp;
- }
-
- if (FileCount || ServerFileCount)
- {
- Log().debug("{} files registered, {} server files (took {})",
- FileCount,
- ServerFileCount,
- NiceTimeSpanMs(Timer.getElapsedTimeMs()));
-
- if (FileCount != ServerFileCount)
+ else
{
- Log().warn("client/server file list mismatch: {} vs {}", FileCount, ServerFileCount);
+ Log().warn("invalid file");
}
}
+
+ Log().debug("added {} file(s) in {}", FileCount, NiceTimeSpanMs(Timer.getElapsedTimeMs()));
}
for (CbFieldView& Entry : Core["meta"sv])
@@ -510,7 +460,7 @@ ProjectStore::Project::Read()
spdlog::info("reading config for project '{}' from {}", Identifier, ProjectStateFilePath);
- CasBlobFile Blob;
+ BasicFile Blob;
Blob.Open(ProjectStateFilePath, false);
IoBuffer Obj = Blob.ReadAll();
@@ -551,7 +501,7 @@ ProjectStore::Project::Write()
spdlog::info("persisting config for project '{}' to {}", Identifier, ProjectStateFilePath);
- CasBlobFile Blob;
+ BasicFile Blob;
Blob.Open(ProjectStateFilePath, true);
Blob.Write(Mem.Data(), Mem.Size(), 0);
Blob.Flush();
@@ -1114,7 +1064,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects)
// the prep step rejected the chunk. This should be fixed since there's
// a performance cost associated with any file system activity
- bool IsValid = true;
+ bool IsValid = true;
std::vector<IoHash> MissingChunks;
CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer {
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-03 12:59:31 +0000