From d795184a5c29727079903298a762c0059362d478 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Fri, 21 May 2021 09:52:28 +0200 Subject: =?UTF-8?q?Added=20support=20for=20package=20additional=20files.?= =?UTF-8?q?=20Added=20io=20hash=20chunk=20id=20to=E2=80=A6=20(#2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added support for attaching additional files to oplog entries in project store --- zenserver/projectstore.cpp | 132 +++++++++++++++------------------------------ 1 file changed, 42 insertions(+), 90 deletions(-) (limited to 'zenserver/projectstore.cpp') diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index 0d7ad0f16..8753d50fc 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -276,9 +276,9 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId) return m_CasStore.FindChunk(ChunkIt->second); } - if (auto FileIt = m_ServerFileMap.find(ChunkId); FileIt != m_ServerFileMap.end()) + if (auto FileIt = m_FileMap.find(ChunkId); FileIt != m_FileMap.end()) { - std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second; + std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second.ServerPath; return IoBufferBuilder::MakeFromFile(FilePath.native().c_str()); } @@ -296,20 +296,35 @@ ProjectStore::Oplog::IterateFileMap(std::function {}", BulkDataId, BulkDataHash); } - if (CbFieldView FilesArray = Core["files"sv]) + if (Core["files"sv]) { - int FileCount = 0; - int ServerFileCount = 0; - - std::atomic InvalidOp{false}; - Stopwatch Timer; + int32_t FileCount = 0; - std::future f0 = std::async(std::launch::async, [&] { - for (CbFieldView& Entry : FilesArray) - { - CbObjectView FileObj = Entry.AsObjectView(); - const Oid FileId = FileObj["id"sv].AsObjectId(); - - if (auto PathField = FileObj["path"sv]) - { - AddFileMapping(FileId, PathField.AsString()); - - // Log().debug("file {} -> {}", FileId, PathString); - - ++FileCount; - } - else - { - // Every file entry needs to specify a path - InvalidOp = true; - break; - } - - if (InvalidOp.load(std::memory_order::relaxed)) - { - break; - } - } - }); - - std::future f1 = std::async(std::launch::async, [&] { - CbArrayView ServerFilesArray = Core["serverfiles"sv].AsArrayView(); + for (CbFieldView& Entry : Core["files"sv]) + { + CbObjectView FileObj = Entry.AsObjectView(); + const Oid FileId = FileObj["id"sv].AsObjectId(); + IoHash FileDataHash = FileObj["data"sv].AsBinaryAttachment(); + std::string_view ServerPath = FileObj["serverpath"sv].AsString(); + std::string_view ClientPath = FileObj["clientpath"sv].AsString(); - for (CbFieldView& Entry : ServerFilesArray) + if (AddFileMapping(FileId, FileDataHash, ServerPath, ClientPath)) { - CbObjectView FileObj = Entry.AsObjectView(); - const Oid FileId = FileObj["id"sv].AsObjectId(); - - if (auto PathField = FileObj["path"sv]) - { - AddServerFileMapping(FileId, PathField.AsString()); - - // m_log.debug("file {} -> {}", FileId, PathString); - - ++ServerFileCount; - } - else - { - // Every file entry needs to specify a path - InvalidOp = true; - break; - } - - if (InvalidOp.load(std::memory_order::relaxed)) - { - break; - } + ++FileCount; } - }); - - f0.wait(); - f1.wait(); - - if (InvalidOp) - { - return kInvalidOp; - } - - if (FileCount || ServerFileCount) - { - Log().debug("{} files registered, {} server files (took {})", - FileCount, - ServerFileCount, - NiceTimeSpanMs(Timer.getElapsedTimeMs())); - - if (FileCount != ServerFileCount) + else { - Log().warn("client/server file list mismatch: {} vs {}", FileCount, ServerFileCount); + Log().warn("invalid file"); } } + + Log().debug("added {} file(s) in {}", FileCount, NiceTimeSpanMs(Timer.getElapsedTimeMs())); } for (CbFieldView& Entry : Core["meta"sv]) @@ -1114,7 +1066,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) // the prep step rejected the chunk. This should be fixed since there's // a performance cost associated with any file system activity - bool IsValid = true; + bool IsValid = true; std::vector MissingChunks; CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer { -- cgit v1.2.3 From 83e3c364e6403883cd60e7dda10165b6e84269f3 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 21 May 2021 20:40:09 +0200 Subject: Moved CasBlobFile into basicfile.h --- zenserver/projectstore.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'zenserver/projectstore.cpp') diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index 8753d50fc..e30742e33 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #pragma comment(lib, "Rpcrt4.lib") // RocksDB made me do this #include -- cgit v1.2.3 From edb2ea0cae30aeac74d1d48ab9754754f010c818 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 21 May 2021 20:43:36 +0200 Subject: Renamed CasBlobFile -> BasicFile --- zenserver/projectstore.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'zenserver/projectstore.cpp') diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index e30742e33..c99fa4232 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -229,7 +229,7 @@ private: std::filesystem::path m_OplogStoragePath; RwLock m_RwLock; TCasLogFile m_Oplog; - CasBlobFile m_OpBlobs; + BasicFile m_OpBlobs; std::atomic m_NextOpsOffset{0}; uint64_t m_OpsAlign = 32; std::atomic m_MaxLsn{0}; @@ -463,7 +463,7 @@ ProjectStore::Project::Read() spdlog::info("reading config for project '{}' from {}", Identifier, ProjectStateFilePath); - CasBlobFile Blob; + BasicFile Blob; Blob.Open(ProjectStateFilePath, false); IoBuffer Obj = Blob.ReadAll(); @@ -504,7 +504,7 @@ ProjectStore::Project::Write() spdlog::info("persisting config for project '{}' to {}", Identifier, ProjectStateFilePath); - CasBlobFile Blob; + BasicFile Blob; Blob.Open(ProjectStateFilePath, true); Blob.Write(Mem.Data(), Mem.Size(), 0); Blob.Flush(); -- cgit v1.2.3 From c0760dafc01a21abffb207a2598b978930d6fcb9 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sat, 22 May 2021 11:40:03 +0200 Subject: clang-format --- zenserver/projectstore.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver/projectstore.cpp') diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index c99fa4232..b353a6eac 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -11,9 +11,9 @@ #include #include #include +#include #include #include -#include #pragma comment(lib, "Rpcrt4.lib") // RocksDB made me do this #include -- cgit v1.2.3 From da806396c4a2b8d1bf097f31d28b03e4e8a29336 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 24 May 2021 22:24:28 +0200 Subject: Eliminated exploratory LMDB/RocksDB code from default path --- zenserver/projectstore.cpp | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) (limited to 'zenserver/projectstore.cpp') diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index b353a6eac..f99bfdcac 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -31,6 +31,8 @@ namespace zen { namespace rocksdb = ROCKSDB_NAMESPACE; using namespace fmt::literals; +bool UseRocksDb = false; + ////////////////////////////////////////////////////////////////////////// struct ProjectStore::OplogStorage : public RefCounted @@ -44,12 +46,6 @@ struct ProjectStore::OplogStorage : public RefCounted Log().info("closing oplog storage at {}", m_OplogStoragePath); Flush(); - if (m_LmdbEnv) - { - mdb_env_close(m_LmdbEnv); - m_LmdbEnv = nullptr; - } - if (m_RocksDb) { // Column families must be torn down before database is closed @@ -94,15 +90,7 @@ struct ProjectStore::OplogStorage : public RefCounted ZEN_ASSERT(IsPow2(m_OpsAlign)); ZEN_ASSERT(!(m_NextOpsOffset & (m_OpsAlign - 1))); - { - std::string LmdbPath = WideToUtf8((m_OplogStoragePath / "ops.zdb").native().c_str()); - - int rc = mdb_env_create(&m_LmdbEnv); - rc = mdb_env_set_mapsize(m_LmdbEnv, 8 * 1024 * 1024); - rc = mdb_env_set_maxreaders(m_LmdbEnv, 256); - rc = mdb_env_open(m_LmdbEnv, LmdbPath.c_str(), MDB_NOSUBDIR | MDB_WRITEMAP | MDB_NOMETASYNC | MDB_NOSYNC, 0666); - } - + if (UseRocksDb) { std::string RocksdbPath = WideToUtf8((m_OplogStoragePath / "ops.rdb").native().c_str()); @@ -233,7 +221,6 @@ private: std::atomic m_NextOpsOffset{0}; uint64_t m_OpsAlign = 32; std::atomic m_MaxLsn{0}; - MDB_env* m_LmdbEnv = nullptr; std::unique_ptr m_RocksDb; std::vector m_RocksDbColumnHandles; }; -- cgit v1.2.3 From 317b7654133b59d20554162da4a251bdf7794cb1 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 25 May 2021 09:28:21 +0200 Subject: Compile out all rocksdb code for a smaller binary --- zenserver/projectstore.cpp | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) (limited to 'zenserver/projectstore.cpp') diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index f99bfdcac..c94a8fe35 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -15,10 +15,13 @@ #include #include -#pragma comment(lib, "Rpcrt4.lib") // RocksDB made me do this -#include +#define USE_ROCKSDB 0 + +#if USE_ROCKSDB +# pragma comment(lib, "Rpcrt4.lib") // RocksDB made me do this +# include +#endif -#include #include #include #include @@ -28,10 +31,11 @@ namespace zen { -namespace rocksdb = ROCKSDB_NAMESPACE; using namespace fmt::literals; -bool UseRocksDb = false; +#if USE_ROCKSDB +namespace rocksdb = ROCKSDB_NAMESPACE; +#endif ////////////////////////////////////////////////////////////////////////// @@ -46,6 +50,7 @@ struct ProjectStore::OplogStorage : public RefCounted Log().info("closing oplog storage at {}", m_OplogStoragePath); Flush(); +#if USE_ROCKSDB if (m_RocksDb) { // Column families must be torn down before database is closed @@ -61,6 +66,7 @@ struct ProjectStore::OplogStorage : public RefCounted Log().warn("db close error reported for '{}' : '{}'", m_OplogStoragePath, Status.getState()); } } +#endif } [[nodiscard]] bool Exists() { return Exists(m_OplogStoragePath); } @@ -90,7 +96,7 @@ struct ProjectStore::OplogStorage : public RefCounted ZEN_ASSERT(IsPow2(m_OpsAlign)); ZEN_ASSERT(!(m_NextOpsOffset & (m_OpsAlign - 1))); - if (UseRocksDb) +#if USE_ROCKSDB { std::string RocksdbPath = WideToUtf8((m_OplogStoragePath / "ops.rdb").native().c_str()); @@ -132,6 +138,7 @@ struct ProjectStore::OplogStorage : public RefCounted m_RocksDb.reset(Db); } +#endif } void ReplayLog(std::function&& Handler) @@ -221,8 +228,11 @@ private: std::atomic m_NextOpsOffset{0}; uint64_t m_OpsAlign = 32; std::atomic m_MaxLsn{0}; + +#if USE_ROCKSDB std::unique_ptr m_RocksDb; std::vector m_RocksDbColumnHandles; +#endif }; ////////////////////////////////////////////////////////////////////////// -- cgit v1.2.3