diff options
| author | Stefan Boberg <[email protected]> | 2026-02-24 15:02:58 +0100 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2026-02-24 15:02:58 +0100 |
| commit | 2af00d3ff720969fb3b4d471778efcf8c7b3fad4 (patch) | |
| tree | 9193eb3ff8cc9f3fb5e3342d94bde8c193a977ce /src | |
| parent | added zencore/filesystem.h include (diff) | |
| parent | Various bug fixes (#778) (diff) | |
| download | zen-sb/spdlog-out.tar.xz zen-sb/spdlog-out.zip | |
Merge branch 'main' into sb/spdlog-outsb/spdlog-out
Diffstat (limited to 'src')
39 files changed, 800 insertions, 284 deletions
diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h index 87ef8091b..83bcf8893 100644 --- a/src/zen/cmds/admin_cmd.h +++ b/src/zen/cmds/admin_cmd.h @@ -13,6 +13,9 @@ namespace zen { class ScrubCommand : public StorageCommand { public: + static constexpr char Name[] = "scrub"; + static constexpr char Description[] = "Scrub zen storage (verify data integrity)"; + ScrubCommand(); ~ScrubCommand(); @@ -20,7 +23,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"scrub", "Scrub zen storage"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_DryRun = false; bool m_NoGc = false; @@ -33,6 +36,9 @@ private: class GcCommand : public StorageCommand { public: + static constexpr char Name[] = "gc"; + static constexpr char Description[] = "Garbage collect zen storage"; + GcCommand(); ~GcCommand(); @@ -40,7 +46,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"gc", "Garbage collect zen storage"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_SmallObjects{false}; bool m_SkipCid{false}; @@ -62,6 +68,9 @@ private: class GcStatusCommand : public StorageCommand { public: + static constexpr char Name[] = "gc-status"; + static constexpr char Description[] = "Garbage collect zen storage status check"; + GcStatusCommand(); ~GcStatusCommand(); @@ -69,7 +78,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"gc-status", "Garbage collect zen storage status check"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_Details = false; }; @@ -77,6 +86,9 @@ private: class GcStopCommand : public StorageCommand { public: + static constexpr char Name[] = "gc-stop"; + static constexpr char Description[] = "Request cancel of running garbage collection in zen storage"; + GcStopCommand(); ~GcStopCommand(); @@ -84,7 +96,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"gc-stop", "Request cancel of running garbage collection in zen storage"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; @@ -93,6 +105,9 @@ private: class JobCommand : public ZenCmdBase { public: + static constexpr char Name[] = "jobs"; + static constexpr char Description[] = "Show/cancel zen background jobs"; + JobCommand(); ~JobCommand(); @@ -100,7 +115,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"jobs", "Show/cancel zen background jobs"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::uint64_t m_JobId = 0; bool m_Cancel = 0; @@ -111,6 +126,9 @@ private: class LoggingCommand : public ZenCmdBase { public: + static constexpr char Name[] = "logs"; + static constexpr char Description[] = "Show/control zen logging"; + LoggingCommand(); ~LoggingCommand(); @@ -118,7 +136,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"logs", "Show/control zen logging"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_CacheWriteLog; std::string m_CacheAccessLog; @@ -133,6 +151,9 @@ private: class FlushCommand : public StorageCommand { public: + static constexpr char Name[] = "flush"; + static constexpr char Description[] = "Flush storage"; + FlushCommand(); ~FlushCommand(); @@ -140,7 +161,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"flush", "Flush zen storage"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; @@ -149,6 +170,9 @@ private: class CopyStateCommand : public StorageCommand { public: + static constexpr char Name[] = "copy-state"; + static constexpr char Description[] = "Copy zen server disk state"; + CopyStateCommand(); ~CopyStateCommand(); @@ -156,7 +180,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"copy-state", "Copy zen server disk state"}; + cxxopts::Options m_Options{Name, Description}; std::filesystem::path m_DataPath; std::filesystem::path m_TargetPath; bool m_SkipLogs = false; diff --git a/src/zen/cmds/bench_cmd.h b/src/zen/cmds/bench_cmd.h index ed123be75..7fbf85340 100644 --- a/src/zen/cmds/bench_cmd.h +++ b/src/zen/cmds/bench_cmd.h @@ -9,6 +9,9 @@ namespace zen { class BenchCommand : public ZenCmdBase { public: + static constexpr char Name[] = "bench"; + static constexpr char Description[] = "Utility command for benchmarking"; + BenchCommand(); ~BenchCommand(); @@ -17,7 +20,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"bench", "Benchmarking utility command"}; + cxxopts::Options m_Options{Name, Description}; bool m_PurgeStandbyLists = false; bool m_SingleProcess = false; }; diff --git a/src/zen/cmds/cache_cmd.h b/src/zen/cmds/cache_cmd.h index 4dc05bbdc..4f5b90f4d 100644 --- a/src/zen/cmds/cache_cmd.h +++ b/src/zen/cmds/cache_cmd.h @@ -9,6 +9,9 @@ namespace zen { class DropCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "drop"; + static constexpr char Description[] = "Drop cache namespace or bucket"; + DropCommand(); ~DropCommand(); @@ -16,7 +19,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"drop", "Drop cache namespace or bucket"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_NamespaceName; std::string m_BucketName; @@ -25,13 +28,16 @@ private: class CacheInfoCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "cache-info"; + static constexpr char Description[] = "Info on cache, namespace or bucket"; + CacheInfoCommand(); ~CacheInfoCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"cache-info", "Info on cache, namespace or bucket"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_NamespaceName; std::string m_SizeInfoBucketNames; @@ -42,26 +48,32 @@ private: class CacheStatsCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "cache-stats"; + static constexpr char Description[] = "Stats on cache"; + CacheStatsCommand(); ~CacheStatsCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"cache-stats", "Stats info on cache"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; class CacheDetailsCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "cache-details"; + static constexpr char Description[] = "Details on cache"; + CacheDetailsCommand(); ~CacheDetailsCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"cache-details", "Detailed info on cache"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_CSV = false; bool m_Details = false; diff --git a/src/zen/cmds/copy_cmd.h b/src/zen/cmds/copy_cmd.h index e1a5dcb82..757a8e691 100644 --- a/src/zen/cmds/copy_cmd.h +++ b/src/zen/cmds/copy_cmd.h @@ -11,6 +11,9 @@ namespace zen { class CopyCommand : public ZenCmdBase { public: + static constexpr char Name[] = "copy"; + static constexpr char Description[] = "Copy file(s)"; + CopyCommand(); ~CopyCommand(); @@ -19,7 +22,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"copy", "Copy files efficiently"}; + cxxopts::Options m_Options{Name, Description}; std::filesystem::path m_CopySource; std::filesystem::path m_CopyTarget; bool m_NoClone = false; diff --git a/src/zen/cmds/dedup_cmd.h b/src/zen/cmds/dedup_cmd.h index 5b8387dd2..835b35e92 100644 --- a/src/zen/cmds/dedup_cmd.h +++ b/src/zen/cmds/dedup_cmd.h @@ -11,6 +11,9 @@ namespace zen { class DedupCommand : public ZenCmdBase { public: + static constexpr char Name[] = "dedup"; + static constexpr char Description[] = "Dedup files"; + DedupCommand(); ~DedupCommand(); @@ -19,7 +22,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"dedup", "Deduplicate files"}; + cxxopts::Options m_Options{Name, Description}; std::vector<std::string> m_Positional; std::filesystem::path m_DedupSource; std::filesystem::path m_DedupTarget; diff --git a/src/zen/cmds/info_cmd.h b/src/zen/cmds/info_cmd.h index 231565bfd..dc108b8a2 100644 --- a/src/zen/cmds/info_cmd.h +++ b/src/zen/cmds/info_cmd.h @@ -9,6 +9,9 @@ namespace zen { class InfoCommand : public ZenCmdBase { public: + static constexpr char Name[] = "info"; + static constexpr char Description[] = "Show high level Zen server information"; + InfoCommand(); ~InfoCommand(); @@ -17,7 +20,7 @@ public: // virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"info", "Show high level zen store information"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; diff --git a/src/zen/cmds/print_cmd.h b/src/zen/cmds/print_cmd.h index 6c1529b7c..f4a97e218 100644 --- a/src/zen/cmds/print_cmd.h +++ b/src/zen/cmds/print_cmd.h @@ -11,6 +11,9 @@ namespace zen { class PrintCommand : public ZenCmdBase { public: + static constexpr char Name[] = "print"; + static constexpr char Description[] = "Print compact binary object"; + PrintCommand(); ~PrintCommand(); @@ -19,7 +22,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"print", "Print compact binary object"}; + cxxopts::Options m_Options{Name, Description}; std::filesystem::path m_Filename; bool m_ShowCbObjectTypeInfo = false; }; @@ -29,6 +32,9 @@ private: class PrintPackageCommand : public ZenCmdBase { public: + static constexpr char Name[] = "printpackage"; + static constexpr char Description[] = "Print compact binary package"; + PrintPackageCommand(); ~PrintPackageCommand(); @@ -37,7 +43,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"printpkg", "Print compact binary package"}; + cxxopts::Options m_Options{Name, Description}; std::filesystem::path m_Filename; bool m_ShowCbObjectTypeInfo = false; }; diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h index 56ef858f5..e415b41b7 100644 --- a/src/zen/cmds/projectstore_cmd.h +++ b/src/zen/cmds/projectstore_cmd.h @@ -16,6 +16,9 @@ class ProjectStoreCommand : public ZenCmdBase class DropProjectCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-drop"; + static constexpr char Description[] = "Drop project or project oplog"; + DropProjectCommand(); ~DropProjectCommand(); @@ -23,7 +26,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-drop", "Drop project or project oplog"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -33,13 +36,16 @@ private: class ProjectInfoCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-info"; + static constexpr char Description[] = "Info on project or project oplog"; + ProjectInfoCommand(); ~ProjectInfoCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-info", "Info on project or project oplog"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -48,6 +54,9 @@ private: class CreateProjectCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-create"; + static constexpr char Description[] = "Create a project"; + CreateProjectCommand(); ~CreateProjectCommand(); @@ -55,7 +64,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-create", "Create project, the project must not already exist."}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectId; std::string m_RootDir; @@ -68,6 +77,9 @@ private: class CreateOplogCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-create"; + static constexpr char Description[] = "Create a project oplog"; + CreateOplogCommand(); ~CreateOplogCommand(); @@ -75,7 +87,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-create", "Create oplog in an existing project, the oplog must not already exist."}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectId; std::string m_OplogId; @@ -86,6 +98,9 @@ private: class ExportOplogCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-export"; + static constexpr char Description[] = "Export project store oplog"; + ExportOplogCommand(); ~ExportOplogCommand(); @@ -93,8 +108,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-export", - "Export project store oplog to cloud (--cloud), file system (--file) or other Zen instance (--zen)"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -145,6 +159,9 @@ private: class ImportOplogCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-import"; + static constexpr char Description[] = "Import project store oplog"; + ImportOplogCommand(); ~ImportOplogCommand(); @@ -152,8 +169,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-import", - "Import project store oplog from cloud (--cloud), file system (--file) or other Zen instance (--zen)"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -198,14 +214,16 @@ private: class SnapshotOplogCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-snapshot"; + static constexpr char Description[] = "Snapshot project store oplog"; + SnapshotOplogCommand(); ~SnapshotOplogCommand(); - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-snapshot", "Snapshot external file references in project store oplog into zen"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -214,26 +232,32 @@ private: class ProjectStatsCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-stats"; + static constexpr char Description[] = "Stats on project store"; + ProjectStatsCommand(); ~ProjectStatsCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-stats", "Stats info on project store"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; class ProjectOpDetailsCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-op-details"; + static constexpr char Description[] = "Detail info on ops inside a project store oplog"; + ProjectOpDetailsCommand(); ~ProjectOpDetailsCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-op-details", "Detail info on ops inside a project store oplog"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_Details = false; bool m_OpDetails = false; @@ -247,13 +271,16 @@ private: class OplogMirrorCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-mirror"; + static constexpr char Description[] = "Mirror project store oplog to file system"; + OplogMirrorCommand(); ~OplogMirrorCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-mirror", "Mirror oplog to file system"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -268,13 +295,16 @@ private: class OplogValidateCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-validate"; + static constexpr char Description[] = "Validate oplog for missing references"; + OplogValidateCommand(); ~OplogValidateCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-validate", "Validate oplog for missing references"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; diff --git a/src/zen/cmds/rpcreplay_cmd.h b/src/zen/cmds/rpcreplay_cmd.h index a6363b614..332a3126c 100644 --- a/src/zen/cmds/rpcreplay_cmd.h +++ b/src/zen/cmds/rpcreplay_cmd.h @@ -9,6 +9,9 @@ namespace zen { class RpcStartRecordingCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "rpc-record-start"; + static constexpr char Description[] = "Starts recording of cache rpc requests on a host"; + RpcStartRecordingCommand(); ~RpcStartRecordingCommand(); @@ -16,7 +19,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"rpc-record-start", "Starts recording of cache rpc requests on a host"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_RecordingPath; }; @@ -24,6 +27,9 @@ private: class RpcStopRecordingCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "rpc-record-stop"; + static constexpr char Description[] = "Stops recording of cache rpc requests on a host"; + RpcStopRecordingCommand(); ~RpcStopRecordingCommand(); @@ -31,13 +37,16 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"rpc-record-stop", "Stops recording of cache rpc requests on a host"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; class RpcReplayCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "rpc-record-replay"; + static constexpr char Description[] = "Replays a previously recorded session of rpc requests"; + RpcReplayCommand(); ~RpcReplayCommand(); @@ -45,7 +54,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"rpc-record-replay", "Replays a previously recorded session of cache rpc requests to a target host"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_RecordingPath; bool m_OnHost = false; diff --git a/src/zen/cmds/run_cmd.h b/src/zen/cmds/run_cmd.h index 570a2e63a..300c08c5b 100644 --- a/src/zen/cmds/run_cmd.h +++ b/src/zen/cmds/run_cmd.h @@ -9,6 +9,9 @@ namespace zen { class RunCommand : public ZenCmdBase { public: + static constexpr char Name[] = "run"; + static constexpr char Description[] = "Run command with special options"; + RunCommand(); ~RunCommand(); @@ -17,7 +20,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"run", "Run executable"}; + cxxopts::Options m_Options{Name, Description}; int m_RunCount = 0; int m_RunTime = -1; std::string m_BaseDirectory; diff --git a/src/zen/cmds/serve_cmd.h b/src/zen/cmds/serve_cmd.h index ac74981f2..22f430948 100644 --- a/src/zen/cmds/serve_cmd.h +++ b/src/zen/cmds/serve_cmd.h @@ -11,6 +11,9 @@ namespace zen { class ServeCommand : public ZenCmdBase { public: + static constexpr char Name[] = "serve"; + static constexpr char Description[] = "Serve files from a directory"; + ServeCommand(); ~ServeCommand(); @@ -18,7 +21,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"serve", "Serve files from a tree"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; diff --git a/src/zen/cmds/status_cmd.h b/src/zen/cmds/status_cmd.h index dc103a196..df5df3066 100644 --- a/src/zen/cmds/status_cmd.h +++ b/src/zen/cmds/status_cmd.h @@ -11,6 +11,9 @@ namespace zen { class StatusCommand : public ZenCmdBase { public: + static constexpr char Name[] = "status"; + static constexpr char Description[] = "Show zen status"; + StatusCommand(); ~StatusCommand(); @@ -20,7 +23,7 @@ public: private: int GetLockFileEffectivePort() const; - cxxopts::Options m_Options{"status", "Show zen status"}; + cxxopts::Options m_Options{Name, Description}; uint16_t m_Port = 0; std::filesystem::path m_DataDir; }; diff --git a/src/zen/cmds/top_cmd.h b/src/zen/cmds/top_cmd.h index 74167ecfd..aeb196558 100644 --- a/src/zen/cmds/top_cmd.h +++ b/src/zen/cmds/top_cmd.h @@ -9,6 +9,9 @@ namespace zen { class TopCommand : public ZenCmdBase { public: + static constexpr char Name[] = "top"; + static constexpr char Description[] = "Monitor zen server activity"; + TopCommand(); ~TopCommand(); @@ -16,12 +19,15 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"top", "Show dev UI"}; + cxxopts::Options m_Options{Name, Description}; }; class PsCommand : public ZenCmdBase { public: + static constexpr char Name[] = "ps"; + static constexpr char Description[] = "Enumerate running zen server instances"; + PsCommand(); ~PsCommand(); @@ -29,7 +35,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"ps", "Enumerate running Zen server instances"}; + cxxopts::Options m_Options{Name, Description}; }; } // namespace zen diff --git a/src/zen/cmds/trace_cmd.h b/src/zen/cmds/trace_cmd.h index a6c9742b7..6eb0ba22b 100644 --- a/src/zen/cmds/trace_cmd.h +++ b/src/zen/cmds/trace_cmd.h @@ -6,11 +6,12 @@ namespace zen { -/** Scrub storage - */ class TraceCommand : public ZenCmdBase { public: + static constexpr char Name[] = "trace"; + static constexpr char Description[] = "Control zen realtime tracing"; + TraceCommand(); ~TraceCommand(); @@ -18,7 +19,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"trace", "Control zen realtime tracing"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_Stop = false; std::string m_TraceHost; diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h index 2e822d5fc..270db7f88 100644 --- a/src/zen/cmds/up_cmd.h +++ b/src/zen/cmds/up_cmd.h @@ -11,6 +11,9 @@ namespace zen { class UpCommand : public ZenCmdBase { public: + static constexpr char Name[] = "up"; + static constexpr char Description[] = "Bring zen server up"; + UpCommand(); ~UpCommand(); @@ -18,7 +21,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"up", "Bring up zen service"}; + cxxopts::Options m_Options{Name, Description}; uint16_t m_Port = 0; bool m_ShowConsole = false; bool m_ShowLog = false; @@ -28,6 +31,9 @@ private: class AttachCommand : public ZenCmdBase { public: + static constexpr char Name[] = "attach"; + static constexpr char Description[] = "Add a sponsor process to a running zen service"; + AttachCommand(); ~AttachCommand(); @@ -35,7 +41,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"}; + cxxopts::Options m_Options{Name, Description}; uint16_t m_Port = 0; int m_OwnerPid = 0; std::filesystem::path m_DataDir; @@ -44,6 +50,9 @@ private: class DownCommand : public ZenCmdBase { public: + static constexpr char Name[] = "down"; + static constexpr char Description[] = "Bring zen server down"; + DownCommand(); ~DownCommand(); @@ -51,7 +60,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"down", "Bring down zen service"}; + cxxopts::Options m_Options{Name, Description}; uint16_t m_Port = 0; bool m_ForceTerminate = false; std::filesystem::path m_ProgramBaseDir; diff --git a/src/zen/cmds/vfs_cmd.h b/src/zen/cmds/vfs_cmd.h index 5deaa02fa..9009c774b 100644 --- a/src/zen/cmds/vfs_cmd.h +++ b/src/zen/cmds/vfs_cmd.h @@ -9,6 +9,9 @@ namespace zen { class VfsCommand : public StorageCommand { public: + static constexpr char Name[] = "vfs"; + static constexpr char Description[] = "Manage virtual file system"; + VfsCommand(); ~VfsCommand(); @@ -16,7 +19,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"vfs", "Manage virtual file system"}; + cxxopts::Options m_Options{Name, Description}; std::string m_Verb; std::string m_HostName; diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 1b0578851..7e500f35e 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -379,56 +379,56 @@ main(int argc, char** argv) const char* CmdSummary; } Commands[] = { // clang-format off - {"attach", &AttachCmd, "Add a sponsor process to a running zen service"}, - {"bench", &BenchCmd, "Utility command for benchmarking"}, - {BuildsCommand::Name, &BuildsCmd, BuildsCommand::Description}, - {"cache-details", &CacheDetailsCmd, "Details on cache"}, - {"cache-info", &CacheInfoCmd, "Info on cache, namespace or bucket"}, + {AttachCommand::Name, &AttachCmd, AttachCommand::Description}, + {BenchCommand::Name, &BenchCmd, BenchCommand::Description}, + {BuildsCommand::Name, &BuildsCmd, BuildsCommand::Description}, + {CacheDetailsCommand::Name, &CacheDetailsCmd, CacheDetailsCommand::Description}, + {CacheInfoCommand::Name, &CacheInfoCmd, CacheInfoCommand::Description}, {CacheGetCommand::Name, &CacheGetCmd, CacheGetCommand::Description}, {CacheGenerateCommand::Name, &CacheGenerateCmd, CacheGenerateCommand::Description}, - {"cache-stats", &CacheStatsCmd, "Stats on cache"}, - {"copy", &CopyCmd, "Copy file(s)"}, - {"copy-state", &CopyStateCmd, "Copy zen server disk state"}, - {"dedup", &DedupCmd, "Dedup files"}, - {"down", &DownCmd, "Bring zen server down"}, - {"drop", &DropCmd, "Drop cache namespace or bucket"}, + {CacheStatsCommand::Name, &CacheStatsCmd, CacheStatsCommand::Description}, + {CopyCommand::Name, &CopyCmd, CopyCommand::Description}, + {CopyStateCommand::Name, &CopyStateCmd, CopyStateCommand::Description}, + {DedupCommand::Name, &DedupCmd, DedupCommand::Description}, + {DownCommand::Name, &DownCmd, DownCommand::Description}, + {DropCommand::Name, &DropCmd, DropCommand::Description}, #if ZEN_WITH_COMPUTE_SERVICES {ExecCommand::Name, &ExecCmd, ExecCommand::Description}, #endif - {"gc-status", &GcStatusCmd, "Garbage collect zen storage status check"}, - {"gc-stop", &GcStopCmd, "Request cancel of running garbage collection in zen storage"}, - {"gc", &GcCmd, "Garbage collect zen storage"}, - {"info", &InfoCmd, "Show high level Zen server information"}, - {"jobs", &JobCmd, "Show/cancel zen background jobs"}, - {"logs", &LoggingCmd, "Show/control zen logging"}, - {"oplog-create", &CreateOplogCmd, "Create a project oplog"}, - {"oplog-export", &ExportOplogCmd, "Export project store oplog"}, - {"oplog-import", &ImportOplogCmd, "Import project store oplog"}, - {"oplog-mirror", &OplogMirrorCmd, "Mirror project store oplog to file system"}, - {"oplog-snapshot", &SnapshotOplogCmd, "Snapshot project store oplog"}, + {GcStatusCommand::Name, &GcStatusCmd, GcStatusCommand::Description}, + {GcStopCommand::Name, &GcStopCmd, GcStopCommand::Description}, + {GcCommand::Name, &GcCmd, GcCommand::Description}, + {InfoCommand::Name, &InfoCmd, InfoCommand::Description}, + {JobCommand::Name, &JobCmd, JobCommand::Description}, + {LoggingCommand::Name, &LoggingCmd, LoggingCommand::Description}, + {CreateOplogCommand::Name, &CreateOplogCmd, CreateOplogCommand::Description}, + {ExportOplogCommand::Name, &ExportOplogCmd, ExportOplogCommand::Description}, + {ImportOplogCommand::Name, &ImportOplogCmd, ImportOplogCommand::Description}, + {OplogMirrorCommand::Name, &OplogMirrorCmd, OplogMirrorCommand::Description}, + {SnapshotOplogCommand::Name, &SnapshotOplogCmd, SnapshotOplogCommand::Description}, {OplogDownloadCommand::Name, &OplogDownload, OplogDownloadCommand::Description}, - {"oplog-validate", &OplogValidateCmd, "Validate oplog for missing references"}, - {"print", &PrintCmd, "Print compact binary object"}, - {"printpackage", &PrintPkgCmd, "Print compact binary package"}, - {"project-create", &CreateProjectCmd, "Create a project"}, - {"project-op-details", &ProjectOpDetailsCmd, "Detail info on ops inside a project store oplog"}, - {"project-drop", &ProjectDropCmd, "Drop project or project oplog"}, - {"project-info", &ProjectInfoCmd, "Info on project or project oplog"}, - {"project-stats", &ProjectStatsCmd, "Stats on project store"}, - {"ps", &PsCmd, "Enumerate running zen server instances"}, - {"rpc-record-replay", &RpcReplayCmd, "Replays a previously recorded session of rpc requests"}, - {"rpc-record-start", &RpcStartRecordingCmd, "Starts recording of cache rpc requests on a host"}, - {"rpc-record-stop", &RpcStopRecordingCmd, "Stops recording of cache rpc requests on a host"}, - {"run", &RunCmd, "Run command with special options"}, - {"scrub", &ScrubCmd, "Scrub zen storage (verify data integrity)"}, - {"serve", &ServeCmd, "Serve files from a directory"}, - {"status", &StatusCmd, "Show zen status"}, - {"top", &TopCmd, "Monitor zen server activity"}, - {"trace", &TraceCmd, "Control zen realtime tracing"}, - {"up", &UpCmd, "Bring zen server up"}, + {OplogValidateCommand::Name, &OplogValidateCmd, OplogValidateCommand::Description}, + {PrintCommand::Name, &PrintCmd, PrintCommand::Description}, + {PrintPackageCommand::Name, &PrintPkgCmd, PrintPackageCommand::Description}, + {CreateProjectCommand::Name, &CreateProjectCmd, CreateProjectCommand::Description}, + {ProjectOpDetailsCommand::Name, &ProjectOpDetailsCmd, ProjectOpDetailsCommand::Description}, + {DropProjectCommand::Name, &ProjectDropCmd, DropProjectCommand::Description}, + {ProjectInfoCommand::Name, &ProjectInfoCmd, ProjectInfoCommand::Description}, + {ProjectStatsCommand::Name, &ProjectStatsCmd, ProjectStatsCommand::Description}, + {PsCommand::Name, &PsCmd, PsCommand::Description}, + {RpcReplayCommand::Name, &RpcReplayCmd, RpcReplayCommand::Description}, + {RpcStartRecordingCommand::Name, &RpcStartRecordingCmd, RpcStartRecordingCommand::Description}, + {RpcStopRecordingCommand::Name, &RpcStopRecordingCmd, RpcStopRecordingCommand::Description}, + {RunCommand::Name, &RunCmd, RunCommand::Description}, + {ScrubCommand::Name, &ScrubCmd, ScrubCommand::Description}, + {ServeCommand::Name, &ServeCmd, ServeCommand::Description}, + {StatusCommand::Name, &StatusCmd, StatusCommand::Description}, + {TopCommand::Name, &TopCmd, TopCommand::Description}, + {TraceCommand::Name, &TraceCmd, TraceCommand::Description}, + {UpCommand::Name, &UpCmd, UpCommand::Description}, {VersionCommand::Name, &VersionCmd, VersionCommand::Description}, - {"vfs", &VfsCmd, "Manage virtual file system"}, - {"flush", &FlushCmd, "Flush storage"}, + {VfsCommand::Name, &VfsCmd, VfsCommand::Description}, + {FlushCommand::Name, &FlushCmd, FlushCommand::Description}, {WipeCommand::Name, &WipeCmd, WipeCommand::Description}, {WorkspaceCommand::Name, &WorkspaceCmd, WorkspaceCommand::Description}, {WorkspaceShareCommand::Name, &WorkspaceShareCmd, WorkspaceShareCommand::Description}, diff --git a/src/zencompute-test/xmake.lua b/src/zencompute-test/xmake.lua index 860bf655b..1207bdefd 100644 --- a/src/zencompute-test/xmake.lua +++ b/src/zencompute-test/xmake.lua @@ -6,4 +6,3 @@ target("zencompute-test") add_headerfiles("**.h") add_files("*.cpp") add_deps("zencompute", "zencore") - diff --git a/src/zencore/base64.cpp b/src/zencore/base64.cpp index 1f56ee6c3..fdf5f2d66 100644 --- a/src/zencore/base64.cpp +++ b/src/zencore/base64.cpp @@ -1,6 +1,10 @@ // Copyright Epic Games, Inc. All Rights Reserved. #include <zencore/base64.h> +#include <zencore/string.h> +#include <zencore/testing.h> + +#include <string> namespace zen { @@ -11,7 +15,6 @@ static const uint8_t EncodingAlphabet[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; /** The table used to convert an ascii character into a 6 bit value */ -#if 0 static const uint8_t DecodingAlphabet[256] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x00-0x0f 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x10-0x1f @@ -30,7 +33,6 @@ static const uint8_t DecodingAlphabet[256] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xe0-0xef 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // 0xf0-0xff }; -#endif // 0 template<typename CharType> uint32_t @@ -104,4 +106,190 @@ Base64::Encode(const uint8_t* Source, uint32_t Length, CharType* Dest) template uint32_t Base64::Encode<char>(const uint8_t* Source, uint32_t Length, char* Dest); template uint32_t Base64::Encode<wchar_t>(const uint8_t* Source, uint32_t Length, wchar_t* Dest); +template<typename CharType> +bool +Base64::Decode(const CharType* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength) +{ + // Length must be a multiple of 4 + if (Length % 4 != 0) + { + OutLength = 0; + return false; + } + + uint8_t* DecodedBytes = Dest; + + // Process 4 encoded characters at a time, producing 3 decoded bytes + while (Length > 0) + { + // Count padding characters at the end + uint32_t PadCount = 0; + if (Source[3] == '=') + { + PadCount++; + if (Source[2] == '=') + { + PadCount++; + } + } + + // Look up each character in the decoding table + uint8_t A = DecodingAlphabet[static_cast<uint8_t>(Source[0])]; + uint8_t B = DecodingAlphabet[static_cast<uint8_t>(Source[1])]; + uint8_t C = (PadCount >= 2) ? 0 : DecodingAlphabet[static_cast<uint8_t>(Source[2])]; + uint8_t D = (PadCount >= 1) ? 0 : DecodingAlphabet[static_cast<uint8_t>(Source[3])]; + + // Check for invalid characters (0xFF means not in the base64 alphabet) + if (A == 0xFF || B == 0xFF || C == 0xFF || D == 0xFF) + { + OutLength = 0; + return false; + } + + // Reconstruct the 24-bit value from 4 6-bit chunks + uint32_t ByteTriplet = (A << 18) | (B << 12) | (C << 6) | D; + + // Extract the 3 bytes + *DecodedBytes++ = static_cast<uint8_t>(ByteTriplet >> 16); + if (PadCount < 2) + { + *DecodedBytes++ = static_cast<uint8_t>((ByteTriplet >> 8) & 0xFF); + } + if (PadCount < 1) + { + *DecodedBytes++ = static_cast<uint8_t>(ByteTriplet & 0xFF); + } + + Source += 4; + Length -= 4; + } + + OutLength = uint32_t(DecodedBytes - Dest); + return true; +} + +template bool Base64::Decode<char>(const char* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength); +template bool Base64::Decode<wchar_t>(const wchar_t* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength); + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +#if ZEN_WITH_TESTS + +using namespace std::string_literals; + +TEST_CASE("Base64") +{ + auto EncodeString = [](std::string_view Input) -> std::string { + std::string Result; + Result.resize(Base64::GetEncodedDataSize(uint32_t(Input.size()))); + Base64::Encode(reinterpret_cast<const uint8_t*>(Input.data()), uint32_t(Input.size()), Result.data()); + return Result; + }; + + auto DecodeString = [](std::string_view Input) -> std::string { + std::string Result; + Result.resize(Base64::GetMaxDecodedDataSize(uint32_t(Input.size()))); + uint32_t DecodedLength = 0; + bool Success = Base64::Decode(Input.data(), uint32_t(Input.size()), reinterpret_cast<uint8_t*>(Result.data()), DecodedLength); + CHECK(Success); + Result.resize(DecodedLength); + return Result; + }; + + SUBCASE("Encode") + { + CHECK(EncodeString("") == ""s); + CHECK(EncodeString("f") == "Zg=="s); + CHECK(EncodeString("fo") == "Zm8="s); + CHECK(EncodeString("foo") == "Zm9v"s); + CHECK(EncodeString("foob") == "Zm9vYg=="s); + CHECK(EncodeString("fooba") == "Zm9vYmE="s); + CHECK(EncodeString("foobar") == "Zm9vYmFy"s); + } + + SUBCASE("Decode") + { + CHECK(DecodeString("") == ""s); + CHECK(DecodeString("Zg==") == "f"s); + CHECK(DecodeString("Zm8=") == "fo"s); + CHECK(DecodeString("Zm9v") == "foo"s); + CHECK(DecodeString("Zm9vYg==") == "foob"s); + CHECK(DecodeString("Zm9vYmE=") == "fooba"s); + CHECK(DecodeString("Zm9vYmFy") == "foobar"s); + } + + SUBCASE("RoundTrip") + { + auto RoundTrip = [&](const std::string& Input) { + std::string Encoded = EncodeString(Input); + std::string Decoded = DecodeString(Encoded); + CHECK(Decoded == Input); + }; + + RoundTrip("Hello, World!"); + RoundTrip("Base64 encoding test with various lengths"); + RoundTrip("A"); + RoundTrip("AB"); + RoundTrip("ABC"); + RoundTrip("ABCD"); + RoundTrip("\x00\x01\x02\xff\xfe\xfd"s); + } + + SUBCASE("BinaryRoundTrip") + { + // Test with all byte values 0-255 + uint8_t AllBytes[256]; + for (int i = 0; i < 256; ++i) + { + AllBytes[i] = static_cast<uint8_t>(i); + } + + char Encoded[Base64::GetEncodedDataSize(256) + 1]; + Base64::Encode(AllBytes, 256, Encoded); + + uint8_t Decoded[256]; + uint32_t DecodedLength = 0; + bool Success = Base64::Decode(Encoded, uint32_t(strlen(Encoded)), Decoded, DecodedLength); + CHECK(Success); + CHECK(DecodedLength == 256); + CHECK(memcmp(AllBytes, Decoded, 256) == 0); + } + + SUBCASE("DecodeInvalidInput") + { + uint8_t Dest[64]; + uint32_t OutLength = 0; + + // Length not a multiple of 4 + CHECK_FALSE(Base64::Decode("abc", 3u, Dest, OutLength)); + + // Invalid character + CHECK_FALSE(Base64::Decode("ab!d", 4u, Dest, OutLength)); + } + + SUBCASE("EncodedDataSize") + { + CHECK(Base64::GetEncodedDataSize(0) == 0); + CHECK(Base64::GetEncodedDataSize(1) == 4); + CHECK(Base64::GetEncodedDataSize(2) == 4); + CHECK(Base64::GetEncodedDataSize(3) == 4); + CHECK(Base64::GetEncodedDataSize(4) == 8); + CHECK(Base64::GetEncodedDataSize(5) == 8); + CHECK(Base64::GetEncodedDataSize(6) == 8); + } + + SUBCASE("MaxDecodedDataSize") + { + CHECK(Base64::GetMaxDecodedDataSize(0) == 0); + CHECK(Base64::GetMaxDecodedDataSize(4) == 3); + CHECK(Base64::GetMaxDecodedDataSize(8) == 6); + CHECK(Base64::GetMaxDecodedDataSize(12) == 9); + } +} + +#endif + } // namespace zen diff --git a/src/zencore/compactbinaryyaml.cpp b/src/zencore/compactbinaryyaml.cpp index 5122e952a..b308af418 100644 --- a/src/zencore/compactbinaryyaml.cpp +++ b/src/zencore/compactbinaryyaml.cpp @@ -14,11 +14,6 @@ #include <string_view> #include <vector> -ZEN_THIRD_PARTY_INCLUDES_START -#include <ryml.hpp> -#include <ryml_std.hpp> -ZEN_THIRD_PARTY_INCLUDES_END - namespace zen { ////////////////////////////////////////////////////////////////////////// @@ -26,193 +21,349 @@ namespace zen { class CbYamlWriter { public: - explicit CbYamlWriter(StringBuilderBase& InBuilder) : m_StrBuilder(InBuilder) { m_NodeStack.push_back(m_Tree.rootref()); } + explicit CbYamlWriter(StringBuilderBase& InBuilder) : m_Builder(InBuilder) {} void WriteField(CbFieldView Field) { - ryml::NodeRef Node; + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); - if (m_IsFirst) + switch (Type) { - Node = Top(); + case CbFieldType::Object: + case CbFieldType::UniformObject: + WriteMapEntries(Field, 0); + break; + case CbFieldType::Array: + case CbFieldType::UniformArray: + WriteSeqEntries(Field, 0); + break; + default: + WriteScalarValue(Field); + m_Builder << '\n'; + break; + } + } + + void WriteMapEntry(CbFieldView Field, int32_t Indent) + { + WriteIndent(Indent); + WriteMapEntryContent(Field, Indent); + } + + void WriteSeqEntry(CbFieldView Field, int32_t Indent) + { + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); - m_IsFirst = false; + if (Type == CbFieldType::Object || Type == CbFieldType::UniformObject) + { + bool First = true; + for (CbFieldView MapChild : Field) + { + if (First) + { + WriteIndent(Indent); + m_Builder << "- "; + First = false; + } + else + { + WriteIndent(Indent + 1); + } + WriteMapEntryContent(MapChild, Indent + 1); + } + } + else if (Type == CbFieldType::Array || Type == CbFieldType::UniformArray) + { + WriteIndent(Indent); + m_Builder << "-\n"; + WriteSeqEntries(Field, Indent + 1); } else { - Node = Top().append_child(); + WriteIndent(Indent); + m_Builder << "- "; + WriteScalarValue(Field); + m_Builder << '\n'; } + } - if (std::u8string_view Name = Field.GetU8Name(); !Name.empty()) +private: + void WriteMapEntries(CbFieldView MapField, int32_t Indent) + { + for (CbFieldView Child : MapField) { - Node.set_key_serialized(ryml::csubstr((const char*)Name.data(), Name.size())); + WriteIndent(Indent); + WriteMapEntryContent(Child, Indent); } + } + + void WriteMapEntryContent(CbFieldView Field, int32_t Indent) + { + std::u8string_view Name = Field.GetU8Name(); + m_Builder << std::string_view(reinterpret_cast<const char*>(Name.data()), Name.size()); - switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); + + if (IsContainer(Type)) { - case CbFieldType::Null: - Node.set_val("null"); - break; - case CbFieldType::Object: - case CbFieldType::UniformObject: - Node |= ryml::MAP; - m_NodeStack.push_back(Node); - for (CbFieldView It : Field) + m_Builder << ":\n"; + WriteFieldValue(Field, Indent + 1); + } + else + { + m_Builder << ": "; + WriteScalarValue(Field); + m_Builder << '\n'; + } + } + + void WriteSeqEntries(CbFieldView SeqField, int32_t Indent) + { + for (CbFieldView Child : SeqField) + { + CbValue Accessor = Child.GetValue(); + CbFieldType Type = Accessor.GetType(); + + if (Type == CbFieldType::Object || Type == CbFieldType::UniformObject) + { + bool First = true; + for (CbFieldView MapChild : Child) { - WriteField(It); + if (First) + { + WriteIndent(Indent); + m_Builder << "- "; + First = false; + } + else + { + WriteIndent(Indent + 1); + } + WriteMapEntryContent(MapChild, Indent + 1); } - m_NodeStack.pop_back(); + } + else if (Type == CbFieldType::Array || Type == CbFieldType::UniformArray) + { + WriteIndent(Indent); + m_Builder << "-\n"; + WriteSeqEntries(Child, Indent + 1); + } + else + { + WriteIndent(Indent); + m_Builder << "- "; + WriteScalarValue(Child); + m_Builder << '\n'; + } + } + } + + void WriteFieldValue(CbFieldView Field, int32_t Indent) + { + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); + + switch (Type) + { + case CbFieldType::Object: + case CbFieldType::UniformObject: + WriteMapEntries(Field, Indent); break; case CbFieldType::Array: case CbFieldType::UniformArray: - Node |= ryml::SEQ; - m_NodeStack.push_back(Node); - for (CbFieldView It : Field) - { - WriteField(It); - } - m_NodeStack.pop_back(); + WriteSeqEntries(Field, Indent); break; - case CbFieldType::Binary: - { - ExtendableStringBuilder<256> Builder; - const MemoryView Value = Accessor.AsBinary(); - ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); - - Node.set_key_serialized(Builder.c_str()); - } + case CbFieldType::CustomById: + WriteCustomById(Field.GetValue().AsCustomById(), Indent); break; - case CbFieldType::String: - { - const std::u8string_view U8String = Accessor.AsU8String(); - Node.set_val(ryml::csubstr((const char*)U8String.data(), U8String.size())); - } + case CbFieldType::CustomByName: + WriteCustomByName(Field.GetValue().AsCustomByName(), Indent); + break; + default: + WriteScalarValue(Field); + m_Builder << '\n'; + break; + } + } + + void WriteScalarValue(CbFieldView Field) + { + CbValue Accessor = Field.GetValue(); + switch (Accessor.GetType()) + { + case CbFieldType::Null: + m_Builder << "null"; + break; + case CbFieldType::BoolFalse: + m_Builder << "false"; + break; + case CbFieldType::BoolTrue: + m_Builder << "true"; break; case CbFieldType::IntegerPositive: - Node << Accessor.AsIntegerPositive(); + m_Builder << Accessor.AsIntegerPositive(); break; case CbFieldType::IntegerNegative: - Node << Accessor.AsIntegerNegative(); + m_Builder << Accessor.AsIntegerNegative(); break; case CbFieldType::Float32: if (const float Value = Accessor.AsFloat32(); std::isfinite(Value)) - { - Node << Value; - } + m_Builder.Append(fmt::format("{}", Value)); else - { - Node << "null"; - } + m_Builder << "null"; break; case CbFieldType::Float64: if (const double Value = Accessor.AsFloat64(); std::isfinite(Value)) - { - Node << Value; - } + m_Builder.Append(fmt::format("{}", Value)); else + m_Builder << "null"; + break; + case CbFieldType::String: { - Node << "null"; + const std::u8string_view U8String = Accessor.AsU8String(); + WriteString(std::string_view(reinterpret_cast<const char*>(U8String.data()), U8String.size())); } break; - case CbFieldType::BoolFalse: - Node << "false"; - break; - case CbFieldType::BoolTrue: - Node << "true"; + case CbFieldType::Hash: + WriteString(Accessor.AsHash().ToHexString()); break; case CbFieldType::ObjectAttachment: case CbFieldType::BinaryAttachment: - Node << Accessor.AsAttachment().ToHexString(); - break; - case CbFieldType::Hash: - Node << Accessor.AsHash().ToHexString(); + WriteString(Accessor.AsAttachment().ToHexString()); break; case CbFieldType::Uuid: - Node << fmt::format("{}", Accessor.AsUuid()); + WriteString(fmt::format("{}", Accessor.AsUuid())); break; case CbFieldType::DateTime: - Node << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); + WriteString(DateTime(Accessor.AsDateTimeTicks()).ToIso8601()); break; case CbFieldType::TimeSpan: if (const TimeSpan Span(Accessor.AsTimeSpanTicks()); Span.GetDays() == 0) - { - Node << Span.ToString("%h:%m:%s.%n"); - } + WriteString(Span.ToString("%h:%m:%s.%n")); else - { - Node << Span.ToString("%d.%h:%m:%s.%n"); - } + WriteString(Span.ToString("%d.%h:%m:%s.%n")); break; case CbFieldType::ObjectId: - Node << fmt::format("{}", Accessor.AsObjectId()); + WriteString(fmt::format("{}", Accessor.AsObjectId())); break; - case CbFieldType::CustomById: - { - CbCustomById Custom = Accessor.AsCustomById(); + case CbFieldType::Binary: + WriteBase64(Accessor.AsBinary()); + break; + default: + ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType())); + break; + } + } - Node |= ryml::MAP; + void WriteCustomById(CbCustomById Custom, int32_t Indent) + { + WriteIndent(Indent); + m_Builder << "Id: "; + m_Builder.Append(fmt::format("{}", Custom.Id)); + m_Builder << '\n'; + + WriteIndent(Indent); + m_Builder << "Data: "; + WriteBase64(Custom.Data); + m_Builder << '\n'; + } - ryml::NodeRef IdNode = Node.append_child(); - IdNode.set_key("Id"); - IdNode.set_val_serialized(fmt::format("{}", Custom.Id)); + void WriteCustomByName(CbCustomByName Custom, int32_t Indent) + { + WriteIndent(Indent); + m_Builder << "Name: "; + WriteString(std::string_view(reinterpret_cast<const char*>(Custom.Name.data()), Custom.Name.size())); + m_Builder << '\n'; + + WriteIndent(Indent); + m_Builder << "Data: "; + WriteBase64(Custom.Data); + m_Builder << '\n'; + } - ryml::NodeRef DataNode = Node.append_child(); - DataNode.set_key("Data"); + void WriteBase64(MemoryView Value) + { + ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); + ExtendableStringBuilder<256> Buf; + const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); + const size_t EncodedIndex = Buf.AddUninitialized(size_t(EncodedSize)); + Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Buf.Data() + EncodedIndex); + WriteString(Buf.ToView()); + } - ExtendableStringBuilder<256> Builder; - const MemoryView& Value = Custom.Data; - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + void WriteString(std::string_view Str) + { + if (NeedsQuoting(Str)) + { + m_Builder << '\''; + for (char C : Str) + { + if (C == '\'') + m_Builder << "''"; + else + m_Builder << C; + } + m_Builder << '\''; + } + else + { + m_Builder << Str; + } + } - DataNode.set_val_serialized(Builder.c_str()); - } - break; - case CbFieldType::CustomByName: - { - CbCustomByName Custom = Accessor.AsCustomByName(); + void WriteIndent(int32_t Indent) + { + for (int32_t I = 0; I < Indent; ++I) + m_Builder << " "; + } - Node |= ryml::MAP; + static bool NeedsQuoting(std::string_view Str) + { + if (Str.empty()) + return false; - ryml::NodeRef NameNode = Node.append_child(); - NameNode.set_key("Name"); - std::string_view Name = std::string_view((const char*)Custom.Name.data(), Custom.Name.size()); - NameNode.set_val_serialized(std::string(Name)); + char First = Str[0]; + if (First == ' ' || First == '\n' || First == '\t' || First == '\r' || First == '*' || First == '&' || First == '%' || + First == '@' || First == '`') + return true; - ryml::NodeRef DataNode = Node.append_child(); - DataNode.set_key("Data"); + if (Str.size() >= 2 && Str[0] == '<' && Str[1] == '<') + return true; - ExtendableStringBuilder<256> Builder; - const MemoryView& Value = Custom.Data; - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + char Last = Str.back(); + if (Last == ' ' || Last == '\n' || Last == '\t' || Last == '\r') + return true; - DataNode.set_val_serialized(Builder.c_str()); - } - break; - default: - ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType())); - break; + for (char C : Str) + { + if (C == '#' || C == ':' || C == '-' || C == '?' || C == ',' || C == '\n' || C == '{' || C == '}' || C == '[' || C == ']' || + C == '\'' || C == '"') + return true; } - if (m_NodeStack.size() == 1) + return false; + } + + static bool IsContainer(CbFieldType Type) + { + switch (Type) { - std::string Yaml = ryml::emitrs_yaml<std::string>(m_Tree); - m_StrBuilder << Yaml; + case CbFieldType::Object: + case CbFieldType::UniformObject: + case CbFieldType::Array: + case CbFieldType::UniformArray: + case CbFieldType::CustomById: + case CbFieldType::CustomByName: + return true; + default: + return false; } } -private: - StringBuilderBase& m_StrBuilder; - bool m_IsFirst = true; - - ryml::Tree m_Tree; - std::vector<ryml::NodeRef> m_NodeStack; - ryml::NodeRef& Top() { return m_NodeStack.back(); } + StringBuilderBase& m_Builder; }; void @@ -229,6 +380,32 @@ CompactBinaryToYaml(const CbArrayView& Array, StringBuilderBase& Builder) Writer.WriteField(Array.AsFieldView()); } +void +CompactBinaryToYaml(MemoryView Data, StringBuilderBase& InBuilder) +{ + std::vector<CbFieldView> Fields = ReadCompactBinaryStream(Data); + if (Fields.empty()) + return; + + CbYamlWriter Writer(InBuilder); + if (Fields.size() == 1) + { + Writer.WriteField(Fields[0]); + return; + } + + if (Fields[0].HasName()) + { + for (const CbFieldView& Field : Fields) + Writer.WriteMapEntry(Field, 0); + } + else + { + for (const CbFieldView& Field : Fields) + Writer.WriteSeqEntry(Field, 0); + } +} + #if ZEN_WITH_TESTS void cbyaml_forcelink() diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 1a4ee4b9b..553897407 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -1326,11 +1326,6 @@ ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uin { BytesRead = size_t(dwNumberOfBytesRead); } - else if ((BytesRead != NumberOfBytesToRead)) - { - Ec = MakeErrorCode(ERROR_HANDLE_EOF); - return; - } else { Ec = MakeErrorCodeFromLastError(); @@ -1344,20 +1339,15 @@ ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uin { BytesRead = size_t(ReadResult); } - else if ((BytesRead != NumberOfBytesToRead)) - { - Ec = MakeErrorCode(EIO); - return; - } else { Ec = MakeErrorCodeFromLastError(); return; } #endif - Size -= NumberOfBytesToRead; - FileOffset += NumberOfBytesToRead; - Data = reinterpret_cast<uint8_t*>(Data) + NumberOfBytesToRead; + Size -= BytesRead; + FileOffset += BytesRead; + Data = reinterpret_cast<uint8_t*>(Data) + BytesRead; } } diff --git a/src/zencore/include/zencore/base64.h b/src/zencore/include/zencore/base64.h index 4d78b085f..08d9f3043 100644 --- a/src/zencore/include/zencore/base64.h +++ b/src/zencore/include/zencore/base64.h @@ -11,7 +11,11 @@ struct Base64 template<typename CharType> static uint32_t Encode(const uint8_t* Source, uint32_t Length, CharType* Dest); + template<typename CharType> + static bool Decode(const CharType* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength); + static inline constexpr int32_t GetEncodedDataSize(uint32_t Size) { return ((Size + 2) / 3) * 4; } + static inline constexpr int32_t GetMaxDecodedDataSize(uint32_t Length) { return (Length / 4) * 3; } }; } // namespace zen diff --git a/src/zencore/include/zencore/compactbinaryvalue.h b/src/zencore/include/zencore/compactbinaryvalue.h index aa2d2821d..4ce8009b8 100644 --- a/src/zencore/include/zencore/compactbinaryvalue.h +++ b/src/zencore/include/zencore/compactbinaryvalue.h @@ -128,17 +128,21 @@ CbValue::AsString(CbFieldError* OutError, std::string_view Default) const uint32_t ValueSizeByteCount; const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount); - if (OutError) + if (ValueSize >= (uint64_t(1) << 31)) { - if (ValueSize >= (uint64_t(1) << 31)) + if (OutError) { *OutError = CbFieldError::RangeError; - return Default; } + return Default; + } + + if (OutError) + { *OutError = CbFieldError::None; } - return std::string_view(Chars + ValueSizeByteCount, int32_t(ValueSize)); + return std::string_view(Chars + ValueSizeByteCount, size_t(ValueSize)); } inline std::u8string_view @@ -148,17 +152,21 @@ CbValue::AsU8String(CbFieldError* OutError, std::u8string_view Default) const uint32_t ValueSizeByteCount; const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount); - if (OutError) + if (ValueSize >= (uint64_t(1) << 31)) { - if (ValueSize >= (uint64_t(1) << 31)) + if (OutError) { *OutError = CbFieldError::RangeError; - return Default; } + return Default; + } + + if (OutError) + { *OutError = CbFieldError::None; } - return std::u8string_view(Chars + ValueSizeByteCount, int32_t(ValueSize)); + return std::u8string_view(Chars + ValueSizeByteCount, size_t(ValueSize)); } inline uint64_t diff --git a/src/zencore/memtrack/callstacktrace.cpp b/src/zencore/memtrack/callstacktrace.cpp index a5b7fede6..4a7068568 100644 --- a/src/zencore/memtrack/callstacktrace.cpp +++ b/src/zencore/memtrack/callstacktrace.cpp @@ -169,13 +169,13 @@ private: std::atomic_uint64_t Key; std::atomic_uint32_t Value; - inline uint64 GetKey() const { return Key.load(std::memory_order_relaxed); } + inline uint64 GetKey() const { return Key.load(std::memory_order_acquire); } inline uint32_t GetValue() const { return Value.load(std::memory_order_relaxed); } - inline bool IsEmpty() const { return Key.load(std::memory_order_relaxed) == 0; } + inline bool IsEmpty() const { return Key.load(std::memory_order_acquire) == 0; } inline void SetKeyValue(uint64_t InKey, uint32_t InValue) { - Value.store(InValue, std::memory_order_release); - Key.store(InKey, std::memory_order_relaxed); + Value.store(InValue, std::memory_order_relaxed); + Key.store(InKey, std::memory_order_release); } static inline uint32_t KeyHash(uint64_t Key) { return static_cast<uint32_t>(Key); } static inline void ClearEntries(FEncounteredCallstackSetEntry* Entries, int32_t EntryCount) diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp index 0ee863b74..a9aed6309 100644 --- a/src/zencore/string.cpp +++ b/src/zencore/string.cpp @@ -24,6 +24,10 @@ utf16to8_impl(u16bit_iterator StartIt, u16bit_iterator EndIt, ::zen::StringBuild // Take care of surrogate pairs first if (utf8::internal::is_lead_surrogate(cp)) { + if (StartIt == EndIt) + { + break; + } uint32_t trail_surrogate = utf8::internal::mask16(*StartIt++); cp = (cp << 10) + trail_surrogate + utf8::internal::SURROGATE_OFFSET; } diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index 4b6e7c44b..322001ef9 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -32,8 +32,6 @@ target('zencore') add_deps("timesinceprocessstart") add_deps("doctest") add_deps("fmt") - add_deps("ryml") - add_packages("json11") if is_plat("linux", "macosx") then diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index d649e6333..a01789618 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -89,10 +89,10 @@ IsIPv6AvailableSysctl(void) char buf[16]; if (fgets(buf, sizeof(buf), f)) { - fclose(f); // 0 means IPv6 enabled, 1 means disabled val = atoi(buf); } + fclose(f); } return val == 0; @@ -1544,7 +1544,7 @@ struct HttpAcceptor { ZEN_WARN("Unable to initialize asio service, (bind returned '{}')", BindErrorCode.message()); - return 0; + return {}; } if (EffectivePort != BasePort) diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index c640ba90b..6995ffca9 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -25,6 +25,7 @@ # include <zencore/workthreadpool.h> # include "iothreadpool.h" +# include <atomic> # include <http.h> namespace zen { @@ -129,8 +130,8 @@ private: std::unique_ptr<WinIoThreadPool> m_IoThreadPool; - RwLock m_AsyncWorkPoolInitLock; - WorkerThreadPool* m_AsyncWorkPool = nullptr; + RwLock m_AsyncWorkPoolInitLock; + std::atomic<WorkerThreadPool*> m_AsyncWorkPool = nullptr; std::vector<std::wstring> m_BaseUris; // eg: http://*:nnnn/ HTTP_SERVER_SESSION_ID m_HttpSessionId = 0; @@ -1032,8 +1033,10 @@ HttpSysServer::~HttpSysServer() ZEN_ERROR("~HttpSysServer() called without calling Close() first"); } - delete m_AsyncWorkPool; + auto WorkPool = m_AsyncWorkPool.load(std::memory_order_relaxed); m_AsyncWorkPool = nullptr; + + delete WorkPool; } void @@ -1323,17 +1326,17 @@ HttpSysServer::WorkPool() { ZEN_MEMSCOPE(GetHttpsysTag()); - if (!m_AsyncWorkPool) + if (!m_AsyncWorkPool.load(std::memory_order_acquire)) { RwLock::ExclusiveLockScope _(m_AsyncWorkPoolInitLock); - if (!m_AsyncWorkPool) + if (!m_AsyncWorkPool.load(std::memory_order_relaxed)) { - m_AsyncWorkPool = new WorkerThreadPool(m_InitialConfig.AsyncWorkThreadCount, "http_async"); + m_AsyncWorkPool.store(new WorkerThreadPool(m_InitialConfig.AsyncWorkThreadCount, "http_async"), std::memory_order_release); } } - return *m_AsyncWorkPool; + return *m_AsyncWorkPool.load(std::memory_order_relaxed); } void diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp index 2b157581f..1cf451e91 100644 --- a/src/zenserver/frontend/frontend.cpp +++ b/src/zenserver/frontend/frontend.cpp @@ -38,7 +38,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpSt #if ZEN_EMBED_HTML_ZIP // Load an embedded Zip archive IoBuffer HtmlZipDataBuffer(IoBuffer::Wrap, gHtmlZipData, sizeof(gHtmlZipData) - 1); - m_ZipFs = ZipFs(std::move(HtmlZipDataBuffer)); + m_ZipFs = std::make_unique<ZipFs>(std::move(HtmlZipDataBuffer)); #endif if (m_Directory.empty() && !m_ZipFs) @@ -157,9 +157,12 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) } } - if (IoBuffer FileBuffer = m_ZipFs.GetFile(Uri)) + if (m_ZipFs) { - return Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer); + if (IoBuffer FileBuffer = m_ZipFs->GetFile(Uri)) + { + return Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer); + } } Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); diff --git a/src/zenserver/frontend/frontend.h b/src/zenserver/frontend/frontend.h index 84ffaac42..6d8585b72 100644 --- a/src/zenserver/frontend/frontend.h +++ b/src/zenserver/frontend/frontend.h @@ -7,6 +7,7 @@ #include "zipfs.h" #include <filesystem> +#include <memory> namespace zen { @@ -20,9 +21,9 @@ public: virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: - ZipFs m_ZipFs; - std::filesystem::path m_Directory; - HttpStatusService& m_StatusService; + std::unique_ptr<ZipFs> m_ZipFs; + std::filesystem::path m_Directory; + HttpStatusService& m_StatusService; }; } // namespace zen diff --git a/src/zenserver/frontend/zipfs.cpp b/src/zenserver/frontend/zipfs.cpp index f9c2bc8ff..42df0520f 100644 --- a/src/zenserver/frontend/zipfs.cpp +++ b/src/zenserver/frontend/zipfs.cpp @@ -149,13 +149,25 @@ ZipFs::ZipFs(IoBuffer&& Buffer) IoBuffer ZipFs::GetFile(const std::string_view& FileName) const { - FileMap::iterator Iter = m_Files.find(FileName); - if (Iter == m_Files.end()) { - return {}; + RwLock::SharedLockScope _(m_FilesLock); + + FileMap::const_iterator Iter = m_Files.find(FileName); + if (Iter == m_Files.end()) + { + return {}; + } + + const FileItem& Item = Iter->second; + if (Item.GetSize() > 0) + { + return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize()); + } } - FileItem& Item = Iter->second; + RwLock::ExclusiveLockScope _(m_FilesLock); + + FileItem& Item = m_Files.find(FileName)->second; if (Item.GetSize() > 0) { return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize()); diff --git a/src/zenserver/frontend/zipfs.h b/src/zenserver/frontend/zipfs.h index 1fa7da451..645121693 100644 --- a/src/zenserver/frontend/zipfs.h +++ b/src/zenserver/frontend/zipfs.h @@ -3,23 +3,23 @@ #pragma once #include <zencore/iobuffer.h> +#include <zencore/thread.h> #include <unordered_map> namespace zen { -////////////////////////////////////////////////////////////////////////// class ZipFs { public: - ZipFs() = default; - ZipFs(IoBuffer&& Buffer); + explicit ZipFs(IoBuffer&& Buffer); + IoBuffer GetFile(const std::string_view& FileName) const; - inline operator bool() const { return !m_Files.empty(); } private: using FileItem = MemoryView; using FileMap = std::unordered_map<std::string_view, FileItem>; + mutable RwLock m_FilesLock; FileMap mutable m_Files; IoBuffer m_Buffer; }; diff --git a/src/zenserver/hub/hubservice.cpp b/src/zenserver/hub/hubservice.cpp index 4d9da3a57..a00446a75 100644 --- a/src/zenserver/hub/hubservice.cpp +++ b/src/zenserver/hub/hubservice.cpp @@ -151,6 +151,7 @@ struct StorageServerInstance inline uint16_t GetBasePort() const { return m_ServerInstance.GetBasePort(); } private: + void WakeLocked(); RwLock m_Lock; std::string m_ModuleId; std::atomic<bool> m_IsProvisioned{false}; @@ -211,7 +212,7 @@ StorageServerInstance::Provision() if (m_IsHibernated) { - Wake(); + WakeLocked(); } else { @@ -294,9 +295,14 @@ StorageServerInstance::Hibernate() void StorageServerInstance::Wake() { - // Start server in-place using existing data - RwLock::ExclusiveLockScope _(m_Lock); + WakeLocked(); +} + +void +StorageServerInstance::WakeLocked() +{ + // Start server in-place using existing data if (!m_IsHibernated) { diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp index f5ba30616..bf7afcc02 100644 --- a/src/zenserver/storage/buildstore/httpbuildstore.cpp +++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp @@ -185,7 +185,7 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) { const HttpRange& Range = Ranges.front(); const uint64_t BlobSize = Blob.GetSize(); - const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0; + const uint64_t MaxBlobSize = Range.Start < BlobSize ? BlobSize - Range.Start : 0; const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); if (Range.Start + RangeSize > BlobSize) { diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index 04a0781d3..aa37e75fe 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -266,13 +266,12 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload) m_BlobLookup.insert({BlobHash, NewBlobIndex}); } - m_LastAccessTimeUpdateCount++; if (m_TrackedBlobKeys) { m_TrackedBlobKeys->push_back(BlobHash); if (MetadataHash != IoHash::Zero) { - m_TrackedBlobKeys->push_back(BlobHash); + m_TrackedBlobKeys->push_back(MetadataHash); } } } diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index ead7e4f3a..b73b3e6fb 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -626,7 +626,7 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& B return false; } - const uint64_t ExpectedEntryCount = (FileSize - sizeof(sizeof(BucketMetaHeader))) / sizeof(ManifestData); + const uint64_t ExpectedEntryCount = (FileSize - sizeof(BucketMetaHeader)) / sizeof(ManifestData); if (Header.EntryCount > ExpectedEntryCount) { ZEN_WARN( @@ -1057,7 +1057,7 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const return 0; } - const uint64_t ExpectedEntryCount = (FileSize - sizeof(sizeof(cache::impl::CacheBucketIndexHeader))) / sizeof(DiskIndexEntry); + const uint64_t ExpectedEntryCount = (FileSize - sizeof(cache::impl::CacheBucketIndexHeader)) / sizeof(DiskIndexEntry); if (Header.EntryCount > ExpectedEntryCount) { return 0; diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index 94abcf547..e1fd0a3e6 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -966,7 +966,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb } else { - ResponseObject.AddBool(true); + ResponseObject.AddBool(false); } } ResponseObject.EndArray(); diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 52b494e45..4e8475293 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -608,7 +608,10 @@ ZenCacheStore::GetBatch::Commit() m_CacheStore.m_HitCount++; OpScope.SetBytes(Result.Value.GetSize()); } - m_CacheStore.m_MissCount++; + else + { + m_CacheStore.m_MissCount++; + } } } } diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index ed017988f..7402d92d3 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -300,12 +300,12 @@ GetCompactCasResults(CasContainerStrategy& Strategy, }; static void -GetFileCasResults(FileCasStrategy& Strategy, - CasStore::InsertMode Mode, - std::span<IoBuffer> Data, - std::span<IoHash> ChunkHashes, - std::span<size_t> Indexes, - std::vector<CasStore::InsertResult> Results) +GetFileCasResults(FileCasStrategy& Strategy, + CasStore::InsertMode Mode, + std::span<IoBuffer> Data, + std::span<IoHash> ChunkHashes, + std::span<size_t> Indexes, + std::vector<CasStore::InsertResult>& Results) { for (size_t Index : Indexes) { |