aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2026-02-24 15:02:58 +0100
committerGitHub Enterprise <[email protected]>2026-02-24 15:02:58 +0100
commit2af00d3ff720969fb3b4d471778efcf8c7b3fad4 (patch)
tree9193eb3ff8cc9f3fb5e3342d94bde8c193a977ce /src
parentadded zencore/filesystem.h include (diff)
parentVarious bug fixes (#778) (diff)
downloadzen-sb/spdlog-out.tar.xz
zen-sb/spdlog-out.zip
Merge branch 'main' into sb/spdlog-outsb/spdlog-out
Diffstat (limited to 'src')
-rw-r--r--src/zen/cmds/admin_cmd.h40
-rw-r--r--src/zen/cmds/bench_cmd.h5
-rw-r--r--src/zen/cmds/cache_cmd.h20
-rw-r--r--src/zen/cmds/copy_cmd.h5
-rw-r--r--src/zen/cmds/dedup_cmd.h5
-rw-r--r--src/zen/cmds/info_cmd.h5
-rw-r--r--src/zen/cmds/print_cmd.h10
-rw-r--r--src/zen/cmds/projectstore_cmd.h58
-rw-r--r--src/zen/cmds/rpcreplay_cmd.h15
-rw-r--r--src/zen/cmds/run_cmd.h5
-rw-r--r--src/zen/cmds/serve_cmd.h5
-rw-r--r--src/zen/cmds/status_cmd.h5
-rw-r--r--src/zen/cmds/top_cmd.h10
-rw-r--r--src/zen/cmds/trace_cmd.h7
-rw-r--r--src/zen/cmds/up_cmd.h15
-rw-r--r--src/zen/cmds/vfs_cmd.h5
-rw-r--r--src/zen/zen.cpp86
-rw-r--r--src/zencompute-test/xmake.lua1
-rw-r--r--src/zencore/base64.cpp192
-rw-r--r--src/zencore/compactbinaryyaml.cpp427
-rw-r--r--src/zencore/filesystem.cpp16
-rw-r--r--src/zencore/include/zencore/base64.h4
-rw-r--r--src/zencore/include/zencore/compactbinaryvalue.h24
-rw-r--r--src/zencore/memtrack/callstacktrace.cpp8
-rw-r--r--src/zencore/string.cpp4
-rw-r--r--src/zencore/xmake.lua2
-rw-r--r--src/zenhttp/servers/httpasio.cpp4
-rw-r--r--src/zenhttp/servers/httpsys.cpp17
-rw-r--r--src/zenserver/frontend/frontend.cpp9
-rw-r--r--src/zenserver/frontend/frontend.h7
-rw-r--r--src/zenserver/frontend/zipfs.cpp20
-rw-r--r--src/zenserver/frontend/zipfs.h8
-rw-r--r--src/zenserver/hub/hubservice.cpp12
-rw-r--r--src/zenserver/storage/buildstore/httpbuildstore.cpp2
-rw-r--r--src/zenstore/buildstore/buildstore.cpp3
-rw-r--r--src/zenstore/cache/cachedisklayer.cpp4
-rw-r--r--src/zenstore/cache/cacherpc.cpp2
-rw-r--r--src/zenstore/cache/structuredcachestore.cpp5
-rw-r--r--src/zenstore/cas.cpp12
39 files changed, 800 insertions, 284 deletions
diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h
index 87ef8091b..83bcf8893 100644
--- a/src/zen/cmds/admin_cmd.h
+++ b/src/zen/cmds/admin_cmd.h
@@ -13,6 +13,9 @@ namespace zen {
class ScrubCommand : public StorageCommand
{
public:
+ static constexpr char Name[] = "scrub";
+ static constexpr char Description[] = "Scrub zen storage (verify data integrity)";
+
ScrubCommand();
~ScrubCommand();
@@ -20,7 +23,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"scrub", "Scrub zen storage"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
bool m_DryRun = false;
bool m_NoGc = false;
@@ -33,6 +36,9 @@ private:
class GcCommand : public StorageCommand
{
public:
+ static constexpr char Name[] = "gc";
+ static constexpr char Description[] = "Garbage collect zen storage";
+
GcCommand();
~GcCommand();
@@ -40,7 +46,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"gc", "Garbage collect zen storage"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
bool m_SmallObjects{false};
bool m_SkipCid{false};
@@ -62,6 +68,9 @@ private:
class GcStatusCommand : public StorageCommand
{
public:
+ static constexpr char Name[] = "gc-status";
+ static constexpr char Description[] = "Garbage collect zen storage status check";
+
GcStatusCommand();
~GcStatusCommand();
@@ -69,7 +78,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"gc-status", "Garbage collect zen storage status check"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
bool m_Details = false;
};
@@ -77,6 +86,9 @@ private:
class GcStopCommand : public StorageCommand
{
public:
+ static constexpr char Name[] = "gc-stop";
+ static constexpr char Description[] = "Request cancel of running garbage collection in zen storage";
+
GcStopCommand();
~GcStopCommand();
@@ -84,7 +96,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"gc-stop", "Request cancel of running garbage collection in zen storage"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
};
@@ -93,6 +105,9 @@ private:
class JobCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "jobs";
+ static constexpr char Description[] = "Show/cancel zen background jobs";
+
JobCommand();
~JobCommand();
@@ -100,7 +115,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"jobs", "Show/cancel zen background jobs"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::uint64_t m_JobId = 0;
bool m_Cancel = 0;
@@ -111,6 +126,9 @@ private:
class LoggingCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "logs";
+ static constexpr char Description[] = "Show/control zen logging";
+
LoggingCommand();
~LoggingCommand();
@@ -118,7 +136,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"logs", "Show/control zen logging"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_CacheWriteLog;
std::string m_CacheAccessLog;
@@ -133,6 +151,9 @@ private:
class FlushCommand : public StorageCommand
{
public:
+ static constexpr char Name[] = "flush";
+ static constexpr char Description[] = "Flush storage";
+
FlushCommand();
~FlushCommand();
@@ -140,7 +161,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"flush", "Flush zen storage"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
};
@@ -149,6 +170,9 @@ private:
class CopyStateCommand : public StorageCommand
{
public:
+ static constexpr char Name[] = "copy-state";
+ static constexpr char Description[] = "Copy zen server disk state";
+
CopyStateCommand();
~CopyStateCommand();
@@ -156,7 +180,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"copy-state", "Copy zen server disk state"};
+ cxxopts::Options m_Options{Name, Description};
std::filesystem::path m_DataPath;
std::filesystem::path m_TargetPath;
bool m_SkipLogs = false;
diff --git a/src/zen/cmds/bench_cmd.h b/src/zen/cmds/bench_cmd.h
index ed123be75..7fbf85340 100644
--- a/src/zen/cmds/bench_cmd.h
+++ b/src/zen/cmds/bench_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
class BenchCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "bench";
+ static constexpr char Description[] = "Utility command for benchmarking";
+
BenchCommand();
~BenchCommand();
@@ -17,7 +20,7 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"bench", "Benchmarking utility command"};
+ cxxopts::Options m_Options{Name, Description};
bool m_PurgeStandbyLists = false;
bool m_SingleProcess = false;
};
diff --git a/src/zen/cmds/cache_cmd.h b/src/zen/cmds/cache_cmd.h
index 4dc05bbdc..4f5b90f4d 100644
--- a/src/zen/cmds/cache_cmd.h
+++ b/src/zen/cmds/cache_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
class DropCommand : public CacheStoreCommand
{
public:
+ static constexpr char Name[] = "drop";
+ static constexpr char Description[] = "Drop cache namespace or bucket";
+
DropCommand();
~DropCommand();
@@ -16,7 +19,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"drop", "Drop cache namespace or bucket"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_NamespaceName;
std::string m_BucketName;
@@ -25,13 +28,16 @@ private:
class CacheInfoCommand : public CacheStoreCommand
{
public:
+ static constexpr char Name[] = "cache-info";
+ static constexpr char Description[] = "Info on cache, namespace or bucket";
+
CacheInfoCommand();
~CacheInfoCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"cache-info", "Info on cache, namespace or bucket"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_NamespaceName;
std::string m_SizeInfoBucketNames;
@@ -42,26 +48,32 @@ private:
class CacheStatsCommand : public CacheStoreCommand
{
public:
+ static constexpr char Name[] = "cache-stats";
+ static constexpr char Description[] = "Stats on cache";
+
CacheStatsCommand();
~CacheStatsCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"cache-stats", "Stats info on cache"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
};
class CacheDetailsCommand : public CacheStoreCommand
{
public:
+ static constexpr char Name[] = "cache-details";
+ static constexpr char Description[] = "Details on cache";
+
CacheDetailsCommand();
~CacheDetailsCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"cache-details", "Detailed info on cache"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
bool m_CSV = false;
bool m_Details = false;
diff --git a/src/zen/cmds/copy_cmd.h b/src/zen/cmds/copy_cmd.h
index e1a5dcb82..757a8e691 100644
--- a/src/zen/cmds/copy_cmd.h
+++ b/src/zen/cmds/copy_cmd.h
@@ -11,6 +11,9 @@ namespace zen {
class CopyCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "copy";
+ static constexpr char Description[] = "Copy file(s)";
+
CopyCommand();
~CopyCommand();
@@ -19,7 +22,7 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"copy", "Copy files efficiently"};
+ cxxopts::Options m_Options{Name, Description};
std::filesystem::path m_CopySource;
std::filesystem::path m_CopyTarget;
bool m_NoClone = false;
diff --git a/src/zen/cmds/dedup_cmd.h b/src/zen/cmds/dedup_cmd.h
index 5b8387dd2..835b35e92 100644
--- a/src/zen/cmds/dedup_cmd.h
+++ b/src/zen/cmds/dedup_cmd.h
@@ -11,6 +11,9 @@ namespace zen {
class DedupCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "dedup";
+ static constexpr char Description[] = "Dedup files";
+
DedupCommand();
~DedupCommand();
@@ -19,7 +22,7 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"dedup", "Deduplicate files"};
+ cxxopts::Options m_Options{Name, Description};
std::vector<std::string> m_Positional;
std::filesystem::path m_DedupSource;
std::filesystem::path m_DedupTarget;
diff --git a/src/zen/cmds/info_cmd.h b/src/zen/cmds/info_cmd.h
index 231565bfd..dc108b8a2 100644
--- a/src/zen/cmds/info_cmd.h
+++ b/src/zen/cmds/info_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
class InfoCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "info";
+ static constexpr char Description[] = "Show high level Zen server information";
+
InfoCommand();
~InfoCommand();
@@ -17,7 +20,7 @@ public:
// virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"info", "Show high level zen store information"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
};
diff --git a/src/zen/cmds/print_cmd.h b/src/zen/cmds/print_cmd.h
index 6c1529b7c..f4a97e218 100644
--- a/src/zen/cmds/print_cmd.h
+++ b/src/zen/cmds/print_cmd.h
@@ -11,6 +11,9 @@ namespace zen {
class PrintCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "print";
+ static constexpr char Description[] = "Print compact binary object";
+
PrintCommand();
~PrintCommand();
@@ -19,7 +22,7 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"print", "Print compact binary object"};
+ cxxopts::Options m_Options{Name, Description};
std::filesystem::path m_Filename;
bool m_ShowCbObjectTypeInfo = false;
};
@@ -29,6 +32,9 @@ private:
class PrintPackageCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "printpackage";
+ static constexpr char Description[] = "Print compact binary package";
+
PrintPackageCommand();
~PrintPackageCommand();
@@ -37,7 +43,7 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"printpkg", "Print compact binary package"};
+ cxxopts::Options m_Options{Name, Description};
std::filesystem::path m_Filename;
bool m_ShowCbObjectTypeInfo = false;
};
diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h
index 56ef858f5..e415b41b7 100644
--- a/src/zen/cmds/projectstore_cmd.h
+++ b/src/zen/cmds/projectstore_cmd.h
@@ -16,6 +16,9 @@ class ProjectStoreCommand : public ZenCmdBase
class DropProjectCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "project-drop";
+ static constexpr char Description[] = "Drop project or project oplog";
+
DropProjectCommand();
~DropProjectCommand();
@@ -23,7 +26,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"project-drop", "Drop project or project oplog"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
@@ -33,13 +36,16 @@ private:
class ProjectInfoCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "project-info";
+ static constexpr char Description[] = "Info on project or project oplog";
+
ProjectInfoCommand();
~ProjectInfoCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"project-info", "Info on project or project oplog"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
@@ -48,6 +54,9 @@ private:
class CreateProjectCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "project-create";
+ static constexpr char Description[] = "Create a project";
+
CreateProjectCommand();
~CreateProjectCommand();
@@ -55,7 +64,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"project-create", "Create project, the project must not already exist."};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectId;
std::string m_RootDir;
@@ -68,6 +77,9 @@ private:
class CreateOplogCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "oplog-create";
+ static constexpr char Description[] = "Create a project oplog";
+
CreateOplogCommand();
~CreateOplogCommand();
@@ -75,7 +87,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"oplog-create", "Create oplog in an existing project, the oplog must not already exist."};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectId;
std::string m_OplogId;
@@ -86,6 +98,9 @@ private:
class ExportOplogCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "oplog-export";
+ static constexpr char Description[] = "Export project store oplog";
+
ExportOplogCommand();
~ExportOplogCommand();
@@ -93,8 +108,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"oplog-export",
- "Export project store oplog to cloud (--cloud), file system (--file) or other Zen instance (--zen)"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
@@ -145,6 +159,9 @@ private:
class ImportOplogCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "oplog-import";
+ static constexpr char Description[] = "Import project store oplog";
+
ImportOplogCommand();
~ImportOplogCommand();
@@ -152,8 +169,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"oplog-import",
- "Import project store oplog from cloud (--cloud), file system (--file) or other Zen instance (--zen)"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
@@ -198,14 +214,16 @@ private:
class SnapshotOplogCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "oplog-snapshot";
+ static constexpr char Description[] = "Snapshot project store oplog";
+
SnapshotOplogCommand();
~SnapshotOplogCommand();
-
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"oplog-snapshot", "Snapshot external file references in project store oplog into zen"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
@@ -214,26 +232,32 @@ private:
class ProjectStatsCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "project-stats";
+ static constexpr char Description[] = "Stats on project store";
+
ProjectStatsCommand();
~ProjectStatsCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"project-stats", "Stats info on project store"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
};
class ProjectOpDetailsCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "project-op-details";
+ static constexpr char Description[] = "Detail info on ops inside a project store oplog";
+
ProjectOpDetailsCommand();
~ProjectOpDetailsCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"project-op-details", "Detail info on ops inside a project store oplog"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
bool m_Details = false;
bool m_OpDetails = false;
@@ -247,13 +271,16 @@ private:
class OplogMirrorCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "oplog-mirror";
+ static constexpr char Description[] = "Mirror project store oplog to file system";
+
OplogMirrorCommand();
~OplogMirrorCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"oplog-mirror", "Mirror oplog to file system"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
@@ -268,13 +295,16 @@ private:
class OplogValidateCommand : public ProjectStoreCommand
{
public:
+ static constexpr char Name[] = "oplog-validate";
+ static constexpr char Description[] = "Validate oplog for missing references";
+
OplogValidateCommand();
~OplogValidateCommand();
virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"oplog-validate", "Validate oplog for missing references"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
diff --git a/src/zen/cmds/rpcreplay_cmd.h b/src/zen/cmds/rpcreplay_cmd.h
index a6363b614..332a3126c 100644
--- a/src/zen/cmds/rpcreplay_cmd.h
+++ b/src/zen/cmds/rpcreplay_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
class RpcStartRecordingCommand : public CacheStoreCommand
{
public:
+ static constexpr char Name[] = "rpc-record-start";
+ static constexpr char Description[] = "Starts recording of cache rpc requests on a host";
+
RpcStartRecordingCommand();
~RpcStartRecordingCommand();
@@ -16,7 +19,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"rpc-record-start", "Starts recording of cache rpc requests on a host"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_RecordingPath;
};
@@ -24,6 +27,9 @@ private:
class RpcStopRecordingCommand : public CacheStoreCommand
{
public:
+ static constexpr char Name[] = "rpc-record-stop";
+ static constexpr char Description[] = "Stops recording of cache rpc requests on a host";
+
RpcStopRecordingCommand();
~RpcStopRecordingCommand();
@@ -31,13 +37,16 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"rpc-record-stop", "Stops recording of cache rpc requests on a host"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
};
class RpcReplayCommand : public CacheStoreCommand
{
public:
+ static constexpr char Name[] = "rpc-record-replay";
+ static constexpr char Description[] = "Replays a previously recorded session of rpc requests";
+
RpcReplayCommand();
~RpcReplayCommand();
@@ -45,7 +54,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"rpc-record-replay", "Replays a previously recorded session of cache rpc requests to a target host"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_RecordingPath;
bool m_OnHost = false;
diff --git a/src/zen/cmds/run_cmd.h b/src/zen/cmds/run_cmd.h
index 570a2e63a..300c08c5b 100644
--- a/src/zen/cmds/run_cmd.h
+++ b/src/zen/cmds/run_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
class RunCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "run";
+ static constexpr char Description[] = "Run command with special options";
+
RunCommand();
~RunCommand();
@@ -17,7 +20,7 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"run", "Run executable"};
+ cxxopts::Options m_Options{Name, Description};
int m_RunCount = 0;
int m_RunTime = -1;
std::string m_BaseDirectory;
diff --git a/src/zen/cmds/serve_cmd.h b/src/zen/cmds/serve_cmd.h
index ac74981f2..22f430948 100644
--- a/src/zen/cmds/serve_cmd.h
+++ b/src/zen/cmds/serve_cmd.h
@@ -11,6 +11,9 @@ namespace zen {
class ServeCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "serve";
+ static constexpr char Description[] = "Serve files from a directory";
+
ServeCommand();
~ServeCommand();
@@ -18,7 +21,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"serve", "Serve files from a tree"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
std::string m_ProjectName;
std::string m_OplogName;
diff --git a/src/zen/cmds/status_cmd.h b/src/zen/cmds/status_cmd.h
index dc103a196..df5df3066 100644
--- a/src/zen/cmds/status_cmd.h
+++ b/src/zen/cmds/status_cmd.h
@@ -11,6 +11,9 @@ namespace zen {
class StatusCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "status";
+ static constexpr char Description[] = "Show zen status";
+
StatusCommand();
~StatusCommand();
@@ -20,7 +23,7 @@ public:
private:
int GetLockFileEffectivePort() const;
- cxxopts::Options m_Options{"status", "Show zen status"};
+ cxxopts::Options m_Options{Name, Description};
uint16_t m_Port = 0;
std::filesystem::path m_DataDir;
};
diff --git a/src/zen/cmds/top_cmd.h b/src/zen/cmds/top_cmd.h
index 74167ecfd..aeb196558 100644
--- a/src/zen/cmds/top_cmd.h
+++ b/src/zen/cmds/top_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
class TopCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "top";
+ static constexpr char Description[] = "Monitor zen server activity";
+
TopCommand();
~TopCommand();
@@ -16,12 +19,15 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"top", "Show dev UI"};
+ cxxopts::Options m_Options{Name, Description};
};
class PsCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "ps";
+ static constexpr char Description[] = "Enumerate running zen server instances";
+
PsCommand();
~PsCommand();
@@ -29,7 +35,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"ps", "Enumerate running Zen server instances"};
+ cxxopts::Options m_Options{Name, Description};
};
} // namespace zen
diff --git a/src/zen/cmds/trace_cmd.h b/src/zen/cmds/trace_cmd.h
index a6c9742b7..6eb0ba22b 100644
--- a/src/zen/cmds/trace_cmd.h
+++ b/src/zen/cmds/trace_cmd.h
@@ -6,11 +6,12 @@
namespace zen {
-/** Scrub storage
- */
class TraceCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "trace";
+ static constexpr char Description[] = "Control zen realtime tracing";
+
TraceCommand();
~TraceCommand();
@@ -18,7 +19,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"trace", "Control zen realtime tracing"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_HostName;
bool m_Stop = false;
std::string m_TraceHost;
diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h
index 2e822d5fc..270db7f88 100644
--- a/src/zen/cmds/up_cmd.h
+++ b/src/zen/cmds/up_cmd.h
@@ -11,6 +11,9 @@ namespace zen {
class UpCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "up";
+ static constexpr char Description[] = "Bring zen server up";
+
UpCommand();
~UpCommand();
@@ -18,7 +21,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"up", "Bring up zen service"};
+ cxxopts::Options m_Options{Name, Description};
uint16_t m_Port = 0;
bool m_ShowConsole = false;
bool m_ShowLog = false;
@@ -28,6 +31,9 @@ private:
class AttachCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "attach";
+ static constexpr char Description[] = "Add a sponsor process to a running zen service";
+
AttachCommand();
~AttachCommand();
@@ -35,7 +41,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"};
+ cxxopts::Options m_Options{Name, Description};
uint16_t m_Port = 0;
int m_OwnerPid = 0;
std::filesystem::path m_DataDir;
@@ -44,6 +50,9 @@ private:
class DownCommand : public ZenCmdBase
{
public:
+ static constexpr char Name[] = "down";
+ static constexpr char Description[] = "Bring zen server down";
+
DownCommand();
~DownCommand();
@@ -51,7 +60,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"down", "Bring down zen service"};
+ cxxopts::Options m_Options{Name, Description};
uint16_t m_Port = 0;
bool m_ForceTerminate = false;
std::filesystem::path m_ProgramBaseDir;
diff --git a/src/zen/cmds/vfs_cmd.h b/src/zen/cmds/vfs_cmd.h
index 5deaa02fa..9009c774b 100644
--- a/src/zen/cmds/vfs_cmd.h
+++ b/src/zen/cmds/vfs_cmd.h
@@ -9,6 +9,9 @@ namespace zen {
class VfsCommand : public StorageCommand
{
public:
+ static constexpr char Name[] = "vfs";
+ static constexpr char Description[] = "Manage virtual file system";
+
VfsCommand();
~VfsCommand();
@@ -16,7 +19,7 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"vfs", "Manage virtual file system"};
+ cxxopts::Options m_Options{Name, Description};
std::string m_Verb;
std::string m_HostName;
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index 1b0578851..7e500f35e 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -379,56 +379,56 @@ main(int argc, char** argv)
const char* CmdSummary;
} Commands[] = {
// clang-format off
- {"attach", &AttachCmd, "Add a sponsor process to a running zen service"},
- {"bench", &BenchCmd, "Utility command for benchmarking"},
- {BuildsCommand::Name, &BuildsCmd, BuildsCommand::Description},
- {"cache-details", &CacheDetailsCmd, "Details on cache"},
- {"cache-info", &CacheInfoCmd, "Info on cache, namespace or bucket"},
+ {AttachCommand::Name, &AttachCmd, AttachCommand::Description},
+ {BenchCommand::Name, &BenchCmd, BenchCommand::Description},
+ {BuildsCommand::Name, &BuildsCmd, BuildsCommand::Description},
+ {CacheDetailsCommand::Name, &CacheDetailsCmd, CacheDetailsCommand::Description},
+ {CacheInfoCommand::Name, &CacheInfoCmd, CacheInfoCommand::Description},
{CacheGetCommand::Name, &CacheGetCmd, CacheGetCommand::Description},
{CacheGenerateCommand::Name, &CacheGenerateCmd, CacheGenerateCommand::Description},
- {"cache-stats", &CacheStatsCmd, "Stats on cache"},
- {"copy", &CopyCmd, "Copy file(s)"},
- {"copy-state", &CopyStateCmd, "Copy zen server disk state"},
- {"dedup", &DedupCmd, "Dedup files"},
- {"down", &DownCmd, "Bring zen server down"},
- {"drop", &DropCmd, "Drop cache namespace or bucket"},
+ {CacheStatsCommand::Name, &CacheStatsCmd, CacheStatsCommand::Description},
+ {CopyCommand::Name, &CopyCmd, CopyCommand::Description},
+ {CopyStateCommand::Name, &CopyStateCmd, CopyStateCommand::Description},
+ {DedupCommand::Name, &DedupCmd, DedupCommand::Description},
+ {DownCommand::Name, &DownCmd, DownCommand::Description},
+ {DropCommand::Name, &DropCmd, DropCommand::Description},
#if ZEN_WITH_COMPUTE_SERVICES
{ExecCommand::Name, &ExecCmd, ExecCommand::Description},
#endif
- {"gc-status", &GcStatusCmd, "Garbage collect zen storage status check"},
- {"gc-stop", &GcStopCmd, "Request cancel of running garbage collection in zen storage"},
- {"gc", &GcCmd, "Garbage collect zen storage"},
- {"info", &InfoCmd, "Show high level Zen server information"},
- {"jobs", &JobCmd, "Show/cancel zen background jobs"},
- {"logs", &LoggingCmd, "Show/control zen logging"},
- {"oplog-create", &CreateOplogCmd, "Create a project oplog"},
- {"oplog-export", &ExportOplogCmd, "Export project store oplog"},
- {"oplog-import", &ImportOplogCmd, "Import project store oplog"},
- {"oplog-mirror", &OplogMirrorCmd, "Mirror project store oplog to file system"},
- {"oplog-snapshot", &SnapshotOplogCmd, "Snapshot project store oplog"},
+ {GcStatusCommand::Name, &GcStatusCmd, GcStatusCommand::Description},
+ {GcStopCommand::Name, &GcStopCmd, GcStopCommand::Description},
+ {GcCommand::Name, &GcCmd, GcCommand::Description},
+ {InfoCommand::Name, &InfoCmd, InfoCommand::Description},
+ {JobCommand::Name, &JobCmd, JobCommand::Description},
+ {LoggingCommand::Name, &LoggingCmd, LoggingCommand::Description},
+ {CreateOplogCommand::Name, &CreateOplogCmd, CreateOplogCommand::Description},
+ {ExportOplogCommand::Name, &ExportOplogCmd, ExportOplogCommand::Description},
+ {ImportOplogCommand::Name, &ImportOplogCmd, ImportOplogCommand::Description},
+ {OplogMirrorCommand::Name, &OplogMirrorCmd, OplogMirrorCommand::Description},
+ {SnapshotOplogCommand::Name, &SnapshotOplogCmd, SnapshotOplogCommand::Description},
{OplogDownloadCommand::Name, &OplogDownload, OplogDownloadCommand::Description},
- {"oplog-validate", &OplogValidateCmd, "Validate oplog for missing references"},
- {"print", &PrintCmd, "Print compact binary object"},
- {"printpackage", &PrintPkgCmd, "Print compact binary package"},
- {"project-create", &CreateProjectCmd, "Create a project"},
- {"project-op-details", &ProjectOpDetailsCmd, "Detail info on ops inside a project store oplog"},
- {"project-drop", &ProjectDropCmd, "Drop project or project oplog"},
- {"project-info", &ProjectInfoCmd, "Info on project or project oplog"},
- {"project-stats", &ProjectStatsCmd, "Stats on project store"},
- {"ps", &PsCmd, "Enumerate running zen server instances"},
- {"rpc-record-replay", &RpcReplayCmd, "Replays a previously recorded session of rpc requests"},
- {"rpc-record-start", &RpcStartRecordingCmd, "Starts recording of cache rpc requests on a host"},
- {"rpc-record-stop", &RpcStopRecordingCmd, "Stops recording of cache rpc requests on a host"},
- {"run", &RunCmd, "Run command with special options"},
- {"scrub", &ScrubCmd, "Scrub zen storage (verify data integrity)"},
- {"serve", &ServeCmd, "Serve files from a directory"},
- {"status", &StatusCmd, "Show zen status"},
- {"top", &TopCmd, "Monitor zen server activity"},
- {"trace", &TraceCmd, "Control zen realtime tracing"},
- {"up", &UpCmd, "Bring zen server up"},
+ {OplogValidateCommand::Name, &OplogValidateCmd, OplogValidateCommand::Description},
+ {PrintCommand::Name, &PrintCmd, PrintCommand::Description},
+ {PrintPackageCommand::Name, &PrintPkgCmd, PrintPackageCommand::Description},
+ {CreateProjectCommand::Name, &CreateProjectCmd, CreateProjectCommand::Description},
+ {ProjectOpDetailsCommand::Name, &ProjectOpDetailsCmd, ProjectOpDetailsCommand::Description},
+ {DropProjectCommand::Name, &ProjectDropCmd, DropProjectCommand::Description},
+ {ProjectInfoCommand::Name, &ProjectInfoCmd, ProjectInfoCommand::Description},
+ {ProjectStatsCommand::Name, &ProjectStatsCmd, ProjectStatsCommand::Description},
+ {PsCommand::Name, &PsCmd, PsCommand::Description},
+ {RpcReplayCommand::Name, &RpcReplayCmd, RpcReplayCommand::Description},
+ {RpcStartRecordingCommand::Name, &RpcStartRecordingCmd, RpcStartRecordingCommand::Description},
+ {RpcStopRecordingCommand::Name, &RpcStopRecordingCmd, RpcStopRecordingCommand::Description},
+ {RunCommand::Name, &RunCmd, RunCommand::Description},
+ {ScrubCommand::Name, &ScrubCmd, ScrubCommand::Description},
+ {ServeCommand::Name, &ServeCmd, ServeCommand::Description},
+ {StatusCommand::Name, &StatusCmd, StatusCommand::Description},
+ {TopCommand::Name, &TopCmd, TopCommand::Description},
+ {TraceCommand::Name, &TraceCmd, TraceCommand::Description},
+ {UpCommand::Name, &UpCmd, UpCommand::Description},
{VersionCommand::Name, &VersionCmd, VersionCommand::Description},
- {"vfs", &VfsCmd, "Manage virtual file system"},
- {"flush", &FlushCmd, "Flush storage"},
+ {VfsCommand::Name, &VfsCmd, VfsCommand::Description},
+ {FlushCommand::Name, &FlushCmd, FlushCommand::Description},
{WipeCommand::Name, &WipeCmd, WipeCommand::Description},
{WorkspaceCommand::Name, &WorkspaceCmd, WorkspaceCommand::Description},
{WorkspaceShareCommand::Name, &WorkspaceShareCmd, WorkspaceShareCommand::Description},
diff --git a/src/zencompute-test/xmake.lua b/src/zencompute-test/xmake.lua
index 860bf655b..1207bdefd 100644
--- a/src/zencompute-test/xmake.lua
+++ b/src/zencompute-test/xmake.lua
@@ -6,4 +6,3 @@ target("zencompute-test")
add_headerfiles("**.h")
add_files("*.cpp")
add_deps("zencompute", "zencore")
-
diff --git a/src/zencore/base64.cpp b/src/zencore/base64.cpp
index 1f56ee6c3..fdf5f2d66 100644
--- a/src/zencore/base64.cpp
+++ b/src/zencore/base64.cpp
@@ -1,6 +1,10 @@
// Copyright Epic Games, Inc. All Rights Reserved.
#include <zencore/base64.h>
+#include <zencore/string.h>
+#include <zencore/testing.h>
+
+#include <string>
namespace zen {
@@ -11,7 +15,6 @@ static const uint8_t EncodingAlphabet[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G',
'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'};
/** The table used to convert an ascii character into a 6 bit value */
-#if 0
static const uint8_t DecodingAlphabet[256] = {
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x00-0x0f
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x10-0x1f
@@ -30,7 +33,6 @@ static const uint8_t DecodingAlphabet[256] = {
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xe0-0xef
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // 0xf0-0xff
};
-#endif // 0
template<typename CharType>
uint32_t
@@ -104,4 +106,190 @@ Base64::Encode(const uint8_t* Source, uint32_t Length, CharType* Dest)
template uint32_t Base64::Encode<char>(const uint8_t* Source, uint32_t Length, char* Dest);
template uint32_t Base64::Encode<wchar_t>(const uint8_t* Source, uint32_t Length, wchar_t* Dest);
+template<typename CharType>
+bool
+Base64::Decode(const CharType* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength)
+{
+ // Length must be a multiple of 4
+ if (Length % 4 != 0)
+ {
+ OutLength = 0;
+ return false;
+ }
+
+ uint8_t* DecodedBytes = Dest;
+
+ // Process 4 encoded characters at a time, producing 3 decoded bytes
+ while (Length > 0)
+ {
+ // Count padding characters at the end
+ uint32_t PadCount = 0;
+ if (Source[3] == '=')
+ {
+ PadCount++;
+ if (Source[2] == '=')
+ {
+ PadCount++;
+ }
+ }
+
+ // Look up each character in the decoding table
+ uint8_t A = DecodingAlphabet[static_cast<uint8_t>(Source[0])];
+ uint8_t B = DecodingAlphabet[static_cast<uint8_t>(Source[1])];
+ uint8_t C = (PadCount >= 2) ? 0 : DecodingAlphabet[static_cast<uint8_t>(Source[2])];
+ uint8_t D = (PadCount >= 1) ? 0 : DecodingAlphabet[static_cast<uint8_t>(Source[3])];
+
+ // Check for invalid characters (0xFF means not in the base64 alphabet)
+ if (A == 0xFF || B == 0xFF || C == 0xFF || D == 0xFF)
+ {
+ OutLength = 0;
+ return false;
+ }
+
+ // Reconstruct the 24-bit value from 4 6-bit chunks
+ uint32_t ByteTriplet = (A << 18) | (B << 12) | (C << 6) | D;
+
+ // Extract the 3 bytes
+ *DecodedBytes++ = static_cast<uint8_t>(ByteTriplet >> 16);
+ if (PadCount < 2)
+ {
+ *DecodedBytes++ = static_cast<uint8_t>((ByteTriplet >> 8) & 0xFF);
+ }
+ if (PadCount < 1)
+ {
+ *DecodedBytes++ = static_cast<uint8_t>(ByteTriplet & 0xFF);
+ }
+
+ Source += 4;
+ Length -= 4;
+ }
+
+ OutLength = uint32_t(DecodedBytes - Dest);
+ return true;
+}
+
+template bool Base64::Decode<char>(const char* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength);
+template bool Base64::Decode<wchar_t>(const wchar_t* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength);
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Testing related code follows...
+//
+
+#if ZEN_WITH_TESTS
+
+using namespace std::string_literals;
+
+TEST_CASE("Base64")
+{
+ auto EncodeString = [](std::string_view Input) -> std::string {
+ std::string Result;
+ Result.resize(Base64::GetEncodedDataSize(uint32_t(Input.size())));
+ Base64::Encode(reinterpret_cast<const uint8_t*>(Input.data()), uint32_t(Input.size()), Result.data());
+ return Result;
+ };
+
+ auto DecodeString = [](std::string_view Input) -> std::string {
+ std::string Result;
+ Result.resize(Base64::GetMaxDecodedDataSize(uint32_t(Input.size())));
+ uint32_t DecodedLength = 0;
+ bool Success = Base64::Decode(Input.data(), uint32_t(Input.size()), reinterpret_cast<uint8_t*>(Result.data()), DecodedLength);
+ CHECK(Success);
+ Result.resize(DecodedLength);
+ return Result;
+ };
+
+ SUBCASE("Encode")
+ {
+ CHECK(EncodeString("") == ""s);
+ CHECK(EncodeString("f") == "Zg=="s);
+ CHECK(EncodeString("fo") == "Zm8="s);
+ CHECK(EncodeString("foo") == "Zm9v"s);
+ CHECK(EncodeString("foob") == "Zm9vYg=="s);
+ CHECK(EncodeString("fooba") == "Zm9vYmE="s);
+ CHECK(EncodeString("foobar") == "Zm9vYmFy"s);
+ }
+
+ SUBCASE("Decode")
+ {
+ CHECK(DecodeString("") == ""s);
+ CHECK(DecodeString("Zg==") == "f"s);
+ CHECK(DecodeString("Zm8=") == "fo"s);
+ CHECK(DecodeString("Zm9v") == "foo"s);
+ CHECK(DecodeString("Zm9vYg==") == "foob"s);
+ CHECK(DecodeString("Zm9vYmE=") == "fooba"s);
+ CHECK(DecodeString("Zm9vYmFy") == "foobar"s);
+ }
+
+ SUBCASE("RoundTrip")
+ {
+ auto RoundTrip = [&](const std::string& Input) {
+ std::string Encoded = EncodeString(Input);
+ std::string Decoded = DecodeString(Encoded);
+ CHECK(Decoded == Input);
+ };
+
+ RoundTrip("Hello, World!");
+ RoundTrip("Base64 encoding test with various lengths");
+ RoundTrip("A");
+ RoundTrip("AB");
+ RoundTrip("ABC");
+ RoundTrip("ABCD");
+ RoundTrip("\x00\x01\x02\xff\xfe\xfd"s);
+ }
+
+ SUBCASE("BinaryRoundTrip")
+ {
+ // Test with all byte values 0-255
+ uint8_t AllBytes[256];
+ for (int i = 0; i < 256; ++i)
+ {
+ AllBytes[i] = static_cast<uint8_t>(i);
+ }
+
+ char Encoded[Base64::GetEncodedDataSize(256) + 1];
+ Base64::Encode(AllBytes, 256, Encoded);
+
+ uint8_t Decoded[256];
+ uint32_t DecodedLength = 0;
+ bool Success = Base64::Decode(Encoded, uint32_t(strlen(Encoded)), Decoded, DecodedLength);
+ CHECK(Success);
+ CHECK(DecodedLength == 256);
+ CHECK(memcmp(AllBytes, Decoded, 256) == 0);
+ }
+
+ SUBCASE("DecodeInvalidInput")
+ {
+ uint8_t Dest[64];
+ uint32_t OutLength = 0;
+
+ // Length not a multiple of 4
+ CHECK_FALSE(Base64::Decode("abc", 3u, Dest, OutLength));
+
+ // Invalid character
+ CHECK_FALSE(Base64::Decode("ab!d", 4u, Dest, OutLength));
+ }
+
+ SUBCASE("EncodedDataSize")
+ {
+ CHECK(Base64::GetEncodedDataSize(0) == 0);
+ CHECK(Base64::GetEncodedDataSize(1) == 4);
+ CHECK(Base64::GetEncodedDataSize(2) == 4);
+ CHECK(Base64::GetEncodedDataSize(3) == 4);
+ CHECK(Base64::GetEncodedDataSize(4) == 8);
+ CHECK(Base64::GetEncodedDataSize(5) == 8);
+ CHECK(Base64::GetEncodedDataSize(6) == 8);
+ }
+
+ SUBCASE("MaxDecodedDataSize")
+ {
+ CHECK(Base64::GetMaxDecodedDataSize(0) == 0);
+ CHECK(Base64::GetMaxDecodedDataSize(4) == 3);
+ CHECK(Base64::GetMaxDecodedDataSize(8) == 6);
+ CHECK(Base64::GetMaxDecodedDataSize(12) == 9);
+ }
+}
+
+#endif
+
} // namespace zen
diff --git a/src/zencore/compactbinaryyaml.cpp b/src/zencore/compactbinaryyaml.cpp
index 5122e952a..b308af418 100644
--- a/src/zencore/compactbinaryyaml.cpp
+++ b/src/zencore/compactbinaryyaml.cpp
@@ -14,11 +14,6 @@
#include <string_view>
#include <vector>
-ZEN_THIRD_PARTY_INCLUDES_START
-#include <ryml.hpp>
-#include <ryml_std.hpp>
-ZEN_THIRD_PARTY_INCLUDES_END
-
namespace zen {
//////////////////////////////////////////////////////////////////////////
@@ -26,193 +21,349 @@ namespace zen {
class CbYamlWriter
{
public:
- explicit CbYamlWriter(StringBuilderBase& InBuilder) : m_StrBuilder(InBuilder) { m_NodeStack.push_back(m_Tree.rootref()); }
+ explicit CbYamlWriter(StringBuilderBase& InBuilder) : m_Builder(InBuilder) {}
void WriteField(CbFieldView Field)
{
- ryml::NodeRef Node;
+ CbValue Accessor = Field.GetValue();
+ CbFieldType Type = Accessor.GetType();
- if (m_IsFirst)
+ switch (Type)
{
- Node = Top();
+ case CbFieldType::Object:
+ case CbFieldType::UniformObject:
+ WriteMapEntries(Field, 0);
+ break;
+ case CbFieldType::Array:
+ case CbFieldType::UniformArray:
+ WriteSeqEntries(Field, 0);
+ break;
+ default:
+ WriteScalarValue(Field);
+ m_Builder << '\n';
+ break;
+ }
+ }
+
+ void WriteMapEntry(CbFieldView Field, int32_t Indent)
+ {
+ WriteIndent(Indent);
+ WriteMapEntryContent(Field, Indent);
+ }
+
+ void WriteSeqEntry(CbFieldView Field, int32_t Indent)
+ {
+ CbValue Accessor = Field.GetValue();
+ CbFieldType Type = Accessor.GetType();
- m_IsFirst = false;
+ if (Type == CbFieldType::Object || Type == CbFieldType::UniformObject)
+ {
+ bool First = true;
+ for (CbFieldView MapChild : Field)
+ {
+ if (First)
+ {
+ WriteIndent(Indent);
+ m_Builder << "- ";
+ First = false;
+ }
+ else
+ {
+ WriteIndent(Indent + 1);
+ }
+ WriteMapEntryContent(MapChild, Indent + 1);
+ }
+ }
+ else if (Type == CbFieldType::Array || Type == CbFieldType::UniformArray)
+ {
+ WriteIndent(Indent);
+ m_Builder << "-\n";
+ WriteSeqEntries(Field, Indent + 1);
}
else
{
- Node = Top().append_child();
+ WriteIndent(Indent);
+ m_Builder << "- ";
+ WriteScalarValue(Field);
+ m_Builder << '\n';
}
+ }
- if (std::u8string_view Name = Field.GetU8Name(); !Name.empty())
+private:
+ void WriteMapEntries(CbFieldView MapField, int32_t Indent)
+ {
+ for (CbFieldView Child : MapField)
{
- Node.set_key_serialized(ryml::csubstr((const char*)Name.data(), Name.size()));
+ WriteIndent(Indent);
+ WriteMapEntryContent(Child, Indent);
}
+ }
+
+ void WriteMapEntryContent(CbFieldView Field, int32_t Indent)
+ {
+ std::u8string_view Name = Field.GetU8Name();
+ m_Builder << std::string_view(reinterpret_cast<const char*>(Name.data()), Name.size());
- switch (CbValue Accessor = Field.GetValue(); Accessor.GetType())
+ CbValue Accessor = Field.GetValue();
+ CbFieldType Type = Accessor.GetType();
+
+ if (IsContainer(Type))
{
- case CbFieldType::Null:
- Node.set_val("null");
- break;
- case CbFieldType::Object:
- case CbFieldType::UniformObject:
- Node |= ryml::MAP;
- m_NodeStack.push_back(Node);
- for (CbFieldView It : Field)
+ m_Builder << ":\n";
+ WriteFieldValue(Field, Indent + 1);
+ }
+ else
+ {
+ m_Builder << ": ";
+ WriteScalarValue(Field);
+ m_Builder << '\n';
+ }
+ }
+
+ void WriteSeqEntries(CbFieldView SeqField, int32_t Indent)
+ {
+ for (CbFieldView Child : SeqField)
+ {
+ CbValue Accessor = Child.GetValue();
+ CbFieldType Type = Accessor.GetType();
+
+ if (Type == CbFieldType::Object || Type == CbFieldType::UniformObject)
+ {
+ bool First = true;
+ for (CbFieldView MapChild : Child)
{
- WriteField(It);
+ if (First)
+ {
+ WriteIndent(Indent);
+ m_Builder << "- ";
+ First = false;
+ }
+ else
+ {
+ WriteIndent(Indent + 1);
+ }
+ WriteMapEntryContent(MapChild, Indent + 1);
}
- m_NodeStack.pop_back();
+ }
+ else if (Type == CbFieldType::Array || Type == CbFieldType::UniformArray)
+ {
+ WriteIndent(Indent);
+ m_Builder << "-\n";
+ WriteSeqEntries(Child, Indent + 1);
+ }
+ else
+ {
+ WriteIndent(Indent);
+ m_Builder << "- ";
+ WriteScalarValue(Child);
+ m_Builder << '\n';
+ }
+ }
+ }
+
+ void WriteFieldValue(CbFieldView Field, int32_t Indent)
+ {
+ CbValue Accessor = Field.GetValue();
+ CbFieldType Type = Accessor.GetType();
+
+ switch (Type)
+ {
+ case CbFieldType::Object:
+ case CbFieldType::UniformObject:
+ WriteMapEntries(Field, Indent);
break;
case CbFieldType::Array:
case CbFieldType::UniformArray:
- Node |= ryml::SEQ;
- m_NodeStack.push_back(Node);
- for (CbFieldView It : Field)
- {
- WriteField(It);
- }
- m_NodeStack.pop_back();
+ WriteSeqEntries(Field, Indent);
break;
- case CbFieldType::Binary:
- {
- ExtendableStringBuilder<256> Builder;
- const MemoryView Value = Accessor.AsBinary();
- ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024);
- const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
- const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize));
- Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex);
-
- Node.set_key_serialized(Builder.c_str());
- }
+ case CbFieldType::CustomById:
+ WriteCustomById(Field.GetValue().AsCustomById(), Indent);
break;
- case CbFieldType::String:
- {
- const std::u8string_view U8String = Accessor.AsU8String();
- Node.set_val(ryml::csubstr((const char*)U8String.data(), U8String.size()));
- }
+ case CbFieldType::CustomByName:
+ WriteCustomByName(Field.GetValue().AsCustomByName(), Indent);
+ break;
+ default:
+ WriteScalarValue(Field);
+ m_Builder << '\n';
+ break;
+ }
+ }
+
+ void WriteScalarValue(CbFieldView Field)
+ {
+ CbValue Accessor = Field.GetValue();
+ switch (Accessor.GetType())
+ {
+ case CbFieldType::Null:
+ m_Builder << "null";
+ break;
+ case CbFieldType::BoolFalse:
+ m_Builder << "false";
+ break;
+ case CbFieldType::BoolTrue:
+ m_Builder << "true";
break;
case CbFieldType::IntegerPositive:
- Node << Accessor.AsIntegerPositive();
+ m_Builder << Accessor.AsIntegerPositive();
break;
case CbFieldType::IntegerNegative:
- Node << Accessor.AsIntegerNegative();
+ m_Builder << Accessor.AsIntegerNegative();
break;
case CbFieldType::Float32:
if (const float Value = Accessor.AsFloat32(); std::isfinite(Value))
- {
- Node << Value;
- }
+ m_Builder.Append(fmt::format("{}", Value));
else
- {
- Node << "null";
- }
+ m_Builder << "null";
break;
case CbFieldType::Float64:
if (const double Value = Accessor.AsFloat64(); std::isfinite(Value))
- {
- Node << Value;
- }
+ m_Builder.Append(fmt::format("{}", Value));
else
+ m_Builder << "null";
+ break;
+ case CbFieldType::String:
{
- Node << "null";
+ const std::u8string_view U8String = Accessor.AsU8String();
+ WriteString(std::string_view(reinterpret_cast<const char*>(U8String.data()), U8String.size()));
}
break;
- case CbFieldType::BoolFalse:
- Node << "false";
- break;
- case CbFieldType::BoolTrue:
- Node << "true";
+ case CbFieldType::Hash:
+ WriteString(Accessor.AsHash().ToHexString());
break;
case CbFieldType::ObjectAttachment:
case CbFieldType::BinaryAttachment:
- Node << Accessor.AsAttachment().ToHexString();
- break;
- case CbFieldType::Hash:
- Node << Accessor.AsHash().ToHexString();
+ WriteString(Accessor.AsAttachment().ToHexString());
break;
case CbFieldType::Uuid:
- Node << fmt::format("{}", Accessor.AsUuid());
+ WriteString(fmt::format("{}", Accessor.AsUuid()));
break;
case CbFieldType::DateTime:
- Node << DateTime(Accessor.AsDateTimeTicks()).ToIso8601();
+ WriteString(DateTime(Accessor.AsDateTimeTicks()).ToIso8601());
break;
case CbFieldType::TimeSpan:
if (const TimeSpan Span(Accessor.AsTimeSpanTicks()); Span.GetDays() == 0)
- {
- Node << Span.ToString("%h:%m:%s.%n");
- }
+ WriteString(Span.ToString("%h:%m:%s.%n"));
else
- {
- Node << Span.ToString("%d.%h:%m:%s.%n");
- }
+ WriteString(Span.ToString("%d.%h:%m:%s.%n"));
break;
case CbFieldType::ObjectId:
- Node << fmt::format("{}", Accessor.AsObjectId());
+ WriteString(fmt::format("{}", Accessor.AsObjectId()));
break;
- case CbFieldType::CustomById:
- {
- CbCustomById Custom = Accessor.AsCustomById();
+ case CbFieldType::Binary:
+ WriteBase64(Accessor.AsBinary());
+ break;
+ default:
+ ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType()));
+ break;
+ }
+ }
- Node |= ryml::MAP;
+ void WriteCustomById(CbCustomById Custom, int32_t Indent)
+ {
+ WriteIndent(Indent);
+ m_Builder << "Id: ";
+ m_Builder.Append(fmt::format("{}", Custom.Id));
+ m_Builder << '\n';
+
+ WriteIndent(Indent);
+ m_Builder << "Data: ";
+ WriteBase64(Custom.Data);
+ m_Builder << '\n';
+ }
- ryml::NodeRef IdNode = Node.append_child();
- IdNode.set_key("Id");
- IdNode.set_val_serialized(fmt::format("{}", Custom.Id));
+ void WriteCustomByName(CbCustomByName Custom, int32_t Indent)
+ {
+ WriteIndent(Indent);
+ m_Builder << "Name: ";
+ WriteString(std::string_view(reinterpret_cast<const char*>(Custom.Name.data()), Custom.Name.size()));
+ m_Builder << '\n';
+
+ WriteIndent(Indent);
+ m_Builder << "Data: ";
+ WriteBase64(Custom.Data);
+ m_Builder << '\n';
+ }
- ryml::NodeRef DataNode = Node.append_child();
- DataNode.set_key("Data");
+ void WriteBase64(MemoryView Value)
+ {
+ ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024);
+ ExtendableStringBuilder<256> Buf;
+ const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
+ const size_t EncodedIndex = Buf.AddUninitialized(size_t(EncodedSize));
+ Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Buf.Data() + EncodedIndex);
+ WriteString(Buf.ToView());
+ }
- ExtendableStringBuilder<256> Builder;
- const MemoryView& Value = Custom.Data;
- const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
- const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize));
- Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex);
+ void WriteString(std::string_view Str)
+ {
+ if (NeedsQuoting(Str))
+ {
+ m_Builder << '\'';
+ for (char C : Str)
+ {
+ if (C == '\'')
+ m_Builder << "''";
+ else
+ m_Builder << C;
+ }
+ m_Builder << '\'';
+ }
+ else
+ {
+ m_Builder << Str;
+ }
+ }
- DataNode.set_val_serialized(Builder.c_str());
- }
- break;
- case CbFieldType::CustomByName:
- {
- CbCustomByName Custom = Accessor.AsCustomByName();
+ void WriteIndent(int32_t Indent)
+ {
+ for (int32_t I = 0; I < Indent; ++I)
+ m_Builder << " ";
+ }
- Node |= ryml::MAP;
+ static bool NeedsQuoting(std::string_view Str)
+ {
+ if (Str.empty())
+ return false;
- ryml::NodeRef NameNode = Node.append_child();
- NameNode.set_key("Name");
- std::string_view Name = std::string_view((const char*)Custom.Name.data(), Custom.Name.size());
- NameNode.set_val_serialized(std::string(Name));
+ char First = Str[0];
+ if (First == ' ' || First == '\n' || First == '\t' || First == '\r' || First == '*' || First == '&' || First == '%' ||
+ First == '@' || First == '`')
+ return true;
- ryml::NodeRef DataNode = Node.append_child();
- DataNode.set_key("Data");
+ if (Str.size() >= 2 && Str[0] == '<' && Str[1] == '<')
+ return true;
- ExtendableStringBuilder<256> Builder;
- const MemoryView& Value = Custom.Data;
- const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
- const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize));
- Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex);
+ char Last = Str.back();
+ if (Last == ' ' || Last == '\n' || Last == '\t' || Last == '\r')
+ return true;
- DataNode.set_val_serialized(Builder.c_str());
- }
- break;
- default:
- ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType()));
- break;
+ for (char C : Str)
+ {
+ if (C == '#' || C == ':' || C == '-' || C == '?' || C == ',' || C == '\n' || C == '{' || C == '}' || C == '[' || C == ']' ||
+ C == '\'' || C == '"')
+ return true;
}
- if (m_NodeStack.size() == 1)
+ return false;
+ }
+
+ static bool IsContainer(CbFieldType Type)
+ {
+ switch (Type)
{
- std::string Yaml = ryml::emitrs_yaml<std::string>(m_Tree);
- m_StrBuilder << Yaml;
+ case CbFieldType::Object:
+ case CbFieldType::UniformObject:
+ case CbFieldType::Array:
+ case CbFieldType::UniformArray:
+ case CbFieldType::CustomById:
+ case CbFieldType::CustomByName:
+ return true;
+ default:
+ return false;
}
}
-private:
- StringBuilderBase& m_StrBuilder;
- bool m_IsFirst = true;
-
- ryml::Tree m_Tree;
- std::vector<ryml::NodeRef> m_NodeStack;
- ryml::NodeRef& Top() { return m_NodeStack.back(); }
+ StringBuilderBase& m_Builder;
};
void
@@ -229,6 +380,32 @@ CompactBinaryToYaml(const CbArrayView& Array, StringBuilderBase& Builder)
Writer.WriteField(Array.AsFieldView());
}
+void
+CompactBinaryToYaml(MemoryView Data, StringBuilderBase& InBuilder)
+{
+ std::vector<CbFieldView> Fields = ReadCompactBinaryStream(Data);
+ if (Fields.empty())
+ return;
+
+ CbYamlWriter Writer(InBuilder);
+ if (Fields.size() == 1)
+ {
+ Writer.WriteField(Fields[0]);
+ return;
+ }
+
+ if (Fields[0].HasName())
+ {
+ for (const CbFieldView& Field : Fields)
+ Writer.WriteMapEntry(Field, 0);
+ }
+ else
+ {
+ for (const CbFieldView& Field : Fields)
+ Writer.WriteSeqEntry(Field, 0);
+ }
+}
+
#if ZEN_WITH_TESTS
void
cbyaml_forcelink()
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 1a4ee4b9b..553897407 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -1326,11 +1326,6 @@ ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uin
{
BytesRead = size_t(dwNumberOfBytesRead);
}
- else if ((BytesRead != NumberOfBytesToRead))
- {
- Ec = MakeErrorCode(ERROR_HANDLE_EOF);
- return;
- }
else
{
Ec = MakeErrorCodeFromLastError();
@@ -1344,20 +1339,15 @@ ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uin
{
BytesRead = size_t(ReadResult);
}
- else if ((BytesRead != NumberOfBytesToRead))
- {
- Ec = MakeErrorCode(EIO);
- return;
- }
else
{
Ec = MakeErrorCodeFromLastError();
return;
}
#endif
- Size -= NumberOfBytesToRead;
- FileOffset += NumberOfBytesToRead;
- Data = reinterpret_cast<uint8_t*>(Data) + NumberOfBytesToRead;
+ Size -= BytesRead;
+ FileOffset += BytesRead;
+ Data = reinterpret_cast<uint8_t*>(Data) + BytesRead;
}
}
diff --git a/src/zencore/include/zencore/base64.h b/src/zencore/include/zencore/base64.h
index 4d78b085f..08d9f3043 100644
--- a/src/zencore/include/zencore/base64.h
+++ b/src/zencore/include/zencore/base64.h
@@ -11,7 +11,11 @@ struct Base64
template<typename CharType>
static uint32_t Encode(const uint8_t* Source, uint32_t Length, CharType* Dest);
+ template<typename CharType>
+ static bool Decode(const CharType* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength);
+
static inline constexpr int32_t GetEncodedDataSize(uint32_t Size) { return ((Size + 2) / 3) * 4; }
+ static inline constexpr int32_t GetMaxDecodedDataSize(uint32_t Length) { return (Length / 4) * 3; }
};
} // namespace zen
diff --git a/src/zencore/include/zencore/compactbinaryvalue.h b/src/zencore/include/zencore/compactbinaryvalue.h
index aa2d2821d..4ce8009b8 100644
--- a/src/zencore/include/zencore/compactbinaryvalue.h
+++ b/src/zencore/include/zencore/compactbinaryvalue.h
@@ -128,17 +128,21 @@ CbValue::AsString(CbFieldError* OutError, std::string_view Default) const
uint32_t ValueSizeByteCount;
const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount);
- if (OutError)
+ if (ValueSize >= (uint64_t(1) << 31))
{
- if (ValueSize >= (uint64_t(1) << 31))
+ if (OutError)
{
*OutError = CbFieldError::RangeError;
- return Default;
}
+ return Default;
+ }
+
+ if (OutError)
+ {
*OutError = CbFieldError::None;
}
- return std::string_view(Chars + ValueSizeByteCount, int32_t(ValueSize));
+ return std::string_view(Chars + ValueSizeByteCount, size_t(ValueSize));
}
inline std::u8string_view
@@ -148,17 +152,21 @@ CbValue::AsU8String(CbFieldError* OutError, std::u8string_view Default) const
uint32_t ValueSizeByteCount;
const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount);
- if (OutError)
+ if (ValueSize >= (uint64_t(1) << 31))
{
- if (ValueSize >= (uint64_t(1) << 31))
+ if (OutError)
{
*OutError = CbFieldError::RangeError;
- return Default;
}
+ return Default;
+ }
+
+ if (OutError)
+ {
*OutError = CbFieldError::None;
}
- return std::u8string_view(Chars + ValueSizeByteCount, int32_t(ValueSize));
+ return std::u8string_view(Chars + ValueSizeByteCount, size_t(ValueSize));
}
inline uint64_t
diff --git a/src/zencore/memtrack/callstacktrace.cpp b/src/zencore/memtrack/callstacktrace.cpp
index a5b7fede6..4a7068568 100644
--- a/src/zencore/memtrack/callstacktrace.cpp
+++ b/src/zencore/memtrack/callstacktrace.cpp
@@ -169,13 +169,13 @@ private:
std::atomic_uint64_t Key;
std::atomic_uint32_t Value;
- inline uint64 GetKey() const { return Key.load(std::memory_order_relaxed); }
+ inline uint64 GetKey() const { return Key.load(std::memory_order_acquire); }
inline uint32_t GetValue() const { return Value.load(std::memory_order_relaxed); }
- inline bool IsEmpty() const { return Key.load(std::memory_order_relaxed) == 0; }
+ inline bool IsEmpty() const { return Key.load(std::memory_order_acquire) == 0; }
inline void SetKeyValue(uint64_t InKey, uint32_t InValue)
{
- Value.store(InValue, std::memory_order_release);
- Key.store(InKey, std::memory_order_relaxed);
+ Value.store(InValue, std::memory_order_relaxed);
+ Key.store(InKey, std::memory_order_release);
}
static inline uint32_t KeyHash(uint64_t Key) { return static_cast<uint32_t>(Key); }
static inline void ClearEntries(FEncounteredCallstackSetEntry* Entries, int32_t EntryCount)
diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp
index 0ee863b74..a9aed6309 100644
--- a/src/zencore/string.cpp
+++ b/src/zencore/string.cpp
@@ -24,6 +24,10 @@ utf16to8_impl(u16bit_iterator StartIt, u16bit_iterator EndIt, ::zen::StringBuild
// Take care of surrogate pairs first
if (utf8::internal::is_lead_surrogate(cp))
{
+ if (StartIt == EndIt)
+ {
+ break;
+ }
uint32_t trail_surrogate = utf8::internal::mask16(*StartIt++);
cp = (cp << 10) + trail_surrogate + utf8::internal::SURROGATE_OFFSET;
}
diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua
index 4b6e7c44b..322001ef9 100644
--- a/src/zencore/xmake.lua
+++ b/src/zencore/xmake.lua
@@ -32,8 +32,6 @@ target('zencore')
add_deps("timesinceprocessstart")
add_deps("doctest")
add_deps("fmt")
- add_deps("ryml")
-
add_packages("json11")
if is_plat("linux", "macosx") then
diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp
index d649e6333..a01789618 100644
--- a/src/zenhttp/servers/httpasio.cpp
+++ b/src/zenhttp/servers/httpasio.cpp
@@ -89,10 +89,10 @@ IsIPv6AvailableSysctl(void)
char buf[16];
if (fgets(buf, sizeof(buf), f))
{
- fclose(f);
// 0 means IPv6 enabled, 1 means disabled
val = atoi(buf);
}
+ fclose(f);
}
return val == 0;
@@ -1544,7 +1544,7 @@ struct HttpAcceptor
{
ZEN_WARN("Unable to initialize asio service, (bind returned '{}')", BindErrorCode.message());
- return 0;
+ return {};
}
if (EffectivePort != BasePort)
diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp
index c640ba90b..6995ffca9 100644
--- a/src/zenhttp/servers/httpsys.cpp
+++ b/src/zenhttp/servers/httpsys.cpp
@@ -25,6 +25,7 @@
# include <zencore/workthreadpool.h>
# include "iothreadpool.h"
+# include <atomic>
# include <http.h>
namespace zen {
@@ -129,8 +130,8 @@ private:
std::unique_ptr<WinIoThreadPool> m_IoThreadPool;
- RwLock m_AsyncWorkPoolInitLock;
- WorkerThreadPool* m_AsyncWorkPool = nullptr;
+ RwLock m_AsyncWorkPoolInitLock;
+ std::atomic<WorkerThreadPool*> m_AsyncWorkPool = nullptr;
std::vector<std::wstring> m_BaseUris; // eg: http://*:nnnn/
HTTP_SERVER_SESSION_ID m_HttpSessionId = 0;
@@ -1032,8 +1033,10 @@ HttpSysServer::~HttpSysServer()
ZEN_ERROR("~HttpSysServer() called without calling Close() first");
}
- delete m_AsyncWorkPool;
+ auto WorkPool = m_AsyncWorkPool.load(std::memory_order_relaxed);
m_AsyncWorkPool = nullptr;
+
+ delete WorkPool;
}
void
@@ -1323,17 +1326,17 @@ HttpSysServer::WorkPool()
{
ZEN_MEMSCOPE(GetHttpsysTag());
- if (!m_AsyncWorkPool)
+ if (!m_AsyncWorkPool.load(std::memory_order_acquire))
{
RwLock::ExclusiveLockScope _(m_AsyncWorkPoolInitLock);
- if (!m_AsyncWorkPool)
+ if (!m_AsyncWorkPool.load(std::memory_order_relaxed))
{
- m_AsyncWorkPool = new WorkerThreadPool(m_InitialConfig.AsyncWorkThreadCount, "http_async");
+ m_AsyncWorkPool.store(new WorkerThreadPool(m_InitialConfig.AsyncWorkThreadCount, "http_async"), std::memory_order_release);
}
}
- return *m_AsyncWorkPool;
+ return *m_AsyncWorkPool.load(std::memory_order_relaxed);
}
void
diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp
index 2b157581f..1cf451e91 100644
--- a/src/zenserver/frontend/frontend.cpp
+++ b/src/zenserver/frontend/frontend.cpp
@@ -38,7 +38,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpSt
#if ZEN_EMBED_HTML_ZIP
// Load an embedded Zip archive
IoBuffer HtmlZipDataBuffer(IoBuffer::Wrap, gHtmlZipData, sizeof(gHtmlZipData) - 1);
- m_ZipFs = ZipFs(std::move(HtmlZipDataBuffer));
+ m_ZipFs = std::make_unique<ZipFs>(std::move(HtmlZipDataBuffer));
#endif
if (m_Directory.empty() && !m_ZipFs)
@@ -157,9 +157,12 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request)
}
}
- if (IoBuffer FileBuffer = m_ZipFs.GetFile(Uri))
+ if (m_ZipFs)
{
- return Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer);
+ if (IoBuffer FileBuffer = m_ZipFs->GetFile(Uri))
+ {
+ return Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer);
+ }
}
Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv);
diff --git a/src/zenserver/frontend/frontend.h b/src/zenserver/frontend/frontend.h
index 84ffaac42..6d8585b72 100644
--- a/src/zenserver/frontend/frontend.h
+++ b/src/zenserver/frontend/frontend.h
@@ -7,6 +7,7 @@
#include "zipfs.h"
#include <filesystem>
+#include <memory>
namespace zen {
@@ -20,9 +21,9 @@ public:
virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
- ZipFs m_ZipFs;
- std::filesystem::path m_Directory;
- HttpStatusService& m_StatusService;
+ std::unique_ptr<ZipFs> m_ZipFs;
+ std::filesystem::path m_Directory;
+ HttpStatusService& m_StatusService;
};
} // namespace zen
diff --git a/src/zenserver/frontend/zipfs.cpp b/src/zenserver/frontend/zipfs.cpp
index f9c2bc8ff..42df0520f 100644
--- a/src/zenserver/frontend/zipfs.cpp
+++ b/src/zenserver/frontend/zipfs.cpp
@@ -149,13 +149,25 @@ ZipFs::ZipFs(IoBuffer&& Buffer)
IoBuffer
ZipFs::GetFile(const std::string_view& FileName) const
{
- FileMap::iterator Iter = m_Files.find(FileName);
- if (Iter == m_Files.end())
{
- return {};
+ RwLock::SharedLockScope _(m_FilesLock);
+
+ FileMap::const_iterator Iter = m_Files.find(FileName);
+ if (Iter == m_Files.end())
+ {
+ return {};
+ }
+
+ const FileItem& Item = Iter->second;
+ if (Item.GetSize() > 0)
+ {
+ return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize());
+ }
}
- FileItem& Item = Iter->second;
+ RwLock::ExclusiveLockScope _(m_FilesLock);
+
+ FileItem& Item = m_Files.find(FileName)->second;
if (Item.GetSize() > 0)
{
return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize());
diff --git a/src/zenserver/frontend/zipfs.h b/src/zenserver/frontend/zipfs.h
index 1fa7da451..645121693 100644
--- a/src/zenserver/frontend/zipfs.h
+++ b/src/zenserver/frontend/zipfs.h
@@ -3,23 +3,23 @@
#pragma once
#include <zencore/iobuffer.h>
+#include <zencore/thread.h>
#include <unordered_map>
namespace zen {
-//////////////////////////////////////////////////////////////////////////
class ZipFs
{
public:
- ZipFs() = default;
- ZipFs(IoBuffer&& Buffer);
+ explicit ZipFs(IoBuffer&& Buffer);
+
IoBuffer GetFile(const std::string_view& FileName) const;
- inline operator bool() const { return !m_Files.empty(); }
private:
using FileItem = MemoryView;
using FileMap = std::unordered_map<std::string_view, FileItem>;
+ mutable RwLock m_FilesLock;
FileMap mutable m_Files;
IoBuffer m_Buffer;
};
diff --git a/src/zenserver/hub/hubservice.cpp b/src/zenserver/hub/hubservice.cpp
index 4d9da3a57..a00446a75 100644
--- a/src/zenserver/hub/hubservice.cpp
+++ b/src/zenserver/hub/hubservice.cpp
@@ -151,6 +151,7 @@ struct StorageServerInstance
inline uint16_t GetBasePort() const { return m_ServerInstance.GetBasePort(); }
private:
+ void WakeLocked();
RwLock m_Lock;
std::string m_ModuleId;
std::atomic<bool> m_IsProvisioned{false};
@@ -211,7 +212,7 @@ StorageServerInstance::Provision()
if (m_IsHibernated)
{
- Wake();
+ WakeLocked();
}
else
{
@@ -294,9 +295,14 @@ StorageServerInstance::Hibernate()
void
StorageServerInstance::Wake()
{
- // Start server in-place using existing data
-
RwLock::ExclusiveLockScope _(m_Lock);
+ WakeLocked();
+}
+
+void
+StorageServerInstance::WakeLocked()
+{
+ // Start server in-place using existing data
if (!m_IsHibernated)
{
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp
index f5ba30616..bf7afcc02 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp
@@ -185,7 +185,7 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
{
const HttpRange& Range = Ranges.front();
const uint64_t BlobSize = Blob.GetSize();
- const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0;
+ const uint64_t MaxBlobSize = Range.Start < BlobSize ? BlobSize - Range.Start : 0;
const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize);
if (Range.Start + RangeSize > BlobSize)
{
diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp
index 04a0781d3..aa37e75fe 100644
--- a/src/zenstore/buildstore/buildstore.cpp
+++ b/src/zenstore/buildstore/buildstore.cpp
@@ -266,13 +266,12 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload)
m_BlobLookup.insert({BlobHash, NewBlobIndex});
}
- m_LastAccessTimeUpdateCount++;
if (m_TrackedBlobKeys)
{
m_TrackedBlobKeys->push_back(BlobHash);
if (MetadataHash != IoHash::Zero)
{
- m_TrackedBlobKeys->push_back(BlobHash);
+ m_TrackedBlobKeys->push_back(MetadataHash);
}
}
}
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index ead7e4f3a..b73b3e6fb 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -626,7 +626,7 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& B
return false;
}
- const uint64_t ExpectedEntryCount = (FileSize - sizeof(sizeof(BucketMetaHeader))) / sizeof(ManifestData);
+ const uint64_t ExpectedEntryCount = (FileSize - sizeof(BucketMetaHeader)) / sizeof(ManifestData);
if (Header.EntryCount > ExpectedEntryCount)
{
ZEN_WARN(
@@ -1057,7 +1057,7 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const
return 0;
}
- const uint64_t ExpectedEntryCount = (FileSize - sizeof(sizeof(cache::impl::CacheBucketIndexHeader))) / sizeof(DiskIndexEntry);
+ const uint64_t ExpectedEntryCount = (FileSize - sizeof(cache::impl::CacheBucketIndexHeader)) / sizeof(DiskIndexEntry);
if (Header.EntryCount > ExpectedEntryCount)
{
return 0;
diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp
index 94abcf547..e1fd0a3e6 100644
--- a/src/zenstore/cache/cacherpc.cpp
+++ b/src/zenstore/cache/cacherpc.cpp
@@ -966,7 +966,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb
}
else
{
- ResponseObject.AddBool(true);
+ ResponseObject.AddBool(false);
}
}
ResponseObject.EndArray();
diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp
index 52b494e45..4e8475293 100644
--- a/src/zenstore/cache/structuredcachestore.cpp
+++ b/src/zenstore/cache/structuredcachestore.cpp
@@ -608,7 +608,10 @@ ZenCacheStore::GetBatch::Commit()
m_CacheStore.m_HitCount++;
OpScope.SetBytes(Result.Value.GetSize());
}
- m_CacheStore.m_MissCount++;
+ else
+ {
+ m_CacheStore.m_MissCount++;
+ }
}
}
}
diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp
index ed017988f..7402d92d3 100644
--- a/src/zenstore/cas.cpp
+++ b/src/zenstore/cas.cpp
@@ -300,12 +300,12 @@ GetCompactCasResults(CasContainerStrategy& Strategy,
};
static void
-GetFileCasResults(FileCasStrategy& Strategy,
- CasStore::InsertMode Mode,
- std::span<IoBuffer> Data,
- std::span<IoHash> ChunkHashes,
- std::span<size_t> Indexes,
- std::vector<CasStore::InsertResult> Results)
+GetFileCasResults(FileCasStrategy& Strategy,
+ CasStore::InsertMode Mode,
+ std::span<IoBuffer> Data,
+ std::span<IoHash> ChunkHashes,
+ std::span<size_t> Indexes,
+ std::vector<CasStore::InsertResult>& Results)
{
for (size_t Index : Indexes)
{