aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore4
-rw-r--r--xmake.lua50
-rw-r--r--zenserver/compute/apply.cpp31
-rw-r--r--zenserver/compute/apply.h2
-rw-r--r--zenserver/config.cpp180
-rw-r--r--zenserver/config.h44
-rw-r--r--zenserver/upstream/upstreamapply.cpp40
-rw-r--r--zenserver/zenserver.cpp101
-rw-r--r--zenutil/zenserverprocess.cpp2
9 files changed, 284 insertions, 170 deletions
diff --git a/.gitignore b/.gitignore
index bf237a7b3..204a2c274 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,9 +20,12 @@
x64/
x86/
bld/
+build/
[Bb]in/
[Oo]bj/
[Ll]og/
+vsxmake20*/
+vs20*/
# Visual Studio Code options directory
.vscode/
@@ -217,6 +220,7 @@ __pycache__/
.minio_data/
.test/
.xmake/
+.gdb_history
# Tags
TAGS
diff --git a/xmake.lua b/xmake.lua
index ced2b55ce..49989615c 100644
--- a/xmake.lua
+++ b/xmake.lua
@@ -73,3 +73,53 @@ includes("zenstore", "zenstore-test")
includes("zenutil")
includes("zenserver", "zenserver-test")
includes("zen")
+
+task("bundle")
+ on_run(function()
+ import("detect.tools.find_zip")
+ import("detect.tools.find_7z")
+
+ -- copy files
+ local dirs = {
+ binaries = "./build/windows/x64/release",
+ bundles = "./build/bundles",
+ bundle = "./build/bundles/zenzerver-win64"
+ }
+
+ local files = {
+ dirs.binaries .. "/zenserver.exe",
+ dirs.binaries .. "/zenserver.pdb",
+ "./vcpkg_installed/x64-windows-static/tools/sentry-native/crashpad_handler.exe"
+ }
+
+ os.mkdir(dirs.bundles)
+ os.mkdir(dirs.bundle)
+
+ for _,file in ipairs(files) do
+ printf("copy '%s' -> '%s'\n", file, dirs.bundle)
+ os.cp(file, dirs.bundle)
+ end
+
+ -- create archive
+ local bundle_name = "zenserver-win64.zip"
+
+ local zip_cmd = find_7z()
+ assert(zip_cmd)
+
+ local zip_args = {}
+ table.insert(zip_args, "a")
+ table.insert(zip_args, dirs.bundles .. "/" .. bundle_name)
+ table.insert(zip_args, dirs.bundle .. "/*.*")
+
+ printf("creating bundle '%s'...", dirs.bundles .. "/" .. bundle_name)
+ os.runv(zip_cmd, zip_args)
+ os.rm(dirs.bundle)
+
+ printf(" Ok!")
+ end)
+
+ set_menu {
+ usage = "xmake bundle",
+ description = "Create zip bundle from binaries",
+ options = {}
+ }
diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp
index 1f18b054f..ae4dd4528 100644
--- a/zenserver/compute/apply.cpp
+++ b/zenserver/compute/apply.cpp
@@ -578,9 +578,13 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
if (NeedList.empty())
{
// We already have everything
+ CbObject Output;
+ HttpResponseCode ResponseCode = ExecActionUpstream(Worker, RequestObject, Output);
- CbObject Output = ExecActionUpstream(Worker, RequestObject);
-
+ if (ResponseCode != HttpResponseCode::OK)
+ {
+ return HttpReq.WriteResponse(ResponseCode);
+ }
return HttpReq.WriteResponse(HttpResponseCode::OK, Output);
}
@@ -638,8 +642,13 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
zen::NiceBytes(TotalNewBytes),
NewAttachmentCount);
- CbObject Output = ExecActionUpstream(Worker, ActionObj);
+ CbObject Output;
+ HttpResponseCode ResponseCode = ExecActionUpstream(Worker, ActionObj, Output);
+ if (ResponseCode != HttpResponseCode::OK)
+ {
+ return HttpReq.WriteResponse(ResponseCode);
+ }
return HttpReq.WriteResponse(HttpResponseCode::OK, Output);
}
break;
@@ -881,8 +890,8 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action)
return OutputPackage;
}
-CbObject
-HttpFunctionService::ExecActionUpstream(const WorkerDesc& Worker, CbObject Action)
+HttpResponseCode
+HttpFunctionService::ExecActionUpstream(const WorkerDesc& Worker, CbObject Action, CbObject& Object)
{
const IoHash WorkerId = Worker.Descriptor.GetHash();
const IoHash ActionId = Action.GetHash();
@@ -895,14 +904,19 @@ HttpFunctionService::ExecActionUpstream(const WorkerDesc& Worker, CbObject Actio
if (!EnqueueResult.Success)
{
- throw std::runtime_error("Error enqueuing upstream task");
+ ZEN_ERROR(
+ "Error enqueuing upstream Action {}/{}",
+ WorkerId.ToHexString(),
+ ActionId.ToHexString());
+ return HttpResponseCode::InternalServerError;
}
CbObjectWriter Writer;
Writer.AddHash("worker", WorkerId);
Writer.AddHash("action", ActionId);
- return std::move(Writer.Save());
+ Object = std::move(Writer.Save());
+ return HttpResponseCode::OK;
}
HttpResponseCode
@@ -932,8 +946,7 @@ HttpFunctionService::ExecActionUpstreamResult(const IoHash& WorkerId, const IoHa
Completed.Error.Reason,
Completed.Error.ErrorCode);
- throw std::runtime_error(
- "Action {}/{} failed"_format(WorkerId.ToHexString(), ActionId.ToHexString()).c_str());
+ return HttpResponseCode::InternalServerError;
}
ZEN_INFO("Action {}/{} completed with {} attachments ({} compressed, {} uncompressed)",
diff --git a/zenserver/compute/apply.h b/zenserver/compute/apply.h
index 15cda4750..0d6edf119 100644
--- a/zenserver/compute/apply.h
+++ b/zenserver/compute/apply.h
@@ -46,7 +46,7 @@ private:
[[nodiscard]] std::filesystem::path CreateNewSandbox();
[[nodiscard]] CbPackage ExecAction(const WorkerDesc& Worker, CbObject Action);
- [[nodiscard]] CbObject ExecActionUpstream(const WorkerDesc& Worker, CbObject Action);
+ [[nodiscard]] HttpResponseCode ExecActionUpstream(const WorkerDesc& Worker, CbObject Action, CbObject& Object);
[[nodiscard]] HttpResponseCode ExecActionUpstreamResult(const IoHash& WorkerId, const IoHash& ActionId, CbPackage& Package);
RwLock m_WorkerLock;
diff --git a/zenserver/config.cpp b/zenserver/config.cpp
index aa61fa5a8..0a812b5a2 100644
--- a/zenserver/config.cpp
+++ b/zenserver/config.cpp
@@ -112,8 +112,10 @@ ParseUpstreamCachePolicy(std::string_view Options)
}
}
+void ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions);
+
void
-ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, ZenServiceConfig& ServiceConfig)
+ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
{
#if ZEN_WITH_HTTPSYS
const char* DefaultHttp = "httpsys";
@@ -124,22 +126,23 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
cxxopts::Options options("zenserver", "Zen Server");
options.add_options()("dedicated",
"Enable dedicated server mode",
- cxxopts::value<bool>(GlobalOptions.IsDedicated)->default_value("false"));
- options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(GlobalOptions.IsDebug)->default_value("false"));
+ cxxopts::value<bool>(ServerOptions.IsDedicated)->default_value("false"));
+ options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(ServerOptions.IsDebug)->default_value("false"));
options.add_options()("help", "Show command line help");
- options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(GlobalOptions.IsTest)->default_value("false"));
- options.add_options()("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(GlobalOptions.LogId));
- options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::filesystem::path>(GlobalOptions.DataDir));
- options.add_options()("content-dir", "Frontend content directory", cxxopts::value<std::filesystem::path>(GlobalOptions.ContentDir));
- options.add_options()("abslog", "Path to log file", cxxopts::value<std::filesystem::path>(GlobalOptions.AbsLogFile));
+ options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(ServerOptions.IsTest)->default_value("false"));
+ options.add_options()("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId));
+ options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::filesystem::path>(ServerOptions.DataDir));
+ options.add_options()("content-dir", "Frontend content directory", cxxopts::value<std::filesystem::path>(ServerOptions.ContentDir));
+ options.add_options()("abslog", "Path to log file", cxxopts::value<std::filesystem::path>(ServerOptions.AbsLogFile));
+ options.add_options()("config", "Path to Lua config file", cxxopts::value<std::filesystem::path>(ServerOptions.ConfigFile));
options
- .add_option("lifetime", "", "owner-pid", "Specify owning process id", cxxopts::value<int>(GlobalOptions.OwnerPid), "<identifier>");
+ .add_option("lifetime", "", "owner-pid", "Specify owning process id", cxxopts::value<int>(ServerOptions.OwnerPid), "<identifier>");
options.add_option("lifetime",
"",
"child-id",
"Specify id which can be used to signal parent",
- cxxopts::value<std::string>(GlobalOptions.ChildId),
+ cxxopts::value<std::string>(ServerOptions.ChildId),
"<identifier>");
#if ZEN_PLATFORM_WINDOWS
@@ -147,13 +150,13 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
"",
"install",
"Install zenserver as a Windows service",
- cxxopts::value<bool>(GlobalOptions.InstallService),
+ cxxopts::value<bool>(ServerOptions.InstallService),
"");
options.add_option("lifetime",
"",
"uninstall",
"Uninstall zenserver as a Windows service",
- cxxopts::value<bool>(GlobalOptions.UninstallService),
+ cxxopts::value<bool>(ServerOptions.UninstallService),
"");
#endif
@@ -161,14 +164,14 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
"",
"http",
"Select HTTP server implementation (asio|httpsys|null)",
- cxxopts::value<std::string>(GlobalOptions.HttpServerClass)->default_value(DefaultHttp),
+ cxxopts::value<std::string>(ServerOptions.HttpServerClass)->default_value(DefaultHttp),
"<http class>");
options.add_option("network",
"p",
"port",
"Select HTTP port",
- cxxopts::value<int>(GlobalOptions.BasePort)->default_value("1337"),
+ cxxopts::value<int>(ServerOptions.BasePort)->default_value("1337"),
"<port number>");
#if ZEN_ENABLE_MESH
@@ -176,7 +179,7 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
"m",
"mesh",
"Enable mesh network",
- cxxopts::value<bool>(ServiceConfig.MeshEnabled)->default_value("false"),
+ cxxopts::value<bool>(ServerOptions.MeshEnabled)->default_value("false"),
"");
#endif
@@ -184,7 +187,7 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
"",
"crash",
"Simulate a crash",
- cxxopts::value<bool>(ServiceConfig.ShouldCrash)->default_value("false"),
+ cxxopts::value<bool>(ServerOptions.ShouldCrash)->default_value("false"),
"");
std::string UpstreamCachePolicyOptions;
@@ -199,105 +202,105 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
"",
"upstream-jupiter-url",
"URL to a Jupiter instance",
- cxxopts::value<std::string>(ServiceConfig.UpstreamCacheConfig.JupiterConfig.Url)->default_value(""),
+ cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.Url)->default_value(""),
"");
options.add_option("cache",
"",
"upstream-jupiter-oauth-url",
"URL to the OAuth provier",
- cxxopts::value<std::string>(ServiceConfig.UpstreamCacheConfig.JupiterConfig.OAuthProvider)->default_value(""),
+ cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthProvider)->default_value(""),
"");
options.add_option("cache",
"",
"upstream-jupiter-oauth-clientid",
"The OAuth client ID",
- cxxopts::value<std::string>(ServiceConfig.UpstreamCacheConfig.JupiterConfig.OAuthClientId)->default_value(""),
+ cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientId)->default_value(""),
"");
options.add_option("cache",
"",
"upstream-jupiter-oauth-clientsecret",
"The OAuth client secret",
- cxxopts::value<std::string>(ServiceConfig.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret)->default_value(""),
+ cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret)->default_value(""),
"");
options.add_option("cache",
"",
"upstream-jupiter-namespace",
"The Common Blob Store API namespace",
- cxxopts::value<std::string>(ServiceConfig.UpstreamCacheConfig.JupiterConfig.Namespace)->default_value(""),
+ cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.Namespace)->default_value(""),
"");
options.add_option("cache",
"",
"upstream-jupiter-namespace-ddc",
"The lecacy DDC namespace",
- cxxopts::value<std::string>(ServiceConfig.UpstreamCacheConfig.JupiterConfig.DdcNamespace)->default_value(""),
+ cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.DdcNamespace)->default_value(""),
"");
options.add_option("cache",
"",
"upstream-jupiter-prod",
"Enable Jupiter upstream caching using production settings",
- cxxopts::value<bool>(ServiceConfig.UpstreamCacheConfig.JupiterConfig.UseProductionSettings)->default_value("false"),
+ cxxopts::value<bool>(ServerOptions.UpstreamCacheConfig.JupiterConfig.UseProductionSettings)->default_value("false"),
"");
options.add_option("cache",
"",
"upstream-jupiter-dev",
"Enable Jupiter upstream caching using development settings",
- cxxopts::value<bool>(ServiceConfig.UpstreamCacheConfig.JupiterConfig.UseDevelopmentSettings)->default_value("false"),
+ cxxopts::value<bool>(ServerOptions.UpstreamCacheConfig.JupiterConfig.UseDevelopmentSettings)->default_value("false"),
"");
options.add_option("cache",
"",
"upstream-jupiter-use-legacy-ddc",
"Whether to store derived data using the legacy endpoint",
- cxxopts::value<bool>(ServiceConfig.UpstreamCacheConfig.JupiterConfig.UseLegacyDdc)->default_value("false"),
+ cxxopts::value<bool>(ServerOptions.UpstreamCacheConfig.JupiterConfig.UseLegacyDdc)->default_value("false"),
"");
options.add_option("cache",
"",
"upstream-zen-url",
"URL to remote Zen server. Use a comma separated list to choose the one with the best latency.",
- cxxopts::value<std::vector<std::string>>(ServiceConfig.UpstreamCacheConfig.ZenConfig.Urls)->default_value(""),
+ cxxopts::value<std::vector<std::string>>(ServerOptions.UpstreamCacheConfig.ZenConfig.Urls)->default_value(""),
"");
options.add_option("cache",
"",
"upstream-zen-dns",
"DNS that resolves to one or more Zen server instance(s)",
- cxxopts::value<std::vector<std::string>>(ServiceConfig.UpstreamCacheConfig.ZenConfig.Dns)->default_value(""),
+ cxxopts::value<std::vector<std::string>>(ServerOptions.UpstreamCacheConfig.ZenConfig.Dns)->default_value(""),
"");
options.add_option("cache",
"",
"upstream-thread-count",
"Number of threads used for upstream procsssing",
- cxxopts::value<int32_t>(ServiceConfig.UpstreamCacheConfig.UpstreamThreadCount)->default_value("4"),
+ cxxopts::value<int32_t>(ServerOptions.UpstreamCacheConfig.UpstreamThreadCount)->default_value("4"),
"");
options.add_option("cache",
"",
"upstream-stats",
"Collect performance metrics for upstream endpoints",
- cxxopts::value<bool>(ServiceConfig.UpstreamCacheConfig.StatsEnabled)->default_value("false"),
+ cxxopts::value<bool>(ServerOptions.UpstreamCacheConfig.StatsEnabled)->default_value("false"),
"");
options.add_option("cache",
"",
"upstream-connect-timeout-ms",
"Connect timeout in millisecond(s). Default 5000 ms.",
- cxxopts::value<int32_t>(ServiceConfig.UpstreamCacheConfig.ConnectTimeoutMilliseconds)->default_value("5000"),
+ cxxopts::value<int32_t>(ServerOptions.UpstreamCacheConfig.ConnectTimeoutMilliseconds)->default_value("5000"),
"");
options.add_option("cache",
"",
"upstream-timeout-ms",
"Timeout in millisecond(s). Default 0 ms",
- cxxopts::value<int32_t>(ServiceConfig.UpstreamCacheConfig.TimeoutMilliseconds)->default_value("0"),
+ cxxopts::value<int32_t>(ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds)->default_value("0"),
"");
try
@@ -312,7 +315,16 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
exit(0);
}
- ServiceConfig.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions);
+ ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions);
+
+ if (!ServerOptions.ConfigFile.empty())
+ {
+ ParseConfigFile(ServerOptions.ConfigFile, ServerOptions);
+ }
+ else
+ {
+ ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions);
+ }
}
catch (cxxopts::OptionParseException& e)
{
@@ -321,32 +333,25 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
throw;
}
- if (GlobalOptions.DataDir.empty())
+ if (ServerOptions.DataDir.empty())
{
- GlobalOptions.DataDir = PickDefaultStateDirectory();
+ ServerOptions.DataDir = PickDefaultStateDirectory();
}
}
void
-ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& ServiceConfig)
+ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions)
{
using namespace fmt::literals;
- std::filesystem::path ConfigScript = DataRoot / "zen_cfg.lua";
- zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(ConfigScript.native().c_str());
+ zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(Path.native().c_str());
if (LuaScript)
{
sol::state lua;
- // Provide some context to help derive defaults
- lua.set("dataroot", DataRoot.native());
-
lua.open_libraries(sol::lib::base);
- // We probably want to limit the scope of this so the script won't see
- // any more than it needs to
-
lua.set_function("getenv", [&](const std::string env) -> sol::object {
std::wstring EnvVarValue;
size_t RequiredSize = 0;
@@ -378,14 +383,50 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv
}
catch (std::exception& e)
{
- ZEN_ERROR("config failure: {}", e.what());
+ throw std::runtime_error("failed to load config script ('{}'): {}"_format(Path, e.what()).c_str());
+ }
- throw std::runtime_error("failed to run global config script ('{}'): {}"_format(ConfigScript, e.what()).c_str());
+ if (sol::optional<sol::table> ServerConfig = lua["server"])
+ {
+ if (ServerOptions.DataDir.empty())
+ {
+ if (sol::optional<std::string> Opt = ServerConfig.value()["datadir"])
+ {
+ ServerOptions.DataDir = Opt.value();
+ }
+ }
+
+ if (ServerOptions.ContentDir.empty())
+ {
+ if (sol::optional<std::string> Opt = ServerConfig.value()["contentdir"])
+ {
+ ServerOptions.ContentDir = Opt.value();
+ }
+ }
+
+ if (ServerOptions.AbsLogFile.empty())
+ {
+ if (sol::optional<std::string> Opt = ServerConfig.value()["abslog"])
+ {
+ ServerOptions.AbsLogFile = Opt.value();
+ }
+ }
+
+ ServerOptions.IsDebug = ServerConfig->get_or("debug", ServerOptions.IsDebug);
}
+ if (sol::optional<sol::table> NetworkConfig = lua["network"])
+ {
+ if (sol::optional<std::string> Opt = NetworkConfig.value()["httpserverclass"])
+ {
+ ServerOptions.HttpServerClass = Opt.value();
+ }
+
+ ServerOptions.BasePort = NetworkConfig->get_or<int>("port", ServerOptions.BasePort);
#if ZEN_ENABLE_MESH
- ServiceConfig.MeshEnabled = lua["mesh"]["enable"].get_or(ServiceConfig.MeshEnabled);
+ ServerOptions.MeshEnabled = NetworkConfig->get_or<bool>("meshenabled", ServerOptions.MeshEnabled);
#endif
+ }
auto UpdateStringValueFromConfig = [](const sol::table& Table, std::string_view Key, std::string& OutValue) {
// Update the specified config value unless it has been set, i.e. from command line
@@ -395,56 +436,69 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv
}
};
- if (sol::optional<sol::table> StructuredCacheConfig = lua["structuredcache"])
+ if (sol::optional<sol::table> StructuredCacheConfig = lua["cache"])
{
- ServiceConfig.StructuredCacheEnabled = StructuredCacheConfig->get_or("enable", ServiceConfig.StructuredCacheEnabled);
+ ServerOptions.StructuredCacheEnabled = StructuredCacheConfig->get_or("enable", ServerOptions.StructuredCacheEnabled);
if (auto UpstreamConfig = StructuredCacheConfig->get<sol::optional<sol::table>>("upstream"))
{
- std::string Policy = UpstreamConfig->get_or("policy", std::string());
- ServiceConfig.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(Policy);
- ServiceConfig.UpstreamCacheConfig.UpstreamThreadCount = UpstreamConfig->get_or("upstreamthreadcount", 4);
+ std::string Policy = UpstreamConfig->get_or("policy", std::string());
+ ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(Policy);
+ ServerOptions.UpstreamCacheConfig.UpstreamThreadCount =
+ UpstreamConfig->get_or("upstreamthreadcount", ServerOptions.UpstreamCacheConfig.UpstreamThreadCount);
if (auto JupiterConfig = UpstreamConfig->get<sol::optional<sol::table>>("jupiter"))
{
UpdateStringValueFromConfig(JupiterConfig.value(),
std::string_view("url"),
- ServiceConfig.UpstreamCacheConfig.JupiterConfig.Url);
+ ServerOptions.UpstreamCacheConfig.JupiterConfig.Url);
UpdateStringValueFromConfig(JupiterConfig.value(),
std::string_view("oauthprovider"),
- ServiceConfig.UpstreamCacheConfig.JupiterConfig.OAuthProvider);
+ ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthProvider);
UpdateStringValueFromConfig(JupiterConfig.value(),
std::string_view("oauthclientid"),
- ServiceConfig.UpstreamCacheConfig.JupiterConfig.OAuthClientId);
+ ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientId);
UpdateStringValueFromConfig(JupiterConfig.value(),
std::string_view("oauthclientsecret"),
- ServiceConfig.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret);
+ ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret);
UpdateStringValueFromConfig(JupiterConfig.value(),
std::string_view("namespace"),
- ServiceConfig.UpstreamCacheConfig.JupiterConfig.Namespace);
+ ServerOptions.UpstreamCacheConfig.JupiterConfig.Namespace);
UpdateStringValueFromConfig(JupiterConfig.value(),
std::string_view("ddcnamespace"),
- ServiceConfig.UpstreamCacheConfig.JupiterConfig.DdcNamespace);
+ ServerOptions.UpstreamCacheConfig.JupiterConfig.DdcNamespace);
- ServiceConfig.UpstreamCacheConfig.JupiterConfig.UseDevelopmentSettings =
+ ServerOptions.UpstreamCacheConfig.JupiterConfig.UseDevelopmentSettings =
JupiterConfig->get_or("usedevelopmentsettings",
- ServiceConfig.UpstreamCacheConfig.JupiterConfig.UseDevelopmentSettings);
+ ServerOptions.UpstreamCacheConfig.JupiterConfig.UseDevelopmentSettings);
- ServiceConfig.UpstreamCacheConfig.JupiterConfig.UseLegacyDdc =
- JupiterConfig->get_or("uselegacyddc", ServiceConfig.UpstreamCacheConfig.JupiterConfig.UseLegacyDdc);
+ ServerOptions.UpstreamCacheConfig.JupiterConfig.UseLegacyDdc =
+ JupiterConfig->get_or("uselegacyddc", ServerOptions.UpstreamCacheConfig.JupiterConfig.UseLegacyDdc);
};
if (auto ZenConfig = UpstreamConfig->get<sol::optional<sol::table>>("zen"))
{
if (auto Url = ZenConfig.value().get<sol::optional<std::string>>("url"))
{
- ServiceConfig.UpstreamCacheConfig.ZenConfig.Urls.push_back(Url.value());
+ ServerOptions.UpstreamCacheConfig.ZenConfig.Urls.push_back(Url.value());
}
else if (auto Urls = ZenConfig.value().get<sol::optional<sol::table>>("url"))
{
for (const auto& Kv : Urls.value())
{
- ServiceConfig.UpstreamCacheConfig.ZenConfig.Urls.push_back(Kv.second.as<std::string>());
+ ServerOptions.UpstreamCacheConfig.ZenConfig.Urls.push_back(Kv.second.as<std::string>());
+ }
+ }
+
+ if (auto Dns = ZenConfig.value().get<sol::optional<std::string>>("dns"))
+ {
+ ServerOptions.UpstreamCacheConfig.ZenConfig.Dns.push_back(Dns.value());
+ }
+ else if (auto DnsArray = ZenConfig.value().get<sol::optional<sol::table>>("dns"))
+ {
+ for (const auto& Kv : DnsArray.value())
+ {
+ ServerOptions.UpstreamCacheConfig.ZenConfig.Dns.push_back(Kv.second.as<std::string>());
}
}
}
diff --git a/zenserver/config.h b/zenserver/config.h
index 36156a570..4229c5bcc 100644
--- a/zenserver/config.h
+++ b/zenserver/config.h
@@ -9,23 +9,6 @@
# define ZEN_ENABLE_MESH 0
#endif
-struct ZenServerOptions
-{
- bool IsDebug = false;
- bool IsTest = false;
- bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements
- int BasePort = 1337; // Service listen port (used for both UDP and TCP)
- int OwnerPid = 0; // Parent process id (zero for standalone)
- std::string ChildId; // Id assigned by parent process (used for lifetime management)
- bool InstallService = false; // Flag used to initiate service install (temporary)
- bool UninstallService = false; // Flag used to initiate service uninstall (temporary)
- std::string LogId; // Id for tagging log output
- std::filesystem::path DataDir; // Root directory for state (used for testing)
- std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental)
- std::string HttpServerClass; // Choice of HTTP server implementation
- std::filesystem::path AbsLogFile; // Absolute path to main log file
-};
-
struct ZenUpstreamJupiterConfig
{
std::string Url;
@@ -64,16 +47,29 @@ struct ZenUpstreamCacheConfig
bool StatsEnabled = false;
};
-struct ZenServiceConfig
+struct ZenServerOptions
{
- bool StructuredCacheEnabled = true;
- bool ShouldCrash = false; // Option for testing crash handling
- bool IsFirstRun = false;
+ ZenUpstreamCacheConfig UpstreamCacheConfig;
+ std::filesystem::path DataDir; // Root directory for state (used for testing)
+ std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental)
+ std::filesystem::path AbsLogFile; // Absolute path to main log file
+ std::filesystem::path ConfigFile; // Path to Lua config file
+ std::string ChildId; // Id assigned by parent process (used for lifetime management)
+ std::string LogId; // Id for tagging log output
+ std::string HttpServerClass; // Choice of HTTP server implementation
+ int BasePort = 1337; // Service listen port (used for both UDP and TCP)
+ int OwnerPid = 0; // Parent process id (zero for standalone)
+ bool InstallService = false; // Flag used to initiate service install (temporary)
+ bool UninstallService = false; // Flag used to initiate service uninstall (temporary)
+ bool IsDebug = false;
+ bool IsTest = false;
+ bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements
+ bool StructuredCacheEnabled = true;
+ bool ShouldCrash = false; // Option for testing crash handling
+ bool IsFirstRun = false;
#if ZEN_ENABLE_MESH
bool MeshEnabled = false; // Experimental p2p mesh discovery
#endif
- ZenUpstreamCacheConfig UpstreamCacheConfig;
};
-void ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, ZenServiceConfig& ServiceConfig);
-void ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& ServiceConfig);
+void ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions);
diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp
index 0118902a6..de646a5bb 100644
--- a/zenserver/upstream/upstreamapply.cpp
+++ b/zenserver/upstream/upstreamapply.cpp
@@ -14,6 +14,7 @@
#include <zencore/session.h>
#include <zencore/stats.h>
#include <zencore/stream.h>
+#include <zencore/thread.h>
#include <zencore/timer.h>
#include <zenstore/cas.h>
@@ -36,6 +37,9 @@ namespace zen {
using namespace std::literals;
+static const IoBuffer EmptyBuffer;
+static const IoHash EmptyBufferId = IoHash::HashBuffer(EmptyBuffer);
+
namespace detail {
class HordeUpstreamApplyEndpoint final : public UpstreamApplyEndpoint
@@ -704,6 +708,8 @@ namespace detail {
[[nodiscard]] bool ProcessApplyKey(const UpstreamApplyRecord& ApplyRecord, UpstreamData& Data)
{
+ using namespace fmt::literals;
+
std::string ExecutablePath;
std::map<std::string, std::string> Environment;
std::set<std::filesystem::path> InputFiles;
@@ -735,6 +741,15 @@ namespace detail {
}
}
+ for (auto& It : ApplyRecord.WorkerDescriptor["dirs"sv])
+ {
+ std::string_view Directory = It.AsString();
+ std::string DummyFile = "{}/.zen_empty_file"_format(Directory);
+ InputFiles.insert(DummyFile);
+ Data.Blobs[EmptyBufferId] = EmptyBuffer;
+ InputFileHashes[DummyFile] = EmptyBufferId;
+ }
+
for (auto& It : ApplyRecord.WorkerDescriptor["environment"sv])
{
std::string_view Env = It.AsString();
@@ -809,18 +824,10 @@ namespace detail {
bool Exclusive = ApplyRecord.WorkerDescriptor["exclusive"sv].AsBool();
// TODO: Remove override when Horde accepts the UE style Host Platforms (Win64, Linux, Mac)
- std::string Condition;
- if (HostPlatform == "Win64" || HostPlatform == "Windows")
- {
- Condition = "OSFamily == 'Windows' && Pool == 'Win-RemoteExec'";
- }
- else if (HostPlatform == "Mac")
+ std::string Condition = "Platform == '{}'"_format(HostPlatform);
+ if (HostPlatform == "Win64")
{
- Condition = "OSFamily == 'MacOS'";
- }
- else
- {
- Condition = "OSFamily == '{}'"_format(HostPlatform);
+ Condition += " && Pool == 'Win-RemoteExec'";
}
std::map<std::string_view, int64_t> Resources;
@@ -1199,6 +1206,8 @@ public:
if (m_RunState.IsRunning)
{
+ m_ShutdownEvent.Reset();
+
for (uint32_t Idx = 0; Idx < m_Options.ThreadCount; Idx++)
{
m_UpstreamThreads.emplace_back(&DefaultUpstreamApply::ProcessUpstreamQueue, this);
@@ -1422,11 +1431,9 @@ private:
void ProcessUpstreamUpdates()
{
- const auto& UpdateSleep = std::chrono::seconds(m_Options.UpdatesInterval);
- for (;;)
+ const auto& UpdateSleep = std::chrono::milliseconds(m_Options.UpdatesInterval);
+ while (!m_ShutdownEvent.Wait(uint32_t(UpdateSleep.count())))
{
- std::this_thread::sleep_for(UpdateSleep);
-
if (!m_RunState.IsRunning)
{
break;
@@ -1489,6 +1496,8 @@ private:
{
if (m_RunState.Stop())
{
+ m_ShutdownEvent.Set();
+
m_UpstreamQueue.CompleteAdding();
for (std::thread& Thread : m_UpstreamThreads)
{
@@ -1536,6 +1545,7 @@ private:
UpstreamApplyTasks m_ApplyTasks;
std::mutex m_ApplyTasksMutex;
std::vector<std::unique_ptr<UpstreamApplyEndpoint>> m_Endpoints;
+ Event m_ShutdownEvent;
std::vector<std::thread> m_UpstreamThreads;
std::thread m_UpstreamUpdatesThread;
std::thread m_EndpointMonitorThread;
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index b36d84c22..b1989e1bc 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -151,12 +151,12 @@ namespace utils {
class ZenServer : public IHttpStatusProvider
{
public:
- void Initialize(ZenServiceConfig& ServiceConfig, const ZenServerOptions& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry)
+ void Initialize(const ZenServerOptions& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry)
{
using namespace fmt::literals;
m_ServerEntry = ServerEntry;
- m_DebugOptionForcedCrash = ServiceConfig.ShouldCrash;
+ m_DebugOptionForcedCrash = ServerOptions.ShouldCrash;
const int ParentPid = ServerOptions.OwnerPid;
if (ParentPid)
@@ -188,7 +188,7 @@ public:
throw std::runtime_error("Failed to create mutex '{}' - is another instance already running?"_format(MutexName).c_str());
}
- InitializeState(ServiceConfig);
+ InitializeState(ServerOptions);
m_HealthService.SetHealthInfo({.DataRoot = m_DataRoot,
.AbsLogPath = ServerOptions.AbsLogFile,
@@ -231,9 +231,9 @@ public:
zen::CreateDirectories(ApplySandboxDir);
m_HttpFunctionService = std::make_unique<zen::HttpFunctionService>(*m_CasStore, *m_CidStore, ApplySandboxDir);
- if (ServiceConfig.StructuredCacheEnabled)
+ if (ServerOptions.StructuredCacheEnabled)
{
- InitializeStructuredCache(ServiceConfig);
+ InitializeStructuredCache(ServerOptions);
}
else
{
@@ -241,7 +241,7 @@ public:
}
#if ZEN_ENABLE_MESH
- if (ServiceConfig.MeshEnabled)
+ if (ServerOptions.MeshEnabled)
{
StartMesh(BasePort);
}
@@ -285,8 +285,8 @@ public:
}
}
- void InitializeState(ZenServiceConfig& ServiceConfig);
- void InitializeStructuredCache(ZenServiceConfig& ServiceConfig);
+ void InitializeState(const ZenServerOptions& ServerOptions);
+ void InitializeStructuredCache(const ZenServerOptions& ServerOptions);
#if ZEN_ENABLE_MESH
void StartMesh(int BasePort)
@@ -529,7 +529,7 @@ ZenServer::OnReady()
}
void
-ZenServer::InitializeState(ZenServiceConfig& ServiceConfig)
+ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
{
// Check root manifest to deal with schema versioning
@@ -542,7 +542,7 @@ ZenServer::InitializeState(ZenServiceConfig& ServiceConfig)
if (ManifestData.ErrorCode)
{
- if (ServiceConfig.IsFirstRun)
+ if (ServerOptions.IsFirstRun)
{
ZEN_INFO("Initializing state at '{}'", m_DataRoot);
@@ -623,7 +623,7 @@ ZenServer::InitializeState(ZenServiceConfig& ServiceConfig)
}
void
-ZenServer::InitializeStructuredCache(ZenServiceConfig& ServiceConfig)
+ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
{
using namespace std::literals;
auto ValueOrDefault = [](std::string_view Value, std::string_view Default) { return Value.empty() ? Default : Value; };
@@ -632,13 +632,13 @@ ZenServer::InitializeStructuredCache(ZenServiceConfig& ServiceConfig)
m_CacheStore = std::make_unique<ZenCacheStore>(m_DataRoot / "cache");
std::unique_ptr<zen::UpstreamCache> UpstreamCache;
- if (ServiceConfig.UpstreamCacheConfig.CachePolicy != UpstreamCachePolicy::Disabled)
+ if (ServerOptions.UpstreamCacheConfig.CachePolicy != UpstreamCachePolicy::Disabled)
{
- const ZenUpstreamCacheConfig& UpstreamConfig = ServiceConfig.UpstreamCacheConfig;
+ const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig;
zen::UpstreamCacheOptions UpstreamOptions;
- UpstreamOptions.ReadUpstream = (uint8_t(ServiceConfig.UpstreamCacheConfig.CachePolicy) & uint8_t(UpstreamCachePolicy::Read)) != 0;
- UpstreamOptions.WriteUpstream = (uint8_t(ServiceConfig.UpstreamCacheConfig.CachePolicy) & uint8_t(UpstreamCachePolicy::Write)) != 0;
+ UpstreamOptions.ReadUpstream = (uint8_t(ServerOptions.UpstreamCacheConfig.CachePolicy) & uint8_t(UpstreamCachePolicy::Read)) != 0;
+ UpstreamOptions.WriteUpstream = (uint8_t(ServerOptions.UpstreamCacheConfig.CachePolicy) & uint8_t(UpstreamCachePolicy::Write)) != 0;
if (UpstreamConfig.UpstreamThreadCount < 32)
{
@@ -747,11 +747,7 @@ ZenServer::InitializeStructuredCache(ZenServiceConfig& ServiceConfig)
class ZenWindowsService : public WindowsService
{
public:
- ZenWindowsService(ZenServerOptions& GlobalOptions, ZenServiceConfig& ServiceConfig)
- : m_GlobalOptions(GlobalOptions)
- , m_ServiceConfig(ServiceConfig)
- {
- }
+ ZenWindowsService(ZenServerOptions& ServerOptions) : m_ServerOptions(ServerOptions) {}
ZenWindowsService(const ZenWindowsService&) = delete;
ZenWindowsService& operator=(const ZenWindowsService&) = delete;
@@ -759,8 +755,7 @@ public:
virtual int Run() override;
private:
- ZenServerOptions& m_GlobalOptions;
- ZenServiceConfig& m_ServiceConfig;
+ ZenServerOptions& m_ServerOptions;
zen::LockFile m_LockFile;
};
@@ -779,8 +774,7 @@ ZenWindowsService::Run()
auto _ = zen::MakeGuard([] { sentry_close(); });
#endif
- auto& GlobalOptions = m_GlobalOptions;
- auto& ServiceConfig = m_ServiceConfig;
+ auto& ServerOptions = m_ServerOptions;
try
{
@@ -788,13 +782,13 @@ ZenWindowsService::Run()
std::error_code Ec;
- std::filesystem::path LockFilePath = GlobalOptions.DataDir / ".lock";
+ std::filesystem::path LockFilePath = ServerOptions.DataDir / ".lock";
bool IsReady = false;
auto MakeLockData = [&] {
CbObjectWriter Cbo;
- Cbo << "pid" << _getpid() << "data" << ToUtf8(GlobalOptions.DataDir) << "port" << GlobalOptions.BasePort << "session_id"
+ Cbo << "pid" << _getpid() << "data" << ToUtf8(ServerOptions.DataDir) << "port" << ServerOptions.BasePort << "session_id"
<< GetSessionId() << "ready" << IsReady;
return Cbo.Save();
};
@@ -808,21 +802,15 @@ ZenWindowsService::Run()
std::exit(99);
}
- InitializeLogging(GlobalOptions);
-
- // Prototype config system, we'll see how this pans out
- //
- // TODO: we need to report any parse errors here
-
- ParseServiceConfig(GlobalOptions.DataDir, /* out */ ServiceConfig);
+ InitializeLogging(ServerOptions);
- ZEN_INFO(ZEN_APP_NAME " - starting on port {}, build '{}'", GlobalOptions.BasePort, BUILD_VERSION);
+ ZEN_INFO(ZEN_APP_NAME " - starting on port {}, build '{}'", ServerOptions.BasePort, BUILD_VERSION);
ZenServerState ServerState;
ServerState.Initialize();
ServerState.Sweep();
- ZenServerState::ZenServerEntry* Entry = ServerState.Lookup(GlobalOptions.BasePort);
+ ZenServerState::ZenServerEntry* Entry = ServerState.Lookup(ServerOptions.BasePort);
if (Entry)
{
@@ -830,34 +818,34 @@ ZenWindowsService::Run()
ZEN_WARN("Looks like there is already a process listening to this port (pid: {})", Entry->Pid);
- if (GlobalOptions.OwnerPid)
+ if (ServerOptions.OwnerPid)
{
- Entry->AddSponsorProcess(GlobalOptions.OwnerPid);
+ Entry->AddSponsorProcess(ServerOptions.OwnerPid);
std::exit(0);
}
}
- Entry = ServerState.Register(GlobalOptions.BasePort);
+ Entry = ServerState.Register(ServerOptions.BasePort);
- if (GlobalOptions.OwnerPid)
+ if (ServerOptions.OwnerPid)
{
- Entry->AddSponsorProcess(GlobalOptions.OwnerPid);
+ Entry->AddSponsorProcess(ServerOptions.OwnerPid);
}
std::unique_ptr<std::thread> ShutdownThread;
std::unique_ptr<zen::NamedEvent> ShutdownEvent;
zen::ExtendableStringBuilder<64> ShutdownEventName;
- ShutdownEventName << "Zen_" << GlobalOptions.BasePort << "_Shutdown";
+ ShutdownEventName << "Zen_" << ServerOptions.BasePort << "_Shutdown";
ShutdownEvent.reset(new zen::NamedEvent{ShutdownEventName});
ZenServer Server;
- Server.SetDataRoot(GlobalOptions.DataDir);
- Server.SetContentRoot(GlobalOptions.ContentDir);
- Server.SetTestMode(GlobalOptions.IsTest);
- Server.SetDedicatedMode(GlobalOptions.IsDedicated);
- Server.Initialize(ServiceConfig, GlobalOptions, Entry);
+ Server.SetDataRoot(ServerOptions.DataDir);
+ Server.SetContentRoot(ServerOptions.ContentDir);
+ Server.SetTestMode(ServerOptions.IsTest);
+ Server.SetDedicatedMode(ServerOptions.IsDedicated);
+ Server.Initialize(ServerOptions, Entry);
// Monitor shutdown signals
@@ -878,9 +866,9 @@ ZenWindowsService::Run()
m_LockFile.Update(MakeLockData(), Ec);
- if (!GlobalOptions.ChildId.empty())
+ if (!ServerOptions.ChildId.empty())
{
- zen::NamedEvent ParentEvent{GlobalOptions.ChildId};
+ zen::NamedEvent ParentEvent{ServerOptions.ChildId};
ParentEvent.Set();
}
});
@@ -938,25 +926,24 @@ main(int argc, char* argv[])
try
{
- ZenServerOptions GlobalOptions;
- ZenServiceConfig ServiceConfig;
- ParseGlobalCliOptions(argc, argv, GlobalOptions, ServiceConfig);
+ ZenServerOptions ServerOptions;
+ ParseCliOptions(argc, argv, ServerOptions);
- if (!std::filesystem::exists(GlobalOptions.DataDir))
+ if (!std::filesystem::exists(ServerOptions.DataDir))
{
- ServiceConfig.IsFirstRun = true;
- std::filesystem::create_directories(GlobalOptions.DataDir);
+ ServerOptions.IsFirstRun = true;
+ std::filesystem::create_directories(ServerOptions.DataDir);
}
#if ZEN_PLATFORM_WINDOWS
- if (GlobalOptions.InstallService)
+ if (ServerOptions.InstallService)
{
WindowsService::Install();
std::exit(0);
}
- if (GlobalOptions.UninstallService)
+ if (ServerOptions.UninstallService)
{
WindowsService::Delete();
@@ -964,7 +951,7 @@ main(int argc, char* argv[])
}
#endif
- ZenWindowsService App(GlobalOptions, ServiceConfig);
+ ZenWindowsService App(ServerOptions);
return App.ServiceMain();
}
catch (std::exception& Ex)
diff --git a/zenutil/zenserverprocess.cpp b/zenutil/zenserverprocess.cpp
index dbe27ef20..e366a917f 100644
--- a/zenutil/zenserverprocess.cpp
+++ b/zenutil/zenserverprocess.cpp
@@ -305,7 +305,7 @@ ZenServerState::ZenServerEntry::AddSponsorProcess(uint32_t PidToAdd)
if (PidEntry.load(std::memory_order::memory_order_relaxed) == 0)
{
uint32_t Expected = 0;
- if (PidEntry.compare_exchange_strong(Expected, uint16_t(PidToAdd)))
+ if (PidEntry.compare_exchange_strong(Expected, PidToAdd))
{
// Success!
return true;
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-30 03:31:16 +0000