diff options
Diffstat (limited to 'zenserver')
26 files changed, 400 insertions, 171 deletions
diff --git a/zenserver/admin/admin.cpp b/zenserver/admin/admin.cpp index 07211cbeb..44406fa70 100644 --- a/zenserver/admin/admin.cpp +++ b/zenserver/admin/admin.cpp @@ -1,7 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#pragma once - #include "admin.h" #include <zencore/compactbinarybuilder.h> diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 726bd7cdb..107af2333 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -10,7 +10,7 @@ #include <zencore/stream.h> #include <zencore/timer.h> #include <zenhttp/httpserver.h> -#include <zenstore/CAS.h> +#include <zenstore/cas.h> #include <zenutil/cache/cache.h> //#include "cachekey.h" @@ -155,7 +155,7 @@ void HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Bucket) { ZEN_UNUSED(Request, Bucket); - switch (auto Verb = Request.RequestVerb()) + switch (Request.RequestVerb()) { using enum HttpVerb; @@ -178,13 +178,16 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, return Request.WriteResponse(HttpResponseCode::NotFound); } break; + + default: + break; } } void HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy) { - switch (auto Verb = Request.RequestVerb()) + switch (Request.RequestVerb()) { using enum HttpVerb; @@ -445,7 +448,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (StoreUpstream) { ZEN_ASSERT(m_UpstreamCache); - auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kBinary, .CacheKey = {Ref.BucketSegment, Ref.HashKey}}); + m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kBinary, .Key = {Ref.BucketSegment, Ref.HashKey}}); } Request.WriteResponse(HttpResponseCode::Created); @@ -489,9 +492,9 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (StoreUpstream && !IsPartialRecord) { ZEN_ASSERT(m_UpstreamCache); - auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbObject, - .CacheKey = {Ref.BucketSegment, Ref.HashKey}, - .PayloadIds = std::move(ValidAttachments)}); + m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbObject, + .Key = {Ref.BucketSegment, Ref.HashKey}, + .PayloadIds = std::move(ValidAttachments)}); } Request.WriteResponse(HttpResponseCode::Created); @@ -571,9 +574,9 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (StoreUpstream && !IsPartialRecord) { ZEN_ASSERT(m_UpstreamCache); - auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbPackage, - .CacheKey = {Ref.BucketSegment, Ref.HashKey}, - .PayloadIds = std::move(ValidAttachments)}); + m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbPackage, + .Key = {Ref.BucketSegment, Ref.HashKey}, + .PayloadIds = std::move(ValidAttachments)}); } Request.WriteResponse(HttpResponseCode::Created); @@ -587,7 +590,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request void HttpStructuredCacheService::HandleCachePayloadRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy) { - switch (auto Verb = Request.RequestVerb()) + switch (Request.RequestVerb()) { using enum HttpVerb; @@ -620,7 +623,7 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques { Payload = UpstreamResult.Value; IoHash ChunkHash = IoHash::HashBuffer(Payload); - CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, ChunkHash); + /*CasStore::InsertResult Result = */ m_CasStore.InsertChunk(Payload, ChunkHash); InUpstreamCache = true; m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash); @@ -777,7 +780,7 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& void HttpStructuredCacheService::HandleRpcRequest(zen::HttpServerRequest& Request) { - switch (auto Verb = Request.RequestVerb()) + switch (Request.RequestVerb()) { using enum HttpVerb; @@ -928,8 +931,8 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req { ZEN_DEBUG("Uncompressed payload '{}' from upstream cache record '{}/{}'", HashView.AsHash(), - Params.CacheKey.Bucket, - Params.CacheKey.Hash); + Params.Key.Bucket, + Params.Key.Hash); Count.Invalid++; } } @@ -949,8 +952,8 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req if (CacheValue) { ZEN_DEBUG("HIT - '{}/{}' {} '{}' attachments '{}/{}/{}' (new/valid/total) (UPSTREAM)", - Params.CacheKey.Bucket, - Params.CacheKey.Hash, + Params.Key.Bucket, + Params.Key.Hash, NiceBytes(CacheValue.GetSize()), ToString(HttpContentType::kCbPackage), Count.New, @@ -960,7 +963,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req CacheValue.SetContentType(ZenContentType::kCbObject); CacheValues[Params.KeyIndex] = CacheValue; - m_CacheStore.Put(Params.CacheKey.Bucket, Params.CacheKey.Hash, {.Value = CacheValue}); + m_CacheStore.Put(Params.Key.Bucket, Params.Key.Hash, {.Value = CacheValue}); m_CacheStats.HitCount++; m_CacheStats.UpstreamHitCount++; @@ -968,7 +971,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req else { const bool IsPartial = Count.Valid != Count.Total; - ZEN_DEBUG("MISS - '{}/{}' {}", Params.CacheKey.Bucket, Params.CacheKey.Hash, IsPartial ? "(partial)"sv : ""sv); + ZEN_DEBUG("MISS - '{}/{}' {}", Params.Key.Bucket, Params.Key.Hash, IsPartial ? "(partial)"sv : ""sv); m_CacheStats.MissCount++; } }; @@ -1134,7 +1137,7 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re const auto OnCachePayloadGetComplete = [this, &ChunkRequests, &Chunks](CachePayloadGetCompleteParams&& Params) { if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Params.Payload))) { - auto InsertResult = m_CidStore.AddChunk(Compressed); + m_CidStore.AddChunk(Compressed); ZEN_DEBUG("HIT - '{}/{}/{}' {} ({})", Params.Request.Key.Bucket, diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 8b9ce8ff9..2b43cddeb 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -3,7 +3,6 @@ #include "structuredcachestore.h" #include <zencore/except.h> -#include <zencore/windows.h> #include <zencore/compactbinary.h> #include <zencore/compactbinarybuilder.h> @@ -23,6 +22,10 @@ #include <zenstore/cidstore.h> #include <zenstore/gc.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #include <concepts> #include <filesystem> #include <ranges> @@ -277,6 +280,9 @@ ZenCacheMemoryLayer::CacheBucket::GarbageCollect(GcContext& GcCtx) case ZenContentType::kBinary: break; + + default: + break; } } } @@ -471,17 +477,17 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo } void -ZenCacheDiskLayer::CacheBucket::BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey) +ZenCacheDiskLayer::CacheBucket::BuildPath(PathBuilderBase& Path, const IoHash& HashKey) { char HexString[sizeof(HashKey.Hash) * 2]; ToHexBytes(HashKey.Hash, sizeof HashKey.Hash, HexString); - Path.Append(m_BucketDir.c_str()); + Path.Append(m_BucketDir); Path.Append(L"/blob/"); Path.AppendAsciiRange(HexString, HexString + 3); - Path.Append(L"/"); + Path.AppendSeparator(); Path.AppendAsciiRange(HexString + 3, HexString + 5); - Path.Append(L"/"); + Path.AppendSeparator(); Path.AppendAsciiRange(HexString + 5, HexString + sizeof(HexString)); } @@ -502,12 +508,12 @@ ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc, Zen bool ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue) { - WideStringBuilder<128> DataFilePath; + PathBuilder<128> DataFilePath; BuildPath(DataFilePath, HashKey); RwLock::SharedLockScope ValueLock(LockForHash(HashKey)); - if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.c_str())) + if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.ToPath())) { OutValue.Value = Data; OutValue.Value.SetContentType(Loc.GetContentType()); @@ -708,7 +714,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c { RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey)); - WideStringBuilder<128> DataFilePath; + PathBuilder<128> DataFilePath; BuildPath(DataFilePath, HashKey); TemporaryFile DataFile; @@ -730,7 +736,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c // Move file into place (atomically) - std::filesystem::path FsPath{DataFilePath.c_str()}; + std::filesystem::path FsPath{DataFilePath.ToPath()}; DataFile.MoveTemporaryIntoPlace(FsPath, Ec); @@ -740,7 +746,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c do { - std::filesystem::path ParentPath = std::filesystem::path(DataFilePath.c_str()).parent_path(); + std::filesystem::path ParentPath = FsPath.parent_path(); CreateDirectories(ParentPath); DataFile.MoveTemporaryIntoPlace(FsPath, Ec); @@ -765,7 +771,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c if (Ec) { - throw std::system_error(Ec, "Failed to finalize file '{}'"_format(WideToUtf8(DataFilePath))); + throw std::system_error(Ec, "Failed to finalize file '{}'"_format(DataFilePath.ToUtf8())); } } @@ -907,11 +913,11 @@ ZenCacheDiskLayer::DiscoverBuckets() virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override { - Dirs.push_back(std::wstring(DirectoryName)); + Dirs.push_back((decltype(Dirs)::value_type)(DirectoryName)); return false; } - std::vector<std::wstring> Dirs; + std::vector<std::filesystem::path::string_type> Dirs; } Visit; Traversal.TraverseFileSystem(m_RootDir, Visit); @@ -920,11 +926,15 @@ ZenCacheDiskLayer::DiscoverBuckets() RwLock::ExclusiveLockScope _(m_Lock); - for (const std::wstring& BucketName : Visit.Dirs) + for (const auto& BucketName : Visit.Dirs) { // New bucket needs to be created +#if ZEN_PLATFORM_WINDOWS std::string BucketName8 = WideToUtf8(BucketName); +#else + const auto& BucketName8 = BucketName; +#endif if (auto It = m_Buckets.find(BucketName8); It != m_Buckets.end()) { diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index 0dfcbc5ca..e3013b53f 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -11,10 +11,9 @@ #include <zenstore/cas.h> #include <zenstore/caslog.h> -#pragma warning(push) -#pragma warning(disable : 4127) +ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_map.h> -#pragma warning(pop) +ZEN_THIRD_PARTY_INCLUDES_END #include <compare> #include <filesystem> @@ -22,7 +21,7 @@ namespace zen { -class WideStringBuilderBase; +class PathBuilderBase; class CasStore; /****************************************************************************** @@ -187,7 +186,7 @@ private: tsl::robin_map<IoHash, DiskLocation, IoHash::Hasher> m_Index; uint64_t m_WriteCursor = 0; - void BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey); + void BuildPath(PathBuilderBase& Path, const IoHash& HashKey); void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value); bool GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue); bool GetInlineCacheValue(const DiskLocation& Loc, ZenCacheValue& OutValue); diff --git a/zenserver/casstore.cpp b/zenserver/casstore.cpp index 88525bd36..872a40df8 100644 --- a/zenserver/casstore.cpp +++ b/zenserver/casstore.cpp @@ -51,6 +51,9 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store) return ServerRequest.WriteResponse(HttpResponseCode::OK); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPut | HttpVerb::kHead); diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index ae4dd4528..ddf7ad36c 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -2,6 +2,8 @@ #include "apply.h" +#if ZEN_WITH_COMPUTE_SERVICES + #include <upstream/jupiter.h> #include <upstream/upstreamapply.h> #include <zencore/compactbinary.h> @@ -14,20 +16,18 @@ #include <zencore/iobuffer.h> #include <zencore/iohash.h> #include <zencore/scopeguard.h> -#include <zencore/windows.h> -#include <zenstore/CAS.h> +#include <zenstore/cas.h> #include <zenstore/cidstore.h> -#if ZEN_PLATFORM_WINDOWS +#include <zencore/windows.h> ZEN_THIRD_PARTY_INCLUDES_START -# include <AccCtrl.h> -# include <AclAPI.h> -# include <sddl.h> -# include <UserEnv.h> -# pragma comment(lib, "UserEnv.lib") -# include <atlbase.h> +#include <AccCtrl.h> +#include <AclAPI.h> +#include <sddl.h> +#include <UserEnv.h> +#pragma comment(lib, "UserEnv.lib") +#include <atlbase.h> ZEN_THIRD_PARTY_INCLUDES_END -#endif #include <filesystem> #include <span> @@ -136,6 +136,8 @@ BasicFunctionJob::ExitCode() return gsl::narrow_cast<int>(Ec); } +//////////////////////////////////////////////////////////////////////////////// + struct SandboxedFunctionJob { SandboxedFunctionJob() = default; @@ -326,6 +328,8 @@ SandboxedFunctionJob::SpawnJob(std::filesystem::path ExePath) return true; } +//////////////////////////////////////////////////////////////////////////////// + HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, const std::filesystem::path& BaseDir) : m_Log(logging::Get("apply")) , m_CasStore(Store) @@ -448,6 +452,8 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, SharedBuffer Decompressed = DataView.Decompress(); const uint64_t DecompressedSize = DataView.GetRawSize(); + ZEN_UNUSED(DataHash); + TotalAttachmentBytes += DecompressedSize; ++AttachmentCount; @@ -478,9 +484,15 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, return HttpReq.WriteResponse(HttpResponseCode::NoContent); } break; + + default: + break; } } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -497,6 +509,9 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, case HttpVerb::kPost: break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -622,6 +637,8 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, const IoHash DataHash = Attachment.GetHash(); CompressedBuffer DataView = Attachment.AsCompressedBinary(); + ZEN_UNUSED(DataHash); + const uint64_t CompressedSize = DataView.GetCompressedSize(); TotalAttachmentBytes += CompressedSize; @@ -652,8 +669,14 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, return HttpReq.WriteResponse(HttpResponseCode::OK, Output); } break; + + default: + break; } break; + + default: + break; } }, HttpVerb::kPost); @@ -961,3 +984,5 @@ HttpFunctionService::ExecActionUpstreamResult(const IoHash& WorkerId, const IoHa } } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/compute/apply.h b/zenserver/compute/apply.h index 0d6edf119..af8668ee2 100644 --- a/zenserver/compute/apply.h +++ b/zenserver/compute/apply.h @@ -2,6 +2,14 @@ #pragma once +#include <zencore/zencore.h> + +#if !defined(ZEN_WITH_COMPUTE_SERVICES) +# define ZEN_WITH_COMPUTE_SERVICES ZEN_PLATFORM_WINDOWS +#endif + +#if ZEN_WITH_COMPUTE_SERVICES + #include <zencore/compactbinary.h> #include <zencore/iohash.h> #include <zencore/logging.h> @@ -54,3 +62,5 @@ private: }; } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/config.cpp b/zenserver/config.cpp index 0a812b5a2..354df7b25 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -9,16 +9,18 @@ #include <zencore/string.h> #include <zenhttp/zenhttp.h> -#pragma warning(push) -#pragma warning(disable : 4267) // warning C4267: '=': conversion from 'size_t' to 'US', possible loss of data +ZEN_THIRD_PARTY_INCLUDES_START #include <cxxopts.hpp> -#pragma warning(pop) - #include <fmt/format.h> #include <zencore/logging.h> #include <sol/sol.hpp> +ZEN_THIRD_PARTY_INCLUDES_END -#include <conio.h> +#if ZEN_PLATFORM_WINDOWS +# include <conio.h> +#else +# include <pwd.h> +#endif #if ZEN_PLATFORM_WINDOWS @@ -86,7 +88,9 @@ PickDefaultStateDirectory() std::filesystem::path PickDefaultStateDirectory() { - return std::filesystem::path("~/.zen"); + int UserId = getuid(); + const passwd* Passwd = getpwuid(UserId); + return std::filesystem::path(Passwd->pw_dir) / ".zen"; } #endif @@ -183,6 +187,22 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) ""); #endif +#if ZEN_WITH_TRACE + options.add_option("ue-trace", + "", + "tracehost", + "Hostname to send the trace to", + cxxopts::value<std::string>(ServerOptions.TraceHost)->default_value(""), + ""); + + options.add_option("ue-trace", + "", + "tracefile", + "Path to write a trace to", + cxxopts::value<std::string>(ServerOptions.TraceFile)->default_value(""), + ""); +#endif // ZEN_WITH_TRACE + options.add_option("diagnostics", "", "crash", @@ -310,8 +330,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) if (result.count("help")) { zen::logging::ConsoleLog().info("{}", options.help()); +#if ZEN_PLATFORM_WINDOWS zen::logging::ConsoleLog().info("Press any key to exit!"); _getch(); +#else + // Assume the user's in a terminal on all other platforms and that + // they'll use less/more/etc. if need be. +#endif exit(0); } @@ -344,7 +369,7 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio { using namespace fmt::literals; - zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(Path.native().c_str()); + zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(Path); if (LuaScript) { @@ -353,6 +378,7 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio lua.open_libraries(sol::lib::base); lua.set_function("getenv", [&](const std::string env) -> sol::object { +#if ZEN_PLATFORM_WINDOWS std::wstring EnvVarValue; size_t RequiredSize = 0; std::wstring EnvWide = zen::Utf8ToWide(env); @@ -364,6 +390,10 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio EnvVarValue.resize(RequiredSize); _wgetenv_s(&RequiredSize, EnvVarValue.data(), RequiredSize, EnvWide.c_str()); return sol::make_object(lua, zen::WideToUtf8(EnvVarValue.c_str())); +#else + ZEN_UNUSED(env); + return sol::make_object(lua, sol::lua_nil); +#endif }); try diff --git a/zenserver/config.h b/zenserver/config.h index 4229c5bcc..74e444339 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -4,6 +4,7 @@ #include <filesystem> #include <string> +#include <vector> #ifndef ZEN_ENABLE_MESH # define ZEN_ENABLE_MESH 0 @@ -67,6 +68,10 @@ struct ZenServerOptions bool StructuredCacheEnabled = true; bool ShouldCrash = false; // Option for testing crash handling bool IsFirstRun = false; +#if ZEN_WITH_TRACE + std::string TraceHost; // Host name or IP address to send trace data to + std::string TraceFile; // Path of a file to write a trace +#endif #if ZEN_ENABLE_MESH bool MeshEnabled = false; // Experimental p2p mesh discovery #endif diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp index 6e2559f1f..44a0e2ec6 100644 --- a/zenserver/diag/logging.cpp +++ b/zenserver/diag/logging.cpp @@ -16,6 +16,7 @@ ZEN_THIRD_PARTY_INCLUDES_START #include <spdlog/sinks/stdout_color_sinks.h> ZEN_THIRD_PARTY_INCLUDES_END +#include <zencore/filesystem.h> #include <zencore/string.h> #include <memory> @@ -168,6 +169,7 @@ private: bool EnableVTMode() { +#if ZEN_PLATFORM_WINDOWS // Set output mode to handle virtual terminal sequences HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE); if (hOut == INVALID_HANDLE_VALUE) @@ -186,6 +188,7 @@ EnableVTMode() { return false; } +#endif return true; } @@ -230,13 +233,13 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) auto ConsoleSink = std::make_shared<spdlog::sinks::ansicolor_stdout_sink_mt>(); #if 0 - auto FileSink = std::make_shared<spdlog::sinks::daily_file_sink_mt>(zen::WideToUtf8(LogPath.c_str()), + auto FileSink = std::make_shared<spdlog::sinks::daily_file_sink_mt>(zen::PathToUtf8(LogPath), 0, 0, /* truncate */ false, uint16_t(/* max files */ 14)); #else - auto FileSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::WideToUtf8(LogPath.c_str()), + auto FileSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::PathToUtf8(LogPath), /* max size */ 128 * 1024 * 1024, /* max files */ 16, /* rotate on open */ true); @@ -264,7 +267,7 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) std::filesystem::path HttpLogPath = GlobalOptions.DataDir / "logs/http.log"; - auto HttpSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::WideToUtf8(HttpLogPath.c_str()), + auto HttpSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::PathToUtf8(HttpLogPath), /* max size */ 128 * 1024 * 1024, /* max files */ 16, /* rotate on open */ true); diff --git a/zenserver/experimental/frontend.cpp b/zenserver/experimental/frontend.cpp index 98d570cfe..4bd3ec90a 100644 --- a/zenserver/experimental/frontend.cpp +++ b/zenserver/experimental/frontend.cpp @@ -54,7 +54,7 @@ body { <pre> __________ _________ __ \____ / ____ ____ / _____/_/ |_ ____ _______ ____ - / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \ + / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \ / /_ \ ___/ | | \ / \ | | ( <_> ) | | \/\ ___/ /_______ \ \___ >|___| //_______ / |__| \____/ |__| \___ > \/ \/ \/ \/ \/ diff --git a/zenserver/experimental/usnjournal.cpp b/zenserver/experimental/usnjournal.cpp index 9422dd485..5aceaad62 100644 --- a/zenserver/experimental/usnjournal.cpp +++ b/zenserver/experimental/usnjournal.cpp @@ -1,11 +1,14 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#include <zencore/zencore.h> + +#if ZEN_PLATFORM_WINDOWS + #include "usnjournal.h" #include <zencore/except.h> #include <zencore/logging.h> #include <zencore/timer.h> -#include <zencore/zencore.h> ZEN_THIRD_PARTY_INCLUDES_START #include <atlfile.h> @@ -344,3 +347,5 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) } } // namespace zen + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/zenserver/experimental/usnjournal.h b/zenserver/experimental/usnjournal.h index db1f59abc..2f666383b 100644 --- a/zenserver/experimental/usnjournal.h +++ b/zenserver/experimental/usnjournal.h @@ -2,6 +2,8 @@ #pragma once +#if ZEN_PLATFORM_WINDOWS + #include <zencore/windows.h> #include <zencore/zencore.h> @@ -63,3 +65,5 @@ private: }; } // namespace zen + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index 73d61c124..8ce418ef5 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -11,11 +11,14 @@ #include <zencore/stream.h> #include <zencore/string.h> #include <zencore/timer.h> -#include <zencore/windows.h> #include <zenstore/basicfile.h> #include <zenstore/cas.h> #include <zenstore/caslog.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #define USE_ROCKSDB 0 #if USE_ROCKSDB @@ -114,7 +117,7 @@ struct ProjectStore::OplogStorage : public RefCounted #if USE_ROCKSDB { - std::string RocksdbPath = WideToUtf8((m_OplogStoragePath / "ops.rdb").native().c_str()); + std::string RocksdbPath = PathToUtf8(m_OplogStoragePath / "ops.rdb"); ZEN_DEBUG("opening rocksdb db at '{}'", RocksdbPath); @@ -184,8 +187,8 @@ struct ProjectStore::OplogStorage : public RefCounted CbObject Op(SharedBuffer::MakeView(OpBuffer.Data(), OpBuffer.Size())); m_NextOpsOffset = - Max(m_NextOpsOffset.load(std::memory_order::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign)); - m_MaxLsn = Max(m_MaxLsn.load(std::memory_order::memory_order_relaxed), LogEntry.OpLsn); + Max(m_NextOpsOffset.load(std::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign)); + m_MaxLsn = Max(m_MaxLsn.load(std::memory_order_relaxed), LogEntry.OpLsn); Handler(Op, LogEntry); }); @@ -275,8 +278,8 @@ private: ProjectStore::Oplog::Oplog(std::string_view Id, Project* Project, CidStore& Store, std::filesystem::path BasePath) : m_OuterProject(Project) , m_CidStore(Store) -, m_OplogId(Id) , m_BasePath(BasePath) +, m_OplogId(Id) { m_Storage = new OplogStorage(this, m_BasePath); const bool StoreExists = m_Storage->Exists(); @@ -331,7 +334,7 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId) std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second.ServerPath; - IoBuffer FileChunk = IoBufferBuilder::MakeFromFile(FilePath.native().c_str()); + IoBuffer FileChunk = IoBufferBuilder::MakeFromFile(FilePath); FileChunk.SetContentType(ZenContentType::kBinary); return FileChunk; @@ -411,16 +414,6 @@ ProjectStore::Oplog::AddFileMapping(Oid FileId, IoHash Hash, std::string_view Se return false; } - if (ServerPath[0] == '/' || ClientPath[0] != '/') - { - // This is a special case just to enable tests to use absolute paths. We might want - // to have configuration to control this - if (ServerPath[1] != ':') - { - return false; - } - } - FileMapEntry Entry; Entry.ServerPath = ServerPath; Entry.ClientPath = ClientPath; @@ -626,7 +619,7 @@ ProjectStore::Project::Write() CbObjectWriter Cfg; Cfg << "id" << Identifier; - Cfg << "root" << WideToUtf8(RootDir.c_str()); + Cfg << "root" << PathToUtf8(RootDir); Cfg << "project" << ProjectRootDir; Cfg << "engine" << EngineRootDir; @@ -725,8 +718,6 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId) void ProjectStore::Project::DeleteOplog(std::string_view OplogId) { - bool Exists = false; - { RwLock::ExclusiveLockScope _(m_ProjectLock); @@ -734,8 +725,6 @@ ProjectStore::Project::DeleteOplog(std::string_view OplogId) if (OplogIt != m_Oplogs.end()) { - Exists = true; - m_Oplogs.erase(OplogIt); } } @@ -776,8 +765,8 @@ ProjectStore::Project::Scrub(ScrubContext& Ctx) ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath) : m_Log(zen::logging::Get("project")) -, m_ProjectBasePath(BasePath) , m_CidStore(Store) +, m_ProjectBasePath(BasePath) { ZEN_INFO("initializing project store at '{}'", BasePath); // m_Log.set_level(spdlog::level::debug); @@ -1492,7 +1481,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) ProjectStore::Oplog& Log = *OplogIt; CbObjectWriter Cb; - Cb << "id"sv << Log.OplogId() << "project"sv << Prj.Identifier << "tempdir"sv << Log.TempDir(); + Cb << "id"sv << Log.OplogId() << "project"sv << Prj.Identifier << "tempdir"sv << Log.TempPath().c_str(); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Cb.Save()); } @@ -1531,6 +1520,9 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) return Req.ServerRequest().WriteResponse(HttpResponseCode::OK); } break; + + default: + break; } }, HttpVerb::kPost | HttpVerb::kGet | HttpVerb::kDelete); @@ -1628,7 +1620,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) const ProjectStore::Project& Prj = *ProjectIt; CbObjectWriter Response; - Response << "id" << Prj.Identifier << "root" << WideToUtf8(Prj.RootDir.c_str()); + Response << "id" << Prj.Identifier << "root" << PathToUtf8(Prj.RootDir); Response.BeginArray("oplogs"sv); Prj.IterateOplogs([&](const ProjectStore::Oplog& I) { Response << "id"sv << I.OplogId(); }); @@ -1654,6 +1646,9 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) return Req.ServerRequest().WriteResponse(HttpResponseCode::NoContent); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost | HttpVerb::kDelete); @@ -1680,6 +1675,7 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request) ////////////////////////////////////////////////////////////////////////// +#if ZEN_PLATFORM_WINDOWS class SecurityAttributes { public: @@ -1712,6 +1708,12 @@ public: } } }; +#else +struct AnyUserSecurityAttributes +{ + int Attributes() { return 0666; } +}; +#endif // ZEN_PLATFORM_WINDOWS ////////////////////////////////////////////////////////////////////////// @@ -1782,6 +1784,7 @@ private: asio::thread_pool m_WorkerThreadPool; asio::io_context m_IoContext; +#if ZEN_PLATFORM_WINDOWS class PipeConnection { enum PipeState @@ -1940,6 +1943,15 @@ private: uint8_t m_MsgBuffer[16384]; }; +#else + class PipeConnection + { + public: + PipeConnection(LocalProjectImpl*) {} + void Accept() {} + void Disconnect() {} + }; +#endif AnyUserSecurityAttributes m_AnyUserSecurityAttributes; std::vector<PipeConnection*> m_ServicePipes; diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h index c9f49217a..cebe26886 100644 --- a/zenserver/projectstore.h +++ b/zenserver/projectstore.h @@ -97,7 +97,6 @@ public: const std::string& OplogId() const { return m_OplogId; } - const std::wstring& TempDir() const { return m_TempPath.native(); } const std::filesystem::path& TempPath() const { return m_TempPath; } spdlog::logger& Log() { return m_OuterProject->Log(); } diff --git a/zenserver/testing/launch.cpp b/zenserver/testing/launch.cpp index 55695ac9c..013aa28b8 100644 --- a/zenserver/testing/launch.cpp +++ b/zenserver/testing/launch.cpp @@ -2,6 +2,8 @@ #include "launch.h" +#if ZEN_WITH_COMPUTE_SERVICES + #include <zencore/compactbinary.h> #include <zencore/compactbinarybuilder.h> #include <zencore/filesystem.h> @@ -10,16 +12,17 @@ #include <zencore/iohash.h> #include <zencore/logging.h> #include <zencore/windows.h> -#include <zenstore/CAS.h> +#include <zenstore/cas.h> +ZEN_THIRD_PARTY_INCLUDES_START #include <AccCtrl.h> #include <AclAPI.h> +#include <atlbase.h> #include <sddl.h> - #include <UserEnv.h> +ZEN_THIRD_PARTY_INCLUDES_END #pragma comment(lib, "UserEnv.lib") -#include <atlbase.h> #include <filesystem> #include <span> @@ -127,6 +130,8 @@ BasicJob::ExitCode() return gsl::narrow_cast<int>(Ec); } +//////////////////////////////////////////////////////////////////////////////// + struct SandboxedJob { SandboxedJob() = default; @@ -317,6 +322,8 @@ SandboxedJob::SpawnJob(std::filesystem::path ExePath) return true; } +//////////////////////////////////////////////////////////////////////////////// + HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::path& SandboxBaseDir) : m_Log(logging::Get("exec")) , m_CasStore(Store) @@ -336,6 +343,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat case HttpVerb::kPost: break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -362,6 +372,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat Job.SpawnJob("c:\\windows\\system32\\cmd.exe"); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -413,6 +426,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat return HttpReq.WriteResponse(HttpResponseCode::OK, Response); } break; + + default: + break; } }, HttpVerb::kPost); @@ -496,6 +512,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -530,3 +549,5 @@ HttpLaunchService::CreateNewSandbox() } } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/testing/launch.h b/zenserver/testing/launch.h index 49f12e2ec..13e3db4df 100644 --- a/zenserver/testing/launch.h +++ b/zenserver/testing/launch.h @@ -2,6 +2,14 @@ #pragma once +#include <zencore/zencore.h> + +#if !defined(ZEN_WITH_COMPUTE_SERVICES) +# define ZEN_WITH_COMPUTE_SERVICES ZEN_PLATFORM_WINDOWS +#endif + +#if ZEN_WITH_COMPUTE_SERVICES + #include <zencore/logging.h> #include <zenhttp/httpserver.h> @@ -36,3 +44,5 @@ private: }; } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index f9be068ec..c3480ff56 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -2,7 +2,6 @@ #include "jupiter.h" -#include "cache/structuredcachestore.h" #include "diag/formatters.h" #include "diag/logging.h" @@ -58,7 +57,7 @@ namespace detail { cpr::Session& GetSession() { return Session; } private: - friend class CloudCacheClient; + friend class zen::CloudCacheClient; CloudCacheClient& OwnerClient; CloudCacheAccessToken AccessToken; @@ -119,7 +118,7 @@ CloudCacheSession::GetDerivedData(std::string_view BucketId, std::string_view Ke return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, + .ErrorCode = !Success ? int32_t(Response.status_code) : 0, .Success = Success}; } @@ -168,7 +167,7 @@ CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key, ZenConte return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, + .ErrorCode = !Success ? int32_t(Response.status_code) : 0, .Success = Success}; } @@ -209,7 +208,7 @@ CloudCacheSession::GetBlob(const IoHash& Key) return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, + .ErrorCode = !Success ? int32_t(Response.status_code) : 0, .Success = Success}; } @@ -249,7 +248,7 @@ CloudCacheSession::GetCompressedBlob(const IoHash& Key) return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, + .ErrorCode = !Success ? int32_t(Response.status_code) : 0, .Success = Success}; } @@ -289,7 +288,7 @@ CloudCacheSession::GetObject(const IoHash& Key) return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, + .ErrorCode = !Success ? int32_t(Response.status_code) : 0, .Success = Success}; } @@ -331,7 +330,7 @@ CloudCacheSession::PutDerivedData(std::string_view BucketId, std::string_view Ke return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, + .ErrorCode = !Success ? int32_t(Response.status_code) : 0, .Success = Success}; } @@ -388,7 +387,7 @@ CloudCacheSession::PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer PutRefResult Result; Result.Success = (Response.status_code == 200 || Response.status_code == 201); - Result.ErrorCode = !Result.Success ? Response.status_code : 0; + Result.ErrorCode = !Result.Success ? int32_t(Response.status_code) : 0; Result.Bytes = Response.uploaded_bytes; Result.ElapsedSeconds = Response.elapsed; @@ -453,7 +452,7 @@ CloudCacheSession::FinalizeRef(std::string_view BucketId, const IoHash& Key, con FinalizeRefResult Result; Result.Success = (Response.status_code == 200 || Response.status_code == 201); - Result.ErrorCode = !Result.Success ? Response.status_code : 0; + Result.ErrorCode = !Result.Success ? int32_t(Response.status_code) : 0; Result.Bytes = Response.uploaded_bytes; Result.ElapsedSeconds = Response.elapsed; @@ -508,7 +507,7 @@ CloudCacheSession::PutBlob(const IoHash& Key, IoBuffer Blob) return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, + .ErrorCode = !Success ? int32_t(Response.status_code) : 0, .Success = Success}; } @@ -546,7 +545,7 @@ CloudCacheSession::PutCompressedBlob(const IoHash& Key, IoBuffer Blob) return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, + .ErrorCode = !Success ? int32_t(Response.status_code) : 0, .Success = Success}; } @@ -584,7 +583,7 @@ CloudCacheSession::PutObject(const IoHash& Key, IoBuffer Object) return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, + .ErrorCode = !Success ? int32_t(Response.status_code) : 0, .Success = Success}; } @@ -621,7 +620,11 @@ CloudCacheSession::RefExists(std::string_view BucketId, const IoHash& Key) const bool Success = Response.status_code == 200; - return {.ElapsedSeconds = Response.elapsed, .ErrorCode = !Success ? Response.status_code : 0, .Success = Success}; + return { + .ElapsedSeconds = Response.elapsed, + .ErrorCode = !Success ? int32_t(Response.status_code) : 0, + .Success = Success + }; } CloudCacheResult @@ -692,7 +695,11 @@ CloudCacheSession::PostComputeTasks(std::string_view ChannelId, IoBuffer TasksDa const bool Success = Response.status_code == 200; - return {.ElapsedSeconds = Response.elapsed, .ErrorCode = !Success ? Response.status_code : 0, .Success = Success}; + return { + .ElapsedSeconds = Response.elapsed, + .ErrorCode = !Success ? int32_t(Response.status_code) : 0, + .Success = Success + }; } CloudCacheResult @@ -731,7 +738,7 @@ CloudCacheSession::GetComputeUpdates(std::string_view ChannelId, const uint32_t return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, + .ErrorCode = !Success ? int32_t(Response.status_code) : 0, .Success = Success}; } @@ -771,7 +778,7 @@ CloudCacheSession::GetObjectTree(const IoHash& Key) return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, + .ErrorCode = !Success ? int32_t(Response.status_code) : 0, .Success = Success}; } diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp index 05be5f65c..f673ec3b3 100644 --- a/zenserver/upstream/upstreamapply.cpp +++ b/zenserver/upstream/upstreamapply.cpp @@ -1,6 +1,9 @@ // Copyright Epic Games, Inc. All Rights Reserved. #include "upstreamapply.h" + +#if ZEN_WITH_COMPUTE_SERVICES + #include "jupiter.h" #include "zen.h" @@ -1569,3 +1572,5 @@ MakeHordeUpstreamEndpoint(const CloudCacheClientOptions& Options, CasStore& CasS } } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/upstream/upstreamapply.h b/zenserver/upstream/upstreamapply.h index 98f193c02..8196c3b40 100644 --- a/zenserver/upstream/upstreamapply.h +++ b/zenserver/upstream/upstreamapply.h @@ -2,6 +2,10 @@ #pragma once +#include "compute/apply.h" + +#if ZEN_WITH_COMPUTE_SERVICES + #include <zencore/compactbinarypackage.h> #include <zencore/iobuffer.h> #include <zencore/iohash.h> @@ -170,3 +174,5 @@ std::unique_ptr<UpstreamApplyEndpoint> MakeHordeUpstreamEndpoint(const CloudCach CidStore& CidStore); } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index e2dc09872..616cd4146 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -195,7 +195,7 @@ namespace detail { } } - OnComplete({.CacheKey = CacheKey, .KeyIndex = Index, .Record = Record, .Package = Package}); + OnComplete({.Key = CacheKey, .KeyIndex = Index, .Record = Record, .Package = Package}); } return Result; @@ -271,16 +271,16 @@ namespace detail { if (CacheRecord.Type == ZenContentType::kBinary) { CloudCacheResult Result; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { if (m_UseLegacyDdc) { - Result = Session.PutDerivedData(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue); + Result = Session.PutDerivedData(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RecordValue); } else { - Result = Session.PutRef(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + Result = Session.PutRef(CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, RecordValue, ZenContentType::kBinary); } @@ -336,15 +336,15 @@ namespace detail { for (int32_t Attempt = 0; Attempt < MaxAttempts && !RefResult.Success; Attempt++) { RefResult = - Session.PutRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue, ZenContentType::kCbObject); + Session.PutRef(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RecordValue, ZenContentType::kCbObject); } m_HealthOk = RefResult.ErrorCode == 0; if (!RefResult.Success) { - return {.Reason = "upload cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + return {.Reason = "upload cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, RefResult.Reason), .Success = false}; } @@ -359,13 +359,13 @@ namespace detail { } const IoHash RefHash = IoHash::HashBuffer(RecordValue); - FinalizeRefResult FinalizeResult = Session.FinalizeRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RefHash); + FinalizeRefResult FinalizeResult = Session.FinalizeRef(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RefHash); m_HealthOk = FinalizeResult.ErrorCode == 0; if (!FinalizeResult.Success) { - return {.Reason = "finalize cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + return {.Reason = "finalize cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, FinalizeResult.Reason), .Success = false}; } @@ -377,13 +377,13 @@ namespace detail { return {.Reason = std::move(Reason), .Success = false}; } - FinalizeResult = Session.FinalizeRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RefHash); + FinalizeResult = Session.FinalizeRef(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RefHash); m_HealthOk = FinalizeResult.ErrorCode == 0; if (!FinalizeResult.Success) { - return {.Reason = "finalize '{}/{}' FAILED, reason '{}'"_format(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + return {.Reason = "finalize '{}/{}' FAILED, reason '{}'"_format(CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, FinalizeResult.Reason), .Success = false}; } @@ -396,8 +396,8 @@ namespace detail { Sb << MissingHash.ToHexString() << ","; } - return {.Reason = "finalize '{}/{}' FAILED, still needs payload(s) '{}'"_format(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + return {.Reason = "finalize '{}/{}' FAILED, still needs payload(s) '{}'"_format(CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, Sb.ToString()), .Success = false}; } @@ -607,7 +607,7 @@ namespace detail { { const size_t Index = IndexMap[LocalIndex++]; OnComplete( - {.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = Record.AsObjectView(), .Package = BatchResponse}); + {.Key = CacheKeys[Index], .KeyIndex = Index, .Record = Record.AsObjectView(), .Package = BatchResponse}); } return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = true}; @@ -620,7 +620,7 @@ namespace detail { for (size_t Index : KeyIndex) { - OnComplete({.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()}); + OnComplete({.Key = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()}); } return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}; @@ -772,10 +772,10 @@ namespace detail { Package.Save(MemStream); IoBuffer PackagePayload(IoBuffer::Wrap, MemStream.Data(), MemStream.Size()); - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { - Result = Session.PutCacheRecord(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + Result = Session.PutCacheRecord(CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, PackagePayload, CacheRecord.Type); @@ -790,10 +790,10 @@ namespace detail { for (size_t Idx = 0, Count = Payloads.size(); Idx < Count; Idx++) { Result.Success = false; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { - Result = Session.PutCachePayload(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + Result = Session.PutCachePayload(CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, CacheRecord.PayloadIds[Idx], Payloads[Idx]); @@ -813,10 +813,10 @@ namespace detail { } Result.Success = false; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { Result = - Session.PutCacheRecord(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue, CacheRecord.Type); + Session.PutCacheRecord(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RecordValue, CacheRecord.Type); m_HealthOk = Result.ErrorCode == 0; } @@ -1099,7 +1099,7 @@ public: for (size_t Index : MissingKeys) { - OnComplete({.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()}); + OnComplete({.Key = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()}); } } @@ -1236,11 +1236,11 @@ private: ZenCacheValue CacheValue; std::vector<IoBuffer> Payloads; - if (!m_CacheStore.Get(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, CacheValue)) + if (!m_CacheStore.Get(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, CacheValue)) { ZEN_WARN("process upstream FAILED, '{}/{}', cache record doesn't exist", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash); + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash); return; } @@ -1253,8 +1253,8 @@ private: else { ZEN_WARN("process upstream FAILED, '{}/{}/{}', payload doesn't exist in CAS", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, PayloadId); return; } @@ -1270,8 +1270,8 @@ private: if (!Result.Success) { ZEN_WARN("upload cache record '{}/{}' FAILED, endpoint '{}', reason '{}'", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, Endpoint->GetEndpointInfo().Url, Result.Reason); } @@ -1293,8 +1293,8 @@ private: catch (std::exception& Err) { ZEN_ERROR("upload cache record '{}/{}' FAILED, reason '{}'", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, Err.what()); } } diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index 12287198d..c463c4996 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -25,7 +25,7 @@ struct ZenStructuredCacheClientOptions; struct UpstreamCacheRecord { ZenContentType Type = ZenContentType::kBinary; - CacheKey CacheKey; + CacheKey Key; std::vector<IoHash> PayloadIds; }; @@ -83,7 +83,7 @@ struct UpstreamEndpointStats struct CacheRecordGetCompleteParams { - const CacheKey& CacheKey; + const CacheKey& Key; size_t KeyIndex = ~size_t(0); const CbObjectView& Record; const CbPackage& Package; diff --git a/zenserver/upstream/zen.h b/zenserver/upstream/zen.h index d549c2fc4..83e54465a 100644 --- a/zenserver/upstream/zen.h +++ b/zenserver/upstream/zen.h @@ -10,14 +10,14 @@ #include <zencore/uid.h> #include <zencore/zencore.h> -#pragma warning(push) -#pragma warning(disable : 4127) +ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_map.h> -#pragma warning(pop) +ZEN_THIRD_PARTY_INCLUDES_END #include <asio.hpp> #include <chrono> +#include <list> struct ZenCacheValue; diff --git a/zenserver/windows/service.cpp b/zenserver/windows/service.cpp index 23cefb7b5..4af9926a6 100644 --- a/zenserver/windows/service.cpp +++ b/zenserver/windows/service.cpp @@ -2,6 +2,10 @@ #include "service.h" +#include <zencore/zencore.h> + +#if ZEN_PLATFORM_WINDOWS + #include <zencore/except.h> #include <zencore/zencore.h> @@ -638,3 +642,5 @@ SvcReportEvent(LPTSTR szFunction) // DeregisterEventSource(hEventSource); //} } + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua index bffb5d9d2..d53fdd4f0 100644 --- a/zenserver/xmake.lua +++ b/zenserver/xmake.lua @@ -18,6 +18,7 @@ target("zenserver") end add_options("vfs") + add_options("compute") add_packages( "vcpkg::sentry-native", @@ -42,7 +43,6 @@ target("zenserver") else commit = "dbg-" .. commit end - target:add("defines","BUILD_VERSION=\"" .. commit .. "\"") - print("build version " .. commit) + target:add("defines", "BUILD_VERSION=\"" .. commit .. "\"") end end) diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index b1989e1bc..c2d2c1d39 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -12,13 +12,17 @@ #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/timer.h> -#include <zencore/windows.h> +#include <zencore/trace.h> #include <zenhttp/httpserver.h> #include <zenstore/basicfile.h> #include <zenstore/cas.h> #include <zenstore/cidstore.h> #include <zenutil/zenserverprocess.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #if ZEN_USE_MIMALLOC ZEN_THIRD_PARTY_INCLUDES_START # include <mimalloc-new-delete.h> @@ -138,7 +142,7 @@ namespace utils { if (!ErrorCode) { - for (const asio::ip::tcp::endpoint& Ep : Endpoints) + for (const asio::ip::tcp::endpoint Ep : Endpoints) { OutEndpoints.push_back("http://{}:{}"_format(Ep.address().to_string(), Ep.port())); } @@ -221,6 +225,7 @@ public: m_HttpProjectService.reset(new zen::HttpProjectService{*m_CidStore, m_ProjectStore}); m_LocalProjectService = zen::LocalProjectService::New(*m_CasStore, m_ProjectStore); +#if ZEN_WITH_COMPUTE_SERVICES ZEN_INFO("instantiating compute services"); std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox"; @@ -230,6 +235,7 @@ public: std::filesystem::path ApplySandboxDir = m_DataRoot / "exec" / "apply"; zen::CreateDirectories(ApplySandboxDir); m_HttpFunctionService = std::make_unique<zen::HttpFunctionService>(*m_CasStore, *m_CidStore, ApplySandboxDir); +#endif // ZEN_WITH_COMPUTE_SERVICES if (ServerOptions.StructuredCacheEnabled) { @@ -267,6 +273,7 @@ public: m_Http->RegisterService(*m_StructuredCacheService); } +#if ZEN_WITH_COMPUTE_SERVICES if (m_HttpLaunchService) { m_Http->RegisterService(*m_HttpLaunchService); @@ -276,6 +283,7 @@ public: { m_Http->RegisterService(*m_HttpFunctionService); } +#endif // ZEN_WITH_COMPUTE_SERVICES m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot); @@ -325,7 +333,7 @@ public: if (m_DebugOptionForcedCrash) { - __debugbreak(); + ZEN_DEBUG_BREAK(); } const bool IsInteractiveMode = zen::IsInteractiveSession() && !m_TestMode; @@ -386,7 +394,7 @@ public: for (auto& PidEntry : m_ServerEntry->SponsorPids) { - if (uint32_t ThisPid = PidEntry.load(std::memory_order::memory_order_relaxed)) + if (uint32_t ThisPid = PidEntry.load(std::memory_order_relaxed)) { if (PidEntry.compare_exchange_strong(ThisPid, 0)) { @@ -505,13 +513,15 @@ private: zen::HttpCasService m_CasService{*m_CasStore}; zen::RefPtr<zen::ProjectStore> m_ProjectStore; zen::Ref<zen::LocalProjectService> m_LocalProjectService; - std::unique_ptr<zen::HttpLaunchService> m_HttpLaunchService; std::unique_ptr<zen::HttpProjectService> m_HttpProjectService; std::unique_ptr<zen::HttpStructuredCacheService> m_StructuredCacheService; zen::HttpAdminService m_AdminService; zen::HttpHealthService m_HealthService; zen::Mesh m_ZenMesh{m_IoContext}; +#if ZEN_WITH_COMPUTE_SERVICES + std::unique_ptr<zen::HttpLaunchService> m_HttpLaunchService; std::unique_ptr<zen::HttpFunctionService> m_HttpFunctionService; +#endif std::unique_ptr<zen::HttpFrontendService> m_FrontendService; bool m_DebugOptionForcedCrash = false; @@ -742,33 +752,35 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) std::move(UpstreamCache))); } -} // namespace zen +//////////////////////////////////////////////////////////////////////////////// -class ZenWindowsService : public WindowsService +class ZenEntryPoint { public: - ZenWindowsService(ZenServerOptions& ServerOptions) : m_ServerOptions(ServerOptions) {} - - ZenWindowsService(const ZenWindowsService&) = delete; - ZenWindowsService& operator=(const ZenWindowsService&) = delete; - - virtual int Run() override; + ZenEntryPoint(ZenServerOptions& ServerOptions); + ZenEntryPoint(const ZenEntryPoint&) = delete; + ZenEntryPoint& operator = (const ZenEntryPoint&) = delete; + int Run(); private: - ZenServerOptions& m_ServerOptions; - zen::LockFile m_LockFile; + ZenServerOptions& m_ServerOptions; + zen::LockFile m_LockFile; }; -int -ZenWindowsService::Run() +ZenEntryPoint::ZenEntryPoint(ZenServerOptions& ServerOptions) +: m_ServerOptions(ServerOptions) { - using namespace zen; +} +int +ZenEntryPoint::Run() +{ #if USE_SENTRY // Initialize sentry.io client sentry_options_t* SentryOptions = sentry_options_new(); sentry_options_set_dsn(SentryOptions, "https://[email protected]/5919284"); + sentry_options_set_database_path(SentryOptions, PathToUtf8(m_ServerOptions.DataDir / ".sentry-native").c_str()); sentry_init(SentryOptions); auto _ = zen::MakeGuard([] { sentry_close(); }); @@ -788,7 +800,7 @@ ZenWindowsService::Run() auto MakeLockData = [&] { CbObjectWriter Cbo; - Cbo << "pid" << _getpid() << "data" << ToUtf8(ServerOptions.DataDir) << "port" << ServerOptions.BasePort << "session_id" + Cbo << "pid" << zen::GetCurrentProcessId() << "data" << PathToUtf8(ServerOptions.DataDir) << "port" << ServerOptions.BasePort << "session_id" << GetSessionId() << "ready" << IsReady; return Cbo.Save(); }; @@ -856,6 +868,10 @@ ZenWindowsService::Run() ZEN_INFO("shutdown signal received"); Server.RequestExit(0); } + else + { + ZEN_INFO("shutdown signal wait() failed"); + } }}); // If we have a parent process, establish the mechanisms we need @@ -889,6 +905,39 @@ ZenWindowsService::Run() return 0; } +} // namespace zen + +//////////////////////////////////////////////////////////////////////////////// + +#if ZEN_PLATFORM_WINDOWS + +class ZenWindowsService : public WindowsService +{ +public: + ZenWindowsService(ZenServerOptions& ServerOptions) + : m_EntryPoint(ServerOptions) + { + } + + ZenWindowsService(const ZenWindowsService&) = delete; + ZenWindowsService& operator=(const ZenWindowsService&) = delete; + + virtual int Run() override; + +private: + zen::ZenEntryPoint m_EntryPoint; +}; + +int +ZenWindowsService::Run() +{ + return m_EntryPoint.Run(); +} + +#endif // ZEN_PLATFORM_WINDOWS + +//////////////////////////////////////////////////////////////////////////////// + #if ZEN_WITH_TESTS int test_main(int argc, char** argv) @@ -935,6 +984,17 @@ main(int argc, char* argv[]) std::filesystem::create_directories(ServerOptions.DataDir); } +#if ZEN_WITH_TRACE + if (ServerOptions.TraceHost.size()) + { + TraceInit(ServerOptions.TraceHost.c_str(), TraceType::Network); + } + else if (ServerOptions.TraceFile.size()) + { + TraceInit(ServerOptions.TraceFile.c_str(), TraceType::File); + } +#endif // ZEN_WITH_TRACE + #if ZEN_PLATFORM_WINDOWS if (ServerOptions.InstallService) { @@ -949,10 +1009,18 @@ main(int argc, char* argv[]) std::exit(0); } -#endif ZenWindowsService App(ServerOptions); return App.ServiceMain(); +#else + if (ServerOptions.InstallService || ServerOptions.UninstallService) + { + throw std::runtime_error("Service mode is not supported on this platform"); + } + + ZenEntryPoint App(ServerOptions); + return App.Run(); +#endif // ZEN_PLATFORM_WINDOWS } catch (std::exception& Ex) { |