diff options
| author | Per Larsson <[email protected]> | 2021-12-14 12:34:47 +0100 |
|---|---|---|
| committer | Per Larsson <[email protected]> | 2021-12-14 12:34:47 +0100 |
| commit | b6c6568e1618f10d2160d836b65e35586e3c740f (patch) | |
| tree | f6a929cf918850bbba87d0ee67cd3482b2d50e24 /zenserver | |
| parent | Fixed bug in z$ service returning partial cache records and enable small obje... (diff) | |
| parent | Partial revert b363c5b (diff) | |
| download | zen-b6c6568e1618f10d2160d836b65e35586e3c740f.tar.xz zen-b6c6568e1618f10d2160d836b65e35586e3c740f.zip | |
Merged main.
Diffstat (limited to 'zenserver')
28 files changed, 511 insertions, 342 deletions
diff --git a/zenserver/admin/admin.cpp b/zenserver/admin/admin.cpp index 15314b12a..01c35a1d8 100644 --- a/zenserver/admin/admin.cpp +++ b/zenserver/admin/admin.cpp @@ -1,7 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#pragma once - #include "admin.h" #include <zencore/compactbinarybuilder.h> diff --git a/zenserver/cache/cachetracking.cpp b/zenserver/cache/cachetracking.cpp index d1c99a597..ae132f37f 100644 --- a/zenserver/cache/cachetracking.cpp +++ b/zenserver/cache/cachetracking.cpp @@ -2,24 +2,26 @@ #include "cachetracking.h" -#include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinaryvalue.h> -#include <zencore/endian.h> -#include <zencore/filesystem.h> -#include <zencore/logging.h> -#include <zencore/scopeguard.h> -#include <zencore/string.h> +#if ZEN_USE_CACHE_TRACKER -#include <zencore/testing.h> -#include <zencore/testutils.h> +# include <zencore/compactbinarybuilder.h> +# include <zencore/compactbinaryvalue.h> +# include <zencore/endian.h> +# include <zencore/filesystem.h> +# include <zencore/logging.h> +# include <zencore/scopeguard.h> +# include <zencore/string.h> + +# include <zencore/testing.h> +# include <zencore/testutils.h> ZEN_THIRD_PARTY_INCLUDES_START -#pragma comment(lib, "Rpcrt4.lib") // RocksDB made me do this -#include <fmt/format.h> -#include <rocksdb/db.h> -#include <tsl/robin_map.h> -#include <tsl/robin_set.h> -#include <gsl/gsl-lite.hpp> +# pragma comment(lib, "Rpcrt4.lib") // RocksDB made me do this +# include <fmt/format.h> +# include <rocksdb/db.h> +# include <tsl/robin_map.h> +# include <tsl/robin_set.h> +# include <gsl/gsl-lite.hpp> ZEN_THIRD_PARTY_INCLUDES_END namespace zen { @@ -150,7 +152,7 @@ struct ZenCacheTracker::Impl { std::filesystem::path StatsDbPath{StateDirectory / ".zdb"}; - std::string RocksdbPath = ToUtf8(StatsDbPath); + std::string RocksdbPath = StatsDbPath.string(); ZEN_DEBUG("opening tracker db at '{}'", RocksdbPath); @@ -291,7 +293,7 @@ ZenCacheTracker::IterateSnapshots(std::function<void(uint64_t TimeStamp, CbObjec m_Impl->IterateSnapshots(std::move(Callback)); } -#if ZEN_WITH_TESTS +# if ZEN_WITH_TESTS TEST_CASE("z$.tracker") { @@ -365,7 +367,7 @@ TEST_CASE("z$.tracker") CHECK_EQ(SnapshotCount, 11); } -#endif +# endif void cachetracker_forcelink() @@ -373,3 +375,5 @@ cachetracker_forcelink() } } // namespace zen + +#endif // ZEN_USE_CACHE_TRACKER diff --git a/zenserver/cache/cachetracking.h b/zenserver/cache/cachetracking.h index 06109ebb0..fdfe1a4c7 100644 --- a/zenserver/cache/cachetracking.h +++ b/zenserver/cache/cachetracking.h @@ -10,6 +10,9 @@ namespace zen { +#define ZEN_USE_CACHE_TRACKER 0 +#if ZEN_USE_CACHE_TRACKER + class CbObject; /** @@ -33,4 +36,6 @@ private: void cachetracker_forcelink(); +#endif // ZEN_USE_CACHE_TRACKER + } // namespace zen diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 22c8c4afb..b322a5d1b 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -13,7 +13,7 @@ #include <zencore/stream.h> #include <zencore/timer.h> #include <zenhttp/httpserver.h> -#include <zenstore/CAS.h> +#include <zenstore/cas.h> #include <zenutil/cache/cache.h> //#include "cachekey.h" @@ -152,7 +152,7 @@ void HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Bucket) { ZEN_UNUSED(Request, Bucket); - switch (auto Verb = Request.RequestVerb()) + switch (Request.RequestVerb()) { using enum HttpVerb; @@ -175,13 +175,16 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, return Request.WriteResponse(HttpResponseCode::NotFound); } break; + + default: + break; } } void HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy) { - switch (auto Verb = Request.RequestVerb()) + switch (Request.RequestVerb()) { using enum HttpVerb; @@ -442,7 +445,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (StoreUpstream) { ZEN_ASSERT(m_UpstreamCache); - auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kBinary, .CacheKey = {Ref.BucketSegment, Ref.HashKey}}); + m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kBinary, .Key = {Ref.BucketSegment, Ref.HashKey}}); } Request.WriteResponse(HttpResponseCode::Created); @@ -486,9 +489,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (StoreUpstream && !IsPartialRecord) { ZEN_ASSERT(m_UpstreamCache); - auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbObject, - .CacheKey = {Ref.BucketSegment, Ref.HashKey}, - .PayloadIds = std::move(ValidAttachments)}); + m_UpstreamCache->EnqueueUpstream( + {.Type = ZenContentType::kCbObject, .Key = {Ref.BucketSegment, Ref.HashKey}, .PayloadIds = std::move(ValidAttachments)}); } Request.WriteResponse(HttpResponseCode::Created); @@ -568,9 +570,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (StoreUpstream && !IsPartialRecord) { ZEN_ASSERT(m_UpstreamCache); - auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbPackage, - .CacheKey = {Ref.BucketSegment, Ref.HashKey}, - .PayloadIds = std::move(ValidAttachments)}); + m_UpstreamCache->EnqueueUpstream( + {.Type = ZenContentType::kCbPackage, .Key = {Ref.BucketSegment, Ref.HashKey}, .PayloadIds = std::move(ValidAttachments)}); } Request.WriteResponse(HttpResponseCode::Created); @@ -584,7 +585,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request void HttpStructuredCacheService::HandleCachePayloadRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy) { - switch (auto Verb = Request.RequestVerb()) + switch (Request.RequestVerb()) { using enum HttpVerb; @@ -768,7 +769,7 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef& void HttpStructuredCacheService::HandleRpcRequest(zen::HttpServerRequest& Request) { - switch (auto Verb = Request.RequestVerb()) + switch (Request.RequestVerb()) { using enum HttpVerb; @@ -927,8 +928,8 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req { ZEN_DEBUG("Uncompressed payload '{}' from upstream cache record '{}/{}'", HashView.AsHash(), - Params.CacheKey.Bucket, - Params.CacheKey.Hash); + Params.Key.Bucket, + Params.Key.Hash); Count.Invalid++; } } @@ -948,8 +949,8 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req if (CacheValue) { ZEN_DEBUG("HIT - '{}/{}' {} '{}' attachments '{}/{}/{}' (new/valid/total) (UPSTREAM)", - Params.CacheKey.Bucket, - Params.CacheKey.Hash, + Params.Key.Bucket, + Params.Key.Hash, NiceBytes(CacheValue.GetSize()), ToString(HttpContentType::kCbPackage), Count.New, @@ -959,7 +960,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req CacheValue.SetContentType(ZenContentType::kCbObject); CacheValues[Params.KeyIndex] = CacheValue; - m_CacheStore.Put(Params.CacheKey.Bucket, Params.CacheKey.Hash, {.Value = CacheValue}); + m_CacheStore.Put(Params.Key.Bucket, Params.Key.Hash, {.Value = CacheValue}); m_CacheStats.HitCount++; m_CacheStats.UpstreamHitCount++; @@ -967,7 +968,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req else { const bool IsPartial = Count.Valid != Count.Total; - ZEN_DEBUG("MISS - '{}/{}' {}", Params.CacheKey.Bucket, Params.CacheKey.Hash, IsPartial ? "(partial)"sv : ""sv); + ZEN_DEBUG("MISS - '{}/{}' {}", Params.Key.Bucket, Params.Key.Hash, IsPartial ? "(partial)"sv : ""sv); m_CacheStats.MissCount++; } }; @@ -1135,7 +1136,7 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re const auto OnCachePayloadGetComplete = [this, &ChunkRequests, &Chunks](CachePayloadGetCompleteParams&& Params) { if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Params.Payload))) { - auto InsertResult = m_CidStore.AddChunk(Compressed); + m_CidStore.AddChunk(Compressed); ZEN_DEBUG("HIT - '{}/{}/{}' {} ({})", Params.Request.Key.Bucket, diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 44c7b728b..718a8db51 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -2,7 +2,7 @@ #include "structuredcachestore.h" -#include "cachetracking.h" +#include <zencore/except.h> #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinarypackage.h> @@ -23,7 +23,10 @@ #include <zenstore/caslog.h> #include <zenstore/cidstore.h> -#include <chrono> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #include <concepts> #include <memory_resource> #include <ranges> @@ -38,8 +41,7 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { using namespace fmt::literals; -using PathBuilder = WideStringBuilder<256>; -namespace fs = std::filesystem; +namespace fs = std::filesystem; static CbObject LoadCompactBinaryObject(const fs::path& Path) @@ -487,7 +489,7 @@ private: uint64_t m_SobsCursor = 0; std::atomic_uint64_t m_TotalSize{}; - void BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey); + void BuildPath(PathBuilderBase& Path, const IoHash& HashKey); void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value); bool GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue); void DeleteStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, const fs::path& Path, std::error_code& Ec); @@ -619,17 +621,17 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const fs::path& BucketDir, const bool Is } void -ZenCacheDiskLayer::CacheBucket::BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey) +ZenCacheDiskLayer::CacheBucket::BuildPath(PathBuilderBase& Path, const IoHash& HashKey) { char HexString[sizeof(HashKey.Hash) * 2]; ToHexBytes(HashKey.Hash, sizeof HashKey.Hash, HexString); - Path.Append(m_BucketDir.c_str()); + Path.Append(m_BucketDir); Path.Append(L"/blob/"); Path.AppendAsciiRange(HexString, HexString + 3); - Path.Append(L"/"); + Path.AppendSeparator(); Path.AppendAsciiRange(HexString + 3, HexString + 5); - Path.Append(L"/"); + Path.AppendSeparator(); Path.AppendAsciiRange(HexString + 5, HexString + sizeof(HexString)); } @@ -650,12 +652,12 @@ ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc, Zen bool ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue) { - PathBuilder DataFilePath; + PathBuilder<256> DataFilePath; BuildPath(DataFilePath, HashKey); RwLock::SharedLockScope ValueLock(LockForHash(HashKey)); - if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.c_str())) + if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.ToPath())) { OutValue.Value = Data; OutValue.Value.SetContentType(Loc.GetContentType()); @@ -970,8 +972,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx) // Remove all standalone file(s) // NOTE: This can probably be made asynchronously { - std::error_code Ec; - PathBuilder Path; + std::error_code Ec; + PathBuilder<256> Path; for (const auto& Entry : ExpiredEntries) { @@ -988,7 +990,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx) if (Ec) { - ZEN_WARN("delete expired z$ standalone file '{}' FAILED, reason '{}'", WideToUtf8(Path.ToString()), Ec.message()); + ZEN_WARN("delete expired z$ standalone file '{}' FAILED, reason '{}'", Path.ToUtf8(), Ec.message()); Ec.clear(); } } @@ -1168,7 +1170,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c { RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey)); - PathBuilder DataFilePath; + PathBuilder<256> DataFilePath; BuildPath(DataFilePath, HashKey); TemporaryFile DataFile; @@ -1190,7 +1192,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c // Move file into place (atomically) - std::filesystem::path FsPath{DataFilePath.c_str()}; + std::filesystem::path FsPath{DataFilePath.ToPath()}; DataFile.MoveTemporaryIntoPlace(FsPath, Ec); @@ -1200,7 +1202,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c do { - std::filesystem::path ParentPath = std::filesystem::path(DataFilePath.c_str()).parent_path(); + std::filesystem::path ParentPath = FsPath.parent_path(); CreateDirectories(ParentPath); DataFile.MoveTemporaryIntoPlace(FsPath, Ec); @@ -1225,7 +1227,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c if (Ec) { - throw std::system_error(Ec, "Failed to finalize file '{}'"_format(WideToUtf8(DataFilePath))); + throw std::system_error(Ec, "Failed to finalize file '{}'"_format(DataFilePath.ToUtf8())); } } @@ -1371,11 +1373,11 @@ ZenCacheDiskLayer::DiscoverBuckets() virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override { - Dirs.push_back(std::wstring(DirectoryName)); + Dirs.push_back((decltype(Dirs)::value_type)(DirectoryName)); return false; } - std::vector<std::wstring> Dirs; + std::vector<std::filesystem::path::string_type> Dirs; } Visit; Traversal.TraverseFileSystem(m_RootDir, Visit); @@ -1384,11 +1386,15 @@ ZenCacheDiskLayer::DiscoverBuckets() RwLock::ExclusiveLockScope _(m_Lock); - for (const std::wstring& BucketName : Visit.Dirs) + for (const auto& BucketName : Visit.Dirs) { // New bucket needs to be created - const std::string BucketName8 = ToUtf8(BucketName); +#if ZEN_PLATFORM_WINDOWS + std::string BucketName8 = WideToUtf8(BucketName); +#else + const auto& BucketName8 = BucketName; +#endif if (auto It = m_Buckets.find(BucketName8); It != m_Buckets.end()) { diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index b64b1353e..41b47b409 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -23,9 +23,9 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { +class PathBuilderBase; class CasStore; class CasGc; -class WideStringBuilderBase; class ZenCacheTracker; /****************************************************************************** diff --git a/zenserver/casstore.cpp b/zenserver/casstore.cpp index 88525bd36..872a40df8 100644 --- a/zenserver/casstore.cpp +++ b/zenserver/casstore.cpp @@ -51,6 +51,9 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store) return ServerRequest.WriteResponse(HttpResponseCode::OK); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPut | HttpVerb::kHead); diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index 11ae8b41d..d483fd4f7 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -2,35 +2,35 @@ #include "apply.h" -#include <upstream/jupiter.h> -#include <upstream/upstreamapply.h> -#include <zencore/compactbinary.h> -#include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinarypackage.h> -#include <zencore/compress.h> -#include <zencore/except.h> -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/iobuffer.h> -#include <zencore/iohash.h> -#include <zencore/scopeguard.h> -#include <zencore/windows.h> -#include <zenstore/CAS.h> -#include <zenstore/cidstore.h> - -#if ZEN_PLATFORM_WINDOWS +#if ZEN_WITH_COMPUTE_SERVICES + +# include <upstream/jupiter.h> +# include <upstream/upstreamapply.h> +# include <zencore/compactbinary.h> +# include <zencore/compactbinarybuilder.h> +# include <zencore/compactbinarypackage.h> +# include <zencore/compress.h> +# include <zencore/except.h> +# include <zencore/filesystem.h> +# include <zencore/fmtutils.h> +# include <zencore/iobuffer.h> +# include <zencore/iohash.h> +# include <zencore/scopeguard.h> +# include <zenstore/cas.h> +# include <zenstore/cidstore.h> + +# include <zencore/windows.h> ZEN_THIRD_PARTY_INCLUDES_START # include <AccCtrl.h> # include <AclAPI.h> -# include <sddl.h> # include <UserEnv.h> +# include <sddl.h> # pragma comment(lib, "UserEnv.lib") # include <atlbase.h> ZEN_THIRD_PARTY_INCLUDES_END -#endif -#include <filesystem> -#include <span> +# include <filesystem> +# include <span> using namespace std::literals; @@ -136,6 +136,8 @@ BasicFunctionJob::ExitCode() return gsl::narrow_cast<int>(Ec); } +//////////////////////////////////////////////////////////////////////////////// + struct SandboxedFunctionJob { SandboxedFunctionJob() = default; @@ -326,6 +328,8 @@ SandboxedFunctionJob::SpawnJob(std::filesystem::path ExePath) return true; } +//////////////////////////////////////////////////////////////////////////////// + HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, const std::filesystem::path& BaseDir) : m_Log(logging::Get("apply")) , m_CasStore(Store) @@ -448,6 +452,8 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, SharedBuffer Decompressed = DataView.Decompress(); const uint64_t DecompressedSize = DataView.GetRawSize(); + ZEN_UNUSED(DataHash); + TotalAttachmentBytes += DecompressedSize; ++AttachmentCount; @@ -478,9 +484,15 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, return HttpReq.WriteResponse(HttpResponseCode::NoContent); } break; + + default: + break; } } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -497,6 +509,9 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, case HttpVerb::kPost: break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -622,6 +637,8 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, const IoHash DataHash = Attachment.GetHash(); CompressedBuffer DataView = Attachment.AsCompressedBinary(); + ZEN_UNUSED(DataHash); + const uint64_t CompressedSize = DataView.GetCompressedSize(); TotalAttachmentBytes += CompressedSize; @@ -652,8 +669,14 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, return HttpReq.WriteResponse(HttpResponseCode::OK, Output); } break; + + default: + break; } break; + + default: + break; } }, HttpVerb::kPost); @@ -958,3 +981,5 @@ HttpFunctionService::ExecActionUpstreamResult(const IoHash& WorkerId, const IoHa } } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/compute/apply.h b/zenserver/compute/apply.h index 0d6edf119..161e47e06 100644 --- a/zenserver/compute/apply.h +++ b/zenserver/compute/apply.h @@ -2,13 +2,21 @@ #pragma once -#include <zencore/compactbinary.h> -#include <zencore/iohash.h> -#include <zencore/logging.h> -#include <zenhttp/httpserver.h> +#include <zencore/zencore.h> -#include <filesystem> -#include <unordered_map> +#if !defined(ZEN_WITH_COMPUTE_SERVICES) +# define ZEN_WITH_COMPUTE_SERVICES ZEN_PLATFORM_WINDOWS +#endif + +#if ZEN_WITH_COMPUTE_SERVICES + +# include <zencore/compactbinary.h> +# include <zencore/iohash.h> +# include <zencore/logging.h> +# include <zenhttp/httpserver.h> + +# include <filesystem> +# include <unordered_map> namespace zen { @@ -54,3 +62,5 @@ private: }; } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/config.cpp b/zenserver/config.cpp index 5e5676494..b25d05409 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -9,16 +9,18 @@ #include <zencore/string.h> #include <zenhttp/zenhttp.h> -#pragma warning(push) -#pragma warning(disable : 4267) // warning C4267: '=': conversion from 'size_t' to 'US', possible loss of data -#include <cxxopts.hpp> -#pragma warning(pop) - +ZEN_THIRD_PARTY_INCLUDES_START #include <fmt/format.h> #include <zencore/logging.h> +#include <cxxopts.hpp> #include <sol/sol.hpp> +ZEN_THIRD_PARTY_INCLUDES_END -#include <conio.h> +#if ZEN_PLATFORM_WINDOWS +# include <conio.h> +#else +# include <pwd.h> +#endif #if ZEN_PLATFORM_WINDOWS @@ -86,7 +88,9 @@ PickDefaultStateDirectory() std::filesystem::path PickDefaultStateDirectory() { - return std::filesystem::path("~/.zen"); + int UserId = getuid(); + const passwd* Passwd = getpwuid(UserId); + return std::filesystem::path(Passwd->pw_dir) / ".zen"; } #endif @@ -183,6 +187,22 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) ""); #endif +#if ZEN_WITH_TRACE + options.add_option("ue-trace", + "", + "tracehost", + "Hostname to send the trace to", + cxxopts::value<std::string>(ServerOptions.TraceHost)->default_value(""), + ""); + + options.add_option("ue-trace", + "", + "tracefile", + "Path to write a trace to", + cxxopts::value<std::string>(ServerOptions.TraceFile)->default_value(""), + ""); +#endif // ZEN_WITH_TRACE + options.add_option("diagnostics", "", "crash", @@ -337,8 +357,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) if (result.count("help")) { zen::logging::ConsoleLog().info("{}", options.help()); +#if ZEN_PLATFORM_WINDOWS zen::logging::ConsoleLog().info("Press any key to exit!"); _getch(); +#else + // Assume the user's in a terminal on all other platforms and that + // they'll use less/more/etc. if need be. +#endif exit(0); } @@ -371,7 +396,7 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio { using namespace fmt::literals; - zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(Path.native().c_str()); + zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(Path); if (LuaScript) { @@ -380,6 +405,7 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio lua.open_libraries(sol::lib::base); lua.set_function("getenv", [&](const std::string env) -> sol::object { +#if ZEN_PLATFORM_WINDOWS std::wstring EnvVarValue; size_t RequiredSize = 0; std::wstring EnvWide = zen::Utf8ToWide(env); @@ -391,6 +417,10 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio EnvVarValue.resize(RequiredSize); _wgetenv_s(&RequiredSize, EnvVarValue.data(), RequiredSize, EnvWide.c_str()); return sol::make_object(lua, zen::WideToUtf8(EnvVarValue.c_str())); +#else + ZEN_UNUSED(env); + return sol::make_object(lua, sol::lua_nil); +#endif }); try diff --git a/zenserver/config.h b/zenserver/config.h index 8a507df39..60e9976e0 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -2,8 +2,10 @@ #pragma once +#include <zencore/zencore.h> #include <filesystem> #include <string> +#include <vector> #ifndef ZEN_ENABLE_MESH # define ZEN_ENABLE_MESH 0 @@ -17,6 +19,10 @@ # define ZEN_USE_EXEC 0 #endif +#ifndef ZEN_WITH_TRACE +# define ZEN_WITH_TRACE 0 +#endif + struct ZenUpstreamJupiterConfig { std::string Url; @@ -101,6 +107,10 @@ struct ZenServerOptions bool StructuredCacheEnabled = true; bool ShouldCrash = false; // Option for testing crash handling bool IsFirstRun = false; +#if ZEN_WITH_TRACE + std::string TraceHost; // Host name or IP address to send trace data to + std::string TraceFile; // Path of a file to write a trace +#endif #if ZEN_ENABLE_MESH bool MeshEnabled = false; // Experimental p2p mesh discovery #endif diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp index 728001202..3a5da2de9 100644 --- a/zenserver/diag/logging.cpp +++ b/zenserver/diag/logging.cpp @@ -171,6 +171,7 @@ private: bool EnableVTMode() { +#if ZEN_PLATFORM_WINDOWS // Set output mode to handle virtual terminal sequences HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE); if (hOut == INVALID_HANDLE_VALUE) @@ -189,6 +190,7 @@ EnableVTMode() { return false; } +#endif return true; } @@ -233,13 +235,13 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) auto ConsoleSink = std::make_shared<spdlog::sinks::ansicolor_stdout_sink_mt>(); #if 0 - auto FileSink = std::make_shared<spdlog::sinks::daily_file_sink_mt>(zen::WideToUtf8(LogPath.c_str()), + auto FileSink = std::make_shared<spdlog::sinks::daily_file_sink_mt>(zen::PathToUtf8(LogPath), 0, 0, /* truncate */ false, uint16_t(/* max files */ 14)); #else - auto FileSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::ToUtf8(LogPath), + auto FileSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::PathToUtf8(LogPath), /* max size */ 128 * 1024 * 1024, /* max files */ 16, /* rotate on open */ true); @@ -267,7 +269,7 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) std::filesystem::path HttpLogPath = GlobalOptions.DataDir / "logs/http.log"; - auto HttpSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::WideToUtf8(HttpLogPath.c_str()), + auto HttpSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::PathToUtf8(HttpLogPath), /* max size */ 128 * 1024 * 1024, /* max files */ 16, /* rotate on open */ true); diff --git a/zenserver/experimental/frontend.cpp b/zenserver/experimental/frontend.cpp index 98d570cfe..4bd3ec90a 100644 --- a/zenserver/experimental/frontend.cpp +++ b/zenserver/experimental/frontend.cpp @@ -54,7 +54,7 @@ body { <pre> __________ _________ __ \____ / ____ ____ / _____/_/ |_ ____ _______ ____ - / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \ + / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \ / /_ \ ___/ | | \ / \ | | ( <_> ) | | \/\ ___/ /_______ \ \___ >|___| //_______ / |__| \____/ |__| \___ > \/ \/ \/ \/ \/ diff --git a/zenserver/experimental/usnjournal.cpp b/zenserver/experimental/usnjournal.cpp index 9422dd485..580d71e45 100644 --- a/zenserver/experimental/usnjournal.cpp +++ b/zenserver/experimental/usnjournal.cpp @@ -1,17 +1,20 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "usnjournal.h" - -#include <zencore/except.h> -#include <zencore/logging.h> -#include <zencore/timer.h> #include <zencore/zencore.h> +#if ZEN_PLATFORM_WINDOWS + +# include "usnjournal.h" + +# include <zencore/except.h> +# include <zencore/logging.h> +# include <zencore/timer.h> + ZEN_THIRD_PARTY_INCLUDES_START -#include <atlfile.h> +# include <atlfile.h> ZEN_THIRD_PARTY_INCLUDES_END -#include <filesystem> +# include <filesystem> namespace zen { @@ -344,3 +347,5 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) } } // namespace zen + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/zenserver/experimental/usnjournal.h b/zenserver/experimental/usnjournal.h index db1f59abc..910eb7d06 100644 --- a/zenserver/experimental/usnjournal.h +++ b/zenserver/experimental/usnjournal.h @@ -2,14 +2,16 @@ #pragma once -#include <zencore/windows.h> -#include <zencore/zencore.h> +#if ZEN_PLATFORM_WINDOWS + +# include <zencore/windows.h> +# include <zencore/zencore.h> ZEN_THIRD_PARTY_INCLUDES_START -#include <winioctl.h> +# include <winioctl.h> ZEN_THIRD_PARTY_INCLUDES_END -#include <filesystem> +# include <filesystem> namespace zen { @@ -63,3 +65,5 @@ private: }; } // namespace zen + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index 8f8b6e163..7e0aed328 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -13,13 +13,16 @@ #include <zencore/testing.h> #include <zencore/testutils.h> #include <zencore/timer.h> -#include <zencore/windows.h> #include <zenstore/basicfile.h> #include <zenstore/cas.h> #include <zenstore/caslog.h> #include "config.h" +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #define USE_ROCKSDB 0 ZEN_THIRD_PARTY_INCLUDES_START @@ -122,7 +125,7 @@ struct ProjectStore::OplogStorage : public RefCounted #if USE_ROCKSDB { - std::string RocksdbPath = WideToUtf8((m_OplogStoragePath / "ops.rdb").native().c_str()); + std::string RocksdbPath = PathToUtf8(m_OplogStoragePath / "ops.rdb"); ZEN_DEBUG("opening rocksdb db at '{}'", RocksdbPath); @@ -200,9 +203,8 @@ struct ProjectStore::OplogStorage : public RefCounted CbObject Op(SharedBuffer::MakeView(OpBuffer.Data(), OpBuffer.Size())); - m_NextOpsOffset = - Max(m_NextOpsOffset.load(std::memory_order::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign)); - m_MaxLsn = Max(m_MaxLsn.load(std::memory_order::memory_order_relaxed), LogEntry.OpLsn); + m_NextOpsOffset = Max(m_NextOpsOffset.load(std::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign)); + m_MaxLsn = Max(m_MaxLsn.load(std::memory_order_relaxed), LogEntry.OpLsn); Handler(Op, LogEntry); }); @@ -299,8 +301,8 @@ private: ProjectStore::Oplog::Oplog(std::string_view Id, Project* Project, CidStore& Store, std::filesystem::path BasePath) : m_OuterProject(Project) , m_CidStore(Store) -, m_OplogId(Id) , m_BasePath(BasePath) +, m_OplogId(Id) { m_Storage = new OplogStorage(this, m_BasePath); const bool StoreExists = m_Storage->Exists(); @@ -385,7 +387,7 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId) std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second.ServerPath; - IoBuffer FileChunk = IoBufferBuilder::MakeFromFile(FilePath.native().c_str()); + IoBuffer FileChunk = IoBufferBuilder::MakeFromFile(FilePath); FileChunk.SetContentType(ZenContentType::kBinary); return FileChunk; @@ -465,16 +467,6 @@ ProjectStore::Oplog::AddFileMapping(Oid FileId, IoHash Hash, std::string_view Se return false; } - if (ServerPath[0] == '/' || ClientPath[0] != '/') - { - // This is a special case just to enable tests to use absolute paths. We might want - // to have configuration to control this - if (ServerPath[1] != ':') - { - return false; - } - } - FileMapEntry Entry; Entry.ServerPath = ServerPath; Entry.ClientPath = ClientPath; @@ -680,7 +672,7 @@ ProjectStore::Project::Write() CbObjectWriter Cfg; Cfg << "id" << Identifier; - Cfg << "root" << WideToUtf8(RootDir.c_str()); + Cfg << "root" << PathToUtf8(RootDir); Cfg << "project" << ProjectRootDir; Cfg << "engine" << EngineRootDir; @@ -779,8 +771,6 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId) void ProjectStore::Project::DeleteOplog(std::string_view OplogId) { - bool Exists = false; - { RwLock::ExclusiveLockScope _(m_ProjectLock); @@ -788,8 +778,6 @@ ProjectStore::Project::DeleteOplog(std::string_view OplogId) if (OplogIt != m_Oplogs.end()) { - Exists = true; - m_Oplogs.erase(OplogIt); } } @@ -1647,7 +1635,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) ProjectStore::Oplog& Log = *OplogIt; CbObjectWriter Cb; - Cb << "id"sv << Log.OplogId() << "project"sv << Prj.Identifier << "tempdir"sv << Log.TempDir(); + Cb << "id"sv << Log.OplogId() << "project"sv << Prj.Identifier << "tempdir"sv << Log.TempPath().c_str(); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Cb.Save()); } @@ -1686,6 +1674,9 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) return Req.ServerRequest().WriteResponse(HttpResponseCode::OK); } break; + + default: + break; } }, HttpVerb::kPost | HttpVerb::kGet | HttpVerb::kDelete); @@ -1783,7 +1774,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) const ProjectStore::Project& Prj = *ProjectIt; CbObjectWriter Response; - Response << "id" << Prj.Identifier << "root" << WideToUtf8(Prj.RootDir.c_str()); + Response << "id" << Prj.Identifier << "root" << PathToUtf8(Prj.RootDir); Response.BeginArray("oplogs"sv); Prj.IterateOplogs([&](const ProjectStore::Oplog& I) { Response << "id"sv << I.OplogId(); }); @@ -1809,6 +1800,9 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) return Req.ServerRequest().WriteResponse(HttpResponseCode::NoContent); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost | HttpVerb::kDelete); @@ -1837,6 +1831,7 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request) ////////////////////////////////////////////////////////////////////////// +# if ZEN_PLATFORM_WINDOWS class SecurityAttributes { public: @@ -1869,6 +1864,12 @@ public: } } }; +# else +struct AnyUserSecurityAttributes +{ + int Attributes() { return 0666; } +}; +# endif // ZEN_PLATFORM_WINDOWS ////////////////////////////////////////////////////////////////////////// @@ -1939,6 +1940,7 @@ private: asio::thread_pool m_WorkerThreadPool; asio::io_context m_IoContext; +# if ZEN_PLATFORM_WINDOWS class PipeConnection { enum PipeState @@ -2097,6 +2099,15 @@ private: uint8_t m_MsgBuffer[16384]; }; +# else + class PipeConnection + { + public: + PipeConnection(LocalProjectImpl*) {} + void Accept() {} + void Disconnect() {} + }; +# endif AnyUserSecurityAttributes m_AnyUserSecurityAttributes; std::vector<PipeConnection*> m_ServicePipes; diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h index 43acdf05f..283dec3b2 100644 --- a/zenserver/projectstore.h +++ b/zenserver/projectstore.h @@ -101,7 +101,6 @@ public: const std::string& OplogId() const { return m_OplogId; } - const std::wstring& TempDir() const { return m_TempPath.native(); } const std::filesystem::path& TempPath() const { return m_TempPath; } spdlog::logger& Log() { return m_OuterProject->Log(); } diff --git a/zenserver/testing/launch.cpp b/zenserver/testing/launch.cpp index 55695ac9c..706594b10 100644 --- a/zenserver/testing/launch.cpp +++ b/zenserver/testing/launch.cpp @@ -2,26 +2,29 @@ #include "launch.h" -#include <zencore/compactbinary.h> -#include <zencore/compactbinarybuilder.h> -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/iobuffer.h> -#include <zencore/iohash.h> -#include <zencore/logging.h> -#include <zencore/windows.h> -#include <zenstore/CAS.h> - -#include <AccCtrl.h> -#include <AclAPI.h> -#include <sddl.h> - -#include <UserEnv.h> -#pragma comment(lib, "UserEnv.lib") - -#include <atlbase.h> -#include <filesystem> -#include <span> +#if ZEN_WITH_COMPUTE_SERVICES + +# include <zencore/compactbinary.h> +# include <zencore/compactbinarybuilder.h> +# include <zencore/filesystem.h> +# include <zencore/fmtutils.h> +# include <zencore/iobuffer.h> +# include <zencore/iohash.h> +# include <zencore/logging.h> +# include <zencore/windows.h> +# include <zenstore/cas.h> + +ZEN_THIRD_PARTY_INCLUDES_START +# include <AccCtrl.h> +# include <AclAPI.h> +# include <UserEnv.h> +# include <atlbase.h> +# include <sddl.h> +ZEN_THIRD_PARTY_INCLUDES_END +# pragma comment(lib, "UserEnv.lib") + +# include <filesystem> +# include <span> using namespace std::literals; @@ -127,6 +130,8 @@ BasicJob::ExitCode() return gsl::narrow_cast<int>(Ec); } +//////////////////////////////////////////////////////////////////////////////// + struct SandboxedJob { SandboxedJob() = default; @@ -317,6 +322,8 @@ SandboxedJob::SpawnJob(std::filesystem::path ExePath) return true; } +//////////////////////////////////////////////////////////////////////////////// + HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::path& SandboxBaseDir) : m_Log(logging::Get("exec")) , m_CasStore(Store) @@ -336,13 +343,16 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat case HttpVerb::kPost: break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); // Experimental -#if 0 +# if 0 m_Router.RegisterRoute( "jobs/sandbox", [this](HttpRouterRequest& Req) { @@ -362,10 +372,13 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat Job.SpawnJob("c:\\windows\\system32\\cmd.exe"); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); -#endif +# endif m_Router.RegisterRoute( "jobs/prep", @@ -413,6 +426,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat return HttpReq.WriteResponse(HttpResponseCode::OK, Response); } break; + + default: + break; } }, HttpVerb::kPost); @@ -496,6 +512,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -530,3 +549,5 @@ HttpLaunchService::CreateNewSandbox() } } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/testing/launch.h b/zenserver/testing/launch.h index 49f12e2ec..925fa18b0 100644 --- a/zenserver/testing/launch.h +++ b/zenserver/testing/launch.h @@ -2,10 +2,18 @@ #pragma once -#include <zencore/logging.h> -#include <zenhttp/httpserver.h> +#include <zencore/zencore.h> -#include <filesystem> +#if !defined(ZEN_WITH_COMPUTE_SERVICES) +# define ZEN_WITH_COMPUTE_SERVICES ZEN_PLATFORM_WINDOWS +#endif + +#if ZEN_WITH_COMPUTE_SERVICES + +# include <zencore/logging.h> +# include <zenhttp/httpserver.h> + +# include <filesystem> namespace zen { @@ -36,3 +44,5 @@ private: }; } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index f9be068ec..177184591 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -2,7 +2,6 @@ #include "jupiter.h" -#include "cache/structuredcachestore.h" #include "diag/formatters.h" #include "diag/logging.h" @@ -58,7 +57,7 @@ namespace detail { cpr::Session& GetSession() { return Session; } private: - friend class CloudCacheClient; + friend class zen::CloudCacheClient; CloudCacheClient& OwnerClient; CloudCacheAccessToken AccessToken; @@ -116,11 +115,7 @@ CloudCacheSession::GetDerivedData(std::string_view BucketId, std::string_view Ke const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - return {.Response = Buffer, - .Bytes = Response.downloaded_bytes, - .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, - .Success = Success}; + return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } CloudCacheResult @@ -165,11 +160,7 @@ CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key, ZenConte const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - return {.Response = Buffer, - .Bytes = Response.downloaded_bytes, - .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, - .Success = Success}; + return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } CloudCacheResult @@ -206,11 +197,7 @@ CloudCacheSession::GetBlob(const IoHash& Key) const IoBuffer Buffer = Success && Response.text.size() > 0 ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - return {.Response = Buffer, - .Bytes = Response.downloaded_bytes, - .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, - .Success = Success}; + return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } CloudCacheResult @@ -246,11 +233,7 @@ CloudCacheSession::GetCompressedBlob(const IoHash& Key) const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - return {.Response = Buffer, - .Bytes = Response.downloaded_bytes, - .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, - .Success = Success}; + return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } CloudCacheResult @@ -286,11 +269,7 @@ CloudCacheSession::GetObject(const IoHash& Key) const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - return {.Response = Buffer, - .Bytes = Response.downloaded_bytes, - .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, - .Success = Success}; + return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } CloudCacheResult @@ -327,12 +306,9 @@ CloudCacheSession::PutDerivedData(std::string_view BucketId, std::string_view Ke return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - const bool Success = Response.status_code == 200 || Response.status_code == 201; - return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, - .Success = Success}; + .Success = (Response.status_code == 200 || Response.status_code == 201)}; } CloudCacheResult @@ -388,7 +364,6 @@ CloudCacheSession::PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer PutRefResult Result; Result.Success = (Response.status_code == 200 || Response.status_code == 201); - Result.ErrorCode = !Result.Success ? Response.status_code : 0; Result.Bytes = Response.uploaded_bytes; Result.ElapsedSeconds = Response.elapsed; @@ -453,7 +428,6 @@ CloudCacheSession::FinalizeRef(std::string_view BucketId, const IoHash& Key, con FinalizeRefResult Result; Result.Success = (Response.status_code == 200 || Response.status_code == 201); - Result.ErrorCode = !Result.Success ? Response.status_code : 0; Result.Bytes = Response.uploaded_bytes; Result.ElapsedSeconds = Response.elapsed; @@ -504,12 +478,9 @@ CloudCacheSession::PutBlob(const IoHash& Key, IoBuffer Blob) return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - const bool Success = Response.status_code == 200 || Response.status_code == 201; - return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, - .Success = Success}; + .Success = (Response.status_code == 200 || Response.status_code == 201)}; } CloudCacheResult @@ -542,12 +513,9 @@ CloudCacheSession::PutCompressedBlob(const IoHash& Key, IoBuffer Blob) return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - const bool Success = Response.status_code == 200 || Response.status_code == 201; - return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, - .Success = Success}; + .Success = (Response.status_code == 200 || Response.status_code == 201)}; } CloudCacheResult @@ -580,12 +548,9 @@ CloudCacheSession::PutObject(const IoHash& Key, IoBuffer Object) return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - const bool Success = Response.status_code == 200 || Response.status_code == 201; - return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, - .Success = Success}; + .Success = (Response.status_code == 200 || Response.status_code == 201)}; } CloudCacheResult @@ -619,9 +584,7 @@ CloudCacheSession::RefExists(std::string_view BucketId, const IoHash& Key) return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - const bool Success = Response.status_code == 200; - - return {.ElapsedSeconds = Response.elapsed, .ErrorCode = !Success ? Response.status_code : 0, .Success = Success}; + return {.ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } CloudCacheResult @@ -690,9 +653,7 @@ CloudCacheSession::PostComputeTasks(std::string_view ChannelId, IoBuffer TasksDa return {.ErrorCode = 401, .Reason = std::string("Invalid access token")}; } - const bool Success = Response.status_code == 200; - - return {.ElapsedSeconds = Response.elapsed, .ErrorCode = !Success ? Response.status_code : 0, .Success = Success}; + return {.ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } CloudCacheResult @@ -728,11 +689,7 @@ CloudCacheSession::GetComputeUpdates(std::string_view ChannelId, const uint32_t const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - return {.Response = Buffer, - .Bytes = Response.downloaded_bytes, - .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, - .Success = Success}; + return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } CloudCacheResult @@ -768,11 +725,7 @@ CloudCacheSession::GetObjectTree(const IoHash& Key) const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); - return {.Response = Buffer, - .Bytes = Response.downloaded_bytes, - .ElapsedSeconds = Response.elapsed, - .ErrorCode = !Success ? Response.status_code : 0, - .Success = Success}; + return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; } std::vector<IoHash> diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp index 05be5f65c..f8a8c3e62 100644 --- a/zenserver/upstream/upstreamapply.cpp +++ b/zenserver/upstream/upstreamapply.cpp @@ -1,37 +1,40 @@ // Copyright Epic Games, Inc. All Rights Reserved. #include "upstreamapply.h" -#include "jupiter.h" -#include "zen.h" - -#include <zencore/blockingqueue.h> -#include <zencore/compactbinary.h> -#include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinarypackage.h> -#include <zencore/compactbinaryvalidation.h> -#include <zencore/compress.h> -#include <zencore/fmtutils.h> -#include <zencore/session.h> -#include <zencore/stats.h> -#include <zencore/stream.h> -#include <zencore/thread.h> -#include <zencore/timer.h> - -#include <zenstore/cas.h> -#include <zenstore/cidstore.h> - -#include "cache/structuredcachestore.h" -#include "diag/logging.h" - -#include <fmt/format.h> - -#include <algorithm> -#include <atomic> -#include <map> -#include <set> -#include <stack> -#include <thread> -#include <unordered_map> + +#if ZEN_WITH_COMPUTE_SERVICES + +# include "jupiter.h" +# include "zen.h" + +# include <zencore/blockingqueue.h> +# include <zencore/compactbinary.h> +# include <zencore/compactbinarybuilder.h> +# include <zencore/compactbinarypackage.h> +# include <zencore/compactbinaryvalidation.h> +# include <zencore/compress.h> +# include <zencore/fmtutils.h> +# include <zencore/session.h> +# include <zencore/stats.h> +# include <zencore/stream.h> +# include <zencore/thread.h> +# include <zencore/timer.h> + +# include <zenstore/cas.h> +# include <zenstore/cidstore.h> + +# include "cache/structuredcachestore.h" +# include "diag/logging.h" + +# include <fmt/format.h> + +# include <algorithm> +# include <atomic> +# include <map> +# include <set> +# include <stack> +# include <thread> +# include <unordered_map> namespace zen { @@ -1569,3 +1572,5 @@ MakeHordeUpstreamEndpoint(const CloudCacheClientOptions& Options, CasStore& CasS } } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/upstream/upstreamapply.h b/zenserver/upstream/upstreamapply.h index 98f193c02..ed73ec7f8 100644 --- a/zenserver/upstream/upstreamapply.h +++ b/zenserver/upstream/upstreamapply.h @@ -2,16 +2,20 @@ #pragma once -#include <zencore/compactbinarypackage.h> -#include <zencore/iobuffer.h> -#include <zencore/iohash.h> -#include <zencore/zencore.h> +#include "compute/apply.h" -#include <atomic> -#include <chrono> -#include <memory> -#include <unordered_map> -#include <unordered_set> +#if ZEN_WITH_COMPUTE_SERVICES + +# include <zencore/compactbinarypackage.h> +# include <zencore/iobuffer.h> +# include <zencore/iohash.h> +# include <zencore/zencore.h> + +# include <atomic> +# include <chrono> +# include <memory> +# include <unordered_map> +# include <unordered_set> namespace zen { @@ -170,3 +174,5 @@ std::unique_ptr<UpstreamApplyEndpoint> MakeHordeUpstreamEndpoint(const CloudCach CidStore& CidStore); } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index e2dc09872..6283457e3 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -195,7 +195,7 @@ namespace detail { } } - OnComplete({.CacheKey = CacheKey, .KeyIndex = Index, .Record = Record, .Package = Package}); + OnComplete({.Key = CacheKey, .KeyIndex = Index, .Record = Record, .Package = Package}); } return Result; @@ -271,18 +271,15 @@ namespace detail { if (CacheRecord.Type == ZenContentType::kBinary) { CloudCacheResult Result; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { if (m_UseLegacyDdc) { - Result = Session.PutDerivedData(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue); + Result = Session.PutDerivedData(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RecordValue); } else { - Result = Session.PutRef(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, - RecordValue, - ZenContentType::kBinary); + Result = Session.PutRef(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RecordValue, ZenContentType::kBinary); } } @@ -335,16 +332,15 @@ namespace detail { PutRefResult RefResult; for (int32_t Attempt = 0; Attempt < MaxAttempts && !RefResult.Success; Attempt++) { - RefResult = - Session.PutRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue, ZenContentType::kCbObject); + RefResult = Session.PutRef(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RecordValue, ZenContentType::kCbObject); } m_HealthOk = RefResult.ErrorCode == 0; if (!RefResult.Success) { - return {.Reason = "upload cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + return {.Reason = "upload cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, RefResult.Reason), .Success = false}; } @@ -359,13 +355,13 @@ namespace detail { } const IoHash RefHash = IoHash::HashBuffer(RecordValue); - FinalizeRefResult FinalizeResult = Session.FinalizeRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RefHash); + FinalizeRefResult FinalizeResult = Session.FinalizeRef(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RefHash); m_HealthOk = FinalizeResult.ErrorCode == 0; if (!FinalizeResult.Success) { - return {.Reason = "finalize cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + return {.Reason = "finalize cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, FinalizeResult.Reason), .Success = false}; } @@ -377,13 +373,13 @@ namespace detail { return {.Reason = std::move(Reason), .Success = false}; } - FinalizeResult = Session.FinalizeRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RefHash); + FinalizeResult = Session.FinalizeRef(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RefHash); m_HealthOk = FinalizeResult.ErrorCode == 0; if (!FinalizeResult.Success) { - return {.Reason = "finalize '{}/{}' FAILED, reason '{}'"_format(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + return {.Reason = "finalize '{}/{}' FAILED, reason '{}'"_format(CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, FinalizeResult.Reason), .Success = false}; } @@ -396,8 +392,8 @@ namespace detail { Sb << MissingHash.ToHexString() << ","; } - return {.Reason = "finalize '{}/{}' FAILED, still needs payload(s) '{}'"_format(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + return {.Reason = "finalize '{}/{}' FAILED, still needs payload(s) '{}'"_format(CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, Sb.ToString()), .Success = false}; } @@ -606,8 +602,7 @@ namespace detail { for (size_t LocalIndex = 0; CbFieldView Record : BatchResponse.GetObject()["Result"sv]) { const size_t Index = IndexMap[LocalIndex++]; - OnComplete( - {.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = Record.AsObjectView(), .Package = BatchResponse}); + OnComplete({.Key = CacheKeys[Index], .KeyIndex = Index, .Record = Record.AsObjectView(), .Package = BatchResponse}); } return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = true}; @@ -620,7 +615,7 @@ namespace detail { for (size_t Index : KeyIndex) { - OnComplete({.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()}); + OnComplete({.Key = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()}); } return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}; @@ -772,12 +767,9 @@ namespace detail { Package.Save(MemStream); IoBuffer PackagePayload(IoBuffer::Wrap, MemStream.Data(), MemStream.Size()); - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { - Result = Session.PutCacheRecord(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, - PackagePayload, - CacheRecord.Type); + Result = Session.PutCacheRecord(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, PackagePayload, CacheRecord.Type); m_HealthOk = Result.ErrorCode == 0; } @@ -790,10 +782,10 @@ namespace detail { for (size_t Idx = 0, Count = Payloads.size(); Idx < Count; Idx++) { Result.Success = false; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { - Result = Session.PutCachePayload(CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + Result = Session.PutCachePayload(CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, CacheRecord.PayloadIds[Idx], Payloads[Idx]); @@ -813,10 +805,9 @@ namespace detail { } Result.Success = false; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { - Result = - Session.PutCacheRecord(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue, CacheRecord.Type); + Result = Session.PutCacheRecord(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RecordValue, CacheRecord.Type); m_HealthOk = Result.ErrorCode == 0; } @@ -1099,7 +1090,7 @@ public: for (size_t Index : MissingKeys) { - OnComplete({.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()}); + OnComplete({.Key = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()}); } } @@ -1236,11 +1227,9 @@ private: ZenCacheValue CacheValue; std::vector<IoBuffer> Payloads; - if (!m_CacheStore.Get(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, CacheValue)) + if (!m_CacheStore.Get(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, CacheValue)) { - ZEN_WARN("process upstream FAILED, '{}/{}', cache record doesn't exist", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash); + ZEN_WARN("process upstream FAILED, '{}/{}', cache record doesn't exist", CacheRecord.Key.Bucket, CacheRecord.Key.Hash); return; } @@ -1253,8 +1242,8 @@ private: else { ZEN_WARN("process upstream FAILED, '{}/{}/{}', payload doesn't exist in CAS", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, PayloadId); return; } @@ -1270,8 +1259,8 @@ private: if (!Result.Success) { ZEN_WARN("upload cache record '{}/{}' FAILED, endpoint '{}', reason '{}'", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, Endpoint->GetEndpointInfo().Url, Result.Reason); } @@ -1292,10 +1281,7 @@ private: } catch (std::exception& Err) { - ZEN_ERROR("upload cache record '{}/{}' FAILED, reason '{}'", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, - Err.what()); + ZEN_ERROR("upload cache record '{}/{}' FAILED, reason '{}'", CacheRecord.Key.Bucket, CacheRecord.Key.Hash, Err.what()); } } diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h index 12287198d..c463c4996 100644 --- a/zenserver/upstream/upstreamcache.h +++ b/zenserver/upstream/upstreamcache.h @@ -25,7 +25,7 @@ struct ZenStructuredCacheClientOptions; struct UpstreamCacheRecord { ZenContentType Type = ZenContentType::kBinary; - CacheKey CacheKey; + CacheKey Key; std::vector<IoHash> PayloadIds; }; @@ -83,7 +83,7 @@ struct UpstreamEndpointStats struct CacheRecordGetCompleteParams { - const CacheKey& CacheKey; + const CacheKey& Key; size_t KeyIndex = ~size_t(0); const CbObjectView& Record; const CbPackage& Package; diff --git a/zenserver/upstream/zen.h b/zenserver/upstream/zen.h index df975df1f..e97ce755d 100644 --- a/zenserver/upstream/zen.h +++ b/zenserver/upstream/zen.h @@ -16,6 +16,7 @@ ZEN_THIRD_PARTY_INCLUDES_START ZEN_THIRD_PARTY_INCLUDES_END #include <chrono> +#include <list> struct ZenCacheValue; diff --git a/zenserver/windows/service.cpp b/zenserver/windows/service.cpp index 23cefb7b5..89bacab0b 100644 --- a/zenserver/windows/service.cpp +++ b/zenserver/windows/service.cpp @@ -2,14 +2,18 @@ #include "service.h" -#include <zencore/except.h> #include <zencore/zencore.h> -#include <stdio.h> -#include <tchar.h> -#include <zencore/windows.h> +#if ZEN_PLATFORM_WINDOWS -#define SVCNAME L"Zen Store" +# include <zencore/except.h> +# include <zencore/zencore.h> + +# include <stdio.h> +# include <tchar.h> +# include <zencore/windows.h> + +# define SVCNAME L"Zen Store" SERVICE_STATUS gSvcStatus; SERVICE_STATUS_HANDLE gSvcStatusHandle; @@ -638,3 +642,5 @@ SvcReportEvent(LPTSTR szFunction) // DeregisterEventSource(hEventSource); //} } + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua index 2fe32112b..9e12df3e3 100644 --- a/zenserver/xmake.lua +++ b/zenserver/xmake.lua @@ -18,6 +18,7 @@ target("zenserver") end add_options("vfs") + add_options("compute") add_packages( "vcpkg::sentry-native", @@ -43,7 +44,6 @@ target("zenserver") else commit = "dbg-" .. commit end - target:add("defines","BUILD_VERSION=\"" .. commit .. "\"") - print("build version " .. commit) + target:add("defines", "BUILD_VERSION=\"" .. commit .. "\"") end end) diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index e14f93f5b..49bb3004d 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -12,13 +12,17 @@ #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/timer.h> -#include <zencore/windows.h> +#include <zencore/trace.h> #include <zenhttp/httpserver.h> #include <zenstore/basicfile.h> #include <zenstore/cas.h> #include <zenstore/cidstore.h> #include <zenutil/zenserverprocess.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #if ZEN_USE_MIMALLOC ZEN_THIRD_PARTY_INCLUDES_START # include <mimalloc-new-delete.h> @@ -138,7 +142,7 @@ namespace utils { if (!ErrorCode) { - for (const asio::ip::tcp::endpoint& Ep : Endpoints) + for (const asio::ip::tcp::endpoint Ep : Endpoints) { OutEndpoints.push_back("http://{}:{}"_format(Ep.address().to_string(), Ep.port())); } @@ -225,17 +229,23 @@ public: m_LocalProjectService = zen::LocalProjectService::New(*m_CasStore, m_ProjectStore); #endif - ZEN_INFO("instantiating compute services"); - #if ZEN_USE_EXEC std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox"; zen::CreateDirectories(SandboxDir); m_HttpLaunchService = std::make_unique<zen::HttpLaunchService>(*m_CasStore, SandboxDir); #endif +#if ZEN_WITH_COMPUTE_SERVICES + ZEN_INFO("instantiating compute services"); + + std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox"; + zen::CreateDirectories(SandboxDir); + m_HttpLaunchService = std::make_unique<zen::HttpLaunchService>(*m_CasStore, SandboxDir); + std::filesystem::path ApplySandboxDir = m_DataRoot / "exec" / "apply"; zen::CreateDirectories(ApplySandboxDir); m_HttpFunctionService = std::make_unique<zen::HttpFunctionService>(*m_CasStore, *m_CidStore, ApplySandboxDir); +#endif // ZEN_WITH_COMPUTE_SERVICES if (ServerOptions.StructuredCacheEnabled) { @@ -273,17 +283,17 @@ public: m_Http->RegisterService(*m_StructuredCacheService); } -#if ZEN_USE_EXEC +#if ZEN_WITH_COMPUTE_SERVICES if (m_HttpLaunchService) { m_Http->RegisterService(*m_HttpLaunchService); } -#endif if (m_HttpFunctionService) { m_Http->RegisterService(*m_HttpFunctionService); } +#endif // ZEN_WITH_COMPUTE_SERVICES m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot); @@ -343,7 +353,7 @@ public: if (m_DebugOptionForcedCrash) { - __debugbreak(); + ZEN_DEBUG_BREAK(); } const bool IsInteractiveMode = zen::IsInteractiveSession() && !m_TestMode; @@ -404,7 +414,7 @@ public: for (auto& PidEntry : m_ServerEntry->SponsorPids) { - if (uint32_t ThisPid = PidEntry.load(std::memory_order::memory_order_relaxed)) + if (uint32_t ThisPid = PidEntry.load(std::memory_order_relaxed)) { if (PidEntry.compare_exchange_strong(ThisPid, 0)) { @@ -523,13 +533,17 @@ private: zen::HttpTestingService m_TestingService; zen::HttpCasService m_CasService{*m_CasStore}; zen::RefPtr<zen::ProjectStore> m_ProjectStore; + zen::Ref<zen::LocalProjectService> m_LocalProjectService; std::unique_ptr<zen::HttpProjectService> m_HttpProjectService; std::unique_ptr<zen::HttpStructuredCacheService> m_StructuredCacheService; zen::HttpAdminService m_AdminService{m_GcScheduler}; zen::HttpHealthService m_HealthService; zen::Mesh m_ZenMesh{m_IoContext}; - std::unique_ptr<zen::HttpFunctionService> m_HttpFunctionService; - std::unique_ptr<zen::HttpFrontendService> m_FrontendService; +#if ZEN_WITH_COMPUTE_SERVICES + std::unique_ptr<zen::HttpLaunchService> m_HttpLaunchService; + std::unique_ptr<zen::HttpFunctionService> m_HttpFunctionService; +#endif + std::unique_ptr<zen::HttpFrontendService> m_FrontendService; #if ZEN_USE_EXEC std::unique_ptr<zen::HttpLaunchService> m_HttpLaunchService; @@ -766,33 +780,34 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) new zen::HttpStructuredCacheService(*m_CacheStore, *m_CidStore, m_StatsService, m_StatusService, std::move(UpstreamCache))); } -} // namespace zen +//////////////////////////////////////////////////////////////////////////////// -class ZenWindowsService : public WindowsService +class ZenEntryPoint { public: - ZenWindowsService(ZenServerOptions& ServerOptions) : m_ServerOptions(ServerOptions) {} - - ZenWindowsService(const ZenWindowsService&) = delete; - ZenWindowsService& operator=(const ZenWindowsService&) = delete; - - virtual int Run() override; + ZenEntryPoint(ZenServerOptions& ServerOptions); + ZenEntryPoint(const ZenEntryPoint&) = delete; + ZenEntryPoint& operator=(const ZenEntryPoint&) = delete; + int Run(); private: ZenServerOptions& m_ServerOptions; zen::LockFile m_LockFile; }; -int -ZenWindowsService::Run() +ZenEntryPoint::ZenEntryPoint(ZenServerOptions& ServerOptions) : m_ServerOptions(ServerOptions) { - using namespace zen; +} +int +ZenEntryPoint::Run() +{ #if USE_SENTRY // Initialize sentry.io client sentry_options_t* SentryOptions = sentry_options_new(); sentry_options_set_dsn(SentryOptions, "https://[email protected]/5919284"); + sentry_options_set_database_path(SentryOptions, PathToUtf8(m_ServerOptions.DataDir / ".sentry-native").c_str()); sentry_init(SentryOptions); auto _ = zen::MakeGuard([] { sentry_close(); }); @@ -812,8 +827,8 @@ ZenWindowsService::Run() auto MakeLockData = [&] { CbObjectWriter Cbo; - Cbo << "pid" << _getpid() << "data" << ToUtf8(ServerOptions.DataDir) << "port" << ServerOptions.BasePort << "session_id" - << GetSessionId() << "ready" << IsReady; + Cbo << "pid" << zen::GetCurrentProcessId() << "data" << PathToUtf8(ServerOptions.DataDir) << "port" << ServerOptions.BasePort + << "session_id" << GetSessionId() << "ready" << IsReady; return Cbo.Save(); }; @@ -880,6 +895,10 @@ ZenWindowsService::Run() ZEN_INFO("shutdown signal received"); Server.RequestExit(0); } + else + { + ZEN_INFO("shutdown signal wait() failed"); + } }}); // If we have a parent process, establish the mechanisms we need @@ -913,6 +932,36 @@ ZenWindowsService::Run() return 0; } +} // namespace zen + +//////////////////////////////////////////////////////////////////////////////// + +#if ZEN_PLATFORM_WINDOWS + +class ZenWindowsService : public WindowsService +{ +public: + ZenWindowsService(ZenServerOptions& ServerOptions) : m_EntryPoint(ServerOptions) {} + + ZenWindowsService(const ZenWindowsService&) = delete; + ZenWindowsService& operator=(const ZenWindowsService&) = delete; + + virtual int Run() override; + +private: + zen::ZenEntryPoint m_EntryPoint; +}; + +int +ZenWindowsService::Run() +{ + return m_EntryPoint.Run(); +} + +#endif // ZEN_PLATFORM_WINDOWS + +//////////////////////////////////////////////////////////////////////////////// + #if ZEN_WITH_TESTS int test_main(int argc, char** argv) @@ -960,6 +1009,17 @@ main(int argc, char* argv[]) std::filesystem::create_directories(ServerOptions.DataDir); } +#if ZEN_WITH_TRACE + if (ServerOptions.TraceHost.size()) + { + TraceInit(ServerOptions.TraceHost.c_str(), TraceType::Network); + } + else if (ServerOptions.TraceFile.size()) + { + TraceInit(ServerOptions.TraceFile.c_str(), TraceType::File); + } +#endif // ZEN_WITH_TRACE + #if ZEN_PLATFORM_WINDOWS if (ServerOptions.InstallService) { @@ -974,10 +1034,18 @@ main(int argc, char* argv[]) std::exit(0); } -#endif ZenWindowsService App(ServerOptions); return App.ServiceMain(); +#else + if (ServerOptions.InstallService || ServerOptions.UninstallService) + { + throw std::runtime_error("Service mode is not supported on this platform"); + } + + ZenEntryPoint App(ServerOptions); + return App.Run(); +#endif // ZEN_PLATFORM_WINDOWS } catch (std::exception& Ex) { |