aboutsummaryrefslogtreecommitdiff
path: root/zenserver
diff options
context:
space:
mode:
authorPer Larsson <[email protected]>2021-12-14 12:34:47 +0100
committerPer Larsson <[email protected]>2021-12-14 12:34:47 +0100
commitb6c6568e1618f10d2160d836b65e35586e3c740f (patch)
treef6a929cf918850bbba87d0ee67cd3482b2d50e24 /zenserver
parentFixed bug in z$ service returning partial cache records and enable small obje... (diff)
parentPartial revert b363c5b (diff)
downloadzen-b6c6568e1618f10d2160d836b65e35586e3c740f.tar.xz
zen-b6c6568e1618f10d2160d836b65e35586e3c740f.zip
Merged main.
Diffstat (limited to 'zenserver')
-rw-r--r--zenserver/admin/admin.cpp2
-rw-r--r--zenserver/cache/cachetracking.cpp40
-rw-r--r--zenserver/cache/cachetracking.h5
-rw-r--r--zenserver/cache/structuredcache.cpp39
-rw-r--r--zenserver/cache/structuredcachestore.cpp50
-rw-r--r--zenserver/cache/structuredcachestore.h2
-rw-r--r--zenserver/casstore.cpp3
-rw-r--r--zenserver/compute/apply.cpp67
-rw-r--r--zenserver/compute/apply.h22
-rw-r--r--zenserver/config.cpp46
-rw-r--r--zenserver/config.h10
-rw-r--r--zenserver/diag/logging.cpp8
-rw-r--r--zenserver/experimental/frontend.cpp2
-rw-r--r--zenserver/experimental/usnjournal.cpp19
-rw-r--r--zenserver/experimental/usnjournal.h12
-rw-r--r--zenserver/projectstore.cpp59
-rw-r--r--zenserver/projectstore.h1
-rw-r--r--zenserver/testing/launch.cpp65
-rw-r--r--zenserver/testing/launch.h16
-rw-r--r--zenserver/upstream/jupiter.cpp75
-rw-r--r--zenserver/upstream/upstreamapply.cpp67
-rw-r--r--zenserver/upstream/upstreamapply.h24
-rw-r--r--zenserver/upstream/upstreamcache.cpp78
-rw-r--r--zenserver/upstream/upstreamcache.h4
-rw-r--r--zenserver/upstream/zen.h1
-rw-r--r--zenserver/windows/service.cpp16
-rw-r--r--zenserver/xmake.lua4
-rw-r--r--zenserver/zenserver.cpp116
28 files changed, 511 insertions, 342 deletions
diff --git a/zenserver/admin/admin.cpp b/zenserver/admin/admin.cpp
index 15314b12a..01c35a1d8 100644
--- a/zenserver/admin/admin.cpp
+++ b/zenserver/admin/admin.cpp
@@ -1,7 +1,5 @@
// Copyright Epic Games, Inc. All Rights Reserved.
-#pragma once
-
#include "admin.h"
#include <zencore/compactbinarybuilder.h>
diff --git a/zenserver/cache/cachetracking.cpp b/zenserver/cache/cachetracking.cpp
index d1c99a597..ae132f37f 100644
--- a/zenserver/cache/cachetracking.cpp
+++ b/zenserver/cache/cachetracking.cpp
@@ -2,24 +2,26 @@
#include "cachetracking.h"
-#include <zencore/compactbinarybuilder.h>
-#include <zencore/compactbinaryvalue.h>
-#include <zencore/endian.h>
-#include <zencore/filesystem.h>
-#include <zencore/logging.h>
-#include <zencore/scopeguard.h>
-#include <zencore/string.h>
+#if ZEN_USE_CACHE_TRACKER
-#include <zencore/testing.h>
-#include <zencore/testutils.h>
+# include <zencore/compactbinarybuilder.h>
+# include <zencore/compactbinaryvalue.h>
+# include <zencore/endian.h>
+# include <zencore/filesystem.h>
+# include <zencore/logging.h>
+# include <zencore/scopeguard.h>
+# include <zencore/string.h>
+
+# include <zencore/testing.h>
+# include <zencore/testutils.h>
ZEN_THIRD_PARTY_INCLUDES_START
-#pragma comment(lib, "Rpcrt4.lib") // RocksDB made me do this
-#include <fmt/format.h>
-#include <rocksdb/db.h>
-#include <tsl/robin_map.h>
-#include <tsl/robin_set.h>
-#include <gsl/gsl-lite.hpp>
+# pragma comment(lib, "Rpcrt4.lib") // RocksDB made me do this
+# include <fmt/format.h>
+# include <rocksdb/db.h>
+# include <tsl/robin_map.h>
+# include <tsl/robin_set.h>
+# include <gsl/gsl-lite.hpp>
ZEN_THIRD_PARTY_INCLUDES_END
namespace zen {
@@ -150,7 +152,7 @@ struct ZenCacheTracker::Impl
{
std::filesystem::path StatsDbPath{StateDirectory / ".zdb"};
- std::string RocksdbPath = ToUtf8(StatsDbPath);
+ std::string RocksdbPath = StatsDbPath.string();
ZEN_DEBUG("opening tracker db at '{}'", RocksdbPath);
@@ -291,7 +293,7 @@ ZenCacheTracker::IterateSnapshots(std::function<void(uint64_t TimeStamp, CbObjec
m_Impl->IterateSnapshots(std::move(Callback));
}
-#if ZEN_WITH_TESTS
+# if ZEN_WITH_TESTS
TEST_CASE("z$.tracker")
{
@@ -365,7 +367,7 @@ TEST_CASE("z$.tracker")
CHECK_EQ(SnapshotCount, 11);
}
-#endif
+# endif
void
cachetracker_forcelink()
@@ -373,3 +375,5 @@ cachetracker_forcelink()
}
} // namespace zen
+
+#endif // ZEN_USE_CACHE_TRACKER
diff --git a/zenserver/cache/cachetracking.h b/zenserver/cache/cachetracking.h
index 06109ebb0..fdfe1a4c7 100644
--- a/zenserver/cache/cachetracking.h
+++ b/zenserver/cache/cachetracking.h
@@ -10,6 +10,9 @@
namespace zen {
+#define ZEN_USE_CACHE_TRACKER 0
+#if ZEN_USE_CACHE_TRACKER
+
class CbObject;
/**
@@ -33,4 +36,6 @@ private:
void cachetracker_forcelink();
+#endif // ZEN_USE_CACHE_TRACKER
+
} // namespace zen
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 22c8c4afb..b322a5d1b 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -13,7 +13,7 @@
#include <zencore/stream.h>
#include <zencore/timer.h>
#include <zenhttp/httpserver.h>
-#include <zenstore/CAS.h>
+#include <zenstore/cas.h>
#include <zenutil/cache/cache.h>
//#include "cachekey.h"
@@ -152,7 +152,7 @@ void
HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Bucket)
{
ZEN_UNUSED(Request, Bucket);
- switch (auto Verb = Request.RequestVerb())
+ switch (Request.RequestVerb())
{
using enum HttpVerb;
@@ -175,13 +175,16 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request,
return Request.WriteResponse(HttpResponseCode::NotFound);
}
break;
+
+ default:
+ break;
}
}
void
HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy)
{
- switch (auto Verb = Request.RequestVerb())
+ switch (Request.RequestVerb())
{
using enum HttpVerb;
@@ -442,7 +445,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
if (StoreUpstream)
{
ZEN_ASSERT(m_UpstreamCache);
- auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kBinary, .CacheKey = {Ref.BucketSegment, Ref.HashKey}});
+ m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kBinary, .Key = {Ref.BucketSegment, Ref.HashKey}});
}
Request.WriteResponse(HttpResponseCode::Created);
@@ -486,9 +489,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
if (StoreUpstream && !IsPartialRecord)
{
ZEN_ASSERT(m_UpstreamCache);
- auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbObject,
- .CacheKey = {Ref.BucketSegment, Ref.HashKey},
- .PayloadIds = std::move(ValidAttachments)});
+ m_UpstreamCache->EnqueueUpstream(
+ {.Type = ZenContentType::kCbObject, .Key = {Ref.BucketSegment, Ref.HashKey}, .PayloadIds = std::move(ValidAttachments)});
}
Request.WriteResponse(HttpResponseCode::Created);
@@ -568,9 +570,8 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
if (StoreUpstream && !IsPartialRecord)
{
ZEN_ASSERT(m_UpstreamCache);
- auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbPackage,
- .CacheKey = {Ref.BucketSegment, Ref.HashKey},
- .PayloadIds = std::move(ValidAttachments)});
+ m_UpstreamCache->EnqueueUpstream(
+ {.Type = ZenContentType::kCbPackage, .Key = {Ref.BucketSegment, Ref.HashKey}, .PayloadIds = std::move(ValidAttachments)});
}
Request.WriteResponse(HttpResponseCode::Created);
@@ -584,7 +585,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
void
HttpStructuredCacheService::HandleCachePayloadRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy)
{
- switch (auto Verb = Request.RequestVerb())
+ switch (Request.RequestVerb())
{
using enum HttpVerb;
@@ -768,7 +769,7 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef&
void
HttpStructuredCacheService::HandleRpcRequest(zen::HttpServerRequest& Request)
{
- switch (auto Verb = Request.RequestVerb())
+ switch (Request.RequestVerb())
{
using enum HttpVerb;
@@ -927,8 +928,8 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req
{
ZEN_DEBUG("Uncompressed payload '{}' from upstream cache record '{}/{}'",
HashView.AsHash(),
- Params.CacheKey.Bucket,
- Params.CacheKey.Hash);
+ Params.Key.Bucket,
+ Params.Key.Hash);
Count.Invalid++;
}
}
@@ -948,8 +949,8 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req
if (CacheValue)
{
ZEN_DEBUG("HIT - '{}/{}' {} '{}' attachments '{}/{}/{}' (new/valid/total) (UPSTREAM)",
- Params.CacheKey.Bucket,
- Params.CacheKey.Hash,
+ Params.Key.Bucket,
+ Params.Key.Hash,
NiceBytes(CacheValue.GetSize()),
ToString(HttpContentType::kCbPackage),
Count.New,
@@ -959,7 +960,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req
CacheValue.SetContentType(ZenContentType::kCbObject);
CacheValues[Params.KeyIndex] = CacheValue;
- m_CacheStore.Put(Params.CacheKey.Bucket, Params.CacheKey.Hash, {.Value = CacheValue});
+ m_CacheStore.Put(Params.Key.Bucket, Params.Key.Hash, {.Value = CacheValue});
m_CacheStats.HitCount++;
m_CacheStats.UpstreamHitCount++;
@@ -967,7 +968,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Req
else
{
const bool IsPartial = Count.Valid != Count.Total;
- ZEN_DEBUG("MISS - '{}/{}' {}", Params.CacheKey.Bucket, Params.CacheKey.Hash, IsPartial ? "(partial)"sv : ""sv);
+ ZEN_DEBUG("MISS - '{}/{}' {}", Params.Key.Bucket, Params.Key.Hash, IsPartial ? "(partial)"sv : ""sv);
m_CacheStats.MissCount++;
}
};
@@ -1135,7 +1136,7 @@ HttpStructuredCacheService::HandleRpcGetCachePayloads(zen::HttpServerRequest& Re
const auto OnCachePayloadGetComplete = [this, &ChunkRequests, &Chunks](CachePayloadGetCompleteParams&& Params) {
if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Params.Payload)))
{
- auto InsertResult = m_CidStore.AddChunk(Compressed);
+ m_CidStore.AddChunk(Compressed);
ZEN_DEBUG("HIT - '{}/{}/{}' {} ({})",
Params.Request.Key.Bucket,
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 44c7b728b..718a8db51 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -2,7 +2,7 @@
#include "structuredcachestore.h"
-#include "cachetracking.h"
+#include <zencore/except.h>
#include <zencore/compactbinarybuilder.h>
#include <zencore/compactbinarypackage.h>
@@ -23,7 +23,10 @@
#include <zenstore/caslog.h>
#include <zenstore/cidstore.h>
-#include <chrono>
+#if ZEN_PLATFORM_WINDOWS
+# include <zencore/windows.h>
+#endif
+
#include <concepts>
#include <memory_resource>
#include <ranges>
@@ -38,8 +41,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
namespace zen {
using namespace fmt::literals;
-using PathBuilder = WideStringBuilder<256>;
-namespace fs = std::filesystem;
+namespace fs = std::filesystem;
static CbObject
LoadCompactBinaryObject(const fs::path& Path)
@@ -487,7 +489,7 @@ private:
uint64_t m_SobsCursor = 0;
std::atomic_uint64_t m_TotalSize{};
- void BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey);
+ void BuildPath(PathBuilderBase& Path, const IoHash& HashKey);
void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
bool GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue);
void DeleteStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, const fs::path& Path, std::error_code& Ec);
@@ -619,17 +621,17 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const fs::path& BucketDir, const bool Is
}
void
-ZenCacheDiskLayer::CacheBucket::BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey)
+ZenCacheDiskLayer::CacheBucket::BuildPath(PathBuilderBase& Path, const IoHash& HashKey)
{
char HexString[sizeof(HashKey.Hash) * 2];
ToHexBytes(HashKey.Hash, sizeof HashKey.Hash, HexString);
- Path.Append(m_BucketDir.c_str());
+ Path.Append(m_BucketDir);
Path.Append(L"/blob/");
Path.AppendAsciiRange(HexString, HexString + 3);
- Path.Append(L"/");
+ Path.AppendSeparator();
Path.AppendAsciiRange(HexString + 3, HexString + 5);
- Path.Append(L"/");
+ Path.AppendSeparator();
Path.AppendAsciiRange(HexString + 5, HexString + sizeof(HexString));
}
@@ -650,12 +652,12 @@ ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc, Zen
bool
ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue)
{
- PathBuilder DataFilePath;
+ PathBuilder<256> DataFilePath;
BuildPath(DataFilePath, HashKey);
RwLock::SharedLockScope ValueLock(LockForHash(HashKey));
- if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.c_str()))
+ if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.ToPath()))
{
OutValue.Value = Data;
OutValue.Value.SetContentType(Loc.GetContentType());
@@ -970,8 +972,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
// Remove all standalone file(s)
// NOTE: This can probably be made asynchronously
{
- std::error_code Ec;
- PathBuilder Path;
+ std::error_code Ec;
+ PathBuilder<256> Path;
for (const auto& Entry : ExpiredEntries)
{
@@ -988,7 +990,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
if (Ec)
{
- ZEN_WARN("delete expired z$ standalone file '{}' FAILED, reason '{}'", WideToUtf8(Path.ToString()), Ec.message());
+ ZEN_WARN("delete expired z$ standalone file '{}' FAILED, reason '{}'", Path.ToUtf8(), Ec.message());
Ec.clear();
}
}
@@ -1168,7 +1170,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
{
RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey));
- PathBuilder DataFilePath;
+ PathBuilder<256> DataFilePath;
BuildPath(DataFilePath, HashKey);
TemporaryFile DataFile;
@@ -1190,7 +1192,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
// Move file into place (atomically)
- std::filesystem::path FsPath{DataFilePath.c_str()};
+ std::filesystem::path FsPath{DataFilePath.ToPath()};
DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
@@ -1200,7 +1202,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
do
{
- std::filesystem::path ParentPath = std::filesystem::path(DataFilePath.c_str()).parent_path();
+ std::filesystem::path ParentPath = FsPath.parent_path();
CreateDirectories(ParentPath);
DataFile.MoveTemporaryIntoPlace(FsPath, Ec);
@@ -1225,7 +1227,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
if (Ec)
{
- throw std::system_error(Ec, "Failed to finalize file '{}'"_format(WideToUtf8(DataFilePath)));
+ throw std::system_error(Ec, "Failed to finalize file '{}'"_format(DataFilePath.ToUtf8()));
}
}
@@ -1371,11 +1373,11 @@ ZenCacheDiskLayer::DiscoverBuckets()
virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
{
- Dirs.push_back(std::wstring(DirectoryName));
+ Dirs.push_back((decltype(Dirs)::value_type)(DirectoryName));
return false;
}
- std::vector<std::wstring> Dirs;
+ std::vector<std::filesystem::path::string_type> Dirs;
} Visit;
Traversal.TraverseFileSystem(m_RootDir, Visit);
@@ -1384,11 +1386,15 @@ ZenCacheDiskLayer::DiscoverBuckets()
RwLock::ExclusiveLockScope _(m_Lock);
- for (const std::wstring& BucketName : Visit.Dirs)
+ for (const auto& BucketName : Visit.Dirs)
{
// New bucket needs to be created
- const std::string BucketName8 = ToUtf8(BucketName);
+#if ZEN_PLATFORM_WINDOWS
+ std::string BucketName8 = WideToUtf8(BucketName);
+#else
+ const auto& BucketName8 = BucketName;
+#endif
if (auto It = m_Buckets.find(BucketName8); It != m_Buckets.end())
{
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index b64b1353e..41b47b409 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -23,9 +23,9 @@ ZEN_THIRD_PARTY_INCLUDES_END
namespace zen {
+class PathBuilderBase;
class CasStore;
class CasGc;
-class WideStringBuilderBase;
class ZenCacheTracker;
/******************************************************************************
diff --git a/zenserver/casstore.cpp b/zenserver/casstore.cpp
index 88525bd36..872a40df8 100644
--- a/zenserver/casstore.cpp
+++ b/zenserver/casstore.cpp
@@ -51,6 +51,9 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store)
return ServerRequest.WriteResponse(HttpResponseCode::OK);
}
break;
+
+ default:
+ break;
}
},
HttpVerb::kGet | HttpVerb::kPut | HttpVerb::kHead);
diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp
index 11ae8b41d..d483fd4f7 100644
--- a/zenserver/compute/apply.cpp
+++ b/zenserver/compute/apply.cpp
@@ -2,35 +2,35 @@
#include "apply.h"
-#include <upstream/jupiter.h>
-#include <upstream/upstreamapply.h>
-#include <zencore/compactbinary.h>
-#include <zencore/compactbinarybuilder.h>
-#include <zencore/compactbinarypackage.h>
-#include <zencore/compress.h>
-#include <zencore/except.h>
-#include <zencore/filesystem.h>
-#include <zencore/fmtutils.h>
-#include <zencore/iobuffer.h>
-#include <zencore/iohash.h>
-#include <zencore/scopeguard.h>
-#include <zencore/windows.h>
-#include <zenstore/CAS.h>
-#include <zenstore/cidstore.h>
-
-#if ZEN_PLATFORM_WINDOWS
+#if ZEN_WITH_COMPUTE_SERVICES
+
+# include <upstream/jupiter.h>
+# include <upstream/upstreamapply.h>
+# include <zencore/compactbinary.h>
+# include <zencore/compactbinarybuilder.h>
+# include <zencore/compactbinarypackage.h>
+# include <zencore/compress.h>
+# include <zencore/except.h>
+# include <zencore/filesystem.h>
+# include <zencore/fmtutils.h>
+# include <zencore/iobuffer.h>
+# include <zencore/iohash.h>
+# include <zencore/scopeguard.h>
+# include <zenstore/cas.h>
+# include <zenstore/cidstore.h>
+
+# include <zencore/windows.h>
ZEN_THIRD_PARTY_INCLUDES_START
# include <AccCtrl.h>
# include <AclAPI.h>
-# include <sddl.h>
# include <UserEnv.h>
+# include <sddl.h>
# pragma comment(lib, "UserEnv.lib")
# include <atlbase.h>
ZEN_THIRD_PARTY_INCLUDES_END
-#endif
-#include <filesystem>
-#include <span>
+# include <filesystem>
+# include <span>
using namespace std::literals;
@@ -136,6 +136,8 @@ BasicFunctionJob::ExitCode()
return gsl::narrow_cast<int>(Ec);
}
+////////////////////////////////////////////////////////////////////////////////
+
struct SandboxedFunctionJob
{
SandboxedFunctionJob() = default;
@@ -326,6 +328,8 @@ SandboxedFunctionJob::SpawnJob(std::filesystem::path ExePath)
return true;
}
+////////////////////////////////////////////////////////////////////////////////
+
HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, const std::filesystem::path& BaseDir)
: m_Log(logging::Get("apply"))
, m_CasStore(Store)
@@ -448,6 +452,8 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
SharedBuffer Decompressed = DataView.Decompress();
const uint64_t DecompressedSize = DataView.GetRawSize();
+ ZEN_UNUSED(DataHash);
+
TotalAttachmentBytes += DecompressedSize;
++AttachmentCount;
@@ -478,9 +484,15 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
return HttpReq.WriteResponse(HttpResponseCode::NoContent);
}
break;
+
+ default:
+ break;
}
}
break;
+
+ default:
+ break;
}
},
HttpVerb::kGet | HttpVerb::kPost);
@@ -497,6 +509,9 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
case HttpVerb::kPost:
break;
+
+ default:
+ break;
}
},
HttpVerb::kGet | HttpVerb::kPost);
@@ -622,6 +637,8 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
const IoHash DataHash = Attachment.GetHash();
CompressedBuffer DataView = Attachment.AsCompressedBinary();
+ ZEN_UNUSED(DataHash);
+
const uint64_t CompressedSize = DataView.GetCompressedSize();
TotalAttachmentBytes += CompressedSize;
@@ -652,8 +669,14 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore,
return HttpReq.WriteResponse(HttpResponseCode::OK, Output);
}
break;
+
+ default:
+ break;
}
break;
+
+ default:
+ break;
}
},
HttpVerb::kPost);
@@ -958,3 +981,5 @@ HttpFunctionService::ExecActionUpstreamResult(const IoHash& WorkerId, const IoHa
}
} // namespace zen
+
+#endif // ZEN_WITH_COMPUTE_SERVICES
diff --git a/zenserver/compute/apply.h b/zenserver/compute/apply.h
index 0d6edf119..161e47e06 100644
--- a/zenserver/compute/apply.h
+++ b/zenserver/compute/apply.h
@@ -2,13 +2,21 @@
#pragma once
-#include <zencore/compactbinary.h>
-#include <zencore/iohash.h>
-#include <zencore/logging.h>
-#include <zenhttp/httpserver.h>
+#include <zencore/zencore.h>
-#include <filesystem>
-#include <unordered_map>
+#if !defined(ZEN_WITH_COMPUTE_SERVICES)
+# define ZEN_WITH_COMPUTE_SERVICES ZEN_PLATFORM_WINDOWS
+#endif
+
+#if ZEN_WITH_COMPUTE_SERVICES
+
+# include <zencore/compactbinary.h>
+# include <zencore/iohash.h>
+# include <zencore/logging.h>
+# include <zenhttp/httpserver.h>
+
+# include <filesystem>
+# include <unordered_map>
namespace zen {
@@ -54,3 +62,5 @@ private:
};
} // namespace zen
+
+#endif // ZEN_WITH_COMPUTE_SERVICES
diff --git a/zenserver/config.cpp b/zenserver/config.cpp
index 5e5676494..b25d05409 100644
--- a/zenserver/config.cpp
+++ b/zenserver/config.cpp
@@ -9,16 +9,18 @@
#include <zencore/string.h>
#include <zenhttp/zenhttp.h>
-#pragma warning(push)
-#pragma warning(disable : 4267) // warning C4267: '=': conversion from 'size_t' to 'US', possible loss of data
-#include <cxxopts.hpp>
-#pragma warning(pop)
-
+ZEN_THIRD_PARTY_INCLUDES_START
#include <fmt/format.h>
#include <zencore/logging.h>
+#include <cxxopts.hpp>
#include <sol/sol.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
-#include <conio.h>
+#if ZEN_PLATFORM_WINDOWS
+# include <conio.h>
+#else
+# include <pwd.h>
+#endif
#if ZEN_PLATFORM_WINDOWS
@@ -86,7 +88,9 @@ PickDefaultStateDirectory()
std::filesystem::path
PickDefaultStateDirectory()
{
- return std::filesystem::path("~/.zen");
+ int UserId = getuid();
+ const passwd* Passwd = getpwuid(UserId);
+ return std::filesystem::path(Passwd->pw_dir) / ".zen";
}
#endif
@@ -183,6 +187,22 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
"");
#endif
+#if ZEN_WITH_TRACE
+ options.add_option("ue-trace",
+ "",
+ "tracehost",
+ "Hostname to send the trace to",
+ cxxopts::value<std::string>(ServerOptions.TraceHost)->default_value(""),
+ "");
+
+ options.add_option("ue-trace",
+ "",
+ "tracefile",
+ "Path to write a trace to",
+ cxxopts::value<std::string>(ServerOptions.TraceFile)->default_value(""),
+ "");
+#endif // ZEN_WITH_TRACE
+
options.add_option("diagnostics",
"",
"crash",
@@ -337,8 +357,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
if (result.count("help"))
{
zen::logging::ConsoleLog().info("{}", options.help());
+#if ZEN_PLATFORM_WINDOWS
zen::logging::ConsoleLog().info("Press any key to exit!");
_getch();
+#else
+ // Assume the user's in a terminal on all other platforms and that
+ // they'll use less/more/etc. if need be.
+#endif
exit(0);
}
@@ -371,7 +396,7 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio
{
using namespace fmt::literals;
- zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(Path.native().c_str());
+ zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(Path);
if (LuaScript)
{
@@ -380,6 +405,7 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio
lua.open_libraries(sol::lib::base);
lua.set_function("getenv", [&](const std::string env) -> sol::object {
+#if ZEN_PLATFORM_WINDOWS
std::wstring EnvVarValue;
size_t RequiredSize = 0;
std::wstring EnvWide = zen::Utf8ToWide(env);
@@ -391,6 +417,10 @@ ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptio
EnvVarValue.resize(RequiredSize);
_wgetenv_s(&RequiredSize, EnvVarValue.data(), RequiredSize, EnvWide.c_str());
return sol::make_object(lua, zen::WideToUtf8(EnvVarValue.c_str()));
+#else
+ ZEN_UNUSED(env);
+ return sol::make_object(lua, sol::lua_nil);
+#endif
});
try
diff --git a/zenserver/config.h b/zenserver/config.h
index 8a507df39..60e9976e0 100644
--- a/zenserver/config.h
+++ b/zenserver/config.h
@@ -2,8 +2,10 @@
#pragma once
+#include <zencore/zencore.h>
#include <filesystem>
#include <string>
+#include <vector>
#ifndef ZEN_ENABLE_MESH
# define ZEN_ENABLE_MESH 0
@@ -17,6 +19,10 @@
# define ZEN_USE_EXEC 0
#endif
+#ifndef ZEN_WITH_TRACE
+# define ZEN_WITH_TRACE 0
+#endif
+
struct ZenUpstreamJupiterConfig
{
std::string Url;
@@ -101,6 +107,10 @@ struct ZenServerOptions
bool StructuredCacheEnabled = true;
bool ShouldCrash = false; // Option for testing crash handling
bool IsFirstRun = false;
+#if ZEN_WITH_TRACE
+ std::string TraceHost; // Host name or IP address to send trace data to
+ std::string TraceFile; // Path of a file to write a trace
+#endif
#if ZEN_ENABLE_MESH
bool MeshEnabled = false; // Experimental p2p mesh discovery
#endif
diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp
index 728001202..3a5da2de9 100644
--- a/zenserver/diag/logging.cpp
+++ b/zenserver/diag/logging.cpp
@@ -171,6 +171,7 @@ private:
bool
EnableVTMode()
{
+#if ZEN_PLATFORM_WINDOWS
// Set output mode to handle virtual terminal sequences
HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE);
if (hOut == INVALID_HANDLE_VALUE)
@@ -189,6 +190,7 @@ EnableVTMode()
{
return false;
}
+#endif
return true;
}
@@ -233,13 +235,13 @@ InitializeLogging(const ZenServerOptions& GlobalOptions)
auto ConsoleSink = std::make_shared<spdlog::sinks::ansicolor_stdout_sink_mt>();
#if 0
- auto FileSink = std::make_shared<spdlog::sinks::daily_file_sink_mt>(zen::WideToUtf8(LogPath.c_str()),
+ auto FileSink = std::make_shared<spdlog::sinks::daily_file_sink_mt>(zen::PathToUtf8(LogPath),
0,
0,
/* truncate */ false,
uint16_t(/* max files */ 14));
#else
- auto FileSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::ToUtf8(LogPath),
+ auto FileSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::PathToUtf8(LogPath),
/* max size */ 128 * 1024 * 1024,
/* max files */ 16,
/* rotate on open */ true);
@@ -267,7 +269,7 @@ InitializeLogging(const ZenServerOptions& GlobalOptions)
std::filesystem::path HttpLogPath = GlobalOptions.DataDir / "logs/http.log";
- auto HttpSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::WideToUtf8(HttpLogPath.c_str()),
+ auto HttpSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::PathToUtf8(HttpLogPath),
/* max size */ 128 * 1024 * 1024,
/* max files */ 16,
/* rotate on open */ true);
diff --git a/zenserver/experimental/frontend.cpp b/zenserver/experimental/frontend.cpp
index 98d570cfe..4bd3ec90a 100644
--- a/zenserver/experimental/frontend.cpp
+++ b/zenserver/experimental/frontend.cpp
@@ -54,7 +54,7 @@ body {
<pre>
__________ _________ __
\____ / ____ ____ / _____/_/ |_ ____ _______ ____
- / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \
+ / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \
/ /_ \ ___/ | | \ / \ | | ( <_> ) | | \/\ ___/
/_______ \ \___ >|___| //_______ / |__| \____/ |__| \___ >
\/ \/ \/ \/ \/
diff --git a/zenserver/experimental/usnjournal.cpp b/zenserver/experimental/usnjournal.cpp
index 9422dd485..580d71e45 100644
--- a/zenserver/experimental/usnjournal.cpp
+++ b/zenserver/experimental/usnjournal.cpp
@@ -1,17 +1,20 @@
// Copyright Epic Games, Inc. All Rights Reserved.
-#include "usnjournal.h"
-
-#include <zencore/except.h>
-#include <zencore/logging.h>
-#include <zencore/timer.h>
#include <zencore/zencore.h>
+#if ZEN_PLATFORM_WINDOWS
+
+# include "usnjournal.h"
+
+# include <zencore/except.h>
+# include <zencore/logging.h>
+# include <zencore/timer.h>
+
ZEN_THIRD_PARTY_INCLUDES_START
-#include <atlfile.h>
+# include <atlfile.h>
ZEN_THIRD_PARTY_INCLUDES_END
-#include <filesystem>
+# include <filesystem>
namespace zen {
@@ -344,3 +347,5 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath)
}
} // namespace zen
+
+#endif // ZEN_PLATFORM_WINDOWS
diff --git a/zenserver/experimental/usnjournal.h b/zenserver/experimental/usnjournal.h
index db1f59abc..910eb7d06 100644
--- a/zenserver/experimental/usnjournal.h
+++ b/zenserver/experimental/usnjournal.h
@@ -2,14 +2,16 @@
#pragma once
-#include <zencore/windows.h>
-#include <zencore/zencore.h>
+#if ZEN_PLATFORM_WINDOWS
+
+# include <zencore/windows.h>
+# include <zencore/zencore.h>
ZEN_THIRD_PARTY_INCLUDES_START
-#include <winioctl.h>
+# include <winioctl.h>
ZEN_THIRD_PARTY_INCLUDES_END
-#include <filesystem>
+# include <filesystem>
namespace zen {
@@ -63,3 +65,5 @@ private:
};
} // namespace zen
+
+#endif // ZEN_PLATFORM_WINDOWS
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 8f8b6e163..7e0aed328 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -13,13 +13,16 @@
#include <zencore/testing.h>
#include <zencore/testutils.h>
#include <zencore/timer.h>
-#include <zencore/windows.h>
#include <zenstore/basicfile.h>
#include <zenstore/cas.h>
#include <zenstore/caslog.h>
#include "config.h"
+#if ZEN_PLATFORM_WINDOWS
+# include <zencore/windows.h>
+#endif
+
#define USE_ROCKSDB 0
ZEN_THIRD_PARTY_INCLUDES_START
@@ -122,7 +125,7 @@ struct ProjectStore::OplogStorage : public RefCounted
#if USE_ROCKSDB
{
- std::string RocksdbPath = WideToUtf8((m_OplogStoragePath / "ops.rdb").native().c_str());
+ std::string RocksdbPath = PathToUtf8(m_OplogStoragePath / "ops.rdb");
ZEN_DEBUG("opening rocksdb db at '{}'", RocksdbPath);
@@ -200,9 +203,8 @@ struct ProjectStore::OplogStorage : public RefCounted
CbObject Op(SharedBuffer::MakeView(OpBuffer.Data(), OpBuffer.Size()));
- m_NextOpsOffset =
- Max(m_NextOpsOffset.load(std::memory_order::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign));
- m_MaxLsn = Max(m_MaxLsn.load(std::memory_order::memory_order_relaxed), LogEntry.OpLsn);
+ m_NextOpsOffset = Max(m_NextOpsOffset.load(std::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign));
+ m_MaxLsn = Max(m_MaxLsn.load(std::memory_order_relaxed), LogEntry.OpLsn);
Handler(Op, LogEntry);
});
@@ -299,8 +301,8 @@ private:
ProjectStore::Oplog::Oplog(std::string_view Id, Project* Project, CidStore& Store, std::filesystem::path BasePath)
: m_OuterProject(Project)
, m_CidStore(Store)
-, m_OplogId(Id)
, m_BasePath(BasePath)
+, m_OplogId(Id)
{
m_Storage = new OplogStorage(this, m_BasePath);
const bool StoreExists = m_Storage->Exists();
@@ -385,7 +387,7 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId)
std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second.ServerPath;
- IoBuffer FileChunk = IoBufferBuilder::MakeFromFile(FilePath.native().c_str());
+ IoBuffer FileChunk = IoBufferBuilder::MakeFromFile(FilePath);
FileChunk.SetContentType(ZenContentType::kBinary);
return FileChunk;
@@ -465,16 +467,6 @@ ProjectStore::Oplog::AddFileMapping(Oid FileId, IoHash Hash, std::string_view Se
return false;
}
- if (ServerPath[0] == '/' || ClientPath[0] != '/')
- {
- // This is a special case just to enable tests to use absolute paths. We might want
- // to have configuration to control this
- if (ServerPath[1] != ':')
- {
- return false;
- }
- }
-
FileMapEntry Entry;
Entry.ServerPath = ServerPath;
Entry.ClientPath = ClientPath;
@@ -680,7 +672,7 @@ ProjectStore::Project::Write()
CbObjectWriter Cfg;
Cfg << "id" << Identifier;
- Cfg << "root" << WideToUtf8(RootDir.c_str());
+ Cfg << "root" << PathToUtf8(RootDir);
Cfg << "project" << ProjectRootDir;
Cfg << "engine" << EngineRootDir;
@@ -779,8 +771,6 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId)
void
ProjectStore::Project::DeleteOplog(std::string_view OplogId)
{
- bool Exists = false;
-
{
RwLock::ExclusiveLockScope _(m_ProjectLock);
@@ -788,8 +778,6 @@ ProjectStore::Project::DeleteOplog(std::string_view OplogId)
if (OplogIt != m_Oplogs.end())
{
- Exists = true;
-
m_Oplogs.erase(OplogIt);
}
}
@@ -1647,7 +1635,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
ProjectStore::Oplog& Log = *OplogIt;
CbObjectWriter Cb;
- Cb << "id"sv << Log.OplogId() << "project"sv << Prj.Identifier << "tempdir"sv << Log.TempDir();
+ Cb << "id"sv << Log.OplogId() << "project"sv << Prj.Identifier << "tempdir"sv << Log.TempPath().c_str();
Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Cb.Save());
}
@@ -1686,6 +1674,9 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
return Req.ServerRequest().WriteResponse(HttpResponseCode::OK);
}
break;
+
+ default:
+ break;
}
},
HttpVerb::kPost | HttpVerb::kGet | HttpVerb::kDelete);
@@ -1783,7 +1774,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
const ProjectStore::Project& Prj = *ProjectIt;
CbObjectWriter Response;
- Response << "id" << Prj.Identifier << "root" << WideToUtf8(Prj.RootDir.c_str());
+ Response << "id" << Prj.Identifier << "root" << PathToUtf8(Prj.RootDir);
Response.BeginArray("oplogs"sv);
Prj.IterateOplogs([&](const ProjectStore::Oplog& I) { Response << "id"sv << I.OplogId(); });
@@ -1809,6 +1800,9 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
return Req.ServerRequest().WriteResponse(HttpResponseCode::NoContent);
}
break;
+
+ default:
+ break;
}
},
HttpVerb::kGet | HttpVerb::kPost | HttpVerb::kDelete);
@@ -1837,6 +1831,7 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request)
//////////////////////////////////////////////////////////////////////////
+# if ZEN_PLATFORM_WINDOWS
class SecurityAttributes
{
public:
@@ -1869,6 +1864,12 @@ public:
}
}
};
+# else
+struct AnyUserSecurityAttributes
+{
+ int Attributes() { return 0666; }
+};
+# endif // ZEN_PLATFORM_WINDOWS
//////////////////////////////////////////////////////////////////////////
@@ -1939,6 +1940,7 @@ private:
asio::thread_pool m_WorkerThreadPool;
asio::io_context m_IoContext;
+# if ZEN_PLATFORM_WINDOWS
class PipeConnection
{
enum PipeState
@@ -2097,6 +2099,15 @@ private:
uint8_t m_MsgBuffer[16384];
};
+# else
+ class PipeConnection
+ {
+ public:
+ PipeConnection(LocalProjectImpl*) {}
+ void Accept() {}
+ void Disconnect() {}
+ };
+# endif
AnyUserSecurityAttributes m_AnyUserSecurityAttributes;
std::vector<PipeConnection*> m_ServicePipes;
diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h
index 43acdf05f..283dec3b2 100644
--- a/zenserver/projectstore.h
+++ b/zenserver/projectstore.h
@@ -101,7 +101,6 @@ public:
const std::string& OplogId() const { return m_OplogId; }
- const std::wstring& TempDir() const { return m_TempPath.native(); }
const std::filesystem::path& TempPath() const { return m_TempPath; }
spdlog::logger& Log() { return m_OuterProject->Log(); }
diff --git a/zenserver/testing/launch.cpp b/zenserver/testing/launch.cpp
index 55695ac9c..706594b10 100644
--- a/zenserver/testing/launch.cpp
+++ b/zenserver/testing/launch.cpp
@@ -2,26 +2,29 @@
#include "launch.h"
-#include <zencore/compactbinary.h>
-#include <zencore/compactbinarybuilder.h>
-#include <zencore/filesystem.h>
-#include <zencore/fmtutils.h>
-#include <zencore/iobuffer.h>
-#include <zencore/iohash.h>
-#include <zencore/logging.h>
-#include <zencore/windows.h>
-#include <zenstore/CAS.h>
-
-#include <AccCtrl.h>
-#include <AclAPI.h>
-#include <sddl.h>
-
-#include <UserEnv.h>
-#pragma comment(lib, "UserEnv.lib")
-
-#include <atlbase.h>
-#include <filesystem>
-#include <span>
+#if ZEN_WITH_COMPUTE_SERVICES
+
+# include <zencore/compactbinary.h>
+# include <zencore/compactbinarybuilder.h>
+# include <zencore/filesystem.h>
+# include <zencore/fmtutils.h>
+# include <zencore/iobuffer.h>
+# include <zencore/iohash.h>
+# include <zencore/logging.h>
+# include <zencore/windows.h>
+# include <zenstore/cas.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+# include <AccCtrl.h>
+# include <AclAPI.h>
+# include <UserEnv.h>
+# include <atlbase.h>
+# include <sddl.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+# pragma comment(lib, "UserEnv.lib")
+
+# include <filesystem>
+# include <span>
using namespace std::literals;
@@ -127,6 +130,8 @@ BasicJob::ExitCode()
return gsl::narrow_cast<int>(Ec);
}
+////////////////////////////////////////////////////////////////////////////////
+
struct SandboxedJob
{
SandboxedJob() = default;
@@ -317,6 +322,8 @@ SandboxedJob::SpawnJob(std::filesystem::path ExePath)
return true;
}
+////////////////////////////////////////////////////////////////////////////////
+
HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::path& SandboxBaseDir)
: m_Log(logging::Get("exec"))
, m_CasStore(Store)
@@ -336,13 +343,16 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat
case HttpVerb::kPost:
break;
+
+ default:
+ break;
}
},
HttpVerb::kGet | HttpVerb::kPost);
// Experimental
-#if 0
+# if 0
m_Router.RegisterRoute(
"jobs/sandbox",
[this](HttpRouterRequest& Req) {
@@ -362,10 +372,13 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat
Job.SpawnJob("c:\\windows\\system32\\cmd.exe");
}
break;
+
+ default:
+ break;
}
},
HttpVerb::kGet | HttpVerb::kPost);
-#endif
+# endif
m_Router.RegisterRoute(
"jobs/prep",
@@ -413,6 +426,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat
return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
}
break;
+
+ default:
+ break;
}
},
HttpVerb::kPost);
@@ -496,6 +512,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat
return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
}
break;
+
+ default:
+ break;
}
},
HttpVerb::kGet | HttpVerb::kPost);
@@ -530,3 +549,5 @@ HttpLaunchService::CreateNewSandbox()
}
} // namespace zen
+
+#endif // ZEN_WITH_COMPUTE_SERVICES
diff --git a/zenserver/testing/launch.h b/zenserver/testing/launch.h
index 49f12e2ec..925fa18b0 100644
--- a/zenserver/testing/launch.h
+++ b/zenserver/testing/launch.h
@@ -2,10 +2,18 @@
#pragma once
-#include <zencore/logging.h>
-#include <zenhttp/httpserver.h>
+#include <zencore/zencore.h>
-#include <filesystem>
+#if !defined(ZEN_WITH_COMPUTE_SERVICES)
+# define ZEN_WITH_COMPUTE_SERVICES ZEN_PLATFORM_WINDOWS
+#endif
+
+#if ZEN_WITH_COMPUTE_SERVICES
+
+# include <zencore/logging.h>
+# include <zenhttp/httpserver.h>
+
+# include <filesystem>
namespace zen {
@@ -36,3 +44,5 @@ private:
};
} // namespace zen
+
+#endif // ZEN_WITH_COMPUTE_SERVICES
diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp
index f9be068ec..177184591 100644
--- a/zenserver/upstream/jupiter.cpp
+++ b/zenserver/upstream/jupiter.cpp
@@ -2,7 +2,6 @@
#include "jupiter.h"
-#include "cache/structuredcachestore.h"
#include "diag/formatters.h"
#include "diag/logging.h"
@@ -58,7 +57,7 @@ namespace detail {
cpr::Session& GetSession() { return Session; }
private:
- friend class CloudCacheClient;
+ friend class zen::CloudCacheClient;
CloudCacheClient& OwnerClient;
CloudCacheAccessToken AccessToken;
@@ -116,11 +115,7 @@ CloudCacheSession::GetDerivedData(std::string_view BucketId, std::string_view Ke
const bool Success = Response.status_code == 200;
const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer();
- return {.Response = Buffer,
- .Bytes = Response.downloaded_bytes,
- .ElapsedSeconds = Response.elapsed,
- .ErrorCode = !Success ? Response.status_code : 0,
- .Success = Success};
+ return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success};
}
CloudCacheResult
@@ -165,11 +160,7 @@ CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key, ZenConte
const bool Success = Response.status_code == 200;
const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer();
- return {.Response = Buffer,
- .Bytes = Response.downloaded_bytes,
- .ElapsedSeconds = Response.elapsed,
- .ErrorCode = !Success ? Response.status_code : 0,
- .Success = Success};
+ return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success};
}
CloudCacheResult
@@ -206,11 +197,7 @@ CloudCacheSession::GetBlob(const IoHash& Key)
const IoBuffer Buffer =
Success && Response.text.size() > 0 ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer();
- return {.Response = Buffer,
- .Bytes = Response.downloaded_bytes,
- .ElapsedSeconds = Response.elapsed,
- .ErrorCode = !Success ? Response.status_code : 0,
- .Success = Success};
+ return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success};
}
CloudCacheResult
@@ -246,11 +233,7 @@ CloudCacheSession::GetCompressedBlob(const IoHash& Key)
const bool Success = Response.status_code == 200;
const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer();
- return {.Response = Buffer,
- .Bytes = Response.downloaded_bytes,
- .ElapsedSeconds = Response.elapsed,
- .ErrorCode = !Success ? Response.status_code : 0,
- .Success = Success};
+ return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success};
}
CloudCacheResult
@@ -286,11 +269,7 @@ CloudCacheSession::GetObject(const IoHash& Key)
const bool Success = Response.status_code == 200;
const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer();
- return {.Response = Buffer,
- .Bytes = Response.downloaded_bytes,
- .ElapsedSeconds = Response.elapsed,
- .ErrorCode = !Success ? Response.status_code : 0,
- .Success = Success};
+ return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success};
}
CloudCacheResult
@@ -327,12 +306,9 @@ CloudCacheSession::PutDerivedData(std::string_view BucketId, std::string_view Ke
return {.ErrorCode = 401, .Reason = std::string("Invalid access token")};
}
- const bool Success = Response.status_code == 200 || Response.status_code == 201;
-
return {.Bytes = Response.uploaded_bytes,
.ElapsedSeconds = Response.elapsed,
- .ErrorCode = !Success ? Response.status_code : 0,
- .Success = Success};
+ .Success = (Response.status_code == 200 || Response.status_code == 201)};
}
CloudCacheResult
@@ -388,7 +364,6 @@ CloudCacheSession::PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer
PutRefResult Result;
Result.Success = (Response.status_code == 200 || Response.status_code == 201);
- Result.ErrorCode = !Result.Success ? Response.status_code : 0;
Result.Bytes = Response.uploaded_bytes;
Result.ElapsedSeconds = Response.elapsed;
@@ -453,7 +428,6 @@ CloudCacheSession::FinalizeRef(std::string_view BucketId, const IoHash& Key, con
FinalizeRefResult Result;
Result.Success = (Response.status_code == 200 || Response.status_code == 201);
- Result.ErrorCode = !Result.Success ? Response.status_code : 0;
Result.Bytes = Response.uploaded_bytes;
Result.ElapsedSeconds = Response.elapsed;
@@ -504,12 +478,9 @@ CloudCacheSession::PutBlob(const IoHash& Key, IoBuffer Blob)
return {.ErrorCode = 401, .Reason = std::string("Invalid access token")};
}
- const bool Success = Response.status_code == 200 || Response.status_code == 201;
-
return {.Bytes = Response.uploaded_bytes,
.ElapsedSeconds = Response.elapsed,
- .ErrorCode = !Success ? Response.status_code : 0,
- .Success = Success};
+ .Success = (Response.status_code == 200 || Response.status_code == 201)};
}
CloudCacheResult
@@ -542,12 +513,9 @@ CloudCacheSession::PutCompressedBlob(const IoHash& Key, IoBuffer Blob)
return {.ErrorCode = 401, .Reason = std::string("Invalid access token")};
}
- const bool Success = Response.status_code == 200 || Response.status_code == 201;
-
return {.Bytes = Response.uploaded_bytes,
.ElapsedSeconds = Response.elapsed,
- .ErrorCode = !Success ? Response.status_code : 0,
- .Success = Success};
+ .Success = (Response.status_code == 200 || Response.status_code == 201)};
}
CloudCacheResult
@@ -580,12 +548,9 @@ CloudCacheSession::PutObject(const IoHash& Key, IoBuffer Object)
return {.ErrorCode = 401, .Reason = std::string("Invalid access token")};
}
- const bool Success = Response.status_code == 200 || Response.status_code == 201;
-
return {.Bytes = Response.uploaded_bytes,
.ElapsedSeconds = Response.elapsed,
- .ErrorCode = !Success ? Response.status_code : 0,
- .Success = Success};
+ .Success = (Response.status_code == 200 || Response.status_code == 201)};
}
CloudCacheResult
@@ -619,9 +584,7 @@ CloudCacheSession::RefExists(std::string_view BucketId, const IoHash& Key)
return {.ErrorCode = 401, .Reason = std::string("Invalid access token")};
}
- const bool Success = Response.status_code == 200;
-
- return {.ElapsedSeconds = Response.elapsed, .ErrorCode = !Success ? Response.status_code : 0, .Success = Success};
+ return {.ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200};
}
CloudCacheResult
@@ -690,9 +653,7 @@ CloudCacheSession::PostComputeTasks(std::string_view ChannelId, IoBuffer TasksDa
return {.ErrorCode = 401, .Reason = std::string("Invalid access token")};
}
- const bool Success = Response.status_code == 200;
-
- return {.ElapsedSeconds = Response.elapsed, .ErrorCode = !Success ? Response.status_code : 0, .Success = Success};
+ return {.ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200};
}
CloudCacheResult
@@ -728,11 +689,7 @@ CloudCacheSession::GetComputeUpdates(std::string_view ChannelId, const uint32_t
const bool Success = Response.status_code == 200;
const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer();
- return {.Response = Buffer,
- .Bytes = Response.downloaded_bytes,
- .ElapsedSeconds = Response.elapsed,
- .ErrorCode = !Success ? Response.status_code : 0,
- .Success = Success};
+ return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success};
}
CloudCacheResult
@@ -768,11 +725,7 @@ CloudCacheSession::GetObjectTree(const IoHash& Key)
const bool Success = Response.status_code == 200;
const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer();
- return {.Response = Buffer,
- .Bytes = Response.downloaded_bytes,
- .ElapsedSeconds = Response.elapsed,
- .ErrorCode = !Success ? Response.status_code : 0,
- .Success = Success};
+ return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success};
}
std::vector<IoHash>
diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp
index 05be5f65c..f8a8c3e62 100644
--- a/zenserver/upstream/upstreamapply.cpp
+++ b/zenserver/upstream/upstreamapply.cpp
@@ -1,37 +1,40 @@
// Copyright Epic Games, Inc. All Rights Reserved.
#include "upstreamapply.h"
-#include "jupiter.h"
-#include "zen.h"
-
-#include <zencore/blockingqueue.h>
-#include <zencore/compactbinary.h>
-#include <zencore/compactbinarybuilder.h>
-#include <zencore/compactbinarypackage.h>
-#include <zencore/compactbinaryvalidation.h>
-#include <zencore/compress.h>
-#include <zencore/fmtutils.h>
-#include <zencore/session.h>
-#include <zencore/stats.h>
-#include <zencore/stream.h>
-#include <zencore/thread.h>
-#include <zencore/timer.h>
-
-#include <zenstore/cas.h>
-#include <zenstore/cidstore.h>
-
-#include "cache/structuredcachestore.h"
-#include "diag/logging.h"
-
-#include <fmt/format.h>
-
-#include <algorithm>
-#include <atomic>
-#include <map>
-#include <set>
-#include <stack>
-#include <thread>
-#include <unordered_map>
+
+#if ZEN_WITH_COMPUTE_SERVICES
+
+# include "jupiter.h"
+# include "zen.h"
+
+# include <zencore/blockingqueue.h>
+# include <zencore/compactbinary.h>
+# include <zencore/compactbinarybuilder.h>
+# include <zencore/compactbinarypackage.h>
+# include <zencore/compactbinaryvalidation.h>
+# include <zencore/compress.h>
+# include <zencore/fmtutils.h>
+# include <zencore/session.h>
+# include <zencore/stats.h>
+# include <zencore/stream.h>
+# include <zencore/thread.h>
+# include <zencore/timer.h>
+
+# include <zenstore/cas.h>
+# include <zenstore/cidstore.h>
+
+# include "cache/structuredcachestore.h"
+# include "diag/logging.h"
+
+# include <fmt/format.h>
+
+# include <algorithm>
+# include <atomic>
+# include <map>
+# include <set>
+# include <stack>
+# include <thread>
+# include <unordered_map>
namespace zen {
@@ -1569,3 +1572,5 @@ MakeHordeUpstreamEndpoint(const CloudCacheClientOptions& Options, CasStore& CasS
}
} // namespace zen
+
+#endif // ZEN_WITH_COMPUTE_SERVICES
diff --git a/zenserver/upstream/upstreamapply.h b/zenserver/upstream/upstreamapply.h
index 98f193c02..ed73ec7f8 100644
--- a/zenserver/upstream/upstreamapply.h
+++ b/zenserver/upstream/upstreamapply.h
@@ -2,16 +2,20 @@
#pragma once
-#include <zencore/compactbinarypackage.h>
-#include <zencore/iobuffer.h>
-#include <zencore/iohash.h>
-#include <zencore/zencore.h>
+#include "compute/apply.h"
-#include <atomic>
-#include <chrono>
-#include <memory>
-#include <unordered_map>
-#include <unordered_set>
+#if ZEN_WITH_COMPUTE_SERVICES
+
+# include <zencore/compactbinarypackage.h>
+# include <zencore/iobuffer.h>
+# include <zencore/iohash.h>
+# include <zencore/zencore.h>
+
+# include <atomic>
+# include <chrono>
+# include <memory>
+# include <unordered_map>
+# include <unordered_set>
namespace zen {
@@ -170,3 +174,5 @@ std::unique_ptr<UpstreamApplyEndpoint> MakeHordeUpstreamEndpoint(const CloudCach
CidStore& CidStore);
} // namespace zen
+
+#endif // ZEN_WITH_COMPUTE_SERVICES
diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp
index e2dc09872..6283457e3 100644
--- a/zenserver/upstream/upstreamcache.cpp
+++ b/zenserver/upstream/upstreamcache.cpp
@@ -195,7 +195,7 @@ namespace detail {
}
}
- OnComplete({.CacheKey = CacheKey, .KeyIndex = Index, .Record = Record, .Package = Package});
+ OnComplete({.Key = CacheKey, .KeyIndex = Index, .Record = Record, .Package = Package});
}
return Result;
@@ -271,18 +271,15 @@ namespace detail {
if (CacheRecord.Type == ZenContentType::kBinary)
{
CloudCacheResult Result;
- for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
+ for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
{
if (m_UseLegacyDdc)
{
- Result = Session.PutDerivedData(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue);
+ Result = Session.PutDerivedData(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RecordValue);
}
else
{
- Result = Session.PutRef(CacheRecord.CacheKey.Bucket,
- CacheRecord.CacheKey.Hash,
- RecordValue,
- ZenContentType::kBinary);
+ Result = Session.PutRef(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RecordValue, ZenContentType::kBinary);
}
}
@@ -335,16 +332,15 @@ namespace detail {
PutRefResult RefResult;
for (int32_t Attempt = 0; Attempt < MaxAttempts && !RefResult.Success; Attempt++)
{
- RefResult =
- Session.PutRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue, ZenContentType::kCbObject);
+ RefResult = Session.PutRef(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RecordValue, ZenContentType::kCbObject);
}
m_HealthOk = RefResult.ErrorCode == 0;
if (!RefResult.Success)
{
- return {.Reason = "upload cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.CacheKey.Bucket,
- CacheRecord.CacheKey.Hash,
+ return {.Reason = "upload cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.Key.Bucket,
+ CacheRecord.Key.Hash,
RefResult.Reason),
.Success = false};
}
@@ -359,13 +355,13 @@ namespace detail {
}
const IoHash RefHash = IoHash::HashBuffer(RecordValue);
- FinalizeRefResult FinalizeResult = Session.FinalizeRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RefHash);
+ FinalizeRefResult FinalizeResult = Session.FinalizeRef(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RefHash);
m_HealthOk = FinalizeResult.ErrorCode == 0;
if (!FinalizeResult.Success)
{
- return {.Reason = "finalize cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.CacheKey.Bucket,
- CacheRecord.CacheKey.Hash,
+ return {.Reason = "finalize cache record '{}/{}' FAILED, reason '{}'"_format(CacheRecord.Key.Bucket,
+ CacheRecord.Key.Hash,
FinalizeResult.Reason),
.Success = false};
}
@@ -377,13 +373,13 @@ namespace detail {
return {.Reason = std::move(Reason), .Success = false};
}
- FinalizeResult = Session.FinalizeRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RefHash);
+ FinalizeResult = Session.FinalizeRef(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RefHash);
m_HealthOk = FinalizeResult.ErrorCode == 0;
if (!FinalizeResult.Success)
{
- return {.Reason = "finalize '{}/{}' FAILED, reason '{}'"_format(CacheRecord.CacheKey.Bucket,
- CacheRecord.CacheKey.Hash,
+ return {.Reason = "finalize '{}/{}' FAILED, reason '{}'"_format(CacheRecord.Key.Bucket,
+ CacheRecord.Key.Hash,
FinalizeResult.Reason),
.Success = false};
}
@@ -396,8 +392,8 @@ namespace detail {
Sb << MissingHash.ToHexString() << ",";
}
- return {.Reason = "finalize '{}/{}' FAILED, still needs payload(s) '{}'"_format(CacheRecord.CacheKey.Bucket,
- CacheRecord.CacheKey.Hash,
+ return {.Reason = "finalize '{}/{}' FAILED, still needs payload(s) '{}'"_format(CacheRecord.Key.Bucket,
+ CacheRecord.Key.Hash,
Sb.ToString()),
.Success = false};
}
@@ -606,8 +602,7 @@ namespace detail {
for (size_t LocalIndex = 0; CbFieldView Record : BatchResponse.GetObject()["Result"sv])
{
const size_t Index = IndexMap[LocalIndex++];
- OnComplete(
- {.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = Record.AsObjectView(), .Package = BatchResponse});
+ OnComplete({.Key = CacheKeys[Index], .KeyIndex = Index, .Record = Record.AsObjectView(), .Package = BatchResponse});
}
return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = true};
@@ -620,7 +615,7 @@ namespace detail {
for (size_t Index : KeyIndex)
{
- OnComplete({.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()});
+ OnComplete({.Key = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()});
}
return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}};
@@ -772,12 +767,9 @@ namespace detail {
Package.Save(MemStream);
IoBuffer PackagePayload(IoBuffer::Wrap, MemStream.Data(), MemStream.Size());
- for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
+ for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
{
- Result = Session.PutCacheRecord(CacheRecord.CacheKey.Bucket,
- CacheRecord.CacheKey.Hash,
- PackagePayload,
- CacheRecord.Type);
+ Result = Session.PutCacheRecord(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, PackagePayload, CacheRecord.Type);
m_HealthOk = Result.ErrorCode == 0;
}
@@ -790,10 +782,10 @@ namespace detail {
for (size_t Idx = 0, Count = Payloads.size(); Idx < Count; Idx++)
{
Result.Success = false;
- for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
+ for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
{
- Result = Session.PutCachePayload(CacheRecord.CacheKey.Bucket,
- CacheRecord.CacheKey.Hash,
+ Result = Session.PutCachePayload(CacheRecord.Key.Bucket,
+ CacheRecord.Key.Hash,
CacheRecord.PayloadIds[Idx],
Payloads[Idx]);
@@ -813,10 +805,9 @@ namespace detail {
}
Result.Success = false;
- for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
+ for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++)
{
- Result =
- Session.PutCacheRecord(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue, CacheRecord.Type);
+ Result = Session.PutCacheRecord(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, RecordValue, CacheRecord.Type);
m_HealthOk = Result.ErrorCode == 0;
}
@@ -1099,7 +1090,7 @@ public:
for (size_t Index : MissingKeys)
{
- OnComplete({.CacheKey = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()});
+ OnComplete({.Key = CacheKeys[Index], .KeyIndex = Index, .Record = CbObjectView(), .Package = CbPackage()});
}
}
@@ -1236,11 +1227,9 @@ private:
ZenCacheValue CacheValue;
std::vector<IoBuffer> Payloads;
- if (!m_CacheStore.Get(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, CacheValue))
+ if (!m_CacheStore.Get(CacheRecord.Key.Bucket, CacheRecord.Key.Hash, CacheValue))
{
- ZEN_WARN("process upstream FAILED, '{}/{}', cache record doesn't exist",
- CacheRecord.CacheKey.Bucket,
- CacheRecord.CacheKey.Hash);
+ ZEN_WARN("process upstream FAILED, '{}/{}', cache record doesn't exist", CacheRecord.Key.Bucket, CacheRecord.Key.Hash);
return;
}
@@ -1253,8 +1242,8 @@ private:
else
{
ZEN_WARN("process upstream FAILED, '{}/{}/{}', payload doesn't exist in CAS",
- CacheRecord.CacheKey.Bucket,
- CacheRecord.CacheKey.Hash,
+ CacheRecord.Key.Bucket,
+ CacheRecord.Key.Hash,
PayloadId);
return;
}
@@ -1270,8 +1259,8 @@ private:
if (!Result.Success)
{
ZEN_WARN("upload cache record '{}/{}' FAILED, endpoint '{}', reason '{}'",
- CacheRecord.CacheKey.Bucket,
- CacheRecord.CacheKey.Hash,
+ CacheRecord.Key.Bucket,
+ CacheRecord.Key.Hash,
Endpoint->GetEndpointInfo().Url,
Result.Reason);
}
@@ -1292,10 +1281,7 @@ private:
}
catch (std::exception& Err)
{
- ZEN_ERROR("upload cache record '{}/{}' FAILED, reason '{}'",
- CacheRecord.CacheKey.Bucket,
- CacheRecord.CacheKey.Hash,
- Err.what());
+ ZEN_ERROR("upload cache record '{}/{}' FAILED, reason '{}'", CacheRecord.Key.Bucket, CacheRecord.Key.Hash, Err.what());
}
}
diff --git a/zenserver/upstream/upstreamcache.h b/zenserver/upstream/upstreamcache.h
index 12287198d..c463c4996 100644
--- a/zenserver/upstream/upstreamcache.h
+++ b/zenserver/upstream/upstreamcache.h
@@ -25,7 +25,7 @@ struct ZenStructuredCacheClientOptions;
struct UpstreamCacheRecord
{
ZenContentType Type = ZenContentType::kBinary;
- CacheKey CacheKey;
+ CacheKey Key;
std::vector<IoHash> PayloadIds;
};
@@ -83,7 +83,7 @@ struct UpstreamEndpointStats
struct CacheRecordGetCompleteParams
{
- const CacheKey& CacheKey;
+ const CacheKey& Key;
size_t KeyIndex = ~size_t(0);
const CbObjectView& Record;
const CbPackage& Package;
diff --git a/zenserver/upstream/zen.h b/zenserver/upstream/zen.h
index df975df1f..e97ce755d 100644
--- a/zenserver/upstream/zen.h
+++ b/zenserver/upstream/zen.h
@@ -16,6 +16,7 @@ ZEN_THIRD_PARTY_INCLUDES_START
ZEN_THIRD_PARTY_INCLUDES_END
#include <chrono>
+#include <list>
struct ZenCacheValue;
diff --git a/zenserver/windows/service.cpp b/zenserver/windows/service.cpp
index 23cefb7b5..89bacab0b 100644
--- a/zenserver/windows/service.cpp
+++ b/zenserver/windows/service.cpp
@@ -2,14 +2,18 @@
#include "service.h"
-#include <zencore/except.h>
#include <zencore/zencore.h>
-#include <stdio.h>
-#include <tchar.h>
-#include <zencore/windows.h>
+#if ZEN_PLATFORM_WINDOWS
-#define SVCNAME L"Zen Store"
+# include <zencore/except.h>
+# include <zencore/zencore.h>
+
+# include <stdio.h>
+# include <tchar.h>
+# include <zencore/windows.h>
+
+# define SVCNAME L"Zen Store"
SERVICE_STATUS gSvcStatus;
SERVICE_STATUS_HANDLE gSvcStatusHandle;
@@ -638,3 +642,5 @@ SvcReportEvent(LPTSTR szFunction)
// DeregisterEventSource(hEventSource);
//}
}
+
+#endif // ZEN_PLATFORM_WINDOWS
diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua
index 2fe32112b..9e12df3e3 100644
--- a/zenserver/xmake.lua
+++ b/zenserver/xmake.lua
@@ -18,6 +18,7 @@ target("zenserver")
end
add_options("vfs")
+ add_options("compute")
add_packages(
"vcpkg::sentry-native",
@@ -43,7 +44,6 @@ target("zenserver")
else
commit = "dbg-" .. commit
end
- target:add("defines","BUILD_VERSION=\"" .. commit .. "\"")
- print("build version " .. commit)
+ target:add("defines", "BUILD_VERSION=\"" .. commit .. "\"")
end
end)
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index e14f93f5b..49bb3004d 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -12,13 +12,17 @@
#include <zencore/string.h>
#include <zencore/thread.h>
#include <zencore/timer.h>
-#include <zencore/windows.h>
+#include <zencore/trace.h>
#include <zenhttp/httpserver.h>
#include <zenstore/basicfile.h>
#include <zenstore/cas.h>
#include <zenstore/cidstore.h>
#include <zenutil/zenserverprocess.h>
+#if ZEN_PLATFORM_WINDOWS
+# include <zencore/windows.h>
+#endif
+
#if ZEN_USE_MIMALLOC
ZEN_THIRD_PARTY_INCLUDES_START
# include <mimalloc-new-delete.h>
@@ -138,7 +142,7 @@ namespace utils {
if (!ErrorCode)
{
- for (const asio::ip::tcp::endpoint& Ep : Endpoints)
+ for (const asio::ip::tcp::endpoint Ep : Endpoints)
{
OutEndpoints.push_back("http://{}:{}"_format(Ep.address().to_string(), Ep.port()));
}
@@ -225,17 +229,23 @@ public:
m_LocalProjectService = zen::LocalProjectService::New(*m_CasStore, m_ProjectStore);
#endif
- ZEN_INFO("instantiating compute services");
-
#if ZEN_USE_EXEC
std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox";
zen::CreateDirectories(SandboxDir);
m_HttpLaunchService = std::make_unique<zen::HttpLaunchService>(*m_CasStore, SandboxDir);
#endif
+#if ZEN_WITH_COMPUTE_SERVICES
+ ZEN_INFO("instantiating compute services");
+
+ std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox";
+ zen::CreateDirectories(SandboxDir);
+ m_HttpLaunchService = std::make_unique<zen::HttpLaunchService>(*m_CasStore, SandboxDir);
+
std::filesystem::path ApplySandboxDir = m_DataRoot / "exec" / "apply";
zen::CreateDirectories(ApplySandboxDir);
m_HttpFunctionService = std::make_unique<zen::HttpFunctionService>(*m_CasStore, *m_CidStore, ApplySandboxDir);
+#endif // ZEN_WITH_COMPUTE_SERVICES
if (ServerOptions.StructuredCacheEnabled)
{
@@ -273,17 +283,17 @@ public:
m_Http->RegisterService(*m_StructuredCacheService);
}
-#if ZEN_USE_EXEC
+#if ZEN_WITH_COMPUTE_SERVICES
if (m_HttpLaunchService)
{
m_Http->RegisterService(*m_HttpLaunchService);
}
-#endif
if (m_HttpFunctionService)
{
m_Http->RegisterService(*m_HttpFunctionService);
}
+#endif // ZEN_WITH_COMPUTE_SERVICES
m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot);
@@ -343,7 +353,7 @@ public:
if (m_DebugOptionForcedCrash)
{
- __debugbreak();
+ ZEN_DEBUG_BREAK();
}
const bool IsInteractiveMode = zen::IsInteractiveSession() && !m_TestMode;
@@ -404,7 +414,7 @@ public:
for (auto& PidEntry : m_ServerEntry->SponsorPids)
{
- if (uint32_t ThisPid = PidEntry.load(std::memory_order::memory_order_relaxed))
+ if (uint32_t ThisPid = PidEntry.load(std::memory_order_relaxed))
{
if (PidEntry.compare_exchange_strong(ThisPid, 0))
{
@@ -523,13 +533,17 @@ private:
zen::HttpTestingService m_TestingService;
zen::HttpCasService m_CasService{*m_CasStore};
zen::RefPtr<zen::ProjectStore> m_ProjectStore;
+ zen::Ref<zen::LocalProjectService> m_LocalProjectService;
std::unique_ptr<zen::HttpProjectService> m_HttpProjectService;
std::unique_ptr<zen::HttpStructuredCacheService> m_StructuredCacheService;
zen::HttpAdminService m_AdminService{m_GcScheduler};
zen::HttpHealthService m_HealthService;
zen::Mesh m_ZenMesh{m_IoContext};
- std::unique_ptr<zen::HttpFunctionService> m_HttpFunctionService;
- std::unique_ptr<zen::HttpFrontendService> m_FrontendService;
+#if ZEN_WITH_COMPUTE_SERVICES
+ std::unique_ptr<zen::HttpLaunchService> m_HttpLaunchService;
+ std::unique_ptr<zen::HttpFunctionService> m_HttpFunctionService;
+#endif
+ std::unique_ptr<zen::HttpFrontendService> m_FrontendService;
#if ZEN_USE_EXEC
std::unique_ptr<zen::HttpLaunchService> m_HttpLaunchService;
@@ -766,33 +780,34 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
new zen::HttpStructuredCacheService(*m_CacheStore, *m_CidStore, m_StatsService, m_StatusService, std::move(UpstreamCache)));
}
-} // namespace zen
+////////////////////////////////////////////////////////////////////////////////
-class ZenWindowsService : public WindowsService
+class ZenEntryPoint
{
public:
- ZenWindowsService(ZenServerOptions& ServerOptions) : m_ServerOptions(ServerOptions) {}
-
- ZenWindowsService(const ZenWindowsService&) = delete;
- ZenWindowsService& operator=(const ZenWindowsService&) = delete;
-
- virtual int Run() override;
+ ZenEntryPoint(ZenServerOptions& ServerOptions);
+ ZenEntryPoint(const ZenEntryPoint&) = delete;
+ ZenEntryPoint& operator=(const ZenEntryPoint&) = delete;
+ int Run();
private:
ZenServerOptions& m_ServerOptions;
zen::LockFile m_LockFile;
};
-int
-ZenWindowsService::Run()
+ZenEntryPoint::ZenEntryPoint(ZenServerOptions& ServerOptions) : m_ServerOptions(ServerOptions)
{
- using namespace zen;
+}
+int
+ZenEntryPoint::Run()
+{
#if USE_SENTRY
// Initialize sentry.io client
sentry_options_t* SentryOptions = sentry_options_new();
sentry_options_set_dsn(SentryOptions, "https://[email protected]/5919284");
+ sentry_options_set_database_path(SentryOptions, PathToUtf8(m_ServerOptions.DataDir / ".sentry-native").c_str());
sentry_init(SentryOptions);
auto _ = zen::MakeGuard([] { sentry_close(); });
@@ -812,8 +827,8 @@ ZenWindowsService::Run()
auto MakeLockData = [&] {
CbObjectWriter Cbo;
- Cbo << "pid" << _getpid() << "data" << ToUtf8(ServerOptions.DataDir) << "port" << ServerOptions.BasePort << "session_id"
- << GetSessionId() << "ready" << IsReady;
+ Cbo << "pid" << zen::GetCurrentProcessId() << "data" << PathToUtf8(ServerOptions.DataDir) << "port" << ServerOptions.BasePort
+ << "session_id" << GetSessionId() << "ready" << IsReady;
return Cbo.Save();
};
@@ -880,6 +895,10 @@ ZenWindowsService::Run()
ZEN_INFO("shutdown signal received");
Server.RequestExit(0);
}
+ else
+ {
+ ZEN_INFO("shutdown signal wait() failed");
+ }
}});
// If we have a parent process, establish the mechanisms we need
@@ -913,6 +932,36 @@ ZenWindowsService::Run()
return 0;
}
+} // namespace zen
+
+////////////////////////////////////////////////////////////////////////////////
+
+#if ZEN_PLATFORM_WINDOWS
+
+class ZenWindowsService : public WindowsService
+{
+public:
+ ZenWindowsService(ZenServerOptions& ServerOptions) : m_EntryPoint(ServerOptions) {}
+
+ ZenWindowsService(const ZenWindowsService&) = delete;
+ ZenWindowsService& operator=(const ZenWindowsService&) = delete;
+
+ virtual int Run() override;
+
+private:
+ zen::ZenEntryPoint m_EntryPoint;
+};
+
+int
+ZenWindowsService::Run()
+{
+ return m_EntryPoint.Run();
+}
+
+#endif // ZEN_PLATFORM_WINDOWS
+
+////////////////////////////////////////////////////////////////////////////////
+
#if ZEN_WITH_TESTS
int
test_main(int argc, char** argv)
@@ -960,6 +1009,17 @@ main(int argc, char* argv[])
std::filesystem::create_directories(ServerOptions.DataDir);
}
+#if ZEN_WITH_TRACE
+ if (ServerOptions.TraceHost.size())
+ {
+ TraceInit(ServerOptions.TraceHost.c_str(), TraceType::Network);
+ }
+ else if (ServerOptions.TraceFile.size())
+ {
+ TraceInit(ServerOptions.TraceFile.c_str(), TraceType::File);
+ }
+#endif // ZEN_WITH_TRACE
+
#if ZEN_PLATFORM_WINDOWS
if (ServerOptions.InstallService)
{
@@ -974,10 +1034,18 @@ main(int argc, char* argv[])
std::exit(0);
}
-#endif
ZenWindowsService App(ServerOptions);
return App.ServiceMain();
+#else
+ if (ServerOptions.InstallService || ServerOptions.UninstallService)
+ {
+ throw std::runtime_error("Service mode is not supported on this platform");
+ }
+
+ ZenEntryPoint App(ServerOptions);
+ return App.Run();
+#endif // ZEN_PLATFORM_WINDOWS
}
catch (std::exception& Ex)
{