aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/zen/cmds/builds_cmd.cpp54
-rw-r--r--src/zenserver/buildstore/buildstore.cpp75
-rw-r--r--src/zenserver/buildstore/buildstore.h30
-rw-r--r--src/zenserver/buildstore/httpbuildstore.cpp28
-rw-r--r--src/zenserver/buildstore/httpbuildstore.h28
-rw-r--r--src/zenserver/config.h1
-rw-r--r--src/zenserver/zenserver.cpp10
-rw-r--r--src/zenserver/zenserver.h2
-rw-r--r--src/zenutil/include/zenutil/chunkedcontent.h135
-rw-r--r--src/zenutil/include/zenutil/logging/rotatingfilesink.h1
10 files changed, 348 insertions, 16 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 11efc42ec..5ed3642d8 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -50,6 +50,8 @@ ZEN_THIRD_PARTY_INCLUDES_END
#define EXTRA_VERIFY 0
+#define ZEN_CLOUD_STORAGE "Cloud Storage"
+
namespace zen {
namespace {
static std::atomic<bool> AbortFlag = false;
@@ -761,6 +763,8 @@ namespace {
#endif // EXTRA_VERIFY
}
+ bool g_UseDeltaEncoding = true;
+
void WriteBuildContentToCompactBinary(CbObjectWriter& PartManifestWriter,
const SourcePlatform Platform,
std::span<const std::filesystem::path> Paths,
@@ -771,7 +775,7 @@ namespace {
std::span<const uint32_t> ChunkCounts,
std::span<const IoHash> LocalChunkHashes,
std::span<const uint64_t> LocalChunkRawSizes,
- std::vector<uint32_t> AbsoluteChunkOrders,
+ const std::vector<uint32_t>& AbsoluteChunkOrders,
const std::span<const uint32_t> LooseLocalChunkIndexes,
const std::span<IoHash> BlockHashes)
{
@@ -805,7 +809,15 @@ namespace {
{
compactbinary_helpers::WriteArray(SequenceRawHashes, "sequenceRawHashes"sv, PartManifestWriter);
compactbinary_helpers::WriteArray(ChunkCounts, "chunkcounts"sv, PartManifestWriter);
- compactbinary_helpers::WriteArray(AbsoluteChunkOrders, "chunkorders"sv, PartManifestWriter);
+
+ if (g_UseDeltaEncoding)
+ {
+ compactbinary_helpers::WriteDeltaArray(AbsoluteChunkOrders, "chunkorders_delta"sv, PartManifestWriter);
+ }
+ else
+ {
+ compactbinary_helpers::WriteArray(AbsoluteChunkOrders, "chunkorders"sv, PartManifestWriter);
+ }
}
PartManifestWriter.EndObject(); // chunkedContent
@@ -918,7 +930,14 @@ namespace {
{
throw std::runtime_error(fmt::format("Number of chunk count entries does not match number of paths"));
}
- compactbinary_helpers::ReadArray("chunkorders"sv, ChunkContentView, OutAbsoluteChunkOrders);
+ if (ChunkContentView["chunkorders_delta"sv])
+ {
+ compactbinary_helpers::ReadDeltaArray("chunkorders_delta"sv, ChunkContentView, OutAbsoluteChunkOrders);
+ }
+ else
+ {
+ compactbinary_helpers::ReadArray("chunkorders"sv, ChunkContentView, OutAbsoluteChunkOrders);
+ }
}
else if (FilesObject["chunkcounts"sv])
{
@@ -2984,7 +3003,7 @@ namespace {
(FindBlocksStats.AcceptedByteCount + FindBlocksStats.AcceptedReduntantByteCount)
: 0.0;
ZEN_CONSOLE(
- "Found {} chunks in {} ({}) blocks eligeble for reuse in {}\n"
+ "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n"
" Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n"
" Accepting {} ({}) redundant chunks ({:.1f}%)\n"
" Rejected {} ({}) chunks in {} blocks\n"
@@ -7202,6 +7221,12 @@ BuildsCommand::BuildsCommand()
"<manifestpath>");
m_UploadOptions
.add_option("", "", "verify", "Enable post upload verify of all uploaded data", cxxopts::value(m_PostUploadVerify), "<verify>");
+ m_UploadOptions.add_option("",
+ "",
+ "allow-deltaencoding",
+ "Allow efficient encoding of build manifest. Defaults to true.",
+ cxxopts::value(g_UseDeltaEncoding),
+ "<allowdeltaencoding>");
m_UploadOptions.parse_positional({"local-path", "build-id"});
m_UploadOptions.positional_help("local-path build-id");
@@ -7729,21 +7754,20 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
std::string StorageName;
if (!m_BuildsUrl.empty())
{
- ZEN_CONSOLE("Downloading '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'",
+ ZEN_CONSOLE("Downloading Build '{}' to '{}' from {}. SessionId: '{}'. Namespace '{}', Bucket '{}'",
BuildId,
Path,
m_BuildsUrl,
Http.GetSessionId(),
m_Namespace,
- m_Bucket,
- BuildId);
+ m_Bucket);
Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName);
- StorageName = "Cloud DDC";
+ StorageName = ZEN_CLOUD_STORAGE;
}
else if (!m_StoragePath.empty())
{
std::filesystem::path StoragePath = StringToPath(m_StoragePath);
- ZEN_CONSOLE("Downloading '{}' to '{}' from folder {}. BuildId '{}'", BuildId, Path, StoragePath, BuildId);
+ ZEN_CONSOLE("Downloading Build '{}' to '{}' from folder {}", BuildId, Path, StoragePath);
Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
StorageName = fmt::format("Disk {}", StoragePath.stem());
}
@@ -7828,7 +7852,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
std::string StorageName;
if (!m_BuildsUrl.empty())
{
- ZEN_CONSOLE("Downloading {} to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'",
+ ZEN_CONSOLE("Downloading {} to '{}' from {}. SessionId: '{}'. Namespace '{}', Bucket '{}'",
FormatArray<std::string>(m_BuildIds, " "sv),
Path,
m_BuildsUrl,
@@ -7836,12 +7860,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
m_Namespace,
m_Bucket);
Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName);
- StorageName = "Cloud DDC";
+ StorageName = ZEN_CLOUD_STORAGE;
}
else if (!m_StoragePath.empty())
{
std::filesystem::path StoragePath = StringToPath(m_StoragePath);
- ZEN_CONSOLE("Downloading {}'to '{}' from folder {}", FormatArray<std::string>(m_BuildIds, " "sv), Path, StoragePath);
+ ZEN_CONSOLE("Downloading '{}' to '{}' from folder '{}'", FormatArray<std::string>(m_BuildIds, " "sv), Path, StoragePath);
Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
StorageName = fmt::format("Disk {}", StoragePath.stem());
}
@@ -7929,7 +7953,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
m_Bucket,
BuildId);
Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName);
- StorageName = "Cloud DDC";
+ StorageName = ZEN_CLOUD_STORAGE;
}
else if (!StoragePath.empty())
{
@@ -8222,7 +8246,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
m_Bucket,
BuildId);
Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName);
- StorageName = "Cloud DDC";
+ StorageName = ZEN_CLOUD_STORAGE;
}
else if (!m_StoragePath.empty())
{
@@ -8292,7 +8316,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
m_Bucket,
BuildId);
Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName);
- StorageName = "Cloud DDC";
+ StorageName = ZEN_CLOUD_STORAGE;
}
else if (!m_StoragePath.empty())
{
diff --git a/src/zenserver/buildstore/buildstore.cpp b/src/zenserver/buildstore/buildstore.cpp
new file mode 100644
index 000000000..81dffc09c
--- /dev/null
+++ b/src/zenserver/buildstore/buildstore.cpp
@@ -0,0 +1,75 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "buildstore.h"
+
+#include <zencore/testing.h>
+#include <zencore/testutils.h>
+
+#include <zencore/uid.h>
+#include <zencore/xxhash.h>
+
+namespace zen {
+
+BuildStore::BuildStore(BuildStoreConfig Config)
+{
+ const uint64_t MaxBlockSize = 256 * 1024 * 1024;
+ const uint64_t MaxBlockCount = 32 * 1024;
+ m_BlockStore.Initialize(Config.RootDirectory, MaxBlockSize, MaxBlockCount);
+}
+
+BuildStore::~BuildStore()
+{
+}
+
+// TODO: reconsider key size
+inline Oid
+IdFromKey(std::string_view Key)
+{
+ XXH3_128 Hash = XXH3_128::HashMemory(Key.data(), Key.size());
+
+ Oid Id;
+ memcpy(&Id.OidBits, Hash.Hash, sizeof Id);
+ return Id;
+}
+
+void
+BuildStore::Put(std::string_view Key, IoBuffer Value)
+{
+ ZEN_UNUSED(Key, Value);
+}
+
+IoBuffer
+BuildStore::Get(std::string_view Key)
+{
+ ZEN_UNUSED(Key);
+ return {};
+}
+
+/*
+ ___________ __
+ \__ ___/___ _______/ |_ ______
+ | |_/ __ \ / ___/\ __\/ ___/
+ | |\ ___/ \___ \ | | \___ \
+ |____| \___ >____ > |__| /____ >
+ \/ \/ \/
+*/
+
+#if ZEN_WITH_TESTS
+
+TEST_CASE("BuildStore")
+{
+ ScopedTemporaryDirectory _;
+
+ BuildStoreConfig Config;
+ Config.RootDirectory = _.Path() / "build_store";
+ BuildStore Store(Config);
+}
+
+void
+buildstore_forcelink()
+{
+}
+
+#endif
+
+} // namespace zen
diff --git a/src/zenserver/buildstore/buildstore.h b/src/zenserver/buildstore/buildstore.h
new file mode 100644
index 000000000..b2801fc7f
--- /dev/null
+++ b/src/zenserver/buildstore/buildstore.h
@@ -0,0 +1,30 @@
+
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenstore/blockstore.h>
+
+#include <filesystem>
+
+namespace zen {
+
+struct BuildStoreConfig
+{
+ std::filesystem::path RootDirectory;
+};
+
+class BuildStore
+{
+public:
+ BuildStore(BuildStoreConfig Config);
+ ~BuildStore();
+
+ void Put(std::string_view Key, IoBuffer Value);
+ IoBuffer Get(std::string_view Key);
+
+private:
+ BlockStore m_BlockStore;
+};
+
+void buildstore_forcelink();
+
+} // namespace zen
diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp
new file mode 100644
index 000000000..1a0998c15
--- /dev/null
+++ b/src/zenserver/buildstore/httpbuildstore.cpp
@@ -0,0 +1,28 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "httpbuildstore.h"
+
+namespace zen {
+
+HttpBuildStoreService::HttpBuildStoreService(BuildStoreConfig Cfg) : m_Config(Cfg)
+{
+ m_BuildStore = std::make_unique<BuildStore>(m_Config);
+}
+
+HttpBuildStoreService::~HttpBuildStoreService()
+{
+}
+
+const char*
+HttpBuildStoreService::HttpBuildStoreService::BaseUri() const
+{
+ return "/builds/";
+}
+
+void
+HttpBuildStoreService::HandleRequest(zen::HttpServerRequest& Request)
+{
+ ZEN_UNUSED(Request);
+}
+
+} // namespace zen
diff --git a/src/zenserver/buildstore/httpbuildstore.h b/src/zenserver/buildstore/httpbuildstore.h
new file mode 100644
index 000000000..d34035d0f
--- /dev/null
+++ b/src/zenserver/buildstore/httpbuildstore.h
@@ -0,0 +1,28 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "buildstore.h"
+
+#include <zenhttp/httpserver.h>
+
+#include <filesystem>
+
+namespace zen {
+
+class HttpBuildStoreService final : public zen::HttpService
+{
+public:
+ HttpBuildStoreService(BuildStoreConfig Cfg);
+ virtual ~HttpBuildStoreService();
+
+ virtual const char* BaseUri() const override;
+ virtual void HandleRequest(zen::HttpServerRequest& Request) override;
+
+private:
+ BuildStoreConfig m_Config;
+
+ std::unique_ptr<BuildStore> m_BuildStore;
+};
+
+} // namespace zen
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index c7781aada..d2965b819 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -172,6 +172,7 @@ struct ZenServerOptions
bool SentryAllowPII = false; // Allow personally identifiable information in sentry crash reports
bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux)
bool ObjectStoreEnabled = false;
+ bool BuildStoreEnabled = false;
bool NoConsoleOutput = false; // Control default use of stdout for diagnostics
std::string Loggers[zen::logging::level::LogLevelCount];
std::string ScrubOptions;
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index f84bc0b00..ace8229d7 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -310,6 +310,15 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
m_Http->RegisterService(*m_ObjStoreService);
}
+ if (ServerOptions.BuildStoreEnabled)
+ {
+ BuildStoreConfig ObjCfg;
+ ObjCfg.RootDirectory = m_DataRoot / "builds";
+
+ m_BuildStoreService = std::make_unique<HttpBuildStoreService>(std::move(ObjCfg));
+ m_Http->RegisterService(*m_BuildStoreService);
+ }
+
#if ZEN_WITH_VFS
m_VfsService = std::make_unique<VfsService>();
m_VfsService->AddService(Ref<ProjectStore>(m_ProjectStore));
@@ -1056,6 +1065,7 @@ void
zenserver_forcelinktests()
{
zen::prj_forcelink();
+ zen::buildstore_forcelink();
}
#endif
diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h
index 80054dc35..b698f001b 100644
--- a/src/zenserver/zenserver.h
+++ b/src/zenserver/zenserver.h
@@ -25,6 +25,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
#include <zenstore/cache/structuredcachestore.h>
#include <zenstore/gc.h>
#include "admin/admin.h"
+#include "buildstore/httpbuildstore.h"
#include "cache/httpstructuredcache.h"
#include "diag/diagsvcs.h"
#include "frontend/frontend.h"
@@ -140,6 +141,7 @@ private:
HttpHealthService m_HealthService;
std::unique_ptr<HttpFrontendService> m_FrontendService;
std::unique_ptr<HttpObjectStoreService> m_ObjStoreService;
+ std::unique_ptr<HttpBuildStoreService> m_BuildStoreService;
std::unique_ptr<VfsService> m_VfsService;
std::unique_ptr<JobQueue> m_JobQueue;
std::unique_ptr<HttpAdminService> m_AdminService;
diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h
index 57b55cb8e..002af1f75 100644
--- a/src/zenutil/include/zenutil/chunkedcontent.h
+++ b/src/zenutil/include/zenutil/chunkedcontent.h
@@ -195,6 +195,61 @@ namespace compactbinary_helpers {
WriteArray(std::span<const Type>(Values), ArrayName, Output);
}
+ inline void WriteDeltaArray(std::span<const uint32_t> Values, std::string_view ArrayName, CbWriter& Output)
+ {
+ Output.BeginArray(ArrayName);
+ uint32_t PreviousValue = 0;
+ uint64_t PreviousDeltaZigZag = 1;
+ uint32_t RunLength = 0;
+
+ auto EmitEndRun = [&] {
+ switch (RunLength)
+ {
+ case 0:
+ return;
+ case 1:
+ Output.AddInteger(PreviousDeltaZigZag);
+ break;
+ case 2:
+ Output.AddInteger(PreviousDeltaZigZag);
+ Output.AddInteger(PreviousDeltaZigZag);
+ break;
+ default: // >= 3
+ Output.AddInteger(1);
+ Output.AddInteger(RunLength - 3);
+ break;
+ }
+
+ RunLength = 0;
+ };
+
+ for (const uint32_t Value : Values)
+ {
+ int64_t Delta = int64_t(Value) - PreviousValue;
+ const uint64_t DeltaZigZag = (Delta >= 0) ? (Delta << 1) : (((-Delta) << 1) | 1);
+
+ if (DeltaZigZag == PreviousDeltaZigZag)
+ {
+ ++RunLength;
+ }
+ else
+ {
+ EmitEndRun();
+ Output.AddInteger(DeltaZigZag);
+
+ PreviousDeltaZigZag = DeltaZigZag;
+ }
+ PreviousValue = Value;
+ }
+ EmitEndRun();
+ Output.EndArray();
+ }
+
+ inline void WriteDeltaArray(const std::vector<uint32_t>& Values, std::string_view ArrayName, CbWriter& Output)
+ {
+ WriteDeltaArray(std::span<const uint32_t>(Values), ArrayName, Output);
+ }
+
template<>
inline void WriteArray(std::span<const std::filesystem::path> Values, std::string_view ArrayName, CbWriter& Output)
{
@@ -237,6 +292,86 @@ namespace compactbinary_helpers {
}
}
+ inline uint64_t ReadDeltaArray(std::string_view ArrayName, CbObjectView Input, std::vector<uint32_t>& Result)
+ {
+ CbArrayView Array = Input[ArrayName].AsArrayView();
+
+ uint64_t EncodedEntryCount = 0;
+
+ {
+ // Count entries for reserve
+
+ uint64_t EntryCount = 0;
+
+ bool InRun = false;
+
+ for (CbFieldView ItemView : Array)
+ {
+ ++EncodedEntryCount;
+
+ const uint64_t DeltaZigZag = ItemView.AsUInt64();
+ if (InRun)
+ {
+ uint64_t RunLength = DeltaZigZag + 3;
+
+ EntryCount += RunLength;
+
+ InRun = false;
+ }
+ else if (DeltaZigZag == 1)
+ {
+ // Encoded run, next value is the repeat count
+ InRun = true;
+ }
+ else
+ {
+ ++EntryCount;
+ }
+ }
+
+ Result.reserve(EntryCount);
+ }
+
+ uint32_t PreviousValue = 0;
+ uint64_t PreviousDeltaZigZag = 1;
+
+ auto EmitEntry = [&](uint64_t DeltaZigZag) {
+ PreviousDeltaZigZag = DeltaZigZag;
+ const int64_t Delta = (DeltaZigZag & 1) ? -int64_t((DeltaZigZag >> 1)) : int64_t(DeltaZigZag >> 1);
+ PreviousValue = uint32_t(PreviousValue + Delta);
+ Result.push_back(PreviousValue);
+ };
+
+ bool InRun = false;
+
+ for (CbFieldView ItemView : Array)
+ {
+ const uint64_t DeltaZigZag = ItemView.AsUInt64();
+ if (InRun)
+ {
+ uint64_t RunLength = DeltaZigZag + 3;
+
+ while (RunLength--)
+ {
+ EmitEntry(PreviousDeltaZigZag);
+ }
+
+ InRun = false;
+ }
+ else if (DeltaZigZag == 1)
+ {
+ // Encoded run, next value is the repeat count
+ InRun = true;
+ }
+ else
+ {
+ EmitEntry(DeltaZigZag);
+ }
+ }
+
+ return EncodedEntryCount;
+ }
+
inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector<uint64_t>& Result)
{
CbArrayView Array = Input[ArrayName].AsArrayView();
diff --git a/src/zenutil/include/zenutil/logging/rotatingfilesink.h b/src/zenutil/include/zenutil/logging/rotatingfilesink.h
index 758722156..cd28bdcb2 100644
--- a/src/zenutil/include/zenutil/logging/rotatingfilesink.h
+++ b/src/zenutil/include/zenutil/logging/rotatingfilesink.h
@@ -27,7 +27,6 @@ public:
{
ZEN_MEMSCOPE(ELLMTag::Logging);
- ZEN_MEMSCOPE(ELLMTag::Logging);
std::error_code Ec;
if (RotateOnOpen)
{
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-26 12:54:14 +0000