aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorZousar Shaker <[email protected]>2025-04-24 08:26:29 -0600
committerGitHub Enterprise <[email protected]>2025-04-24 08:26:29 -0600
commit787449efb4de24fd12f3af3c4e466a9629203108 (patch)
treecb4a0e84a0381e9b4a087401037ba5837e8e65d7 /src
parentChangelog update terminology (diff)
parent5.6.6-pre1 (diff)
downloadzen-787449efb4de24fd12f3af3c4e466a9629203108.tar.xz
zen-787449efb4de24fd12f3af3c4e466a9629203108.zip
Merge branch 'main' into zs/zencli-list-namespaces-buckets
Diffstat (limited to 'src')
-rw-r--r--src/transports/transport-sdk/include/transportplugin.h42
-rw-r--r--src/transports/winsock/winsock.cpp16
-rw-r--r--src/zen/cmds/admin_cmd.cpp19
-rw-r--r--src/zen/cmds/admin_cmd.h8
-rw-r--r--src/zen/cmds/builds_cmd.cpp760
-rw-r--r--src/zen/cmds/builds_cmd.h50
-rw-r--r--src/zen/cmds/cache_cmd.cpp22
-rw-r--r--src/zen/cmds/cache_cmd.h18
-rw-r--r--src/zen/cmds/copy_cmd.cpp7
-rw-r--r--src/zen/cmds/copy_cmd.h10
-rw-r--r--src/zen/cmds/dedup_cmd.h4
-rw-r--r--src/zen/cmds/status_cmd.cpp9
-rw-r--r--src/zen/cmds/status_cmd.h6
-rw-r--r--src/zen/cmds/up_cmd.cpp41
-rw-r--r--src/zen/cmds/up_cmd.h28
-rw-r--r--src/zen/cmds/wipe_cmd.cpp575
-rw-r--r--src/zen/cmds/wipe_cmd.h36
-rw-r--r--src/zen/cmds/workspaces_cmd.cpp55
-rw-r--r--src/zen/cmds/workspaces_cmd.h28
-rw-r--r--src/zen/zen.cpp3
-rw-r--r--src/zen/zen.h5
-rw-r--r--src/zencore/filesystem.cpp16
-rw-r--r--src/zencore/include/zencore/filesystem.h2
-rw-r--r--src/zencore/include/zencore/process.h3
-rw-r--r--src/zencore/process.cpp142
-rw-r--r--src/zenhttp/httpclient.cpp4
-rw-r--r--src/zenhttp/httpserver.cpp187
-rw-r--r--src/zenhttp/include/zenhttp/httpserver.h15
-rw-r--r--src/zenhttp/servers/httpmulti.cpp4
-rw-r--r--src/zenhttp/transports/dlltransport.cpp92
-rw-r--r--src/zenhttp/transports/dlltransport.h2
-rw-r--r--src/zenserver/config.cpp112
-rw-r--r--src/zenserver/config.h1
-rw-r--r--src/zenserver/config/luaconfig.cpp23
-rw-r--r--src/zenserver/config/luaconfig.h5
-rw-r--r--src/zenserver/frontend/html.zipbin156798 -> 157061 bytes
-rw-r--r--src/zenserver/frontend/html/indexer/indexer.js2
-rw-r--r--src/zenserver/frontend/html/pages/entry.js65
-rw-r--r--src/zenstore/cache/cachedisklayer.cpp2
-rw-r--r--src/zenutil/commandlineoptions.cpp213
-rw-r--r--src/zenutil/include/zenutil/commandlineoptions.h28
-rw-r--r--src/zenutil/zenutil.cpp2
42 files changed, 1735 insertions, 927 deletions
diff --git a/src/transports/transport-sdk/include/transportplugin.h b/src/transports/transport-sdk/include/transportplugin.h
index 4347868e6..a78a758bc 100644
--- a/src/transports/transport-sdk/include/transportplugin.h
+++ b/src/transports/transport-sdk/include/transportplugin.h
@@ -17,10 +17,14 @@
namespace zen {
+// Current API version, value will be incremented to represent breaking changes
+static const uint32_t kTransportApiVersion = 1;
+
class TransportConnection;
class TransportPlugin;
class TransportServerConnection;
class TransportServer;
+class TransportLogger;
/*************************************************************************
@@ -60,6 +64,29 @@ public:
virtual TransportServerConnection* CreateConnectionHandler(TransportConnection* Connection) = 0;
};
+/** Logger interface
+
+ There will be one instance of this provided by the system to the transport plugin
+
+ The plugin can use this to log messages back to zen server
+
+ */
+class TransportLogger
+{
+public:
+ enum class LogLevel : uint32_t
+ {
+ Trace = 0,
+ Debug = 1,
+ Info = 2,
+ Warn = 3,
+ Err = 4,
+ Critical = 5,
+ };
+
+ virtual void LogMessage(LogLevel Level, const char* Message) = 0;
+};
+
/*************************************************************************
The following interfaces are to be implemented by transport plugins.
@@ -116,7 +143,18 @@ public:
extern "C"
{
- DLL_TRANSPORT_API zen::TransportPlugin* CreateTransportPlugin();
+ /** Provide information about plugin version
+
+ Fills out API version (kTransportApiVersion) plugin was built against.
+ Fills out plugin own version ever increasing version number,
+ a copy of plugin with higher version will be used.
+ */
+ DLL_TRANSPORT_API void GetTransportPluginVersion(uint32_t* OutApiVersion, uint32_t* OutPluginVersion);
+
+ // Return nullptr if requested api version mismatches api version plugin was built against
+ DLL_TRANSPORT_API zen::TransportPlugin* CreateTransportPlugin(zen::TransportLogger* Logger);
}
-typedef zen::TransportPlugin* (*PfnCreateTransportPlugin)();
+typedef void (*PfnGetTransportPluginVersion)(uint32_t* OutApiVersion, uint32_t* OutPluginVersion);
+
+typedef zen::TransportPlugin* (*PfnCreateTransportPlugin)(zen::TransportLogger* Logger);
diff --git a/src/transports/winsock/winsock.cpp b/src/transports/winsock/winsock.cpp
index 1c3ee909a..f98984726 100644
--- a/src/transports/winsock/winsock.cpp
+++ b/src/transports/winsock/winsock.cpp
@@ -364,8 +364,22 @@ WinsockTransportPlugin::IsAvailable()
//////////////////////////////////////////////////////////////////////////
+void
+GetTransportPluginVersion(uint32_t* OutApiVersion, uint32_t* OutPluginVersion)
+{
+ if (OutApiVersion != nullptr)
+ {
+ *OutApiVersion = kTransportApiVersion;
+ }
+
+ if (OutPluginVersion != nullptr)
+ {
+ *OutPluginVersion = 1;
+ }
+}
+
TransportPlugin*
-CreateTransportPlugin()
+CreateTransportPlugin([[maybe_unused]] TransportLogger* Logger)
{
return new WinsockTransportPlugin;
}
diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp
index 573639c2d..a7cfa6a4e 100644
--- a/src/zen/cmds/admin_cmd.cpp
+++ b/src/zen/cmds/admin_cmd.cpp
@@ -714,29 +714,26 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
throw OptionParseException("data path must be given");
}
- std::filesystem::path DataPath = StringToPath(m_DataPath);
- std::filesystem::path TargetPath = StringToPath(m_TargetPath);
-
- if (!IsDir(DataPath))
+ if (!IsDir(m_DataPath))
{
throw OptionParseException("data path must exist");
}
- if (TargetPath.empty())
+ if (m_TargetPath.empty())
{
throw OptionParseException("target path must be given");
}
- std::filesystem::path RootManifestPath = DataPath / "root_manifest";
- std::filesystem::path TargetRootManifestPath = TargetPath / "root_manifest";
+ std::filesystem::path RootManifestPath = m_DataPath / "root_manifest";
+ std::filesystem::path TargetRootManifestPath = m_TargetPath / "root_manifest";
if (!TryCopy(RootManifestPath, TargetRootManifestPath))
{
throw OptionParseException("data path is invalid, missing root_manifest");
}
- std::filesystem::path CachePath = DataPath / "cache";
- std::filesystem::path TargetCachePath = TargetPath / "cache";
+ std::filesystem::path CachePath = m_DataPath / "cache";
+ std::filesystem::path TargetCachePath = m_TargetPath / "cache";
// Copy cache state
DirectoryContent CacheDirectoryContent;
@@ -781,8 +778,8 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
}
- std::filesystem::path CasPath = DataPath / "cas";
- std::filesystem::path TargetCasPath = TargetPath / "cas";
+ std::filesystem::path CasPath = m_DataPath / "cas";
+ std::filesystem::path TargetCasPath = m_TargetPath / "cas";
{
std::filesystem::path UCasRootPath = CasPath / ".ucas_root";
diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h
index 8b6d3e258..c593b2cac 100644
--- a/src/zen/cmds/admin_cmd.h
+++ b/src/zen/cmds/admin_cmd.h
@@ -155,10 +155,10 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"copy-state", "Copy zen server disk state"};
- std::string m_DataPath;
- std::string m_TargetPath;
- bool m_SkipLogs = false;
+ cxxopts::Options m_Options{"copy-state", "Copy zen server disk state"};
+ std::filesystem::path m_DataPath;
+ std::filesystem::path m_TargetPath;
+ bool m_SkipLogs = false;
};
} // namespace zen
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 624bb2270..15c635594 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -164,24 +164,6 @@ namespace {
);
- std::filesystem::path MakeSafeAbsolutePath(std::filesystem::path Path)
- {
- std::filesystem::path AbsolutePath = std::filesystem::absolute(Path).make_preferred();
-#if ZEN_PLATFORM_WINDOWS && 1
- const std::string_view Prefix = "\\\\?\\";
- const std::u8string PrefixU8(Prefix.begin(), Prefix.end());
- std::u8string PathString = AbsolutePath.u8string();
- if (!PathString.empty() && !PathString.starts_with(PrefixU8))
- {
- PathString.insert(0, PrefixU8);
- return std::filesystem::path(PathString);
- }
-#endif
- return AbsolutePath;
- }
-
- std::filesystem::path MakeSafeAbsolutePath(const std::string PathString) { return MakeSafeAbsolutePath(StringToPath(PathString)); }
-
bool IsFileWithRetry(const std::filesystem::path& Path)
{
std::error_code Ec;
@@ -690,75 +672,6 @@ namespace {
return CacheFolderPath / RawHash.ToHexString();
}
- ChunkedFolderContent ScanAndChunkFolder(
- GetFolderContentStatistics& GetFolderContentStats,
- ChunkingStatistics& ChunkingStats,
- const std::filesystem::path& Path,
- std::function<bool(const std::string_view& RelativePath)>&& IsAcceptedFolder,
- std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile,
- ChunkingController& ChunkController)
- {
- ZEN_TRACE_CPU("ScanAndChunkFolder");
-
- FolderContent Content = GetFolderContent(
- GetFolderContentStats,
- Path,
- std::move(IsAcceptedFolder),
- std::move(IsAcceptedFile),
- GetIOWorkerPool(),
- UsePlainProgress ? 5000 : 200,
- [](bool, std::ptrdiff_t) {},
- AbortFlag);
- if (AbortFlag)
- {
- return {};
- }
-
- ProgressBar ProgressBar(UsePlainProgress);
- FilteredRate FilteredBytesHashed;
- FilteredBytesHashed.Start();
- ChunkedFolderContent FolderContent = ChunkFolderContent(
- ChunkingStats,
- GetIOWorkerPool(),
- Path,
- Content,
- ChunkController,
- UsePlainProgress ? 5000 : 200,
- [&](bool, std::ptrdiff_t) {
- FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load());
- std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
- ChunkingStats.FilesProcessed.load(),
- GetFolderContentStats.AcceptedFileCount.load(),
- NiceBytes(ChunkingStats.BytesHashed.load()),
- NiceBytes(GetFolderContentStats.FoundFileByteCount),
- NiceNum(FilteredBytesHashed.GetCurrent()),
- ChunkingStats.UniqueChunksFound.load(),
- NiceBytes(ChunkingStats.UniqueBytesFound.load()));
- ProgressBar.UpdateState({.Task = "Scanning files ",
- .Details = Details,
- .TotalCount = GetFolderContentStats.AcceptedFileByteCount,
- .RemainingCount = GetFolderContentStats.AcceptedFileByteCount - ChunkingStats.BytesHashed.load()},
- false);
- },
- AbortFlag);
- if (AbortFlag)
- {
- return {};
- }
- FilteredBytesHashed.Stop();
- ProgressBar.Finish();
-
- ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
- ChunkingStats.FilesProcessed.load(),
- NiceBytes(ChunkingStats.BytesHashed.load()),
- ChunkingStats.UniqueChunksFound.load(),
- NiceBytes(ChunkingStats.UniqueBytesFound.load()),
- Path,
- NiceTimeSpanMs((GetFolderContentStats.ElapsedWallTimeUS + ChunkingStats.ElapsedWallTimeUS) / 1000),
- NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
- return FolderContent;
- };
-
struct DiskStatistics
{
std::atomic<uint64_t> OpenReadCount = 0;
@@ -2562,7 +2475,7 @@ namespace {
BlockIndexes.push_back(It->second);
TotalBlocksSize += NewBlocks.BlockSizes[It->second];
}
- if (auto ChunkIndexIt = Lookup.ChunkHashToChunkIndex.find(RawHash); ChunkIndexIt != Lookup.ChunkHashToChunkIndex.end())
+ else if (auto ChunkIndexIt = Lookup.ChunkHashToChunkIndex.find(RawHash); ChunkIndexIt != Lookup.ChunkHashToChunkIndex.end())
{
const uint32_t ChunkIndex = ChunkIndexIt->second;
if (auto LooseOrderIndexIt = ChunkIndexToLooseChunkOrderIndex.find(ChunkIndex);
@@ -2572,6 +2485,11 @@ namespace {
TotalLooseChunksSize += Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
}
}
+ else
+ {
+ throw std::runtime_error(
+ fmt::format("Can not upload requested build blob {} as it was not generated by this upload", RawHash));
+ }
}
uint64_t TotalRawSize = TotalLooseChunksSize + TotalBlocksSize;
@@ -3169,8 +3087,9 @@ namespace {
auto ParseManifest = [](const std::filesystem::path& Path,
const std::filesystem::path& ManifestPath) -> std::vector<std::filesystem::path> {
std::vector<std::filesystem::path> AssetPaths;
- std::filesystem::path AbsoluteManifestPath = ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath;
- IoBuffer ManifestContent = ReadFile(AbsoluteManifestPath).Flatten();
+ std::filesystem::path AbsoluteManifestPath =
+ MakeSafeAbsolutePath(ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath);
+ IoBuffer ManifestContent = ReadFile(AbsoluteManifestPath).Flatten();
std::string_view ManifestString((const char*)ManifestContent.GetView().GetData(), ManifestContent.GetSize());
std::string_view::size_type Offset = 0;
while (Offset < ManifestContent.GetSize())
@@ -3730,7 +3649,8 @@ namespace {
UploadAttachments(PutBuildPartResult.second);
}
- while (!AbortFlag)
+ uint32_t FinalizeBuildPartRetryCount = 5;
+ while (!AbortFlag && (FinalizeBuildPartRetryCount--) > 0)
{
Stopwatch FinalizeBuildPartTimer;
std::vector<IoHash> Needs = Storage.BuildStorage->FinalizeBuildPart(BuildId, BuildPartId, PartHash);
@@ -8175,13 +8095,13 @@ namespace {
return RemoteContent;
}
- ChunkedFolderContent GetLocalContent(GetFolderContentStatistics& LocalFolderScanStats,
- ChunkingStatistics& ChunkingStats,
- const std::filesystem::path& Path,
- const std::filesystem::path& StateFilePath,
- ChunkingController& ChunkController,
- const ChunkedFolderContent& ReferenceContent,
- FolderContent& OutLocalFolderContent)
+ ChunkedFolderContent GetLocalContent(GetFolderContentStatistics& LocalFolderScanStats,
+ ChunkingStatistics& ChunkingStats,
+ const std::filesystem::path& Path,
+ const std::filesystem::path& StateFilePath,
+ ChunkingController& ChunkController,
+ std::span<const std::filesystem::path> ReferencePaths,
+ FolderContent& OutLocalFolderContent)
{
FolderContent LocalFolderState;
ChunkedFolderContent LocalContent;
@@ -8193,7 +8113,7 @@ namespace {
ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs()));
}
{
- const uint32_t LocalPathCount = gsl::narrow<uint32_t>(ReferenceContent.Paths.size());
+ const uint32_t LocalPathCount = gsl::narrow<uint32_t>(ReferencePaths.size());
const uint32_t RemotePathCount = gsl::narrow<uint32_t>(LocalFolderState.Paths.size());
std::vector<std::filesystem::path> PathsToCheck;
@@ -8208,7 +8128,7 @@ namespace {
PathsToCheck.push_back(LocalPath);
}
- for (const std::filesystem::path& RemotePath : ReferenceContent.Paths)
+ for (const std::filesystem::path& RemotePath : ReferencePaths)
{
if (FileSet.insert(RemotePath.generic_string()).second)
{
@@ -8351,6 +8271,56 @@ namespace {
return LocalContent;
}
+ ChunkedFolderContent ScanAndChunkFolder(
+ GetFolderContentStatistics& GetFolderContentStats,
+ ChunkingStatistics& ChunkingStats,
+ const std::filesystem::path& Path,
+ std::function<bool(const std::string_view& RelativePath)>&& IsAcceptedFolder,
+ std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile,
+ ChunkingController& ChunkController)
+ {
+ Stopwatch Timer;
+
+ ZEN_TRACE_CPU("ScanAndChunkFolder");
+
+ FolderContent Content = GetFolderContent(
+ GetFolderContentStats,
+ Path,
+ std::move(IsAcceptedFolder),
+ std::move(IsAcceptedFile),
+ GetIOWorkerPool(),
+ UsePlainProgress ? 5000 : 200,
+ [](bool, std::ptrdiff_t) {},
+ AbortFlag);
+ if (AbortFlag)
+ {
+ return {};
+ }
+
+ FolderContent _;
+ ChunkedFolderContent Result = GetLocalContent(GetFolderContentStats,
+ ChunkingStats,
+ Path,
+ ZenStateFilePath(Path / ZenFolderName),
+ ChunkController,
+ Content.Paths,
+ _);
+
+ const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0));
+ const uint64_t ChunkedRawSize =
+ std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0));
+
+ ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
+ Result.Paths.size(),
+ NiceBytes(TotalRawSize),
+ Result.ChunkedContent.ChunkHashes.size(),
+ NiceBytes(ChunkedRawSize),
+ Path,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+ NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ return Result;
+ };
+
void DownloadFolder(StorageInstance& Storage,
const Oid& BuildId,
const std::vector<Oid>& BuildPartIds,
@@ -8415,7 +8385,7 @@ namespace {
Path,
ZenStateFilePath(ZenFolderPath),
*ChunkController,
- RemoteContent,
+ RemoteContent.Paths,
LocalFolderContent);
}
}
@@ -8712,7 +8682,7 @@ namespace {
double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0;
- ZEN_CONSOLE("{} ({}) files removed, {} ({}) files added, {} ({} {:.1f}%) files kept",
+ ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept",
RemovedHashes.size(),
NiceBytes(RemovedSize),
AddedHashes.size(),
@@ -8747,7 +8717,7 @@ namespace {
double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0;
double NewPercent = CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0;
- ZEN_CONSOLE("Found {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.",
+ ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.",
FoundChunkCount,
NiceBytes(FoundChunkSize),
FoundPercent,
@@ -8769,7 +8739,7 @@ BuildsCommand::BuildsCommand()
m_Options.add_options()("h,help", "Print help");
auto AddSystemOptions = [this](cxxopts::Options& Ops) {
- Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value<std::string>(m_SystemRootDir), "<systemdir>");
+ Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>");
};
auto AddAuthOptions = [this](cxxopts::Options& Ops) {
@@ -8890,7 +8860,7 @@ BuildsCommand::BuildsCommand()
Ops.add_option("",
"",
"boost-workers",
- "Increase the number of worker threads - may cause computer to less responsive",
+ "Increase the number of worker threads - may cause computer to be less responsive",
cxxopts::value(m_BoostWorkerThreads),
"<boostworkers>");
};
@@ -9218,8 +9188,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
std::filesystem::path SystemRootDir;
auto ParseSystemOptions = [&]() {
- SystemRootDir = m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : MakeSafeAbsolutePath(m_SystemRootDir);
+ if (m_SystemRootDir.empty())
+ {
+ m_SystemRootDir = PickDefaultSystemRootDirectory();
+ }
+ MakeSafeAbsolutePathÍnPlace(m_SystemRootDir);
};
+ ParseSystemOptions();
auto ParseStorageOptions = [&]() {
if (!m_OverrideHost.empty() || !m_Host.empty())
@@ -9238,6 +9213,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help()));
}
+ MakeSafeAbsolutePathÍnPlace(m_StoragePath);
};
std::unique_ptr<AuthMgr> Auth;
@@ -9247,7 +9223,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
.RetryCount = 2};
auto CreateAuthMgr = [&]() {
- ZEN_ASSERT(!SystemRootDir.empty());
+ ZEN_ASSERT(!m_SystemRootDir.empty());
if (!Auth)
{
if (m_EncryptionKey.empty())
@@ -9262,7 +9238,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZEN_CONSOLE("Warning: Using default encryption initialization vector");
}
- AuthConfig AuthMgrConfig = {.RootDirectory = SystemRootDir / "auth",
+ AuthConfig AuthMgrConfig = {.RootDirectory = m_SystemRootDir / "auth",
.EncryptionKey = AesKey256Bit::FromString(m_EncryptionKey),
.EncryptionIV = AesIV128Bit::FromString(m_EncryptionIV)};
if (!AuthMgrConfig.EncryptionKey.IsValid())
@@ -9278,7 +9254,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
};
auto ParseAuthOptions = [&]() {
- ParseSystemOptions();
if (!m_OpenIdProviderUrl.empty() && !m_OpenIdClientId.empty())
{
CreateAuthMgr();
@@ -9325,7 +9300,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else if (!m_AccessTokenPath.empty())
{
- std::string ResolvedAccessToken = ReadAccessTokenFromFile(MakeSafeAbsolutePath(m_AccessTokenPath));
+ MakeSafeAbsolutePathÍnPlace(m_AccessTokenPath);
+ std::string ResolvedAccessToken = ReadAccessTokenFromFile(m_AccessTokenPath);
if (!ResolvedAccessToken.empty())
{
ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken);
@@ -9567,10 +9543,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else if (!m_StoragePath.empty())
{
- std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath);
- StorageDescription = fmt::format("folder {}", StoragePath);
- Result.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
- Result.StorageName = fmt::format("Disk {}", StoragePath.stem());
+ StorageDescription = fmt::format("folder {}", m_StoragePath);
+ Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec);
+ Result.StorageName = fmt::format("Disk {}", m_StoragePath.stem());
}
else
{
@@ -9614,6 +9589,137 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return Result;
};
+ auto ParsePath = [&]() {
+ if (m_Path.empty())
+ {
+ throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help()));
+ }
+ MakeSafeAbsolutePathÍnPlace(m_Path);
+ };
+
+ auto ParseDiffPath = [&]() {
+ if (m_DiffPath.empty())
+ {
+ throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help()));
+ }
+ MakeSafeAbsolutePathÍnPlace(m_DiffPath);
+ };
+
+ auto ParseBlobHash = [&]() -> IoHash {
+ if (m_BlobHash.empty())
+ {
+ throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help()));
+ }
+
+ IoHash BlobHash;
+ if (!IoHash::TryParse(m_BlobHash, BlobHash))
+ {
+ throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help()));
+ }
+
+ return BlobHash;
+ };
+
+ auto ParseBuildId = [&]() -> Oid {
+ if (m_BuildId.length() != Oid::StringLength)
+ {
+ throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help()));
+ }
+ else if (Oid BuildId = Oid::FromHexString(m_BuildId); BuildId == Oid::Zero)
+ {
+ throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help()));
+ }
+ else
+ {
+ return BuildId;
+ }
+ };
+
+ auto ParseBuildPartId = [&]() -> Oid {
+ if (m_BuildPartId.length() != Oid::StringLength)
+ {
+ throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help()));
+ }
+ else if (Oid BuildPartId = Oid::FromHexString(m_BuildPartId); BuildPartId == Oid::Zero)
+ {
+ throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help()));
+ }
+ else
+ {
+ return BuildPartId;
+ }
+ };
+
+ auto ParseBuildPartIds = [&]() -> std::vector<Oid> {
+ std::vector<Oid> BuildPartIds;
+ for (const std::string& BuildPartId : m_BuildPartIds)
+ {
+ BuildPartIds.push_back(Oid::TryFromHexString(BuildPartId));
+ if (BuildPartIds.back() == Oid::Zero)
+ {
+ throw zen::OptionParseException(fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, m_DownloadOptions.help()));
+ }
+ }
+ return BuildPartIds;
+ };
+
+ auto ParseBuildMetadata = [&]() -> CbObject {
+ if (m_CreateBuild)
+ {
+ if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty())
+ {
+ throw zen::OptionParseException(fmt::format("Options for builds target are missing\n{}", m_UploadOptions.help()));
+ }
+ if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty())
+ {
+ throw zen::OptionParseException(fmt::format("Conflicting options for builds target\n{}", m_UploadOptions.help()));
+ }
+
+ if (!m_BuildMetadataPath.empty())
+ {
+ MakeSafeAbsolutePathÍnPlace(m_BuildMetadataPath);
+ IoBuffer MetaDataJson = ReadFile(m_BuildMetadataPath).Flatten();
+ std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
+ std::string JsonError;
+ CbObject MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
+ if (!JsonError.empty())
+ {
+ throw std::runtime_error(
+ fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_BuildMetadataPath, JsonError));
+ }
+ return MetaData;
+ }
+ if (!m_BuildMetadata.empty())
+ {
+ CbObjectWriter MetaDataWriter(1024);
+ ForEachStrTok(m_BuildMetadata, ';', [&](std::string_view Pair) {
+ size_t SplitPos = Pair.find('=');
+ if (SplitPos == std::string::npos || SplitPos == 0)
+ {
+ throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair));
+ }
+ MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1));
+ return true;
+ });
+ return MetaDataWriter.Save();
+ }
+ }
+ else
+ {
+ if (!m_BuildMetadataPath.empty())
+ {
+ throw zen::OptionParseException(
+ fmt::format("metadata-path option is only valid if creating a build\n{}", m_UploadOptions.help()));
+ }
+ if (!m_BuildMetadata.empty())
+ {
+ throw zen::OptionParseException(
+ fmt::format("metadata option is only valid if creating a build\n{}", m_UploadOptions.help()));
+ }
+ }
+ return {};
+ };
+
BoostWorkerThreads = m_BoostWorkerThreads;
try
@@ -9671,6 +9777,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (SubOption == &m_ListOptions)
{
+ MakeSafeAbsolutePathÍnPlace(m_ListQueryPath);
+ MakeSafeAbsolutePathÍnPlace(m_ListResultPath);
+
if (!m_ListResultPath.empty())
{
ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
@@ -9685,14 +9794,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- std::filesystem::path ListQueryPath = MakeSafeAbsolutePath(m_ListQueryPath);
- if (ToLower(ListQueryPath.extension().string()) == ".cbo")
+ if (ToLower(m_ListQueryPath.extension().string()) == ".cbo")
{
- QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(ListQueryPath));
+ QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_ListQueryPath));
}
else
{
- IoBuffer MetaDataJson = ReadFile(ListQueryPath).Flatten();
+ IoBuffer MetaDataJson = ReadFile(m_ListQueryPath).Flatten();
std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
std::string JsonError;
QueryObject = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
@@ -9707,19 +9815,22 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
BuildStorage::Statistics StorageStats;
BuildStorageCache::Statistics StorageCacheStats;
- const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty()
- ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName
- : MakeSafeAbsolutePath(m_ZenFolderPath);
- CreateDirectories(ZenFolderPath);
- auto _ = MakeGuard([ZenFolderPath]() {
- if (CleanDirectory(ZenFolderPath, {}))
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+ CreateDirectories(m_ZenFolderPath);
+ auto _ = MakeGuard([this]() {
+ if (CleanDirectory(m_ZenFolderPath, {}))
{
std::error_code DummyEc;
- RemoveDir(ZenFolderPath, DummyEc);
+ RemoveDir(m_ZenFolderPath, DummyEc);
}
});
- StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath));
+ StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath));
CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject);
ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None);
@@ -9731,17 +9842,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath);
- if (ToLower(ListResultPath.extension().string()) == ".cbo")
+ if (ToLower(m_ListResultPath.extension().string()) == ".cbo")
{
MemoryView ResponseView = Response.GetView();
- WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
+ WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize()));
}
else
{
ExtendableStringBuilder<1024> SB;
CompactBinaryToJson(Response.GetView(), SB);
- WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
+ WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
}
}
@@ -9752,130 +9862,53 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
- if (m_Path.empty())
- {
- throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help()));
- }
-
- if (m_CreateBuild)
- {
- if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty())
- {
- throw zen::OptionParseException(fmt::format("Options for builds target are missing\n{}", m_UploadOptions.help()));
- }
- if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty())
- {
- throw zen::OptionParseException(fmt::format("Conflicting options for builds target\n{}", m_UploadOptions.help()));
- }
- }
- else
- {
- if (!m_BuildMetadataPath.empty())
- {
- throw zen::OptionParseException(
- fmt::format("metadata-path option is only valid if creating a build\n{}", m_UploadOptions.help()));
- }
- if (!m_BuildMetadata.empty())
- {
- throw zen::OptionParseException(
- fmt::format("metadata option is only valid if creating a build\n{}", m_UploadOptions.help()));
- }
- }
-
- std::filesystem::path Path = MakeSafeAbsolutePath(m_Path);
+ ParsePath();
if (m_BuildPartName.empty())
{
- m_BuildPartName = Path.filename().string();
- }
-
- const bool GeneratedBuildId = m_BuildId.empty();
- if (GeneratedBuildId)
- {
- m_BuildId = Oid::NewOid().ToString();
- }
- else if (m_BuildId.length() != Oid::StringLength)
- {
- throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help()));
- }
- else if (Oid::FromHexString(m_BuildId) == Oid::Zero)
- {
- throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help()));
+ m_BuildPartName = m_Path.filename().string();
}
- const bool GeneratedBuildPartId = m_BuildPartId.empty();
- if (GeneratedBuildPartId)
- {
- m_BuildPartId = Oid::NewOid().ToString();
- }
- else if (m_BuildPartId.length() != Oid::StringLength)
+ const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId();
+ if (m_BuildId.empty())
{
- throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help()));
+ m_BuildId = BuildId.ToString();
}
- else if (Oid::FromHexString(m_BuildPartId) == Oid::Zero)
+ const Oid BuildPartId = m_BuildPartId.empty() ? Oid::NewOid() : ParseBuildPartId();
+ if (m_BuildPartId.empty())
{
- throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help()));
+ m_BuildPartId = BuildPartId.ToString();
}
- const Oid BuildId = Oid::FromHexString(m_BuildId);
- const Oid BuildPartId = Oid::FromHexString(m_BuildPartId);
-
BuildStorage::Statistics StorageStats;
BuildStorageCache::Statistics StorageCacheStats;
- const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty()
- ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName
- : MakeSafeAbsolutePath(m_ZenFolderPath);
- CreateDirectories(ZenFolderPath);
- auto _ = MakeGuard([ZenFolderPath]() {
- if (CleanDirectory(ZenFolderPath, {}))
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+ CreateDirectories(m_ZenFolderPath);
+ auto _ = MakeGuard([this]() {
+ if (CleanDirectory(m_ZenFolderPath, {}))
{
std::error_code DummyEc;
- RemoveDir(ZenFolderPath, DummyEc);
+ RemoveDir(m_ZenFolderPath, DummyEc);
}
});
- StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath));
+ StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath));
- CbObject MetaData;
- if (m_CreateBuild)
- {
- if (!m_BuildMetadataPath.empty())
- {
- std::filesystem::path MetadataPath = MakeSafeAbsolutePath(m_BuildMetadataPath);
- IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten();
- std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize());
- std::string JsonError;
- MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject();
- if (!JsonError.empty())
- {
- throw std::runtime_error(
- fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_BuildMetadataPath, JsonError));
- }
- }
- if (!m_BuildMetadata.empty())
- {
- CbObjectWriter MetaDataWriter(1024);
- ForEachStrTok(m_BuildMetadata, ';', [&](std::string_view Pair) {
- size_t SplitPos = Pair.find('=');
- if (SplitPos == std::string::npos || SplitPos == 0)
- {
- throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair));
- }
- MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1));
- return true;
- });
- MetaData = MetaDataWriter.Save();
- }
- }
+ CbObject MetaData = ParseBuildMetadata();
UploadFolder(Storage,
BuildId,
BuildPartId,
m_BuildPartName,
- Path,
- ZenFolderPath,
- m_ManifestPath.empty() ? std::filesystem::path{} : MakeSafeAbsolutePath(m_ManifestPath),
+ m_Path,
+ m_ZenFolderPath,
+ m_ManifestPath,
m_FindBlockMaxCount,
m_BlockReuseMinPercentLimit,
m_AllowMultiparts,
@@ -9911,26 +9944,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
- ParseSystemOptions();
-
- if (m_Path.empty())
- {
- throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help()));
- }
- if (m_BuildId.empty())
- {
- throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help()));
- }
- Oid BuildId = Oid::TryFromHexString(m_BuildId);
- if (BuildId == Oid::Zero)
- {
- throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help()));
- }
+ ParsePath();
- if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
- {
- throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help()));
- }
+ const Oid BuildId = ParseBuildId();
if (m_PostDownloadVerify && m_PrimeCacheOnly)
{
@@ -9948,34 +9964,26 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZEN_WARN("ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled");
}
- std::vector<Oid> BuildPartIds;
- for (const std::string& BuildPartId : m_BuildPartIds)
+ std::vector<Oid> BuildPartIds = ParseBuildPartIds();
+
+ if (m_ZenFolderPath.empty())
{
- BuildPartIds.push_back(Oid::TryFromHexString(BuildPartId));
- if (BuildPartIds.back() == Oid::Zero)
- {
- throw zen::OptionParseException(
- fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, m_DownloadOptions.help()));
- }
+ m_ZenFolderPath = m_Path / ZenFolderName;
}
-
- std::filesystem::path Path = MakeSafeAbsolutePath(m_Path);
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
BuildStorage::Statistics StorageStats;
BuildStorageCache::Statistics StorageCacheStats;
- const std::filesystem::path ZenFolderPath =
- m_ZenFolderPath.empty() ? Path / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath);
-
- StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath));
+ StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath));
DownloadFolder(Storage,
BuildId,
BuildPartIds,
m_BuildPartNames,
- Path,
- ZenFolderPath,
- SystemRootDir,
+ m_Path,
+ m_ZenFolderPath,
+ m_SystemRootDir,
m_AllowMultiparts,
m_AllowPartialBlockRequests && !m_PrimeCacheOnly,
m_Clean,
@@ -9983,79 +9991,43 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
m_PrimeCacheOnly,
m_EnableScavenging);
- if (false)
- {
- ZEN_CONSOLE(
- "{}:\n"
- "Read: {}\n"
- "Write: {}\n"
- "Requests: {}\n"
- "Avg Request Time: {}\n"
- "Avg I/O Time: {}",
- Storage.StorageName,
- NiceBytes(StorageStats.TotalBytesRead.load()),
- NiceBytes(StorageStats.TotalBytesWritten.load()),
- StorageStats.TotalRequestCount.load(),
- StorageStats.TotalExecutionTimeUs.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0,
- StorageStats.TotalRequestCount.load() > 0
- ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load())
- : 0);
- }
-
return AbortFlag ? 11 : 0;
}
if (SubOption == &m_DiffOptions)
{
- if (m_Path.empty())
- {
- throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help()));
- }
- if (m_DiffPath.empty())
- {
- throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help()));
- }
- std::filesystem::path Path = MakeSafeAbsolutePath(m_Path);
- std::filesystem::path DiffPath = MakeSafeAbsolutePath(m_DiffPath);
- DiffFolders(Path, DiffPath, m_OnlyChunked);
+ ParsePath();
+ ParseDiffPath();
+
+ DiffFolders(m_Path, m_DiffPath, m_OnlyChunked);
return AbortFlag ? 11 : 0;
}
if (SubOption == &m_FetchBlobOptions)
{
ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
- if (m_BlobHash.empty())
- {
- throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help()));
- }
-
- IoHash BlobHash;
- if (!IoHash::TryParse(m_BlobHash, BlobHash))
- {
- throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help()));
- }
+ IoHash BlobHash = ParseBlobHash();
const Oid BuildId = Oid::FromHexString(m_BuildId);
- std::filesystem::path Path = MakeSafeAbsolutePath(m_Path);
-
BuildStorage::Statistics StorageStats;
BuildStorageCache::Statistics StorageCacheStats;
- const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty()
- ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName
- : MakeSafeAbsolutePath(m_ZenFolderPath);
- CreateDirectories(ZenFolderPath);
- auto _ = MakeGuard([ZenFolderPath]() {
- if (CleanDirectory(ZenFolderPath, {}))
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+ CreateDirectories(m_ZenFolderPath);
+ auto _ = MakeGuard([this]() {
+ if (CleanDirectory(m_ZenFolderPath, {}))
{
std::error_code DummyEc;
- RemoveDir(ZenFolderPath, DummyEc);
+ RemoveDir(m_ZenFolderPath, DummyEc);
}
});
- StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath));
+ StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath));
uint64_t CompressedSize;
uint64_t DecompressedSize;
@@ -10075,41 +10047,34 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL);
- if (m_BuildId.empty())
- {
- throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help()));
- }
- Oid BuildId = Oid::TryFromHexString(m_BuildId);
- if (BuildId == Oid::Zero)
- {
- throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help()));
- }
+ Oid BuildId = ParseBuildId();
if (!m_BuildPartName.empty() && !m_BuildPartId.empty())
{
throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help()));
}
- std::filesystem::path Path = MakeSafeAbsolutePath(m_Path);
-
BuildStorage::Statistics StorageStats;
BuildStorageCache::Statistics StorageCacheStats;
- const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty()
- ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName
- : MakeSafeAbsolutePath(m_ZenFolderPath);
- CreateDirectories(ZenFolderPath);
- auto _ = MakeGuard([ZenFolderPath]() {
- if (CleanDirectory(ZenFolderPath, {}))
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
+
+ CreateDirectories(m_ZenFolderPath);
+ auto _ = MakeGuard([this]() {
+ if (CleanDirectory(m_ZenFolderPath, {}))
{
std::error_code DummyEc;
- RemoveDir(ZenFolderPath, DummyEc);
+ RemoveDir(m_ZenFolderPath, DummyEc);
}
});
- StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath));
+ StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath));
- Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId);
+ const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId();
ValidateStatistics ValidateStats;
DownloadStatistics DownloadStats;
@@ -10120,35 +10085,23 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (SubOption == &m_MultiTestDownloadOptions)
{
- SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
- CreateDirectories(SystemRootDir);
- CleanDirectory(SystemRootDir, {});
- auto _ = MakeGuard([&]() { DeleteDirectories(SystemRootDir); });
+ m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
+ CreateDirectories(m_SystemRootDir);
+ CleanDirectory(m_SystemRootDir, {});
+ auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); });
+
+ ParsePath();
- if (m_Path.empty())
+ if (m_ZenFolderPath.empty())
{
- throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help()));
+ m_ZenFolderPath = m_Path / ZenFolderName;
}
-
- // m_StoragePath = "D:\\buildstorage";
- // m_Path = "F:\\Saved\\DownloadedBuilds\\++Fortnite+Main-CL-XXXXXXXX\\WindowsClient";
- // std::vector<std::string> BuildIdStrings{"07d3942f0e7f4ca1b13b0587",
- // "07d394eed89d769f2254e75d",
- // "07d3953f22fa3f8000fa6f0a",
- // "07d3959df47ed1f42ddbe44c",
- // "07d395fa7803d50804f14417",
- // "07d3964f919d577a321a1fdd",
- // "07d396a6ce875004e16b9528"};
-
- std::filesystem::path Path = MakeSafeAbsolutePath(m_Path);
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
BuildStorage::Statistics StorageStats;
BuildStorageCache::Statistics StorageCacheStats;
- const std::filesystem::path ZenFolderPath =
- m_ZenFolderPath.empty() ? Path / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath);
-
- StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath));
+ StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath));
Stopwatch Timer;
for (const std::string& BuildIdString : m_BuildIds)
@@ -10162,9 +10115,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
BuildId,
{},
{},
- Path,
- ZenFolderPath,
- SystemRootDir,
+ m_Path,
+ m_ZenFolderPath,
+ m_SystemRootDir,
m_AllowMultiparts,
m_AllowPartialBlockRequests,
BuildIdString == m_BuildIds.front(),
@@ -10184,47 +10137,41 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (SubOption == &m_TestOptions)
{
- SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
- CreateDirectories(SystemRootDir);
- CleanDirectory(SystemRootDir, {});
- auto _ = MakeGuard([&]() { DeleteDirectories(SystemRootDir); });
-
- if (m_Path.empty())
- {
- throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help()));
- }
+ m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred();
+ CreateDirectories(m_SystemRootDir);
+ CleanDirectory(m_SystemRootDir, {});
+ auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); });
- std::filesystem::path Path = MakeSafeAbsolutePath(m_Path);
+ ParsePath();
m_BuildId = Oid::NewOid().ToString();
- m_BuildPartName = Path.filename().string();
+ m_BuildPartName = m_Path.filename().string();
m_BuildPartId = Oid::NewOid().ToString();
m_CreateBuild = true;
const Oid BuildId = Oid::FromHexString(m_BuildId);
const Oid BuildPartId = Oid::FromHexString(m_BuildPartId);
- std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath);
-
- if (m_OverrideHost.empty() && StoragePath.empty())
+ if (m_OverrideHost.empty() && m_StoragePath.empty())
{
- StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
- CreateDirectories(StoragePath);
- CleanDirectory(StoragePath, {});
- m_StoragePath = StoragePath.generic_string();
+ m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred();
+ CreateDirectories(m_StoragePath);
+ CleanDirectory(m_StoragePath, {});
+ m_StoragePath = m_StoragePath.generic_string();
}
+
auto __ = MakeGuard([&]() {
- if (m_OverrideHost.empty() && StoragePath.empty())
+ if (m_OverrideHost.empty() && m_StoragePath.empty())
{
- DeleteDirectories(StoragePath);
+ DeleteDirectories(m_StoragePath);
}
});
BuildStorage::Statistics StorageStats;
BuildStorageCache::Statistics StorageCacheStats;
- const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_test");
- const std::filesystem::path DownloadPath2 = Path.parent_path() / (m_BuildPartName + "_test2");
+ const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_test");
+ const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2");
auto ___ = MakeGuard([DownloadPath, DownloadPath2]() {
CleanDirectory(DownloadPath, true);
@@ -10233,10 +10180,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
DeleteDirectories(DownloadPath2);
});
- const std::filesystem::path ZenFolderPath =
- m_ZenFolderPath.empty() ? DownloadPath / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath);
+ if (m_ZenFolderPath.empty())
+ {
+ m_ZenFolderPath = m_Path / ZenFolderName;
+ }
+ MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath);
- StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath));
+ StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath));
auto MakeMetaData = [](const Oid& BuildId) -> CbObject {
CbObjectWriter BuildMetaDataWriter;
@@ -10263,8 +10213,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
BuildId,
BuildPartId,
m_BuildPartName,
- Path,
- ZenFolderPath,
+ m_Path,
+ m_ZenFolderPath,
{},
m_FindBlockMaxCount,
m_BlockReuseMinPercentLimit,
@@ -10285,8 +10235,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{BuildPartId},
{},
DownloadPath,
- ZenFolderPath,
- SystemRootDir,
+ DownloadPath / ZenFolderName,
+ m_SystemRootDir,
m_AllowMultiparts,
m_AllowPartialBlockRequests,
true,
@@ -10309,8 +10259,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{BuildPartId},
{},
DownloadPath,
- ZenFolderPath,
- SystemRootDir,
+ DownloadPath / ZenFolderName,
+ m_SystemRootDir,
m_AllowMultiparts,
m_AllowPartialBlockRequests,
false,
@@ -10428,8 +10378,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{BuildPartId},
{},
DownloadPath,
- ZenFolderPath,
- SystemRootDir,
+ DownloadPath / ZenFolderName,
+ m_SystemRootDir,
m_AllowMultiparts,
m_AllowPartialBlockRequests,
false,
@@ -10459,7 +10409,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
BuildPartId2,
m_BuildPartName,
DownloadPath,
- ZenFolderPath,
+ m_ZenFolderPath,
{},
m_FindBlockMaxCount,
m_BlockReuseMinPercentLimit,
@@ -10480,8 +10430,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{BuildPartId},
{},
DownloadPath,
- ZenFolderPath,
- SystemRootDir,
+ DownloadPath / ZenFolderName,
+ m_SystemRootDir,
m_AllowMultiparts,
m_AllowPartialBlockRequests,
false,
@@ -10500,8 +10450,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{BuildPartId2},
{},
DownloadPath,
- ZenFolderPath,
- SystemRootDir,
+ DownloadPath / ZenFolderName,
+ m_SystemRootDir,
m_AllowMultiparts,
m_AllowPartialBlockRequests,
false,
@@ -10520,8 +10470,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{BuildPartId2},
{},
DownloadPath,
- ZenFolderPath,
- SystemRootDir,
+ DownloadPath / ZenFolderName,
+ m_SystemRootDir,
m_AllowMultiparts,
m_AllowPartialBlockRequests,
false,
@@ -10540,8 +10490,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{BuildPartId},
{},
DownloadPath2,
- ZenFolderPath,
- SystemRootDir,
+ DownloadPath2 / ZenFolderName,
+ m_SystemRootDir,
m_AllowMultiparts,
m_AllowPartialBlockRequests,
false,
diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h
index f44e758ae..01d510e1b 100644
--- a/src/zen/cmds/builds_cmd.h
+++ b/src/zen/cmds/builds_cmd.h
@@ -25,13 +25,13 @@ public:
private:
cxxopts::Options m_Options{Name, Description};
- std::string m_SystemRootDir;
+ std::filesystem::path m_SystemRootDir;
bool m_PlainProgress = false;
bool m_Verbose = false;
bool m_BoostWorkerThreads = false;
- std::string m_ZenFolderPath;
+ std::filesystem::path m_ZenFolderPath;
// cloud builds
std::string m_OverrideHost;
@@ -41,29 +41,29 @@ private:
std::string m_Bucket;
// file storage
- std::string m_StoragePath;
- bool m_WriteMetadataAsJson = false;
+ std::filesystem::path m_StoragePath;
+ bool m_WriteMetadataAsJson = false;
// cache
std::string m_ZenCacheHost;
bool m_PrimeCacheOnly = false;
- std::string m_BuildId;
- bool m_CreateBuild = false;
- std::string m_BuildMetadataPath;
- std::string m_BuildMetadata;
- std::string m_BuildPartName; // Defaults to name of leaf folder in m_Path
- std::string m_BuildPartId; // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName
- bool m_Clean = false;
- uint8_t m_BlockReuseMinPercentLimit = 85;
- bool m_AllowMultiparts = true;
- bool m_AllowPartialBlockRequests = true;
- std::string m_ManifestPath;
+ std::string m_BuildId;
+ bool m_CreateBuild = false;
+ std::filesystem::path m_BuildMetadataPath;
+ std::string m_BuildMetadata;
+ std::string m_BuildPartName; // Defaults to name of leaf folder in m_Path
+ std::string m_BuildPartId; // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName
+ bool m_Clean = false;
+ uint8_t m_BlockReuseMinPercentLimit = 85;
+ bool m_AllowMultiparts = true;
+ bool m_AllowPartialBlockRequests = true;
+ std::string m_ManifestPath; // Not a std::filesystem::path since it can be relative to m_Path
// Direct access token (may expire)
- std::string m_AccessToken;
- std::string m_AccessTokenEnv;
- std::string m_AccessTokenPath;
+ std::string m_AccessToken;
+ std::string m_AccessTokenEnv;
+ std::filesystem::path m_AccessTokenPath;
// Auth manager token encryption
std::string m_EncryptionKey; // 256 bit AES encryption key
@@ -88,11 +88,11 @@ private:
std::string m_ListNamespacesResultPath;
bool m_ListNamespacesRecursive = false;
- cxxopts::Options m_ListOptions{"list", "List available builds"};
- std::string m_ListQueryPath;
- std::string m_ListResultPath;
+ cxxopts::Options m_ListOptions{"list", "List available builds"};
+ std::filesystem::path m_ListQueryPath;
+ std::filesystem::path m_ListResultPath;
- std::string m_Path;
+ std::filesystem::path m_Path;
cxxopts::Options m_UploadOptions{"upload", "Upload a folder"};
uint64_t m_FindBlockMaxCount = 10000;
@@ -104,9 +104,9 @@ private:
bool m_PostDownloadVerify = false;
bool m_EnableScavenging = true;
- cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"};
- std::string m_DiffPath;
- bool m_OnlyChunked = false;
+ cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"};
+ std::filesystem::path m_DiffPath;
+ bool m_OnlyChunked = false;
cxxopts::Options m_FetchBlobOptions{"fetch-blob", "Fetch a blob from remote store"};
std::string m_BlobHash;
diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp
index 185edc35d..4412eaf34 100644
--- a/src/zen/cmds/cache_cmd.cpp
+++ b/src/zen/cmds/cache_cmd.cpp
@@ -625,22 +625,20 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
HttpClient Http(m_HostName);
- std::filesystem::path TargetPath;
if (!m_OutputPath.empty())
{
- TargetPath = std::filesystem::path(m_OutputPath);
- if (IsDir(TargetPath))
+ if (IsDir(m_OutputPath))
{
- TargetPath = TargetPath / (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash);
+ m_OutputPath = m_OutputPath / (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash);
}
else
{
- CreateDirectories(TargetPath.parent_path());
+ CreateDirectories(m_OutputPath.parent_path());
}
}
- if (TargetPath.empty())
+ if (m_OutputPath.empty())
{
- TargetPath = (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash);
+ m_OutputPath = (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash);
}
std::string Url = fmt::format("/z$/{}/{}/{}", m_Namespace, m_Bucket, m_ValueKey);
@@ -670,17 +668,17 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
else
{
- WriteFile(TargetPath, IoBuffer(IoBuffer::Wrap, StringData.data(), StringData.length()));
- ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(StringData.length()), TargetPath, ToString(ChunkData.GetContentType()));
+ WriteFile(m_OutputPath, IoBuffer(IoBuffer::Wrap, StringData.data(), StringData.length()));
+ ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(StringData.length()), m_OutputPath, ToString(ChunkData.GetContentType()));
}
}
else
{
- if (!MoveToFile(TargetPath, ChunkData))
+ if (!MoveToFile(m_OutputPath, ChunkData))
{
- WriteFile(TargetPath, ChunkData);
+ WriteFile(m_OutputPath, ChunkData);
}
- ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(ChunkData.GetSize()), TargetPath, ToString(ChunkData.GetContentType()));
+ ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(ChunkData.GetSize()), m_OutputPath, ToString(ChunkData.GetContentType()));
}
}
else
diff --git a/src/zen/cmds/cache_cmd.h b/src/zen/cmds/cache_cmd.h
index 73702cada..b8a319359 100644
--- a/src/zen/cmds/cache_cmd.h
+++ b/src/zen/cmds/cache_cmd.h
@@ -108,15 +108,15 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{Name, Description};
- std::string m_HostName;
- std::string m_Namespace;
- std::string m_Bucket;
- std::string m_ValueKey;
- std::string m_AttachmentHash;
- std::string m_OutputPath;
- bool m_AsText = false;
- bool m_Decompress = true;
+ cxxopts::Options m_Options{Name, Description};
+ std::string m_HostName;
+ std::string m_Namespace;
+ std::string m_Bucket;
+ std::string m_ValueKey;
+ std::string m_AttachmentHash;
+ std::filesystem::path m_OutputPath;
+ bool m_AsText = false;
+ bool m_Decompress = true;
};
} // namespace zen
diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp
index 53e80c896..e86b6964c 100644
--- a/src/zen/cmds/copy_cmd.cpp
+++ b/src/zen/cmds/copy_cmd.cpp
@@ -42,11 +42,8 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (m_CopyTarget.empty())
throw std::runtime_error("No target specified");
- std::filesystem::path FromPath;
- std::filesystem::path ToPath;
-
- FromPath = m_CopySource;
- ToPath = m_CopyTarget;
+ std::filesystem::path FromPath = m_CopySource;
+ std::filesystem::path ToPath = m_CopyTarget;
std::error_code Ec;
std::filesystem::path FromCanonical = std::filesystem::canonical(FromPath, Ec);
diff --git a/src/zen/cmds/copy_cmd.h b/src/zen/cmds/copy_cmd.h
index 876aff3f5..e9735c159 100644
--- a/src/zen/cmds/copy_cmd.h
+++ b/src/zen/cmds/copy_cmd.h
@@ -19,11 +19,11 @@ public:
virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
private:
- cxxopts::Options m_Options{"copy", "Copy files efficiently"};
- std::string m_CopySource;
- std::string m_CopyTarget;
- bool m_NoClone = false;
- bool m_MustClone = false;
+ cxxopts::Options m_Options{"copy", "Copy files efficiently"};
+ std::filesystem::path m_CopySource;
+ std::filesystem::path m_CopyTarget;
+ bool m_NoClone = false;
+ bool m_MustClone = false;
};
} // namespace zen
diff --git a/src/zen/cmds/dedup_cmd.h b/src/zen/cmds/dedup_cmd.h
index c4f0068e4..2721be2b9 100644
--- a/src/zen/cmds/dedup_cmd.h
+++ b/src/zen/cmds/dedup_cmd.h
@@ -21,8 +21,8 @@ public:
private:
cxxopts::Options m_Options{"dedup", "Deduplicate files"};
std::vector<std::string> m_Positional;
- std::string m_DedupSource;
- std::string m_DedupTarget;
+ std::filesystem::path m_DedupSource;
+ std::filesystem::path m_DedupTarget;
size_t m_SizeThreshold = 1024 * 1024;
};
diff --git a/src/zen/cmds/status_cmd.cpp b/src/zen/cmds/status_cmd.cpp
index 88c0b22a2..2b507e43d 100644
--- a/src/zen/cmds/status_cmd.cpp
+++ b/src/zen/cmds/status_cmd.cpp
@@ -32,17 +32,16 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
uint16_t EffectivePort = 0;
if (!m_DataDir.empty())
{
- std::filesystem::path DataDir = StringToPath(m_DataDir);
- if (!IsFile(DataDir / ".lock"))
+ if (!IsFile(m_DataDir / ".lock"))
{
- ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir);
+ ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir);
return 1;
}
- LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock")));
+ LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock")));
std::string Reason;
if (!ValidateLockFileInfo(Info, Reason))
{
- ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason);
+ ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
return 1;
}
EffectivePort = Info.EffectiveListenPort;
diff --git a/src/zen/cmds/status_cmd.h b/src/zen/cmds/status_cmd.h
index 00ad0e758..46bda9ee6 100644
--- a/src/zen/cmds/status_cmd.h
+++ b/src/zen/cmds/status_cmd.h
@@ -20,9 +20,9 @@ public:
private:
int GetLockFileEffectivePort() const;
- cxxopts::Options m_Options{"status", "Show zen status"};
- uint16_t m_Port = 0;
- std::string m_DataDir;
+ cxxopts::Options m_Options{"status", "Show zen status"};
+ uint16_t m_Port = 0;
+ std::filesystem::path m_DataDir;
};
} // namespace zen
diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp
index aacc115a0..f3bf2f66d 100644
--- a/src/zen/cmds/up_cmd.cpp
+++ b/src/zen/cmds/up_cmd.cpp
@@ -77,15 +77,13 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
}
}
- std::filesystem::path ProgramBaseDir = StringToPath(m_ProgramBaseDir);
-
- if (ProgramBaseDir.empty())
+ if (m_ProgramBaseDir.empty())
{
std::filesystem::path ExePath = zen::GetRunningExecutablePath();
- ProgramBaseDir = ExePath.parent_path();
+ m_ProgramBaseDir = ExePath.parent_path();
}
ZenServerEnvironment ServerEnvironment;
- ServerEnvironment.Initialize(ProgramBaseDir);
+ ServerEnvironment.Initialize(m_ProgramBaseDir);
ZenServerInstance Server(ServerEnvironment);
std::string ServerArguments = GlobalOptions.PassthroughCommandLine;
if ((m_Port != 0) && (ServerArguments.find("--port"sv) == std::string::npos))
@@ -155,20 +153,18 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
Instance.Sweep();
ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port);
- std::filesystem::path DataDir = StringToPath(m_DataDir);
-
- if (!DataDir.empty())
+ if (!m_DataDir.empty())
{
- if (!IsFile(DataDir / ".lock"))
+ if (!IsFile(m_DataDir / ".lock"))
{
- ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir);
+ ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir);
return 1;
}
- LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock")));
+ LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock")));
std::string Reason;
if (!ValidateLockFileInfo(Info, Reason))
{
- ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason);
+ ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
return 1;
}
Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort);
@@ -218,27 +214,24 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
Instance.Initialize();
ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port);
- std::filesystem::path ProgramBaseDir = StringToPath(m_ProgramBaseDir);
- if (ProgramBaseDir.empty())
+ if (m_ProgramBaseDir.empty())
{
std::filesystem::path ExePath = GetRunningExecutablePath();
- ProgramBaseDir = ExePath.parent_path();
+ m_ProgramBaseDir = ExePath.parent_path();
}
- std::filesystem::path DataDir = StringToPath(m_DataDir);
-
- if (!DataDir.empty())
+ if (!m_DataDir.empty())
{
- if (!IsFile(DataDir / ".lock"))
+ if (!IsFile(m_DataDir / ".lock"))
{
- ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir);
+ ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir);
return 1;
}
- LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock")));
+ LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock")));
std::string Reason;
if (!ValidateLockFileInfo(Info, Reason))
{
- ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason);
+ ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
return 1;
}
Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort);
@@ -251,7 +244,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
try
{
ZenServerEnvironment ServerEnvironment;
- ServerEnvironment.Initialize(ProgramBaseDir);
+ ServerEnvironment.Initialize(m_ProgramBaseDir);
ZenServerInstance Server(ServerEnvironment);
Server.AttachToRunningServer(EntryPort);
@@ -316,7 +309,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
if (m_ForceTerminate)
{
// Try to find the running executable by path name
- std::filesystem::path ServerExePath = ProgramBaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
+ std::filesystem::path ServerExePath = m_ProgramBaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL;
ProcessHandle RunningProcess;
if (std::error_code Ec = FindProcess(ServerExePath, RunningProcess); !Ec)
{
diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h
index 32d8ddab3..c9af16749 100644
--- a/src/zen/cmds/up_cmd.h
+++ b/src/zen/cmds/up_cmd.h
@@ -18,11 +18,11 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"up", "Bring up zen service"};
- uint16_t m_Port = 0;
- bool m_ShowConsole = false;
- bool m_ShowLog = false;
- std::string m_ProgramBaseDir;
+ cxxopts::Options m_Options{"up", "Bring up zen service"};
+ uint16_t m_Port = 0;
+ bool m_ShowConsole = false;
+ bool m_ShowLog = false;
+ std::filesystem::path m_ProgramBaseDir;
};
class AttachCommand : public ZenCmdBase
@@ -35,10 +35,10 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"};
- uint16_t m_Port = 0;
- int m_OwnerPid = 0;
- std::string m_DataDir;
+ cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"};
+ uint16_t m_Port = 0;
+ int m_OwnerPid = 0;
+ std::filesystem::path m_DataDir;
};
class DownCommand : public ZenCmdBase
@@ -51,11 +51,11 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{"down", "Bring down zen service"};
- uint16_t m_Port = 0;
- bool m_ForceTerminate = false;
- std::string m_ProgramBaseDir;
- std::string m_DataDir;
+ cxxopts::Options m_Options{"down", "Bring down zen service"};
+ uint16_t m_Port = 0;
+ bool m_ForceTerminate = false;
+ std::filesystem::path m_ProgramBaseDir;
+ std::filesystem::path m_DataDir;
};
} // namespace zen
diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp
new file mode 100644
index 000000000..2b4e9ab3c
--- /dev/null
+++ b/src/zen/cmds/wipe_cmd.cpp
@@ -0,0 +1,575 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "wipe_cmd.h"
+
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/string.h>
+#include <zencore/timer.h>
+#include <zencore/trace.h>
+#include <zenutil/parallellwork.h>
+#include <zenutil/workerpools.h>
+
+#include <signal.h>
+
+#include <iostream>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+#include <tsl/robin_set.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#if ZEN_PLATFORM_WINDOWS
+# include <zencore/windows.h>
+#else
+# include <fcntl.h>
+# include <sys/file.h>
+# include <sys/stat.h>
+# include <unistd.h>
+#endif
+
+namespace zen {
+
+namespace {
+ static std::atomic<bool> AbortFlag = false;
+ static bool IsVerbose = false;
+ static bool Quiet = false;
+ static bool UsePlainProgress = false;
+ const bool SingleThreaded = false;
+ bool BoostWorkerThreads = true;
+
+ WorkerThreadPool& GetIOWorkerPool()
+ {
+ return SingleThreaded ? GetSyncWorkerPool()
+ : BoostWorkerThreads ? GetLargeWorkerPool(EWorkloadType::Burst)
+ : GetMediumWorkerPool(EWorkloadType::Burst);
+ }
+
+#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \
+ if (IsVerbose) \
+ { \
+ ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__); \
+ }
+
+ static void SignalCallbackHandler(int SigNum)
+ {
+ if (SigNum == SIGINT)
+ {
+ AbortFlag = true;
+ }
+#if ZEN_PLATFORM_WINDOWS
+ if (SigNum == SIGBREAK)
+ {
+ AbortFlag = true;
+ }
+#endif // ZEN_PLATFORM_WINDOWS
+ }
+
+ bool IsReadOnly(uint32_t Attributes)
+ {
+#if ZEN_PLATFORM_WINDOWS
+ return IsFileAttributeReadOnly(Attributes);
+#else
+ return IsFileModeReadOnly(Attributes);
+#endif
+ }
+
+ bool IsFileWithRetry(const std::filesystem::path& Path)
+ {
+ std::error_code Ec;
+ bool Result = IsFile(Path, Ec);
+ for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+ {
+ Sleep(100 + int(Retries * 50));
+ Ec.clear();
+ Result = IsFile(Path, Ec);
+ }
+ if (Ec)
+ {
+ zen::ThrowSystemError(Ec.value(), Ec.message());
+ }
+ return Result;
+ }
+
+ bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly)
+ {
+ std::error_code Ec;
+ bool Result = SetFileReadOnly(Path, ReadOnly, Ec);
+ for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+ {
+ Sleep(100 + int(Retries * 50));
+ if (!IsFileWithRetry(Path))
+ {
+ return false;
+ }
+ Ec.clear();
+ Result = SetFileReadOnly(Path, ReadOnly, Ec);
+ }
+ if (Ec)
+ {
+ zen::ThrowSystemError(Ec.value(), Ec.message());
+ }
+ return Result;
+ }
+
+ void RemoveFileWithRetry(const std::filesystem::path& Path)
+ {
+ std::error_code Ec;
+ RemoveFile(Path, Ec);
+ for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+ {
+ Sleep(100 + int(Retries * 50));
+ if (!IsFileWithRetry(Path))
+ {
+ return;
+ }
+ Ec.clear();
+ RemoveFile(Path, Ec);
+ }
+ if (Ec)
+ {
+ zen::ThrowSystemError(Ec.value(), Ec.message());
+ }
+ }
+
+ void RemoveDirWithRetry(const std::filesystem::path& Path)
+ {
+ std::error_code Ec;
+ RemoveDir(Path, Ec);
+ for (size_t Retries = 0; Ec && Retries < 3; Retries++)
+ {
+ Sleep(100 + int(Retries * 50));
+ if (!IsDir(Path))
+ {
+ return;
+ }
+ Ec.clear();
+ RemoveDir(Path, Ec);
+ }
+ if (Ec)
+ {
+ zen::ThrowSystemError(Ec.value(), Ec.message());
+ }
+ }
+
+ bool CleanDirectory(const std::filesystem::path& Path,
+ std::span<const std::string_view> ExcludeDirectories,
+ bool RemoveReadonly,
+ bool Dryrun)
+ {
+ ZEN_TRACE_CPU("CleanDirectory");
+ Stopwatch Timer;
+
+ ProgressBar Progress(UsePlainProgress);
+
+ std::atomic<bool> CleanWipe = true;
+ std::atomic<uint64_t> DiscoveredItemCount = 0;
+ std::atomic<uint64_t> DeletedItemCount = 0;
+ std::atomic<uint64_t> DeletedByteCount = 0;
+ std::atomic<uint64_t> FailedDeleteCount = 0;
+
+ std::vector<std::filesystem::path> SubdirectoriesToDelete;
+ tsl::robin_map<IoHash, size_t, IoHash::Hasher> SubdirectoriesToDeleteLookup;
+ tsl::robin_set<IoHash, IoHash::Hasher> SubdirectoriesToKeep;
+ RwLock SubdirectoriesLock;
+
+ auto AddFoundDirectory = [&](std::filesystem::path Directory, bool Keep) -> bool {
+ bool Added = false;
+ if (Keep)
+ {
+ bool IsLeaf = true;
+ while (Directory != Path)
+ {
+ const std::string DirectoryString = Directory.generic_string();
+ IoHash DirectoryNameHash = IoHash::HashBuffer(DirectoryString.data(), DirectoryString.length());
+ RwLock::ExclusiveLockScope _(SubdirectoriesLock);
+ if (auto It = SubdirectoriesToKeep.find(DirectoryNameHash); It == SubdirectoriesToKeep.end())
+ {
+ SubdirectoriesToKeep.insert(DirectoryNameHash);
+ if (IsLeaf)
+ {
+ Added = true;
+ }
+ }
+ else
+ {
+ break;
+ }
+ Directory = Directory.parent_path();
+ IsLeaf = false;
+ }
+ }
+ else
+ {
+ bool IsLeaf = true;
+ while (Directory != Path)
+ {
+ const std::string DirectoryString = Directory.generic_string();
+ IoHash DirectoryNameHash = IoHash::HashBuffer(DirectoryString.data(), DirectoryString.length());
+ RwLock::ExclusiveLockScope _(SubdirectoriesLock);
+ if (SubdirectoriesToKeep.contains(DirectoryNameHash))
+ {
+ break;
+ }
+ if (auto It = SubdirectoriesToDeleteLookup.find(DirectoryNameHash); It == SubdirectoriesToDeleteLookup.end())
+ {
+ SubdirectoriesToDeleteLookup.insert({DirectoryNameHash, SubdirectoriesToDelete.size()});
+ SubdirectoriesToDelete.push_back(Directory);
+ if (IsLeaf)
+ {
+ Added = true;
+ }
+ }
+ else
+ {
+ break;
+ }
+ Directory = Directory.parent_path();
+ IsLeaf = false;
+ }
+ }
+ return Added;
+ };
+
+ ParallellWork Work(AbortFlag);
+
+ struct AsyncVisitor : public GetDirectoryContentVisitor
+ {
+ AsyncVisitor(const std::filesystem::path& InPath,
+ std::atomic<bool>& InCleanWipe,
+ std::atomic<uint64_t>& InDiscoveredItemCount,
+ std::atomic<uint64_t>& InDeletedItemCount,
+ std::atomic<uint64_t>& InDeletedByteCount,
+ std::atomic<uint64_t>& InFailedDeleteCount,
+ std::span<const std::string_view> InExcludeDirectories,
+ bool InRemoveReadonly,
+ bool InDryrun,
+ const std::function<bool(std::filesystem::path, bool)>& InAddFoundDirectoryFunc)
+ : Path(InPath)
+ , CleanWipe(InCleanWipe)
+ , DiscoveredItemCount(InDiscoveredItemCount)
+ , DeletedItemCount(InDeletedItemCount)
+ , DeletedByteCount(InDeletedByteCount)
+ , FailedDeleteCount(InFailedDeleteCount)
+ , ExcludeDirectories(InExcludeDirectories)
+ , RemoveReadonly(InRemoveReadonly)
+ , Dryrun(InDryrun)
+ , AddFoundDirectoryFunc(InAddFoundDirectoryFunc)
+ {
+ }
+ virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override
+ {
+ ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory");
+ if (!AbortFlag)
+ {
+ if (!RelativeRoot.empty())
+ {
+ DiscoveredItemCount++;
+ }
+ if (!Content.FileNames.empty())
+ {
+ DiscoveredItemCount += Content.FileNames.size();
+
+ const std::string RelativeRootString = RelativeRoot.generic_string();
+ bool RemoveContent = true;
+ for (const std::string_view ExcludeDirectory : ExcludeDirectories)
+ {
+ if (RelativeRootString.starts_with(ExcludeDirectory))
+ {
+ if (RelativeRootString.length() > ExcludeDirectory.length())
+ {
+ const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()];
+ if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' ||
+ MaybePathDelimiter == std::filesystem::path::preferred_separator)
+ {
+ RemoveContent = false;
+ break;
+ }
+ }
+ else
+ {
+ RemoveContent = false;
+ break;
+ }
+ }
+ }
+
+ const std::filesystem::path ParentPath = Path / RelativeRoot;
+ bool KeepDirectory = RelativeRoot.empty();
+
+ if (RemoveContent)
+ {
+ ZEN_TRACE_CPU("DeleteFiles");
+ uint64_t RemovedCount = 0;
+ for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++)
+ {
+ const std::filesystem::path& FileName = Content.FileNames[FileIndex];
+ const std::filesystem::path FilePath = (ParentPath / FileName).make_preferred();
+ try
+ {
+ const uint32_t Attributes = Content.FileAttributes[FileIndex];
+ const bool IsReadonly = IsReadOnly(Attributes);
+ bool RemoveFile = false;
+ if (IsReadonly)
+ {
+ if (RemoveReadonly)
+ {
+ if (!Dryrun)
+ {
+ SetFileReadOnlyWithRetry(FilePath, false);
+ }
+ RemoveFile = true;
+ }
+ }
+ else
+ {
+ RemoveFile = true;
+ }
+
+ if (RemoveFile)
+ {
+ if (!Dryrun)
+ {
+ RemoveFileWithRetry(FilePath);
+ }
+ DeletedItemCount++;
+ DeletedByteCount += Content.FileSizes[FileIndex];
+ RemovedCount++;
+ ZEN_CONSOLE_VERBOSE("Removed file {}", FilePath);
+ }
+ else
+ {
+ ZEN_CONSOLE_VERBOSE("Skipped readonly file {}", FilePath);
+ KeepDirectory = true;
+ }
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_WARN("Failed removing file {}. Reason: {}", FilePath, Ex.what());
+ FailedDeleteCount++;
+ CleanWipe = false;
+ KeepDirectory = true;
+ }
+ }
+ ZEN_CONSOLE_VERBOSE("Removed {} files in {}", RemovedCount, ParentPath);
+ }
+ else
+ {
+ ZEN_CONSOLE_VERBOSE("Skipped removal of {} files in {}", Content.FileNames.size(), ParentPath);
+ }
+ bool Added = AddFoundDirectoryFunc(ParentPath, KeepDirectory);
+ if (Added)
+ {
+ ZEN_CONSOLE_VERBOSE("{} directory {}", KeepDirectory ? "Keeping" : "Removing", ParentPath);
+ }
+ }
+ }
+ }
+ const std::filesystem::path& Path;
+ std::atomic<bool>& CleanWipe;
+ std::atomic<uint64_t>& DiscoveredItemCount;
+ std::atomic<uint64_t>& DeletedItemCount;
+ std::atomic<uint64_t>& DeletedByteCount;
+ std::atomic<uint64_t>& FailedDeleteCount;
+ std::span<const std::string_view> ExcludeDirectories;
+ const bool RemoveReadonly;
+ const bool Dryrun;
+ std::function<bool(std::filesystem::path, bool)> AddFoundDirectoryFunc;
+ } Visitor(Path,
+ CleanWipe,
+ DiscoveredItemCount,
+ DeletedItemCount,
+ DeletedByteCount,
+ FailedDeleteCount,
+ ExcludeDirectories,
+ RemoveReadonly,
+ Dryrun,
+ AddFoundDirectory);
+
+ uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs();
+
+ GetDirectoryContent(Path,
+ DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive |
+ DirectoryContentFlags::IncludeFileSizes | DirectoryContentFlags::IncludeAttributes,
+ Visitor,
+ GetIOWorkerPool(),
+ Work.PendingWork());
+
+ Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, ptrdiff_t PendingWork) {
+ if (Quiet)
+ {
+ return;
+ }
+ ZEN_UNUSED(IsAborted, PendingWork);
+ LastUpdateTimeMs = Timer.GetElapsedTimeMs();
+
+ uint64_t Deleted = DeletedItemCount.load();
+ uint64_t DeletedBytes = DeletedByteCount.load();
+ uint64_t Discovered = DiscoveredItemCount.load();
+ Progress.UpdateState({.Task = "Removing files ",
+ .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
+ .TotalCount = Discovered,
+ .RemainingCount = Discovered - Deleted},
+ false);
+ });
+
+ std::vector<std::filesystem::path> DirectoriesToDelete;
+ DirectoriesToDelete.reserve(SubdirectoriesToDelete.size());
+ for (auto It : SubdirectoriesToDeleteLookup)
+ {
+ const IoHash& DirHash = It.first;
+ if (auto KeepIt = SubdirectoriesToKeep.find(DirHash); KeepIt == SubdirectoriesToKeep.end())
+ {
+ DirectoriesToDelete.emplace_back(std::move(SubdirectoriesToDelete[It.second]));
+ }
+ }
+
+ std::sort(DirectoriesToDelete.begin(),
+ DirectoriesToDelete.end(),
+ [](const std::filesystem::path& Lhs, const std::filesystem::path& Rhs) {
+ return Lhs.string().length() > Rhs.string().length();
+ });
+
+ for (size_t SubDirectoryIndex = 0; SubDirectoryIndex < DirectoriesToDelete.size(); SubDirectoryIndex++)
+ {
+ ZEN_TRACE_CPU("DeleteDirs");
+ const std::filesystem::path& DirectoryToDelete = DirectoriesToDelete[SubDirectoryIndex];
+ try
+ {
+ if (!Dryrun)
+ {
+ RemoveDirWithRetry(DirectoryToDelete);
+ }
+ ZEN_CONSOLE_VERBOSE("Removed directory {}", DirectoryToDelete);
+ DeletedItemCount++;
+ }
+ catch (const std::exception& Ex)
+ {
+ if (!Quiet)
+ {
+ ZEN_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what());
+ }
+ CleanWipe = false;
+ FailedDeleteCount++;
+ }
+
+ uint64_t NowMs = Timer.GetElapsedTimeMs();
+ if ((NowMs - LastUpdateTimeMs) >= (UsePlainProgress ? 5000 : 200))
+ {
+ LastUpdateTimeMs = NowMs;
+
+ uint64_t Deleted = DeletedItemCount.load();
+ uint64_t DeletedBytes = DeletedByteCount.load();
+ uint64_t Discovered = DiscoveredItemCount.load();
+ Progress.UpdateState({.Task = "Removing folders",
+ .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)),
+ .TotalCount = DirectoriesToDelete.size(),
+ .RemainingCount = DirectoriesToDelete.size() - SubDirectoryIndex},
+ false);
+ }
+ }
+
+ Progress.Finish();
+
+ uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
+ if (!Quiet)
+ {
+ ZEN_CONSOLE("Wiped folder '{}' {} ({}) ({} failed) in {}",
+ Path,
+ DeletedItemCount.load(),
+ NiceBytes(DeletedByteCount.load()),
+ FailedDeleteCount.load(),
+ NiceTimeSpanMs(ElapsedTimeMs));
+ }
+ if (FailedDeleteCount.load() > 0)
+ {
+ throw std::runtime_error(fmt::format("Failed to delete {} files/directories in '{}'", FailedDeleteCount.load(), Path));
+ }
+ return CleanWipe;
+ }
+} // namespace
+
+WipeCommand::WipeCommand()
+{
+ m_Options.add_options()("h,help", "Print help");
+ m_Options.add_option("", "d", "directory", "Directory to wipe", cxxopts::value(m_Directory), "<directory>");
+ m_Options.add_option("", "r", "keep-readonly", "Leave read-only files", cxxopts::value(m_KeepReadOnlyFiles), "<keepreadonly>");
+ m_Options.add_option("", "q", "quiet", "Reduce output to console", cxxopts::value(m_Quiet), "<quiet>");
+ m_Options.add_option("", "y", "yes", "Don't query for confirmation", cxxopts::value(m_Yes), "<yes>");
+ m_Options.add_option("", "", "dryrun", "Do a dry run without deleting anything", cxxopts::value(m_Dryrun), "<dryrun>");
+ m_Options.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), "<progress>");
+ m_Options.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>");
+ m_Options.add_option("",
+ "",
+ "boost-workers",
+ "Increase the number of worker threads - may cause computer to be less responsive",
+ cxxopts::value(m_BoostWorkerThreads),
+ "<boostworkers>");
+
+ m_Options.parse_positional({"directory"});
+}
+
+WipeCommand::~WipeCommand() = default;
+
+int
+WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+ ZEN_UNUSED(GlobalOptions);
+
+ signal(SIGINT, SignalCallbackHandler);
+#if ZEN_PLATFORM_WINDOWS
+ signal(SIGBREAK, SignalCallbackHandler);
+#endif // ZEN_PLATFORM_WINDOWS
+
+ if (!ZenCmdBase::ParseOptions(argc, argv))
+ {
+ return 0;
+ }
+
+ Quiet = m_Quiet;
+ IsVerbose = m_Verbose;
+ UsePlainProgress = IsVerbose || m_PlainProgress;
+ BoostWorkerThreads = m_BoostWorkerThreads;
+
+ MakeSafeAbsolutePathÍnPlace(m_Directory);
+
+ if (!IsDir(m_Directory))
+ {
+ return 0;
+ }
+
+ while (!m_Yes)
+ {
+ const std::string Prompt = fmt::format("Do you want to wipe directory '{}'? (yes/no) ", m_Directory);
+ printf("%s", Prompt.c_str());
+ std::string Reponse;
+ std::getline(std::cin, Reponse);
+ Reponse = ToLower(Reponse);
+ if (Reponse == "y" || Reponse == "yes")
+ {
+ m_Yes = true;
+ }
+ else if (Reponse == "n" || Reponse == "no")
+ {
+ return 0;
+ }
+ }
+
+ try
+ {
+ CleanDirectory(m_Directory, {}, !m_KeepReadOnlyFiles, m_Dryrun);
+ }
+ catch (std::exception& Ex)
+ {
+ if (!m_Quiet)
+ {
+ ZEN_ERROR("{}", Ex.what());
+ }
+ return 3;
+ }
+
+ return 0;
+}
+
+} // namespace zen
diff --git a/src/zen/cmds/wipe_cmd.h b/src/zen/cmds/wipe_cmd.h
new file mode 100644
index 000000000..0e910bb81
--- /dev/null
+++ b/src/zen/cmds/wipe_cmd.h
@@ -0,0 +1,36 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "../zen.h"
+
+namespace zen {
+
+/** Wipe directories
+ */
+class WipeCommand : public ZenCmdBase
+{
+public:
+ static constexpr char Name[] = "wipe";
+ static constexpr char Description[] = "Wipe the contents of a directory";
+
+ WipeCommand();
+ ~WipeCommand();
+
+ virtual cxxopts::Options& Options() override { return m_Options; }
+ virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+ virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; }
+
+private:
+ cxxopts::Options m_Options{Name, Description};
+ std::filesystem::path m_Directory;
+ bool m_KeepReadOnlyFiles = true;
+ bool m_Quiet = false;
+ bool m_Yes = false;
+ bool m_PlainProgress = false;
+ bool m_Verbose = false;
+ bool m_Dryrun = false;
+ bool m_BoostWorkerThreads = false;
+};
+
+} // namespace zen
diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp
index 5f3f8f7ca..773734f12 100644
--- a/src/zen/cmds/workspaces_cmd.cpp
+++ b/src/zen/cmds/workspaces_cmd.cpp
@@ -139,18 +139,16 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
m_HostName = ResolveTargetHostSpec(m_HostName);
- std::filesystem::path SystemRootDir = StringToPath(m_SystemRootDir);
-
- if (SystemRootDir.empty())
+ if (m_SystemRootDir.empty())
{
- SystemRootDir = PickDefaultSystemRootDirectory();
- if (SystemRootDir.empty())
+ m_SystemRootDir = PickDefaultSystemRootDirectory();
+ if (m_SystemRootDir.empty())
{
throw zen::OptionParseException("unable to resolve system root directory");
}
}
- std::filesystem::path StatePath = SystemRootDir / "workspaces";
+ std::filesystem::path StatePath = m_SystemRootDir / "workspaces";
if (!ParseOptions(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data()))
{
@@ -392,18 +390,20 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
m_HostName = ResolveTargetHostSpec(m_HostName);
- std::filesystem::path SystemRootDir = StringToPath(m_SystemRootDir);
-
- if (SystemRootDir.empty())
+ if (m_SystemRootDir.empty())
{
- SystemRootDir = PickDefaultSystemRootDirectory();
- if (SystemRootDir.empty())
+ m_SystemRootDir = PickDefaultSystemRootDirectory();
+ if (m_SystemRootDir.empty())
{
throw zen::OptionParseException("unable to resolve system root directory");
}
}
+ else
+ {
+ MakeSafeAbsolutePathÍnPlace(m_SystemRootDir);
+ }
- std::filesystem::path StatePath = SystemRootDir / "workspaces";
+ std::filesystem::path StatePath = m_SystemRootDir / "workspaces";
if (!ParseOptions(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data()))
{
@@ -412,8 +412,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
if (SubOption == &m_CreateOptions)
{
- std::filesystem::path WorkspaceRoot = StringToPath(m_WorkspaceRoot);
- if (WorkspaceRoot.empty())
+ if (m_WorkspaceRoot.empty())
{
if (m_WorkspaceId.empty())
{
@@ -432,15 +431,15 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
ZEN_CONSOLE("Workspace {} does not exist", m_WorkspaceId);
return 0;
}
- WorkspaceRoot = WorkspaceConfig.RootPath;
+ m_WorkspaceRoot = WorkspaceConfig.RootPath;
}
else
{
- RemoveTrailingPathSeparator(WorkspaceRoot);
+ RemoveTrailingPathSeparator(m_WorkspaceRoot);
if (m_WorkspaceId.empty())
{
- m_WorkspaceId = Workspaces::PathToId(WorkspaceRoot).ToString();
- ZEN_CONSOLE("Using generated workspace id {} from path '{}'", m_WorkspaceId, WorkspaceRoot);
+ m_WorkspaceId = Workspaces::PathToId(m_WorkspaceRoot).ToString();
+ ZEN_CONSOLE("Using generated workspace id {} from path '{}'", m_WorkspaceId, m_WorkspaceRoot);
}
else
{
@@ -449,25 +448,23 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
throw zen::OptionParseException(fmt::format("workspace id '{}' is invalid", m_WorkspaceId));
}
}
- if (Workspaces::AddWorkspace(Log(), StatePath, {.Id = Oid::FromHexString(m_WorkspaceId), .RootPath = WorkspaceRoot}))
+ if (Workspaces::AddWorkspace(Log(), StatePath, {.Id = Oid::FromHexString(m_WorkspaceId), .RootPath = m_WorkspaceRoot}))
{
- ZEN_CONSOLE("Created workspace {} using root path '{}'", m_WorkspaceId, WorkspaceRoot);
+ ZEN_CONSOLE("Created workspace {} using root path '{}'", m_WorkspaceId, m_WorkspaceRoot);
}
else
{
- ZEN_CONSOLE("Using existing workspace {} with root path '{}'", m_WorkspaceId, WorkspaceRoot);
+ ZEN_CONSOLE("Using existing workspace {} with root path '{}'", m_WorkspaceId, m_WorkspaceRoot);
}
}
- std::filesystem::path SharePath = StringToPath(m_SharePath);
-
- RemoveLeadingPathSeparator(SharePath);
- RemoveTrailingPathSeparator(SharePath);
+ RemoveLeadingPathSeparator(m_SharePath);
+ RemoveTrailingPathSeparator(m_SharePath);
if (m_ShareId.empty())
{
- m_ShareId = Workspaces::PathToId(SharePath).ToString();
- ZEN_CONSOLE("Using generated share id {}, for path '{}'", m_ShareId, SharePath);
+ m_ShareId = Workspaces::PathToId(m_SharePath).ToString();
+ ZEN_CONSOLE("Using generated share id {}, for path '{}'", m_ShareId, m_SharePath);
}
if (Oid::TryFromHexString(m_ShareId) == Oid::Zero)
@@ -476,8 +473,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char**
}
if (Workspaces::AddWorkspaceShare(Log(),
- WorkspaceRoot,
- {.Id = Oid::FromHexString(m_ShareId), .SharePath = SharePath, .Alias = m_Alias}))
+ m_WorkspaceRoot,
+ {.Id = Oid::FromHexString(m_ShareId), .SharePath = m_SharePath, .Alias = m_Alias}))
{
if (!m_HostName.empty())
{
diff --git a/src/zen/cmds/workspaces_cmd.h b/src/zen/cmds/workspaces_cmd.h
index 86452e25e..d85d8f7d8 100644
--- a/src/zen/cmds/workspaces_cmd.h
+++ b/src/zen/cmds/workspaces_cmd.h
@@ -21,9 +21,9 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{Name, Description};
- std::string m_HostName;
- std::string m_SystemRootDir;
+ cxxopts::Options m_Options{Name, Description};
+ std::string m_HostName;
+ std::filesystem::path m_SystemRootDir;
std::string m_Verb; // create, info, remove
@@ -53,17 +53,17 @@ public:
virtual cxxopts::Options& Options() override { return m_Options; }
private:
- cxxopts::Options m_Options{Name, Description};
- std::string m_HostName;
- std::string m_SystemRootDir;
- std::string m_WorkspaceId;
- std::string m_WorkspaceRoot;
- std::string m_Verb; // create, info, remove
- std::string m_ShareId;
- std::string m_Alias;
-
- cxxopts::Options m_CreateOptions{"create", "Create a workspace share"};
- std::string m_SharePath;
+ cxxopts::Options m_Options{Name, Description};
+ std::string m_HostName;
+ std::filesystem::path m_SystemRootDir;
+ std::string m_WorkspaceId;
+ std::filesystem::path m_WorkspaceRoot;
+ std::string m_Verb; // create, info, remove
+ std::string m_ShareId;
+ std::string m_Alias;
+
+ cxxopts::Options m_CreateOptions{"create", "Create a workspace share"};
+ std::filesystem::path m_SharePath;
bool m_Refresh = false;
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index 5ce0a89ec..e442f8a4b 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -23,6 +23,7 @@
#include "cmds/up_cmd.h"
#include "cmds/version_cmd.h"
#include "cmds/vfs_cmd.h"
+#include "cmds/wipe_cmd.h"
#include "cmds/workspaces_cmd.h"
#include <zencore/callstack.h>
@@ -557,6 +558,7 @@ main(int argc, char** argv)
UpCommand UpCmd;
VersionCommand VersionCmd;
VfsCommand VfsCmd;
+ WipeCommand WipeCmd;
WorkspaceCommand WorkspaceCmd;
WorkspaceShareCommand WorkspaceShareCmd;
@@ -613,6 +615,7 @@ main(int argc, char** argv)
{"version", &VersionCmd, "Get zen server version"},
{"vfs", &VfsCmd, "Manage virtual file system"},
{"flush", &FlushCmd, "Flush storage"},
+ {WipeCommand::Name, &WipeCmd, WipeCommand::Description},
{WorkspaceCommand::Name, &WorkspaceCmd, WorkspaceCommand::Description},
{WorkspaceShareCommand::Name, &WorkspaceShareCmd, WorkspaceShareCommand::Description},
// clang-format on
diff --git a/src/zen/zen.h b/src/zen/zen.h
index 6765101db..dd0fd44b3 100644
--- a/src/zen/zen.h
+++ b/src/zen/zen.h
@@ -5,10 +5,7 @@
#include <zencore/except.h>
#include <zencore/timer.h>
#include <zencore/zencore.h>
-
-ZEN_THIRD_PARTY_INCLUDES_START
-#include <cxxopts.hpp>
-ZEN_THIRD_PARTY_INCLUDES_END
+#include <zenutil/commandlineoptions.h>
namespace cpr {
class Response;
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index ad796cb4a..018330d9b 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -2670,22 +2670,6 @@ SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly)
return Result;
}
-std::filesystem::path
-StringToPath(const std::string_view& Path)
-{
- std::string_view UnquotedPath = Path;
-
- if (Path.length() > 2 && Path.front() == '\"' && Path.back() == '\"')
- {
- UnquotedPath = Path.substr(1, Path.length() - 2);
- }
- if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator))
- {
- UnquotedPath = UnquotedPath.substr(0, UnquotedPath.length() - 1);
- }
- return std::filesystem::path(UnquotedPath).make_preferred();
-}
-
//////////////////////////////////////////////////////////////////////////
//
// Testing related code follows...
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index 66deffa6f..1bc3943df 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -372,8 +372,6 @@ uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly);
bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec);
bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly);
-std::filesystem::path StringToPath(const std::string_view& Path);
-
//////////////////////////////////////////////////////////////////////////
void filesystem_forcelink(); // internal
diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h
index 0c5931ba0..d1394cd9a 100644
--- a/src/zencore/include/zencore/process.h
+++ b/src/zencore/include/zencore/process.h
@@ -100,9 +100,6 @@ int GetProcessId(CreateProcResult ProcId);
std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc);
std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle);
-std::vector<std::string> ParseCommandLine(std::string_view CommandLine);
-std::vector<char*> StripCommandlineQuotes(std::vector<std::string>& InOutArgs);
-
void process_forcelink(); // internal
} // namespace zen
diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp
index 2fe5b8948..48efc3f85 100644
--- a/src/zencore/process.cpp
+++ b/src/zencore/process.cpp
@@ -1047,118 +1047,6 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand
#endif // ZEN_PLATFORM_LINUX
}
-std::vector<std::string>
-ParseCommandLine(std::string_view CommandLine)
-{
- auto IsWhitespaceOrEnd = [](std::string_view CommandLine, std::string::size_type Pos) {
- if (Pos == CommandLine.length())
- {
- return true;
- }
- if (CommandLine[Pos] == ' ')
- {
- return true;
- }
- return false;
- };
-
- bool IsParsingArg = false;
- bool IsInQuote = false;
-
- std::string::size_type Pos = 0;
- std::string::size_type ArgStart = 0;
- std::vector<std::string> Args;
- while (Pos < CommandLine.length())
- {
- if (IsInQuote)
- {
- if (CommandLine[Pos] == '"' && IsWhitespaceOrEnd(CommandLine, Pos + 1))
- {
- Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart + 1)));
- Pos++;
- IsInQuote = false;
- IsParsingArg = false;
- }
- else
- {
- Pos++;
- }
- }
- else if (IsParsingArg)
- {
- ZEN_ASSERT(Pos > ArgStart);
- if (CommandLine[Pos] == ' ')
- {
- Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart)));
- Pos++;
- IsParsingArg = false;
- }
- else if (CommandLine[Pos] == '"')
- {
- IsInQuote = true;
- Pos++;
- }
- else
- {
- Pos++;
- }
- }
- else if (CommandLine[Pos] == '"')
- {
- IsInQuote = true;
- IsParsingArg = true;
- ArgStart = Pos;
- Pos++;
- }
- else if (CommandLine[Pos] != ' ')
- {
- IsParsingArg = true;
- ArgStart = Pos;
- Pos++;
- }
- else
- {
- Pos++;
- }
- }
- if (IsParsingArg)
- {
- ZEN_ASSERT(Pos > ArgStart);
- Args.push_back(std::string(CommandLine.substr(ArgStart)));
- }
-
- return Args;
-}
-
-std::vector<char*>
-StripCommandlineQuotes(std::vector<std::string>& InOutArgs)
-{
- std::vector<char*> RawArgs;
- RawArgs.reserve(InOutArgs.size());
- for (std::string& Arg : InOutArgs)
- {
- std::string::size_type EscapedQuotePos = Arg.find("\\\"", 1);
- while (EscapedQuotePos != std::string::npos && Arg.rfind('\"', EscapedQuotePos - 1) != std::string::npos)
- {
- Arg.erase(EscapedQuotePos, 1);
- EscapedQuotePos = Arg.find("\\\"", EscapedQuotePos);
- }
-
- if (Arg.starts_with("\""))
- {
- if (Arg.find('"', 1) == Arg.length() - 1)
- {
- if (Arg.find(' ', 1) == std::string::npos)
- {
- Arg = Arg.substr(1, Arg.length() - 2);
- }
- }
- }
- RawArgs.push_back(const_cast<char*>(Arg.c_str()));
- }
- return RawArgs;
-}
-
#if ZEN_WITH_TESTS
void
@@ -1235,36 +1123,6 @@ TEST_CASE("BuildArgV")
}
}
-TEST_CASE("CommandLine")
-{
- std::vector<std::string> v1 = ParseCommandLine("c:\\my\\exe.exe \"quoted arg\" \"one\",two,\"three\\\"");
- CHECK_EQ(v1[0], "c:\\my\\exe.exe");
- CHECK_EQ(v1[1], "\"quoted arg\"");
- CHECK_EQ(v1[2], "\"one\",two,\"three\\\"");
-
- std::vector<std::string> v2 = ParseCommandLine(
- "--tracehost 127.0.0.1 builds download --url=https://jupiter.devtools.epicgames.com --namespace=ue.oplog "
- "--bucket=citysample.packaged-build.fortnite-main.windows \"c:\\just\\a\\path\" "
- "--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\" \"D:\\Dev\\Spaced Folder\\Target\\\" "
- "--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\\\" 07dn23ifiwesnvoasjncasab --build-part-name win64,linux,ps5");
-
- std::vector<char*> v2Stripped = StripCommandlineQuotes(v2);
- CHECK_EQ(v2Stripped[0], std::string("--tracehost"));
- CHECK_EQ(v2Stripped[1], std::string("127.0.0.1"));
- CHECK_EQ(v2Stripped[2], std::string("builds"));
- CHECK_EQ(v2Stripped[3], std::string("download"));
- CHECK_EQ(v2Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com"));
- CHECK_EQ(v2Stripped[5], std::string("--namespace=ue.oplog"));
- CHECK_EQ(v2Stripped[6], std::string("--bucket=citysample.packaged-build.fortnite-main.windows"));
- CHECK_EQ(v2Stripped[7], std::string("c:\\just\\a\\path"));
- CHECK_EQ(v2Stripped[8], std::string("--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\""));
- CHECK_EQ(v2Stripped[9], std::string("\"D:\\Dev\\Spaced Folder\\Target\""));
- CHECK_EQ(v2Stripped[10], std::string("--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\""));
- CHECK_EQ(v2Stripped[11], std::string("07dn23ifiwesnvoasjncasab"));
- CHECK_EQ(v2Stripped[12], std::string("--build-part-name"));
- CHECK_EQ(v2Stripped[13], std::string("win64,linux,ps5"));
-}
-
TEST_SUITE_END(/* core.process */);
#endif
diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp
index 763f3262a..ca1b820c9 100644
--- a/src/zenhttp/httpclient.cpp
+++ b/src/zenhttp/httpclient.cpp
@@ -365,9 +365,10 @@ ShouldRetry(const cpr::Response& Response)
{
case cpr::ErrorCode::OK:
break;
- case cpr::ErrorCode::OPERATION_TIMEDOUT:
+ case cpr::ErrorCode::INTERNAL_ERROR:
case cpr::ErrorCode::NETWORK_RECEIVE_ERROR:
case cpr::ErrorCode::NETWORK_SEND_FAILURE:
+ case cpr::ErrorCode::OPERATION_TIMEDOUT:
return true;
default:
return false;
@@ -377,6 +378,7 @@ ShouldRetry(const cpr::Response& Response)
case HttpResponseCode::RequestTimeout:
case HttpResponseCode::TooManyRequests:
case HttpResponseCode::InternalServerError:
+ case HttpResponseCode::BadGateway:
case HttpResponseCode::ServiceUnavailable:
case HttpResponseCode::GatewayTimeout:
return true;
diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp
index 27a09f339..764f2a2a7 100644
--- a/src/zenhttp/httpserver.cpp
+++ b/src/zenhttp/httpserver.cpp
@@ -787,120 +787,131 @@ HttpRpcHandler::AddRpc(std::string_view RpcId, std::function<void(CbObject& RpcA
//////////////////////////////////////////////////////////////////////////
-enum class HttpServerClass
-{
- kHttpAsio,
- kHttpSys,
- kHttpPlugin,
- kHttpMulti,
- kHttpNull
-};
-
Ref<HttpServer>
-CreateHttpServerClass(HttpServerClass Class, const HttpServerConfig& Config)
+CreateHttpServerClass(const std::string_view ServerClass, const HttpServerConfig& Config)
{
- switch (Class)
+ if (ServerClass == "asio"sv)
{
- default:
- case HttpServerClass::kHttpAsio:
- ZEN_INFO("using asio HTTP server implementation");
- return CreateHttpAsioServer(Config.ForceLoopback, Config.ThreadCount);
-
- case HttpServerClass::kHttpMulti:
- {
- ZEN_INFO("using multi HTTP server implementation");
- Ref<HttpMultiServer> Server{new HttpMultiServer()};
-
- // This is hardcoded for now, but should be configurable in the future
- Server->AddServer(CreateHttpServerClass(HttpServerClass::kHttpSys, Config));
- Server->AddServer(CreateHttpServerClass(HttpServerClass::kHttpPlugin, Config));
+ ZEN_INFO("using asio HTTP server implementation")
+ return CreateHttpAsioServer(Config.ForceLoopback, Config.ThreadCount);
+ }
+#if ZEN_WITH_HTTPSYS
+ else if (ServerClass == "httpsys"sv)
+ {
+ ZEN_INFO("using http.sys server implementation")
+ return Ref<HttpServer>(CreateHttpSysServer({.ThreadCount = Config.ThreadCount,
+ .AsyncWorkThreadCount = Config.HttpSys.AsyncWorkThreadCount,
+ .IsAsyncResponseEnabled = Config.HttpSys.IsAsyncResponseEnabled,
+ .IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled,
+ .IsDedicatedServer = Config.IsDedicatedServer,
+ .ForceLoopback = Config.ForceLoopback}));
+ }
+#endif
+ else if (ServerClass == "null"sv)
+ {
+ ZEN_INFO("using null HTTP server implementation")
+ return Ref<HttpServer>(new HttpNullServer);
+ }
+ else
+ {
+ ZEN_WARN("unknown HTTP server implementation '{}', falling back to default", ServerClass)
- return Server;
- }
+#if ZEN_WITH_HTTPSYS
+ return CreateHttpServerClass("httpsys"sv, Config);
+#else
+ return CreateHttpServerClass("asio"sv, Config);
+#endif
+ }
+}
#if ZEN_WITH_PLUGINS
- case HttpServerClass::kHttpPlugin:
- {
- ZEN_INFO("using plugin HTTP server implementation");
- Ref<HttpPluginServer> Server{CreateHttpPluginServer()};
+Ref<HttpServer>
+CreateHttpServerPlugin(const HttpServerPluginConfig& PluginConfig)
+{
+ const std::string& PluginName = PluginConfig.PluginName;
- // This is hardcoded for now, but should be configurable in the future
+ ZEN_INFO("using '{}' plugin HTTP server implementation", PluginName)
+ if (PluginName.starts_with("builtin:"sv))
+ {
# if 0
- Ref<TransportPlugin> WinsockPlugin{CreateSocketTransportPlugin()};
- WinsockPlugin->Configure("port", "8558");
- Server->AddPlugin(WinsockPlugin);
-# endif
+ Ref<TransportPlugin> Plugin = {};
+ if (PluginName == "builtin:winsock"sv)
+ {
+ Plugin = CreateSocketTransportPlugin();
+ }
+ else if (PluginName == "builtin:asio"sv)
+ {
+ Plugin = CreateAsioTransportPlugin();
+ }
+ else
+ {
+ ZEN_WARN("Unknown builtin plugin '{}'", PluginName)
+ return {};
+ }
-# if 0
- Ref<TransportPlugin> AsioPlugin{CreateAsioTransportPlugin()};
- AsioPlugin->Configure("port", "8558");
- Server->AddPlugin(AsioPlugin);
-# endif
+ ZEN_ASSERT(!Plugin.IsNull());
-# if 1
- Ref<DllTransportPlugin> DllPlugin{CreateDllTransportPlugin()};
- DllPlugin->LoadDll("winsock");
- DllPlugin->ConfigureDll("winsock", "port", "8558");
- Server->AddPlugin(DllPlugin);
-# endif
+ for (const std::pair<std::string, std::string>& Option : PluginConfig.PluginOptions)
+ {
+ Plugin->Configure(Option.first.c_str(), Option.second.c_str());
+ }
- return Server;
- }
-#endif
+ Ref<HttpPluginServer> Server{CreateHttpPluginServer()};
+ Server->AddPlugin(Plugin);
+ return Server;
+# else
+ ZEN_WARN("Builtin plugin '{}' is not supported", PluginName)
+ return {};
+# endif
+ }
-#if ZEN_WITH_HTTPSYS
- case HttpServerClass::kHttpSys:
- ZEN_INFO("using http.sys server implementation");
- return Ref<HttpServer>(CreateHttpSysServer({.ThreadCount = Config.ThreadCount,
- .AsyncWorkThreadCount = Config.HttpSys.AsyncWorkThreadCount,
- .IsAsyncResponseEnabled = Config.HttpSys.IsAsyncResponseEnabled,
- .IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled,
- .IsDedicatedServer = Config.IsDedicatedServer,
- .ForceLoopback = Config.ForceLoopback}));
-#endif
+ Ref<DllTransportPlugin> DllPlugin{CreateDllTransportPlugin()};
+ if (!DllPlugin->LoadDll(PluginName))
+ {
+ return {};
+ }
- case HttpServerClass::kHttpNull:
- ZEN_INFO("using null HTTP server implementation");
- return Ref<HttpServer>(new HttpNullServer);
+ for (const std::pair<std::string, std::string>& Option : PluginConfig.PluginOptions)
+ {
+ DllPlugin->ConfigureDll(PluginName, Option.first.c_str(), Option.second.c_str());
}
+
+ Ref<HttpPluginServer> Server{CreateHttpPluginServer()};
+ Server->AddPlugin(DllPlugin);
+ return Server;
}
+#endif
Ref<HttpServer>
CreateHttpServer(const HttpServerConfig& Config)
{
using namespace std::literals;
- HttpServerClass Class = HttpServerClass::kHttpNull;
-
-#if ZEN_WITH_HTTPSYS
- Class = HttpServerClass::kHttpSys;
-#else
- Class = HttpServerClass::kHttpAsio;
-#endif
-
- if (Config.ServerClass == "asio"sv)
- {
- Class = HttpServerClass::kHttpAsio;
- }
- else if (Config.ServerClass == "httpsys"sv)
- {
- Class = HttpServerClass::kHttpSys;
- }
- else if (Config.ServerClass == "plugin"sv)
- {
- Class = HttpServerClass::kHttpPlugin;
- }
- else if (Config.ServerClass == "null"sv)
+#if ZEN_WITH_PLUGINS
+ if (Config.PluginConfigs.empty())
{
- Class = HttpServerClass::kHttpNull;
+ return CreateHttpServerClass(Config.ServerClass, Config);
}
- else if (Config.ServerClass == "multi"sv)
+ else
{
- Class = HttpServerClass::kHttpMulti;
- }
+ Ref<HttpMultiServer> Server{new HttpMultiServer()};
+ Server->AddServer(CreateHttpServerClass(Config.ServerClass, Config));
- return CreateHttpServerClass(Class, Config);
+ for (const HttpServerPluginConfig& PluginConfig : Config.PluginConfigs)
+ {
+ Ref<HttpServer> PluginServer = CreateHttpServerPlugin(PluginConfig);
+ if (!PluginServer.IsNull())
+ {
+ Server->AddServer(PluginServer);
+ }
+ }
+
+ return Server;
+ }
+#else
+ return CreateHttpServerClass(Config.ServerClass, Config);
+#endif
}
//////////////////////////////////////////////////////////////////////////
diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h
index 217455dba..03e547bf3 100644
--- a/src/zenhttp/include/zenhttp/httpserver.h
+++ b/src/zenhttp/include/zenhttp/httpserver.h
@@ -184,12 +184,19 @@ public:
virtual void Close() = 0;
};
+struct HttpServerPluginConfig
+{
+ std::string PluginName;
+ std::vector<std::pair<std::string, std::string>> PluginOptions;
+};
+
struct HttpServerConfig
{
- bool IsDedicatedServer = false; // Should be set to true for shared servers
- std::string ServerClass; // Choice of HTTP server implementation
- bool ForceLoopback = false;
- unsigned int ThreadCount = 0;
+ bool IsDedicatedServer = false; // Should be set to true for shared servers
+ std::string ServerClass; // Choice of HTTP server implementation
+ std::vector<HttpServerPluginConfig> PluginConfigs;
+ bool ForceLoopback = false;
+ unsigned int ThreadCount = 0;
struct
{
diff --git a/src/zenhttp/servers/httpmulti.cpp b/src/zenhttp/servers/httpmulti.cpp
index 2a6a90d2e..f4dc1e15b 100644
--- a/src/zenhttp/servers/httpmulti.cpp
+++ b/src/zenhttp/servers/httpmulti.cpp
@@ -103,6 +103,10 @@ HttpMultiServer::RequestExit()
void
HttpMultiServer::Close()
{
+ for (auto& Server : m_Servers)
+ {
+ Server->Close();
+ }
}
void
diff --git a/src/zenhttp/transports/dlltransport.cpp b/src/zenhttp/transports/dlltransport.cpp
index e09e62ec5..fb3dd23b5 100644
--- a/src/zenhttp/transports/dlltransport.cpp
+++ b/src/zenhttp/transports/dlltransport.cpp
@@ -21,18 +21,31 @@ namespace zen {
//////////////////////////////////////////////////////////////////////////
+class DllTransportLogger : public TransportLogger, public RefCounted
+{
+public:
+ DllTransportLogger(std::string_view PluginName);
+ virtual ~DllTransportLogger() = default;
+
+ void LogMessage(LogLevel Level, const char* Message) override;
+
+private:
+ std::string m_PluginName;
+};
+
struct LoadedDll
{
std::string Name;
std::filesystem::path LoadedFromPath;
+ DllTransportLogger* Logger = nullptr;
Ref<TransportPlugin> Plugin;
};
class DllTransportPluginImpl : public DllTransportPlugin, RefCounted
{
public:
- DllTransportPluginImpl();
- ~DllTransportPluginImpl();
+ DllTransportPluginImpl() = default;
+ ~DllTransportPluginImpl() = default;
virtual uint32_t AddRef() const override;
virtual uint32_t Release() const override;
@@ -42,7 +55,7 @@ public:
virtual const char* GetDebugName() override;
virtual bool IsAvailable() override;
- virtual void LoadDll(std::string_view Name) override;
+ virtual bool LoadDll(std::string_view Name) override;
virtual void ConfigureDll(std::string_view Name, const char* OptionTag, const char* OptionValue) override;
private:
@@ -51,12 +64,27 @@ private:
std::vector<LoadedDll> m_Transports;
};
-DllTransportPluginImpl::DllTransportPluginImpl()
+DllTransportLogger::DllTransportLogger(std::string_view PluginName) : m_PluginName(PluginName)
{
}
-DllTransportPluginImpl::~DllTransportPluginImpl()
+void
+DllTransportLogger::LogMessage(LogLevel PluginLogLevel, const char* Message)
{
+ logging::level::LogLevel Level;
+ // clang-format off
+ switch (PluginLogLevel)
+ {
+ case LogLevel::Trace: Level = logging::level::Trace; break;
+ case LogLevel::Debug: Level = logging::level::Debug; break;
+ case LogLevel::Info: Level = logging::level::Info; break;
+ case LogLevel::Warn: Level = logging::level::Warn; break;
+ case LogLevel::Err: Level = logging::level::Err; break;
+ case LogLevel::Critical: Level = logging::level::Critical; break;
+ default: Level = logging::level::Off; break;
+ }
+ // clang-format on
+ ZEN_LOG(Log(), Level, "[{}] {}", m_PluginName, Message)
}
uint32_t
@@ -109,6 +137,7 @@ DllTransportPluginImpl::Shutdown()
try
{
Transport.Plugin->Shutdown();
+ Transport.Logger->Release();
}
catch (const std::exception&)
{
@@ -143,42 +172,73 @@ DllTransportPluginImpl::ConfigureDll(std::string_view Name, const char* OptionTa
}
}
-void
+bool
DllTransportPluginImpl::LoadDll(std::string_view Name)
{
RwLock::ExclusiveLockScope _(m_Lock);
- ExtendableStringBuilder<128> DllPath;
- DllPath << Name << ".dll";
+ ExtendableStringBuilder<1024> DllPath;
+ DllPath << Name;
+ if (!Name.ends_with(".dll"))
+ {
+ DllPath << ".dll";
+ }
+
+ std::string FileName = std::filesystem::path(DllPath.c_str()).filename().replace_extension().string();
+
HMODULE DllHandle = LoadLibraryA(DllPath.c_str());
if (!DllHandle)
{
- std::error_code Ec = MakeErrorCodeFromLastError();
-
- throw std::system_error(Ec, fmt::format("failed to load transport DLL from '{}'", DllPath));
+ ZEN_WARN("Failed to load transport DLL from '{}' due to '{}'", DllPath, GetLastErrorAsString())
+ return false;
}
- TransportPlugin* CreateTransportPlugin();
+ PfnGetTransportPluginVersion GetVersion = (PfnGetTransportPluginVersion)GetProcAddress(DllHandle, "GetTransportPluginVersion");
+ PfnCreateTransportPlugin CreatePlugin = (PfnCreateTransportPlugin)GetProcAddress(DllHandle, "CreateTransportPlugin");
+
+ uint32_t APIVersion = 0;
+ uint32_t PluginVersion = 0;
+
+ if (GetVersion)
+ {
+ GetVersion(&APIVersion, &PluginVersion);
+ }
- PfnCreateTransportPlugin CreatePlugin = (PfnCreateTransportPlugin)GetProcAddress(DllHandle, "CreateTransportPlugin");
+ const bool bValidApiVersion = APIVersion == kTransportApiVersion;
- if (!CreatePlugin)
+ if (!GetVersion || !CreatePlugin || !bValidApiVersion)
{
std::error_code Ec = MakeErrorCodeFromLastError();
FreeLibrary(DllHandle);
- throw std::system_error(Ec, fmt::format("API mismatch detected in transport DLL loaded from '{}'", DllPath));
+ if (GetVersion && !bValidApiVersion)
+ {
+ ZEN_WARN("Failed to load transport DLL from '{}' due to invalid API version {}, supported API version is {}",
+ DllPath,
+ APIVersion,
+ kTransportApiVersion)
+ }
+ else
+ {
+ ZEN_WARN("Failed to load transport DLL from '{}' due to not finding GetTransportPluginVersion or CreateTransportPlugin",
+ DllPath)
+ }
+
+ return false;
}
LoadedDll NewDll;
NewDll.Name = Name;
NewDll.LoadedFromPath = DllPath.c_str();
- NewDll.Plugin = CreatePlugin();
+ NewDll.Logger = new DllTransportLogger(FileName);
+ NewDll.Logger->AddRef();
+ NewDll.Plugin = CreatePlugin(NewDll.Logger);
m_Transports.emplace_back(std::move(NewDll));
+ return true;
}
DllTransportPlugin*
diff --git a/src/zenhttp/transports/dlltransport.h b/src/zenhttp/transports/dlltransport.h
index 9346a10ce..c49f888da 100644
--- a/src/zenhttp/transports/dlltransport.h
+++ b/src/zenhttp/transports/dlltransport.h
@@ -15,7 +15,7 @@ namespace zen {
class DllTransportPlugin : public TransportPlugin
{
public:
- virtual void LoadDll(std::string_view Name) = 0;
+ virtual bool LoadDll(std::string_view Name) = 0;
virtual void ConfigureDll(std::string_view Name, const char* OptionTag, const char* OptionValue) = 0;
};
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 31c104110..e81e8eb54 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -15,12 +15,14 @@
#include <zencore/logging.h>
#include <zencore/string.h>
#include <zenhttp/zenhttp.h>
+#include <zenutil/commandlineoptions.h>
ZEN_THIRD_PARTY_INCLUDES_START
#include <fmt/format.h>
#include <fmt/ranges.h>
#include <zencore/logging.h>
#include <cxxopts.hpp>
+#include <json11.hpp>
#include <sol/sol.hpp>
ZEN_THIRD_PARTY_INCLUDES_END
@@ -178,27 +180,6 @@ ParseBucketConfigs(std::span<std::string> Buckets)
return Cfg;
}
-static std::string
-MakeSafePath(const std::string_view Path)
-{
-#if ZEN_PLATFORM_WINDOWS
- if (Path.empty())
- {
- return std::string(Path);
- }
-
- std::string FixedPath(Path);
- std::replace(FixedPath.begin(), FixedPath.end(), '/', '\\');
- if (!FixedPath.starts_with("\\\\?\\"))
- {
- FixedPath.insert(0, "\\\\?\\");
- }
- return FixedPath;
-#else
- return std::string(Path);
-#endif
-};
-
class CachePolicyOption : public LuaConfig::OptionValue
{
public:
@@ -324,7 +305,7 @@ public:
std::string Name = Bucket.value().get_or("name", std::string("Default"));
std::string Directory = Bucket.value().get_or("directory", std::string());
- Value.Buckets.push_back({.Name = std::move(Name), .Directory = LuaConfig::MakeSafePath(Directory)});
+ Value.Buckets.push_back({.Name = std::move(Name), .Directory = MakeSafeAbsolutePath(Directory)});
}
}
}
@@ -369,6 +350,7 @@ ParseConfigFile(const std::filesystem::path& Path,
LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv);
LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv);
LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv);
+ LuaOptions.AddOption("server.pluginsconfigfile"sv, ServerOptions.PluginsConfigFile, "plugins-config"sv);
LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv);
LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv);
LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "quiet"sv);
@@ -379,7 +361,7 @@ ParseConfigFile(const std::filesystem::path& Path,
////// buildsstore
LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv);
- LuaOptions.AddOption("buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit");
+ LuaOptions.AddOption("server.buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit");
////// network
LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv);
@@ -525,7 +507,7 @@ ParseConfigFile(const std::filesystem::path& Path,
if (!OutputConfigFile.empty())
{
- std::filesystem::path WritePath(MakeSafePath(OutputConfigFile));
+ std::filesystem::path WritePath(MakeSafeAbsolutePath(OutputConfigFile));
zen::ExtendableStringBuilder<512> ConfigStringBuilder;
LuaOptions.Print(ConfigStringBuilder, CmdLineResult);
zen::BasicFile Output;
@@ -535,6 +517,68 @@ ParseConfigFile(const std::filesystem::path& Path,
}
void
+ParsePluginsConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions, int BasePort)
+{
+ using namespace std::literals;
+
+ IoBuffer Body = IoBufferBuilder::MakeFromFile(Path);
+ std::string JsonText(reinterpret_cast<const char*>(Body.GetData()), Body.GetSize());
+ std::string JsonError;
+ json11::Json PluginsInfo = json11::Json::parse(JsonText, JsonError);
+ if (!JsonError.empty())
+ {
+ throw std::runtime_error(fmt::format("failed parsing json file '{}'. Reason: '{}'", Path, JsonError));
+ }
+ for (const json11::Json& PluginInfo : PluginsInfo.array_items())
+ {
+ if (!PluginInfo.is_object())
+ {
+ throw std::runtime_error(fmt::format("the json file '{}' does not contain a valid plugin definition, object expected, got '{}'",
+ Path,
+ PluginInfo.dump()));
+ }
+
+ HttpServerPluginConfig Config = {};
+
+ bool bNeedsPort = true;
+
+ for (const std::pair<const std::string, json11::Json>& Items : PluginInfo.object_items())
+ {
+ if (!Items.second.is_string())
+ {
+ throw std::runtime_error(
+ fmt::format("the json file '{}' does not contain a valid plugins definition, string expected, got '{}'",
+ Path,
+ Items.second.dump()));
+ }
+
+ const std::string& Name = Items.first;
+ const std::string& Value = Items.second.string_value();
+
+ if (Name == "name"sv)
+ Config.PluginName = Value;
+ else
+ {
+ Config.PluginOptions.push_back({Name, Value});
+
+ if (Name == "port"sv)
+ {
+ bNeedsPort = false;
+ }
+ }
+ }
+
+ // add a default base port in case if json config didn't provide one
+ if (bNeedsPort)
+ {
+ Config.PluginOptions.push_back({"port", std::to_string(BasePort)});
+ }
+
+ ServerOptions.HttpServerConfig.PluginConfigs.push_back(Config);
+ }
+}
+
+void
ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
{
const char* DefaultHttp = "asio";
@@ -566,6 +610,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
std::string ContentDir;
std::string AbsLogFile;
std::string ConfigFile;
+ std::string PluginsConfigFile;
std::string OutputConfigFile;
std::string BaseSnapshotDir;
@@ -593,6 +638,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
"Exit immediately after initialization is complete",
cxxopts::value<bool>(ServerOptions.IsPowerCycle));
options.add_options()("config", "Path to Lua config file", cxxopts::value<std::string>(ConfigFile));
+ options.add_options()("plugins-config", "Path to plugins config file", cxxopts::value<std::string>(PluginsConfigFile));
options.add_options()("write-config", "Path to output Lua config file", cxxopts::value<std::string>(OutputConfigFile));
options.add_options()("no-sentry",
"Disable Sentry crash handler",
@@ -1110,12 +1156,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
}
logging::RefreshLogLevels();
- ServerOptions.SystemRootDir = MakeSafePath(SystemRootDir);
- ServerOptions.DataDir = MakeSafePath(DataDir);
- ServerOptions.BaseSnapshotDir = MakeSafePath(BaseSnapshotDir);
- ServerOptions.ContentDir = MakeSafePath(ContentDir);
- ServerOptions.AbsLogFile = MakeSafePath(AbsLogFile);
- ServerOptions.ConfigFile = MakeSafePath(ConfigFile);
+ ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir);
+ ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir);
+ ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir);
+ ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir);
+ ServerOptions.AbsLogFile = MakeSafeAbsolutePath(AbsLogFile);
+ ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile);
+ ServerOptions.PluginsConfigFile = MakeSafeAbsolutePath(PluginsConfigFile);
ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions);
if (!BaseSnapshotDir.empty())
@@ -1149,6 +1196,11 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, Result, OutputConfigFile);
}
+ if (!ServerOptions.PluginsConfigFile.empty())
+ {
+ ParsePluginsConfigFile(ServerOptions.PluginsConfigFile, ServerOptions, ServerOptions.BasePort);
+ }
+
ValidateOptions(ServerOptions);
}
catch (const zen::OptionParseException& e)
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index 6bd7aa357..bd277cd88 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -165,6 +165,7 @@ struct ZenServerOptions
std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental)
std::filesystem::path AbsLogFile; // Absolute path to main log file
std::filesystem::path ConfigFile; // Path to Lua config file
+ std::filesystem::path PluginsConfigFile; // Path to plugins config file
std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start)
std::string ChildId; // Id assigned by parent process (used for lifetime management)
std::string LogId; // Id for tagging log output
diff --git a/src/zenserver/config/luaconfig.cpp b/src/zenserver/config/luaconfig.cpp
index f742fa34a..2c54de29e 100644
--- a/src/zenserver/config/luaconfig.cpp
+++ b/src/zenserver/config/luaconfig.cpp
@@ -4,27 +4,6 @@
namespace zen::LuaConfig {
-std::string
-MakeSafePath(const std::string_view Path)
-{
-#if ZEN_PLATFORM_WINDOWS
- if (Path.empty())
- {
- return std::string(Path);
- }
-
- std::string FixedPath(Path);
- std::replace(FixedPath.begin(), FixedPath.end(), '/', '\\');
- if (!FixedPath.starts_with("\\\\?\\"))
- {
- FixedPath.insert(0, "\\\\?\\");
- }
- return FixedPath;
-#else
- return std::string(Path);
-#endif
-};
-
void
EscapeBackslash(std::string& InOutString)
{
@@ -101,7 +80,7 @@ FilePathOption::Parse(sol::object Object)
std::string Str = Object.as<std::string>();
if (!Str.empty())
{
- Value = MakeSafePath(Str);
+ Value = MakeSafeAbsolutePath(Str);
}
}
diff --git a/src/zenserver/config/luaconfig.h b/src/zenserver/config/luaconfig.h
index 76b3088a3..ce7013a9a 100644
--- a/src/zenserver/config/luaconfig.h
+++ b/src/zenserver/config/luaconfig.h
@@ -4,10 +4,10 @@
#include <zenbase/concepts.h>
#include <zencore/fmtutils.h>
+#include <zenutil/commandlineoptions.h>
ZEN_THIRD_PARTY_INCLUDES_START
#include <fmt/format.h>
-#include <cxxopts.hpp>
#include <sol/sol.hpp>
ZEN_THIRD_PARTY_INCLUDES_END
@@ -20,8 +20,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
namespace zen::LuaConfig {
-std::string MakeSafePath(const std::string_view Path);
-void EscapeBackslash(std::string& InOutString);
+void EscapeBackslash(std::string& InOutString);
class OptionValue
{
diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip
index c28404da6..eadfcf84c 100644
--- a/src/zenserver/frontend/html.zip
+++ b/src/zenserver/frontend/html.zip
Binary files differ
diff --git a/src/zenserver/frontend/html/indexer/indexer.js b/src/zenserver/frontend/html/indexer/indexer.js
index 16b91e130..688bc71b0 100644
--- a/src/zenserver/frontend/html/indexer/indexer.js
+++ b/src/zenserver/frontend/html/indexer/indexer.js
@@ -61,7 +61,7 @@ class Indexer
{
for (const page of this._pages)
for (const [_, name, size, raw_size] of page)
- yield [name, size|0, raw_size|0];
+ yield [name, size|0n, raw_size|0n];
}
}
diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js
index 41fc47218..54fb11c18 100644
--- a/src/zenserver/frontend/html/pages/entry.js
+++ b/src/zenserver/frontend/html/pages/entry.js
@@ -79,10 +79,32 @@ export class Page extends ZenPage
async _build_meta(section, entry)
{
var tree = {}
- const cookart = this._find_iohash_field(entry, "CookPackageArtifacts");
- if (cookart != null)
+
+ for (const field of entry)
{
- tree["cook"] = { CookPackageArtifacts: cookart};
+ var visibleKey = undefined;
+ const name = field.get_name();
+ if (name == "CookPackageArtifacts")
+ {
+ visibleKey = name;
+ }
+ else if (name.startsWith("meta."))
+ {
+ visibleKey = name.slice(5);
+ }
+
+ if (visibleKey != undefined)
+ {
+ var found_value = field.as_value();
+ if (found_value instanceof Uint8Array)
+ {
+ var ret = "";
+ for (var x of found_value)
+ ret += x.toString(16).padStart(2, "0");
+ tree[visibleKey] = ret;
+ }
+ }
+
}
if (Object.keys(tree).length == 0)
@@ -90,28 +112,25 @@ export class Page extends ZenPage
const sub_section = section.add_section("meta");
- for (const cat_name in tree)
+ const table = sub_section.add_widget(
+ Table,
+ ["name", "actions"], Table.Flag_PackRight
+ );
+ for (const key in tree)
{
- const cat_section = sub_section.add_section(cat_name);
- const table = cat_section.add_widget(
- Table,
- ["name", "actions"], Table.Flag_PackRight
+ const row = table.add_row(key);
+ const value = tree[key];
+
+ const project = this.get_param("project");
+ const oplog = this.get_param("oplog");
+ const link = row.get_cell(0).link(
+ "/" + ["prj", project, "oplog", oplog, value+".json"].join("/")
);
- Object.entries(tree[cat_name]).forEach(([key, value]) =>
- {
- const row = table.add_row(key);
-
- const project = this.get_param("project");
- const oplog = this.get_param("oplog");
- const link = row.get_cell(0).link(
- "/" + ["prj", project, "oplog", oplog, value+".json"].join("/")
- );
-
- const action_tb = new Toolbar(row.get_cell(-1), true);
- action_tb.left().add("copy-hash").on_click(async (v) => {
- await navigator.clipboard.writeText(v);
- }, value);
- });
+
+ const action_tb = new Toolbar(row.get_cell(-1), true);
+ action_tb.left().add("copy-hash").on_click(async (v) => {
+ await navigator.clipboard.writeText(v);
+ }, value);
}
}
diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp
index e4d962b56..4f72a711a 100644
--- a/src/zenstore/cache/cachedisklayer.cpp
+++ b/src/zenstore/cache/cachedisklayer.cpp
@@ -3572,7 +3572,7 @@ ZenCacheDiskLayer::~ZenCacheDiskLayer()
}
template<typename T, typename U>
-struct equal_to_2 : public eastl::binary_function<T, U, bool>
+struct equal_to_2
{
constexpr bool operator()(const T& a, const U& b) const { return a == b; }
diff --git a/src/zenutil/commandlineoptions.cpp b/src/zenutil/commandlineoptions.cpp
new file mode 100644
index 000000000..0dffa42f0
--- /dev/null
+++ b/src/zenutil/commandlineoptions.cpp
@@ -0,0 +1,213 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenutil/commandlineoptions.h>
+
+#include <filesystem>
+#if ZEN_WITH_TESTS
+# include <zencore/testing.h>
+#endif // ZEN_WITH_TESTS
+
+void
+cxxopts::values::parse_value(const std::string& text, std::filesystem::path& value)
+{
+ value = zen::StringToPath(text);
+}
+
+namespace zen {
+
+std::vector<std::string>
+ParseCommandLine(std::string_view CommandLine)
+{
+ auto IsWhitespaceOrEnd = [](std::string_view CommandLine, std::string::size_type Pos) {
+ if (Pos == CommandLine.length())
+ {
+ return true;
+ }
+ if (CommandLine[Pos] == ' ')
+ {
+ return true;
+ }
+ return false;
+ };
+
+ bool IsParsingArg = false;
+ bool IsInQuote = false;
+
+ std::string::size_type Pos = 0;
+ std::string::size_type ArgStart = 0;
+ std::vector<std::string> Args;
+ while (Pos < CommandLine.length())
+ {
+ if (IsInQuote)
+ {
+ if (CommandLine[Pos] == '"' && IsWhitespaceOrEnd(CommandLine, Pos + 1))
+ {
+ Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart + 1)));
+ Pos++;
+ IsInQuote = false;
+ IsParsingArg = false;
+ }
+ else
+ {
+ Pos++;
+ }
+ }
+ else if (IsParsingArg)
+ {
+ ZEN_ASSERT(Pos > ArgStart);
+ if (CommandLine[Pos] == ' ')
+ {
+ Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart)));
+ Pos++;
+ IsParsingArg = false;
+ }
+ else if (CommandLine[Pos] == '"')
+ {
+ IsInQuote = true;
+ Pos++;
+ }
+ else
+ {
+ Pos++;
+ }
+ }
+ else if (CommandLine[Pos] == '"')
+ {
+ IsInQuote = true;
+ IsParsingArg = true;
+ ArgStart = Pos;
+ Pos++;
+ }
+ else if (CommandLine[Pos] != ' ')
+ {
+ IsParsingArg = true;
+ ArgStart = Pos;
+ Pos++;
+ }
+ else
+ {
+ Pos++;
+ }
+ }
+ if (IsParsingArg)
+ {
+ ZEN_ASSERT(Pos > ArgStart);
+ Args.push_back(std::string(CommandLine.substr(ArgStart)));
+ }
+
+ return Args;
+}
+
+std::vector<char*>
+StripCommandlineQuotes(std::vector<std::string>& InOutArgs)
+{
+ std::vector<char*> RawArgs;
+ RawArgs.reserve(InOutArgs.size());
+ for (std::string& Arg : InOutArgs)
+ {
+ std::string::size_type EscapedQuotePos = Arg.find("\\\"", 1);
+ while (EscapedQuotePos != std::string::npos && Arg.rfind('\"', EscapedQuotePos - 1) != std::string::npos)
+ {
+ Arg.erase(EscapedQuotePos, 1);
+ EscapedQuotePos = Arg.find("\\\"", EscapedQuotePos);
+ }
+
+ if (Arg.starts_with("\""))
+ {
+ if (Arg.find('"', 1) == Arg.length() - 1)
+ {
+ if (Arg.find(' ', 1) == std::string::npos)
+ {
+ Arg = Arg.substr(1, Arg.length() - 2);
+ }
+ }
+ }
+ RawArgs.push_back(const_cast<char*>(Arg.c_str()));
+ }
+ return RawArgs;
+}
+
+void
+MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path)
+{
+ if (!Path.empty())
+ {
+ std::filesystem::path AbsolutePath = std::filesystem::absolute(Path).make_preferred();
+#if ZEN_PLATFORM_WINDOWS
+ const std::string_view Prefix = "\\\\?\\";
+ const std::u8string PrefixU8(Prefix.begin(), Prefix.end());
+ std::u8string PathString = AbsolutePath.u8string();
+ if (!PathString.empty() && !PathString.starts_with(PrefixU8))
+ {
+ PathString.insert(0, PrefixU8);
+ Path = PathString;
+ }
+#endif // ZEN_PLATFORM_WINDOWS
+ }
+}
+
+std::filesystem::path
+MakeSafeAbsolutePath(const std::filesystem::path& Path)
+{
+ std::filesystem::path Tmp(Path);
+ MakeSafeAbsolutePathÍnPlace(Tmp);
+ return Tmp;
+}
+
+std::filesystem::path
+StringToPath(const std::string_view& Path)
+{
+ std::string_view UnquotedPath = Path;
+
+ if (UnquotedPath.length() > 2 && UnquotedPath.front() == '\"' && UnquotedPath.back() == '\"')
+ {
+ UnquotedPath = UnquotedPath.substr(1, UnquotedPath.length() - 2);
+ }
+
+ if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator))
+ {
+ UnquotedPath = UnquotedPath.substr(0, UnquotedPath.length() - 1);
+ }
+
+ return std::filesystem::path(UnquotedPath).make_preferred();
+}
+
+#if ZEN_WITH_TESTS
+
+void
+commandlineoptions_forcelink()
+{
+}
+
+TEST_CASE("CommandLine")
+{
+ std::vector<std::string> v1 = ParseCommandLine("c:\\my\\exe.exe \"quoted arg\" \"one\",two,\"three\\\"");
+ CHECK_EQ(v1[0], "c:\\my\\exe.exe");
+ CHECK_EQ(v1[1], "\"quoted arg\"");
+ CHECK_EQ(v1[2], "\"one\",two,\"three\\\"");
+
+ std::vector<std::string> v2 = ParseCommandLine(
+ "--tracehost 127.0.0.1 builds download --url=https://jupiter.devtools.epicgames.com --namespace=ue.oplog "
+ "--bucket=citysample.packaged-build.fortnite-main.windows \"c:\\just\\a\\path\" "
+ "--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\" \"D:\\Dev\\Spaced Folder\\Target\\\" "
+ "--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\\\" 07dn23ifiwesnvoasjncasab --build-part-name win64,linux,ps5");
+
+ std::vector<char*> v2Stripped = StripCommandlineQuotes(v2);
+ CHECK_EQ(v2Stripped[0], std::string("--tracehost"));
+ CHECK_EQ(v2Stripped[1], std::string("127.0.0.1"));
+ CHECK_EQ(v2Stripped[2], std::string("builds"));
+ CHECK_EQ(v2Stripped[3], std::string("download"));
+ CHECK_EQ(v2Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com"));
+ CHECK_EQ(v2Stripped[5], std::string("--namespace=ue.oplog"));
+ CHECK_EQ(v2Stripped[6], std::string("--bucket=citysample.packaged-build.fortnite-main.windows"));
+ CHECK_EQ(v2Stripped[7], std::string("c:\\just\\a\\path"));
+ CHECK_EQ(v2Stripped[8], std::string("--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\""));
+ CHECK_EQ(v2Stripped[9], std::string("\"D:\\Dev\\Spaced Folder\\Target\""));
+ CHECK_EQ(v2Stripped[10], std::string("--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\""));
+ CHECK_EQ(v2Stripped[11], std::string("07dn23ifiwesnvoasjncasab"));
+ CHECK_EQ(v2Stripped[12], std::string("--build-part-name"));
+ CHECK_EQ(v2Stripped[13], std::string("win64,linux,ps5"));
+}
+
+#endif
+} // namespace zen
diff --git a/src/zenutil/include/zenutil/commandlineoptions.h b/src/zenutil/include/zenutil/commandlineoptions.h
new file mode 100644
index 000000000..b7581f6cd
--- /dev/null
+++ b/src/zenutil/include/zenutil/commandlineoptions.h
@@ -0,0 +1,28 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+#include <filesystem>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+
+namespace cxxopts::values {
+// We declare this specialization before including cxxopts to make it stick
+void parse_value(const std::string& text, std::filesystem::path& value);
+} // namespace cxxopts::values
+
+#include <cxxopts.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+std::vector<std::string> ParseCommandLine(std::string_view CommandLine);
+std::vector<char*> StripCommandlineQuotes(std::vector<std::string>& InOutArgs);
+void MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path);
+[[nodiscard]] std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path);
+std::filesystem::path StringToPath(const std::string_view& Path);
+
+void commandlineoptions_forcelink(); // internal
+
+} // namespace zen
diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp
index 19eb63ce9..aff9156f4 100644
--- a/src/zenutil/zenutil.cpp
+++ b/src/zenutil/zenutil.cpp
@@ -7,6 +7,7 @@
# include <zenutil/cache/cacherequests.h>
# include <zenutil/cache/rpcrecording.h>
# include <zenutil/chunkedfile.h>
+# include <zenutil/commandlineoptions.h>
namespace zen {
@@ -17,6 +18,7 @@ zenutil_forcelinktests()
cache::rpcrecord_forcelink();
cacherequests_forcelink();
chunkedfile_forcelink();
+ commandlineoptions_forcelink();
}
} // namespace zen