diff options
| author | Zousar Shaker <[email protected]> | 2025-04-24 08:26:29 -0600 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-04-24 08:26:29 -0600 |
| commit | 787449efb4de24fd12f3af3c4e466a9629203108 (patch) | |
| tree | cb4a0e84a0381e9b4a087401037ba5837e8e65d7 /src | |
| parent | Changelog update terminology (diff) | |
| parent | 5.6.6-pre1 (diff) | |
| download | zen-787449efb4de24fd12f3af3c4e466a9629203108.tar.xz zen-787449efb4de24fd12f3af3c4e466a9629203108.zip | |
Merge branch 'main' into zs/zencli-list-namespaces-buckets
Diffstat (limited to 'src')
42 files changed, 1735 insertions, 927 deletions
diff --git a/src/transports/transport-sdk/include/transportplugin.h b/src/transports/transport-sdk/include/transportplugin.h index 4347868e6..a78a758bc 100644 --- a/src/transports/transport-sdk/include/transportplugin.h +++ b/src/transports/transport-sdk/include/transportplugin.h @@ -17,10 +17,14 @@ namespace zen { +// Current API version, value will be incremented to represent breaking changes +static const uint32_t kTransportApiVersion = 1; + class TransportConnection; class TransportPlugin; class TransportServerConnection; class TransportServer; +class TransportLogger; /************************************************************************* @@ -60,6 +64,29 @@ public: virtual TransportServerConnection* CreateConnectionHandler(TransportConnection* Connection) = 0; }; +/** Logger interface + + There will be one instance of this provided by the system to the transport plugin + + The plugin can use this to log messages back to zen server + + */ +class TransportLogger +{ +public: + enum class LogLevel : uint32_t + { + Trace = 0, + Debug = 1, + Info = 2, + Warn = 3, + Err = 4, + Critical = 5, + }; + + virtual void LogMessage(LogLevel Level, const char* Message) = 0; +}; + /************************************************************************* The following interfaces are to be implemented by transport plugins. @@ -116,7 +143,18 @@ public: extern "C" { - DLL_TRANSPORT_API zen::TransportPlugin* CreateTransportPlugin(); + /** Provide information about plugin version + + Fills out API version (kTransportApiVersion) plugin was built against. + Fills out plugin own version ever increasing version number, + a copy of plugin with higher version will be used. + */ + DLL_TRANSPORT_API void GetTransportPluginVersion(uint32_t* OutApiVersion, uint32_t* OutPluginVersion); + + // Return nullptr if requested api version mismatches api version plugin was built against + DLL_TRANSPORT_API zen::TransportPlugin* CreateTransportPlugin(zen::TransportLogger* Logger); } -typedef zen::TransportPlugin* (*PfnCreateTransportPlugin)(); +typedef void (*PfnGetTransportPluginVersion)(uint32_t* OutApiVersion, uint32_t* OutPluginVersion); + +typedef zen::TransportPlugin* (*PfnCreateTransportPlugin)(zen::TransportLogger* Logger); diff --git a/src/transports/winsock/winsock.cpp b/src/transports/winsock/winsock.cpp index 1c3ee909a..f98984726 100644 --- a/src/transports/winsock/winsock.cpp +++ b/src/transports/winsock/winsock.cpp @@ -364,8 +364,22 @@ WinsockTransportPlugin::IsAvailable() ////////////////////////////////////////////////////////////////////////// +void +GetTransportPluginVersion(uint32_t* OutApiVersion, uint32_t* OutPluginVersion) +{ + if (OutApiVersion != nullptr) + { + *OutApiVersion = kTransportApiVersion; + } + + if (OutPluginVersion != nullptr) + { + *OutPluginVersion = 1; + } +} + TransportPlugin* -CreateTransportPlugin() +CreateTransportPlugin([[maybe_unused]] TransportLogger* Logger) { return new WinsockTransportPlugin; } diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp index 573639c2d..a7cfa6a4e 100644 --- a/src/zen/cmds/admin_cmd.cpp +++ b/src/zen/cmds/admin_cmd.cpp @@ -714,29 +714,26 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw OptionParseException("data path must be given"); } - std::filesystem::path DataPath = StringToPath(m_DataPath); - std::filesystem::path TargetPath = StringToPath(m_TargetPath); - - if (!IsDir(DataPath)) + if (!IsDir(m_DataPath)) { throw OptionParseException("data path must exist"); } - if (TargetPath.empty()) + if (m_TargetPath.empty()) { throw OptionParseException("target path must be given"); } - std::filesystem::path RootManifestPath = DataPath / "root_manifest"; - std::filesystem::path TargetRootManifestPath = TargetPath / "root_manifest"; + std::filesystem::path RootManifestPath = m_DataPath / "root_manifest"; + std::filesystem::path TargetRootManifestPath = m_TargetPath / "root_manifest"; if (!TryCopy(RootManifestPath, TargetRootManifestPath)) { throw OptionParseException("data path is invalid, missing root_manifest"); } - std::filesystem::path CachePath = DataPath / "cache"; - std::filesystem::path TargetCachePath = TargetPath / "cache"; + std::filesystem::path CachePath = m_DataPath / "cache"; + std::filesystem::path TargetCachePath = m_TargetPath / "cache"; // Copy cache state DirectoryContent CacheDirectoryContent; @@ -781,8 +778,8 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path CasPath = DataPath / "cas"; - std::filesystem::path TargetCasPath = TargetPath / "cas"; + std::filesystem::path CasPath = m_DataPath / "cas"; + std::filesystem::path TargetCasPath = m_TargetPath / "cas"; { std::filesystem::path UCasRootPath = CasPath / ".ucas_root"; diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h index 8b6d3e258..c593b2cac 100644 --- a/src/zen/cmds/admin_cmd.h +++ b/src/zen/cmds/admin_cmd.h @@ -155,10 +155,10 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"copy-state", "Copy zen server disk state"}; - std::string m_DataPath; - std::string m_TargetPath; - bool m_SkipLogs = false; + cxxopts::Options m_Options{"copy-state", "Copy zen server disk state"}; + std::filesystem::path m_DataPath; + std::filesystem::path m_TargetPath; + bool m_SkipLogs = false; }; } // namespace zen diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 624bb2270..15c635594 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -164,24 +164,6 @@ namespace { ); - std::filesystem::path MakeSafeAbsolutePath(std::filesystem::path Path) - { - std::filesystem::path AbsolutePath = std::filesystem::absolute(Path).make_preferred(); -#if ZEN_PLATFORM_WINDOWS && 1 - const std::string_view Prefix = "\\\\?\\"; - const std::u8string PrefixU8(Prefix.begin(), Prefix.end()); - std::u8string PathString = AbsolutePath.u8string(); - if (!PathString.empty() && !PathString.starts_with(PrefixU8)) - { - PathString.insert(0, PrefixU8); - return std::filesystem::path(PathString); - } -#endif - return AbsolutePath; - } - - std::filesystem::path MakeSafeAbsolutePath(const std::string PathString) { return MakeSafeAbsolutePath(StringToPath(PathString)); } - bool IsFileWithRetry(const std::filesystem::path& Path) { std::error_code Ec; @@ -690,75 +672,6 @@ namespace { return CacheFolderPath / RawHash.ToHexString(); } - ChunkedFolderContent ScanAndChunkFolder( - GetFolderContentStatistics& GetFolderContentStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - std::function<bool(const std::string_view& RelativePath)>&& IsAcceptedFolder, - std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile, - ChunkingController& ChunkController) - { - ZEN_TRACE_CPU("ScanAndChunkFolder"); - - FolderContent Content = GetFolderContent( - GetFolderContentStats, - Path, - std::move(IsAcceptedFolder), - std::move(IsAcceptedFile), - GetIOWorkerPool(), - UsePlainProgress ? 5000 : 200, - [](bool, std::ptrdiff_t) {}, - AbortFlag); - if (AbortFlag) - { - return {}; - } - - ProgressBar ProgressBar(UsePlainProgress); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkedFolderContent FolderContent = ChunkFolderContent( - ChunkingStats, - GetIOWorkerPool(), - Path, - Content, - ChunkController, - UsePlainProgress ? 5000 : 200, - [&](bool, std::ptrdiff_t) { - FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - ChunkingStats.FilesProcessed.load(), - GetFolderContentStats.AcceptedFileCount.load(), - NiceBytes(ChunkingStats.BytesHashed.load()), - NiceBytes(GetFolderContentStats.FoundFileByteCount), - NiceNum(FilteredBytesHashed.GetCurrent()), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = GetFolderContentStats.AcceptedFileByteCount, - .RemainingCount = GetFolderContentStats.AcceptedFileByteCount - ChunkingStats.BytesHashed.load()}, - false); - }, - AbortFlag); - if (AbortFlag) - { - return {}; - } - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - - ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", - ChunkingStats.FilesProcessed.load(), - NiceBytes(ChunkingStats.BytesHashed.load()), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load()), - Path, - NiceTimeSpanMs((GetFolderContentStats.ElapsedWallTimeUS + ChunkingStats.ElapsedWallTimeUS) / 1000), - NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); - return FolderContent; - }; - struct DiskStatistics { std::atomic<uint64_t> OpenReadCount = 0; @@ -2562,7 +2475,7 @@ namespace { BlockIndexes.push_back(It->second); TotalBlocksSize += NewBlocks.BlockSizes[It->second]; } - if (auto ChunkIndexIt = Lookup.ChunkHashToChunkIndex.find(RawHash); ChunkIndexIt != Lookup.ChunkHashToChunkIndex.end()) + else if (auto ChunkIndexIt = Lookup.ChunkHashToChunkIndex.find(RawHash); ChunkIndexIt != Lookup.ChunkHashToChunkIndex.end()) { const uint32_t ChunkIndex = ChunkIndexIt->second; if (auto LooseOrderIndexIt = ChunkIndexToLooseChunkOrderIndex.find(ChunkIndex); @@ -2572,6 +2485,11 @@ namespace { TotalLooseChunksSize += Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; } } + else + { + throw std::runtime_error( + fmt::format("Can not upload requested build blob {} as it was not generated by this upload", RawHash)); + } } uint64_t TotalRawSize = TotalLooseChunksSize + TotalBlocksSize; @@ -3169,8 +3087,9 @@ namespace { auto ParseManifest = [](const std::filesystem::path& Path, const std::filesystem::path& ManifestPath) -> std::vector<std::filesystem::path> { std::vector<std::filesystem::path> AssetPaths; - std::filesystem::path AbsoluteManifestPath = ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath; - IoBuffer ManifestContent = ReadFile(AbsoluteManifestPath).Flatten(); + std::filesystem::path AbsoluteManifestPath = + MakeSafeAbsolutePath(ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath); + IoBuffer ManifestContent = ReadFile(AbsoluteManifestPath).Flatten(); std::string_view ManifestString((const char*)ManifestContent.GetView().GetData(), ManifestContent.GetSize()); std::string_view::size_type Offset = 0; while (Offset < ManifestContent.GetSize()) @@ -3730,7 +3649,8 @@ namespace { UploadAttachments(PutBuildPartResult.second); } - while (!AbortFlag) + uint32_t FinalizeBuildPartRetryCount = 5; + while (!AbortFlag && (FinalizeBuildPartRetryCount--) > 0) { Stopwatch FinalizeBuildPartTimer; std::vector<IoHash> Needs = Storage.BuildStorage->FinalizeBuildPart(BuildId, BuildPartId, PartHash); @@ -8175,13 +8095,13 @@ namespace { return RemoteContent; } - ChunkedFolderContent GetLocalContent(GetFolderContentStatistics& LocalFolderScanStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - const std::filesystem::path& StateFilePath, - ChunkingController& ChunkController, - const ChunkedFolderContent& ReferenceContent, - FolderContent& OutLocalFolderContent) + ChunkedFolderContent GetLocalContent(GetFolderContentStatistics& LocalFolderScanStats, + ChunkingStatistics& ChunkingStats, + const std::filesystem::path& Path, + const std::filesystem::path& StateFilePath, + ChunkingController& ChunkController, + std::span<const std::filesystem::path> ReferencePaths, + FolderContent& OutLocalFolderContent) { FolderContent LocalFolderState; ChunkedFolderContent LocalContent; @@ -8193,7 +8113,7 @@ namespace { ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); } { - const uint32_t LocalPathCount = gsl::narrow<uint32_t>(ReferenceContent.Paths.size()); + const uint32_t LocalPathCount = gsl::narrow<uint32_t>(ReferencePaths.size()); const uint32_t RemotePathCount = gsl::narrow<uint32_t>(LocalFolderState.Paths.size()); std::vector<std::filesystem::path> PathsToCheck; @@ -8208,7 +8128,7 @@ namespace { PathsToCheck.push_back(LocalPath); } - for (const std::filesystem::path& RemotePath : ReferenceContent.Paths) + for (const std::filesystem::path& RemotePath : ReferencePaths) { if (FileSet.insert(RemotePath.generic_string()).second) { @@ -8351,6 +8271,56 @@ namespace { return LocalContent; } + ChunkedFolderContent ScanAndChunkFolder( + GetFolderContentStatistics& GetFolderContentStats, + ChunkingStatistics& ChunkingStats, + const std::filesystem::path& Path, + std::function<bool(const std::string_view& RelativePath)>&& IsAcceptedFolder, + std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile, + ChunkingController& ChunkController) + { + Stopwatch Timer; + + ZEN_TRACE_CPU("ScanAndChunkFolder"); + + FolderContent Content = GetFolderContent( + GetFolderContentStats, + Path, + std::move(IsAcceptedFolder), + std::move(IsAcceptedFile), + GetIOWorkerPool(), + UsePlainProgress ? 5000 : 200, + [](bool, std::ptrdiff_t) {}, + AbortFlag); + if (AbortFlag) + { + return {}; + } + + FolderContent _; + ChunkedFolderContent Result = GetLocalContent(GetFolderContentStats, + ChunkingStats, + Path, + ZenStateFilePath(Path / ZenFolderName), + ChunkController, + Content.Paths, + _); + + const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0)); + const uint64_t ChunkedRawSize = + std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0)); + + ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", + Result.Paths.size(), + NiceBytes(TotalRawSize), + Result.ChunkedContent.ChunkHashes.size(), + NiceBytes(ChunkedRawSize), + Path, + NiceTimeSpanMs(Timer.GetElapsedTimeMs()), + NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + return Result; + }; + void DownloadFolder(StorageInstance& Storage, const Oid& BuildId, const std::vector<Oid>& BuildPartIds, @@ -8415,7 +8385,7 @@ namespace { Path, ZenStateFilePath(ZenFolderPath), *ChunkController, - RemoteContent, + RemoteContent.Paths, LocalFolderContent); } } @@ -8712,7 +8682,7 @@ namespace { double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0; - ZEN_CONSOLE("{} ({}) files removed, {} ({}) files added, {} ({} {:.1f}%) files kept", + ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept", RemovedHashes.size(), NiceBytes(RemovedSize), AddedHashes.size(), @@ -8747,7 +8717,7 @@ namespace { double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0; double NewPercent = CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0; - ZEN_CONSOLE("Found {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.", + ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.", FoundChunkCount, NiceBytes(FoundChunkSize), FoundPercent, @@ -8769,7 +8739,7 @@ BuildsCommand::BuildsCommand() m_Options.add_options()("h,help", "Print help"); auto AddSystemOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value<std::string>(m_SystemRootDir), "<systemdir>"); + Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), "<systemdir>"); }; auto AddAuthOptions = [this](cxxopts::Options& Ops) { @@ -8890,7 +8860,7 @@ BuildsCommand::BuildsCommand() Ops.add_option("", "", "boost-workers", - "Increase the number of worker threads - may cause computer to less responsive", + "Increase the number of worker threads - may cause computer to be less responsive", cxxopts::value(m_BoostWorkerThreads), "<boostworkers>"); }; @@ -9218,8 +9188,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::filesystem::path SystemRootDir; auto ParseSystemOptions = [&]() { - SystemRootDir = m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : MakeSafeAbsolutePath(m_SystemRootDir); + if (m_SystemRootDir.empty()) + { + m_SystemRootDir = PickDefaultSystemRootDirectory(); + } + MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); }; + ParseSystemOptions(); auto ParseStorageOptions = [&]() { if (!m_OverrideHost.empty() || !m_Host.empty()) @@ -9238,6 +9213,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); } + MakeSafeAbsolutePathÍnPlace(m_StoragePath); }; std::unique_ptr<AuthMgr> Auth; @@ -9247,7 +9223,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) .RetryCount = 2}; auto CreateAuthMgr = [&]() { - ZEN_ASSERT(!SystemRootDir.empty()); + ZEN_ASSERT(!m_SystemRootDir.empty()); if (!Auth) { if (m_EncryptionKey.empty()) @@ -9262,7 +9238,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("Warning: Using default encryption initialization vector"); } - AuthConfig AuthMgrConfig = {.RootDirectory = SystemRootDir / "auth", + AuthConfig AuthMgrConfig = {.RootDirectory = m_SystemRootDir / "auth", .EncryptionKey = AesKey256Bit::FromString(m_EncryptionKey), .EncryptionIV = AesIV128Bit::FromString(m_EncryptionIV)}; if (!AuthMgrConfig.EncryptionKey.IsValid()) @@ -9278,7 +9254,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; auto ParseAuthOptions = [&]() { - ParseSystemOptions(); if (!m_OpenIdProviderUrl.empty() && !m_OpenIdClientId.empty()) { CreateAuthMgr(); @@ -9325,7 +9300,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (!m_AccessTokenPath.empty()) { - std::string ResolvedAccessToken = ReadAccessTokenFromFile(MakeSafeAbsolutePath(m_AccessTokenPath)); + MakeSafeAbsolutePathÍnPlace(m_AccessTokenPath); + std::string ResolvedAccessToken = ReadAccessTokenFromFile(m_AccessTokenPath); if (!ResolvedAccessToken.empty()) { ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); @@ -9567,10 +9543,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (!m_StoragePath.empty()) { - std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath); - StorageDescription = fmt::format("folder {}", StoragePath); - Result.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - Result.StorageName = fmt::format("Disk {}", StoragePath.stem()); + StorageDescription = fmt::format("folder {}", m_StoragePath); + Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + Result.StorageName = fmt::format("Disk {}", m_StoragePath.stem()); } else { @@ -9614,6 +9589,137 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return Result; }; + auto ParsePath = [&]() { + if (m_Path.empty()) + { + throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help())); + } + MakeSafeAbsolutePathÍnPlace(m_Path); + }; + + auto ParseDiffPath = [&]() { + if (m_DiffPath.empty()) + { + throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); + } + MakeSafeAbsolutePathÍnPlace(m_DiffPath); + }; + + auto ParseBlobHash = [&]() -> IoHash { + if (m_BlobHash.empty()) + { + throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); + } + + IoHash BlobHash; + if (!IoHash::TryParse(m_BlobHash, BlobHash)) + { + throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help())); + } + + return BlobHash; + }; + + auto ParseBuildId = [&]() -> Oid { + if (m_BuildId.length() != Oid::StringLength) + { + throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); + } + else if (Oid BuildId = Oid::FromHexString(m_BuildId); BuildId == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); + } + else + { + return BuildId; + } + }; + + auto ParseBuildPartId = [&]() -> Oid { + if (m_BuildPartId.length() != Oid::StringLength) + { + throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); + } + else if (Oid BuildPartId = Oid::FromHexString(m_BuildPartId); BuildPartId == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); + } + else + { + return BuildPartId; + } + }; + + auto ParseBuildPartIds = [&]() -> std::vector<Oid> { + std::vector<Oid> BuildPartIds; + for (const std::string& BuildPartId : m_BuildPartIds) + { + BuildPartIds.push_back(Oid::TryFromHexString(BuildPartId)); + if (BuildPartIds.back() == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, m_DownloadOptions.help())); + } + } + return BuildPartIds; + }; + + auto ParseBuildMetadata = [&]() -> CbObject { + if (m_CreateBuild) + { + if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty()) + { + throw zen::OptionParseException(fmt::format("Options for builds target are missing\n{}", m_UploadOptions.help())); + } + if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty()) + { + throw zen::OptionParseException(fmt::format("Conflicting options for builds target\n{}", m_UploadOptions.help())); + } + + if (!m_BuildMetadataPath.empty()) + { + MakeSafeAbsolutePathÍnPlace(m_BuildMetadataPath); + IoBuffer MetaDataJson = ReadFile(m_BuildMetadataPath).Flatten(); + std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); + std::string JsonError; + CbObject MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); + if (!JsonError.empty()) + { + throw std::runtime_error( + fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_BuildMetadataPath, JsonError)); + } + return MetaData; + } + if (!m_BuildMetadata.empty()) + { + CbObjectWriter MetaDataWriter(1024); + ForEachStrTok(m_BuildMetadata, ';', [&](std::string_view Pair) { + size_t SplitPos = Pair.find('='); + if (SplitPos == std::string::npos || SplitPos == 0) + { + throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair)); + } + MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1)); + return true; + }); + return MetaDataWriter.Save(); + } + } + else + { + if (!m_BuildMetadataPath.empty()) + { + throw zen::OptionParseException( + fmt::format("metadata-path option is only valid if creating a build\n{}", m_UploadOptions.help())); + } + if (!m_BuildMetadata.empty()) + { + throw zen::OptionParseException( + fmt::format("metadata option is only valid if creating a build\n{}", m_UploadOptions.help())); + } + } + return {}; + }; + BoostWorkerThreads = m_BoostWorkerThreads; try @@ -9671,6 +9777,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_ListOptions) { + MakeSafeAbsolutePathÍnPlace(m_ListQueryPath); + MakeSafeAbsolutePathÍnPlace(m_ListResultPath); + if (!m_ListResultPath.empty()) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); @@ -9685,14 +9794,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - std::filesystem::path ListQueryPath = MakeSafeAbsolutePath(m_ListQueryPath); - if (ToLower(ListQueryPath.extension().string()) == ".cbo") + if (ToLower(m_ListQueryPath.extension().string()) == ".cbo") { - QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(ListQueryPath)); + QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_ListQueryPath)); } else { - IoBuffer MetaDataJson = ReadFile(ListQueryPath).Flatten(); + IoBuffer MetaDataJson = ReadFile(m_ListQueryPath).Flatten(); std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); std::string JsonError; QueryObject = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); @@ -9707,19 +9815,22 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() - ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName - : MakeSafeAbsolutePath(m_ZenFolderPath); - CreateDirectories(ZenFolderPath); - auto _ = MakeGuard([ZenFolderPath]() { - if (CleanDirectory(ZenFolderPath, {})) + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + + CreateDirectories(m_ZenFolderPath); + auto _ = MakeGuard([this]() { + if (CleanDirectory(m_ZenFolderPath, {})) { std::error_code DummyEc; - RemoveDir(ZenFolderPath, DummyEc); + RemoveDir(m_ZenFolderPath, DummyEc); } }); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); @@ -9731,17 +9842,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath); - if (ToLower(ListResultPath.extension().string()) == ".cbo") + if (ToLower(m_ListResultPath.extension().string()) == ".cbo") { MemoryView ResponseView = Response.GetView(); - WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); + WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); } else { ExtendableStringBuilder<1024> SB; CompactBinaryToJson(Response.GetView(), SB); - WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); + WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); } } @@ -9752,130 +9862,53 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); - if (m_Path.empty()) - { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help())); - } - - if (m_CreateBuild) - { - if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty()) - { - throw zen::OptionParseException(fmt::format("Options for builds target are missing\n{}", m_UploadOptions.help())); - } - if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty()) - { - throw zen::OptionParseException(fmt::format("Conflicting options for builds target\n{}", m_UploadOptions.help())); - } - } - else - { - if (!m_BuildMetadataPath.empty()) - { - throw zen::OptionParseException( - fmt::format("metadata-path option is only valid if creating a build\n{}", m_UploadOptions.help())); - } - if (!m_BuildMetadata.empty()) - { - throw zen::OptionParseException( - fmt::format("metadata option is only valid if creating a build\n{}", m_UploadOptions.help())); - } - } - - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); + ParsePath(); if (m_BuildPartName.empty()) { - m_BuildPartName = Path.filename().string(); - } - - const bool GeneratedBuildId = m_BuildId.empty(); - if (GeneratedBuildId) - { - m_BuildId = Oid::NewOid().ToString(); - } - else if (m_BuildId.length() != Oid::StringLength) - { - throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); - } - else if (Oid::FromHexString(m_BuildId) == Oid::Zero) - { - throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); + m_BuildPartName = m_Path.filename().string(); } - const bool GeneratedBuildPartId = m_BuildPartId.empty(); - if (GeneratedBuildPartId) - { - m_BuildPartId = Oid::NewOid().ToString(); - } - else if (m_BuildPartId.length() != Oid::StringLength) + const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId(); + if (m_BuildId.empty()) { - throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); + m_BuildId = BuildId.ToString(); } - else if (Oid::FromHexString(m_BuildPartId) == Oid::Zero) + const Oid BuildPartId = m_BuildPartId.empty() ? Oid::NewOid() : ParseBuildPartId(); + if (m_BuildPartId.empty()) { - throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); + m_BuildPartId = BuildPartId.ToString(); } - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() - ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName - : MakeSafeAbsolutePath(m_ZenFolderPath); - CreateDirectories(ZenFolderPath); - auto _ = MakeGuard([ZenFolderPath]() { - if (CleanDirectory(ZenFolderPath, {})) + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + + CreateDirectories(m_ZenFolderPath); + auto _ = MakeGuard([this]() { + if (CleanDirectory(m_ZenFolderPath, {})) { std::error_code DummyEc; - RemoveDir(ZenFolderPath, DummyEc); + RemoveDir(m_ZenFolderPath, DummyEc); } }); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); - CbObject MetaData; - if (m_CreateBuild) - { - if (!m_BuildMetadataPath.empty()) - { - std::filesystem::path MetadataPath = MakeSafeAbsolutePath(m_BuildMetadataPath); - IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten(); - std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); - std::string JsonError; - MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); - if (!JsonError.empty()) - { - throw std::runtime_error( - fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_BuildMetadataPath, JsonError)); - } - } - if (!m_BuildMetadata.empty()) - { - CbObjectWriter MetaDataWriter(1024); - ForEachStrTok(m_BuildMetadata, ';', [&](std::string_view Pair) { - size_t SplitPos = Pair.find('='); - if (SplitPos == std::string::npos || SplitPos == 0) - { - throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair)); - } - MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1)); - return true; - }); - MetaData = MetaDataWriter.Save(); - } - } + CbObject MetaData = ParseBuildMetadata(); UploadFolder(Storage, BuildId, BuildPartId, m_BuildPartName, - Path, - ZenFolderPath, - m_ManifestPath.empty() ? std::filesystem::path{} : MakeSafeAbsolutePath(m_ManifestPath), + m_Path, + m_ZenFolderPath, + m_ManifestPath, m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, m_AllowMultiparts, @@ -9911,26 +9944,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); - ParseSystemOptions(); - - if (m_Path.empty()) - { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); - } - if (m_BuildId.empty()) - { - throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help())); - } - Oid BuildId = Oid::TryFromHexString(m_BuildId); - if (BuildId == Oid::Zero) - { - throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help())); - } + ParsePath(); - if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) - { - throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); - } + const Oid BuildId = ParseBuildId(); if (m_PostDownloadVerify && m_PrimeCacheOnly) { @@ -9948,34 +9964,26 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_WARN("ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled"); } - std::vector<Oid> BuildPartIds; - for (const std::string& BuildPartId : m_BuildPartIds) + std::vector<Oid> BuildPartIds = ParseBuildPartIds(); + + if (m_ZenFolderPath.empty()) { - BuildPartIds.push_back(Oid::TryFromHexString(BuildPartId)); - if (BuildPartIds.back() == Oid::Zero) - { - throw zen::OptionParseException( - fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, m_DownloadOptions.help())); - } + m_ZenFolderPath = m_Path / ZenFolderName; } - - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? Path / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); - - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); DownloadFolder(Storage, BuildId, BuildPartIds, m_BuildPartNames, - Path, - ZenFolderPath, - SystemRootDir, + m_Path, + m_ZenFolderPath, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests && !m_PrimeCacheOnly, m_Clean, @@ -9983,79 +9991,43 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_PrimeCacheOnly, m_EnableScavenging); - if (false) - { - ZEN_CONSOLE( - "{}:\n" - "Read: {}\n" - "Write: {}\n" - "Requests: {}\n" - "Avg Request Time: {}\n" - "Avg I/O Time: {}", - Storage.StorageName, - NiceBytes(StorageStats.TotalBytesRead.load()), - NiceBytes(StorageStats.TotalBytesWritten.load()), - StorageStats.TotalRequestCount.load(), - StorageStats.TotalExecutionTimeUs.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0, - StorageStats.TotalRequestCount.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0); - } - return AbortFlag ? 11 : 0; } if (SubOption == &m_DiffOptions) { - if (m_Path.empty()) - { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); - } - if (m_DiffPath.empty()) - { - throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); - } - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); - std::filesystem::path DiffPath = MakeSafeAbsolutePath(m_DiffPath); - DiffFolders(Path, DiffPath, m_OnlyChunked); + ParsePath(); + ParseDiffPath(); + + DiffFolders(m_Path, m_DiffPath, m_OnlyChunked); return AbortFlag ? 11 : 0; } if (SubOption == &m_FetchBlobOptions) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); - if (m_BlobHash.empty()) - { - throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); - } - - IoHash BlobHash; - if (!IoHash::TryParse(m_BlobHash, BlobHash)) - { - throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help())); - } + IoHash BlobHash = ParseBlobHash(); const Oid BuildId = Oid::FromHexString(m_BuildId); - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); - BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() - ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName - : MakeSafeAbsolutePath(m_ZenFolderPath); - CreateDirectories(ZenFolderPath); - auto _ = MakeGuard([ZenFolderPath]() { - if (CleanDirectory(ZenFolderPath, {})) + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + + CreateDirectories(m_ZenFolderPath); + auto _ = MakeGuard([this]() { + if (CleanDirectory(m_ZenFolderPath, {})) { std::error_code DummyEc; - RemoveDir(ZenFolderPath, DummyEc); + RemoveDir(m_ZenFolderPath, DummyEc); } }); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); uint64_t CompressedSize; uint64_t DecompressedSize; @@ -10075,41 +10047,34 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); - if (m_BuildId.empty()) - { - throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help())); - } - Oid BuildId = Oid::TryFromHexString(m_BuildId); - if (BuildId == Oid::Zero) - { - throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help())); - } + Oid BuildId = ParseBuildId(); if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) { throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); - BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() - ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName - : MakeSafeAbsolutePath(m_ZenFolderPath); - CreateDirectories(ZenFolderPath); - auto _ = MakeGuard([ZenFolderPath]() { - if (CleanDirectory(ZenFolderPath, {})) + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + + CreateDirectories(m_ZenFolderPath); + auto _ = MakeGuard([this]() { + if (CleanDirectory(m_ZenFolderPath, {})) { std::error_code DummyEc; - RemoveDir(ZenFolderPath, DummyEc); + RemoveDir(m_ZenFolderPath, DummyEc); } }); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); - Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); + const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(); ValidateStatistics ValidateStats; DownloadStatistics DownloadStats; @@ -10120,35 +10085,23 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_MultiTestDownloadOptions) { - SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(SystemRootDir); - CleanDirectory(SystemRootDir, {}); - auto _ = MakeGuard([&]() { DeleteDirectories(SystemRootDir); }); + m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_SystemRootDir); + CleanDirectory(m_SystemRootDir, {}); + auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); }); + + ParsePath(); - if (m_Path.empty()) + if (m_ZenFolderPath.empty()) { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); + m_ZenFolderPath = m_Path / ZenFolderName; } - - // m_StoragePath = "D:\\buildstorage"; - // m_Path = "F:\\Saved\\DownloadedBuilds\\++Fortnite+Main-CL-XXXXXXXX\\WindowsClient"; - // std::vector<std::string> BuildIdStrings{"07d3942f0e7f4ca1b13b0587", - // "07d394eed89d769f2254e75d", - // "07d3953f22fa3f8000fa6f0a", - // "07d3959df47ed1f42ddbe44c", - // "07d395fa7803d50804f14417", - // "07d3964f919d577a321a1fdd", - // "07d396a6ce875004e16b9528"}; - - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? Path / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); - - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) @@ -10162,9 +10115,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildId, {}, {}, - Path, - ZenFolderPath, - SystemRootDir, + m_Path, + m_ZenFolderPath, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, BuildIdString == m_BuildIds.front(), @@ -10184,47 +10137,41 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_TestOptions) { - SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(SystemRootDir); - CleanDirectory(SystemRootDir, {}); - auto _ = MakeGuard([&]() { DeleteDirectories(SystemRootDir); }); - - if (m_Path.empty()) - { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); - } + m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_SystemRootDir); + CleanDirectory(m_SystemRootDir, {}); + auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); }); - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); + ParsePath(); m_BuildId = Oid::NewOid().ToString(); - m_BuildPartName = Path.filename().string(); + m_BuildPartName = m_Path.filename().string(); m_BuildPartId = Oid::NewOid().ToString(); m_CreateBuild = true; const Oid BuildId = Oid::FromHexString(m_BuildId); const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath); - - if (m_OverrideHost.empty() && StoragePath.empty()) + if (m_OverrideHost.empty() && m_StoragePath.empty()) { - StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); - CreateDirectories(StoragePath); - CleanDirectory(StoragePath, {}); - m_StoragePath = StoragePath.generic_string(); + m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); + CreateDirectories(m_StoragePath); + CleanDirectory(m_StoragePath, {}); + m_StoragePath = m_StoragePath.generic_string(); } + auto __ = MakeGuard([&]() { - if (m_OverrideHost.empty() && StoragePath.empty()) + if (m_OverrideHost.empty() && m_StoragePath.empty()) { - DeleteDirectories(StoragePath); + DeleteDirectories(m_StoragePath); } }); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_test"); - const std::filesystem::path DownloadPath2 = Path.parent_path() / (m_BuildPartName + "_test2"); + const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_test"); + const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2"); auto ___ = MakeGuard([DownloadPath, DownloadPath2]() { CleanDirectory(DownloadPath, true); @@ -10233,10 +10180,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) DeleteDirectories(DownloadPath2); }); - const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? DownloadPath / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = m_Path / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); auto MakeMetaData = [](const Oid& BuildId) -> CbObject { CbObjectWriter BuildMetaDataWriter; @@ -10263,8 +10213,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildId, BuildPartId, m_BuildPartName, - Path, - ZenFolderPath, + m_Path, + m_ZenFolderPath, {}, m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, @@ -10285,8 +10235,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, - ZenFolderPath, - SystemRootDir, + DownloadPath / ZenFolderName, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, true, @@ -10309,8 +10259,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, - ZenFolderPath, - SystemRootDir, + DownloadPath / ZenFolderName, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -10428,8 +10378,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, - ZenFolderPath, - SystemRootDir, + DownloadPath / ZenFolderName, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -10459,7 +10409,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId2, m_BuildPartName, DownloadPath, - ZenFolderPath, + m_ZenFolderPath, {}, m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, @@ -10480,8 +10430,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, - ZenFolderPath, - SystemRootDir, + DownloadPath / ZenFolderName, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -10500,8 +10450,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId2}, {}, DownloadPath, - ZenFolderPath, - SystemRootDir, + DownloadPath / ZenFolderName, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -10520,8 +10470,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId2}, {}, DownloadPath, - ZenFolderPath, - SystemRootDir, + DownloadPath / ZenFolderName, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -10540,8 +10490,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath2, - ZenFolderPath, - SystemRootDir, + DownloadPath2 / ZenFolderName, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index f44e758ae..01d510e1b 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -25,13 +25,13 @@ public: private: cxxopts::Options m_Options{Name, Description}; - std::string m_SystemRootDir; + std::filesystem::path m_SystemRootDir; bool m_PlainProgress = false; bool m_Verbose = false; bool m_BoostWorkerThreads = false; - std::string m_ZenFolderPath; + std::filesystem::path m_ZenFolderPath; // cloud builds std::string m_OverrideHost; @@ -41,29 +41,29 @@ private: std::string m_Bucket; // file storage - std::string m_StoragePath; - bool m_WriteMetadataAsJson = false; + std::filesystem::path m_StoragePath; + bool m_WriteMetadataAsJson = false; // cache std::string m_ZenCacheHost; bool m_PrimeCacheOnly = false; - std::string m_BuildId; - bool m_CreateBuild = false; - std::string m_BuildMetadataPath; - std::string m_BuildMetadata; - std::string m_BuildPartName; // Defaults to name of leaf folder in m_Path - std::string m_BuildPartId; // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName - bool m_Clean = false; - uint8_t m_BlockReuseMinPercentLimit = 85; - bool m_AllowMultiparts = true; - bool m_AllowPartialBlockRequests = true; - std::string m_ManifestPath; + std::string m_BuildId; + bool m_CreateBuild = false; + std::filesystem::path m_BuildMetadataPath; + std::string m_BuildMetadata; + std::string m_BuildPartName; // Defaults to name of leaf folder in m_Path + std::string m_BuildPartId; // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName + bool m_Clean = false; + uint8_t m_BlockReuseMinPercentLimit = 85; + bool m_AllowMultiparts = true; + bool m_AllowPartialBlockRequests = true; + std::string m_ManifestPath; // Not a std::filesystem::path since it can be relative to m_Path // Direct access token (may expire) - std::string m_AccessToken; - std::string m_AccessTokenEnv; - std::string m_AccessTokenPath; + std::string m_AccessToken; + std::string m_AccessTokenEnv; + std::filesystem::path m_AccessTokenPath; // Auth manager token encryption std::string m_EncryptionKey; // 256 bit AES encryption key @@ -88,11 +88,11 @@ private: std::string m_ListNamespacesResultPath; bool m_ListNamespacesRecursive = false; - cxxopts::Options m_ListOptions{"list", "List available builds"}; - std::string m_ListQueryPath; - std::string m_ListResultPath; + cxxopts::Options m_ListOptions{"list", "List available builds"}; + std::filesystem::path m_ListQueryPath; + std::filesystem::path m_ListResultPath; - std::string m_Path; + std::filesystem::path m_Path; cxxopts::Options m_UploadOptions{"upload", "Upload a folder"}; uint64_t m_FindBlockMaxCount = 10000; @@ -104,9 +104,9 @@ private: bool m_PostDownloadVerify = false; bool m_EnableScavenging = true; - cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"}; - std::string m_DiffPath; - bool m_OnlyChunked = false; + cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"}; + std::filesystem::path m_DiffPath; + bool m_OnlyChunked = false; cxxopts::Options m_FetchBlobOptions{"fetch-blob", "Fetch a blob from remote store"}; std::string m_BlobHash; diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp index 185edc35d..4412eaf34 100644 --- a/src/zen/cmds/cache_cmd.cpp +++ b/src/zen/cmds/cache_cmd.cpp @@ -625,22 +625,20 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) HttpClient Http(m_HostName); - std::filesystem::path TargetPath; if (!m_OutputPath.empty()) { - TargetPath = std::filesystem::path(m_OutputPath); - if (IsDir(TargetPath)) + if (IsDir(m_OutputPath)) { - TargetPath = TargetPath / (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); + m_OutputPath = m_OutputPath / (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); } else { - CreateDirectories(TargetPath.parent_path()); + CreateDirectories(m_OutputPath.parent_path()); } } - if (TargetPath.empty()) + if (m_OutputPath.empty()) { - TargetPath = (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); + m_OutputPath = (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); } std::string Url = fmt::format("/z$/{}/{}/{}", m_Namespace, m_Bucket, m_ValueKey); @@ -670,17 +668,17 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - WriteFile(TargetPath, IoBuffer(IoBuffer::Wrap, StringData.data(), StringData.length())); - ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(StringData.length()), TargetPath, ToString(ChunkData.GetContentType())); + WriteFile(m_OutputPath, IoBuffer(IoBuffer::Wrap, StringData.data(), StringData.length())); + ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(StringData.length()), m_OutputPath, ToString(ChunkData.GetContentType())); } } else { - if (!MoveToFile(TargetPath, ChunkData)) + if (!MoveToFile(m_OutputPath, ChunkData)) { - WriteFile(TargetPath, ChunkData); + WriteFile(m_OutputPath, ChunkData); } - ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(ChunkData.GetSize()), TargetPath, ToString(ChunkData.GetContentType())); + ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(ChunkData.GetSize()), m_OutputPath, ToString(ChunkData.GetContentType())); } } else diff --git a/src/zen/cmds/cache_cmd.h b/src/zen/cmds/cache_cmd.h index 73702cada..b8a319359 100644 --- a/src/zen/cmds/cache_cmd.h +++ b/src/zen/cmds/cache_cmd.h @@ -108,15 +108,15 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_Namespace; - std::string m_Bucket; - std::string m_ValueKey; - std::string m_AttachmentHash; - std::string m_OutputPath; - bool m_AsText = false; - bool m_Decompress = true; + cxxopts::Options m_Options{Name, Description}; + std::string m_HostName; + std::string m_Namespace; + std::string m_Bucket; + std::string m_ValueKey; + std::string m_AttachmentHash; + std::filesystem::path m_OutputPath; + bool m_AsText = false; + bool m_Decompress = true; }; } // namespace zen diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp index 53e80c896..e86b6964c 100644 --- a/src/zen/cmds/copy_cmd.cpp +++ b/src/zen/cmds/copy_cmd.cpp @@ -42,11 +42,8 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_CopyTarget.empty()) throw std::runtime_error("No target specified"); - std::filesystem::path FromPath; - std::filesystem::path ToPath; - - FromPath = m_CopySource; - ToPath = m_CopyTarget; + std::filesystem::path FromPath = m_CopySource; + std::filesystem::path ToPath = m_CopyTarget; std::error_code Ec; std::filesystem::path FromCanonical = std::filesystem::canonical(FromPath, Ec); diff --git a/src/zen/cmds/copy_cmd.h b/src/zen/cmds/copy_cmd.h index 876aff3f5..e9735c159 100644 --- a/src/zen/cmds/copy_cmd.h +++ b/src/zen/cmds/copy_cmd.h @@ -19,11 +19,11 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"copy", "Copy files efficiently"}; - std::string m_CopySource; - std::string m_CopyTarget; - bool m_NoClone = false; - bool m_MustClone = false; + cxxopts::Options m_Options{"copy", "Copy files efficiently"}; + std::filesystem::path m_CopySource; + std::filesystem::path m_CopyTarget; + bool m_NoClone = false; + bool m_MustClone = false; }; } // namespace zen diff --git a/src/zen/cmds/dedup_cmd.h b/src/zen/cmds/dedup_cmd.h index c4f0068e4..2721be2b9 100644 --- a/src/zen/cmds/dedup_cmd.h +++ b/src/zen/cmds/dedup_cmd.h @@ -21,8 +21,8 @@ public: private: cxxopts::Options m_Options{"dedup", "Deduplicate files"}; std::vector<std::string> m_Positional; - std::string m_DedupSource; - std::string m_DedupTarget; + std::filesystem::path m_DedupSource; + std::filesystem::path m_DedupTarget; size_t m_SizeThreshold = 1024 * 1024; }; diff --git a/src/zen/cmds/status_cmd.cpp b/src/zen/cmds/status_cmd.cpp index 88c0b22a2..2b507e43d 100644 --- a/src/zen/cmds/status_cmd.cpp +++ b/src/zen/cmds/status_cmd.cpp @@ -32,17 +32,16 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) uint16_t EffectivePort = 0; if (!m_DataDir.empty()) { - std::filesystem::path DataDir = StringToPath(m_DataDir); - if (!IsFile(DataDir / ".lock")) + if (!IsFile(m_DataDir / ".lock")) { - ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); + ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir); return 1; } - LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock"))); + LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); std::string Reason; if (!ValidateLockFileInfo(Info, Reason)) { - ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason); + ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); return 1; } EffectivePort = Info.EffectiveListenPort; diff --git a/src/zen/cmds/status_cmd.h b/src/zen/cmds/status_cmd.h index 00ad0e758..46bda9ee6 100644 --- a/src/zen/cmds/status_cmd.h +++ b/src/zen/cmds/status_cmd.h @@ -20,9 +20,9 @@ public: private: int GetLockFileEffectivePort() const; - cxxopts::Options m_Options{"status", "Show zen status"}; - uint16_t m_Port = 0; - std::string m_DataDir; + cxxopts::Options m_Options{"status", "Show zen status"}; + uint16_t m_Port = 0; + std::filesystem::path m_DataDir; }; } // namespace zen diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp index aacc115a0..f3bf2f66d 100644 --- a/src/zen/cmds/up_cmd.cpp +++ b/src/zen/cmds/up_cmd.cpp @@ -77,15 +77,13 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path ProgramBaseDir = StringToPath(m_ProgramBaseDir); - - if (ProgramBaseDir.empty()) + if (m_ProgramBaseDir.empty()) { std::filesystem::path ExePath = zen::GetRunningExecutablePath(); - ProgramBaseDir = ExePath.parent_path(); + m_ProgramBaseDir = ExePath.parent_path(); } ZenServerEnvironment ServerEnvironment; - ServerEnvironment.Initialize(ProgramBaseDir); + ServerEnvironment.Initialize(m_ProgramBaseDir); ZenServerInstance Server(ServerEnvironment); std::string ServerArguments = GlobalOptions.PassthroughCommandLine; if ((m_Port != 0) && (ServerArguments.find("--port"sv) == std::string::npos)) @@ -155,20 +153,18 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Instance.Sweep(); ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port); - std::filesystem::path DataDir = StringToPath(m_DataDir); - - if (!DataDir.empty()) + if (!m_DataDir.empty()) { - if (!IsFile(DataDir / ".lock")) + if (!IsFile(m_DataDir / ".lock")) { - ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); + ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir); return 1; } - LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock"))); + LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); std::string Reason; if (!ValidateLockFileInfo(Info, Reason)) { - ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason); + ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); return 1; } Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort); @@ -218,27 +214,24 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Instance.Initialize(); ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port); - std::filesystem::path ProgramBaseDir = StringToPath(m_ProgramBaseDir); - if (ProgramBaseDir.empty()) + if (m_ProgramBaseDir.empty()) { std::filesystem::path ExePath = GetRunningExecutablePath(); - ProgramBaseDir = ExePath.parent_path(); + m_ProgramBaseDir = ExePath.parent_path(); } - std::filesystem::path DataDir = StringToPath(m_DataDir); - - if (!DataDir.empty()) + if (!m_DataDir.empty()) { - if (!IsFile(DataDir / ".lock")) + if (!IsFile(m_DataDir / ".lock")) { - ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); + ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir); return 1; } - LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock"))); + LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); std::string Reason; if (!ValidateLockFileInfo(Info, Reason)) { - ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason); + ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); return 1; } Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort); @@ -251,7 +244,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) try { ZenServerEnvironment ServerEnvironment; - ServerEnvironment.Initialize(ProgramBaseDir); + ServerEnvironment.Initialize(m_ProgramBaseDir); ZenServerInstance Server(ServerEnvironment); Server.AttachToRunningServer(EntryPort); @@ -316,7 +309,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_ForceTerminate) { // Try to find the running executable by path name - std::filesystem::path ServerExePath = ProgramBaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL; + std::filesystem::path ServerExePath = m_ProgramBaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL; ProcessHandle RunningProcess; if (std::error_code Ec = FindProcess(ServerExePath, RunningProcess); !Ec) { diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h index 32d8ddab3..c9af16749 100644 --- a/src/zen/cmds/up_cmd.h +++ b/src/zen/cmds/up_cmd.h @@ -18,11 +18,11 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"up", "Bring up zen service"}; - uint16_t m_Port = 0; - bool m_ShowConsole = false; - bool m_ShowLog = false; - std::string m_ProgramBaseDir; + cxxopts::Options m_Options{"up", "Bring up zen service"}; + uint16_t m_Port = 0; + bool m_ShowConsole = false; + bool m_ShowLog = false; + std::filesystem::path m_ProgramBaseDir; }; class AttachCommand : public ZenCmdBase @@ -35,10 +35,10 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"}; - uint16_t m_Port = 0; - int m_OwnerPid = 0; - std::string m_DataDir; + cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"}; + uint16_t m_Port = 0; + int m_OwnerPid = 0; + std::filesystem::path m_DataDir; }; class DownCommand : public ZenCmdBase @@ -51,11 +51,11 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"down", "Bring down zen service"}; - uint16_t m_Port = 0; - bool m_ForceTerminate = false; - std::string m_ProgramBaseDir; - std::string m_DataDir; + cxxopts::Options m_Options{"down", "Bring down zen service"}; + uint16_t m_Port = 0; + bool m_ForceTerminate = false; + std::filesystem::path m_ProgramBaseDir; + std::filesystem::path m_DataDir; }; } // namespace zen diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp new file mode 100644 index 000000000..2b4e9ab3c --- /dev/null +++ b/src/zen/cmds/wipe_cmd.cpp @@ -0,0 +1,575 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "wipe_cmd.h" + +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/string.h> +#include <zencore/timer.h> +#include <zencore/trace.h> +#include <zenutil/parallellwork.h> +#include <zenutil/workerpools.h> + +#include <signal.h> + +#include <iostream> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <tsl/robin_map.h> +#include <tsl/robin_set.h> +ZEN_THIRD_PARTY_INCLUDES_END + +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#else +# include <fcntl.h> +# include <sys/file.h> +# include <sys/stat.h> +# include <unistd.h> +#endif + +namespace zen { + +namespace { + static std::atomic<bool> AbortFlag = false; + static bool IsVerbose = false; + static bool Quiet = false; + static bool UsePlainProgress = false; + const bool SingleThreaded = false; + bool BoostWorkerThreads = true; + + WorkerThreadPool& GetIOWorkerPool() + { + return SingleThreaded ? GetSyncWorkerPool() + : BoostWorkerThreads ? GetLargeWorkerPool(EWorkloadType::Burst) + : GetMediumWorkerPool(EWorkloadType::Burst); + } + +#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ + if (IsVerbose) \ + { \ + ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__); \ + } + + static void SignalCallbackHandler(int SigNum) + { + if (SigNum == SIGINT) + { + AbortFlag = true; + } +#if ZEN_PLATFORM_WINDOWS + if (SigNum == SIGBREAK) + { + AbortFlag = true; + } +#endif // ZEN_PLATFORM_WINDOWS + } + + bool IsReadOnly(uint32_t Attributes) + { +#if ZEN_PLATFORM_WINDOWS + return IsFileAttributeReadOnly(Attributes); +#else + return IsFileModeReadOnly(Attributes); +#endif + } + + bool IsFileWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + bool Result = IsFile(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + Result = IsFile(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + return Result; + } + + bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly) + { + std::error_code Ec; + bool Result = SetFileReadOnly(Path, ReadOnly, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsFileWithRetry(Path)) + { + return false; + } + Ec.clear(); + Result = SetFileReadOnly(Path, ReadOnly, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + return Result; + } + + void RemoveFileWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + RemoveFile(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsFileWithRetry(Path)) + { + return; + } + Ec.clear(); + RemoveFile(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + + void RemoveDirWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + RemoveDir(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsDir(Path)) + { + return; + } + Ec.clear(); + RemoveDir(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + + bool CleanDirectory(const std::filesystem::path& Path, + std::span<const std::string_view> ExcludeDirectories, + bool RemoveReadonly, + bool Dryrun) + { + ZEN_TRACE_CPU("CleanDirectory"); + Stopwatch Timer; + + ProgressBar Progress(UsePlainProgress); + + std::atomic<bool> CleanWipe = true; + std::atomic<uint64_t> DiscoveredItemCount = 0; + std::atomic<uint64_t> DeletedItemCount = 0; + std::atomic<uint64_t> DeletedByteCount = 0; + std::atomic<uint64_t> FailedDeleteCount = 0; + + std::vector<std::filesystem::path> SubdirectoriesToDelete; + tsl::robin_map<IoHash, size_t, IoHash::Hasher> SubdirectoriesToDeleteLookup; + tsl::robin_set<IoHash, IoHash::Hasher> SubdirectoriesToKeep; + RwLock SubdirectoriesLock; + + auto AddFoundDirectory = [&](std::filesystem::path Directory, bool Keep) -> bool { + bool Added = false; + if (Keep) + { + bool IsLeaf = true; + while (Directory != Path) + { + const std::string DirectoryString = Directory.generic_string(); + IoHash DirectoryNameHash = IoHash::HashBuffer(DirectoryString.data(), DirectoryString.length()); + RwLock::ExclusiveLockScope _(SubdirectoriesLock); + if (auto It = SubdirectoriesToKeep.find(DirectoryNameHash); It == SubdirectoriesToKeep.end()) + { + SubdirectoriesToKeep.insert(DirectoryNameHash); + if (IsLeaf) + { + Added = true; + } + } + else + { + break; + } + Directory = Directory.parent_path(); + IsLeaf = false; + } + } + else + { + bool IsLeaf = true; + while (Directory != Path) + { + const std::string DirectoryString = Directory.generic_string(); + IoHash DirectoryNameHash = IoHash::HashBuffer(DirectoryString.data(), DirectoryString.length()); + RwLock::ExclusiveLockScope _(SubdirectoriesLock); + if (SubdirectoriesToKeep.contains(DirectoryNameHash)) + { + break; + } + if (auto It = SubdirectoriesToDeleteLookup.find(DirectoryNameHash); It == SubdirectoriesToDeleteLookup.end()) + { + SubdirectoriesToDeleteLookup.insert({DirectoryNameHash, SubdirectoriesToDelete.size()}); + SubdirectoriesToDelete.push_back(Directory); + if (IsLeaf) + { + Added = true; + } + } + else + { + break; + } + Directory = Directory.parent_path(); + IsLeaf = false; + } + } + return Added; + }; + + ParallellWork Work(AbortFlag); + + struct AsyncVisitor : public GetDirectoryContentVisitor + { + AsyncVisitor(const std::filesystem::path& InPath, + std::atomic<bool>& InCleanWipe, + std::atomic<uint64_t>& InDiscoveredItemCount, + std::atomic<uint64_t>& InDeletedItemCount, + std::atomic<uint64_t>& InDeletedByteCount, + std::atomic<uint64_t>& InFailedDeleteCount, + std::span<const std::string_view> InExcludeDirectories, + bool InRemoveReadonly, + bool InDryrun, + const std::function<bool(std::filesystem::path, bool)>& InAddFoundDirectoryFunc) + : Path(InPath) + , CleanWipe(InCleanWipe) + , DiscoveredItemCount(InDiscoveredItemCount) + , DeletedItemCount(InDeletedItemCount) + , DeletedByteCount(InDeletedByteCount) + , FailedDeleteCount(InFailedDeleteCount) + , ExcludeDirectories(InExcludeDirectories) + , RemoveReadonly(InRemoveReadonly) + , Dryrun(InDryrun) + , AddFoundDirectoryFunc(InAddFoundDirectoryFunc) + { + } + virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override + { + ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory"); + if (!AbortFlag) + { + if (!RelativeRoot.empty()) + { + DiscoveredItemCount++; + } + if (!Content.FileNames.empty()) + { + DiscoveredItemCount += Content.FileNames.size(); + + const std::string RelativeRootString = RelativeRoot.generic_string(); + bool RemoveContent = true; + for (const std::string_view ExcludeDirectory : ExcludeDirectories) + { + if (RelativeRootString.starts_with(ExcludeDirectory)) + { + if (RelativeRootString.length() > ExcludeDirectory.length()) + { + const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()]; + if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' || + MaybePathDelimiter == std::filesystem::path::preferred_separator) + { + RemoveContent = false; + break; + } + } + else + { + RemoveContent = false; + break; + } + } + } + + const std::filesystem::path ParentPath = Path / RelativeRoot; + bool KeepDirectory = RelativeRoot.empty(); + + if (RemoveContent) + { + ZEN_TRACE_CPU("DeleteFiles"); + uint64_t RemovedCount = 0; + for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++) + { + const std::filesystem::path& FileName = Content.FileNames[FileIndex]; + const std::filesystem::path FilePath = (ParentPath / FileName).make_preferred(); + try + { + const uint32_t Attributes = Content.FileAttributes[FileIndex]; + const bool IsReadonly = IsReadOnly(Attributes); + bool RemoveFile = false; + if (IsReadonly) + { + if (RemoveReadonly) + { + if (!Dryrun) + { + SetFileReadOnlyWithRetry(FilePath, false); + } + RemoveFile = true; + } + } + else + { + RemoveFile = true; + } + + if (RemoveFile) + { + if (!Dryrun) + { + RemoveFileWithRetry(FilePath); + } + DeletedItemCount++; + DeletedByteCount += Content.FileSizes[FileIndex]; + RemovedCount++; + ZEN_CONSOLE_VERBOSE("Removed file {}", FilePath); + } + else + { + ZEN_CONSOLE_VERBOSE("Skipped readonly file {}", FilePath); + KeepDirectory = true; + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing file {}. Reason: {}", FilePath, Ex.what()); + FailedDeleteCount++; + CleanWipe = false; + KeepDirectory = true; + } + } + ZEN_CONSOLE_VERBOSE("Removed {} files in {}", RemovedCount, ParentPath); + } + else + { + ZEN_CONSOLE_VERBOSE("Skipped removal of {} files in {}", Content.FileNames.size(), ParentPath); + } + bool Added = AddFoundDirectoryFunc(ParentPath, KeepDirectory); + if (Added) + { + ZEN_CONSOLE_VERBOSE("{} directory {}", KeepDirectory ? "Keeping" : "Removing", ParentPath); + } + } + } + } + const std::filesystem::path& Path; + std::atomic<bool>& CleanWipe; + std::atomic<uint64_t>& DiscoveredItemCount; + std::atomic<uint64_t>& DeletedItemCount; + std::atomic<uint64_t>& DeletedByteCount; + std::atomic<uint64_t>& FailedDeleteCount; + std::span<const std::string_view> ExcludeDirectories; + const bool RemoveReadonly; + const bool Dryrun; + std::function<bool(std::filesystem::path, bool)> AddFoundDirectoryFunc; + } Visitor(Path, + CleanWipe, + DiscoveredItemCount, + DeletedItemCount, + DeletedByteCount, + FailedDeleteCount, + ExcludeDirectories, + RemoveReadonly, + Dryrun, + AddFoundDirectory); + + uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs(); + + GetDirectoryContent(Path, + DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | + DirectoryContentFlags::IncludeFileSizes | DirectoryContentFlags::IncludeAttributes, + Visitor, + GetIOWorkerPool(), + Work.PendingWork()); + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, ptrdiff_t PendingWork) { + if (Quiet) + { + return; + } + ZEN_UNUSED(IsAborted, PendingWork); + LastUpdateTimeMs = Timer.GetElapsedTimeMs(); + + uint64_t Deleted = DeletedItemCount.load(); + uint64_t DeletedBytes = DeletedByteCount.load(); + uint64_t Discovered = DiscoveredItemCount.load(); + Progress.UpdateState({.Task = "Removing files ", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = Discovered, + .RemainingCount = Discovered - Deleted}, + false); + }); + + std::vector<std::filesystem::path> DirectoriesToDelete; + DirectoriesToDelete.reserve(SubdirectoriesToDelete.size()); + for (auto It : SubdirectoriesToDeleteLookup) + { + const IoHash& DirHash = It.first; + if (auto KeepIt = SubdirectoriesToKeep.find(DirHash); KeepIt == SubdirectoriesToKeep.end()) + { + DirectoriesToDelete.emplace_back(std::move(SubdirectoriesToDelete[It.second])); + } + } + + std::sort(DirectoriesToDelete.begin(), + DirectoriesToDelete.end(), + [](const std::filesystem::path& Lhs, const std::filesystem::path& Rhs) { + return Lhs.string().length() > Rhs.string().length(); + }); + + for (size_t SubDirectoryIndex = 0; SubDirectoryIndex < DirectoriesToDelete.size(); SubDirectoryIndex++) + { + ZEN_TRACE_CPU("DeleteDirs"); + const std::filesystem::path& DirectoryToDelete = DirectoriesToDelete[SubDirectoryIndex]; + try + { + if (!Dryrun) + { + RemoveDirWithRetry(DirectoryToDelete); + } + ZEN_CONSOLE_VERBOSE("Removed directory {}", DirectoryToDelete); + DeletedItemCount++; + } + catch (const std::exception& Ex) + { + if (!Quiet) + { + ZEN_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what()); + } + CleanWipe = false; + FailedDeleteCount++; + } + + uint64_t NowMs = Timer.GetElapsedTimeMs(); + if ((NowMs - LastUpdateTimeMs) >= (UsePlainProgress ? 5000 : 200)) + { + LastUpdateTimeMs = NowMs; + + uint64_t Deleted = DeletedItemCount.load(); + uint64_t DeletedBytes = DeletedByteCount.load(); + uint64_t Discovered = DiscoveredItemCount.load(); + Progress.UpdateState({.Task = "Removing folders", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = DirectoriesToDelete.size(), + .RemainingCount = DirectoriesToDelete.size() - SubDirectoryIndex}, + false); + } + } + + Progress.Finish(); + + uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); + if (!Quiet) + { + ZEN_CONSOLE("Wiped folder '{}' {} ({}) ({} failed) in {}", + Path, + DeletedItemCount.load(), + NiceBytes(DeletedByteCount.load()), + FailedDeleteCount.load(), + NiceTimeSpanMs(ElapsedTimeMs)); + } + if (FailedDeleteCount.load() > 0) + { + throw std::runtime_error(fmt::format("Failed to delete {} files/directories in '{}'", FailedDeleteCount.load(), Path)); + } + return CleanWipe; + } +} // namespace + +WipeCommand::WipeCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "d", "directory", "Directory to wipe", cxxopts::value(m_Directory), "<directory>"); + m_Options.add_option("", "r", "keep-readonly", "Leave read-only files", cxxopts::value(m_KeepReadOnlyFiles), "<keepreadonly>"); + m_Options.add_option("", "q", "quiet", "Reduce output to console", cxxopts::value(m_Quiet), "<quiet>"); + m_Options.add_option("", "y", "yes", "Don't query for confirmation", cxxopts::value(m_Yes), "<yes>"); + m_Options.add_option("", "", "dryrun", "Do a dry run without deleting anything", cxxopts::value(m_Dryrun), "<dryrun>"); + m_Options.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), "<progress>"); + m_Options.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>"); + m_Options.add_option("", + "", + "boost-workers", + "Increase the number of worker threads - may cause computer to be less responsive", + cxxopts::value(m_BoostWorkerThreads), + "<boostworkers>"); + + m_Options.parse_positional({"directory"}); +} + +WipeCommand::~WipeCommand() = default; + +int +WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + signal(SIGINT, SignalCallbackHandler); +#if ZEN_PLATFORM_WINDOWS + signal(SIGBREAK, SignalCallbackHandler); +#endif // ZEN_PLATFORM_WINDOWS + + if (!ZenCmdBase::ParseOptions(argc, argv)) + { + return 0; + } + + Quiet = m_Quiet; + IsVerbose = m_Verbose; + UsePlainProgress = IsVerbose || m_PlainProgress; + BoostWorkerThreads = m_BoostWorkerThreads; + + MakeSafeAbsolutePathÍnPlace(m_Directory); + + if (!IsDir(m_Directory)) + { + return 0; + } + + while (!m_Yes) + { + const std::string Prompt = fmt::format("Do you want to wipe directory '{}'? (yes/no) ", m_Directory); + printf("%s", Prompt.c_str()); + std::string Reponse; + std::getline(std::cin, Reponse); + Reponse = ToLower(Reponse); + if (Reponse == "y" || Reponse == "yes") + { + m_Yes = true; + } + else if (Reponse == "n" || Reponse == "no") + { + return 0; + } + } + + try + { + CleanDirectory(m_Directory, {}, !m_KeepReadOnlyFiles, m_Dryrun); + } + catch (std::exception& Ex) + { + if (!m_Quiet) + { + ZEN_ERROR("{}", Ex.what()); + } + return 3; + } + + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/wipe_cmd.h b/src/zen/cmds/wipe_cmd.h new file mode 100644 index 000000000..0e910bb81 --- /dev/null +++ b/src/zen/cmds/wipe_cmd.h @@ -0,0 +1,36 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +/** Wipe directories + */ +class WipeCommand : public ZenCmdBase +{ +public: + static constexpr char Name[] = "wipe"; + static constexpr char Description[] = "Wipe the contents of a directory"; + + WipeCommand(); + ~WipeCommand(); + + virtual cxxopts::Options& Options() override { return m_Options; } + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } + +private: + cxxopts::Options m_Options{Name, Description}; + std::filesystem::path m_Directory; + bool m_KeepReadOnlyFiles = true; + bool m_Quiet = false; + bool m_Yes = false; + bool m_PlainProgress = false; + bool m_Verbose = false; + bool m_Dryrun = false; + bool m_BoostWorkerThreads = false; +}; + +} // namespace zen diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp index 5f3f8f7ca..773734f12 100644 --- a/src/zen/cmds/workspaces_cmd.cpp +++ b/src/zen/cmds/workspaces_cmd.cpp @@ -139,18 +139,16 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_HostName = ResolveTargetHostSpec(m_HostName); - std::filesystem::path SystemRootDir = StringToPath(m_SystemRootDir); - - if (SystemRootDir.empty()) + if (m_SystemRootDir.empty()) { - SystemRootDir = PickDefaultSystemRootDirectory(); - if (SystemRootDir.empty()) + m_SystemRootDir = PickDefaultSystemRootDirectory(); + if (m_SystemRootDir.empty()) { throw zen::OptionParseException("unable to resolve system root directory"); } } - std::filesystem::path StatePath = SystemRootDir / "workspaces"; + std::filesystem::path StatePath = m_SystemRootDir / "workspaces"; if (!ParseOptions(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data())) { @@ -392,18 +390,20 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** m_HostName = ResolveTargetHostSpec(m_HostName); - std::filesystem::path SystemRootDir = StringToPath(m_SystemRootDir); - - if (SystemRootDir.empty()) + if (m_SystemRootDir.empty()) { - SystemRootDir = PickDefaultSystemRootDirectory(); - if (SystemRootDir.empty()) + m_SystemRootDir = PickDefaultSystemRootDirectory(); + if (m_SystemRootDir.empty()) { throw zen::OptionParseException("unable to resolve system root directory"); } } + else + { + MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); + } - std::filesystem::path StatePath = SystemRootDir / "workspaces"; + std::filesystem::path StatePath = m_SystemRootDir / "workspaces"; if (!ParseOptions(*SubOption, gsl::narrow<int>(SubCommandArguments.size()), SubCommandArguments.data())) { @@ -412,8 +412,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** if (SubOption == &m_CreateOptions) { - std::filesystem::path WorkspaceRoot = StringToPath(m_WorkspaceRoot); - if (WorkspaceRoot.empty()) + if (m_WorkspaceRoot.empty()) { if (m_WorkspaceId.empty()) { @@ -432,15 +431,15 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ZEN_CONSOLE("Workspace {} does not exist", m_WorkspaceId); return 0; } - WorkspaceRoot = WorkspaceConfig.RootPath; + m_WorkspaceRoot = WorkspaceConfig.RootPath; } else { - RemoveTrailingPathSeparator(WorkspaceRoot); + RemoveTrailingPathSeparator(m_WorkspaceRoot); if (m_WorkspaceId.empty()) { - m_WorkspaceId = Workspaces::PathToId(WorkspaceRoot).ToString(); - ZEN_CONSOLE("Using generated workspace id {} from path '{}'", m_WorkspaceId, WorkspaceRoot); + m_WorkspaceId = Workspaces::PathToId(m_WorkspaceRoot).ToString(); + ZEN_CONSOLE("Using generated workspace id {} from path '{}'", m_WorkspaceId, m_WorkspaceRoot); } else { @@ -449,25 +448,23 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** throw zen::OptionParseException(fmt::format("workspace id '{}' is invalid", m_WorkspaceId)); } } - if (Workspaces::AddWorkspace(Log(), StatePath, {.Id = Oid::FromHexString(m_WorkspaceId), .RootPath = WorkspaceRoot})) + if (Workspaces::AddWorkspace(Log(), StatePath, {.Id = Oid::FromHexString(m_WorkspaceId), .RootPath = m_WorkspaceRoot})) { - ZEN_CONSOLE("Created workspace {} using root path '{}'", m_WorkspaceId, WorkspaceRoot); + ZEN_CONSOLE("Created workspace {} using root path '{}'", m_WorkspaceId, m_WorkspaceRoot); } else { - ZEN_CONSOLE("Using existing workspace {} with root path '{}'", m_WorkspaceId, WorkspaceRoot); + ZEN_CONSOLE("Using existing workspace {} with root path '{}'", m_WorkspaceId, m_WorkspaceRoot); } } - std::filesystem::path SharePath = StringToPath(m_SharePath); - - RemoveLeadingPathSeparator(SharePath); - RemoveTrailingPathSeparator(SharePath); + RemoveLeadingPathSeparator(m_SharePath); + RemoveTrailingPathSeparator(m_SharePath); if (m_ShareId.empty()) { - m_ShareId = Workspaces::PathToId(SharePath).ToString(); - ZEN_CONSOLE("Using generated share id {}, for path '{}'", m_ShareId, SharePath); + m_ShareId = Workspaces::PathToId(m_SharePath).ToString(); + ZEN_CONSOLE("Using generated share id {}, for path '{}'", m_ShareId, m_SharePath); } if (Oid::TryFromHexString(m_ShareId) == Oid::Zero) @@ -476,8 +473,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** } if (Workspaces::AddWorkspaceShare(Log(), - WorkspaceRoot, - {.Id = Oid::FromHexString(m_ShareId), .SharePath = SharePath, .Alias = m_Alias})) + m_WorkspaceRoot, + {.Id = Oid::FromHexString(m_ShareId), .SharePath = m_SharePath, .Alias = m_Alias})) { if (!m_HostName.empty()) { diff --git a/src/zen/cmds/workspaces_cmd.h b/src/zen/cmds/workspaces_cmd.h index 86452e25e..d85d8f7d8 100644 --- a/src/zen/cmds/workspaces_cmd.h +++ b/src/zen/cmds/workspaces_cmd.h @@ -21,9 +21,9 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_SystemRootDir; + cxxopts::Options m_Options{Name, Description}; + std::string m_HostName; + std::filesystem::path m_SystemRootDir; std::string m_Verb; // create, info, remove @@ -53,17 +53,17 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_SystemRootDir; - std::string m_WorkspaceId; - std::string m_WorkspaceRoot; - std::string m_Verb; // create, info, remove - std::string m_ShareId; - std::string m_Alias; - - cxxopts::Options m_CreateOptions{"create", "Create a workspace share"}; - std::string m_SharePath; + cxxopts::Options m_Options{Name, Description}; + std::string m_HostName; + std::filesystem::path m_SystemRootDir; + std::string m_WorkspaceId; + std::filesystem::path m_WorkspaceRoot; + std::string m_Verb; // create, info, remove + std::string m_ShareId; + std::string m_Alias; + + cxxopts::Options m_CreateOptions{"create", "Create a workspace share"}; + std::filesystem::path m_SharePath; bool m_Refresh = false; diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 5ce0a89ec..e442f8a4b 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -23,6 +23,7 @@ #include "cmds/up_cmd.h" #include "cmds/version_cmd.h" #include "cmds/vfs_cmd.h" +#include "cmds/wipe_cmd.h" #include "cmds/workspaces_cmd.h" #include <zencore/callstack.h> @@ -557,6 +558,7 @@ main(int argc, char** argv) UpCommand UpCmd; VersionCommand VersionCmd; VfsCommand VfsCmd; + WipeCommand WipeCmd; WorkspaceCommand WorkspaceCmd; WorkspaceShareCommand WorkspaceShareCmd; @@ -613,6 +615,7 @@ main(int argc, char** argv) {"version", &VersionCmd, "Get zen server version"}, {"vfs", &VfsCmd, "Manage virtual file system"}, {"flush", &FlushCmd, "Flush storage"}, + {WipeCommand::Name, &WipeCmd, WipeCommand::Description}, {WorkspaceCommand::Name, &WorkspaceCmd, WorkspaceCommand::Description}, {WorkspaceShareCommand::Name, &WorkspaceShareCmd, WorkspaceShareCommand::Description}, // clang-format on diff --git a/src/zen/zen.h b/src/zen/zen.h index 6765101db..dd0fd44b3 100644 --- a/src/zen/zen.h +++ b/src/zen/zen.h @@ -5,10 +5,7 @@ #include <zencore/except.h> #include <zencore/timer.h> #include <zencore/zencore.h> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <cxxopts.hpp> -ZEN_THIRD_PARTY_INCLUDES_END +#include <zenutil/commandlineoptions.h> namespace cpr { class Response; diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index ad796cb4a..018330d9b 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -2670,22 +2670,6 @@ SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) return Result; } -std::filesystem::path -StringToPath(const std::string_view& Path) -{ - std::string_view UnquotedPath = Path; - - if (Path.length() > 2 && Path.front() == '\"' && Path.back() == '\"') - { - UnquotedPath = Path.substr(1, Path.length() - 2); - } - if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator)) - { - UnquotedPath = UnquotedPath.substr(0, UnquotedPath.length() - 1); - } - return std::filesystem::path(UnquotedPath).make_preferred(); -} - ////////////////////////////////////////////////////////////////////////// // // Testing related code follows... diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 66deffa6f..1bc3943df 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -372,8 +372,6 @@ uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); -std::filesystem::path StringToPath(const std::string_view& Path); - ////////////////////////////////////////////////////////////////////////// void filesystem_forcelink(); // internal diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index 0c5931ba0..d1394cd9a 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -100,9 +100,6 @@ int GetProcessId(CreateProcResult ProcId); std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc); std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle); -std::vector<std::string> ParseCommandLine(std::string_view CommandLine); -std::vector<char*> StripCommandlineQuotes(std::vector<std::string>& InOutArgs); - void process_forcelink(); // internal } // namespace zen diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 2fe5b8948..48efc3f85 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -1047,118 +1047,6 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand #endif // ZEN_PLATFORM_LINUX } -std::vector<std::string> -ParseCommandLine(std::string_view CommandLine) -{ - auto IsWhitespaceOrEnd = [](std::string_view CommandLine, std::string::size_type Pos) { - if (Pos == CommandLine.length()) - { - return true; - } - if (CommandLine[Pos] == ' ') - { - return true; - } - return false; - }; - - bool IsParsingArg = false; - bool IsInQuote = false; - - std::string::size_type Pos = 0; - std::string::size_type ArgStart = 0; - std::vector<std::string> Args; - while (Pos < CommandLine.length()) - { - if (IsInQuote) - { - if (CommandLine[Pos] == '"' && IsWhitespaceOrEnd(CommandLine, Pos + 1)) - { - Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart + 1))); - Pos++; - IsInQuote = false; - IsParsingArg = false; - } - else - { - Pos++; - } - } - else if (IsParsingArg) - { - ZEN_ASSERT(Pos > ArgStart); - if (CommandLine[Pos] == ' ') - { - Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart))); - Pos++; - IsParsingArg = false; - } - else if (CommandLine[Pos] == '"') - { - IsInQuote = true; - Pos++; - } - else - { - Pos++; - } - } - else if (CommandLine[Pos] == '"') - { - IsInQuote = true; - IsParsingArg = true; - ArgStart = Pos; - Pos++; - } - else if (CommandLine[Pos] != ' ') - { - IsParsingArg = true; - ArgStart = Pos; - Pos++; - } - else - { - Pos++; - } - } - if (IsParsingArg) - { - ZEN_ASSERT(Pos > ArgStart); - Args.push_back(std::string(CommandLine.substr(ArgStart))); - } - - return Args; -} - -std::vector<char*> -StripCommandlineQuotes(std::vector<std::string>& InOutArgs) -{ - std::vector<char*> RawArgs; - RawArgs.reserve(InOutArgs.size()); - for (std::string& Arg : InOutArgs) - { - std::string::size_type EscapedQuotePos = Arg.find("\\\"", 1); - while (EscapedQuotePos != std::string::npos && Arg.rfind('\"', EscapedQuotePos - 1) != std::string::npos) - { - Arg.erase(EscapedQuotePos, 1); - EscapedQuotePos = Arg.find("\\\"", EscapedQuotePos); - } - - if (Arg.starts_with("\"")) - { - if (Arg.find('"', 1) == Arg.length() - 1) - { - if (Arg.find(' ', 1) == std::string::npos) - { - Arg = Arg.substr(1, Arg.length() - 2); - } - } - } - RawArgs.push_back(const_cast<char*>(Arg.c_str())); - } - return RawArgs; -} - #if ZEN_WITH_TESTS void @@ -1235,36 +1123,6 @@ TEST_CASE("BuildArgV") } } -TEST_CASE("CommandLine") -{ - std::vector<std::string> v1 = ParseCommandLine("c:\\my\\exe.exe \"quoted arg\" \"one\",two,\"three\\\""); - CHECK_EQ(v1[0], "c:\\my\\exe.exe"); - CHECK_EQ(v1[1], "\"quoted arg\""); - CHECK_EQ(v1[2], "\"one\",two,\"three\\\""); - - std::vector<std::string> v2 = ParseCommandLine( - "--tracehost 127.0.0.1 builds download --url=https://jupiter.devtools.epicgames.com --namespace=ue.oplog " - "--bucket=citysample.packaged-build.fortnite-main.windows \"c:\\just\\a\\path\" " - "--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\" \"D:\\Dev\\Spaced Folder\\Target\\\" " - "--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\\\" 07dn23ifiwesnvoasjncasab --build-part-name win64,linux,ps5"); - - std::vector<char*> v2Stripped = StripCommandlineQuotes(v2); - CHECK_EQ(v2Stripped[0], std::string("--tracehost")); - CHECK_EQ(v2Stripped[1], std::string("127.0.0.1")); - CHECK_EQ(v2Stripped[2], std::string("builds")); - CHECK_EQ(v2Stripped[3], std::string("download")); - CHECK_EQ(v2Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com")); - CHECK_EQ(v2Stripped[5], std::string("--namespace=ue.oplog")); - CHECK_EQ(v2Stripped[6], std::string("--bucket=citysample.packaged-build.fortnite-main.windows")); - CHECK_EQ(v2Stripped[7], std::string("c:\\just\\a\\path")); - CHECK_EQ(v2Stripped[8], std::string("--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\"")); - CHECK_EQ(v2Stripped[9], std::string("\"D:\\Dev\\Spaced Folder\\Target\"")); - CHECK_EQ(v2Stripped[10], std::string("--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\"")); - CHECK_EQ(v2Stripped[11], std::string("07dn23ifiwesnvoasjncasab")); - CHECK_EQ(v2Stripped[12], std::string("--build-part-name")); - CHECK_EQ(v2Stripped[13], std::string("win64,linux,ps5")); -} - TEST_SUITE_END(/* core.process */); #endif diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 763f3262a..ca1b820c9 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -365,9 +365,10 @@ ShouldRetry(const cpr::Response& Response) { case cpr::ErrorCode::OK: break; - case cpr::ErrorCode::OPERATION_TIMEDOUT: + case cpr::ErrorCode::INTERNAL_ERROR: case cpr::ErrorCode::NETWORK_RECEIVE_ERROR: case cpr::ErrorCode::NETWORK_SEND_FAILURE: + case cpr::ErrorCode::OPERATION_TIMEDOUT: return true; default: return false; @@ -377,6 +378,7 @@ ShouldRetry(const cpr::Response& Response) case HttpResponseCode::RequestTimeout: case HttpResponseCode::TooManyRequests: case HttpResponseCode::InternalServerError: + case HttpResponseCode::BadGateway: case HttpResponseCode::ServiceUnavailable: case HttpResponseCode::GatewayTimeout: return true; diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 27a09f339..764f2a2a7 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -787,120 +787,131 @@ HttpRpcHandler::AddRpc(std::string_view RpcId, std::function<void(CbObject& RpcA ////////////////////////////////////////////////////////////////////////// -enum class HttpServerClass -{ - kHttpAsio, - kHttpSys, - kHttpPlugin, - kHttpMulti, - kHttpNull -}; - Ref<HttpServer> -CreateHttpServerClass(HttpServerClass Class, const HttpServerConfig& Config) +CreateHttpServerClass(const std::string_view ServerClass, const HttpServerConfig& Config) { - switch (Class) + if (ServerClass == "asio"sv) { - default: - case HttpServerClass::kHttpAsio: - ZEN_INFO("using asio HTTP server implementation"); - return CreateHttpAsioServer(Config.ForceLoopback, Config.ThreadCount); - - case HttpServerClass::kHttpMulti: - { - ZEN_INFO("using multi HTTP server implementation"); - Ref<HttpMultiServer> Server{new HttpMultiServer()}; - - // This is hardcoded for now, but should be configurable in the future - Server->AddServer(CreateHttpServerClass(HttpServerClass::kHttpSys, Config)); - Server->AddServer(CreateHttpServerClass(HttpServerClass::kHttpPlugin, Config)); + ZEN_INFO("using asio HTTP server implementation") + return CreateHttpAsioServer(Config.ForceLoopback, Config.ThreadCount); + } +#if ZEN_WITH_HTTPSYS + else if (ServerClass == "httpsys"sv) + { + ZEN_INFO("using http.sys server implementation") + return Ref<HttpServer>(CreateHttpSysServer({.ThreadCount = Config.ThreadCount, + .AsyncWorkThreadCount = Config.HttpSys.AsyncWorkThreadCount, + .IsAsyncResponseEnabled = Config.HttpSys.IsAsyncResponseEnabled, + .IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled, + .IsDedicatedServer = Config.IsDedicatedServer, + .ForceLoopback = Config.ForceLoopback})); + } +#endif + else if (ServerClass == "null"sv) + { + ZEN_INFO("using null HTTP server implementation") + return Ref<HttpServer>(new HttpNullServer); + } + else + { + ZEN_WARN("unknown HTTP server implementation '{}', falling back to default", ServerClass) - return Server; - } +#if ZEN_WITH_HTTPSYS + return CreateHttpServerClass("httpsys"sv, Config); +#else + return CreateHttpServerClass("asio"sv, Config); +#endif + } +} #if ZEN_WITH_PLUGINS - case HttpServerClass::kHttpPlugin: - { - ZEN_INFO("using plugin HTTP server implementation"); - Ref<HttpPluginServer> Server{CreateHttpPluginServer()}; +Ref<HttpServer> +CreateHttpServerPlugin(const HttpServerPluginConfig& PluginConfig) +{ + const std::string& PluginName = PluginConfig.PluginName; - // This is hardcoded for now, but should be configurable in the future + ZEN_INFO("using '{}' plugin HTTP server implementation", PluginName) + if (PluginName.starts_with("builtin:"sv)) + { # if 0 - Ref<TransportPlugin> WinsockPlugin{CreateSocketTransportPlugin()}; - WinsockPlugin->Configure("port", "8558"); - Server->AddPlugin(WinsockPlugin); -# endif + Ref<TransportPlugin> Plugin = {}; + if (PluginName == "builtin:winsock"sv) + { + Plugin = CreateSocketTransportPlugin(); + } + else if (PluginName == "builtin:asio"sv) + { + Plugin = CreateAsioTransportPlugin(); + } + else + { + ZEN_WARN("Unknown builtin plugin '{}'", PluginName) + return {}; + } -# if 0 - Ref<TransportPlugin> AsioPlugin{CreateAsioTransportPlugin()}; - AsioPlugin->Configure("port", "8558"); - Server->AddPlugin(AsioPlugin); -# endif + ZEN_ASSERT(!Plugin.IsNull()); -# if 1 - Ref<DllTransportPlugin> DllPlugin{CreateDllTransportPlugin()}; - DllPlugin->LoadDll("winsock"); - DllPlugin->ConfigureDll("winsock", "port", "8558"); - Server->AddPlugin(DllPlugin); -# endif + for (const std::pair<std::string, std::string>& Option : PluginConfig.PluginOptions) + { + Plugin->Configure(Option.first.c_str(), Option.second.c_str()); + } - return Server; - } -#endif + Ref<HttpPluginServer> Server{CreateHttpPluginServer()}; + Server->AddPlugin(Plugin); + return Server; +# else + ZEN_WARN("Builtin plugin '{}' is not supported", PluginName) + return {}; +# endif + } -#if ZEN_WITH_HTTPSYS - case HttpServerClass::kHttpSys: - ZEN_INFO("using http.sys server implementation"); - return Ref<HttpServer>(CreateHttpSysServer({.ThreadCount = Config.ThreadCount, - .AsyncWorkThreadCount = Config.HttpSys.AsyncWorkThreadCount, - .IsAsyncResponseEnabled = Config.HttpSys.IsAsyncResponseEnabled, - .IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled, - .IsDedicatedServer = Config.IsDedicatedServer, - .ForceLoopback = Config.ForceLoopback})); -#endif + Ref<DllTransportPlugin> DllPlugin{CreateDllTransportPlugin()}; + if (!DllPlugin->LoadDll(PluginName)) + { + return {}; + } - case HttpServerClass::kHttpNull: - ZEN_INFO("using null HTTP server implementation"); - return Ref<HttpServer>(new HttpNullServer); + for (const std::pair<std::string, std::string>& Option : PluginConfig.PluginOptions) + { + DllPlugin->ConfigureDll(PluginName, Option.first.c_str(), Option.second.c_str()); } + + Ref<HttpPluginServer> Server{CreateHttpPluginServer()}; + Server->AddPlugin(DllPlugin); + return Server; } +#endif Ref<HttpServer> CreateHttpServer(const HttpServerConfig& Config) { using namespace std::literals; - HttpServerClass Class = HttpServerClass::kHttpNull; - -#if ZEN_WITH_HTTPSYS - Class = HttpServerClass::kHttpSys; -#else - Class = HttpServerClass::kHttpAsio; -#endif - - if (Config.ServerClass == "asio"sv) - { - Class = HttpServerClass::kHttpAsio; - } - else if (Config.ServerClass == "httpsys"sv) - { - Class = HttpServerClass::kHttpSys; - } - else if (Config.ServerClass == "plugin"sv) - { - Class = HttpServerClass::kHttpPlugin; - } - else if (Config.ServerClass == "null"sv) +#if ZEN_WITH_PLUGINS + if (Config.PluginConfigs.empty()) { - Class = HttpServerClass::kHttpNull; + return CreateHttpServerClass(Config.ServerClass, Config); } - else if (Config.ServerClass == "multi"sv) + else { - Class = HttpServerClass::kHttpMulti; - } + Ref<HttpMultiServer> Server{new HttpMultiServer()}; + Server->AddServer(CreateHttpServerClass(Config.ServerClass, Config)); - return CreateHttpServerClass(Class, Config); + for (const HttpServerPluginConfig& PluginConfig : Config.PluginConfigs) + { + Ref<HttpServer> PluginServer = CreateHttpServerPlugin(PluginConfig); + if (!PluginServer.IsNull()) + { + Server->AddServer(PluginServer); + } + } + + return Server; + } +#else + return CreateHttpServerClass(Config.ServerClass, Config); +#endif } ////////////////////////////////////////////////////////////////////////// diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 217455dba..03e547bf3 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -184,12 +184,19 @@ public: virtual void Close() = 0; }; +struct HttpServerPluginConfig +{ + std::string PluginName; + std::vector<std::pair<std::string, std::string>> PluginOptions; +}; + struct HttpServerConfig { - bool IsDedicatedServer = false; // Should be set to true for shared servers - std::string ServerClass; // Choice of HTTP server implementation - bool ForceLoopback = false; - unsigned int ThreadCount = 0; + bool IsDedicatedServer = false; // Should be set to true for shared servers + std::string ServerClass; // Choice of HTTP server implementation + std::vector<HttpServerPluginConfig> PluginConfigs; + bool ForceLoopback = false; + unsigned int ThreadCount = 0; struct { diff --git a/src/zenhttp/servers/httpmulti.cpp b/src/zenhttp/servers/httpmulti.cpp index 2a6a90d2e..f4dc1e15b 100644 --- a/src/zenhttp/servers/httpmulti.cpp +++ b/src/zenhttp/servers/httpmulti.cpp @@ -103,6 +103,10 @@ HttpMultiServer::RequestExit() void HttpMultiServer::Close() { + for (auto& Server : m_Servers) + { + Server->Close(); + } } void diff --git a/src/zenhttp/transports/dlltransport.cpp b/src/zenhttp/transports/dlltransport.cpp index e09e62ec5..fb3dd23b5 100644 --- a/src/zenhttp/transports/dlltransport.cpp +++ b/src/zenhttp/transports/dlltransport.cpp @@ -21,18 +21,31 @@ namespace zen { ////////////////////////////////////////////////////////////////////////// +class DllTransportLogger : public TransportLogger, public RefCounted +{ +public: + DllTransportLogger(std::string_view PluginName); + virtual ~DllTransportLogger() = default; + + void LogMessage(LogLevel Level, const char* Message) override; + +private: + std::string m_PluginName; +}; + struct LoadedDll { std::string Name; std::filesystem::path LoadedFromPath; + DllTransportLogger* Logger = nullptr; Ref<TransportPlugin> Plugin; }; class DllTransportPluginImpl : public DllTransportPlugin, RefCounted { public: - DllTransportPluginImpl(); - ~DllTransportPluginImpl(); + DllTransportPluginImpl() = default; + ~DllTransportPluginImpl() = default; virtual uint32_t AddRef() const override; virtual uint32_t Release() const override; @@ -42,7 +55,7 @@ public: virtual const char* GetDebugName() override; virtual bool IsAvailable() override; - virtual void LoadDll(std::string_view Name) override; + virtual bool LoadDll(std::string_view Name) override; virtual void ConfigureDll(std::string_view Name, const char* OptionTag, const char* OptionValue) override; private: @@ -51,12 +64,27 @@ private: std::vector<LoadedDll> m_Transports; }; -DllTransportPluginImpl::DllTransportPluginImpl() +DllTransportLogger::DllTransportLogger(std::string_view PluginName) : m_PluginName(PluginName) { } -DllTransportPluginImpl::~DllTransportPluginImpl() +void +DllTransportLogger::LogMessage(LogLevel PluginLogLevel, const char* Message) { + logging::level::LogLevel Level; + // clang-format off + switch (PluginLogLevel) + { + case LogLevel::Trace: Level = logging::level::Trace; break; + case LogLevel::Debug: Level = logging::level::Debug; break; + case LogLevel::Info: Level = logging::level::Info; break; + case LogLevel::Warn: Level = logging::level::Warn; break; + case LogLevel::Err: Level = logging::level::Err; break; + case LogLevel::Critical: Level = logging::level::Critical; break; + default: Level = logging::level::Off; break; + } + // clang-format on + ZEN_LOG(Log(), Level, "[{}] {}", m_PluginName, Message) } uint32_t @@ -109,6 +137,7 @@ DllTransportPluginImpl::Shutdown() try { Transport.Plugin->Shutdown(); + Transport.Logger->Release(); } catch (const std::exception&) { @@ -143,42 +172,73 @@ DllTransportPluginImpl::ConfigureDll(std::string_view Name, const char* OptionTa } } -void +bool DllTransportPluginImpl::LoadDll(std::string_view Name) { RwLock::ExclusiveLockScope _(m_Lock); - ExtendableStringBuilder<128> DllPath; - DllPath << Name << ".dll"; + ExtendableStringBuilder<1024> DllPath; + DllPath << Name; + if (!Name.ends_with(".dll")) + { + DllPath << ".dll"; + } + + std::string FileName = std::filesystem::path(DllPath.c_str()).filename().replace_extension().string(); + HMODULE DllHandle = LoadLibraryA(DllPath.c_str()); if (!DllHandle) { - std::error_code Ec = MakeErrorCodeFromLastError(); - - throw std::system_error(Ec, fmt::format("failed to load transport DLL from '{}'", DllPath)); + ZEN_WARN("Failed to load transport DLL from '{}' due to '{}'", DllPath, GetLastErrorAsString()) + return false; } - TransportPlugin* CreateTransportPlugin(); + PfnGetTransportPluginVersion GetVersion = (PfnGetTransportPluginVersion)GetProcAddress(DllHandle, "GetTransportPluginVersion"); + PfnCreateTransportPlugin CreatePlugin = (PfnCreateTransportPlugin)GetProcAddress(DllHandle, "CreateTransportPlugin"); + + uint32_t APIVersion = 0; + uint32_t PluginVersion = 0; + + if (GetVersion) + { + GetVersion(&APIVersion, &PluginVersion); + } - PfnCreateTransportPlugin CreatePlugin = (PfnCreateTransportPlugin)GetProcAddress(DllHandle, "CreateTransportPlugin"); + const bool bValidApiVersion = APIVersion == kTransportApiVersion; - if (!CreatePlugin) + if (!GetVersion || !CreatePlugin || !bValidApiVersion) { std::error_code Ec = MakeErrorCodeFromLastError(); FreeLibrary(DllHandle); - throw std::system_error(Ec, fmt::format("API mismatch detected in transport DLL loaded from '{}'", DllPath)); + if (GetVersion && !bValidApiVersion) + { + ZEN_WARN("Failed to load transport DLL from '{}' due to invalid API version {}, supported API version is {}", + DllPath, + APIVersion, + kTransportApiVersion) + } + else + { + ZEN_WARN("Failed to load transport DLL from '{}' due to not finding GetTransportPluginVersion or CreateTransportPlugin", + DllPath) + } + + return false; } LoadedDll NewDll; NewDll.Name = Name; NewDll.LoadedFromPath = DllPath.c_str(); - NewDll.Plugin = CreatePlugin(); + NewDll.Logger = new DllTransportLogger(FileName); + NewDll.Logger->AddRef(); + NewDll.Plugin = CreatePlugin(NewDll.Logger); m_Transports.emplace_back(std::move(NewDll)); + return true; } DllTransportPlugin* diff --git a/src/zenhttp/transports/dlltransport.h b/src/zenhttp/transports/dlltransport.h index 9346a10ce..c49f888da 100644 --- a/src/zenhttp/transports/dlltransport.h +++ b/src/zenhttp/transports/dlltransport.h @@ -15,7 +15,7 @@ namespace zen { class DllTransportPlugin : public TransportPlugin { public: - virtual void LoadDll(std::string_view Name) = 0; + virtual bool LoadDll(std::string_view Name) = 0; virtual void ConfigureDll(std::string_view Name, const char* OptionTag, const char* OptionValue) = 0; }; diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 31c104110..e81e8eb54 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -15,12 +15,14 @@ #include <zencore/logging.h> #include <zencore/string.h> #include <zenhttp/zenhttp.h> +#include <zenutil/commandlineoptions.h> ZEN_THIRD_PARTY_INCLUDES_START #include <fmt/format.h> #include <fmt/ranges.h> #include <zencore/logging.h> #include <cxxopts.hpp> +#include <json11.hpp> #include <sol/sol.hpp> ZEN_THIRD_PARTY_INCLUDES_END @@ -178,27 +180,6 @@ ParseBucketConfigs(std::span<std::string> Buckets) return Cfg; } -static std::string -MakeSafePath(const std::string_view Path) -{ -#if ZEN_PLATFORM_WINDOWS - if (Path.empty()) - { - return std::string(Path); - } - - std::string FixedPath(Path); - std::replace(FixedPath.begin(), FixedPath.end(), '/', '\\'); - if (!FixedPath.starts_with("\\\\?\\")) - { - FixedPath.insert(0, "\\\\?\\"); - } - return FixedPath; -#else - return std::string(Path); -#endif -}; - class CachePolicyOption : public LuaConfig::OptionValue { public: @@ -324,7 +305,7 @@ public: std::string Name = Bucket.value().get_or("name", std::string("Default")); std::string Directory = Bucket.value().get_or("directory", std::string()); - Value.Buckets.push_back({.Name = std::move(Name), .Directory = LuaConfig::MakeSafePath(Directory)}); + Value.Buckets.push_back({.Name = std::move(Name), .Directory = MakeSafeAbsolutePath(Directory)}); } } } @@ -369,6 +350,7 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv); LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv); + LuaOptions.AddOption("server.pluginsconfigfile"sv, ServerOptions.PluginsConfigFile, "plugins-config"sv); LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv); LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv); LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "quiet"sv); @@ -379,7 +361,7 @@ ParseConfigFile(const std::filesystem::path& Path, ////// buildsstore LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv); - LuaOptions.AddOption("buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit"); + LuaOptions.AddOption("server.buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit"); ////// network LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv); @@ -525,7 +507,7 @@ ParseConfigFile(const std::filesystem::path& Path, if (!OutputConfigFile.empty()) { - std::filesystem::path WritePath(MakeSafePath(OutputConfigFile)); + std::filesystem::path WritePath(MakeSafeAbsolutePath(OutputConfigFile)); zen::ExtendableStringBuilder<512> ConfigStringBuilder; LuaOptions.Print(ConfigStringBuilder, CmdLineResult); zen::BasicFile Output; @@ -535,6 +517,68 @@ ParseConfigFile(const std::filesystem::path& Path, } void +ParsePluginsConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions, int BasePort) +{ + using namespace std::literals; + + IoBuffer Body = IoBufferBuilder::MakeFromFile(Path); + std::string JsonText(reinterpret_cast<const char*>(Body.GetData()), Body.GetSize()); + std::string JsonError; + json11::Json PluginsInfo = json11::Json::parse(JsonText, JsonError); + if (!JsonError.empty()) + { + throw std::runtime_error(fmt::format("failed parsing json file '{}'. Reason: '{}'", Path, JsonError)); + } + for (const json11::Json& PluginInfo : PluginsInfo.array_items()) + { + if (!PluginInfo.is_object()) + { + throw std::runtime_error(fmt::format("the json file '{}' does not contain a valid plugin definition, object expected, got '{}'", + Path, + PluginInfo.dump())); + } + + HttpServerPluginConfig Config = {}; + + bool bNeedsPort = true; + + for (const std::pair<const std::string, json11::Json>& Items : PluginInfo.object_items()) + { + if (!Items.second.is_string()) + { + throw std::runtime_error( + fmt::format("the json file '{}' does not contain a valid plugins definition, string expected, got '{}'", + Path, + Items.second.dump())); + } + + const std::string& Name = Items.first; + const std::string& Value = Items.second.string_value(); + + if (Name == "name"sv) + Config.PluginName = Value; + else + { + Config.PluginOptions.push_back({Name, Value}); + + if (Name == "port"sv) + { + bNeedsPort = false; + } + } + } + + // add a default base port in case if json config didn't provide one + if (bNeedsPort) + { + Config.PluginOptions.push_back({"port", std::to_string(BasePort)}); + } + + ServerOptions.HttpServerConfig.PluginConfigs.push_back(Config); + } +} + +void ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) { const char* DefaultHttp = "asio"; @@ -566,6 +610,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) std::string ContentDir; std::string AbsLogFile; std::string ConfigFile; + std::string PluginsConfigFile; std::string OutputConfigFile; std::string BaseSnapshotDir; @@ -593,6 +638,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) "Exit immediately after initialization is complete", cxxopts::value<bool>(ServerOptions.IsPowerCycle)); options.add_options()("config", "Path to Lua config file", cxxopts::value<std::string>(ConfigFile)); + options.add_options()("plugins-config", "Path to plugins config file", cxxopts::value<std::string>(PluginsConfigFile)); options.add_options()("write-config", "Path to output Lua config file", cxxopts::value<std::string>(OutputConfigFile)); options.add_options()("no-sentry", "Disable Sentry crash handler", @@ -1110,12 +1156,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) } logging::RefreshLogLevels(); - ServerOptions.SystemRootDir = MakeSafePath(SystemRootDir); - ServerOptions.DataDir = MakeSafePath(DataDir); - ServerOptions.BaseSnapshotDir = MakeSafePath(BaseSnapshotDir); - ServerOptions.ContentDir = MakeSafePath(ContentDir); - ServerOptions.AbsLogFile = MakeSafePath(AbsLogFile); - ServerOptions.ConfigFile = MakeSafePath(ConfigFile); + ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir); + ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir); + ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir); + ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir); + ServerOptions.AbsLogFile = MakeSafeAbsolutePath(AbsLogFile); + ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile); + ServerOptions.PluginsConfigFile = MakeSafeAbsolutePath(PluginsConfigFile); ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions); if (!BaseSnapshotDir.empty()) @@ -1149,6 +1196,11 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, Result, OutputConfigFile); } + if (!ServerOptions.PluginsConfigFile.empty()) + { + ParsePluginsConfigFile(ServerOptions.PluginsConfigFile, ServerOptions, ServerOptions.BasePort); + } + ValidateOptions(ServerOptions); } catch (const zen::OptionParseException& e) diff --git a/src/zenserver/config.h b/src/zenserver/config.h index 6bd7aa357..bd277cd88 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -165,6 +165,7 @@ struct ZenServerOptions std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) std::filesystem::path AbsLogFile; // Absolute path to main log file std::filesystem::path ConfigFile; // Path to Lua config file + std::filesystem::path PluginsConfigFile; // Path to plugins config file std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) std::string ChildId; // Id assigned by parent process (used for lifetime management) std::string LogId; // Id for tagging log output diff --git a/src/zenserver/config/luaconfig.cpp b/src/zenserver/config/luaconfig.cpp index f742fa34a..2c54de29e 100644 --- a/src/zenserver/config/luaconfig.cpp +++ b/src/zenserver/config/luaconfig.cpp @@ -4,27 +4,6 @@ namespace zen::LuaConfig { -std::string -MakeSafePath(const std::string_view Path) -{ -#if ZEN_PLATFORM_WINDOWS - if (Path.empty()) - { - return std::string(Path); - } - - std::string FixedPath(Path); - std::replace(FixedPath.begin(), FixedPath.end(), '/', '\\'); - if (!FixedPath.starts_with("\\\\?\\")) - { - FixedPath.insert(0, "\\\\?\\"); - } - return FixedPath; -#else - return std::string(Path); -#endif -}; - void EscapeBackslash(std::string& InOutString) { @@ -101,7 +80,7 @@ FilePathOption::Parse(sol::object Object) std::string Str = Object.as<std::string>(); if (!Str.empty()) { - Value = MakeSafePath(Str); + Value = MakeSafeAbsolutePath(Str); } } diff --git a/src/zenserver/config/luaconfig.h b/src/zenserver/config/luaconfig.h index 76b3088a3..ce7013a9a 100644 --- a/src/zenserver/config/luaconfig.h +++ b/src/zenserver/config/luaconfig.h @@ -4,10 +4,10 @@ #include <zenbase/concepts.h> #include <zencore/fmtutils.h> +#include <zenutil/commandlineoptions.h> ZEN_THIRD_PARTY_INCLUDES_START #include <fmt/format.h> -#include <cxxopts.hpp> #include <sol/sol.hpp> ZEN_THIRD_PARTY_INCLUDES_END @@ -20,8 +20,7 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen::LuaConfig { -std::string MakeSafePath(const std::string_view Path); -void EscapeBackslash(std::string& InOutString); +void EscapeBackslash(std::string& InOutString); class OptionValue { diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip Binary files differindex c28404da6..eadfcf84c 100644 --- a/src/zenserver/frontend/html.zip +++ b/src/zenserver/frontend/html.zip diff --git a/src/zenserver/frontend/html/indexer/indexer.js b/src/zenserver/frontend/html/indexer/indexer.js index 16b91e130..688bc71b0 100644 --- a/src/zenserver/frontend/html/indexer/indexer.js +++ b/src/zenserver/frontend/html/indexer/indexer.js @@ -61,7 +61,7 @@ class Indexer { for (const page of this._pages) for (const [_, name, size, raw_size] of page) - yield [name, size|0, raw_size|0]; + yield [name, size|0n, raw_size|0n]; } } diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 41fc47218..54fb11c18 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -79,10 +79,32 @@ export class Page extends ZenPage async _build_meta(section, entry) { var tree = {} - const cookart = this._find_iohash_field(entry, "CookPackageArtifacts"); - if (cookart != null) + + for (const field of entry) { - tree["cook"] = { CookPackageArtifacts: cookart}; + var visibleKey = undefined; + const name = field.get_name(); + if (name == "CookPackageArtifacts") + { + visibleKey = name; + } + else if (name.startsWith("meta.")) + { + visibleKey = name.slice(5); + } + + if (visibleKey != undefined) + { + var found_value = field.as_value(); + if (found_value instanceof Uint8Array) + { + var ret = ""; + for (var x of found_value) + ret += x.toString(16).padStart(2, "0"); + tree[visibleKey] = ret; + } + } + } if (Object.keys(tree).length == 0) @@ -90,28 +112,25 @@ export class Page extends ZenPage const sub_section = section.add_section("meta"); - for (const cat_name in tree) + const table = sub_section.add_widget( + Table, + ["name", "actions"], Table.Flag_PackRight + ); + for (const key in tree) { - const cat_section = sub_section.add_section(cat_name); - const table = cat_section.add_widget( - Table, - ["name", "actions"], Table.Flag_PackRight + const row = table.add_row(key); + const value = tree[key]; + + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + const link = row.get_cell(0).link( + "/" + ["prj", project, "oplog", oplog, value+".json"].join("/") ); - Object.entries(tree[cat_name]).forEach(([key, value]) => - { - const row = table.add_row(key); - - const project = this.get_param("project"); - const oplog = this.get_param("oplog"); - const link = row.get_cell(0).link( - "/" + ["prj", project, "oplog", oplog, value+".json"].join("/") - ); - - const action_tb = new Toolbar(row.get_cell(-1), true); - action_tb.left().add("copy-hash").on_click(async (v) => { - await navigator.clipboard.writeText(v); - }, value); - }); + + const action_tb = new Toolbar(row.get_cell(-1), true); + action_tb.left().add("copy-hash").on_click(async (v) => { + await navigator.clipboard.writeText(v); + }, value); } } diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index e4d962b56..4f72a711a 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -3572,7 +3572,7 @@ ZenCacheDiskLayer::~ZenCacheDiskLayer() } template<typename T, typename U> -struct equal_to_2 : public eastl::binary_function<T, U, bool> +struct equal_to_2 { constexpr bool operator()(const T& a, const U& b) const { return a == b; } diff --git a/src/zenutil/commandlineoptions.cpp b/src/zenutil/commandlineoptions.cpp new file mode 100644 index 000000000..0dffa42f0 --- /dev/null +++ b/src/zenutil/commandlineoptions.cpp @@ -0,0 +1,213 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zenutil/commandlineoptions.h> + +#include <filesystem> +#if ZEN_WITH_TESTS +# include <zencore/testing.h> +#endif // ZEN_WITH_TESTS + +void +cxxopts::values::parse_value(const std::string& text, std::filesystem::path& value) +{ + value = zen::StringToPath(text); +} + +namespace zen { + +std::vector<std::string> +ParseCommandLine(std::string_view CommandLine) +{ + auto IsWhitespaceOrEnd = [](std::string_view CommandLine, std::string::size_type Pos) { + if (Pos == CommandLine.length()) + { + return true; + } + if (CommandLine[Pos] == ' ') + { + return true; + } + return false; + }; + + bool IsParsingArg = false; + bool IsInQuote = false; + + std::string::size_type Pos = 0; + std::string::size_type ArgStart = 0; + std::vector<std::string> Args; + while (Pos < CommandLine.length()) + { + if (IsInQuote) + { + if (CommandLine[Pos] == '"' && IsWhitespaceOrEnd(CommandLine, Pos + 1)) + { + Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart + 1))); + Pos++; + IsInQuote = false; + IsParsingArg = false; + } + else + { + Pos++; + } + } + else if (IsParsingArg) + { + ZEN_ASSERT(Pos > ArgStart); + if (CommandLine[Pos] == ' ') + { + Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart))); + Pos++; + IsParsingArg = false; + } + else if (CommandLine[Pos] == '"') + { + IsInQuote = true; + Pos++; + } + else + { + Pos++; + } + } + else if (CommandLine[Pos] == '"') + { + IsInQuote = true; + IsParsingArg = true; + ArgStart = Pos; + Pos++; + } + else if (CommandLine[Pos] != ' ') + { + IsParsingArg = true; + ArgStart = Pos; + Pos++; + } + else + { + Pos++; + } + } + if (IsParsingArg) + { + ZEN_ASSERT(Pos > ArgStart); + Args.push_back(std::string(CommandLine.substr(ArgStart))); + } + + return Args; +} + +std::vector<char*> +StripCommandlineQuotes(std::vector<std::string>& InOutArgs) +{ + std::vector<char*> RawArgs; + RawArgs.reserve(InOutArgs.size()); + for (std::string& Arg : InOutArgs) + { + std::string::size_type EscapedQuotePos = Arg.find("\\\"", 1); + while (EscapedQuotePos != std::string::npos && Arg.rfind('\"', EscapedQuotePos - 1) != std::string::npos) + { + Arg.erase(EscapedQuotePos, 1); + EscapedQuotePos = Arg.find("\\\"", EscapedQuotePos); + } + + if (Arg.starts_with("\"")) + { + if (Arg.find('"', 1) == Arg.length() - 1) + { + if (Arg.find(' ', 1) == std::string::npos) + { + Arg = Arg.substr(1, Arg.length() - 2); + } + } + } + RawArgs.push_back(const_cast<char*>(Arg.c_str())); + } + return RawArgs; +} + +void +MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path) +{ + if (!Path.empty()) + { + std::filesystem::path AbsolutePath = std::filesystem::absolute(Path).make_preferred(); +#if ZEN_PLATFORM_WINDOWS + const std::string_view Prefix = "\\\\?\\"; + const std::u8string PrefixU8(Prefix.begin(), Prefix.end()); + std::u8string PathString = AbsolutePath.u8string(); + if (!PathString.empty() && !PathString.starts_with(PrefixU8)) + { + PathString.insert(0, PrefixU8); + Path = PathString; + } +#endif // ZEN_PLATFORM_WINDOWS + } +} + +std::filesystem::path +MakeSafeAbsolutePath(const std::filesystem::path& Path) +{ + std::filesystem::path Tmp(Path); + MakeSafeAbsolutePathÍnPlace(Tmp); + return Tmp; +} + +std::filesystem::path +StringToPath(const std::string_view& Path) +{ + std::string_view UnquotedPath = Path; + + if (UnquotedPath.length() > 2 && UnquotedPath.front() == '\"' && UnquotedPath.back() == '\"') + { + UnquotedPath = UnquotedPath.substr(1, UnquotedPath.length() - 2); + } + + if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator)) + { + UnquotedPath = UnquotedPath.substr(0, UnquotedPath.length() - 1); + } + + return std::filesystem::path(UnquotedPath).make_preferred(); +} + +#if ZEN_WITH_TESTS + +void +commandlineoptions_forcelink() +{ +} + +TEST_CASE("CommandLine") +{ + std::vector<std::string> v1 = ParseCommandLine("c:\\my\\exe.exe \"quoted arg\" \"one\",two,\"three\\\""); + CHECK_EQ(v1[0], "c:\\my\\exe.exe"); + CHECK_EQ(v1[1], "\"quoted arg\""); + CHECK_EQ(v1[2], "\"one\",two,\"three\\\""); + + std::vector<std::string> v2 = ParseCommandLine( + "--tracehost 127.0.0.1 builds download --url=https://jupiter.devtools.epicgames.com --namespace=ue.oplog " + "--bucket=citysample.packaged-build.fortnite-main.windows \"c:\\just\\a\\path\" " + "--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\" \"D:\\Dev\\Spaced Folder\\Target\\\" " + "--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\\\" 07dn23ifiwesnvoasjncasab --build-part-name win64,linux,ps5"); + + std::vector<char*> v2Stripped = StripCommandlineQuotes(v2); + CHECK_EQ(v2Stripped[0], std::string("--tracehost")); + CHECK_EQ(v2Stripped[1], std::string("127.0.0.1")); + CHECK_EQ(v2Stripped[2], std::string("builds")); + CHECK_EQ(v2Stripped[3], std::string("download")); + CHECK_EQ(v2Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com")); + CHECK_EQ(v2Stripped[5], std::string("--namespace=ue.oplog")); + CHECK_EQ(v2Stripped[6], std::string("--bucket=citysample.packaged-build.fortnite-main.windows")); + CHECK_EQ(v2Stripped[7], std::string("c:\\just\\a\\path")); + CHECK_EQ(v2Stripped[8], std::string("--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\"")); + CHECK_EQ(v2Stripped[9], std::string("\"D:\\Dev\\Spaced Folder\\Target\"")); + CHECK_EQ(v2Stripped[10], std::string("--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\"")); + CHECK_EQ(v2Stripped[11], std::string("07dn23ifiwesnvoasjncasab")); + CHECK_EQ(v2Stripped[12], std::string("--build-part-name")); + CHECK_EQ(v2Stripped[13], std::string("win64,linux,ps5")); +} + +#endif +} // namespace zen diff --git a/src/zenutil/include/zenutil/commandlineoptions.h b/src/zenutil/include/zenutil/commandlineoptions.h new file mode 100644 index 000000000..b7581f6cd --- /dev/null +++ b/src/zenutil/include/zenutil/commandlineoptions.h @@ -0,0 +1,28 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> +#include <filesystem> + +ZEN_THIRD_PARTY_INCLUDES_START + +namespace cxxopts::values { +// We declare this specialization before including cxxopts to make it stick +void parse_value(const std::string& text, std::filesystem::path& value); +} // namespace cxxopts::values + +#include <cxxopts.hpp> +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +std::vector<std::string> ParseCommandLine(std::string_view CommandLine); +std::vector<char*> StripCommandlineQuotes(std::vector<std::string>& InOutArgs); +void MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path); +[[nodiscard]] std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path); +std::filesystem::path StringToPath(const std::string_view& Path); + +void commandlineoptions_forcelink(); // internal + +} // namespace zen diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp index 19eb63ce9..aff9156f4 100644 --- a/src/zenutil/zenutil.cpp +++ b/src/zenutil/zenutil.cpp @@ -7,6 +7,7 @@ # include <zenutil/cache/cacherequests.h> # include <zenutil/cache/rpcrecording.h> # include <zenutil/chunkedfile.h> +# include <zenutil/commandlineoptions.h> namespace zen { @@ -17,6 +18,7 @@ zenutil_forcelinktests() cache::rpcrecord_forcelink(); cacherequests_forcelink(); chunkedfile_forcelink(); + commandlineoptions_forcelink(); } } // namespace zen |