diff options
Diffstat (limited to 'src')
127 files changed, 9008 insertions, 3068 deletions
diff --git a/src/zen/authutils.cpp b/src/zen/authutils.cpp index cf6179b5e..d68e60b11 100644 --- a/src/zen/authutils.cpp +++ b/src/zen/authutils.cpp @@ -255,6 +255,10 @@ AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, HostUrl, Quiet, m_OidcTokenUnattended, Hidden); } + else if (!m_OidcTokenAuthExecutablePath.empty()) + { + throw OptionParseException(fmt::format("'--oidctoken-exe-path' ('{}') does not exist", m_OidcTokenAuthExecutablePath), Ops.help()); + } if (!ClientSettings.AccessTokenProvider) { diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp index 502d1e799..15e854796 100644 --- a/src/zen/cmds/admin_cmd.cpp +++ b/src/zen/cmds/admin_cmd.cpp @@ -21,6 +21,12 @@ ScrubCommand::ScrubCommand() m_Options.add_option("", "n", "dry", "Dry run (do not delete any data)", cxxopts::value(m_DryRun), "<bool>"); m_Options.add_option("", "", "no-gc", "Do not perform GC after scrub pass", cxxopts::value(m_NoGc), "<bool>"); m_Options.add_option("", "", "no-cas", "Do not scrub CAS stores", cxxopts::value(m_NoCas), "<bool>"); + m_Options.add_option("", + "", + "maxtimeslice", + "Number of second Scrub is allowed to run before stopping in seconds (default 300s)", + cxxopts::value(m_MaxTimeSliceSeconds), + "<maxtimeslice>"); } ScrubCommand::~ScrubCommand() = default; @@ -44,7 +50,10 @@ ScrubCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) HttpClient Http(m_HostName); - HttpClient::KeyValueMap Params{{"skipdelete", ToString(m_DryRun)}, {"skipgc", ToString(m_NoGc)}, {"skipcid", ToString(m_NoCas)}}; + HttpClient::KeyValueMap Params{{"skipdelete", ToString(m_DryRun)}, + {"skipgc", ToString(m_NoGc)}, + {"skipcid", ToString(m_NoCas)}, + {"maxtimeslice", fmt::format("{}", m_MaxTimeSliceSeconds)}}; if (HttpClient::Response Response = Http.Post("/admin/scrub"sv, /* headers */ HttpClient::KeyValueMap{}, Params)) { diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h index 4f97b7ad4..87ef8091b 100644 --- a/src/zen/cmds/admin_cmd.h +++ b/src/zen/cmds/admin_cmd.h @@ -22,9 +22,10 @@ public: private: cxxopts::Options m_Options{"scrub", "Scrub zen storage"}; std::string m_HostName; - bool m_DryRun = false; - bool m_NoGc = false; - bool m_NoCas = false; + bool m_DryRun = false; + bool m_NoGc = false; + bool m_NoCas = false; + uint64_t m_MaxTimeSliceSeconds = 300; }; /** Garbage collect storage diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 25f66e0ee..f4edb65ab 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -24,6 +24,7 @@ #include <zenhttp/httpclientauth.h> #include <zenhttp/httpcommon.h> #include <zenremotestore/builds/buildcontent.h> +#include <zenremotestore/builds/buildmanifest.h> #include <zenremotestore/builds/buildsavedstate.h> #include <zenremotestore/builds/buildstoragecache.h> #include <zenremotestore/builds/buildstorageoperations.h> @@ -33,6 +34,7 @@ #include <zenremotestore/chunking/chunkblock.h> #include <zenremotestore/chunking/chunkedcontent.h> #include <zenremotestore/chunking/chunkedfile.h> +#include <zenremotestore/chunking/chunkingcache.h> #include <zenremotestore/chunking/chunkingcontroller.h> #include <zenremotestore/filesystemutils.h> #include <zenremotestore/jupiter/jupiterhost.h> @@ -289,13 +291,6 @@ namespace { return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u; } - bool IncludePath(std::span<const std::string> IncludeWildcards, - std::span<const std::string> ExcludeWildcards, - const std::filesystem::path& Path) - { - return zen::IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(Path.generic_string()), /*CaseSensitive*/ true); - } - class FilteredRate { public: @@ -427,251 +422,269 @@ namespace { NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000)); } - void UploadFolder(OperationLogOutput& Output, - TransferThreadWorkers& Workers, - StorageInstance& Storage, - const Oid& BuildId, - const Oid& BuildPartId, - const std::string_view BuildPartName, - const std::filesystem::path& Path, - const std::filesystem::path& TempDir, - const std::filesystem::path& ManifestPath, - const uint64_t FindBlockMaxCount, - const uint8_t BlockReuseMinPercentLimit, - bool AllowMultiparts, - const CbObject& MetaData, - bool CreateBuild, - bool IgnoreExistingBlocks, - bool UploadToZenCache) + struct UploadFolderOptions + { + std::filesystem::path TempDir; + uint64_t FindBlockMaxCount; + uint8_t BlockReuseMinPercentLimit; + bool AllowMultiparts; + bool CreateBuild; + bool IgnoreExistingBlocks; + bool UploadToZenCache; + const std::vector<std::string>& ExcludeFolders = DefaultExcludeFolders; + const std::vector<std::string>& ExcludeExtensions = DefaultExcludeExtensions; + }; + + std::vector<std::pair<Oid, std::string>> UploadFolder(OperationLogOutput& Output, + TransferThreadWorkers& Workers, + StorageInstance& Storage, + const Oid& BuildId, + const Oid& BuildPartId, + const std::string_view BuildPartName, + const std::filesystem::path& Path, + const std::filesystem::path& ManifestPath, + const CbObject& MetaData, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + const UploadFolderOptions& Options) { ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder"); + + Stopwatch UploadTimer; + + BuildsOperationUploadFolder UploadOp( + Output, + Storage, + AbortFlag, + PauseFlag, + Workers.GetIOWorkerPool(), + Workers.GetNetworkPool(), + BuildId, + Path, + Options.CreateBuild, + std::move(MetaData), + BuildsOperationUploadFolder::Options{.IsQuiet = IsQuiet, + .IsVerbose = IsVerbose, + .DoExtraContentValidation = DoExtraContentVerify, + .FindBlockMaxCount = Options.FindBlockMaxCount, + .BlockReuseMinPercentLimit = Options.BlockReuseMinPercentLimit, + .AllowMultiparts = Options.AllowMultiparts, + .IgnoreExistingBlocks = Options.IgnoreExistingBlocks, + .TempDir = Options.TempDir, + .ExcludeFolders = Options.ExcludeFolders, + .ExcludeExtensions = Options.ExcludeExtensions, + .ZenExcludeManifestName = ZenExcludeManifestName, + .NonCompressableExtensions = DefaultSplitOnlyExtensions, + .PopulateCache = Options.UploadToZenCache}); + + std::vector<std::pair<Oid, std::string>> UploadedParts = + UploadOp.Execute(BuildPartId, BuildPartName, ManifestPath, ChunkController, ChunkCache); + if (AbortFlag) { - Stopwatch UploadTimer; + return {}; + } - BuildsOperationUploadFolder UploadOp( - Output, - Storage, - AbortFlag, - PauseFlag, - Workers.GetIOWorkerPool(), - Workers.GetNetworkPool(), - BuildId, + ZEN_CONSOLE_VERBOSE( + "Folder scanning stats:" + "\n FoundFileCount: {}" + "\n FoundFileByteCount: {}" + "\n AcceptedFileCount: {}" + "\n AcceptedFileByteCount: {}" + "\n ElapsedWallTimeUS: {}", + UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), + NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), + UploadOp.m_LocalFolderScanStats.AcceptedFileCount.load(), + NiceBytes(UploadOp.m_LocalFolderScanStats.AcceptedFileByteCount.load()), + NiceLatencyNs(UploadOp.m_LocalFolderScanStats.ElapsedWallTimeUS * 1000)); + + ZEN_CONSOLE_VERBOSE( + "Chunking stats:" + "\n FilesProcessed: {}" + "\n FilesChunked: {}" + "\n BytesHashed: {}" + "\n UniqueChunksFound: {}" + "\n UniqueSequencesFound: {}" + "\n UniqueBytesFound: {}" + "\n FilesFoundInCache: {}" + "\n ChunksFoundInCache: {}" + "\n FilesStoredInCache: {}" + "\n ChunksStoredInCache: {}" + "\n ElapsedWallTimeUS: {}", + UploadOp.m_ChunkingStats.FilesProcessed.load(), + UploadOp.m_ChunkingStats.FilesChunked.load(), + NiceBytes(UploadOp.m_ChunkingStats.BytesHashed.load()), + UploadOp.m_ChunkingStats.UniqueChunksFound.load(), + UploadOp.m_ChunkingStats.UniqueSequencesFound.load(), + NiceBytes(UploadOp.m_ChunkingStats.UniqueBytesFound.load()), + UploadOp.m_ChunkingStats.FilesFoundInCache.load(), + UploadOp.m_ChunkingStats.ChunksFoundInCache.load(), + NiceBytes(UploadOp.m_ChunkingStats.BytesFoundInCache.load()), + UploadOp.m_ChunkingStats.FilesStoredInCache.load(), + UploadOp.m_ChunkingStats.ChunksStoredInCache.load(), + NiceBytes(UploadOp.m_ChunkingStats.BytesStoredInCache.load()), + NiceLatencyNs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS * 1000)); + + ZEN_CONSOLE_VERBOSE( + "Find block stats:" + "\n FindBlockTimeMS: {}" + "\n PotentialChunkCount: {}" + "\n PotentialChunkByteCount: {}" + "\n FoundBlockCount: {}" + "\n FoundBlockChunkCount: {}" + "\n FoundBlockByteCount: {}" + "\n AcceptedBlockCount: {}" + "\n NewBlocksCount: {}" + "\n NewBlocksChunkCount: {}" + "\n NewBlocksChunkByteCount: {}", + NiceTimeSpanMs(UploadOp.m_FindBlocksStats.FindBlockTimeMS), + UploadOp.m_FindBlocksStats.PotentialChunkCount, + NiceBytes(UploadOp.m_FindBlocksStats.PotentialChunkByteCount), + UploadOp.m_FindBlocksStats.FoundBlockCount, + UploadOp.m_FindBlocksStats.FoundBlockChunkCount, + NiceBytes(UploadOp.m_FindBlocksStats.FoundBlockByteCount), + UploadOp.m_FindBlocksStats.AcceptedBlockCount, + UploadOp.m_FindBlocksStats.NewBlocksCount, + UploadOp.m_FindBlocksStats.NewBlocksChunkCount, + NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount)); + + ZEN_CONSOLE_VERBOSE( + "Reuse block stats:" + "\n AcceptedChunkCount: {}" + "\n AcceptedByteCount: {}" + "\n AcceptedRawByteCount: {}" + "\n RejectedBlockCount: {}" + "\n RejectedChunkCount: {}" + "\n RejectedByteCount: {}" + "\n AcceptedReduntantChunkCount: {}" + "\n AcceptedReduntantByteCount: {}", + UploadOp.m_ReuseBlocksStats.AcceptedChunkCount, + NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedByteCount), + NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedRawByteCount), + UploadOp.m_ReuseBlocksStats.RejectedBlockCount, + UploadOp.m_ReuseBlocksStats.RejectedChunkCount, + NiceBytes(UploadOp.m_ReuseBlocksStats.RejectedByteCount), + UploadOp.m_ReuseBlocksStats.AcceptedReduntantChunkCount, + NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedReduntantByteCount)); + + ZEN_CONSOLE_VERBOSE( + "Generate blocks stats:" + "\n GeneratedBlockByteCount: {}" + "\n GeneratedBlockCount: {}" + "\n GenerateBlocksElapsedWallTimeUS: {}", + NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), + UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), + NiceLatencyNs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000)); + + ZEN_CONSOLE_VERBOSE( + "Generate blocks stats:" + "\n ChunkCount: {}" + "\n ChunkByteCount: {}" + "\n CompressedChunkCount: {}" + "\n CompressChunksElapsedWallTimeUS: {}", + UploadOp.m_LooseChunksStats.ChunkCount, + NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount), + UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), + NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), + NiceLatencyNs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000)); + + ZEN_CONSOLE_VERBOSE( + "Disk stats:" + "\n OpenReadCount: {}" + "\n OpenWriteCount: {}" + "\n ReadCount: {}" + "\n ReadByteCount: {}" + "\n WriteCount: {} ({} cloned)" + "\n WriteByteCount: {} ({} cloned)" + "\n CurrentOpenFileCount: {}", + UploadOp.m_DiskStats.OpenReadCount.load(), + UploadOp.m_DiskStats.OpenWriteCount.load(), + UploadOp.m_DiskStats.ReadCount.load(), + NiceBytes(UploadOp.m_DiskStats.ReadByteCount.load()), + UploadOp.m_DiskStats.WriteCount.load(), + UploadOp.m_DiskStats.CloneCount.load(), + NiceBytes(UploadOp.m_DiskStats.WriteByteCount.load()), + NiceBytes(UploadOp.m_DiskStats.CloneByteCount.load()), + UploadOp.m_DiskStats.CurrentOpenFileCount.load()); + + ZEN_CONSOLE_VERBOSE( + "Upload stats:" + "\n BlockCount: {}" + "\n BlocksBytes: {}" + "\n ChunkCount: {}" + "\n ChunksBytes: {}" + "\n ReadFromDiskBytes: {}" + "\n MultipartAttachmentCount: {}" + "\n ElapsedWallTimeUS: {}", + UploadOp.m_UploadStats.BlockCount.load(), + NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), + UploadOp.m_UploadStats.ChunkCount.load(), + NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), + NiceBytes(UploadOp.m_UploadStats.ReadFromDiskBytes.load()), + UploadOp.m_UploadStats.MultipartAttachmentCount.load(), + NiceLatencyNs(UploadOp.m_UploadStats.ElapsedWallTimeUS * 1000)); + + const double DeltaByteCountPercent = + UploadOp.m_ChunkingStats.BytesHashed > 0 + ? (100.0 * (UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes)) / + (UploadOp.m_ChunkingStats.BytesHashed) + : 0.0; + + const std::string MultipartAttachmentStats = + Options.AllowMultiparts ? fmt::format(" ({} as multipart)", UploadOp.m_UploadStats.MultipartAttachmentCount.load()) : ""; + + if (!IsQuiet) + { + ZEN_CONSOLE( + "Uploaded part {} ('{}') to build {}, {}\n" + " Scanned files: {:>8} ({}), {}B/sec, {}\n" + " New data: {:>8} ({}) {:.1f}%\n" + " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n" + " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n" + " Uploaded: {:>8} ({}), {}bits/sec, {}\n" + " Blocks: {:>8} ({})\n" + " Chunks: {:>8} ({}){}", BuildPartId, BuildPartName, - Path, - ManifestPath, - CreateBuild, - std::move(MetaData), - BuildsOperationUploadFolder::Options{.IsQuiet = IsQuiet, - .IsVerbose = IsVerbose, - .DoExtraContentValidation = DoExtraContentVerify, - .FindBlockMaxCount = FindBlockMaxCount, - .BlockReuseMinPercentLimit = BlockReuseMinPercentLimit, - .AllowMultiparts = AllowMultiparts, - .IgnoreExistingBlocks = IgnoreExistingBlocks, - .TempDir = TempDir, - .ExcludeFolders = DefaultExcludeFolders, - .ExcludeExtensions = DefaultExcludeExtensions, - .ZenExcludeManifestName = ZenExcludeManifestName, - .NonCompressableExtensions = DefaultSplitOnlyExtensions, - .PopulateCache = UploadToZenCache}); - UploadOp.Execute(); - if (AbortFlag) - { - return; - } + BuildId, + NiceTimeSpanMs(UploadTimer.GetElapsedTimeMs()), - ZEN_CONSOLE_VERBOSE( - "Folder scanning stats:" - "\n FoundFileCount: {}" - "\n FoundFileByteCount: {}" - "\n AcceptedFileCount: {}" - "\n AcceptedFileByteCount: {}" - "\n ElapsedWallTimeUS: {}", UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), - UploadOp.m_LocalFolderScanStats.AcceptedFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.AcceptedFileByteCount.load()), - NiceLatencyNs(UploadOp.m_LocalFolderScanStats.ElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Chunking stats:" - "\n FilesProcessed: {}" - "\n FilesChunked: {}" - "\n BytesHashed: {}" - "\n UniqueChunksFound: {}" - "\n UniqueSequencesFound: {}" - "\n UniqueBytesFound: {}" - "\n ElapsedWallTimeUS: {}", - UploadOp.m_ChunkingStats.FilesProcessed.load(), - UploadOp.m_ChunkingStats.FilesChunked.load(), - NiceBytes(UploadOp.m_ChunkingStats.BytesHashed.load()), - UploadOp.m_ChunkingStats.UniqueChunksFound.load(), - UploadOp.m_ChunkingStats.UniqueSequencesFound.load(), - NiceBytes(UploadOp.m_ChunkingStats.UniqueBytesFound.load()), - NiceLatencyNs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Find block stats:" - "\n FindBlockTimeMS: {}" - "\n PotentialChunkCount: {}" - "\n PotentialChunkByteCount: {}" - "\n FoundBlockCount: {}" - "\n FoundBlockChunkCount: {}" - "\n FoundBlockByteCount: {}" - "\n AcceptedBlockCount: {}" - "\n NewBlocksCount: {}" - "\n NewBlocksChunkCount: {}" - "\n NewBlocksChunkByteCount: {}", - NiceTimeSpanMs(UploadOp.m_FindBlocksStats.FindBlockTimeMS), - UploadOp.m_FindBlocksStats.PotentialChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.PotentialChunkByteCount), - UploadOp.m_FindBlocksStats.FoundBlockCount, - UploadOp.m_FindBlocksStats.FoundBlockChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.FoundBlockByteCount), - UploadOp.m_FindBlocksStats.AcceptedBlockCount, - UploadOp.m_FindBlocksStats.NewBlocksCount, - UploadOp.m_FindBlocksStats.NewBlocksChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount)); - - ZEN_CONSOLE_VERBOSE( - "Reuse block stats:" - "\n AcceptedChunkCount: {}" - "\n AcceptedByteCount: {}" - "\n AcceptedRawByteCount: {}" - "\n RejectedBlockCount: {}" - "\n RejectedChunkCount: {}" - "\n RejectedByteCount: {}" - "\n AcceptedReduntantChunkCount: {}" - "\n AcceptedReduntantByteCount: {}", - UploadOp.m_ReuseBlocksStats.AcceptedChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedByteCount), - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedRawByteCount), - UploadOp.m_ReuseBlocksStats.RejectedBlockCount, - UploadOp.m_ReuseBlocksStats.RejectedChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.RejectedByteCount), - UploadOp.m_ReuseBlocksStats.AcceptedReduntantChunkCount, - NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedReduntantByteCount)); - - ZEN_CONSOLE_VERBOSE( - "Generate blocks stats:" - "\n GeneratedBlockByteCount: {}" - "\n GeneratedBlockCount: {}" - "\n GenerateBlocksElapsedWallTimeUS: {}", - NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), + NiceNum(GetBytesPerSecond(UploadOp.m_ChunkingStats.ElapsedWallTimeUS, UploadOp.m_ChunkingStats.BytesHashed)), + NiceTimeSpanMs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS / 1000), + + UploadOp.m_FindBlocksStats.NewBlocksChunkCount + UploadOp.m_LooseChunksStats.CompressedChunkCount, + NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes), + DeltaByteCountPercent, + UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), - NiceLatencyNs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Generate blocks stats:" - "\n ChunkCount: {}" - "\n ChunkByteCount: {}" - "\n CompressedChunkCount: {}" - "\n CompressChunksElapsedWallTimeUS: {}", - UploadOp.m_LooseChunksStats.ChunkCount, - NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount), + NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount), + NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), + NiceNum(GetBytesPerSecond(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, + UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount)), + NiceTimeSpanMs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000), + UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), + NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkRawBytes), NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), - NiceLatencyNs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000)); - - ZEN_CONSOLE_VERBOSE( - "Disk stats:" - "\n OpenReadCount: {}" - "\n OpenWriteCount: {}" - "\n ReadCount: {}" - "\n ReadByteCount: {}" - "\n WriteCount: {} ({} cloned)" - "\n WriteByteCount: {} ({} cloned)" - "\n CurrentOpenFileCount: {}", - UploadOp.m_DiskStats.OpenReadCount.load(), - UploadOp.m_DiskStats.OpenWriteCount.load(), - UploadOp.m_DiskStats.ReadCount.load(), - NiceBytes(UploadOp.m_DiskStats.ReadByteCount.load()), - UploadOp.m_DiskStats.WriteCount.load(), - UploadOp.m_DiskStats.CloneCount.load(), - NiceBytes(UploadOp.m_DiskStats.WriteByteCount.load()), - NiceBytes(UploadOp.m_DiskStats.CloneByteCount.load()), - UploadOp.m_DiskStats.CurrentOpenFileCount.load()); - - ZEN_CONSOLE_VERBOSE( - "Upload stats:" - "\n BlockCount: {}" - "\n BlocksBytes: {}" - "\n ChunkCount: {}" - "\n ChunksBytes: {}" - "\n ReadFromDiskBytes: {}" - "\n MultipartAttachmentCount: {}" - "\n ElapsedWallTimeUS: {}", + NiceNum(GetBytesPerSecond(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS, + UploadOp.m_LooseChunksStats.CompressedChunkRawBytes)), + NiceTimeSpanMs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000), + + UploadOp.m_UploadStats.BlockCount.load() + UploadOp.m_UploadStats.ChunkCount.load(), + NiceBytes(UploadOp.m_UploadStats.BlocksBytes + UploadOp.m_UploadStats.ChunksBytes), + NiceNum(GetBytesPerSecond(UploadOp.m_UploadStats.ElapsedWallTimeUS, + (UploadOp.m_UploadStats.ChunksBytes + UploadOp.m_UploadStats.BlocksBytes) * 8)), + NiceTimeSpanMs(UploadOp.m_UploadStats.ElapsedWallTimeUS / 1000), + UploadOp.m_UploadStats.BlockCount.load(), NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), + UploadOp.m_UploadStats.ChunkCount.load(), NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), - NiceBytes(UploadOp.m_UploadStats.ReadFromDiskBytes.load()), - UploadOp.m_UploadStats.MultipartAttachmentCount.load(), - NiceLatencyNs(UploadOp.m_UploadStats.ElapsedWallTimeUS * 1000)); - - const double DeltaByteCountPercent = - UploadOp.m_ChunkingStats.BytesHashed > 0 - ? (100.0 * (UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes)) / - (UploadOp.m_ChunkingStats.BytesHashed) - : 0.0; - - const std::string MultipartAttachmentStats = - AllowMultiparts ? fmt::format(" ({} as multipart)", UploadOp.m_UploadStats.MultipartAttachmentCount.load()) : ""; - - if (!IsQuiet) - { - ZEN_CONSOLE( - "Uploaded part {} ('{}') to build {}, {}\n" - " Scanned files: {:>8} ({}), {}B/sec, {}\n" - " New data: {:>8} ({}) {:.1f}%\n" - " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n" - " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n" - " Uploaded: {:>8} ({}), {}bits/sec, {}\n" - " Blocks: {:>8} ({})\n" - " Chunks: {:>8} ({}){}", - BuildPartId, - BuildPartName, - BuildId, - NiceTimeSpanMs(UploadTimer.GetElapsedTimeMs()), - - UploadOp.m_LocalFolderScanStats.FoundFileCount.load(), - NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_ChunkingStats.ElapsedWallTimeUS, UploadOp.m_ChunkingStats.BytesHashed)), - NiceTimeSpanMs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS / 1000), - - UploadOp.m_FindBlocksStats.NewBlocksChunkCount + UploadOp.m_LooseChunksStats.CompressedChunkCount, - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes), - DeltaByteCountPercent, - - UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(), - NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount), - NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, - UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount)), - NiceTimeSpanMs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000), - - UploadOp.m_LooseChunksStats.CompressedChunkCount.load(), - NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount), - NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()), - NiceNum(GetBytesPerSecond(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS, - UploadOp.m_LooseChunksStats.ChunkByteCount)), - NiceTimeSpanMs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000), - - UploadOp.m_UploadStats.BlockCount.load() + UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes + UploadOp.m_UploadStats.ChunksBytes), - NiceNum(GetBytesPerSecond(UploadOp.m_UploadStats.ElapsedWallTimeUS, - (UploadOp.m_UploadStats.ChunksBytes + UploadOp.m_UploadStats.BlocksBytes) * 8)), - NiceTimeSpanMs(UploadOp.m_UploadStats.ElapsedWallTimeUS / 1000), - - UploadOp.m_UploadStats.BlockCount.load(), - NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()), - - UploadOp.m_UploadStats.ChunkCount.load(), - NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()), - MultipartAttachmentStats); - } + MultipartAttachmentStats); } + return UploadedParts; } struct VerifyFolderStatistics @@ -682,12 +695,13 @@ namespace { uint64_t VerifyElapsedWallTimeUs = 0; }; - void VerifyFolder(TransferThreadWorkers& Workers, - const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - const std::filesystem::path& Path, - bool VerifyFileHash, - VerifyFolderStatistics& VerifyFolderStats) + void VerifyFolder(TransferThreadWorkers& Workers, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const std::filesystem::path& Path, + const std::vector<std::string>& ExcludeFolders, + bool VerifyFileHash, + VerifyFolderStatistics& VerifyFolderStats) { ZEN_TRACE_CPU("VerifyFolder"); @@ -704,7 +718,7 @@ namespace { RwLock ErrorLock; std::vector<std::string> Errors; - auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders](const std::string_view& RelativePath) -> bool { + auto IsAcceptedFolder = [ExcludeFolders = ExcludeFolders](const std::string_view& RelativePath) -> bool { for (const std::string& ExcludeFolder : ExcludeFolders) { if (RelativePath.starts_with(ExcludeFolder)) @@ -875,107 +889,6 @@ namespace { } } - std::string GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix) - { - ExtendableStringBuilder<512> SB; - std::vector<std::pair<std::string, std::string>> NameStringValuePairs; - for (CbFieldView Field : Object) - { - std::string_view Name = Field.GetName(); - switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) - { - case CbFieldType::String: - NameStringValuePairs.push_back({std::string(Name), std::string(Accessor.AsString())}); - break; - case CbFieldType::IntegerPositive: - NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerPositive())}); - break; - case CbFieldType::IntegerNegative: - NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerNegative())}); - break; - case CbFieldType::Float32: - { - const float Value = Accessor.AsFloat32(); - if (std::isfinite(Value)) - { - NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.9g}", Value)}); - } - else - { - NameStringValuePairs.push_back({std::string(Name), "null"}); - } - } - break; - case CbFieldType::Float64: - { - const double Value = Accessor.AsFloat64(); - if (std::isfinite(Value)) - { - NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.17g}", Value)}); - } - else - { - NameStringValuePairs.push_back({std::string(Name), "null"}); - } - } - break; - case CbFieldType::BoolFalse: - NameStringValuePairs.push_back({std::string(Name), "false"}); - break; - case CbFieldType::BoolTrue: - NameStringValuePairs.push_back({std::string(Name), "true"}); - break; - case CbFieldType::Hash: - { - NameStringValuePairs.push_back({std::string(Name), Accessor.AsHash().ToHexString()}); - } - break; - case CbFieldType::Uuid: - { - StringBuilder<Oid::StringLength + 1> Builder; - Accessor.AsUuid().ToString(Builder); - NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); - } - break; - case CbFieldType::DateTime: - { - ExtendableStringBuilder<64> Builder; - Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); - NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); - } - break; - case CbFieldType::TimeSpan: - { - ExtendableStringBuilder<64> Builder; - const TimeSpan Span(Accessor.AsTimeSpanTicks()); - if (Span.GetDays() == 0) - { - Builder << Span.ToString("%h:%m:%s.%n"); - } - else - { - Builder << Span.ToString("%d.%h:%m:%s.%n"); - } - NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); - break; - } - case CbFieldType::ObjectId: - NameStringValuePairs.push_back({std::string(Name), Accessor.AsObjectId().ToString()}); - break; - } - } - std::string::size_type LongestKey = 0; - for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs) - { - LongestKey = Max(KeyValue.first.length(), LongestKey); - } - for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs) - { - SB.Append(fmt::format("{}{:<{}}: {}{}", Prefix, KeyValue.first, LongestKey, KeyValue.second, Suffix)); - } - return SB.ToString(); - } - CbObject GetBuild(BuildStorageBase& Storage, const Oid& BuildId) { Stopwatch GetBuildTimer; @@ -992,280 +905,6 @@ namespace { return BuildObject; } - std::vector<std::pair<Oid, std::string>> ResolveBuildPartNames(CbObjectView BuildObject, - const Oid& BuildId, - const std::vector<Oid>& BuildPartIds, - std::span<const std::string> BuildPartNames, - std::uint64_t& OutPreferredMultipartChunkSize) - { - std::vector<std::pair<Oid, std::string>> Result; - { - CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); - if (!PartsObject) - { - throw std::runtime_error("Build object does not have a 'parts' object"); - } - - OutPreferredMultipartChunkSize = BuildObject["chunkSize"sv].AsUInt64(OutPreferredMultipartChunkSize); - - std::vector<std::pair<Oid, std::string>> AvailableParts; - - for (CbFieldView PartView : PartsObject) - { - const std::string BuildPartName = std::string(PartView.GetName()); - const Oid BuildPartId = PartView.AsObjectId(); - if (BuildPartId == Oid::Zero) - { - ExtendableStringBuilder<128> SB; - for (CbFieldView ScanPartView : PartsObject) - { - SB.Append(fmt::format("\n {}: {}", ScanPartView.GetName(), ScanPartView.AsObjectId())); - } - throw std::runtime_error( - fmt::format("Build object parts does not have a '{}' object id{}", BuildPartName, SB.ToView())); - } - AvailableParts.push_back({BuildPartId, BuildPartName}); - } - - if (BuildPartIds.empty() && BuildPartNames.empty()) - { - Result = AvailableParts; - } - else - { - for (const std::string& BuildPartName : BuildPartNames) - { - if (auto It = std::find_if(AvailableParts.begin(), - AvailableParts.end(), - [&BuildPartName](const auto& Part) { return Part.second == BuildPartName; }); - It != AvailableParts.end()) - { - Result.push_back(*It); - } - else - { - throw std::runtime_error(fmt::format("Build {} object does not have a part named '{}'", BuildId, BuildPartName)); - } - } - for (const Oid& BuildPartId : BuildPartIds) - { - if (auto It = std::find_if(AvailableParts.begin(), - AvailableParts.end(), - [&BuildPartId](const auto& Part) { return Part.first == BuildPartId; }); - It != AvailableParts.end()) - { - Result.push_back(*It); - } - else - { - throw std::runtime_error(fmt::format("Build {} object does not have a part with id '{}'", BuildId, BuildPartId)); - } - } - } - - if (Result.empty()) - { - throw std::runtime_error(fmt::format("Build object does not have any parts", BuildId)); - } - } - return Result; - } - - ChunkedFolderContent GetRemoteContent(OperationLogOutput& Output, - StorageInstance& Storage, - const Oid& BuildId, - const std::vector<std::pair<Oid, std::string>>& BuildParts, - std::span<const std::string> IncludeWildcards, - std::span<const std::string> ExcludeWildcards, - std::unique_ptr<ChunkingController>& OutChunkController, - std::vector<ChunkedFolderContent>& OutPartContents, - std::vector<ChunkBlockDescription>& OutBlockDescriptions, - std::vector<IoHash>& OutLooseChunkHashes) - { - ZEN_TRACE_CPU("GetRemoteContent"); - - Stopwatch GetBuildPartTimer; - const Oid BuildPartId = BuildParts[0].first; - const std::string_view BuildPartName = BuildParts[0].second; - CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); - if (!IsQuiet) - { - ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", - BuildPartId, - BuildPartName, - NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), - NiceBytes(BuildPartManifest.GetSize())); - ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv)); - } - - { - CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView(); - std::string_view ChunkerName = Chunker["name"sv].AsString(); - CbObjectView Parameters = Chunker["parameters"sv].AsObjectView(); - OutChunkController = CreateChunkingController(ChunkerName, Parameters); - } - - auto ParseBuildPartManifest = [&Output](StorageInstance& Storage, - const Oid& BuildId, - const Oid& BuildPartId, - CbObject BuildPartManifest, - std::span<const std::string> IncludeWildcards, - std::span<const std::string> ExcludeWildcards, - ChunkedFolderContent& OutRemoteContent, - std::vector<ChunkBlockDescription>& OutBlockDescriptions, - std::vector<IoHash>& OutLooseChunkHashes) { - std::vector<uint32_t> AbsoluteChunkOrders; - std::vector<uint64_t> LooseChunkRawSizes; - std::vector<IoHash> BlockRawHashes; - - ReadBuildContentFromCompactBinary(BuildPartManifest, - OutRemoteContent.Platform, - OutRemoteContent.Paths, - OutRemoteContent.RawHashes, - OutRemoteContent.RawSizes, - OutRemoteContent.Attributes, - OutRemoteContent.ChunkedContent.SequenceRawHashes, - OutRemoteContent.ChunkedContent.ChunkCounts, - AbsoluteChunkOrders, - OutLooseChunkHashes, - LooseChunkRawSizes, - BlockRawHashes); - - // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them - - { - bool AttemptFallback = false; - OutBlockDescriptions = GetBlockDescriptions(Output, - *Storage.BuildStorage, - Storage.BuildCacheStorage.get(), - BuildId, - BuildPartId, - BlockRawHashes, - AttemptFallback, - IsQuiet, - IsVerbose); - } - - CalculateLocalChunkOrders(AbsoluteChunkOrders, - OutLooseChunkHashes, - LooseChunkRawSizes, - OutBlockDescriptions, - OutRemoteContent.ChunkedContent.ChunkHashes, - OutRemoteContent.ChunkedContent.ChunkRawSizes, - OutRemoteContent.ChunkedContent.ChunkOrders, - DoExtraContentVerify); - - if (!IncludeWildcards.empty() || !ExcludeWildcards.empty()) - { - std::vector<std::filesystem::path> DeletedPaths; - for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths) - { - if (!IncludePath(IncludeWildcards, ExcludeWildcards, RemotePath)) - { - DeletedPaths.push_back(RemotePath); - } - } - - if (!DeletedPaths.empty()) - { - OutRemoteContent = DeletePathsFromChunkedContent(OutRemoteContent, DeletedPaths); - InlineRemoveUnusedHashes(OutLooseChunkHashes, OutRemoteContent.ChunkedContent.ChunkHashes); - } - } - -#if ZEN_BUILD_DEBUG - ValidateChunkedFolderContent(OutRemoteContent, OutBlockDescriptions, OutLooseChunkHashes, IncludeWildcards, ExcludeWildcards); -#endif // ZEN_BUILD_DEBUG - }; - - OutPartContents.resize(1); - ParseBuildPartManifest(Storage, - BuildId, - BuildPartId, - BuildPartManifest, - IncludeWildcards, - ExcludeWildcards, - OutPartContents[0], - OutBlockDescriptions, - OutLooseChunkHashes); - ChunkedFolderContent RemoteContent; - if (BuildParts.size() > 1) - { - std::vector<ChunkBlockDescription> OverlayBlockDescriptions; - std::vector<IoHash> OverlayLooseChunkHashes; - for (size_t PartIndex = 1; PartIndex < BuildParts.size(); PartIndex++) - { - const Oid& OverlayBuildPartId = BuildParts[PartIndex].first; - const std::string& OverlayBuildPartName = BuildParts[PartIndex].second; - Stopwatch GetOverlayBuildPartTimer; - CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId); - if (!IsQuiet) - { - ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", - OverlayBuildPartId, - OverlayBuildPartName, - NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()), - NiceBytes(OverlayBuildPartManifest.GetSize())); - } - - ChunkedFolderContent OverlayPartContent; - std::vector<ChunkBlockDescription> OverlayPartBlockDescriptions; - std::vector<IoHash> OverlayPartLooseChunkHashes; - - ParseBuildPartManifest(Storage, - BuildId, - OverlayBuildPartId, - OverlayBuildPartManifest, - IncludeWildcards, - ExcludeWildcards, - OverlayPartContent, - OverlayPartBlockDescriptions, - OverlayPartLooseChunkHashes); - OutPartContents.push_back(OverlayPartContent); - OverlayBlockDescriptions.insert(OverlayBlockDescriptions.end(), - OverlayPartBlockDescriptions.begin(), - OverlayPartBlockDescriptions.end()); - OverlayLooseChunkHashes.insert(OverlayLooseChunkHashes.end(), - OverlayPartLooseChunkHashes.begin(), - OverlayPartLooseChunkHashes.end()); - } - - RemoteContent = - MergeChunkedFolderContents(OutPartContents[0], std::span<const ChunkedFolderContent>(OutPartContents).subspan(1)); - { - tsl::robin_set<IoHash> AllBlockHashes; - for (const ChunkBlockDescription& Description : OutBlockDescriptions) - { - AllBlockHashes.insert(Description.BlockHash); - } - for (const ChunkBlockDescription& Description : OverlayBlockDescriptions) - { - if (!AllBlockHashes.contains(Description.BlockHash)) - { - AllBlockHashes.insert(Description.BlockHash); - OutBlockDescriptions.push_back(Description); - } - } - } - { - tsl::robin_set<IoHash> AllLooseChunkHashes(OutLooseChunkHashes.begin(), OutLooseChunkHashes.end()); - for (const IoHash& OverlayLooseChunkHash : OverlayLooseChunkHashes) - { - if (!AllLooseChunkHashes.contains(OverlayLooseChunkHash)) - { - AllLooseChunkHashes.insert(OverlayLooseChunkHash); - OutLooseChunkHashes.push_back(OverlayLooseChunkHash); - } - } - } - } - else - { - RemoteContent = OutPartContents[0]; - } - return RemoteContent; - } - std::vector<std::filesystem::path> GetNewPaths(const std::span<const std::filesystem::path> KnownPaths, const std::span<const std::filesystem::path> Paths) { @@ -1292,6 +931,7 @@ namespace { ChunkingStatistics& ChunkingStats, const std::filesystem::path& Path, ChunkingController& ChunkController, + ChunkingCache& ChunkCache, std::span<const std::filesystem::path> PathsToCheck) { FolderContent FolderState; @@ -1336,6 +976,7 @@ namespace { Path, FolderState, ChunkController, + ChunkCache, GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load()); @@ -1371,7 +1012,8 @@ namespace { ChunkingStatistics& ChunkingStats, const std::filesystem::path& Path, const std::filesystem::path& StateFilePath, - ChunkingController& ChunkController) + ChunkingController& ChunkController, + ChunkingCache& ChunkCache) { Stopwatch ReadStateTimer; bool FileExists = IsFile(StateFilePath); @@ -1458,6 +1100,7 @@ namespace { Path, UpdatedContent, ChunkController, + ChunkCache, GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load()); @@ -1526,7 +1169,8 @@ namespace { const std::filesystem::path& Path, std::function<bool(const std::string_view& RelativePath)>&& IsAcceptedFolder, std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile, - ChunkingController& ChunkController) + ChunkingController& ChunkController, + ChunkingCache& ChunkCache) { Stopwatch Timer; @@ -1546,14 +1190,19 @@ namespace { return {}; } - BuildState LocalContent = - GetLocalContent(Workers, GetFolderContentStats, ChunkingStats, Path, ZenStateFilePath(Path / ZenFolderName), ChunkController) - .State; + BuildState LocalContent = GetLocalContent(Workers, + GetFolderContentStats, + ChunkingStats, + Path, + ZenStateFilePath(Path / ZenFolderName), + ChunkController, + ChunkCache) + .State; std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalContent.ChunkedContent.Paths, Content.Paths); BuildState UntrackedLocalContent = - GetLocalStateFromPaths(Workers, GetFolderContentStats, ChunkingStats, Path, ChunkController, UntrackedPaths).State; + GetLocalStateFromPaths(Workers, GetFolderContentStats, ChunkingStats, Path, ChunkController, ChunkCache, UntrackedPaths).State; ChunkedFolderContent Result = MergeChunkedFolderContents(LocalContent.ChunkedContent, std::vector<ChunkedFolderContent>{UntrackedLocalContent.ChunkedContent}); @@ -1593,6 +1242,7 @@ namespace { uint64_t MaximumInMemoryPayloadSize = 512u * 1024u; bool PopulateCache = true; bool AppendNewContent = false; + std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; }; void DownloadFolder(OperationLogOutput& Output, @@ -1602,6 +1252,7 @@ namespace { const Oid& BuildId, const std::vector<Oid>& BuildPartIds, std::span<const std::string> BuildPartNames, + const std::filesystem::path& DownloadSpecPath, const std::filesystem::path& Path, const DownloadOptions& Options) { @@ -1639,6 +1290,14 @@ namespace { std::vector<std::pair<Oid, std::string>> AllBuildParts = ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); + BuildManifest Manifest; + if (!DownloadSpecPath.empty()) + { + const std::filesystem::path AbsoluteDownloadSpecPath = + DownloadSpecPath.is_relative() ? MakeSafeAbsolutePath(Path / DownloadSpecPath) : MakeSafeAbsolutePath(DownloadSpecPath); + Manifest = ParseBuildManifest(DownloadSpecPath); + } + std::vector<ChunkedFolderContent> PartContents; std::unique_ptr<ChunkingController> ChunkController; @@ -1652,15 +1311,17 @@ namespace { Storage, BuildId, AllBuildParts, + Manifest, Options.IncludeWildcards, Options.ExcludeWildcards, ChunkController, PartContents, BlockDescriptions, - LooseChunkHashes); -#if ZEN_BUILD_DEBUG - ValidateChunkedFolderContent(RemoteContent, BlockDescriptions, LooseChunkHashes, {}, {}); -#endif // ZEN_BUILD_DEBUG + LooseChunkHashes, + IsQuiet, + IsVerbose, + DoExtraContentVerify); + const std::uint64_t LargeAttachmentSize = Options.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; GetFolderContentStatistics LocalFolderScanStats; ChunkingStatistics ChunkingStats; @@ -1676,18 +1337,25 @@ namespace { ZEN_CONSOLE_INFO("Unspecified chunking algorithm, using default"); ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); } + std::unique_ptr<ChunkingCache> ChunkCache(CreateNullChunkingCache()); LocalState = GetLocalContent(Workers, LocalFolderScanStats, ChunkingStats, Path, ZenStateFilePath(Path / ZenFolderName), - *ChunkController); + *ChunkController, + *ChunkCache); std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalState.State.ChunkedContent.Paths, RemoteContent.Paths); - BuildSaveState UntrackedLocalContent = - GetLocalStateFromPaths(Workers, LocalFolderScanStats, ChunkingStats, Path, *ChunkController, UntrackedPaths); + BuildSaveState UntrackedLocalContent = GetLocalStateFromPaths(Workers, + LocalFolderScanStats, + ChunkingStats, + Path, + *ChunkController, + *ChunkCache, + UntrackedPaths); if (!UntrackedLocalContent.State.ChunkedContent.Paths.empty()) { @@ -1834,8 +1502,7 @@ namespace { .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging, .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging || Options.AppendNewContent, .ValidateCompletedSequences = Options.PostDownloadVerify, - .ExcludeFolders = DefaultExcludeFolders, - .ExcludeExtensions = DefaultExcludeExtensions, + .ExcludeFolders = Options.ExcludeFolders, .MaximumInMemoryPayloadSize = Options.MaximumInMemoryPayloadSize, .PopulateCache = Options.PopulateCache}); { @@ -1861,7 +1528,13 @@ namespace { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount); - VerifyFolder(Workers, RemoteContent, RemoteLookup, Path, Options.PostDownloadVerify, VerifyFolderStats); + VerifyFolder(Workers, + RemoteContent, + RemoteLookup, + Path, + Options.ExcludeFolders, + Options.PostDownloadVerify, + VerifyFolderStats); Stopwatch WriteStateTimer; CbObject StateObject = CreateBuildSaveStateObject(LocalState); @@ -1999,88 +1672,158 @@ namespace { const std::vector<Oid>& BuildPartIds, std::span<const std::string> BuildPartNames, std::span<const std::string> IncludeWildcards, - std::span<const std::string> ExcludeWildcards) + std::span<const std::string> ExcludeWildcards, + CbObjectWriter* OptionalStructuredOutput) { std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId); + if (OptionalStructuredOutput != nullptr) + { + OptionalStructuredOutput->AddObjectId("buildId"sv, BuildId); + OptionalStructuredOutput->AddObject("build"sv, BuildObject); + } + std::vector<std::pair<Oid, std::string>> AllBuildParts = ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); - Stopwatch GetBuildPartTimer; - - for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++) + if (!AllBuildParts.empty()) { - const Oid BuildPartId = AllBuildParts[BuildPartIndex].first; - const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second; - CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); + Stopwatch GetBuildPartTimer; - if (!IsQuiet) + if (OptionalStructuredOutput != nullptr) { - ZEN_CONSOLE("{}Part: {} ('{}'):\n", - BuildPartIndex > 0 ? "\n" : "", - BuildPartId, - BuildPartName, - NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), - NiceBytes(BuildPartManifest.GetSize())); + OptionalStructuredOutput->BeginArray("parts"sv); } - std::vector<std::filesystem::path> Paths; - std::vector<IoHash> RawHashes; - std::vector<uint64_t> RawSizes; - std::vector<uint32_t> Attributes; - - SourcePlatform Platform; - std::vector<IoHash> SequenceRawHashes; - std::vector<uint32_t> ChunkCounts; - std::vector<uint32_t> AbsoluteChunkOrders; - std::vector<IoHash> LooseChunkHashes; - std::vector<uint64_t> LooseChunkRawSizes; - std::vector<IoHash> BlockRawHashes; - - ReadBuildContentFromCompactBinary(BuildPartManifest, - Platform, - Paths, - RawHashes, - RawSizes, - Attributes, - SequenceRawHashes, - ChunkCounts, - AbsoluteChunkOrders, - LooseChunkHashes, - LooseChunkRawSizes, - BlockRawHashes); - - std::vector<size_t> Order(Paths.size()); - std::iota(Order.begin(), Order.end(), 0); - - std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) { - const std::filesystem::path& LhsPath = Paths[Lhs]; - const std::filesystem::path& RhsPath = Paths[Rhs]; - return LhsPath < RhsPath; - }); - - for (size_t Index : Order) + for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++) { - const std::filesystem::path& Path = Paths[Index]; - if (IncludePath(IncludeWildcards, ExcludeWildcards, Path)) + const Oid BuildPartId = AllBuildParts[BuildPartIndex].first; + const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second; + CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); + + if (OptionalStructuredOutput != nullptr) { - const IoHash& RawHash = RawHashes[Index]; - const uint64_t RawSize = RawSizes[Index]; - const uint32_t Attribute = Attributes[Index]; - ZEN_UNUSED(Attribute); + OptionalStructuredOutput->BeginObject(); + OptionalStructuredOutput->AddObjectId("id"sv, BuildPartId); + OptionalStructuredOutput->AddString("partName"sv, BuildPartName); + } + { + if (OptionalStructuredOutput != nullptr) + { + } + else if (!IsQuiet) + { + ZEN_CONSOLE("{}Part: {} ('{}'):\n", + BuildPartIndex > 0 ? "\n" : "", + BuildPartId, + BuildPartName, + NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), + NiceBytes(BuildPartManifest.GetSize())); + } - ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash); + std::vector<std::filesystem::path> Paths; + std::vector<IoHash> RawHashes; + std::vector<uint64_t> RawSizes; + std::vector<uint32_t> Attributes; + + SourcePlatform Platform; + std::vector<IoHash> SequenceRawHashes; + std::vector<uint32_t> ChunkCounts; + std::vector<uint32_t> AbsoluteChunkOrders; + std::vector<IoHash> LooseChunkHashes; + std::vector<uint64_t> LooseChunkRawSizes; + std::vector<IoHash> BlockRawHashes; + + ReadBuildContentFromCompactBinary(BuildPartManifest, + Platform, + Paths, + RawHashes, + RawSizes, + Attributes, + SequenceRawHashes, + ChunkCounts, + AbsoluteChunkOrders, + LooseChunkHashes, + LooseChunkRawSizes, + BlockRawHashes); + + std::vector<size_t> Order(Paths.size()); + std::iota(Order.begin(), Order.end(), 0); + + std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) { + const std::filesystem::path& LhsPath = Paths[Lhs]; + const std::filesystem::path& RhsPath = Paths[Rhs]; + return LhsPath < RhsPath; + }); + + if (OptionalStructuredOutput != nullptr) + { + OptionalStructuredOutput->BeginArray("files"sv); + } + { + for (size_t Index : Order) + { + const std::filesystem::path& Path = Paths[Index]; + if (IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(Path.generic_string()), /*CaseSensitive*/ true)) + { + const IoHash& RawHash = RawHashes[Index]; + const uint64_t RawSize = RawSizes[Index]; + const uint32_t Attribute = Attributes[Index]; + + if (OptionalStructuredOutput != nullptr) + { + OptionalStructuredOutput->BeginObject(); + { + OptionalStructuredOutput->AddString("path"sv, fmt::format("{}", Path)); + OptionalStructuredOutput->AddInteger("rawSize"sv, RawSize); + switch (Platform) + { + case SourcePlatform::Windows: + OptionalStructuredOutput->AddInteger("attributes"sv, Attribute); + break; + case SourcePlatform::MacOS: + case SourcePlatform::Linux: + OptionalStructuredOutput->AddString("chmod"sv, fmt::format("{:#04o}", Attribute)); + break; + default: + throw std::runtime_error(fmt::format("Unsupported platform: {}", (int)Platform)); + } + } + OptionalStructuredOutput->EndObject(); + } + else + { + ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash); + } + } + } + } + if (OptionalStructuredOutput != nullptr) + { + OptionalStructuredOutput->EndArray(); // "files" + } + } + if (OptionalStructuredOutput != nullptr) + { + OptionalStructuredOutput->EndObject(); } } + if (OptionalStructuredOutput != nullptr) + { + OptionalStructuredOutput->EndArray(); // parts + } } } - void DiffFolders(TransferThreadWorkers& Workers, - const std::filesystem::path& BasePath, - const std::filesystem::path& ComparePath, - bool OnlyChunked) + void DiffFolders(TransferThreadWorkers& Workers, + const std::filesystem::path& BasePath, + const std::filesystem::path& ComparePath, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + const std::vector<std::string>& ExcludeFolders, + const std::vector<std::string>& ExcludeExtensions) { ZEN_TRACE_CPU("DiffFolders"); @@ -2102,20 +1845,7 @@ namespace { ChunkedFolderContent CompareFolderContent; { - StandardChunkingControllerSettings ChunkingSettings; - std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(ChunkingSettings); - std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; - if (OnlyChunked) - { - ExcludeExtensions.insert(ExcludeExtensions.end(), - ChunkingSettings.SplitOnlyExtensions.begin(), - ChunkingSettings.SplitOnlyExtensions.end()); - ExcludeExtensions.insert(ExcludeExtensions.end(), - ChunkingSettings.SplitAndCompressExtensions.begin(), - ChunkingSettings.SplitAndCompressExtensions.end()); - } - - auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders](const std::string_view& RelativePath) -> bool { + auto IsAcceptedFolder = [ExcludeFolders](const std::string_view& RelativePath) -> bool { for (const std::string& ExcludeFolder : ExcludeFolders) { if (RelativePath.starts_with(ExcludeFolder)) @@ -2154,7 +1884,8 @@ namespace { BasePath, IsAcceptedFolder, IsAcceptedFile, - *ChunkController); + ChunkController, + ChunkCache); if (AbortFlag) { return; @@ -2170,7 +1901,8 @@ namespace { ComparePath, IsAcceptedFolder, IsAcceptedFile, - *ChunkController); + ChunkController, + ChunkCache); if (AbortFlag) { @@ -2387,6 +2119,15 @@ BuildsCommand::BuildsCommand() "<boostworkers>"); }; + auto AddChunkingCacheOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("", + "", + "chunking-cache-path", + "Path to cache for chunking information of scanned files. Default is empty resulting in no caching", + cxxopts::value(m_ChunkingCachePath), + "<chunkingcachepath>"); + }; + auto AddWildcardOptions = [this](cxxopts::Options& Ops) { Ops.add_option("", "", @@ -2404,6 +2145,25 @@ BuildsCommand::BuildsCommand() "<excludewildcard>"); }; + auto AddExcludeFolderOption = [this](cxxopts::Options& Ops) { + Ops.add_option("", + "", + "exclude-folders", + "Names of folders to exclude, separated by ;", + cxxopts::value(m_ExcludeFolders), + "<excludefolders>"); + }; + + auto AddExcludeExtensionsOption = [this](cxxopts::Options& Ops) { + Ops.add_option("", + "", + "exclude-extensions", + "Extensions to exclude, separated by ;" + "include filter", + cxxopts::value(m_ExcludeExtensions), + "<excludeextensions>"); + }; + auto AddMultipartOptions = [this](cxxopts::Options& Ops) { Ops.add_option("", "", @@ -2463,12 +2223,13 @@ BuildsCommand::BuildsCommand() "Enable fetch of buckets within namespaces also", cxxopts::value(m_ListNamespacesRecursive), "<recursive>"); - m_ListNamespacesOptions.add_option("", - "", - "result-path", - "Path to json or compactbinary to write query result to", - cxxopts::value(m_ListResultPath), - "<result-path>"); + m_ListNamespacesOptions.add_option( + "", + "", + "result-path", + "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console", + cxxopts::value(m_ListResultPath), + "<result-path>"); m_ListNamespacesOptions.parse_positional({"result-path"}); m_ListNamespacesOptions.positional_help("result-path"); @@ -2488,7 +2249,7 @@ BuildsCommand::BuildsCommand() m_ListOptions.add_option("", "", "result-path", - "Path to json or compactbinary to write query result to", + "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console", cxxopts::value(m_ListResultPath), "<result-path>"); m_ListOptions.parse_positional({"query-path", "result-path"}); @@ -2503,7 +2264,7 @@ BuildsCommand::BuildsCommand() m_ListBlocksOptions.add_option("", "", "result-path", - "Path to json or compactbinary to write query result to", + "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console", cxxopts::value(m_ListResultPath), "<result-path>"); @@ -2521,6 +2282,9 @@ BuildsCommand::BuildsCommand() AddCacheOptions(m_UploadOptions); AddWorkerOptions(m_UploadOptions); AddZenFolderOptions(m_UploadOptions); + AddExcludeFolderOption(m_UploadOptions); + AddExcludeExtensionsOption(m_UploadOptions); + AddChunkingCacheOptions(m_UploadOptions); m_UploadOptions.add_options()("h,help", "Print help"); m_UploadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); m_UploadOptions.add_option("", @@ -2574,9 +2338,11 @@ BuildsCommand::BuildsCommand() m_UploadOptions.add_option("", "", "manifest-path", - "Path to a text file with one line of <local path>[TAB]<modification date> per file to include.", + "Path to a text file with one line of <local path>[TAB]<modification date> per file to include or a " + "structured .json file describing the parts", cxxopts::value(m_ManifestPath), "<manifestpath>"); + m_UploadOptions .add_option("", "", "verify", "Enable post upload verify of all uploaded data", cxxopts::value(m_PostUploadVerify), "<verify>"); m_UploadOptions.add_option("", @@ -2599,6 +2365,7 @@ BuildsCommand::BuildsCommand() AddWorkerOptions(m_DownloadOptions); AddWildcardOptions(m_DownloadOptions); AddAppendNewContentOptions(m_DownloadOptions); + AddExcludeFolderOption(m_DownloadOptions); m_DownloadOptions.add_option("cache", "", @@ -2646,6 +2413,14 @@ BuildsCommand::BuildsCommand() AddPartialBlockRequestOptions(m_DownloadOptions); + m_DownloadOptions.add_option( + "", + "", + "download-spec-path", + "Path to a text file with one line of <local path> per file to include or a structured .json file describing what to download.", + cxxopts::value(m_DownloadSpecPath), + "<downloadspecpath>"); + m_DownloadOptions .add_option("", "", "verify", "Enable post download verify of all tracked files", cxxopts::value(m_PostDownloadVerify), "<verify>"); m_DownloadOptions.add_option("", @@ -2691,12 +2466,29 @@ BuildsCommand::BuildsCommand() cxxopts::value(m_BuildPartNames), "<name>"); + m_LsOptions.add_option("", + "", + "result-path", + "Path to json (.json) or compactbinary (.cbo) to write output result to. Default is output to console", + cxxopts::value(m_LsResultPath), + "<result-path>"); + + m_LsOptions.add_option("", + "o", + "output-path", + "Path to output, extension .json or .cb (compac binary). Default is output to console", + cxxopts::value(m_LsResultPath), + "<output-path>"); + m_LsOptions.parse_positional({"build-id", "wildcard"}); m_LsOptions.positional_help("build-id wildcard"); // diff AddOutputOptions(m_DiffOptions); AddWorkerOptions(m_DiffOptions); + AddExcludeFolderOption(m_DiffOptions); + AddExcludeExtensionsOption(m_DiffOptions); + AddChunkingCacheOptions(m_DiffOptions); m_DiffOptions.add_options()("h,help", "Print help"); m_DiffOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); m_DiffOptions.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>"); @@ -2722,6 +2514,7 @@ BuildsCommand::BuildsCommand() AddPartialBlockRequestOptions(m_TestOptions); AddWildcardOptions(m_TestOptions); AddAppendNewContentOptions(m_TestOptions); + AddChunkingCacheOptions(m_TestOptions); m_TestOptions.add_option("", "", @@ -3158,9 +2951,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; auto ParseFileFilters = [&](std::vector<std::string>& OutIncludeWildcards, std::vector<std::string>& OutExcludeWildcards) { - auto SplitWildcard = [](const std::string_view Wildcard) -> std::vector<std::string> { - std::vector<std::string> Wildcards; - ForEachStrTok(Wildcard, ';', [&Wildcards](std::string_view Wildcard) { + auto SplitAndAppendWildcard = [](const std::string_view Wildcard, std::vector<std::string>& Output) { + ForEachStrTok(Wildcard, ';', [&Output](std::string_view Wildcard) { if (!Wildcard.empty()) { std::string CleanWildcard(ToLower(Wildcard)); @@ -3177,15 +2969,34 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) CleanWildcard = CleanWildcard.substr(2); } - Wildcards.emplace_back(std::move(CleanWildcard)); + Output.emplace_back(std::move(CleanWildcard)); + } + return true; + }); + }; + + SplitAndAppendWildcard(m_IncludeWildcard, OutIncludeWildcards); + SplitAndAppendWildcard(m_ExcludeWildcard, OutExcludeWildcards); + }; + + auto ParseExcludeFolderAndExtension = [&](std::vector<std::string>& OutExcludeFolders, std::vector<std::string>& OutExcludeExtensions) { + auto SplitAndAppendExclusion = [](const std::string_view Input, std::vector<std::string>& Output) { + ForEachStrTok(Input, ";,", [&Output](std::string_view Exclusion) { + if (!Exclusion.empty()) + { + std::string CleanExclusion(ToLower(Exclusion)); + if (CleanExclusion.length() > 2 && CleanExclusion.front() == '"' && CleanExclusion.back() == '"') + { + CleanExclusion = CleanExclusion.substr(1, CleanExclusion.length() - 2); + } + Output.emplace_back(std::move(CleanExclusion)); } return true; }); - return Wildcards; }; - OutIncludeWildcards = SplitWildcard(m_IncludeWildcard); - OutExcludeWildcards = SplitWildcard(m_ExcludeWildcard); + SplitAndAppendExclusion(m_ExcludeFolders, OutExcludeFolders); + SplitAndAppendExclusion(m_ExcludeExtensions, OutExcludeExtensions); }; auto ParseDiffPath = [&]() { @@ -3401,31 +3212,39 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) LogExecutableVersionAndPid(); } } - CbObject QueryObject; + std::string JsonQuery; if (m_ListQueryPath.empty()) { CbObjectWriter QueryWriter; QueryWriter.BeginObject("query"); QueryWriter.EndObject(); // query - QueryObject = QueryWriter.Save(); + CbObject QueryObject = QueryWriter.Save(); + ExtendableStringBuilder<64> SB; + CompactBinaryToJson(QueryObject, SB); + JsonQuery = SB.ToString(); } else { if (ToLower(m_ListQueryPath.extension().string()) == ".cbo") { - QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_ListQueryPath)); + CbObject QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_ListQueryPath)); + ExtendableStringBuilder<64> SB; + CompactBinaryToJson(QueryObject, SB); + JsonQuery = SB.ToString(); } else { IoBuffer MetaDataJson = ReadFile(m_ListQueryPath).Flatten(); std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); std::string JsonError; - QueryObject = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); + CbObject QueryObject = LoadCompactBinaryFromJson(Json, JsonError) + .AsObject(); // We try to convert it so it is at least reaonably verified in format if (!JsonError.empty()) { throw std::runtime_error( fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_ListQueryPath, JsonError)); } + JsonQuery = std::string(Json); } } @@ -3448,7 +3267,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) /*RequireBucket*/ false, /*BoostCacheBackgroundWorkerPool */ false); - CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject); + CbObject Response = Storage.BuildStorage->ListBuilds(JsonQuery); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::Default) == CbValidateError::None); if (m_ListResultPath.empty()) { @@ -3569,6 +3388,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); @@ -3580,7 +3400,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) /*RequireBucket*/ true, /*BoostCacheBackgroundWorkerPool */ false); - if (m_BuildPartName.empty()) + if (m_BuildPartName.empty() && m_ManifestPath.empty()) { m_BuildPartName = m_Path.filename().string(); } @@ -3590,38 +3410,57 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_BuildId = BuildId.ToString(); } - const Oid BuildPartId = m_BuildPartId.empty() ? Oid::NewOid() : ParseBuildPartId(); - if (m_BuildPartId.empty()) + + Oid BuildPartId; + if (!m_BuildPartId.empty()) { - m_BuildPartId = BuildPartId.ToString(); + BuildPartId = ParseBuildPartId(); } CbObject MetaData = ParseBuildMetadata(); const std::filesystem::path TempDir = ZenTempFolderPath(m_ZenFolderPath); - UploadFolder(*Output, - Workers, - Storage, - BuildId, - BuildPartId, - m_BuildPartName, - m_Path, - TempDir, - m_ManifestPath, - m_FindBlockMaxCount, - m_BlockReuseMinPercentLimit, - m_AllowMultiparts, - MetaData, - m_CreateBuild, - m_Clean, - m_UploadToZenCache); + std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; + std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + + std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); + std::unique_ptr<ChunkingCache> ChunkCache = m_ChunkingCachePath.empty() + ? CreateNullChunkingCache() + : CreateDiskChunkingCache(m_ChunkingCachePath, *ChunkController, 256u * 1024u); + + std::vector<std::pair<Oid, std::string>> UploadedParts = + UploadFolder(*Output, + Workers, + Storage, + BuildId, + BuildPartId, + m_BuildPartName, + m_Path, + m_ManifestPath, + MetaData, + *ChunkController, + *ChunkCache, + UploadFolderOptions{.TempDir = TempDir, + .FindBlockMaxCount = m_FindBlockMaxCount, + .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, + .AllowMultiparts = m_AllowMultiparts, + .CreateBuild = m_CreateBuild, + .IgnoreExistingBlocks = m_Clean, + .UploadToZenCache = m_UploadToZenCache, + .ExcludeFolders = ExcludeFolders, + .ExcludeExtensions = ExcludeExtensions}); if (!AbortFlag) { if (m_PostUploadVerify) { - ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); + // TODO: Validate all parts + for (const auto& Part : UploadedParts) + { + ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, Part.first, Part.second); + } } } @@ -3737,6 +3576,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw OptionParseException("'--append' conflicts with '--clean'", SubOption->help()); } + std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; + std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + DownloadFolder(*Output, Workers, Storage, @@ -3744,6 +3587,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildId, BuildPartIds, BuildPartNames, + m_DownloadSpecPath, m_Path, DownloadOptions{.SystemRootDir = m_SystemRootDir, .ZenFolderPath = m_ZenFolderPath, @@ -3759,7 +3603,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) .ExcludeWildcards = ExcludeWildcards, .MaximumInMemoryPayloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory), .PopulateCache = m_UploadToZenCache, - .AppendNewContent = m_AppendNewContent}); + .AppendNewContent = m_AppendNewContent, + .ExcludeFolders = ExcludeFolders}); if (AbortFlag) { @@ -3769,9 +3614,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_LsOptions) { - if (!IsQuiet) + if (!m_LsResultPath.empty()) { - LogExecutableVersionAndPid(); + if (!IsQuiet) + { + LogExecutableVersionAndPid(); + } } ZenState InstanceState; @@ -3801,7 +3649,30 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::vector<Oid> BuildPartIds = ParseBuildPartIds(); std::vector<std::string> BuildPartNames = ParseBuildPartNames(); - ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards); + std::unique_ptr<CbObjectWriter> StructuredOutput; + if (!m_LsResultPath.empty()) + { + MakeSafeAbsolutePathÍnPlace(m_LsResultPath); + StructuredOutput = std::make_unique<CbObjectWriter>(); + } + + ListBuild(Storage, BuildId, BuildPartIds, BuildPartNames, IncludeWildcards, ExcludeWildcards, StructuredOutput.get()); + + if (StructuredOutput) + { + CbObject Response = StructuredOutput->Save(); + if (ToLower(m_LsResultPath.extension().string()) == ".cbo") + { + MemoryView ResponseView = Response.GetView(); + WriteFile(m_LsResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); + } + else + { + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + WriteFile(m_LsResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); + } + } if (AbortFlag) { @@ -3825,7 +3696,29 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ParsePath(); ParseDiffPath(); - DiffFolders(Workers, m_Path, m_DiffPath, m_OnlyChunked); + MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath); + + std::vector<std::string> ExcludeFolders = DefaultExcludeFolders; + std::vector<std::string> ExcludeExtensions = DefaultExcludeExtensions; + ParseExcludeFolderAndExtension(ExcludeFolders, ExcludeExtensions); + + StandardChunkingControllerSettings ChunkingSettings; + std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(ChunkingSettings); + std::unique_ptr<ChunkingCache> ChunkCache = m_ChunkingCachePath.empty() + ? CreateNullChunkingCache() + : CreateDiskChunkingCache(m_ChunkingCachePath, *ChunkController, 256u * 1024u); + + if (m_OnlyChunked) + { + ExcludeExtensions.insert(ExcludeExtensions.end(), + ChunkingSettings.SplitOnlyExtensions.begin(), + ChunkingSettings.SplitOnlyExtensions.end()); + ExcludeExtensions.insert(ExcludeExtensions.end(), + ChunkingSettings.SplitAndCompressExtensions.begin(), + ChunkingSettings.SplitAndCompressExtensions.end()); + } + + DiffFolders(Workers, m_Path, m_DiffPath, *ChunkController, *ChunkCache, ExcludeFolders, ExcludeExtensions); if (AbortFlag) { throw std::runtime_error("Diff folders aborted"); @@ -4067,8 +3960,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Storage, StorageCacheStats, BuildId, - {}, - {}, + /*BuildPartIds,*/ {}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, m_Path, DownloadOptions{.SystemRootDir = m_SystemRootDir, .ZenFolderPath = m_ZenFolderPath, @@ -4190,6 +4084,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_ZenFolderPath = m_Path / ZenFolderName; } MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath); StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, @@ -4228,7 +4123,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path); - // std::filesystem::path UploadTempDir = m_ZenFolderPath / "upload_tmp"; + + std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); + std::unique_ptr<ChunkingCache> ChunkCache = m_ChunkingCachePath.empty() + ? CreateNullChunkingCache() + : CreateDiskChunkingCache(m_ChunkingCachePath, *ChunkController, 256u * 1024u); UploadFolder(*Output, Workers, @@ -4237,20 +4136,51 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, m_Path, - UploadTempDir, {}, - m_FindBlockMaxCount, - m_BlockReuseMinPercentLimit, - m_AllowMultiparts, MetaData, - true, - false, - m_UploadToZenCache); + *ChunkController, + *ChunkCache, + UploadFolderOptions{.TempDir = UploadTempDir, + .FindBlockMaxCount = m_FindBlockMaxCount, + .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, + .AllowMultiparts = m_AllowMultiparts, + .CreateBuild = true, + .IgnoreExistingBlocks = false, + .UploadToZenCache = m_UploadToZenCache}); + if (AbortFlag) { throw std::runtime_error("Test aborted. (Upload build)"); } + { + ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}' with chunking cache", m_BuildId, BuildPartId, m_BuildPartName, m_Path); + + UploadFolder(*Output, + Workers, + Storage, + Oid::NewOid(), + Oid::NewOid(), + m_BuildPartName, + m_Path, + {}, + MetaData, + *ChunkController, + *ChunkCache, + UploadFolderOptions{.TempDir = UploadTempDir, + .FindBlockMaxCount = m_FindBlockMaxCount, + .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, + .AllowMultiparts = m_AllowMultiparts, + .CreateBuild = true, + .IgnoreExistingBlocks = false, + .UploadToZenCache = m_UploadToZenCache}); + + if (AbortFlag) + { + throw std::runtime_error("Test aborted. (Upload again, chunking is cached)"); + } + } + ValidateBuildPart(*Output, Workers, *Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName); if (!m_IncludeWildcard.empty() || !m_ExcludeWildcard.empty()) @@ -4269,7 +4199,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageCacheStats, BuildId, {BuildPartId}, - {}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, DownloadPath, DownloadOptions{.SystemRootDir = m_SystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, @@ -4300,7 +4231,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageCacheStats, BuildId, {BuildPartId}, - {}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, DownloadPath, DownloadOptions{.SystemRootDir = m_SystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, @@ -4327,7 +4259,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageCacheStats, BuildId, {BuildPartId}, - {}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, DownloadPath, DownloadOptions{.SystemRootDir = m_SystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, @@ -4355,7 +4288,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageCacheStats, BuildId, {BuildPartId}, - {}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, DownloadPath, DownloadOptions{.SystemRootDir = m_SystemRootDir, @@ -4380,7 +4314,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageCacheStats, BuildId, {BuildPartId}, - {}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, DownloadPath, DownloadOptions{.SystemRootDir = m_SystemRootDir, @@ -4497,7 +4432,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageCacheStats, BuildId, {BuildPartId}, - {}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, DownloadPath, DownloadOptions{.SystemRootDir = m_SystemRootDir, @@ -4534,15 +4470,18 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId2, m_BuildPartName, DownloadPath, - UploadTempDir, {}, - m_FindBlockMaxCount, - m_BlockReuseMinPercentLimit, - m_AllowMultiparts, MetaData2, - true, - false, - m_UploadToZenCache); + *ChunkController, + *ChunkCache, + UploadFolderOptions{.TempDir = UploadTempDir, + .FindBlockMaxCount = m_FindBlockMaxCount, + .BlockReuseMinPercentLimit = m_BlockReuseMinPercentLimit, + .AllowMultiparts = m_AllowMultiparts, + .CreateBuild = true, + .IgnoreExistingBlocks = false, + .UploadToZenCache = m_UploadToZenCache}); + if (AbortFlag) { throw std::runtime_error("Test aborted. (Upload scrambled)"); @@ -4557,7 +4496,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageCacheStats, BuildId, {BuildPartId}, - {}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, DownloadPath, DownloadOptions{.SystemRootDir = m_SystemRootDir, @@ -4582,7 +4522,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageCacheStats, BuildId2, {BuildPartId2}, - {}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, DownloadPath, DownloadOptions{.SystemRootDir = m_SystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, @@ -4606,7 +4547,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageCacheStats, BuildId2, {BuildPartId2}, - {}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, DownloadPath, DownloadOptions{.SystemRootDir = m_SystemRootDir, .ZenFolderPath = DownloadPath / ZenFolderName, @@ -4630,7 +4572,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageCacheStats, BuildId, {BuildPartId}, - {}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, DownloadPath2, DownloadOptions{.SystemRootDir = m_SystemRootDir, .ZenFolderPath = DownloadPath2 / ZenFolderName, @@ -4654,7 +4597,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageCacheStats, BuildId, {BuildPartId}, - {}, + /*BuildPartNames*/ {}, + /*ManifestPath*/ {}, DownloadPath3, DownloadOptions{.SystemRootDir = m_SystemRootDir, .ZenFolderPath = DownloadPath3 / ZenFolderName, diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 081a3a460..f5c44ab55 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -72,7 +72,6 @@ private: uint8_t m_BlockReuseMinPercentLimit = 85; bool m_AllowMultiparts = true; std::string m_AllowPartialBlockRequests = "mixed"; - std::string m_ManifestPath; // Not a std::filesystem::path since it can be relative to m_Path AuthCommandLineOptions m_AuthOptions; @@ -90,19 +89,27 @@ private: std::filesystem::path m_Path; - cxxopts::Options m_UploadOptions{"upload", "Upload a folder"}; - uint64_t m_FindBlockMaxCount = 10000; - bool m_PostUploadVerify = false; + std::string m_IncludeWildcard; + std::string m_ExcludeWildcard; + + std::string m_ExcludeFolders; + std::string m_ExcludeExtensions; + + cxxopts::Options m_UploadOptions{"upload", "Upload a folder"}; + uint64_t m_FindBlockMaxCount = 10000; + bool m_PostUploadVerify = false; + std::filesystem::path m_ChunkingCachePath; + std::filesystem::path m_ManifestPath; cxxopts::Options m_DownloadOptions{"download", "Download a folder"}; std::vector<std::string> m_BuildPartNames; std::vector<std::string> m_BuildPartIds; bool m_PostDownloadVerify = false; bool m_EnableScavenging = true; + std::filesystem::path m_DownloadSpecPath; - cxxopts::Options m_LsOptions{"ls", "List the content of uploaded build"}; - std::string m_IncludeWildcard; - std::string m_ExcludeWildcard; + cxxopts::Options m_LsOptions{"ls", "List the content of uploaded build"}; + std::filesystem::path m_LsResultPath; cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"}; std::filesystem::path m_DiffPath; diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 15fb1f16c..4885fd363 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -47,12 +47,13 @@ namespace { #define ZEN_CLOUD_STORAGE "Cloud Storage" - void WriteAuthOptions(CbObjectWriter& Writer, - std::string_view JupiterOpenIdProvider, - std::string_view JupiterAccessToken, - std::string_view JupiterAccessTokenEnv, - std::string_view JupiterAccessTokenPath, - std::string_view OidcTokenAuthExecutablePath) + void WriteAuthOptions(CbObjectWriter& Writer, + std::string_view JupiterOpenIdProvider, + std::string_view JupiterAccessToken, + std::string_view JupiterAccessTokenEnv, + std::string_view JupiterAccessTokenPath, + std::string_view OidcTokenAuthExecutablePath, + cxxopts::Options& Options) { if (!JupiterOpenIdProvider.empty()) { @@ -87,6 +88,11 @@ namespace { { Writer.AddString("oidc-exe-path"sv, OidcTokenExePath.generic_string()); } + else if (!OidcTokenAuthExecutablePath.empty()) + { + throw OptionParseException(fmt::format("'--oidctoken-exe-path' ('{}') does not exist", OidcTokenAuthExecutablePath), + Options.help()); + } } IoBuffer MakeCbObjectPayload(std::function<void(CbObjectWriter& Writer)> WriteCB) @@ -1234,7 +1240,8 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg m_JupiterAccessToken, m_JupiterAccessTokenEnv, m_JupiterAccessTokenPath, - m_OidcTokenAuthExecutablePath); + m_OidcTokenAuthExecutablePath, + m_Options); if (m_JupiterAssumeHttp2) { Writer.AddBool("assumehttp2"sv, true); @@ -1270,7 +1277,8 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg m_JupiterAccessToken, m_JupiterAccessTokenEnv, m_JupiterAccessTokenPath, - m_OidcTokenAuthExecutablePath); + m_OidcTokenAuthExecutablePath, + m_Options); if (m_JupiterAssumeHttp2) { Writer.AddBool("assumehttp2"sv, true); @@ -1664,7 +1672,8 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg m_JupiterAccessToken, m_JupiterAccessTokenEnv, m_JupiterAccessTokenPath, - m_OidcTokenAuthExecutablePath); + m_OidcTokenAuthExecutablePath, + m_Options); if (m_JupiterAssumeHttp2) { Writer.AddBool("assumehttp2"sv, true); @@ -1689,7 +1698,8 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg m_JupiterAccessToken, m_JupiterAccessTokenEnv, m_JupiterAccessTokenPath, - m_OidcTokenAuthExecutablePath); + m_OidcTokenAuthExecutablePath, + m_Options); if (m_JupiterAssumeHttp2) { Writer.AddBool("assumehttp2"sv, true); @@ -2634,8 +2644,6 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a BuildId, {.IsQuiet = m_Quiet, .IsVerbose = m_Verbose, .ForceDownload = m_ForceDownload, .TempFolderPath = StorageTempPath}); - const Oid OplogBuildPartId = State.GetBuildPartId(); - if (!m_Attachments.empty()) { if (m_AttachmentsPath.empty()) diff --git a/src/zen/cmds/service_cmd.cpp b/src/zen/cmds/service_cmd.cpp index 8be76de7c..a781dc340 100644 --- a/src/zen/cmds/service_cmd.cpp +++ b/src/zen/cmds/service_cmd.cpp @@ -13,6 +13,7 @@ # include <zencore/windows.h> # include <shellapi.h> # include <Shlwapi.h> +# pragma comment(lib, "Shlwapi.lib") #endif #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC @@ -37,9 +38,9 @@ namespace zen { namespace { #if ZEN_PLATFORM_WINDOWS - BOOL IsElevated() + BOOL RequiresElevation() { - BOOL fRet = FALSE; + BOOL fRet = TRUE; HANDLE hToken = NULL; if (OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &hToken)) { @@ -47,7 +48,7 @@ namespace { DWORD cbSize = sizeof(TOKEN_ELEVATION); if (GetTokenInformation(hToken, TokenElevation, &Elevation, sizeof(Elevation), &cbSize)) { - fRet = Elevation.TokenIsElevated; + fRet = !Elevation.TokenIsElevated; } } if (hToken) @@ -102,9 +103,13 @@ namespace { } } -#else // ZEN_PLATFORM_WINDOWS +#elif ZEN_PLATFORM_MAC - bool IsElevated() { return geteuid() == 0; } + bool RequiresElevation() { return false; } // Mac service mode commands can run without elevation + +#else + + bool RequiresElevation() { return geteuid() != 0; } #endif // ZEN_PLATFORM_WINDOWS @@ -344,7 +349,7 @@ ServiceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_InstallOptions) { - if (!IsElevated()) + if (RequiresElevation()) { RunElevated(m_AllowElevation); return; @@ -553,7 +558,7 @@ ServiceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw std::runtime_error(fmt::format("Service '{}' is running, stop before uninstalling", m_ServiceName)); } - if (!IsElevated()) + if (RequiresElevation()) { RunElevated(m_AllowElevation); return; @@ -585,7 +590,7 @@ ServiceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - if (!IsElevated()) + if (RequiresElevation()) { RunElevated(m_AllowElevation); return; @@ -617,7 +622,7 @@ ServiceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return; } - if (!IsElevated()) + if (RequiresElevation()) { RunElevated(m_AllowElevation); return; diff --git a/src/zen/cmds/top_cmd.cpp b/src/zen/cmds/top_cmd.cpp index 0e44dbbec..f674db6cd 100644 --- a/src/zen/cmds/top_cmd.cpp +++ b/src/zen/cmds/top_cmd.cpp @@ -4,6 +4,7 @@ #include <zencore/fmtutils.h> #include <zencore/logging.h> +#include <zencore/system.h> #include <zencore/uid.h> #include <zenutil/zenserverprocess.h> @@ -24,6 +25,47 @@ TopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_UNUSED(GlobalOptions, argc, argv); + SystemMetrics Metrics = GetSystemMetrics(); + + struct SystemMetric + { + const char* Name; + uint64_t Value; + } MetricValues[] = { + {"Cpus", Metrics.CpuCount}, + {"Cores", Metrics.CoreCount}, + {"LPs", Metrics.LogicalProcessorCount}, + {"SysMemMiB", Metrics.SystemMemoryMiB}, + {"AvailSysMemMiB", Metrics.AvailSystemMemoryMiB}, + {"VirtMemMiB", Metrics.VirtualMemoryMiB}, + {"AvailVirtMemMiB", Metrics.AvailVirtualMemoryMiB}, + {"PageFileMiB", Metrics.PageFileMiB}, + {"AvailPageFileMiB", Metrics.AvailPageFileMiB}, + }; + + std::vector<size_t> ColumnWidths; + for (const SystemMetric& Metric : MetricValues) + { + size_t NameLen = strlen(Metric.Name); + size_t ValueLen = fmt::formatted_size("{}", Metric.Value); + ColumnWidths.push_back(std::max(NameLen, ValueLen)); + } + + ExtendableStringBuilder<128> Header; + for (size_t i = 0; i < sizeof(MetricValues) / sizeof(MetricValues[0]); ++i) + { + Header << fmt::format("{:<{}} ", MetricValues[i].Name, ColumnWidths[i]); + } + ZEN_CONSOLE("{}", Header); + + // Print values with same adaptive widths + ExtendableStringBuilder<128> Values; + for (size_t i = 0; i < sizeof(MetricValues) / sizeof(MetricValues[0]); ++i) + { + Values << fmt::format("{:>{}} ", MetricValues[i].Value, ColumnWidths[i]); + } + ZEN_CONSOLE("{}\n", Values); + ZenServerState State; if (!State.InitializeReadOnly()) { diff --git a/src/zen/xmake.lua b/src/zen/xmake.lua index 55d073a86..ab094fef3 100644 --- a/src/zen/xmake.lua +++ b/src/zen/xmake.lua @@ -19,8 +19,6 @@ target("zen") add_files("zen.rc") add_ldflags("/subsystem:console,5.02") add_ldflags("/LTCG") - add_links("crypt32", "wldap32", "Ws2_32", "Shlwapi") - add_links("dbghelp", "winhttp", "version") -- for Sentry end if is_plat("macosx") then diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 4d4966222..09a2e4f91 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -296,28 +296,13 @@ ZenCmdBase::LogExecutableVersionAndPid() int main(int argc, char** argv) { - zen::SetCurrentThreadName("main"); - - std::vector<std::string> Args; #if ZEN_PLATFORM_WINDOWS - LPWSTR RawCommandLine = GetCommandLine(); - std::string CommandLine = zen::WideToUtf8(RawCommandLine); - Args = zen::ParseCommandLine(CommandLine); -#else - Args.reserve(argc); - for (int I = 0; I < argc; I++) - { - std::string Arg(argv[I]); - if ((!Arg.empty()) && (Arg != " ")) - { - Args.emplace_back(std::move(Arg)); - } - } -#endif - std::vector<char*> RawArgs = zen::StripCommandlineQuotes(Args); + setlocale(LC_ALL, "en_us.UTF8"); +#endif // ZEN_PLATFORM_WINDOWS + + zen::SetCurrentThreadName("main"); - argc = gsl::narrow<int>(RawArgs.size()); - argv = RawArgs.data(); + zen::CommandLineConverter ArgConverter(argc, argv); using namespace zen; using namespace std::literals; diff --git a/src/zencore-test/zencore-test.cpp b/src/zencore-test/zencore-test.cpp index 327550b32..68fc940ee 100644 --- a/src/zencore-test/zencore-test.cpp +++ b/src/zencore-test/zencore-test.cpp @@ -19,6 +19,10 @@ int main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) { +#if ZEN_PLATFORM_WINDOWS + setlocale(LC_ALL, "en_us.UTF8"); +#endif // ZEN_PLATFORM_WINDOWS + #if ZEN_WITH_TESTS zen::zencore_forcelinktests(); diff --git a/src/zencore/base64.cpp b/src/zencore/base64.cpp index b97dfebbf..1f56ee6c3 100644 --- a/src/zencore/base64.cpp +++ b/src/zencore/base64.cpp @@ -101,7 +101,7 @@ Base64::Encode(const uint8_t* Source, uint32_t Length, CharType* Dest) return uint32_t(EncodedBytes - Dest); } -template ZENCORE_API uint32_t Base64::Encode<char>(const uint8_t* Source, uint32_t Length, char* Dest); -template ZENCORE_API uint32_t Base64::Encode<wchar_t>(const uint8_t* Source, uint32_t Length, wchar_t* Dest); +template uint32_t Base64::Encode<char>(const uint8_t* Source, uint32_t Length, char* Dest); +template uint32_t Base64::Encode<wchar_t>(const uint8_t* Source, uint32_t Length, wchar_t* Dest); } // namespace zen diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index 2fa02937d..bd4d119fb 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -181,7 +181,7 @@ BasicFile::ReadRange(uint64_t FileOffset, uint64_t ByteCount) void BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) { - const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; + const uint64_t MaxChunkSize = 512u * 1024u; std::error_code Ec; ReadFile(m_FileHandle, Data, BytesToRead, FileOffset, MaxChunkSize, Ec); if (Ec) diff --git a/src/zencore/commandline.cpp b/src/zencore/commandline.cpp index 78260aeef..426cf23d6 100644 --- a/src/zencore/commandline.cpp +++ b/src/zencore/commandline.cpp @@ -22,6 +22,10 @@ void IterateCommandlineArgs(std::function<void(const std::string_view& Arg)>& ProcessArg) { #if ZEN_PLATFORM_WINDOWS + // It might seem odd to do this here in addition to at start of main functions but the InitGMalloc() function is called before main (via + // static data) so we need to make sure we set the locale before parsing the command line + setlocale(LC_ALL, "en_us.UTF8"); + int ArgC = 0; const LPWSTR CmdLine = ::GetCommandLineW(); const LPWSTR* ArgV = ::CommandLineToArgvW(CmdLine, &ArgC); diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 7f341818b..92a065707 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -1592,7 +1592,7 @@ ReadStdIn() } FileContents -ReadFile(std::filesystem::path Path) +ReadFile(const std::filesystem::path& Path) { uint64_t FileSizeBytes; void* Handle; @@ -1641,7 +1641,7 @@ ReadFile(std::filesystem::path Path) return Contents; } -ZENCORE_API void +void ScanFile(void* NativeHandle, uint64_t Offset, uint64_t Size, @@ -2669,36 +2669,41 @@ GetDirectoryContent(const std::filesystem::path& RootDir, } if (EnumHasAnyFlags(Flags, DirectoryContentFlags::Recursive)) { - PendingWorkCount.AddCount(1); - try + if (Visitor->AsyncAllowDirectory(Parent, DirectoryName)) { - WorkerPool.ScheduleWork( - [WorkerPool = &WorkerPool, - PendingWorkCount = &PendingWorkCount, - Visitor = Visitor, - Flags = Flags, - Path = std::move(Path), - RelativeRoot = RelativeRoot / DirectoryName]() { - ZEN_ASSERT(Visitor); - auto _ = MakeGuard([&]() { PendingWorkCount->CountDown(); }); - try - { - MultithreadedVisitor SubVisitor(*WorkerPool, *PendingWorkCount, RelativeRoot, Flags, Visitor); - FileSystemTraversal Traversal; - Traversal.TraverseFileSystem(Path, SubVisitor); - Visitor->AsyncVisitDirectory(SubVisitor.RelativeRoot, std::move(SubVisitor.Content)); - } - catch (const std::exception& Ex) - { - ZEN_ERROR("Failed scheduling work to scan subfolder '{}'. Reason: '{}'", Path / RelativeRoot, Ex.what()); - } - }, - WorkerThreadPool::EMode::DisableBacklog); - } - catch (const std::exception& Ex) - { - ZEN_ERROR("Failed scheduling work to scan folder '{}'. Reason: '{}'", Path, Ex.what()); - PendingWorkCount.CountDown(); + PendingWorkCount.AddCount(1); + try + { + WorkerPool.ScheduleWork( + [WorkerPool = &WorkerPool, + PendingWorkCount = &PendingWorkCount, + Visitor = Visitor, + Flags = Flags, + Path = std::move(Path), + RelativeRoot = RelativeRoot / DirectoryName]() { + ZEN_ASSERT(Visitor); + auto _ = MakeGuard([&]() { PendingWorkCount->CountDown(); }); + try + { + MultithreadedVisitor SubVisitor(*WorkerPool, *PendingWorkCount, RelativeRoot, Flags, Visitor); + FileSystemTraversal Traversal; + Traversal.TraverseFileSystem(Path, SubVisitor); + Visitor->AsyncVisitDirectory(SubVisitor.RelativeRoot, std::move(SubVisitor.Content)); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Failed scheduling work to scan subfolder '{}'. Reason: '{}'", + Path / RelativeRoot, + Ex.what()); + } + }, + WorkerThreadPool::EMode::DisableBacklog); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Failed scheduling work to scan folder '{}'. Reason: '{}'", Path, Ex.what()); + PendingWorkCount.CountDown(); + } } } return false; diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h index f5c82b8fe..f397370fc 100644 --- a/src/zencore/include/zencore/basicfile.h +++ b/src/zencore/include/zencore/basicfile.h @@ -192,6 +192,6 @@ private: IoBuffer WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path); -ZENCORE_API void basicfile_forcelink(); +void basicfile_forcelink(); } // namespace zen diff --git a/src/zencore/include/zencore/compactbinary.h b/src/zencore/include/zencore/compactbinary.h index 82ca055ab..b128e4205 100644 --- a/src/zencore/include/zencore/compactbinary.h +++ b/src/zencore/include/zencore/compactbinary.h @@ -138,8 +138,8 @@ public: int GetSeconds() const { return (int)((Ticks / TicksPerSecond) % 60); } - ZENCORE_API std::string ToString(const char* Format) const; - ZENCORE_API std::string ToString() const; + std::string ToString(const char* Format) const; + std::string ToString() const; friend inline DateTime operator+(const DateTime& Lhs, const TimeSpan& Rhs); friend inline DateTime operator+(const TimeSpan& Lhs, const DateTime& Rhs); @@ -492,7 +492,7 @@ class CbFieldView public: CbFieldView() = default; - ZENCORE_API CbFieldView(const void* DataPointer, CbFieldType FieldType = CbFieldType::HasFieldType); + CbFieldView(const void* DataPointer, CbFieldType FieldType = CbFieldType::HasFieldType); /** Construct a field from a value, without access to the name. */ inline explicit CbFieldView(const CbValue& Value); @@ -508,13 +508,13 @@ public: /** Returns the value for unchecked access. Prefer the typed accessors below. */ inline CbValue GetValue() const; - ZENCORE_API MemoryView AsBinaryView(MemoryView Default = MemoryView()); - ZENCORE_API CbObjectView AsObjectView(); - ZENCORE_API CbArrayView AsArrayView(); - ZENCORE_API std::string_view AsString(std::string_view Default = std::string_view()); - ZENCORE_API std::u8string_view AsU8String(std::u8string_view Default = std::u8string_view()); + MemoryView AsBinaryView(MemoryView Default = MemoryView()); + CbObjectView AsObjectView(); + CbArrayView AsArrayView(); + std::string_view AsString(std::string_view Default = std::string_view()); + std::u8string_view AsU8String(std::u8string_view Default = std::u8string_view()); - ZENCORE_API void IterateAttachments(const std::function<void(CbFieldView)>& Visitor) const; + void IterateAttachments(const std::function<void(CbFieldView)>& Visitor) const; /** Access the field as an int8. Returns the provided default on error. */ inline int8_t AsInt8(int8_t Default = 0) { return AsInteger<int8_t>(Default); } @@ -534,53 +534,53 @@ public: inline uint64_t AsUInt64(uint64_t Default = 0) { return AsInteger<uint64_t>(Default); } /** Access the field as a float. Returns the provided default on error. */ - ZENCORE_API float AsFloat(float Default = 0.0f); + float AsFloat(float Default = 0.0f); /** Access the field as a double. Returns the provided default on error. */ - ZENCORE_API double AsDouble(double Default = 0.0); + double AsDouble(double Default = 0.0); /** Access the field as a bool. Returns the provided default on error. */ - ZENCORE_API bool AsBool(bool bDefault = false); + bool AsBool(bool bDefault = false); /** Access the field as a hash referencing a compact binary attachment. Returns the provided default on error. */ - ZENCORE_API IoHash AsObjectAttachment(const IoHash& Default = IoHash()); + IoHash AsObjectAttachment(const IoHash& Default = IoHash()); /** Access the field as a hash referencing a binary attachment. Returns the provided default on error. */ - ZENCORE_API IoHash AsBinaryAttachment(const IoHash& Default = IoHash()); + IoHash AsBinaryAttachment(const IoHash& Default = IoHash()); /** Access the field as a hash referencing an attachment. Returns the provided default on error. */ - ZENCORE_API IoHash AsAttachment(const IoHash& Default = IoHash()); + IoHash AsAttachment(const IoHash& Default = IoHash()); /** Access the field as a hash. Returns the provided default on error. */ - ZENCORE_API IoHash AsHash(const IoHash& Default = IoHash()); + IoHash AsHash(const IoHash& Default = IoHash()); /** Access the field as a UUID. Returns a nil UUID on error. */ - ZENCORE_API Guid AsUuid(); + Guid AsUuid(); /** Access the field as a UUID. Returns the provided default on error. */ - ZENCORE_API Guid AsUuid(const Guid& Default); + Guid AsUuid(const Guid& Default); /** Access the field as an OID. Returns a nil OID on error. */ - ZENCORE_API Oid AsObjectId(); + Oid AsObjectId(); /** Access the field as a OID. Returns the provided default on error. */ - ZENCORE_API Oid AsObjectId(const Oid& Default); + Oid AsObjectId(const Oid& Default); /** Access the field as a custom sub-type with an integer identifier. Returns the provided default on error. */ - ZENCORE_API CbCustomById AsCustomById(CbCustomById Default); + CbCustomById AsCustomById(CbCustomById Default); /** Access the field as a custom sub-type with a string identifier. Returns the provided default on error. */ - ZENCORE_API CbCustomByName AsCustomByName(CbCustomByName Default); + CbCustomByName AsCustomByName(CbCustomByName Default); /** Access the field as a date/time tick count. Returns the provided default on error. */ - ZENCORE_API int64_t AsDateTimeTicks(int64_t Default = 0); + int64_t AsDateTimeTicks(int64_t Default = 0); /** Access the field as a date/time. Returns a date/time at the epoch on error. */ - ZENCORE_API DateTime AsDateTime(); + DateTime AsDateTime(); /** Access the field as a date/time. Returns the provided default on error. */ - ZENCORE_API DateTime AsDateTime(DateTime Default); + DateTime AsDateTime(DateTime Default); /** Access the field as a timespan tick count. Returns the provided default on error. */ - ZENCORE_API int64_t AsTimeSpanTicks(int64_t Default = 0); + int64_t AsTimeSpanTicks(int64_t Default = 0); /** Access the field as a timespan. Returns an empty timespan on error. */ - ZENCORE_API TimeSpan AsTimeSpan(); + TimeSpan AsTimeSpan(); /** Access the field as a timespan. Returns the provided default on error. */ - ZENCORE_API TimeSpan AsTimeSpan(TimeSpan Default); + TimeSpan AsTimeSpan(TimeSpan Default); /** True if the field has a name. */ constexpr inline bool HasName() const { return CbFieldTypeOps::HasFieldName(Type); } @@ -628,12 +628,12 @@ public: constexpr inline CbFieldError GetError() const { return Error; } /** Returns the size of the field in bytes, including the type and name. */ - ZENCORE_API uint64_t GetSize() const; + uint64_t GetSize() const; /** Calculate the hash of the field, including the type and name. */ - ZENCORE_API IoHash GetHash() const; + IoHash GetHash() const; - ZENCORE_API void GetHash(IoHashStream& HashStream) const; + void GetHash(IoHashStream& HashStream) const; /** Feed the field (including type and name) to the stream function */ inline void WriteToStream(auto Hash) const @@ -645,10 +645,10 @@ public: } /** Copy the field into a buffer of exactly GetSize() bytes, including the type and name. */ - ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; + void CopyTo(MutableMemoryView Buffer) const; /** Copy the field into an archive, including its type and name. */ - ZENCORE_API void CopyTo(BinaryWriter& Ar) const; + void CopyTo(BinaryWriter& Ar) const; /** * Whether this field is identical to the other field. @@ -659,10 +659,10 @@ public: * these assumptions do not hold, this may return false for equivalent inputs. Validation can * be performed with ValidateCompactBinary, except for field order and field name case. */ - ZENCORE_API bool Equals(const CbFieldView& Other) const; + bool Equals(const CbFieldView& Other) const; /** Returns a view of the field, including the type and name when present. */ - ZENCORE_API MemoryView GetView() const; + MemoryView GetView() const; /** * Try to get a view of the field as it would be serialized, such as by CopyTo. @@ -682,7 +682,7 @@ public: protected: /** Returns a view of the name and value payload, which excludes the type. */ - ZENCORE_API MemoryView GetViewNoType() const; + MemoryView GetViewNoType() const; /** Returns a view of the value payload, which excludes the type and name. */ inline MemoryView GetPayloadView() const { return MemoryView(Payload, GetPayloadSize()); } @@ -697,7 +697,7 @@ protected: inline const void* GetPayloadEnd() const { return static_cast<const uint8_t*>(Payload) + GetPayloadSize(); } /** Returns the size of the value payload in bytes, which is the field excluding the type and name. */ - ZENCORE_API uint64_t GetPayloadSize() const; + uint64_t GetPayloadSize() const; /** Assign a field from a pointer to its data and an optional externally-provided type. */ inline void Assign(const void* InData, const CbFieldType InType) @@ -719,7 +719,7 @@ private: return IntType(AsInteger(uint64_t(Default), CompactBinaryPrivate::MakeIntegerParams<IntType>())); } - ZENCORE_API uint64_t AsInteger(uint64_t Default, CompactBinaryPrivate::IntegerParams Params); + uint64_t AsInteger(uint64_t Default, CompactBinaryPrivate::IntegerParams Params); private: /** The field type, with the transient HasFieldType flag if the field contains its type. */ @@ -766,11 +766,11 @@ public: inline void Reset() { *this = TCbFieldIterator(); } /** Returns the size of the fields in the range in bytes. */ - ZENCORE_API uint64_t GetRangeSize() const; + uint64_t GetRangeSize() const; /** Calculate the hash of every field in the range. */ - ZENCORE_API IoHash GetRangeHash() const; - ZENCORE_API void GetRangeHash(IoHashStream& Hash) const; + IoHash GetRangeHash() const; + void GetRangeHash(IoHashStream& Hash) const; using FieldType::Equals; @@ -793,10 +793,10 @@ public: } /** Copy the field range into a buffer of exactly GetRangeSize() bytes. */ - ZENCORE_API void CopyRangeTo(MutableMemoryView Buffer) const; + void CopyRangeTo(MutableMemoryView Buffer) const; /** Invoke the visitor for every attachment in the field range. */ - ZENCORE_API void IterateRangeAttachments(const std::function<void(CbFieldView)>& Visitor) const; + void IterateRangeAttachments(const std::function<void(CbFieldView)>& Visitor) const; /** Create a view of every field in the range. */ inline MemoryView GetRangeView() const { return MemoryView(FieldType::GetView().GetData(), FieldsEnd); } @@ -892,12 +892,12 @@ private: /** * Serialize a compact binary array to JSON. */ -ZENCORE_API void CompactBinaryToJson(const CbArrayView& Object, StringBuilderBase& Builder); +void CompactBinaryToJson(const CbArrayView& Object, StringBuilderBase& Builder); /** * Serialize a compact binary array to YAML. */ -ZENCORE_API void CompactBinaryToYaml(const CbArrayView& Object, StringBuilderBase& Builder); +void CompactBinaryToYaml(const CbArrayView& Object, StringBuilderBase& Builder); /** * Array of CbField that have no names. @@ -919,16 +919,16 @@ public: using CbFieldView::TryGetSerializedView; /** Construct an array with no fields. */ - ZENCORE_API CbArrayView(); + CbArrayView(); /** Returns the number of items in the array. */ - ZENCORE_API uint64_t Num() const; + uint64_t Num() const; /** Create an iterator for the fields of this array. */ - ZENCORE_API CbFieldViewIterator CreateViewIterator() const; + CbFieldViewIterator CreateViewIterator() const; /** Visit the fields of this array. */ - ZENCORE_API void VisitFields(ICbVisitor& Visitor); + void VisitFields(ICbVisitor& Visitor); /** Access the array as an array field. */ inline CbFieldView AsFieldView() const { return static_cast<const CbFieldView&>(*this); } @@ -940,12 +940,12 @@ public: inline explicit operator bool() const { return Num() > 0; } /** Returns the size of the array in bytes if serialized by itself with no name. */ - ZENCORE_API uint64_t GetSize() const; + uint64_t GetSize() const; /** Calculate the hash of the array if serialized by itself with no name. */ - ZENCORE_API IoHash GetHash() const; + IoHash GetHash() const; - ZENCORE_API void GetHash(IoHashStream& Stream) const; + void GetHash(IoHashStream& Stream) const; /** * Whether this array is identical to the other array. @@ -956,13 +956,13 @@ public: * these assumptions do not hold, this may return false for equivalent inputs. Validation can * be done with the All mode to check these assumptions about the format of the inputs. */ - ZENCORE_API bool Equals(const CbArrayView& Other) const; + bool Equals(const CbArrayView& Other) const; /** Copy the array into a buffer of exactly GetSize() bytes, with no name. */ - ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; + void CopyTo(MutableMemoryView Buffer) const; /** Copy the array into an archive, including its type and name. */ - ZENCORE_API void CopyTo(BinaryWriter& Ar) const; + void CopyTo(BinaryWriter& Ar) const; ///** Invoke the visitor for every attachment in the array. */ inline void IterateAttachments(const std::function<void(CbFieldView)>& Visitor) const @@ -996,11 +996,11 @@ private: /** * Serialize a compact binary object to JSON. */ -ZENCORE_API void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder, bool AddTypeComment = false); +void CompactBinaryToJson(const CbObjectView& Object, StringBuilderBase& Builder, bool AddTypeComment = false); /** * Serialize a compact binary object to YAML. */ -ZENCORE_API void CompactBinaryToYaml(const CbObjectView& Object, StringBuilderBase& Builder); +void CompactBinaryToYaml(const CbObjectView& Object, StringBuilderBase& Builder); class CbObjectView : protected CbFieldView { @@ -1013,13 +1013,13 @@ public: using CbFieldView::TryGetSerializedView; /** Construct an object with no fields. */ - ZENCORE_API CbObjectView(); + CbObjectView(); /** Create an iterator for the fields of this object. */ - ZENCORE_API CbFieldViewIterator CreateViewIterator() const; + CbFieldViewIterator CreateViewIterator() const; /** Visit the fields of this object. */ - ZENCORE_API void VisitFields(ICbVisitor& Visitor); + void VisitFields(ICbVisitor& Visitor); /** * Find a field by case-sensitive name comparison. @@ -1030,10 +1030,10 @@ public: * @param Name The name of the field. * @return The matching field if found, otherwise a field with no value. */ - ZENCORE_API CbFieldView FindView(std::string_view Name) const; + CbFieldView FindView(std::string_view Name) const; /** Find a field by case-insensitive name comparison. */ - ZENCORE_API CbFieldView FindViewIgnoreCase(std::string_view Name) const; + CbFieldView FindViewIgnoreCase(std::string_view Name) const; /** Find a field by case-sensitive name comparison. */ inline CbFieldView operator[](std::string_view Name) const { return FindView(Name); } @@ -1045,15 +1045,15 @@ public: static inline CbObjectView FromFieldView(const CbFieldView& Field) { return CbObjectView(Field); } /** Whether the object has any fields. */ - ZENCORE_API explicit operator bool() const; + explicit operator bool() const; /** Returns the size of the object in bytes if serialized by itself with no name. */ - ZENCORE_API uint64_t GetSize() const; + uint64_t GetSize() const; /** Calculate the hash of the object if serialized by itself with no name. */ - ZENCORE_API IoHash GetHash() const; + IoHash GetHash() const; - ZENCORE_API void GetHash(IoHashStream& HashStream) const; + void GetHash(IoHashStream& HashStream) const; /** * Whether this object is identical to the other object. @@ -1064,13 +1064,13 @@ public: * these assumptions do not hold, this may return false for equivalent inputs. Validation can * be done with the All mode to check these assumptions about the format of the inputs. */ - ZENCORE_API bool Equals(const CbObjectView& Other) const; + bool Equals(const CbObjectView& Other) const; /** Copy the object into a buffer of exactly GetSize() bytes, with no name. */ - ZENCORE_API void CopyTo(MutableMemoryView Buffer) const; + void CopyTo(MutableMemoryView Buffer) const; /** Copy the field into an archive, including its type and name. */ - ZENCORE_API void CopyTo(BinaryWriter& Ar) const; + void CopyTo(BinaryWriter& Ar) const; ///** Invoke the visitor for every attachment in the object. */ inline void IterateAttachments(const std::function<void(CbFieldView)>& Visitor) const @@ -1263,7 +1263,7 @@ class CbFieldIterator : public TCbFieldIterator<CbField> { public: /** Construct a field range from an owned clone of a range. */ - ZENCORE_API static CbFieldIterator CloneRange(const CbFieldViewIterator& It); + static CbFieldIterator CloneRange(const CbFieldViewIterator& It); /** Construct a field range from an owned clone of a range. */ static inline CbFieldIterator CloneRange(const CbFieldIterator& It) { return CloneRange(CbFieldViewIterator(It)); } @@ -1306,7 +1306,7 @@ public: } /** Returns a buffer that exactly contains the field range. */ - ZENCORE_API SharedBuffer GetRangeBuffer() const; + SharedBuffer GetRangeBuffer() const; private: using TCbFieldIterator::TCbFieldIterator; @@ -1440,22 +1440,22 @@ CbField::AsBinary(const SharedBuffer& Default) && * @param Allocator Allocator for the buffer that the field is loaded into. * @return A field with a reference to the allocated buffer, or a default field on failure. */ -ZENCORE_API CbField LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator); +CbField LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator); -ZENCORE_API CbObject LoadCompactBinaryObject(IoBuffer&& Payload); -ZENCORE_API CbObject LoadCompactBinaryObject(const IoBuffer& Payload); -ZENCORE_API CbObject LoadCompactBinaryObject(CompressedBuffer&& Payload); -ZENCORE_API CbObject LoadCompactBinaryObject(const CompressedBuffer& Payload); +CbObject LoadCompactBinaryObject(IoBuffer&& Payload); +CbObject LoadCompactBinaryObject(const IoBuffer& Payload); +CbObject LoadCompactBinaryObject(CompressedBuffer&& Payload); +CbObject LoadCompactBinaryObject(const CompressedBuffer& Payload); /** * Load a compact binary from JSON. */ -ZENCORE_API CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json, std::string& Error); -ZENCORE_API CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json); +CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json, std::string& Error); +CbFieldIterator LoadCompactBinaryFromJson(std::string_view Json); -ZENCORE_API void SaveCompactBinary(BinaryWriter& Ar, const CbFieldView& Field); -ZENCORE_API void SaveCompactBinary(BinaryWriter& Ar, const CbArrayView& Array); -ZENCORE_API void SaveCompactBinary(BinaryWriter& Ar, const CbObjectView& Object); +void SaveCompactBinary(BinaryWriter& Ar, const CbFieldView& Field); +void SaveCompactBinary(BinaryWriter& Ar, const CbArrayView& Array); +void SaveCompactBinary(BinaryWriter& Ar, const CbObjectView& Object); /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -1473,7 +1473,7 @@ ZENCORE_API void SaveCompactBinary(BinaryWriter& Ar, const CbObjectView& Object) * @param View A memory view that may contain the start of a field. * @param Type HasFieldType means that View contains the type. Otherwise, use the given type. */ -ZENCORE_API uint64_t MeasureCompactBinary(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType); +uint64_t MeasureCompactBinary(MemoryView View, CbFieldType Type = CbFieldType::HasFieldType); /** * Try to determine the type and size of the compact binary field at the start of the view. @@ -1491,10 +1491,7 @@ ZENCORE_API uint64_t MeasureCompactBinary(MemoryView View, CbFieldType Type = Cb * @param InType HasFieldType means that InView contains the type. Otherwise, use the given type. * @return true if the size of the field was determined, otherwise false. */ -ZENCORE_API bool TryMeasureCompactBinary(MemoryView InView, - CbFieldType& OutType, - uint64_t& OutSize, - CbFieldType InType = CbFieldType::HasFieldType); +bool TryMeasureCompactBinary(MemoryView InView, CbFieldType& OutType, uint64_t& OutSize, CbFieldType InType = CbFieldType::HasFieldType); inline CbFieldViewIterator begin(CbFieldView& View) @@ -1520,14 +1517,14 @@ end(CbFieldView&) /** * Serialize serialized compact binary blob to JSON. It must be 0 to n fields with including type for each field */ -ZENCORE_API void CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder, bool AddTypeComment = false); +void CompactBinaryToJson(MemoryView Data, StringBuilderBase& InBuilder, bool AddTypeComment = false); /** * Serialize serialized compact binary blob to YAML. It must be 0 to n fields with including type for each field */ -ZENCORE_API void CompactBinaryToYaml(MemoryView Data, StringBuilderBase& InBuilder); +void CompactBinaryToYaml(MemoryView Data, StringBuilderBase& InBuilder); -ZENCORE_API std::vector<CbFieldView> ReadCompactBinaryStream(MemoryView Data); +std::vector<CbFieldView> ReadCompactBinaryStream(MemoryView Data); void uson_forcelink(); // internal void cbjson_forcelink(); // internal diff --git a/src/zencore/include/zencore/compactbinarybuilder.h b/src/zencore/include/zencore/compactbinarybuilder.h index f11717453..3e695c86f 100644 --- a/src/zencore/include/zencore/compactbinarybuilder.h +++ b/src/zencore/include/zencore/compactbinarybuilder.h @@ -67,14 +67,14 @@ class BinaryWriter; class CbWriter { public: - ZENCORE_API CbWriter(); - ZENCORE_API ~CbWriter(); + CbWriter(); + ~CbWriter(); CbWriter(const CbWriter&) = delete; CbWriter& operator=(const CbWriter&) = delete; /** Empty the writer without releasing any allocated memory. */ - ZENCORE_API void Reset(); + void Reset(); /** * Serialize the field(s) to an owned buffer and return it as an iterator. @@ -82,7 +82,7 @@ public: * It is not valid to call this function in the middle of writing an object, array, or field. * The writer remains valid for further use when this function returns. */ - ZENCORE_API CbFieldIterator Save(); + CbFieldIterator Save(); /** * Serialize the field(s) to memory. @@ -93,16 +93,16 @@ public: * @param Buffer A mutable memory view to write to. Must be exactly GetSaveSize() bytes. * @return An iterator for the field(s) written to the buffer. */ - ZENCORE_API CbFieldViewIterator Save(MutableMemoryView Buffer); + CbFieldViewIterator Save(MutableMemoryView Buffer); - ZENCORE_API void Save(BinaryWriter& Writer); + void Save(BinaryWriter& Writer); /** * The size of buffer (in bytes) required to serialize the fields that have been written. * * It is not valid to call this function in the middle of writing an object, array, or field. */ - ZENCORE_API uint64_t GetSaveSize() const; + uint64_t GetSaveSize() const; /** * Sets the name of the next field to be written. @@ -110,7 +110,7 @@ public: * It is not valid to call this function when writing a field inside an array. * Names must be valid UTF-8 and must be unique within an object. */ - ZENCORE_API CbWriter& SetName(std::string_view Name); + CbWriter& SetName(std::string_view Name); /** Copy the value (not the name) of an existing field. */ inline void AddField(std::string_view Name, const CbFieldView& Value) @@ -119,7 +119,7 @@ public: AddField(Value); } - ZENCORE_API void AddField(const CbFieldView& Value); + void AddField(const CbFieldView& Value); /** Copy the value (not the name) of an existing field. Holds a reference if owned. */ inline void AddField(std::string_view Name, const CbField& Value) @@ -127,7 +127,7 @@ public: SetName(Name); AddField(Value); } - ZENCORE_API void AddField(const CbField& Value); + void AddField(const CbField& Value); /** Begin a new object. Must have a matching call to EndObject. */ inline void BeginObject(std::string_view Name) @@ -135,9 +135,9 @@ public: SetName(Name); BeginObject(); } - ZENCORE_API void BeginObject(); + void BeginObject(); /** End an object after its fields have been written. */ - ZENCORE_API void EndObject(); + void EndObject(); /** Copy the value (not the name) of an existing object. */ inline void AddObject(std::string_view Name, const CbObjectView& Value) @@ -145,14 +145,14 @@ public: SetName(Name); AddObject(Value); } - ZENCORE_API void AddObject(const CbObjectView& Value); + void AddObject(const CbObjectView& Value); /** Copy the value (not the name) of an existing object. Holds a reference if owned. */ inline void AddObject(std::string_view Name, const CbObject& Value) { SetName(Name); AddObject(Value); } - ZENCORE_API void AddObject(const CbObject& Value); + void AddObject(const CbObject& Value); /** Begin a new array. Must have a matching call to EndArray. */ inline void BeginArray(std::string_view Name) @@ -160,9 +160,9 @@ public: SetName(Name); BeginArray(); } - ZENCORE_API void BeginArray(); + void BeginArray(); /** End an array after its fields have been written. */ - ZENCORE_API void EndArray(); + void EndArray(); /** Copy the value (not the name) of an existing array. */ inline void AddArray(std::string_view Name, const CbArrayView& Value) @@ -170,14 +170,14 @@ public: SetName(Name); AddArray(Value); } - ZENCORE_API void AddArray(const CbArrayView& Value); + void AddArray(const CbArrayView& Value); /** Copy the value (not the name) of an existing array. Holds a reference if owned. */ inline void AddArray(std::string_view Name, const CbArray& Value) { SetName(Name); AddArray(Value); } - ZENCORE_API void AddArray(const CbArray& Value); + void AddArray(const CbArray& Value); /** Write a null field. */ inline void AddNull(std::string_view Name) @@ -185,7 +185,7 @@ public: SetName(Name); AddNull(); } - ZENCORE_API void AddNull(); + void AddNull(); /** Write a binary field by copying Size bytes from Value. */ inline void AddBinary(std::string_view Name, const void* Value, uint64_t Size) @@ -193,7 +193,7 @@ public: SetName(Name); AddBinary(Value, Size); } - ZENCORE_API void AddBinary(const void* Value, uint64_t Size); + void AddBinary(const void* Value, uint64_t Size); /** Write a binary field by copying the view. */ inline void AddBinary(std::string_view Name, MemoryView Value) { @@ -208,15 +208,15 @@ public: SetName(Name); AddBinary(std::move(Value)); } - ZENCORE_API void AddBinary(IoBuffer Value); - ZENCORE_API void AddBinary(SharedBuffer Value); + void AddBinary(IoBuffer Value); + void AddBinary(SharedBuffer Value); inline void AddBinary(std::string_view Name, const CompositeBuffer& Buffer) { SetName(Name); AddBinary(Buffer); } - ZENCORE_API void AddBinary(const CompositeBuffer& Buffer); + void AddBinary(const CompositeBuffer& Buffer); /** Write a string field by copying the UTF-8 value. */ inline void AddString(std::string_view Name, std::string_view Value) @@ -224,14 +224,14 @@ public: SetName(Name); AddString(Value); } - ZENCORE_API void AddString(std::string_view Value); + void AddString(std::string_view Value); /** Write a string field by converting the UTF-16 value to UTF-8. */ inline void AddString(std::string_view Name, std::wstring_view Value) { SetName(Name); AddString(Value); } - ZENCORE_API void AddString(std::wstring_view Value); + void AddString(std::wstring_view Value); /** Write an integer field. */ inline void AddInteger(std::string_view Name, int32_t Value) @@ -239,28 +239,28 @@ public: SetName(Name); AddInteger(Value); } - ZENCORE_API void AddInteger(int32_t Value); + void AddInteger(int32_t Value); /** Write an integer field. */ inline void AddInteger(std::string_view Name, int64_t Value) { SetName(Name); AddInteger(Value); } - ZENCORE_API void AddInteger(int64_t Value); + void AddInteger(int64_t Value); /** Write an integer field. */ inline void AddInteger(std::string_view Name, uint32_t Value) { SetName(Name); AddInteger(Value); } - ZENCORE_API void AddInteger(uint32_t Value); + void AddInteger(uint32_t Value); /** Write an integer field. */ inline void AddInteger(std::string_view Name, uint64_t Value) { SetName(Name); AddInteger(Value); } - ZENCORE_API void AddInteger(uint64_t Value); + void AddInteger(uint64_t Value); /** Write a float field from a 32-bit float value. */ inline void AddFloat(std::string_view Name, float Value) @@ -268,7 +268,7 @@ public: SetName(Name); AddFloat(Value); } - ZENCORE_API void AddFloat(float Value); + void AddFloat(float Value); /** Write a float field from a 64-bit float value. */ inline void AddFloat(std::string_view Name, double Value) @@ -276,7 +276,7 @@ public: SetName(Name); AddFloat(Value); } - ZENCORE_API void AddFloat(double Value); + void AddFloat(double Value); /** Write a bool field. */ inline void AddBool(std::string_view Name, bool bValue) @@ -284,7 +284,7 @@ public: SetName(Name); AddBool(bValue); } - ZENCORE_API void AddBool(bool bValue); + void AddBool(bool bValue); /** Write a field referencing a compact binary attachment by its hash. */ inline void AddObjectAttachment(std::string_view Name, const IoHash& Value) @@ -292,7 +292,7 @@ public: SetName(Name); AddObjectAttachment(Value); } - ZENCORE_API void AddObjectAttachment(const IoHash& Value); + void AddObjectAttachment(const IoHash& Value); /** Write a field referencing a binary attachment by its hash. */ inline void AddBinaryAttachment(std::string_view Name, const IoHash& Value) @@ -300,7 +300,7 @@ public: SetName(Name); AddBinaryAttachment(Value); } - ZENCORE_API void AddBinaryAttachment(const IoHash& Value); + void AddBinaryAttachment(const IoHash& Value); /** Write a field referencing the attachment by its hash. */ inline void AddAttachment(std::string_view Name, const CbAttachment& Attachment) @@ -308,7 +308,7 @@ public: SetName(Name); AddAttachment(Attachment); } - ZENCORE_API void AddAttachment(const CbAttachment& Attachment); + void AddAttachment(const CbAttachment& Attachment); /** Write a hash field. */ inline void AddHash(std::string_view Name, const IoHash& Value) @@ -316,7 +316,7 @@ public: SetName(Name); AddHash(Value); } - ZENCORE_API void AddHash(const IoHash& Value); + void AddHash(const IoHash& Value); /** Write a UUID field. */ inline void AddUuid(std::string_view Name, const Guid& Value) @@ -324,7 +324,7 @@ public: SetName(Name); AddUuid(Value); } - ZENCORE_API void AddUuid(const Guid& Value); + void AddUuid(const Guid& Value); /** Write an ObjectId field. */ inline void AddObjectId(std::string_view Name, const Oid& Value) @@ -332,7 +332,7 @@ public: SetName(Name); AddObjectId(Value); } - ZENCORE_API void AddObjectId(const Oid& Value); + void AddObjectId(const Oid& Value); /** Write a date/time field with the specified count of 100ns ticks since the epoch. */ inline void AddDateTimeTicks(std::string_view Name, int64_t Ticks) @@ -340,7 +340,7 @@ public: SetName(Name); AddDateTimeTicks(Ticks); } - ZENCORE_API void AddDateTimeTicks(int64_t Ticks); + void AddDateTimeTicks(int64_t Ticks); /** Write a date/time field. */ inline void AddDateTime(std::string_view Name, DateTime Value) @@ -348,7 +348,7 @@ public: SetName(Name); AddDateTime(Value); } - ZENCORE_API void AddDateTime(DateTime Value); + void AddDateTime(DateTime Value); /** Write a time span field with the specified count of 100ns ticks. */ inline void AddTimeSpanTicks(std::string_view Name, int64_t Ticks) @@ -356,7 +356,7 @@ public: SetName(Name); AddTimeSpanTicks(Ticks); } - ZENCORE_API void AddTimeSpanTicks(int64_t Ticks); + void AddTimeSpanTicks(int64_t Ticks); /** Write a time span field. */ inline void AddTimeSpan(std::string_view Name, TimeSpan Value) @@ -364,7 +364,7 @@ public: SetName(Name); AddTimeSpan(Value); } - ZENCORE_API void AddTimeSpan(TimeSpan Value); + void AddTimeSpan(TimeSpan Value); /** Private flags that are public to work with ENUM_CLASS_FLAGS. */ enum class StateFlags : uint8_t; @@ -373,7 +373,7 @@ public: protected: /** Reserve the specified size up front until the format is optimized. */ - ZENCORE_API explicit CbWriter(int64_t InitialSize); + explicit CbWriter(int64_t InitialSize); private: friend CbWriter& operator<<(CbWriter& Writer, std::string_view NameOrValue); @@ -385,7 +385,7 @@ private: void EndField(CbFieldType Type); /** Set the field name if valid in this state, otherwise write add a string field. */ - ZENCORE_API void SetNameOrAddString(std::string_view NameOrValue); + void SetNameOrAddString(std::string_view NameOrValue); /** Returns a view of the name of the active field, if any, otherwise the empty view. */ std::string_view GetActiveName() const; @@ -447,19 +447,19 @@ public: explicit CbObjectWriter(int64_t InitialSize) : CbWriter(InitialSize) { BeginObject(); } CbObjectWriter() { BeginObject(); } - ZENCORE_API CbObject Save() + CbObject Save() { Finalize(); return CbWriter::Save().AsObject(); } - ZENCORE_API void Save(BinaryWriter& Writer) + void Save(BinaryWriter& Writer) { Finalize(); return CbWriter::Save(Writer); } - ZENCORE_API CbFieldViewIterator Save(MutableMemoryView Buffer) + CbFieldViewIterator Save(MutableMemoryView Buffer) { ZEN_ASSERT(m_Finalized); return CbWriter::Save(Buffer); @@ -655,20 +655,20 @@ operator<<(CbWriter& Writer, const Oid& Value) return Writer; } -ZENCORE_API CbWriter& operator<<(CbWriter& Writer, DateTime Value); -ZENCORE_API CbWriter& operator<<(CbWriter& Writer, TimeSpan Value); +CbWriter& operator<<(CbWriter& Writer, DateTime Value); +CbWriter& operator<<(CbWriter& Writer, TimeSpan Value); -ZENCORE_API inline TimeSpan +inline TimeSpan ToTimeSpan(std::chrono::seconds Secs) { return TimeSpan(0, 0, gsl::narrow<int>(Secs.count())); }; -ZENCORE_API inline TimeSpan +inline TimeSpan ToTimeSpan(std::chrono::milliseconds MS) { return TimeSpan(MS.count() * TimeSpan::TicksPerMillisecond); } -ZENCORE_API inline DateTime +inline DateTime ToDateTime(std::chrono::system_clock::time_point TimePoint) { time_t Time = std::chrono::system_clock::to_time_t(TimePoint); diff --git a/src/zencore/include/zencore/compactbinarypackage.h b/src/zencore/include/zencore/compactbinarypackage.h index 9ec12cb0f..64b62e2c0 100644 --- a/src/zencore/include/zencore/compactbinarypackage.h +++ b/src/zencore/include/zencore/compactbinarypackage.h @@ -51,23 +51,23 @@ public: inline CbAttachment(const CbObject& InValue, const IoHash& Hash) : CbAttachment(InValue, &Hash) {} /** Construct a raw binary attachment. Value is cloned if not owned. */ - ZENCORE_API explicit CbAttachment(const SharedBuffer& InValue) : CbAttachment(CompositeBuffer(InValue)) {} + explicit CbAttachment(const SharedBuffer& InValue) : CbAttachment(CompositeBuffer(InValue)) {} /** Construct a raw binary attachment. Value is cloned if not owned. Hash must match Value. */ - ZENCORE_API CbAttachment(const SharedBuffer& InValue, const IoHash& Hash) : CbAttachment(CompositeBuffer(InValue), Hash) {} + CbAttachment(const SharedBuffer& InValue, const IoHash& Hash) : CbAttachment(CompositeBuffer(InValue), Hash) {} /** Construct a raw binary attachment. Value is cloned if not owned. */ - ZENCORE_API explicit CbAttachment(SharedBuffer&& InValue) : CbAttachment(CompositeBuffer(std::move(InValue))) {} + explicit CbAttachment(SharedBuffer&& InValue) : CbAttachment(CompositeBuffer(std::move(InValue))) {} /** Construct a raw binary attachment. Value is cloned if not owned. Hash must match Value. */ - ZENCORE_API CbAttachment(SharedBuffer&& InValue, const IoHash& Hash) : CbAttachment(CompositeBuffer(std::move(InValue)), Hash) {} + CbAttachment(SharedBuffer&& InValue, const IoHash& Hash) : CbAttachment(CompositeBuffer(std::move(InValue)), Hash) {} /** Construct a compressed binary attachment. Value is cloned if not owned. */ - ZENCORE_API CbAttachment(const CompressedBuffer& InValue, const IoHash& Hash); - ZENCORE_API CbAttachment(CompressedBuffer&& InValue, const IoHash& Hash); + CbAttachment(const CompressedBuffer& InValue, const IoHash& Hash); + CbAttachment(CompressedBuffer&& InValue, const IoHash& Hash); /** Construct a binary attachment. Value is cloned if not owned. */ - ZENCORE_API CbAttachment(CompositeBuffer&& InValue, const IoHash& Hash); + CbAttachment(CompositeBuffer&& InValue, const IoHash& Hash); /** Reset this to a null attachment. */ inline void Reset() { *this = CbAttachment(); } @@ -76,31 +76,31 @@ public: inline explicit operator bool() const { return !IsNull(); } /** Whether the attachment has a value. */ - ZENCORE_API [[nodiscard]] bool IsNull() const; + [[nodiscard]] bool IsNull() const; /** Access the attachment as binary. Defaults to a null buffer on error. */ - ZENCORE_API [[nodiscard]] SharedBuffer AsBinary() const; + [[nodiscard]] SharedBuffer AsBinary() const; /** Access the attachment as raw binary. Defaults to a null buffer on error. */ - ZENCORE_API [[nodiscard]] const CompositeBuffer& AsCompositeBinary() const; + [[nodiscard]] const CompositeBuffer& AsCompositeBinary() const; /** Access the attachment as compressed binary. Defaults to a null buffer if the attachment is null. */ - ZENCORE_API [[nodiscard]] const CompressedBuffer& AsCompressedBinary() const; + [[nodiscard]] const CompressedBuffer& AsCompressedBinary() const; /** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */ - ZENCORE_API [[nodiscard]] CbObject AsObject() const; + [[nodiscard]] CbObject AsObject() const; /** Returns true if the attachment is binary */ - ZENCORE_API [[nodiscard]] bool IsBinary() const; + [[nodiscard]] bool IsBinary() const; /** Returns true if the attachment is compressed binary */ - ZENCORE_API [[nodiscard]] bool IsCompressedBinary() const; + [[nodiscard]] bool IsCompressedBinary() const; /** Returns whether the attachment is an object. */ - ZENCORE_API [[nodiscard]] bool IsObject() const; + [[nodiscard]] bool IsObject() const; /** Returns the hash of the attachment value. */ - ZENCORE_API [[nodiscard]] IoHash GetHash() const; + [[nodiscard]] IoHash GetHash() const; /** Compares attachments by their hash. Any discrepancy in type must be handled externally. */ inline bool operator==(const CbAttachment& Attachment) const { return GetHash() == Attachment.GetHash(); } @@ -114,27 +114,27 @@ public: * * The iterator is advanced as attachment fields are consumed from it. */ - ZENCORE_API bool TryLoad(CbFieldIterator& Fields); + bool TryLoad(CbFieldIterator& Fields); /** * Load the attachment from compact binary as written by Save. */ - ZENCORE_API bool TryLoad(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc); + bool TryLoad(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc); /** * Load the attachment from compact binary as written by Save. */ - ZENCORE_API bool TryLoad(IoBuffer& Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc); + bool TryLoad(IoBuffer& Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc); /** Save the attachment into the writer as a stream of compact binary fields. */ - ZENCORE_API void Save(CbWriter& Writer) const; + void Save(CbWriter& Writer) const; /** Save the attachment into the writer as a stream of compact binary fields. */ - ZENCORE_API void Save(BinaryWriter& Writer) const; + void Save(BinaryWriter& Writer) const; private: - ZENCORE_API CbAttachment(const CbObject& Value, const IoHash* Hash); - ZENCORE_API explicit CbAttachment(CompositeBuffer&& InValue); + CbAttachment(const CbObject& Value, const IoHash* Hash); + explicit CbAttachment(CompositeBuffer&& InValue); IoHash Hash; std::variant<std::nullptr_t, CbObject, CompositeBuffer, CompressedBuffer> Value; @@ -278,7 +278,7 @@ public: * @return The attachment, or null if the attachment is not found. * @note The returned pointer is only valid until the attachments on this package are modified. */ - ZENCORE_API const CbAttachment* FindAttachment(const IoHash& Hash) const; + const CbAttachment* FindAttachment(const IoHash& Hash) const; /** Find an attachment if it exists in the package. */ inline const CbAttachment* FindAttachment(const CbAttachment& Attachment) const { return FindAttachment(Attachment.GetHash()); } @@ -298,13 +298,13 @@ public: * * @return Number of attachments removed, which will be either 0 or 1. */ - ZENCORE_API int32_t RemoveAttachment(const IoHash& Hash); - inline int32_t RemoveAttachment(const CbAttachment& Attachment) { return RemoveAttachment(Attachment.GetHash()); } + int32_t RemoveAttachment(const IoHash& Hash); + inline int32_t RemoveAttachment(const CbAttachment& Attachment) { return RemoveAttachment(Attachment.GetHash()); } /** Compares packages by their object and attachment hashes. */ - ZENCORE_API bool Equals(const CbPackage& Package) const; - inline bool operator==(const CbPackage& Package) const { return Equals(Package); } - inline bool operator!=(const CbPackage& Package) const { return !Equals(Package); } + bool Equals(const CbPackage& Package) const; + inline bool operator==(const CbPackage& Package) const { return Equals(Package); } + inline bool operator!=(const CbPackage& Package) const { return !Equals(Package); } /** * Load the object and attachments from compact binary as written by Save. @@ -314,19 +314,19 @@ public: * * The iterator is advanced as object and attachment fields are consumed from it. */ - ZENCORE_API bool TryLoad(CbFieldIterator& Fields); - ZENCORE_API bool TryLoad(IoBuffer Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc, AttachmentResolver* Mapper = nullptr); - ZENCORE_API bool TryLoad(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc, AttachmentResolver* Mapper = nullptr); + bool TryLoad(CbFieldIterator& Fields); + bool TryLoad(IoBuffer Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc, AttachmentResolver* Mapper = nullptr); + bool TryLoad(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc, AttachmentResolver* Mapper = nullptr); /** Save the object and attachments into the writer as a stream of compact binary fields. */ - ZENCORE_API void Save(CbWriter& Writer) const; + void Save(CbWriter& Writer) const; /** Save the object and attachments into the writer as a stream of compact binary fields. */ - ZENCORE_API void Save(BinaryWriter& Writer) const; + void Save(BinaryWriter& Writer) const; private: - ZENCORE_API void SetObject(CbObject Object, const IoHash* Hash, AttachmentResolver* Resolver); - ZENCORE_API void AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver); + void SetObject(CbObject Object, const IoHash* Hash, AttachmentResolver* Resolver); + void AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver); void GatherAttachments(const CbObject& Object, AttachmentResolver Resolver); diff --git a/src/zencore/include/zencore/compactbinaryvalidation.h b/src/zencore/include/zencore/compactbinaryvalidation.h index ddecc8a38..1fade1bc4 100644 --- a/src/zencore/include/zencore/compactbinaryvalidation.h +++ b/src/zencore/include/zencore/compactbinaryvalidation.h @@ -150,7 +150,7 @@ std::string ToString(const CbValidateError Error); * @param Type HasFieldType means that View contains the type. Otherwise, use the given type. * @return None on success, otherwise the flags for the types of errors that were detected. */ -ZENCORE_API CbValidateError ValidateCompactBinary(MemoryView View, CbValidateMode Mode, CbFieldType Type = CbFieldType::HasFieldType); +CbValidateError ValidateCompactBinary(MemoryView View, CbValidateMode Mode, CbFieldType Type = CbFieldType::HasFieldType); /** * Validate the compact binary data for every field in the view as specified by the mode flags. @@ -161,7 +161,7 @@ ZENCORE_API CbValidateError ValidateCompactBinary(MemoryView View, CbValidateMod * * @see ValidateCompactBinary */ -ZENCORE_API CbValidateError ValidateCompactBinaryRange(MemoryView View, CbValidateMode Mode); +CbValidateError ValidateCompactBinaryRange(MemoryView View, CbValidateMode Mode); /** * Validate the compact binary attachment pointed to by the view as specified by the mode flags. @@ -175,7 +175,7 @@ ZENCORE_API CbValidateError ValidateCompactBinaryRange(MemoryView View, CbValida * @param Mode A combination of the flags for the types of validation to perform. * @return None on success, otherwise the flags for the types of errors that were detected. */ -ZENCORE_API CbValidateError ValidateObjectAttachment(MemoryView View, CbValidateMode Mode); +CbValidateError ValidateObjectAttachment(MemoryView View, CbValidateMode Mode); /** * Validate the compact binary package pointed to by the view as specified by the mode flags. @@ -189,7 +189,7 @@ ZENCORE_API CbValidateError ValidateObjectAttachment(MemoryView View, CbValidate * @param Mode A combination of the flags for the types of validation to perform. * @return None on success, otherwise the flags for the types of errors that were detected. */ -ZENCORE_API CbValidateError ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode); +CbValidateError ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode); /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/zencore/include/zencore/compositebuffer.h b/src/zencore/include/zencore/compositebuffer.h index 1e1611de9..095ba3803 100644 --- a/src/zencore/include/zencore/compositebuffer.h +++ b/src/zencore/include/zencore/compositebuffer.h @@ -43,10 +43,10 @@ public: } /** Reset this to null. */ - ZENCORE_API void Reset(); + void Reset(); /** Returns the total size of the composite buffer in bytes. */ - [[nodiscard]] ZENCORE_API uint64_t GetSize() const; + [[nodiscard]] uint64_t GetSize() const; /** Returns the segments that the buffer is composed from. */ [[nodiscard]] inline std::span<const SharedBuffer> GetSegments() const @@ -61,22 +61,22 @@ public: [[nodiscard]] inline bool IsNull() const { return m_Segments.empty(); } /** Returns true if every segment in the composite buffer is owned. */ - [[nodiscard]] ZENCORE_API bool IsOwned() const; + [[nodiscard]] bool IsOwned() const; /** Returns a copy of the buffer where every segment is owned. */ - [[nodiscard]] ZENCORE_API CompositeBuffer MakeOwned() const&; - [[nodiscard]] ZENCORE_API CompositeBuffer MakeOwned() &&; + [[nodiscard]] CompositeBuffer MakeOwned() const&; + [[nodiscard]] CompositeBuffer MakeOwned() &&; /** Returns the concatenation of the segments into a contiguous buffer. */ [[nodiscard]] inline SharedBuffer Flatten() const& { return ToShared(); } [[nodiscard]] inline SharedBuffer Flatten() && { return ToShared(); } /** Returns the concatenation of the segments into a contiguous buffer. */ - [[nodiscard]] ZENCORE_API SharedBuffer ToShared() const&; - [[nodiscard]] ZENCORE_API SharedBuffer ToShared() &&; + [[nodiscard]] SharedBuffer ToShared() const&; + [[nodiscard]] SharedBuffer ToShared() &&; /** Returns the middle part of the buffer by taking the size starting at the offset. */ - [[nodiscard]] ZENCORE_API CompositeBuffer Mid(uint64_t Offset, uint64_t Size = ~uint64_t(0)) const; + [[nodiscard]] CompositeBuffer Mid(uint64_t Offset, uint64_t Size = ~uint64_t(0)) const; /** * Returns a view of the range if contained by one segment, otherwise a view of a copy of the range. @@ -88,10 +88,10 @@ public: * @param CopyBuffer The buffer to write the copy into if a copy is required. * @param Allocator The optional allocator to use when the copy buffer is required. */ - [[nodiscard]] ZENCORE_API MemoryView ViewOrCopyRange(uint64_t Offset, - uint64_t Size, - UniqueBuffer& CopyBuffer, - std::function<UniqueBuffer(uint64_t Size)> Allocator = UniqueBuffer::Alloc) const; + [[nodiscard]] MemoryView ViewOrCopyRange(uint64_t Offset, + uint64_t Size, + UniqueBuffer& CopyBuffer, + std::function<UniqueBuffer(uint64_t Size)> Allocator = UniqueBuffer::Alloc) const; /** * Copies a range of the buffer to a contiguous region of memory. @@ -99,7 +99,7 @@ public: * @param Target The view to copy to. Must be no larger than the data available at the offset. * @param Offset The byte offset in this buffer to start copying from. */ - ZENCORE_API void CopyTo(MutableMemoryView Target, uint64_t Offset = 0) const; + void CopyTo(MutableMemoryView Target, uint64_t Offset = 0) const; /** * Invokes a visitor with a view of each segment that intersects with a range. @@ -108,19 +108,17 @@ public: * @param Size The number of bytes in the range to visit. * @param Visitor The visitor to invoke from zero to GetSegments().Num() times. */ - ZENCORE_API void IterateRange(uint64_t Offset, uint64_t Size, std::function<void(MemoryView View)> Visitor) const; - ZENCORE_API void IterateRange(uint64_t Offset, - uint64_t Size, - std::function<void(MemoryView View, const SharedBuffer& ViewOuter)> Visitor) const; + void IterateRange(uint64_t Offset, uint64_t Size, std::function<void(MemoryView View)> Visitor) const; + void IterateRange(uint64_t Offset, uint64_t Size, std::function<void(MemoryView View, const SharedBuffer& ViewOuter)> Visitor) const; struct Iterator { size_t SegmentIndex = 0; uint64_t OffsetInSegment = 0; }; - ZENCORE_API Iterator GetIterator(uint64_t Offset) const; - ZENCORE_API MemoryView ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& CopyBuffer) const; - ZENCORE_API void CopyTo(MutableMemoryView Target, Iterator& It) const; + Iterator GetIterator(uint64_t Offset) const; + MemoryView ViewOrCopyRange(Iterator& It, uint64_t Size, UniqueBuffer& CopyBuffer) const; + void CopyTo(MutableMemoryView Target, Iterator& It) const; /** A null composite buffer. */ static const CompositeBuffer Null; diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h index 3802a8c31..ed21296c2 100644 --- a/src/zencore/include/zencore/compress.h +++ b/src/zencore/include/zencore/compress.h @@ -66,15 +66,15 @@ public: * @param BlockSize The power-of-two block size to encode raw data in. 0 is default. * @return An owned compressed buffer, or null on error. */ - [[nodiscard]] ZENCORE_API static CompressedBuffer Compress(const CompositeBuffer& RawData, - OodleCompressor Compressor = OodleCompressor::Mermaid, - OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, - uint64_t BlockSize = 0); - [[nodiscard]] ZENCORE_API static CompressedBuffer Compress(const SharedBuffer& RawData, - OodleCompressor Compressor = OodleCompressor::Mermaid, - OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, - uint64_t BlockSize = 0); - [[nodiscard]] ZENCORE_API static bool CompressToStream( + [[nodiscard]] static CompressedBuffer Compress(const CompositeBuffer& RawData, + OodleCompressor Compressor = OodleCompressor::Mermaid, + OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, + uint64_t BlockSize = 0); + [[nodiscard]] static CompressedBuffer Compress(const SharedBuffer& RawData, + OodleCompressor Compressor = OodleCompressor::Mermaid, + OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, + uint64_t BlockSize = 0); + [[nodiscard]] static bool CompressToStream( const CompositeBuffer& RawData, std::function<void(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback, OodleCompressor Compressor = OodleCompressor::Mermaid, @@ -86,34 +86,26 @@ public: * * @return A compressed buffer, or null on error, such as an invalid format or corrupt header. */ - [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const CompositeBuffer& CompressedData, - IoHash& OutRawHash, - uint64_t& OutRawSize); - [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData, - IoHash& OutRawHash, - uint64_t& OutRawSize); - [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData, - IoHash& OutRawHash, - uint64_t& OutRawSize); - [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData, - IoHash& OutRawHash, - uint64_t& OutRawSize); - [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(IoBuffer&& CompressedData); - [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(CompositeBuffer&& CompressedData); - [[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(IoBuffer&& CompressedData, - IoHash& OutRawHash, - uint64_t& OutRawSize, - uint64_t* OutOptionalTotalCompressedSize); - [[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(const IoBuffer& CompressedData, - IoHash& OutRawHash, - uint64_t& OutRawSize, - uint64_t* OutOptionalTotalCompressedSize); - [[nodiscard]] ZENCORE_API static bool ValidateCompressedHeader(const CompositeBuffer& CompressedData, - IoHash& OutRawHash, - uint64_t& OutRawSize, - uint64_t* OutOptionalTotalCompressedSize); - [[nodiscard]] ZENCORE_API static size_t GetHeaderSizeForNoneEncoder(); - [[nodiscard]] ZENCORE_API static UniqueBuffer CreateHeaderForNoneEncoder(uint64_t RawSize, const BLAKE3& RawHash); + [[nodiscard]] static CompressedBuffer FromCompressed(const CompositeBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize); + [[nodiscard]] static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize); + [[nodiscard]] static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize); + [[nodiscard]] static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize); + [[nodiscard]] static CompressedBuffer FromCompressedNoValidate(IoBuffer&& CompressedData); + [[nodiscard]] static CompressedBuffer FromCompressedNoValidate(CompositeBuffer&& CompressedData); + [[nodiscard]] static bool ValidateCompressedHeader(IoBuffer&& CompressedData, + IoHash& OutRawHash, + uint64_t& OutRawSize, + uint64_t* OutOptionalTotalCompressedSize); + [[nodiscard]] static bool ValidateCompressedHeader(const IoBuffer& CompressedData, + IoHash& OutRawHash, + uint64_t& OutRawSize, + uint64_t* OutOptionalTotalCompressedSize); + [[nodiscard]] static bool ValidateCompressedHeader(const CompositeBuffer& CompressedData, + IoHash& OutRawHash, + uint64_t& OutRawSize, + uint64_t* OutOptionalTotalCompressedSize); + [[nodiscard]] static size_t GetHeaderSizeForNoneEncoder(); + [[nodiscard]] static UniqueBuffer CreateHeaderForNoneEncoder(uint64_t RawSize, const BLAKE3& RawHash); /** Reset this to null. */ inline void Reset() { CompressedData.Reset(); } @@ -139,10 +131,10 @@ public: [[nodiscard]] inline uint64_t GetCompressedSize() const { return CompressedData.GetSize(); } /** Returns the size of the raw data. Zero on error or if this is empty or null. */ - [[nodiscard]] ZENCORE_API uint64_t DecodeRawSize() const; + [[nodiscard]] uint64_t DecodeRawSize() const; /** Returns the hash of the raw data. Zero on error or if this is null. */ - [[nodiscard]] ZENCORE_API IoHash DecodeRawHash() const; + [[nodiscard]] IoHash DecodeRawHash() const; /** * Returns a block aligned range of a compressed buffer. @@ -159,7 +151,7 @@ public: * * @return A sub-range from the compressed buffer that encompasses RawOffset and RawSize */ - [[nodiscard]] ZENCORE_API CompressedBuffer CopyRange(uint64_t RawOffset, uint64_t RawSize = ~uint64_t(0)) const; + [[nodiscard]] CompressedBuffer CopyRange(uint64_t RawOffset, uint64_t RawSize = ~uint64_t(0)) const; /** * Returns a block aligned range of a compressed buffer. @@ -176,7 +168,7 @@ public: * * @return A sub-range from the compressed buffer that encompasses RawOffset and RawSize */ - [[nodiscard]] ZENCORE_API CompressedBuffer GetRange(uint64_t RawOffset, uint64_t RawSize = ~uint64_t(0)) const; + [[nodiscard]] CompressedBuffer GetRange(uint64_t RawOffset, uint64_t RawSize = ~uint64_t(0)) const; /** * Returns the compressor and compression level used by this buffer. @@ -187,28 +179,28 @@ public: * * @return True if parameters were written, otherwise false. */ - [[nodiscard]] ZENCORE_API bool TryGetCompressParameters(OodleCompressor& OutCompressor, - OodleCompressionLevel& OutCompressionLevel, - uint64_t& OutBlockSize) const; + [[nodiscard]] bool TryGetCompressParameters(OodleCompressor& OutCompressor, + OodleCompressionLevel& OutCompressionLevel, + uint64_t& OutBlockSize) const; /** * Decompress into a memory view that is less or equal GetRawSize() bytes. */ - [[nodiscard]] ZENCORE_API bool TryDecompressTo(MutableMemoryView RawView, uint64_t RawOffset = 0) const; + [[nodiscard]] bool TryDecompressTo(MutableMemoryView RawView, uint64_t RawOffset = 0) const; /** * Decompress into an owned buffer. * * @return An owned buffer containing the raw data, or null on error. */ - [[nodiscard]] ZENCORE_API SharedBuffer Decompress(uint64_t RawOffset = 0, uint64_t RawSize = ~uint64_t(0)) const; + [[nodiscard]] SharedBuffer Decompress(uint64_t RawOffset = 0, uint64_t RawSize = ~uint64_t(0)) const; /** * Decompress into an owned composite buffer. * * @return An owned buffer containing the raw data, or null on error. */ - [[nodiscard]] ZENCORE_API CompositeBuffer DecompressToComposite() const; + [[nodiscard]] CompositeBuffer DecompressToComposite() const; /** * Decompress into and call callback for ranges of decompressed data. @@ -216,7 +208,7 @@ public: * * @return True if the buffer is valid and can be decompressed. */ - [[nodiscard]] ZENCORE_API bool DecompressToStream( + [[nodiscard]] bool DecompressToStream( uint64_t RawOffset, uint64_t RawSize, std::function<bool(uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const; diff --git a/src/zencore/include/zencore/except.h b/src/zencore/include/zencore/except.h index c933adfd8..4689c480f 100644 --- a/src/zencore/include/zencore/except.h +++ b/src/zencore/include/zencore/except.h @@ -18,23 +18,23 @@ namespace zen { #if ZEN_PLATFORM_WINDOWS -ZENCORE_API void ThrowSystemException [[noreturn]] (HRESULT hRes, std::string_view Message); +void ThrowSystemException [[noreturn]] (HRESULT hRes, std::string_view Message); #endif // ZEN_PLATFORM_WINDOWS #if defined(__cpp_lib_source_location) -ZENCORE_API void ThrowLastErrorImpl [[noreturn]] (std::string_view Message, const std::source_location& Location); -ZENCORE_API void ThrowOutOfMemoryImpl [[noreturn]] (std::string_view Message, const std::source_location& Location); +void ThrowLastErrorImpl [[noreturn]] (std::string_view Message, const std::source_location& Location); +void ThrowOutOfMemoryImpl [[noreturn]] (std::string_view Message, const std::source_location& Location); # define ThrowLastError(Message) ThrowLastErrorImpl(Message, std::source_location::current()) # define ThrowOutOfMemory(Message) ThrowOutOfMemoryImpl(Message, std::source_location::current()) #else -ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message); -ZENCORE_API void ThrowOutOfMemory [[noreturn]] (std::string_view Message); +void ThrowLastError [[noreturn]] (std::string_view Message); +void ThrowOutOfMemory [[noreturn]] (std::string_view Message); #endif -ZENCORE_API void ThrowSystemError [[noreturn]] (uint32_t ErrorCode, std::string_view Message); +void ThrowSystemError [[noreturn]] (uint32_t ErrorCode, std::string_view Message); -ZENCORE_API std::string GetLastErrorAsString(); -ZENCORE_API std::string GetSystemErrorAsString(uint32_t Win32ErrorCode); +std::string GetLastErrorAsString(); +std::string GetSystemErrorAsString(uint32_t Win32ErrorCode); inline int32_t GetLastError() diff --git a/src/zencore/include/zencore/except_fmt.h b/src/zencore/include/zencore/except_fmt.h new file mode 100644 index 000000000..095a78da7 --- /dev/null +++ b/src/zencore/include/zencore/except_fmt.h @@ -0,0 +1,36 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <fmt/args.h> +#include <fmt/format.h> +#include "except.h" + +namespace zen { + +/** + * Exception helper class to make formatted exception messages slightly easier by + * avoiding the need to explicitly call fmt::format(), making for less visual noise. + * + * Usage: + * + * throw zen::runtime_error("Failed to open file '{}'", FileName); + * + */ +template<typename BaseT> +class format_exception : public BaseT +{ +public: + format_exception(std::string_view Message) : BaseT(std::string(Message)) {} + + template<typename... T> + format_exception(fmt::format_string<T...> fmt, T&&... args) : BaseT(fmt::format(fmt, std::forward<T>(args)...)) + { + } +}; + +using runtime_error = format_exception<std::runtime_error>; +using invalid_argument = format_exception<std::invalid_argument>; +using out_of_range = format_exception<std::out_of_range>; + +} // namespace zen diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index b4906aebf..f28863679 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -8,8 +8,14 @@ #include <zencore/iobuffer.h> #include <zencore/string.h> +ZEN_THIRD_PARTY_INCLUDES_START #include <filesystem> #include <functional> +ZEN_THIRD_PARTY_INCLUDES_END + +#if ZEN_PLATFORM_WINDOWS +# undef CopyFile +#endif namespace zen { @@ -20,128 +26,128 @@ class WorkerThreadPool; /** Delete directory (after deleting any contents) */ -ZENCORE_API bool DeleteDirectories(const std::filesystem::path& Path); +bool DeleteDirectories(const std::filesystem::path& Path); /** Delete directory (after deleting any contents) */ -ZENCORE_API bool DeleteDirectories(const std::filesystem::path& Path, std::error_code& Ec); +bool DeleteDirectories(const std::filesystem::path& Path, std::error_code& Ec); /** Ensure directory exists. - Will also create any required parent direCleanDirectoryctories + Will also create any required parent directories */ -ZENCORE_API bool CreateDirectories(const std::filesystem::path& Path); +bool CreateDirectories(const std::filesystem::path& Path); /** Ensure directory exists. Will also create any required parent directories */ -ZENCORE_API bool CreateDirectories(const std::filesystem::path& Path, std::error_code& Ec); +bool CreateDirectories(const std::filesystem::path& Path, std::error_code& Ec); /** Ensure directory exists and delete contents (if any) before returning */ -ZENCORE_API bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles); +bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles); /** Ensure directory exists and delete contents (if any) before returning */ -ZENCORE_API bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec); +bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec); /** Ensure directory exists and delete contents (if any) before returning */ -ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& Path); +bool CleanDirectoryExceptDotFiles(const std::filesystem::path& Path); /** Map native file handle to a path */ -ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle, std::error_code& Ec); +std::filesystem::path PathFromHandle(void* NativeHandle, std::error_code& Ec); /** Get canonical path name from a generic path */ -ZENCORE_API std::filesystem::path CanonicalPath(std::filesystem::path InPath, std::error_code& Ec); +std::filesystem::path CanonicalPath(std::filesystem::path InPath, std::error_code& Ec); /** Query file size */ -ZENCORE_API bool IsFile(const std::filesystem::path& Path); +bool IsFile(const std::filesystem::path& Path); /** Query file size */ -ZENCORE_API bool IsFile(const std::filesystem::path& Path, std::error_code& Ec); +bool IsFile(const std::filesystem::path& Path, std::error_code& Ec); /** Query file size */ -ZENCORE_API bool IsDir(const std::filesystem::path& Path); +bool IsDir(const std::filesystem::path& Path); /** Query file size */ -ZENCORE_API bool IsDir(const std::filesystem::path& Path, std::error_code& Ec); +bool IsDir(const std::filesystem::path& Path, std::error_code& Ec); /** Query file size */ -ZENCORE_API bool RemoveFile(const std::filesystem::path& Path); +bool RemoveFile(const std::filesystem::path& Path); /** Query file size */ -ZENCORE_API bool RemoveFile(const std::filesystem::path& Path, std::error_code& Ec); +bool RemoveFile(const std::filesystem::path& Path, std::error_code& Ec); /** Query file size */ -ZENCORE_API bool RemoveDir(const std::filesystem::path& Path); +bool RemoveDir(const std::filesystem::path& Path); /** Query file size */ -ZENCORE_API bool RemoveDir(const std::filesystem::path& Path, std::error_code& Ec); +bool RemoveDir(const std::filesystem::path& Path, std::error_code& Ec); /** Query file size */ -ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path); +uint64_t FileSizeFromPath(const std::filesystem::path& Path); /** Query file size */ -ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec); +uint64_t FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec); /** Query file size from native file handle */ -ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle); +uint64_t FileSizeFromHandle(void* NativeHandle); /** Query file size from native file handle */ -ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle, std::error_code& Ec); +uint64_t FileSizeFromHandle(void* NativeHandle, std::error_code& Ec); /** Get a native time tick of last modification time */ -ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec); +uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec); /** Get a native time tick of last modification time */ -ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); +uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); -ZENCORE_API bool TryGetFileProperties(const std::filesystem::path& Path, - uint64_t& OutSize, - uint64_t& OutModificationTick, - uint32_t& OutNativeModeOrAttributes); +bool TryGetFileProperties(const std::filesystem::path& Path, + uint64_t& OutSize, + uint64_t& OutModificationTick, + uint32_t& OutNativeModeOrAttributes); /** Move a file, if the files are not on the same drive the function will fail */ -ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); +void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); /** Move a file, if the files are not on the same drive the function will fail */ -ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); +void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); /** Move a directory, if the files are not on the same drive the function will fail */ -ZENCORE_API void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); +void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); /** Move a directory, if the files are not on the same drive the function will fail */ -ZENCORE_API void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); +void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); -ZENCORE_API std::filesystem::path GetRunningExecutablePath(); +std::filesystem::path GetRunningExecutablePath(); /** Set the max open file handle count to max allowed for the current process on Linux and MacOS */ -ZENCORE_API void MaximizeOpenFileCount(); +void MaximizeOpenFileCount(); -ZENCORE_API bool PrepareFileForScatteredWrite(void* FileHandle, uint64_t FinalSize); +bool PrepareFileForScatteredWrite(void* FileHandle, uint64_t FinalSize); struct FileContents { @@ -149,9 +155,16 @@ struct FileContents std::error_code ErrorCode; IoBuffer Flatten(); + + explicit operator bool() const { return !ErrorCode; } }; -ZENCORE_API FileContents ReadStdIn(); +/** Read all of standard input into a FileContents structure + * + * Note that this will block until end-of-file is reached on standard input + * which could be a very bad idea in some contexts. + */ +FileContents ReadStdIn(); /** Prepare file for reading @@ -159,25 +172,20 @@ ZENCORE_API FileContents ReadStdIn(); IoBuffer referencing the file contents so that it may be read at a later time. This is leveraged to allow sending of data straight from disk cache and other optimizations. */ -ZENCORE_API FileContents ReadFile(std::filesystem::path Path); - -ZENCORE_API bool ScanFile(std::filesystem::path Path, uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc); -ZENCORE_API void WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount); -ZENCORE_API void WriteFile(std::filesystem::path Path, IoBuffer Data); -ZENCORE_API void WriteFile(std::filesystem::path Path, CompositeBuffer Data); -ZENCORE_API bool MoveToFile(std::filesystem::path Path, IoBuffer Data); -ZENCORE_API void ScanFile(void* NativeHandle, - uint64_t Offset, - uint64_t Size, - uint64_t ChunkSize, - std::function<void(const void* Data, size_t Size)>&& ProcessFunc); -ZENCORE_API void WriteFile(void* NativeHandle, - const void* Data, - uint64_t Size, - uint64_t FileOffset, - uint64_t ChunkSize, - std::error_code& Ec); -ZENCORE_API void ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uint64_t ChunkSize, std::error_code& Ec); +FileContents ReadFile(const std::filesystem::path& Path); + +bool ScanFile(std::filesystem::path Path, uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc); +void WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount); +void WriteFile(std::filesystem::path Path, IoBuffer Data); +void WriteFile(std::filesystem::path Path, CompositeBuffer Data); +bool MoveToFile(std::filesystem::path Path, IoBuffer Data); +void ScanFile(void* NativeHandle, + uint64_t Offset, + uint64_t Size, + uint64_t ChunkSize, + std::function<void(const void* Data, size_t Size)>&& ProcessFunc); +void WriteFile(void* NativeHandle, const void* Data, uint64_t Size, uint64_t FileOffset, uint64_t ChunkSize, std::error_code& Ec); +void ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uint64_t ChunkSize, std::error_code& Ec); class CloneQueryInterface { @@ -198,9 +206,9 @@ public: uint64_t TargetFinalSize) = 0; }; -ZENCORE_API std::unique_ptr<CloneQueryInterface> GetCloneQueryInterface(const std::filesystem::path& TargetDirectory); +std::unique_ptr<CloneQueryInterface> GetCloneQueryInterface(const std::filesystem::path& TargetDirectory); -ZENCORE_API bool TryCloneFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToPath); +bool TryCloneFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToPath); struct CopyFileOptions { @@ -208,16 +216,16 @@ struct CopyFileOptions bool MustClone = false; }; -ZENCORE_API bool CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToPath, const CopyFileOptions& Options); -ZENCORE_API void CopyFile(const std::filesystem::path& FromPath, - const std::filesystem::path& ToPath, - const CopyFileOptions& Options, - std::error_code& OutError); -ZENCORE_API void CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options); -ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path); +bool CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToPath, const CopyFileOptions& Options); +void CopyFile(const std::filesystem::path& FromPath, + const std::filesystem::path& ToPath, + const CopyFileOptions& Options, + std::error_code& OutError); +void CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options); +bool SupportsBlockRefCounting(std::filesystem::path Path); -ZENCORE_API void PathToUtf8(const std::filesystem::path& Path, StringBuilderBase& Out); -ZENCORE_API std::string PathToUtf8(const std::filesystem::path& Path); +void PathToUtf8(const std::filesystem::path& Path, StringBuilderBase& Out); +std::string PathToUtf8(const std::filesystem::path& Path); extern template class StringBuilderImpl<std::filesystem::path::value_type>; @@ -291,7 +299,7 @@ struct DiskSpace uint64_t Total{}; }; -ZENCORE_API DiskSpace DiskSpaceInfo(std::filesystem::path Directory, std::error_code& Error); +DiskSpace DiskSpaceInfo(std::filesystem::path Directory, std::error_code& Error); inline bool DiskSpaceInfo(std::filesystem::path Directory, DiskSpace& Space) @@ -368,6 +376,11 @@ public: std::vector<std::filesystem::path> DirectoryNames; std::vector<uint32_t> DirectoryAttributes; }; + virtual bool AsyncAllowDirectory(const std::filesystem::path& Parent, const std::filesystem::path& DirectoryName) const + { + ZEN_UNUSED(Parent, DirectoryName); + return true; + } virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) = 0; }; diff --git a/src/zencore/include/zencore/iobuffer.h b/src/zencore/include/zencore/iobuffer.h index 1b2d382ee..182768ff6 100644 --- a/src/zencore/include/zencore/iobuffer.h +++ b/src/zencore/include/zencore/iobuffer.h @@ -98,9 +98,9 @@ public: { } - ZENCORE_API explicit IoBufferCore(size_t SizeBytes); - ZENCORE_API IoBufferCore(size_t SizeBytes, size_t Alignment); - ZENCORE_API ~IoBufferCore(); + explicit IoBufferCore(size_t SizeBytes); + IoBufferCore(size_t SizeBytes, size_t Alignment); + ~IoBufferCore(); void* operator new(size_t Size); void operator delete(void* Ptr); @@ -129,9 +129,9 @@ public: // - ZENCORE_API void Materialize() const; - ZENCORE_API void DeleteThis() const; - ZENCORE_API void MakeOwned(bool Immutable = true); + void Materialize() const; + void DeleteThis() const; + void MakeOwned(bool Immutable = true); inline void EnsureDataValid() const { @@ -172,7 +172,7 @@ public: inline IoBufferExtendedCore* ExtendedCore(); inline const IoBufferExtendedCore* ExtendedCore() const; - ZENCORE_API void* MutableDataPointer() const; + void* MutableDataPointer() const; inline const void* DataPointer() const { @@ -356,15 +356,15 @@ public: /** Create an uninitialized buffer of the given size */ - ZENCORE_API explicit IoBuffer(size_t InSize); + explicit IoBuffer(size_t InSize); /** Create an uninitialized buffer of the given size with the specified alignment */ - ZENCORE_API explicit IoBuffer(size_t InSize, uint64_t InAlignment); + explicit IoBuffer(size_t InSize, uint64_t InAlignment); /** Create a buffer which references a sequence of bytes inside another buffer */ - ZENCORE_API IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes = ~0ull); + IoBuffer(const IoBuffer& OuterBuffer, size_t Offset, size_t SizeBytes = ~0ull); /** Create a buffer which references a range of bytes which we assume will live * for the entire life time. @@ -376,8 +376,8 @@ public: memcpy(const_cast<void*>(m_Core->DataPointer()), DataPtr, SizeBytes); } - ZENCORE_API IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize, bool IsWholeFile); - ZENCORE_API IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize); + IoBuffer(EFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize, bool IsWholeFile); + IoBuffer(EBorrowedFileTag, void* FileHandle, uint64_t ChunkFileOffset, uint64_t ChunkSize); inline explicit operator bool() const { return !m_Core->IsNull(); } inline operator MemoryView() const& { return MemoryView(m_Core->DataPointer(), m_Core->DataBytes()); } @@ -392,7 +392,7 @@ public: [[nodiscard]] size_t GetSize() const { return m_Core->DataBytes(); } inline void SetContentType(ZenContentType ContentType) { m_Core->SetContentType(ContentType); } [[nodiscard]] inline ZenContentType GetContentType() const { return m_Core->GetContentType(); } - [[nodiscard]] ZENCORE_API bool GetFileReference(IoBufferFileReference& OutRef) const; + [[nodiscard]] bool GetFileReference(IoBufferFileReference& OutRef) const; void SetDeleteOnClose(bool DeleteOnClose); inline MemoryView GetView() const { return MemoryView(m_Core->DataPointer(), m_Core->DataBytes()); } @@ -426,14 +426,14 @@ private: class IoBufferBuilder { public: - ZENCORE_API static IoBuffer MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); - ZENCORE_API static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName); - ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); + static IoBuffer MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); + static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName); + static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); /** Make sure buffer data is memory resident, but avoid memory mapping data from files */ - ZENCORE_API static IoBuffer ReadFromFileMaybe(const IoBuffer& InBuffer); - inline static IoBuffer MakeFromMemory(MemoryView Memory) { return IoBuffer(IoBuffer::Wrap, Memory.GetData(), Memory.GetSize()); } - inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) + static IoBuffer ReadFromFileMaybe(const IoBuffer& InBuffer); + inline static IoBuffer MakeFromMemory(MemoryView Memory) { return IoBuffer(IoBuffer::Wrap, Memory.GetData(), Memory.GetSize()); } + inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { if (Sz) { diff --git a/src/zencore/include/zencore/parallelwork.h b/src/zencore/include/zencore/parallelwork.h index 138d0bc7c..536b0a056 100644 --- a/src/zencore/include/zencore/parallelwork.h +++ b/src/zencore/include/zencore/parallelwork.h @@ -21,7 +21,19 @@ public: typedef std::function<void(std::exception_ptr Ex, std::atomic<bool>& AbortFlag)> ExceptionCallback; typedef std::function<void(bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork)> UpdateCallback; - void ScheduleWork(WorkerThreadPool& WorkerPool, WorkCallback&& Work, ExceptionCallback&& OnError = {}) + inline void ScheduleWork(WorkerThreadPool& WorkerPool, WorkCallback&& Work) { ScheduleWork(WorkerPool, std::move(Work), {}, m_Mode); } + + inline void ScheduleWork(WorkerThreadPool& WorkerPool, WorkCallback&& Work, ExceptionCallback&& OnError) + { + ScheduleWork(WorkerPool, std::move(Work), std::move(OnError), m_Mode); + } + + inline void ScheduleWork(WorkerThreadPool& WorkerPool, WorkCallback&& Work, WorkerThreadPool::EMode Mode) + { + ScheduleWork(WorkerPool, std::move(Work), {}, Mode); + } + + void ScheduleWork(WorkerThreadPool& WorkerPool, WorkCallback&& Work, ExceptionCallback&& OnError, WorkerThreadPool::EMode Mode) { m_PendingWork.AddCount(1); try @@ -42,7 +54,7 @@ public: OnError(std::current_exception(), m_AbortFlag); } }, - m_Mode); + Mode); } catch (const std::exception& Ex) { diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index 04b79a1e0..e3b7a70d7 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -14,26 +14,27 @@ namespace zen { class ProcessHandle { public: - ZENCORE_API ProcessHandle(); + ProcessHandle(); ProcessHandle(const ProcessHandle&) = delete; ProcessHandle& operator=(const ProcessHandle&) = delete; - ZENCORE_API ~ProcessHandle(); - - ZENCORE_API void Initialize(int Pid); - ZENCORE_API void Initialize(int Pid, std::error_code& OutEc); - ZENCORE_API void Initialize(void* ProcessHandle); /// Initialize with an existing handle - takes ownership of the handle - ZENCORE_API [[nodiscard]] bool IsRunning() const; - ZENCORE_API [[nodiscard]] bool IsValid() const; - ZENCORE_API bool Wait(int TimeoutMs = -1); - ZENCORE_API bool Wait(int TimeoutMs, std::error_code& OutEc); - ZENCORE_API int WaitExitCode(); - ZENCORE_API int GetExitCode(); - ZENCORE_API bool Terminate(int ExitCode); - ZENCORE_API void Reset(); - [[nodiscard]] inline int Pid() const { return m_Pid; } - [[nodiscard]] inline void* Handle() const { return m_ProcessHandle; } + ~ProcessHandle(); + + void Initialize(int Pid); + void Initialize(int Pid, std::error_code& OutEc); + void Initialize(void* ProcessHandle); /// Initialize with an existing handle - takes ownership of the handle + [[nodiscard]] bool IsRunning() const; + [[nodiscard]] bool IsValid() const; + bool Wait(int TimeoutMs = -1); + bool Wait(int TimeoutMs, std::error_code& OutEc); + int WaitExitCode(); + int GetExitCode(); + bool Kill(); + bool Terminate(int ExitCode); + void Reset(); + [[nodiscard]] inline int Pid() const { return m_Pid; } + [[nodiscard]] inline void* Handle() const { return m_ProcessHandle; } private: void* m_ProcessHandle = nullptr; @@ -53,6 +54,10 @@ struct CreateProcOptions Flag_Elevated = 1 << 1, Flag_Unelevated = 1 << 2, Flag_NoConsole = 1 << 3, + // This flag creates the new process in a new process group. This is relevant only on Windows, and + // allows sending ctrl-break events to the new process group without also sending it to the current + // process. + Flag_Windows_NewProcessGroup = 1 << 4, }; const std::filesystem::path* WorkingDirectory = nullptr; @@ -66,9 +71,9 @@ using CreateProcResult = void*; // handle to the process using CreateProcResult = int32_t; // pid #endif -ZENCORE_API CreateProcResult CreateProc(const std::filesystem::path& Executable, - std::string_view CommandLine, // should also include arg[0] (executable name) - const CreateProcOptions& Options = {}); +CreateProcResult CreateProc(const std::filesystem::path& Executable, + std::string_view CommandLine, // should also include arg[0] (executable name) + const CreateProcOptions& Options = {}); /** Process monitor - monitors a list of running processes via polling @@ -83,9 +88,9 @@ public: ProcessMonitor(); ~ProcessMonitor(); - ZENCORE_API bool IsRunning(); - ZENCORE_API void AddPid(int Pid); - ZENCORE_API bool IsActive() const; + bool IsRunning(); + void AddPid(int Pid); + bool IsActive() const; private: using HandleType = void*; @@ -94,18 +99,42 @@ private: std::vector<HandleType> m_ProcessHandles; }; -ZENCORE_API bool IsProcessRunning(int pid); -ZENCORE_API bool IsProcessRunning(int pid, std::error_code& OutEc); -ZENCORE_API int GetCurrentProcessId(); -int GetProcessId(CreateProcResult ProcId); +bool IsProcessRunning(int pid); +bool IsProcessRunning(int pid, std::error_code& OutEc); +int GetCurrentProcessId(); +int GetProcessId(CreateProcResult ProcId); std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc); std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle, bool IncludeSelf = true); +/** Wait for all threads in the current process to exit (except the calling thread) + * + * This is only implemented on Windows currently. The use-case for this is to try + * and ensure that all threads have exited before exiting main() in order to + * avoid some issues which can occur if threads are still running during process + * shutdown especially when the CRT is statically linked into the executable. + * + * This is a best-effort function and may return before all threads have exited. + */ +void WaitForThreads(uint64_t WaitTimeMs); + #if ZEN_PLATFORM_LINUX void IgnoreChildSignals(); #endif +struct ProcessMetrics +{ + uint64_t MemoryBytes = 0; + uint64_t KernelTimeMs = 0; + uint64_t UserTimeMs = 0; + uint64_t WorkingSetSize = 0; + uint64_t PeakWorkingSetSize = 0; + uint64_t PagefileUsage = 0; + uint64_t PeakPagefileUsage = 0; +}; + +void GetProcessMetrics(ProcessHandle& Handle, ProcessMetrics& OutMetrics); + void process_forcelink(); // internal } // namespace zen diff --git a/src/zencore/include/zencore/session.h b/src/zencore/include/zencore/session.h index 52289b7ef..10c33da24 100644 --- a/src/zencore/include/zencore/session.h +++ b/src/zencore/include/zencore/session.h @@ -9,7 +9,7 @@ namespace zen { struct Oid; -ZENCORE_API [[nodiscard]] Oid GetSessionId(); -ZENCORE_API [[nodiscard]] std::string_view GetSessionIdString(); +[[nodiscard]] Oid GetSessionId(); +[[nodiscard]] std::string_view GetSessionIdString(); } // namespace zen diff --git a/src/zencore/include/zencore/sharedbuffer.h b/src/zencore/include/zencore/sharedbuffer.h index 7df5109cb..c57e9f568 100644 --- a/src/zencore/include/zencore/sharedbuffer.h +++ b/src/zencore/include/zencore/sharedbuffer.h @@ -28,7 +28,7 @@ public: UniqueBuffer(const UniqueBuffer&) = delete; UniqueBuffer& operator=(const UniqueBuffer&) = delete; - ZENCORE_API explicit UniqueBuffer(IoBufferCore* Owner); + explicit UniqueBuffer(IoBufferCore* Owner); [[nodiscard]] void* GetData() { return m_Buffer ? m_Buffer->MutableDataPointer() : nullptr; } [[nodiscard]] const void* GetData() const { return m_Buffer ? m_Buffer->DataPointer() : nullptr; } @@ -45,23 +45,23 @@ public: [[nodiscard]] inline bool IsNull() const { return m_Buffer.IsNull(); } /** Reset this to null. */ - ZENCORE_API void Reset(); + void Reset(); [[nodiscard]] inline MutableMemoryView GetMutableView() { return MutableMemoryView(GetData(), GetSize()); } [[nodiscard]] inline MemoryView GetView() const { return MemoryView(GetData(), GetSize()); } /** Make an uninitialized owned buffer of the specified size. */ - [[nodiscard]] ZENCORE_API static UniqueBuffer Alloc(uint64_t Size); + [[nodiscard]] static UniqueBuffer Alloc(uint64_t Size); /** Make a non-owned view of the input. */ - [[nodiscard]] ZENCORE_API static UniqueBuffer MakeMutableView(void* DataPtr, uint64_t Size); + [[nodiscard]] static UniqueBuffer MakeMutableView(void* DataPtr, uint64_t Size); /** * Convert this to an immutable shared buffer, leaving this null. * * Steals the buffer owner from the unique buffer. */ - [[nodiscard]] ZENCORE_API SharedBuffer MoveToShared(); + [[nodiscard]] SharedBuffer MoveToShared(); private: // This may be null, for a default constructed UniqueBuffer only @@ -77,11 +77,11 @@ class SharedBuffer { public: SharedBuffer() = default; - ZENCORE_API explicit SharedBuffer(UniqueBuffer&&); + explicit SharedBuffer(UniqueBuffer&&); inline explicit SharedBuffer(IoBufferCore* Owner) : m_Buffer(Owner) {} - ZENCORE_API explicit SharedBuffer(IoBuffer&& Buffer) : m_Buffer(std::move(Buffer.m_Core)) {} - ZENCORE_API explicit SharedBuffer(const IoBuffer& Buffer) : m_Buffer(Buffer.m_Core) {} - ZENCORE_API explicit SharedBuffer(RefPtr<IoBufferCore>&& Owner) : m_Buffer(std::move(Owner)) {} + explicit SharedBuffer(IoBuffer&& Buffer) : m_Buffer(std::move(Buffer.m_Core)) {} + explicit SharedBuffer(const IoBuffer& Buffer) : m_Buffer(Buffer.m_Core) {} + explicit SharedBuffer(RefPtr<IoBufferCore>&& Owner) : m_Buffer(std::move(Owner)) {} [[nodiscard]] const void* GetData() const { @@ -108,8 +108,8 @@ public: } /** Returns a buffer that is owned, by cloning if not owned. */ - [[nodiscard]] ZENCORE_API SharedBuffer MakeOwned() const&; - [[nodiscard]] ZENCORE_API SharedBuffer MakeOwned() &&; + [[nodiscard]] SharedBuffer MakeOwned() const&; + [[nodiscard]] SharedBuffer MakeOwned() &&; [[nodiscard]] bool IsOwned() const { return !m_Buffer || m_Buffer->IsOwned(); } [[nodiscard]] inline bool IsNull() const { return !m_Buffer; } @@ -161,13 +161,13 @@ public: return MakeView(Span.data(), Span.size() * sizeof(typename decltype(Span)::element_type)); } /** Make a non-owned view of the input */ - [[nodiscard]] ZENCORE_API static SharedBuffer MakeView(const void* Data, uint64_t Size); + [[nodiscard]] static SharedBuffer MakeView(const void* Data, uint64_t Size); /** Make a non-owned view of the input */ - [[nodiscard]] ZENCORE_API static SharedBuffer MakeView(MemoryView View, SharedBuffer OuterBuffer); + [[nodiscard]] static SharedBuffer MakeView(MemoryView View, SharedBuffer OuterBuffer); /** Make an owned clone of the buffer */ - [[nodiscard]] ZENCORE_API SharedBuffer Clone(); + [[nodiscard]] SharedBuffer Clone(); /** Make an owned clone of the memory in the input view */ - [[nodiscard]] ZENCORE_API static SharedBuffer Clone(MemoryView View); + [[nodiscard]] static SharedBuffer Clone(MemoryView View); private: RefPtr<IoBufferCore> m_Buffer; diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h index 93f8add0a..cbff6454f 100644 --- a/src/zencore/include/zencore/string.h +++ b/src/zencore/include/zencore/string.h @@ -85,14 +85,14 @@ StringLength(const char16_t* str) // File name helpers // -ZENCORE_API const char* FilepathFindExtension(const std::string_view& path, const char* extensionToMatch = nullptr); +const char* FilepathFindExtension(const std::string_view& path, const char* extensionToMatch = nullptr); ////////////////////////////////////////////////////////////////////////// // Text formatting of numbers // -ZENCORE_API bool ToString(std::span<char> Buffer, uint64_t Num); -ZENCORE_API bool ToString(std::span<char> Buffer, int64_t Num); +bool ToString(std::span<char> Buffer, uint64_t Num); +bool ToString(std::span<char> Buffer, int64_t Num); struct TextNumBase { @@ -120,7 +120,7 @@ class StringBuilderImpl { public: StringBuilderImpl() = default; - ZENCORE_API ~StringBuilderImpl(); + ~StringBuilderImpl(); StringBuilderImpl(const StringBuilderImpl&) = delete; StringBuilderImpl(const StringBuilderImpl&&) = delete; @@ -368,11 +368,11 @@ protected: Extend(ExtraRequired); } - ZENCORE_API void Extend(size_t ExtraCapacity); - ZENCORE_API void* AllocBuffer(size_t ByteCount); - ZENCORE_API void FreeBuffer(void* Buffer, size_t ByteCount); + void Extend(size_t ExtraCapacity); + void* AllocBuffer(size_t ByteCount); + void FreeBuffer(void* Buffer, size_t ByteCount); - ZENCORE_API [[noreturn]] void Fail(const char* FailReason); // note: throws exception + [[noreturn]] void Fail(const char* FailReason); // note: throws exception C* m_Base; C* m_CurPos; @@ -535,6 +535,22 @@ std::string WideToUtf8(const wchar_t* wstr); void WideToUtf8(const std::wstring_view& wstr, StringBuilderBase& out); std::string WideToUtf8(const std::wstring_view Wstr); +// This is a no-op helper to make it easier to write code that works with both +// narrow and wide strings +inline std::string +WideToUtf8(const char* str8) +{ + return str8; +} + +inline std::string +WideToUtf8(const std::string_view str8) +{ + return std::string(str8); +} + +////////////////////////////////////////////////////////////////////////// + inline uint8_t Char2Nibble(char c) { @@ -686,11 +702,11 @@ std::string UrlDecode(std::string_view InUrl); // Format numbers for humans // -ZENCORE_API size_t NiceNumToBuffer(uint64_t Num, std::span<char> Buffer); -ZENCORE_API size_t NiceBytesToBuffer(uint64_t Num, std::span<char> Buffer); -ZENCORE_API size_t NiceByteRateToBuffer(uint64_t Num, uint64_t ms, std::span<char> Buffer); -ZENCORE_API size_t NiceLatencyNsToBuffer(uint64_t NanoSeconds, std::span<char> Buffer); -ZENCORE_API size_t NiceTimeSpanMsToBuffer(uint64_t Milliseconds, std::span<char> Buffer); +size_t NiceNumToBuffer(uint64_t Num, std::span<char> Buffer); +size_t NiceBytesToBuffer(uint64_t Num, std::span<char> Buffer); +size_t NiceByteRateToBuffer(uint64_t Num, uint64_t ms, std::span<char> Buffer); +size_t NiceLatencyNsToBuffer(uint64_t NanoSeconds, std::span<char> Buffer); +size_t NiceTimeSpanMsToBuffer(uint64_t Milliseconds, std::span<char> Buffer); struct NiceBase { @@ -821,19 +837,66 @@ ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func) while (It != End) { - if (*It == Delim) + std::string_view Remaining{It, size_t(ptrdiff_t(End - It))}; + size_t Idx = Remaining.find(Delim, 0); + if (Idx == 0) { It++; continue; } + size_t DelimSize = 0; + + if (Idx == std::string_view::npos) + { + Idx = Remaining.size(); + } + else + { + DelimSize = 1; + } + + Count++; + std::string_view Token{It, Idx}; + if (!Func(Token)) + { + break; + } + + It = It + (Idx + DelimSize); + } + + return Count; +} + +template<typename Fn> +uint32_t +ForEachStrTok(const std::string_view& Str, const char* Delims, Fn&& Func) +{ + const char* It = Str.data(); + const char* End = It + Str.length(); + uint32_t Count = 0; + + while (It != End) + { std::string_view Remaining{It, size_t(ptrdiff_t(End - It))}; - size_t Idx = Remaining.find(Delim, 0); + size_t Idx = Remaining.find_first_of(Delims, 0); + if (Idx == 0) + { + It++; + continue; + } + + size_t DelimSize = 0; if (Idx == std::string_view::npos) { Idx = Remaining.size(); } + else + { + DelimSize = 1; + } Count++; std::string_view Token{It, Idx}; @@ -842,7 +905,7 @@ ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func) break; } - It = It + Idx; + It = It + (Idx + DelimSize); } return Count; diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h index 9fe6dfb9b..de8f9399c 100644 --- a/src/zencore/include/zencore/thread.h +++ b/src/zencore/include/zencore/thread.h @@ -27,11 +27,11 @@ void SetCurrentThreadName(std::string_view ThreadName); class RwLock { public: - ZENCORE_API void AcquireShared() noexcept; - ZENCORE_API void ReleaseShared() noexcept; + void AcquireShared() noexcept; + void ReleaseShared() noexcept; - ZENCORE_API void AcquireExclusive() noexcept; - ZENCORE_API void ReleaseExclusive() noexcept; + void AcquireExclusive() noexcept; + void ReleaseExclusive() noexcept; struct SharedLockScope { @@ -100,8 +100,8 @@ private: class Event { public: - ZENCORE_API Event(); - ZENCORE_API ~Event(); + Event(); + ~Event(); Event(Event&& Rhs) noexcept : m_EventHandle(Rhs.m_EventHandle.load()) { Rhs.m_EventHandle = nullptr; } @@ -110,10 +110,10 @@ public: Event& operator=(Event&& Rhs) = delete; - ZENCORE_API void Set(); - ZENCORE_API void Reset(); - ZENCORE_API bool Wait(int TimeoutMs = -1); - ZENCORE_API void Close(); + void Set(); + void Reset(); + bool Wait(int TimeoutMs = -1); + void Close(); #if ZEN_USE_WINDOWS_EVENTS inline void* GetWindowsHandle() { return m_EventHandle; } @@ -131,11 +131,11 @@ class NamedEvent { public: NamedEvent() = default; - ZENCORE_API explicit NamedEvent(std::string_view EventName); - ZENCORE_API ~NamedEvent(); - ZENCORE_API void Close(); - ZENCORE_API std::error_code Set(); - ZENCORE_API bool Wait(int TimeoutMs = -1); + explicit NamedEvent(std::string_view EventName); + ~NamedEvent(); + void Close(); + std::error_code Set(); + bool Wait(int TimeoutMs = -1); NamedEvent(NamedEvent&& Rhs) noexcept : m_EventHandle(Rhs.m_EventHandle.load()) { Rhs.m_EventHandle = nullptr; } @@ -162,9 +162,9 @@ class NamedMutex public: ~NamedMutex(); - ZENCORE_API [[nodiscard]] bool Create(std::string_view MutexName); + [[nodiscard]] bool Create(std::string_view MutexName); - ZENCORE_API static bool Exists(std::string_view MutexName); + static bool Exists(std::string_view MutexName); private: void* m_MutexHandle = nullptr; diff --git a/src/zencore/include/zencore/timer.h b/src/zencore/include/zencore/timer.h index 767dc4314..1df2b0bc5 100644 --- a/src/zencore/include/zencore/timer.h +++ b/src/zencore/include/zencore/timer.h @@ -16,10 +16,10 @@ namespace zen { // High frequency timers -ZENCORE_API uint64_t GetHifreqTimerValue(); -ZENCORE_API uint64_t GetHifreqTimerFrequency(); -ZENCORE_API double GetHifreqTimerToSeconds(); -ZENCORE_API uint64_t GetHifreqTimerFrequencySafe(); // May be used during static init +uint64_t GetHifreqTimerValue(); +uint64_t GetHifreqTimerFrequency(); +double GetHifreqTimerToSeconds(); +uint64_t GetHifreqTimerFrequencySafe(); // May be used during static init // Query time since process was spawned (returns time in ms) @@ -45,7 +45,7 @@ private: // Low frequency timers namespace detail { - extern ZENCORE_API uint64_t g_LofreqTimerValue; + extern uint64_t g_LofreqTimerValue; } // namespace detail inline uint64_t @@ -54,8 +54,8 @@ GetLofreqTimerValue() return detail::g_LofreqTimerValue; } -ZENCORE_API void UpdateLofreqTimerValue(); -ZENCORE_API uint64_t GetLofreqTimerFrequency(); +void UpdateLofreqTimerValue(); +uint64_t GetLofreqTimerFrequency(); void timer_forcelink(); // internal diff --git a/src/zencore/include/zencore/varint.h b/src/zencore/include/zencore/varint.h index ae9aceed6..9fe905f25 100644 --- a/src/zencore/include/zencore/varint.h +++ b/src/zencore/include/zencore/varint.h @@ -76,14 +76,24 @@ MeasureVarInt(const void* InData) inline uint32_t MeasureVarUInt(uint32_t InValue) { - return uint32_t(int32_t(FloorLog2(InValue)) / 7 + 1); + // return + // This branching implementation is faster than branchless alternatives in real-world benchmarks. + // Replaces: uint32_t(int32_t(FloorLog2(InValue)) / 7 + 1); + return (InValue < (uint32_t(1) << 28)) + ? ((InValue < (uint32_t(1) << 14)) ? ((InValue < (uint32_t(1) << 7)) ? 1 : 2) : ((InValue < (uint32_t(1) << 21)) ? 3 : 4)) + : 5; } /** Measure the number of bytes (1-9) required to encode the 64-bit input. */ inline uint32_t MeasureVarUInt(uint64_t InValue) { - return uint32_t(std::min(int32_t(FloorLog2_64(InValue)) / 7 + 1, 9)); + // This branching implementation is faster than branchless alternatives in real-world benchmarks. + // Replaces: uint32_t(std::min(int32_t(FloorLog2_64(InValue)) / 7 + 1, 9)); + return (InValue < (uint64_t(1) << 28)) + ? ((InValue < (uint64_t(1) << 14)) ? ((InValue < (uint64_t(1) << 7)) ? 1 : 2) : ((InValue < (uint64_t(1) << 21)) ? 3 : 4)) + : ((InValue < (uint64_t(1) << 42)) ? ((InValue < (uint64_t(1) << 35)) ? 5 : 6) + : ((InValue < (uint64_t(1) << 49)) ? 7 : ((InValue < (uint64_t(1) << 56)) ? 8 : 9))); } /** Measure the number of bytes (1-5) required to encode the 32-bit input. \see \ref MeasureVarUInt */ diff --git a/src/zencore/include/zencore/zencore.h b/src/zencore/include/zencore/zencore.h index b5eb3e3e8..177a19fff 100644 --- a/src/zencore/include/zencore/zencore.h +++ b/src/zencore/include/zencore/zencore.h @@ -93,18 +93,17 @@ protected: ////////////////////////////////////////////////////////////////////////// #define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false, __VA_ARGS__) -#define ZENCORE_API // Placeholder to allow DLL configs in the future (maybe) namespace zen { -ZENCORE_API bool IsApplicationExitRequested(); -ZENCORE_API bool RequestApplicationExit(int ExitCode); -ZENCORE_API int ApplicationExitCode(); -ZENCORE_API bool IsDebuggerPresent(); -ZENCORE_API void SetIsInteractiveSession(bool Value); -ZENCORE_API bool IsInteractiveSession(); +bool IsApplicationExitRequested(); +bool RequestApplicationExit(int ExitCode); +int ApplicationExitCode(); +bool IsDebuggerPresent(); +void SetIsInteractiveSession(bool Value); +bool IsInteractiveSession(); -ZENCORE_API void zencore_forcelinktests(); +void zencore_forcelinktests(); } // namespace zen diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 0b25d14f4..56849a10d 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -8,14 +8,19 @@ #include <zencore/scopeguard.h> #include <zencore/string.h> #include <zencore/testing.h> +#include <zencore/timer.h> #include <thread> +ZEN_THIRD_PARTY_INCLUDES_START + #if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> + +# include <Psapi.h> # include <shellapi.h> # include <Shlobj.h> # include <TlHelp32.h> -# include <zencore/windows.h> #else # include <fcntl.h> # include <pthread.h> @@ -35,8 +40,8 @@ # include <sys/sysctl.h> #endif -ZEN_THIRD_PARTY_INCLUDES_START #include <fmt/format.h> + ZEN_THIRD_PARTY_INCLUDES_END namespace zen { @@ -215,6 +220,51 @@ ProcessHandle::IsValid() const } bool +ProcessHandle::Kill() +{ + if (!IsRunning()) + { + return true; + } + +#if ZEN_PLATFORM_WINDOWS + SetConsoleCtrlHandler(nullptr, TRUE); // Prevent this process from terminating itself + auto _ = MakeGuard([] { SetConsoleCtrlHandler(nullptr, FALSE); }); + + // Try graceful shutdown first + if (GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, m_Pid)) + { + // Wait briefly for graceful shutdown + if (WaitForSingleObject(m_ProcessHandle, 5000) == WAIT_OBJECT_0) + { + Reset(); + return true; + } + } + + // Fall back to forceful termination if graceful shutdown failed + if (!TerminateProcess(m_ProcessHandle, 0)) + { + return false; + } +#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + int Res = kill(pid_t(m_Pid), SIGTERM); + if (Res != 0) + { + int err = errno; + if (err != ESRCH) + { + return false; + } + } +#endif + + Reset(); + + return true; +} + +bool ProcessHandle::Terminate(int ExitCode) { if (!IsRunning()) @@ -449,6 +499,10 @@ CreateProcNormal(const std::filesystem::path& Executable, std::string_view Comma { CreationFlags |= CREATE_NO_WINDOW; } + if (Options.Flags & CreateProcOptions::Flag_Windows_NewProcessGroup) + { + CreationFlags |= CREATE_NEW_PROCESS_GROUP; + } const wchar_t* WorkingDir = nullptr; if (Options.WorkingDirectory != nullptr) @@ -1082,6 +1136,90 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand #endif // ZEN_PLATFORM_LINUX } +void +WaitForThreads(uint64_t WaitTimeMs) +{ +#if ZEN_PLATFORM_WINDOWS + auto ThreadSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0); + if (ThreadSnapshot == INVALID_HANDLE_VALUE) + { + return; + } + auto _ = MakeGuard([ThreadSnapshot] { CloseHandle(ThreadSnapshot); }); + + const DWORD CurrentProcessId = ::GetCurrentProcessId(); + const DWORD CurrentThreadId = ::GetCurrentThreadId(); + + Stopwatch Timer; + + do + { + THREADENTRY32 ThreadEntry; + ThreadEntry.dwSize = sizeof(THREADENTRY32); + + uint32_t ThreadCount = 0; + if (Thread32First(ThreadSnapshot, &ThreadEntry)) + { + do + { + if (ThreadEntry.th32OwnerProcessID == CurrentProcessId && ThreadEntry.th32ThreadID != CurrentThreadId) + { + ThreadCount++; + } + } while (Thread32Next(ThreadSnapshot, &ThreadEntry)); + } + + if (ThreadCount <= 1) + { + break; + } + + const uint64_t SleepMs = 10; + Sleep(SleepMs); + } while (Timer.GetElapsedTimeMs() < WaitTimeMs); +#else + ZEN_UNUSED(WaitTimeMs); +#endif +} + +void +GetProcessMetrics(ProcessHandle& Handle, ProcessMetrics& OutMetrics) +{ +#if ZEN_PLATFORM_WINDOWS + FILETIME CreationTime; + FILETIME ExitTime; + FILETIME KernelTime; + FILETIME UserTime; + + if (GetProcessTimes(Handle.Handle(), &CreationTime, &ExitTime, &KernelTime, &UserTime)) + { + ULARGE_INTEGER KTime; + KTime.LowPart = KernelTime.dwLowDateTime; + KTime.HighPart = KernelTime.dwHighDateTime; + + ULARGE_INTEGER UTime; + UTime.LowPart = UserTime.dwLowDateTime; + UTime.HighPart = UserTime.dwHighDateTime; + + OutMetrics.KernelTimeMs = KTime.QuadPart / 10000; + OutMetrics.UserTimeMs = UTime.QuadPart / 10000; + } + + PROCESS_MEMORY_COUNTERS MemCounters; + if (GetProcessMemoryInfo(Handle.Handle(), &MemCounters, sizeof(MemCounters))) + { + OutMetrics.WorkingSetSize = MemCounters.WorkingSetSize; + OutMetrics.PeakWorkingSetSize = MemCounters.PeakWorkingSetSize; + OutMetrics.PagefileUsage = MemCounters.PagefileUsage; + OutMetrics.PeakPagefileUsage = MemCounters.PeakPagefileUsage; + } +#else + // TODO: implement for Linux and Mac + ZEN_UNUSED(Handle); + ZEN_UNUSED(OutMetrics); +#endif +} + #if ZEN_WITH_TESTS void diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp index c8c7c2cde..0ee863b74 100644 --- a/src/zencore/string.cpp +++ b/src/zencore/string.cpp @@ -1067,6 +1067,42 @@ TEST_CASE("string") TokenCount = ForEachStrTok(""sv, ',', [](const std::string_view&) { return true; }); CHECK(TokenCount == ExpectedTokenCount); } + + SUBCASE("ForEachStrTok2") + { + const auto Tokens = "here,is;my,different tokens"sv; + const auto ExpectedTokens = "here,is,my,different,tokens"sv; + int32_t ExpectedTokenCount = 5; + int32_t TokenCount = 0; + StringBuilder<512> Sb; + + TokenCount = ForEachStrTok(Tokens, " ,;", [&Sb](const std::string_view& Token) { + if (Sb.Size()) + { + Sb << ","; + } + Sb << Token; + return true; + }); + + CHECK(TokenCount == ExpectedTokenCount); + CHECK(Sb.ToString() == ExpectedTokens); + + ExpectedTokenCount = 1; + const auto Str = "mosdef"sv; + + Sb.Reset(); + TokenCount = ForEachStrTok(Str, " ,;", [&Sb](const std::string_view& Token) { + Sb << Token; + return true; + }); + CHECK(Sb.ToString() == Str); + CHECK(TokenCount == ExpectedTokenCount); + + ExpectedTokenCount = 0; + TokenCount = ForEachStrTok(""sv, " ,;", [](const std::string_view&) { return true; }); + CHECK(TokenCount == ExpectedTokenCount); + } } #endif diff --git a/src/zenhttp-test/zenhttp-test.cpp b/src/zenhttp-test/zenhttp-test.cpp index d18b2167e..c18759beb 100644 --- a/src/zenhttp-test/zenhttp-test.cpp +++ b/src/zenhttp-test/zenhttp-test.cpp @@ -15,6 +15,10 @@ int main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) { +#if ZEN_PLATFORM_WINDOWS + setlocale(LC_ALL, "en_us.UTF8"); +#endif // ZEN_PLATFORM_WINDOWS + #if ZEN_WITH_TESTS zen::zenhttp_forcelinktests(); diff --git a/src/zenhttp/clients/httpclientcommon.cpp b/src/zenhttp/clients/httpclientcommon.cpp index 8e5136dff..47425e014 100644 --- a/src/zenhttp/clients/httpclientcommon.cpp +++ b/src/zenhttp/clients/httpclientcommon.cpp @@ -309,7 +309,7 @@ namespace detail { void BufferedReadFileStream::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) { - const uint64_t MaxChunkSize = 1u * 1024 * 1024; + const uint64_t MaxChunkSize = 512u * 1024u; std::error_code Ec; ReadFile(m_FileHandle, Data, BytesToRead, FileOffset, MaxChunkSize, Ec); diff --git a/src/zenhttp/clients/httpclientcpr.cpp b/src/zenhttp/clients/httpclientcpr.cpp index 987cdd706..5d92b3b6b 100644 --- a/src/zenhttp/clients/httpclientcpr.cpp +++ b/src/zenhttp/clients/httpclientcpr.cpp @@ -163,7 +163,7 @@ CprHttpClient::~CprHttpClient() HttpClient::Response CprHttpClient::ResponseWithPayload(std::string_view SessionId, - cpr::Response& HttpResponse, + cpr::Response&& HttpResponse, const HttpResponseCode WorkResponseCode, IoBuffer&& Payload) { @@ -235,11 +235,7 @@ CprHttpClient::CommonResponse(std::string_view SessionId, cpr::Response&& HttpRe } else { - return ResponseWithPayload( - SessionId, - HttpResponse, - WorkResponseCode, - Payload ? std::move(Payload) : IoBufferBuilder::MakeCloneFromMemory(HttpResponse.text.data(), HttpResponse.text.size())); + return ResponseWithPayload(SessionId, std::move(HttpResponse), WorkResponseCode, std::move(Payload)); } } diff --git a/src/zenhttp/clients/httpclientcpr.h b/src/zenhttp/clients/httpclientcpr.h index 94d57fb43..40af53b5d 100644 --- a/src/zenhttp/clients/httpclientcpr.h +++ b/src/zenhttp/clients/httpclientcpr.h @@ -160,7 +160,7 @@ private: HttpClient::Response CommonResponse(std::string_view SessionId, cpr::Response&& HttpResponse, IoBuffer&& Payload); HttpClient::Response ResponseWithPayload(std::string_view SessionId, - cpr::Response& HttpResponse, + cpr::Response&& HttpResponse, const HttpResponseCode WorkResponseCode, IoBuffer&& Payload); }; diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 085275195..c4e67d4ed 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -1345,7 +1345,7 @@ TEST_CASE("http.common") "{a}", [&](auto& Req) { HandledA = true; - Captures = {std::string(Req.GetCapture(1))}; + Captures = {std::string(Req.GetCapture(0))}; }, HttpVerb::kGet); diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 074e40734..3438a1471 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -84,8 +84,8 @@ public: */ virtual IoBuffer ReadPayload() = 0; - ZENCORE_API CbObject ReadPayloadObject(); - ZENCORE_API CbPackage ReadPayloadPackage(); + CbObject ReadPayloadObject(); + CbPackage ReadPayloadPackage(); /** Respond with payload diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index a0431b0b1..18a0f6a40 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -4,6 +4,7 @@ #include "httptracer.h" #include <zencore/except.h> +#include <zencore/fmtutils.h> #include <zencore/logging.h> #include <zencore/memory/llm.h> #include <zencore/thread.h> @@ -13,6 +14,8 @@ #include "httpparser.h" +#include <EASTL/fixed_vector.h> + #include <deque> #include <memory> #include <string_view> @@ -28,6 +31,7 @@ ZEN_THIRD_PARTY_INCLUDES_START # include <errno.h> #endif #include <asio.hpp> +#include <asio/stream_file.hpp> ZEN_THIRD_PARTY_INCLUDES_END #define ASIO_VERBOSE_TRACE 0 @@ -154,6 +158,338 @@ Log() ////////////////////////////////////////////////////////////////////////// +#if !defined(ASIO_HAS_FILE) +# define ASIO_HAS_FILE 0 +#endif + +#if defined(ASIO_HAS_WINDOWS_OVERLAPPED_PTR) +# define ZEN_USE_TRANSMITFILE 1 +# define ZEN_USE_ASYNC_SENDFILE 0 +#else +# define ZEN_USE_TRANSMITFILE 0 +# define ZEN_USE_ASYNC_SENDFILE 0 +#endif + +#if ZEN_USE_TRANSMITFILE +template<typename Handler> +void +TransmitFileAsync(asio::ip::tcp::socket& Socket, HANDLE FileHandle, uint64_t ByteOffset, uint32_t ByteSize, Handler&& Cb) +{ +# if ZEN_BUILD_DEBUG + const uint64_t FileSize = FileSizeFromHandle(FileHandle); + const uint64_t SendEndOffset = ByteOffset + ByteSize; + + if (SendEndOffset > FileSize) + { + std::error_code DummyEc; + + ZEN_WARN("TransmitFileAsync (offset {:#x}, size {:#x}) file: '{}' (size {:#x})) tries to transmit {} bytes too many", + ByteOffset, + ByteSize, + PathFromHandle(FileHandle, DummyEc), + FileSizeFromHandle(FileHandle), + SendEndOffset - FileSize); + } +# endif // ZEN_BUILD_DEBUG + + asio::windows::overlapped_ptr OverlappedPtr( + Socket.get_executor(), + [WrappedCb = std::move(Cb), ExpectedBytes = ByteSize, FileHandle, ByteOffset, ByteSize](const std::error_code& Ec, + std::size_t ActualBytesTransferred) { + if (Ec) + { + std::error_code DummyEc; + ZEN_WARN("NOTE: TransmitFileAsync (offset {:#x}, size {:#x}) file: '{}' (size {:#x})) error '{}', transmitted {} bytes", + ByteOffset, + ByteSize, + PathFromHandle(FileHandle, DummyEc), + FileSizeFromHandle(FileHandle), + Ec.message(), + ActualBytesTransferred); + } + WrappedCb(Ec, ActualBytesTransferred); + }); + + OVERLAPPED* RawOverlapped = OverlappedPtr.get(); + RawOverlapped->Offset = uint32_t(ByteOffset & 0xffffFFFFull); + RawOverlapped->OffsetHigh = uint32_t(ByteOffset >> 32); + + const DWORD NumberOfBytesPerSend = 0; // let TransmitFile decide + + const BOOL Ok = + ::TransmitFile(Socket.native_handle(), FileHandle, ByteSize, NumberOfBytesPerSend, RawOverlapped, nullptr, /* dwReserved */ 0); + const DWORD LastError = ::GetLastError(); + + // Check if the operation completed immediately. + if (!Ok && LastError != ERROR_IO_PENDING) + { + // The operation completed immediately, so a completion notification needs + // to be posted. When complete() is called, ownership of the OVERLAPPED- + // derived object passes to the io_context. + + asio::error_code ec(LastError, asio::error::get_system_category()); + OverlappedPtr.complete(ec, 0); + } + else + { + // The operation was successfully initiated, so ownership of the + // OVERLAPPED-derived object has passed to the io_context. + + OverlappedPtr.release(); + } +} +#endif // ZEN_USE_TRANSMITFILE + +#if ZEN_USE_ASYNC_SENDFILE + +// Pipelined file sender that reads from a file and writes to a socket using two buffers +// to pipeline reads and writes. Unfortunately this strategy can't currently be used on +// non-Windows platforms as they don't currently support async file reads. We'll have +// to build a mechanism using a thread pool for that, perhaps with optional support +// for io_uring where available since that should be the most efficient. +// +// In other words, this is not super useful as Windows already has the TransmitFile +// version above, but it's here for completeness and potential future use on other platforms. + +template<class AsyncWriteStream, class CompletionHandler> +class PipelinedFileSender : public std::enable_shared_from_this<PipelinedFileSender<AsyncWriteStream, CompletionHandler>> +{ +public: + PipelinedFileSender(AsyncWriteStream& WriteSocket, + asio::stream_file&& FileToReadFrom, + uint64_t ByteSize, + std::size_t BufferSize, + CompletionHandler&& CompletionToken) + : m_WriteSocket(WriteSocket) + , m_SourceFile(std::move(FileToReadFrom)) + , m_Strand(asio::make_strand(m_WriteSocket.get_executor())) + , m_Buffers{std::vector<char>(BufferSize), std::vector<char>(BufferSize)} + , m_CompletionHandler(std::move(CompletionToken)) + , m_TotalBytesToRead(ByteSize) + , m_RemainingBytesToRead(ByteSize) + { + } + + void Start() { EnqueueRead(); } + +private: + struct Slot + { + std::size_t Size = 0; // valid bytes in buffer + bool Ready = false; // has unwritten data + bool InUse = false; // being written + }; + + void OnSendCompleted(const std::error_code& Ec) + { + // TODO: ensure this behaves properly for instance if the write fails while a read is pending + + if (m_SendCompleted) + { + return; + } + + m_SendCompleted = true; + + // Ensure completion runs on the strand/executor. + + asio::dispatch(m_Strand, [CompletionHandler = std::move(m_CompletionHandler), Ec, TotalBytesWritten = m_TotalBytesWritten]() { + CompletionHandler(Ec, TotalBytesWritten); + }); + } + + void EnqueueRead() + { + asio::dispatch(m_Strand, [self = this->shared_from_this()] { + self->TryPostRead(); + self->PumpWrites(); + }); + } + + void TryPostRead() + { + if (m_IsEof || m_ReadInFlight || m_RemainingBytesToRead == 0) + { + return; + } + + const int ReadSlotIndex = GetFreeSlotIndex(); + if (ReadSlotIndex < 0) + { + // no free slot; wait for writer to free one (not meant to ever happen) + return; + } + + m_ReadInFlight = true; + auto ReadBuffer = asio::buffer(m_Buffers[ReadSlotIndex].data(), zen::Min(m_Buffers[ReadSlotIndex].size(), m_RemainingBytesToRead)); + + asio::async_read( + m_SourceFile, + ReadBuffer, + asio::bind_executor(m_Strand, + [self = this->shared_from_this(), ReadSlotIndex](const std::error_code& Ec, std::size_t ActualBytesRead) { + self->m_ReadInFlight = false; + self->m_RemainingBytesToRead -= ActualBytesRead; + + if (Ec) + { + if (Ec == asio::error::eof) + { + ZEN_ASSERT(self->m_RemainingBytesToRead == 0); + + self->m_IsEof = true; + + // No data produced on EOF; just try to pump whatever is left + self->PumpWrites(); + } + else + { + // read error, cancel everything and let outer completion handler know + self->OnSendCompleted(Ec); + } + } + else + { + // Mark slot as ready with ActualBytesRead valid bytes of data in buffer + self->m_Slots[ReadSlotIndex].Size = ActualBytesRead; + self->m_Slots[ReadSlotIndex].Ready = true; + self->PumpWrites(); + self->TryPostRead(); + } + })); + } + + void PumpWrites() + { + if (m_WriteInFlight) + { + return; + } + + const int WriteSlotIndex = GetReadySlotIndex(); + if (WriteSlotIndex < 0) + { + // No ready data. We're done if EOF/no more data to read and no reads in flight and nothing ready + if (!m_ReadInFlight && (m_IsEof || m_RemainingBytesToRead == 0)) + { + // all done + return OnSendCompleted({}); + } + + return; + } + + m_WriteInFlight = true; + m_Slots[WriteSlotIndex].InUse = true; + + asio::async_write( + m_WriteSocket, + asio::buffer(m_Buffers[WriteSlotIndex].data(), m_Slots[WriteSlotIndex].Size), + asio::bind_executor(m_Strand, + [self = this->shared_from_this(), WriteSlotIndex](const std::error_code& Ec, std::size_t BytesWritten) { + self->m_TotalBytesWritten += BytesWritten; + self->m_WriteInFlight = false; + + if (Ec) + { + self->OnSendCompleted(Ec); + + return; + } + else + { + // Free the slot + self->m_Slots[WriteSlotIndex].Ready = false; + self->m_Slots[WriteSlotIndex].InUse = false; + self->m_Slots[WriteSlotIndex].Size = 0; + + self->TryPostRead(); + self->PumpWrites(); + } + })); + } + + int GetFreeSlotIndex() const + { + for (int i = 0; i < 2; ++i) + { + if (!m_Slots[i].Ready && !m_Slots[i].InUse) + { + return i; + } + } + return -1; + } + + int GetReadySlotIndex() const + { + for (int i = 0; i < 2; ++i) + { + if (m_Slots[i].Ready && !m_Slots[i].InUse) + { + return i; + } + } + return -1; + } + + AsyncWriteStream& m_WriteSocket; + asio::stream_file m_SourceFile; + asio::strand<asio::any_io_executor> m_Strand; + + // There's no synchronization needed for these as all access is via the strand + std::vector<char> m_Buffers[2]; + Slot m_Slots[2]; + + bool m_IsEof = false; + bool m_ReadInFlight = false; + bool m_WriteInFlight = false; + bool m_SendCompleted = false; + + const uint64_t m_TotalBytesToRead = 0; + uint64_t m_RemainingBytesToRead = 0; + uint64_t m_TotalBytesWritten = 0; + + CompletionHandler m_CompletionHandler; +}; + +template<class AsyncWriteStream, class CompletionHandler> +void +SendFileAsync(AsyncWriteStream& WriteSocket, + const auto FileHandle, + uint64_t ByteOffset, + uint64_t ByteSize, + std::size_t BufferSize, + CompletionHandler&& CompletionToken) +{ + HANDLE hReopenedFile = + ReOpenFile(FileHandle, FILE_GENERIC_READ, FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, FILE_FLAG_OVERLAPPED); + + // Note that this assumes ownership of the handle + asio::stream_file SourceFile(WriteSocket.get_executor(), hReopenedFile); + + // TODO: handle any error properly here + SourceFile.seek(ByteOffset, asio::stream_file::seek_set); + + if (BufferSize > ByteSize) + { + BufferSize = ByteSize; + } + + auto op = std::make_shared<PipelinedFileSender<AsyncWriteStream, std::decay_t<CompletionHandler>>>( + WriteSocket, + std::move(SourceFile), + ByteSize, + BufferSize, + std::forward<CompletionHandler>(CompletionToken)); + + // Start the pipeline + op->Start(); +} +#endif // ZEN_USE_ASYNC_SENDFILE + +////////////////////////////////////////////////////////////////////////// + struct HttpAsioServerImpl { public: @@ -191,7 +527,7 @@ public: class HttpAsioServerRequest : public HttpServerRequest { public: - HttpAsioServerRequest(HttpRequestParser& Request, HttpService& Service, IoBuffer PayloadBuffer); + HttpAsioServerRequest(HttpRequestParser& Request, HttpService& Service, IoBuffer PayloadBuffer, uint32_t RequestNumber); ~HttpAsioServerRequest(); virtual Oid ParseSessionId() const override; @@ -210,18 +546,41 @@ public: HttpAsioServerRequest& operator=(const HttpAsioServerRequest&) = delete; HttpRequestParser& m_Request; + uint32_t m_RequestNumber = 0; // Note: different to request ID which is derived from headers IoBuffer m_PayloadBuffer; std::unique_ptr<HttpResponse> m_Response; }; +/** + * HTTP Response representation used internally by the ASIO server + * + * This is used to build up the response headers and payload prior to sending + * it over the network. It's also responsible for managing the send operation itself, + * including ownership of the source buffers until the operation completes. + * + */ struct HttpResponse { public: HttpResponse() = default; - explicit HttpResponse(HttpContentType ContentType) : m_ContentType(ContentType) {} + explicit HttpResponse(HttpContentType ContentType, uint32_t RequestNumber) : m_RequestNumber(RequestNumber), m_ContentType(ContentType) + { + } + + ~HttpResponse() = default; + /** + * Initialize the response for sending a payload made up of multiple blobs + * + * This builds the necessary headers and IO vectors for sending the response + * and also makes sure all buffers are owned for the duration of the + * operation. + * + */ void InitializeForPayload(uint16_t ResponseCode, std::span<IoBuffer> BlobList) { + ZEN_ASSERT(m_State == State::kUninitialized); + ZEN_MEMSCOPE(GetHttpasioTag()); ZEN_TRACE_CPU("asio::InitializeForPayload"); @@ -230,57 +589,124 @@ public: const uint32_t ChunkCount = gsl::narrow<uint32_t>(BlobList.size()); m_DataBuffers.reserve(ChunkCount); + m_IoVecs.reserve(ChunkCount + 1); - for (IoBuffer& Buffer : BlobList) - { -#if 1 - m_DataBuffers.emplace_back(std::move(Buffer)).MakeOwned(); -#else - IoBuffer TempBuffer = std::move(Buffer); - TempBuffer.MakeOwned(); - m_DataBuffers.emplace_back(IoBufferBuilder::ReadFromFileMaybe(TempBuffer)); -#endif - } + m_IoVecs.emplace_back(); // header IoVec - uint64_t LocalDataSize = 0; + m_IoVecCursor = 0; - m_AsioBuffers.push_back({}); // Placeholder for header + uint64_t LocalDataSize = 0; - for (IoBuffer& Buffer : m_DataBuffers) + for (IoBuffer& Buffer : BlobList) { - uint64_t BufferDataSize = Buffer.Size(); + const uint64_t BufferDataSize = Buffer.Size(); ZEN_ASSERT(BufferDataSize); LocalDataSize += BufferDataSize; - IoBufferFileReference FileRef; - if (Buffer.GetFileReference(/* out */ FileRef)) + IoBuffer OwnedBuffer = std::move(Buffer); + + bool ChunkHandled = false; + +#if ZEN_USE_TRANSMITFILE || ZEN_USE_ASYNC_SENDFILE + if (OwnedBuffer.IsWholeFile()) { - // TODO: Use direct file transfer, via TransmitFile/sendfile - // - // this looks like it requires some custom asio plumbing however + if (IoBufferFileReference FileRef; OwnedBuffer.GetFileReference(/* out */ FileRef)) + { +# if ZEN_USE_TRANSMITFILE + // We establish a new handle here to add the FILE_FLAG_OVERLAPPED flag for use during TransmitFile + + HANDLE WinFileHandle = ReOpenFile(FileRef.FileHandle, + FILE_GENERIC_READ, + FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, + FILE_FLAG_OVERLAPPED); + + if (WinFileHandle == INVALID_HANDLE_VALUE) + { + HRESULT hRes = HRESULT_FROM_WIN32(GetLastError()); + std::error_code DummyEc; + ThrowSystemException(hRes, + fmt::format("Failed to ReOpenFile file {}", PathFromHandle(FileRef.FileHandle, DummyEc))); + } + + void* FileHandle = (void*)WinFileHandle; + + OwnedBuffer = IoBufferBuilder::MakeFromFileHandle(FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize); +# else // ZEN_USE_TRANSMITFILE + void* FileHandle = FileRef.FileHandle; + + OwnedBuffer.MakeOwned(); +# endif // ZEN_USE_TRANSMITFILE + + // Since there's a limit to how much data TransmitFile can send in one go, + // we may need to split this into multiple IoVec entries. In this case we'll + // end up reallocating the IoVec array, but this should be rare. - m_AsioBuffers.push_back({Buffer.Data(), Buffer.Size()}); + uint64_t RemainingChunkBytes = FileRef.FileChunkSize; + uint64_t ChunkOffset = FileRef.FileChunkOffset; + + const uint32_t MaxTransmitSize = 1 * 1024 * 1024 * 1024; // 1 GB + + while (RemainingChunkBytes) + { + IoVec Io{.IsFileRef = true}; + + Io.Ref.FileRef.FileHandle = FileHandle; + Io.Ref.FileRef.FileChunkOffset = ChunkOffset; + + if (RemainingChunkBytes > MaxTransmitSize) + { + Io.Ref.FileRef.FileChunkSize = MaxTransmitSize; + RemainingChunkBytes -= MaxTransmitSize; + } + else + { + Io.Ref.FileRef.FileChunkSize = gsl::narrow<uint32_t>(RemainingChunkBytes); + RemainingChunkBytes = 0; + } + + ChunkOffset += Io.Ref.FileRef.FileChunkSize; + + m_IoVecs.push_back(Io); + } + + ChunkHandled = true; + } } - else +#endif // ZEN_USE_TRANSMITFILE || ZEN_USE_ASYNC_SENDFILE + + if (!ChunkHandled) { - // Send from memory + OwnedBuffer.MakeOwned(); + + IoVec Io{.IsFileRef = false}; + + Io.Ref.MemoryRef = {.Data = OwnedBuffer.Data(), .Size = OwnedBuffer.Size()}; - m_AsioBuffers.push_back({Buffer.Data(), Buffer.Size()}); + m_IoVecs.push_back(Io); } + + m_DataBuffers.push_back(std::move(OwnedBuffer)); } + + // Now that we know the full data size, we can build the headers + m_ContentLength = LocalDataSize; std::string_view Headers = GetHeaders(); - m_AsioBuffers[0] = asio::const_buffer(Headers.data(), Headers.size()); + + IoVec& HeaderIo = m_IoVecs[0]; + + HeaderIo.IsFileRef = false; + HeaderIo.Ref.MemoryRef = {.Data = Headers.data(), .Size = Headers.size()}; + + m_State = State::kInitialized; } uint16_t ResponseCode() const { return m_ResponseCode; } uint64_t ContentLength() const { return m_ContentLength; } - const std::vector<asio::const_buffer>& AsioBuffers() const { return m_AsioBuffers; } - std::string_view GetHeaders() { ZEN_MEMSCOPE(GetHttpasioTag()); @@ -299,16 +725,146 @@ public: return m_Headers; } - void SuppressPayload() { m_AsioBuffers.resize(1); } + void SendResponse(asio::ip::tcp::socket& TcpSocket, std::function<void(const asio::error_code& Ec, std::size_t ByteCount)>&& Token) + { + ZEN_ASSERT(m_State == State::kInitialized); + + ZEN_MEMSCOPE(GetHttpasioTag()); + ZEN_TRACE_CPU("asio::SendResponse"); + + m_SendCb = std::move(Token); + m_State = State::kSending; + + SendNextChunk(TcpSocket); + } + + void SendNextChunk(asio::ip::tcp::socket& TcpSocket) + { + ZEN_ASSERT(m_State == State::kSending); + + ZEN_MEMSCOPE(GetHttpasioTag()); + ZEN_TRACE_CPU("asio::SendNextChunk"); + + if (m_IoVecCursor == m_IoVecs.size()) + { + // All data sent, complete the operation + + ZEN_ASSERT(m_SendCb); + + m_State = State::kSent; + + auto CompletionToken = [Self = this, Token = std::move(m_SendCb), TotalBytes = m_TotalBytesSent] { Token({}, TotalBytes); }; + + asio::defer(TcpSocket.get_executor(), std::move(CompletionToken)); + + return; + } + + auto OnCompletion = [this, &TcpSocket](const asio::error_code& Ec, std::size_t ByteCount) { + ZEN_ASSERT(m_State == State::kSending); + + m_TotalBytesSent += ByteCount; + if (Ec) + { + m_State = State::kFailed; + m_SendCb(Ec, m_TotalBytesSent); + } + else + { + SendNextChunk(TcpSocket); + } + }; + + const IoVec& Io = m_IoVecs[m_IoVecCursor++]; + + if (Io.IsFileRef) + { + ZEN_TRACE_VERBOSE("SendNextChunk from FILE, thread: {}, offset: {}, bytes: {}", + zen::GetCurrentThreadId(), + Io.Ref.FileRef.FileChunkOffset, + Io.Ref.FileRef.FileChunkSize); + +#if ZEN_USE_TRANSMITFILE + TransmitFileAsync(TcpSocket, + Io.Ref.FileRef.FileHandle, + Io.Ref.FileRef.FileChunkOffset, + gsl::narrow_cast<uint32_t>(Io.Ref.FileRef.FileChunkSize), + OnCompletion); +#elif ZEN_USE_ASYNC_SENDFILE + SendFileAsync(TcpSocket, + Io.Ref.FileRef.FileHandle, + Io.Ref.FileRef.FileChunkOffset, + Io.Ref.FileRef.FileChunkSize, + 64 * 1024, + OnCompletion); +#else + // This should never occur unless we compile with one + // of the options above + ZEN_WARN("invalid file reference in response"); +#endif + + return; + } + + // Send as many consecutive non-file references as possible in one asio operation + + std::vector<asio::const_buffer> AsioBuffers; + AsioBuffers.push_back(asio::const_buffer{Io.Ref.MemoryRef.Data, Io.Ref.MemoryRef.Size}); + + while (m_IoVecCursor != m_IoVecs.size()) + { + const IoVec& Io2 = m_IoVecs[m_IoVecCursor]; + + if (Io2.IsFileRef) + { + break; + } + + AsioBuffers.push_back(asio::const_buffer{Io2.Ref.MemoryRef.Data, Io2.Ref.MemoryRef.Size}); + ++m_IoVecCursor; + } + + asio::async_write(TcpSocket, std::move(AsioBuffers), asio::transfer_all(), OnCompletion); + } private: - uint16_t m_ResponseCode = 0; - bool m_IsKeepAlive = true; - HttpContentType m_ContentType = HttpContentType::kBinary; - uint64_t m_ContentLength = 0; - std::vector<IoBuffer> m_DataBuffers; - std::vector<asio::const_buffer> m_AsioBuffers; - ExtendableStringBuilder<160> m_Headers; + enum class State : uint8_t + { + kUninitialized, + kInitialized, + kSending, + kSent, + kFailed + }; + + uint32_t m_RequestNumber = 0; + uint16_t m_ResponseCode = 0; + bool m_IsKeepAlive = true; + State m_State = State::kUninitialized; + HttpContentType m_ContentType = HttpContentType::kBinary; + uint64_t m_ContentLength = 0; + eastl::fixed_vector<IoBuffer, 8> m_DataBuffers; // This is here to keep the IoBuffer buffers/handles alive + ExtendableStringBuilder<160> m_Headers; + + struct IoVec + { + bool IsFileRef; + union + { + struct MemoryBuffer + { + const void* Data; + uint64_t Size; + } MemoryRef; + IoBufferFileReference FileRef; + } Ref; + }; + + eastl::fixed_vector<IoVec, 8> m_IoVecs; + unsigned int m_IoVecCursor = 0; + + std::function<void(const asio::error_code& Ec, std::size_t ByteCount)> m_SendCb; + uint64_t m_TotalBytesSent = 0; }; ////////////////////////////////////////////////////////////////////////// @@ -339,37 +895,63 @@ private: kTerminated }; + const char* StateToString(RequestState State) + { + switch (State) + { + case RequestState::kInitialState: + return "InitialState"; + case RequestState::kInitialRead: + return "InitialRead"; + case RequestState::kReadingMore: + return "ReadingMore"; + case RequestState::kWriting: + return "Writing"; + case RequestState::kWritingFinal: + return "WritingFinal"; + case RequestState::kDone: + return "Done"; + case RequestState::kTerminated: + return "Terminated"; + default: + return "Unknown"; + } + } + RequestState m_RequestState = RequestState::kInitialState; HttpRequestParser m_RequestData{*this}; void EnqueueRead(); void OnDataReceived(const asio::error_code& Ec, std::size_t ByteCount); - void OnResponseDataSent(const asio::error_code& Ec, std::size_t ByteCount, bool Pop = false); + void OnResponseDataSent(const asio::error_code& Ec, std::size_t ByteCount, uint32_t RequestNumber, HttpResponse* ResponseToPop); void CloseConnection(); - HttpAsioServerImpl& m_Server; - asio::streambuf m_RequestBuffer; - std::unique_ptr<asio::ip::tcp::socket> m_Socket; - std::atomic<uint32_t> m_RequestCounter{0}; - uint32_t m_ConnectionId = 0; - Ref<IHttpPackageHandler> m_PackageHandler; + HttpAsioServerImpl& m_Server; + asio::streambuf m_RequestBuffer; + std::atomic<uint32_t> m_RequestCounter{0}; + uint32_t m_ConnectionId = 0; + Ref<IHttpPackageHandler> m_PackageHandler; - RwLock m_ResponsesLock; - std::deque<std::unique_ptr<HttpResponse>> m_Responses; + RwLock m_ActiveResponsesLock; + std::deque<std::unique_ptr<HttpResponse>> m_ActiveResponses; + + std::unique_ptr<asio::ip::tcp::socket> m_Socket; }; std::atomic<uint32_t> g_ConnectionIdCounter{0}; HttpServerConnection::HttpServerConnection(HttpAsioServerImpl& Server, std::unique_ptr<asio::ip::tcp::socket>&& Socket) : m_Server(Server) -, m_Socket(std::move(Socket)) , m_ConnectionId(g_ConnectionIdCounter.fetch_add(1)) +, m_Socket(std::move(Socket)) { ZEN_TRACE_VERBOSE("new connection #{}", m_ConnectionId); } HttpServerConnection::~HttpServerConnection() { + RwLock::ExclusiveLockScope _(m_ActiveResponsesLock); + ZEN_TRACE_VERBOSE("destroying connection #{}", m_ConnectionId); } @@ -434,7 +1016,11 @@ HttpServerConnection::OnDataReceived(const asio::error_code& Ec, [[maybe_unused] return; default: - ZEN_WARN("on data received ERROR, connection: {}, reason '{}'", m_ConnectionId, Ec.message()); + ZEN_WARN("on data received ERROR, connection: {} (state: {}), reason '{}'", + m_ConnectionId, + StateToString(m_RequestState), + Ec.message()); + return TerminateConnection(); } } @@ -472,37 +1058,58 @@ HttpServerConnection::OnDataReceived(const asio::error_code& Ec, [[maybe_unused] } void -HttpServerConnection::OnResponseDataSent(const asio::error_code& Ec, [[maybe_unused]] std::size_t ByteCount, bool Pop) +HttpServerConnection::OnResponseDataSent(const asio::error_code& Ec, + [[maybe_unused]] std::size_t ByteCount, + [[maybe_unused]] uint32_t RequestNumber, + HttpResponse* ResponseToPop) { ZEN_MEMSCOPE(GetHttpasioTag()); if (Ec) { - ZEN_WARN("on data sent ERROR, connection: {}, reason: '{}'", m_ConnectionId, Ec.message()); + ZEN_WARN("on data sent ERROR, connection: {} (state: {}), reason: '{}' (bytes: {})", + m_ConnectionId, + StateToString(m_RequestState), + Ec.message(), + ByteCount); + TerminateConnection(); + + return; } - else - { - ZEN_TRACE_VERBOSE("on data sent, connection: {}, request: {}, thread: {}, bytes: {}", - m_ConnectionId, - m_RequestCounter.load(std::memory_order_relaxed), - zen::GetCurrentThreadId(), - NiceBytes(ByteCount)); - if (!m_RequestData.IsKeepAlive()) - { - CloseConnection(); - } - else - { - if (Pop) + ZEN_TRACE_VERBOSE("on data sent, connection: {}, request: {}, thread: {}, bytes: {}", + m_ConnectionId, + RequestNumber, + zen::GetCurrentThreadId(), + NiceBytes(ByteCount)); + + if (ResponseToPop) + { + m_ActiveResponsesLock.WithExclusiveLock([&] { + // Once a response is sent we can release any referenced resources + // + // completion callbacks may be issued out-of-order so we need to + // remove the relevant entry from our active response list, it may + // not be the first + + if (auto It = find_if(begin(m_ActiveResponses), + end(m_ActiveResponses), + [ResponseToPop](const auto& Item) { return Item.get() == ResponseToPop; }); + It != end(m_ActiveResponses)) { - RwLock::ExclusiveLockScope _(m_ResponsesLock); - m_Responses.pop_front(); + m_ActiveResponses.erase(It); } + else + { + ZEN_WARN("response not found"); + } + }); + } - m_RequestCounter.fetch_add(1); - } + if (!m_RequestData.IsKeepAlive()) + { + CloseConnection(); } } @@ -553,13 +1160,13 @@ HttpServerConnection::HandleRequest() m_RequestState = RequestState::kWriting; } + const uint32_t RequestNumber = m_RequestCounter.fetch_add(1); + if (HttpService* Service = m_Server.RouteRequest(m_RequestData.Url())) { ZEN_TRACE_CPU("asio::HandleRequest"); - const uint32_t RequestNumber = m_RequestCounter.load(std::memory_order_relaxed); - - HttpAsioServerRequest Request(m_RequestData, *Service, m_RequestData.Body()); + HttpAsioServerRequest Request(m_RequestData, *Service, m_RequestData.Body(), RequestNumber); ZEN_TRACE_VERBOSE("handle request, connection: {}, request: {}'", m_ConnectionId, RequestNumber); @@ -635,34 +1242,38 @@ HttpServerConnection::HandleRequest() if (m_RequestData.RequestVerb() == HttpVerb::kHead) { - Response->SuppressPayload(); - } + ZEN_TRACE_CPU("asio::async_write"); - const std::vector<asio::const_buffer>& ResponseBuffers = Response->AsioBuffers(); + std::string_view Headers = Response->GetHeaders(); - uint64_t ResponseLength = 0; + std::vector<asio::const_buffer> AsioBuffers; + AsioBuffers.push_back(asio::const_buffer(Headers.data(), Headers.size())); - for (const asio::const_buffer& Buffer : ResponseBuffers) - { - ResponseLength += Buffer.size(); + asio::async_write(*m_Socket.get(), + AsioBuffers, + asio::transfer_all(), + [Conn = AsSharedPtr(), RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) { + Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr); + }); } - + else { - RwLock::ExclusiveLockScope _(m_ResponsesLock); - m_Responses.push_back(std::move(Response)); - } + ZEN_TRACE_CPU("asio::async_write"); - // TODO: should cork/uncork for Linux? + HttpResponse* ResponseRaw = Response.get(); - { - ZEN_TRACE_CPU("asio::async_write"); - asio::async_write(*m_Socket.get(), - ResponseBuffers, - asio::transfer_exactly(ResponseLength), - [Conn = AsSharedPtr()](const asio::error_code& Ec, std::size_t ByteCount) { - Conn->OnResponseDataSent(Ec, ByteCount, true); - }); + m_ActiveResponsesLock.WithExclusiveLock([&] { + // Keep referenced resources alive + m_ActiveResponses.push_back(std::move(Response)); + }); + + ResponseRaw->SendResponse( + *m_Socket, + [Conn = AsSharedPtr(), ResponseRaw, RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) { + Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ ResponseRaw); + }); } + return; } } @@ -681,10 +1292,11 @@ HttpServerConnection::HandleRequest() "\r\n"sv; } - asio::async_write( - *m_Socket.get(), - asio::buffer(Response), - [Conn = AsSharedPtr()](const asio::error_code& Ec, std::size_t ByteCount) { Conn->OnResponseDataSent(Ec, ByteCount); }); + asio::async_write(*m_Socket.get(), + asio::buffer(Response), + [Conn = AsSharedPtr(), RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) { + Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr); + }); } else { @@ -706,10 +1318,11 @@ HttpServerConnection::HandleRequest() "No suitable route found"sv; } - asio::async_write( - *m_Socket.get(), - asio::buffer(Response), - [Conn = AsSharedPtr()](const asio::error_code& Ec, std::size_t ByteCount) { Conn->OnResponseDataSent(Ec, ByteCount); }); + asio::async_write(*m_Socket.get(), + asio::buffer(Response), + [Conn = AsSharedPtr(), RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) { + Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr); + }); } } @@ -1016,9 +1629,13 @@ private: ////////////////////////////////////////////////////////////////////////// -HttpAsioServerRequest::HttpAsioServerRequest(HttpRequestParser& Request, HttpService& Service, IoBuffer PayloadBuffer) +HttpAsioServerRequest::HttpAsioServerRequest(HttpRequestParser& Request, + HttpService& Service, + IoBuffer PayloadBuffer, + uint32_t RequestNumber) : HttpServerRequest(Service) , m_Request(Request) +, m_RequestNumber(RequestNumber) , m_PayloadBuffer(std::move(PayloadBuffer)) { const int PrefixLength = Service.UriPrefixLength(); @@ -1104,7 +1721,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode) ZEN_ASSERT(!m_Response); - m_Response.reset(new HttpResponse(HttpContentType::kBinary)); + m_Response.reset(new HttpResponse(HttpContentType::kBinary, m_RequestNumber)); std::array<IoBuffer, 0> Empty; m_Response->InitializeForPayload((uint16_t)ResponseCode, Empty); @@ -1117,7 +1734,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentT ZEN_ASSERT(!m_Response); - m_Response.reset(new HttpResponse(ContentType)); + m_Response.reset(new HttpResponse(ContentType, m_RequestNumber)); m_Response->InitializeForPayload((uint16_t)ResponseCode, Blobs); } @@ -1127,7 +1744,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentT ZEN_MEMSCOPE(GetHttpasioTag()); ZEN_ASSERT(!m_Response); - m_Response.reset(new HttpResponse(ContentType)); + m_Response.reset(new HttpResponse(ContentType, m_RequestNumber)); IoBuffer MessageBuffer(IoBuffer::Wrap, ResponseString.data(), ResponseString.size()); std::array<IoBuffer, 1> SingleBufferList({MessageBuffer}); @@ -1375,23 +1992,17 @@ HttpAsioServer::OnInitialize(int BasePort, std::filesystem::path DataDir) void HttpAsioServer::OnRun(bool IsInteractive) { - const bool TestMode = !IsInteractive; - - int WaitTimeout = -1; - if (!TestMode) - { - WaitTimeout = 1000; - } + const int WaitTimeout = 1000; #if ZEN_PLATFORM_WINDOWS - if (TestMode == false) + if (IsInteractive) { ZEN_CONSOLE("Zen Server running (asio HTTP). Press ESC or Q to quit"); } do { - if (!TestMode && _kbhit() != 0) + if (IsInteractive && _kbhit() != 0) { char c = (char)_getch(); @@ -1404,7 +2015,7 @@ HttpAsioServer::OnRun(bool IsInteractive) m_ShutdownEvent.Wait(WaitTimeout); } while (!IsApplicationExitRequested()); #else - if (TestMode == false) + if (IsInteractive) { ZEN_CONSOLE("Zen Server running (asio HTTP). Ctrl-C to quit"); } diff --git a/src/zenhttp/servers/httpmulti.cpp b/src/zenhttp/servers/httpmulti.cpp index 6541a1c48..31cb04be5 100644 --- a/src/zenhttp/servers/httpmulti.cpp +++ b/src/zenhttp/servers/httpmulti.cpp @@ -56,23 +56,17 @@ HttpMultiServer::OnInitialize(int BasePort, std::filesystem::path DataDir) void HttpMultiServer::OnRun(bool IsInteractiveSession) { - const bool TestMode = !IsInteractiveSession; - - int WaitTimeout = -1; - if (!TestMode) - { - WaitTimeout = 1000; - } + const int WaitTimeout = 1000; #if ZEN_PLATFORM_WINDOWS - if (TestMode == false) + if (IsInteractiveSession) { ZEN_CONSOLE("Zen Server running (multi server). Press ESC or Q to quit"); } do { - if (!TestMode && _kbhit() != 0) + if (IsInteractiveSession && _kbhit() != 0) { char c = (char)_getch(); @@ -85,7 +79,7 @@ HttpMultiServer::OnRun(bool IsInteractiveSession) m_ShutdownEvent.Wait(WaitTimeout); } while (!IsApplicationExitRequested()); #else - if (TestMode == false) + if (IsInteractiveSession) { ZEN_CONSOLE("Zen Server running (null HTTP). Ctrl-C to quit"); } diff --git a/src/zenhttp/servers/httpnull.cpp b/src/zenhttp/servers/httpnull.cpp index 06838a0ed..0ec1cb3c4 100644 --- a/src/zenhttp/servers/httpnull.cpp +++ b/src/zenhttp/servers/httpnull.cpp @@ -34,23 +34,17 @@ HttpNullServer::OnInitialize(int BasePort, std::filesystem::path DataDir) void HttpNullServer::OnRun(bool IsInteractiveSession) { - const bool TestMode = !IsInteractiveSession; - - int WaitTimeout = -1; - if (!TestMode) - { - WaitTimeout = 1000; - } + const int WaitTimeout = 1000; #if ZEN_PLATFORM_WINDOWS - if (TestMode == false) + if (IsInteractiveSession) { ZEN_CONSOLE("Zen Server running (null HTTP). Press ESC or Q to quit"); } do { - if (!TestMode && _kbhit() != 0) + if (IsInteractiveSession && _kbhit() != 0) { char c = (char)_getch(); @@ -63,7 +57,7 @@ HttpNullServer::OnRun(bool IsInteractiveSession) m_ShutdownEvent.Wait(WaitTimeout); } while (!IsApplicationExitRequested()); #else - if (TestMode == false) + if (IsInteractiveSession) { ZEN_CONSOLE("Zen Server running (null HTTP). Ctrl-C to quit"); } diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp index 7ee4c6e62..b9217ed87 100644 --- a/src/zenhttp/servers/httpplugin.cpp +++ b/src/zenhttp/servers/httpplugin.cpp @@ -790,23 +790,17 @@ HttpPluginServerImpl::OnRun(bool IsInteractive) { ZEN_MEMSCOPE(GetHttppluginTag()); - const bool TestMode = !IsInteractive; - - int WaitTimeout = -1; - if (!TestMode) - { - WaitTimeout = 1000; - } + const int WaitTimeout = 1000; # if ZEN_PLATFORM_WINDOWS - if (TestMode == false) + if (IsInteractive) { ZEN_CONSOLE("Zen Server running (plugin HTTP). Press ESC or Q to quit"); } do { - if (!TestMode && _kbhit() != 0) + if (IsInteractive && _kbhit() != 0) { char c = (char)_getch(); @@ -819,7 +813,7 @@ HttpPluginServerImpl::OnRun(bool IsInteractive) m_ShutdownEvent.Wait(WaitTimeout); } while (!IsApplicationExitRequested()); # else - if (TestMode == false) + if (IsInteractive) { ZEN_CONSOLE("Zen Server running (plugin HTTP). Ctrl-C to quit"); } diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index c555a39b6..54cc0c22d 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -1647,9 +1647,9 @@ HttpSysTransaction::InvokeRequestHandler(HttpService& Service, IoBuffer Payload) std::string_view Verb = ToString(ThisRequest.RequestVerb()); std::string_view Uri = ThisRequest.m_UriUtf8.ToView(); - ExtendableStringBuilder<64> SpanName; - SpanName << Verb << " " << Uri; - otel::ScopedSpan HttpSpan(SpanName.ToView(), [&](otel::Span& Span) { + auto SpanNamer = [&](StringBuilderBase& SpanName) { SpanName << Verb << " " << Uri; }; + + auto SpanAnnotator = [&](otel::Span& Span) { Span.AddAttribute("http.request.method"sv, Verb); Span.AddAttribute("url.path"sv, Uri); // FIXME: should be total size including headers etc according to spec @@ -1661,7 +1661,9 @@ HttpSysTransaction::InvokeRequestHandler(HttpService& Service, IoBuffer Payload) ExtendableStringBuilder<64> ClientAddr; GetAddressString(ClientAddr, SockAddr, /* IncludePort */ false); Span.AddAttribute("client.address"sv, ClientAddr.ToView()); - }); + }; + + otel::ScopedSpan HttpSpan(SpanNamer, SpanAnnotator); # endif if (!HandlePackageOffers(Service, ThisRequest, m_PackageHandler)) diff --git a/src/zennet-test/zennet-test.cpp b/src/zennet-test/zennet-test.cpp index 5e4d29220..bc3b8e8e9 100644 --- a/src/zennet-test/zennet-test.cpp +++ b/src/zennet-test/zennet-test.cpp @@ -16,6 +16,10 @@ int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) { +#if ZEN_PLATFORM_WINDOWS + setlocale(LC_ALL, "en_us.UTF8"); +#endif // ZEN_PLATFORM_WINDOWS + #if ZEN_WITH_TESTS zen::zennet_forcelinktests(); diff --git a/src/zenremotestore-test/zenremotestore-test.cpp b/src/zenremotestore-test/zenremotestore-test.cpp index a49c6d273..5db185041 100644 --- a/src/zenremotestore-test/zenremotestore-test.cpp +++ b/src/zenremotestore-test/zenremotestore-test.cpp @@ -17,6 +17,10 @@ int main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) { +#if ZEN_PLATFORM_WINDOWS + setlocale(LC_ALL, "en_us.UTF8"); +#endif // ZEN_PLATFORM_WINDOWS + #if ZEN_WITH_TESTS zen::zenremotestore_forcelinktests(); diff --git a/src/zenremotestore/builds/buildmanifest.cpp b/src/zenremotestore/builds/buildmanifest.cpp new file mode 100644 index 000000000..051436e96 --- /dev/null +++ b/src/zenremotestore/builds/buildmanifest.cpp @@ -0,0 +1,173 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zenremotestore/builds/buildmanifest.h> + +#include <zencore/compactbinary.h> +#include <zencore/fmtutils.h> + +#if ZEN_WITH_TESTS +# include <zencore/basicfile.h> +# include <zencore/testing.h> +# include <zencore/testutils.h> +#endif // ZEN_WITH_TESTS + +namespace zen { + +using namespace std::literals; + +BuildManifest +ParseBuildManifest(const std::filesystem::path& ManifestPath) +{ + BuildManifest Result; + { + IoBuffer ManifestContent = ReadFile(ManifestPath).Flatten(); + + if (ToLower(ManifestPath.extension().string()) == ".json") + { + IoBuffer MetaDataJson = ReadFile(ManifestPath).Flatten(); + std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); + std::string JsonError; + CbObject Manifest = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); + if (!JsonError.empty()) + { + throw std::runtime_error(fmt::format("Invalid manifest file at {}. '{}'", ManifestPath, JsonError)); + } + CbObjectView PartsObject = Manifest["parts"sv].AsObjectView(); + for (CbFieldView PartsField : PartsObject) + { + std::string_view PartName = PartsField.GetName(); + if (PartName.empty()) + { + throw std::runtime_error(fmt::format("Part {} in manifest file at {} does not have a name. '{}'", + Result.Parts.size() + 1, + ManifestPath, + JsonError)); + } + CbObjectView Part = PartsField.AsObjectView(); + Oid PartId = Part["partId"sv].AsObjectId(); + CbArrayView FilesArray = Part["files"sv].AsArrayView(); + std::vector<std::filesystem::path> Files; + Files.reserve(FilesArray.Num()); + for (CbFieldView FileField : FilesArray) + { + std::filesystem::path File(FileField.AsU8String()); + Files.push_back(File); + } + + Result.Parts.push_back(BuildManifest::Part{.PartId = PartId, .PartName = std::string(PartName), .Files = std::move(Files)}); + } + return Result; + } + else + { + Result.Parts.resize(1); + BuildManifest::Part& SinglePart = Result.Parts.front(); + + std::string_view ManifestString((const char*)ManifestContent.GetView().GetData(), ManifestContent.GetSize()); + std::string_view::size_type Offset = 0; + while (Offset < ManifestContent.GetSize()) + { + size_t PathBreakOffset = ManifestString.find_first_of("\t\r\n", Offset); + if (PathBreakOffset == std::string_view::npos) + { + PathBreakOffset = ManifestContent.GetSize(); + } + std::string_view AssetPath = ManifestString.substr(Offset, PathBreakOffset - Offset); + if (!AssetPath.empty()) + { + SinglePart.Files.push_back(std::filesystem::path(AssetPath)); + } + Offset = PathBreakOffset; + size_t EolOffset = ManifestString.find_first_of("\r\n", Offset); + if (EolOffset == std::string_view::npos) + { + break; + } + Offset = EolOffset; + size_t LineBreakOffset = ManifestString.find_first_not_of("\t\r\n", Offset); + if (LineBreakOffset == std::string_view::npos) + { + break; + } + Offset = LineBreakOffset; + } + } + } + return Result; +} +#if ZEN_WITH_TESTS + +TEST_CASE("buildmanifest.unstructured") +{ + ScopedTemporaryDirectory Root; + std::vector<std::filesystem::path> Files = {"fileA", "dirA/FileB", "dirB/FileC", "dirB/FileD"}; + + { + ExtendableStringBuilder<512> SB; + for (const std::filesystem::path& File : Files) + { + SB << File.generic_string() << "\n"; + } + WriteFile(Root.Path() / "manifest.txt", IoBuffer(IoBuffer::Wrap, SB.ToView().data(), SB.ToView().length())); + } + + BuildManifest Manifest = ParseBuildManifest(Root.Path() / "manifest.txt"); + CHECK_EQ(Manifest.Parts.size(), 1u); + CHECK_EQ(Manifest.Parts[0].PartId, Oid::Zero); + CHECK_EQ(Manifest.Parts[0].PartName, ""); + CHECK_EQ(Manifest.Parts[0].Files, Files); +} + +TEST_CASE("buildmanifest.structured") +{ + ScopedTemporaryDirectory Root; + + std::string Id = Oid::NewOid().ToString(); + + std::string ManifestString = + "{\n" + " \"parts\": {\n" + " \"default\": {\n" + " \"partId\": \"098a2742d46c22a67ab57457\",\n" + " \"files\": [\n" + " \"foo/bar\",\n" + " \"baz.exe\"\n" + " ]\n" + " },\n" + " \"symbols\": {\n" + " \"files\": [\n" + " \"baz.pdb\"\n" + " ]\n" + " }\n" + " }\n" + "}\n"; + + WriteFile(Root.Path() / "manifest.json", IoBuffer(IoBuffer::Wrap, ManifestString.data(), ManifestString.length())); + + const Oid DefaultPartExpectedId = Oid::FromHexString("098a2742d46c22a67ab57457"); + const std::string DefaultPartExpectedName = "default"; + const Oid SymbolPartExpectedId = Oid::Zero; + const std::string SymbolsPartExpectedName = "symbols"; + + BuildManifest Manifest = ParseBuildManifest(Root.Path() / "manifest.json"); + CHECK_EQ(Manifest.Parts.size(), 2u); + CHECK_EQ(Manifest.Parts[0].PartId, DefaultPartExpectedId); + CHECK_EQ(Manifest.Parts[0].PartName, DefaultPartExpectedName); + CHECK_EQ(Manifest.Parts[0].Files.size(), 2u); + CHECK_EQ(Manifest.Parts[0].Files[0].generic_string(), "foo/bar"); + CHECK_EQ(Manifest.Parts[0].Files[1].generic_string(), "baz.exe"); + + CHECK_EQ(Manifest.Parts[1].PartId, SymbolPartExpectedId); + CHECK_EQ(Manifest.Parts[1].PartName, SymbolsPartExpectedName); + CHECK_EQ(Manifest.Parts[1].Files.size(), 1u); + CHECK_EQ(Manifest.Parts[1].Files[0].generic_string(), "baz.pdb"); +} + +void +buildmanifest_forcelink() +{ +} + +#endif // ZEN_WITH_TESTS + +} // namespace zen diff --git a/src/zenremotestore/builds/buildsavedstate.cpp b/src/zenremotestore/builds/buildsavedstate.cpp index 5a86ee865..1d1f4605f 100644 --- a/src/zenremotestore/builds/buildsavedstate.cpp +++ b/src/zenremotestore/builds/buildsavedstate.cpp @@ -207,9 +207,14 @@ ReadBuildSaveStateFile(const std::filesystem::path& StateFilePath) { ZEN_TRACE_CPU("ReadStateFile"); - FileContents FileData = ReadFile(StateFilePath); + IoBuffer DataBuffer; + { + BasicFile Source(StateFilePath, BasicFile::Mode::kRead); + DataBuffer = Source.ReadAll(); + } + CbValidateError ValidateError; - if (CbObject CurrentStateObject = ValidateAndReadCompactBinaryObject(FileData.Flatten(), ValidateError); + if (CbObject CurrentStateObject = ValidateAndReadCompactBinaryObject(std::move(DataBuffer), ValidateError); ValidateError == CbValidateError::None) { if (CurrentStateObject) diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp index b9f5eb07a..2319ad66d 100644 --- a/src/zenremotestore/builds/buildstorageoperations.cpp +++ b/src/zenremotestore/builds/buildstorageoperations.cpp @@ -3,11 +3,13 @@ #include <zenremotestore/builds/buildstorageoperations.h> #include <zenremotestore/builds/buildcontent.h> +#include <zenremotestore/builds/buildmanifest.h> #include <zenremotestore/builds/buildsavedstate.h> #include <zenremotestore/builds/buildstorage.h> #include <zenremotestore/builds/buildstoragecache.h> #include <zenremotestore/builds/buildstorageutil.h> #include <zenremotestore/chunking/chunkblock.h> +#include <zenremotestore/chunking/chunkingcache.h> #include <zenremotestore/chunking/chunkingcontroller.h> #include <zenremotestore/filesystemutils.h> #include <zenremotestore/operationlogoutput.h> @@ -16,6 +18,7 @@ #include <zencore/compactbinary.h> #include <zencore/compactbinaryfile.h> #include <zencore/compactbinaryutil.h> +#include <zencore/compactbinaryvalue.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/parallelwork.h> @@ -23,6 +26,7 @@ #include <zencore/string.h> #include <zencore/timer.h> #include <zencore/trace.h> +#include <zenutil/wildcard.h> #include <numeric> @@ -31,6 +35,12 @@ ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_set.h> ZEN_THIRD_PARTY_INCLUDES_END +#if ZEN_WITH_TESTS +# include <zencore/testing.h> +# include <zencore/testutils.h> +# include <zenremotestore/builds/filebuildstorage.h> +#endif // ZEN_WITH_TESTS + namespace zen { using namespace std::literals; @@ -782,78 +792,17 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) Stopwatch LocalTimer; - for (uint32_t LocalSequenceIndex = 0; - LocalSequenceIndex < m_LocalContent.ChunkedContent.SequenceRawHashes.size() && (RemainingChunkCount > 0); - LocalSequenceIndex++) - { - const IoHash& LocalSequenceRawHash = m_LocalContent.ChunkedContent.SequenceRawHashes[LocalSequenceIndex]; - const uint32_t LocalOrderOffset = m_LocalLookup.SequenceIndexChunkOrderOffset[LocalSequenceIndex]; - - { - uint64_t SourceOffset = 0; - const uint32_t LocalChunkCount = m_LocalContent.ChunkedContent.ChunkCounts[LocalSequenceIndex]; - for (uint32_t LocalOrderIndex = 0; LocalOrderIndex < LocalChunkCount; LocalOrderIndex++) - { - const uint32_t LocalChunkIndex = m_LocalContent.ChunkedContent.ChunkOrders[LocalOrderOffset + LocalOrderIndex]; - const IoHash& LocalChunkHash = m_LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; - const uint64_t LocalChunkRawSize = m_LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex]; - - if (auto RemoteChunkIt = m_RemoteLookup.ChunkHashToChunkIndex.find(LocalChunkHash); - RemoteChunkIt != m_RemoteLookup.ChunkHashToChunkIndex.end()) - { - const uint32_t RemoteChunkIndex = RemoteChunkIt->second; - if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) - { - std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = - GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex); + ScavengeSourceForChunks(RemainingChunkCount, + RemoteChunkIndexNeedsCopyFromLocalFileFlags, + RawHashToCopyChunkDataIndex, + SequenceIndexChunksLeftToWriteCounters, + m_LocalContent, + m_LocalLookup, + CopyChunkDatas, + uint32_t(-1), + m_CacheMappingStats.LocalChunkMatchingRemoteCount, + m_CacheMappingStats.LocalChunkMatchingRemoteByteCount); - if (!ChunkTargetPtrs.empty()) - { - CopyChunkData::ChunkTarget Target = { - .TargetChunkLocationCount = gsl::narrow<uint32_t>(ChunkTargetPtrs.size()), - .RemoteChunkIndex = RemoteChunkIndex, - .CacheFileOffset = SourceOffset}; - if (auto CopySourceIt = RawHashToCopyChunkDataIndex.find(LocalSequenceRawHash); - CopySourceIt != RawHashToCopyChunkDataIndex.end()) - { - CopyChunkData& Data = CopyChunkDatas[CopySourceIt->second]; - if (Data.TargetChunkLocationPtrs.size() > 1024) - { - RawHashToCopyChunkDataIndex.insert_or_assign(LocalSequenceRawHash, CopyChunkDatas.size()); - CopyChunkDatas.push_back( - CopyChunkData{.ScavengeSourceIndex = (uint32_t)-1, - .SourceSequenceIndex = LocalSequenceIndex, - .TargetChunkLocationPtrs = ChunkTargetPtrs, - .ChunkTargets = std::vector<CopyChunkData::ChunkTarget>{Target}}); - } - else - { - Data.TargetChunkLocationPtrs.insert(Data.TargetChunkLocationPtrs.end(), - ChunkTargetPtrs.begin(), - ChunkTargetPtrs.end()); - Data.ChunkTargets.push_back(Target); - } - } - else - { - RawHashToCopyChunkDataIndex.insert_or_assign(LocalSequenceRawHash, CopyChunkDatas.size()); - CopyChunkDatas.push_back( - CopyChunkData{.ScavengeSourceIndex = (uint32_t)-1, - .SourceSequenceIndex = LocalSequenceIndex, - .TargetChunkLocationPtrs = ChunkTargetPtrs, - .ChunkTargets = std::vector<CopyChunkData::ChunkTarget>{Target}}); - } - m_CacheMappingStats.LocalChunkMatchingRemoteCount++; - m_CacheMappingStats.LocalChunkMatchingRemoteByteCount += LocalChunkRawSize; - RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true; - RemainingChunkCount--; - } - } - } - SourceOffset += LocalChunkRawSize; - } - } - } m_CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs(); } @@ -867,86 +816,22 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) ScavengedContentIndex++) { const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengedContentIndex]; - // const std::filesystem::path& ScavengedPath = ScavengedPaths[ScavengedContentIndex]; - const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengedContentIndex]; - - for (uint32_t ScavengedSequenceIndex = 0; - ScavengedSequenceIndex < ScavengedContent.ChunkedContent.SequenceRawHashes.size() && (RemainingChunkCount > 0); - ScavengedSequenceIndex++) - { - const IoHash& ScavengedSequenceRawHash = ScavengedContent.ChunkedContent.SequenceRawHashes[ScavengedSequenceIndex]; - const uint32_t ScavengedOrderOffset = ScavengedLookup.SequenceIndexChunkOrderOffset[ScavengedSequenceIndex]; - - { - uint64_t SourceOffset = 0; - const uint32_t ScavengedChunkCount = ScavengedContent.ChunkedContent.ChunkCounts[ScavengedSequenceIndex]; - for (uint32_t ScavengedOrderIndex = 0; ScavengedOrderIndex < ScavengedChunkCount; ScavengedOrderIndex++) - { - const uint32_t ScavengedChunkIndex = - ScavengedContent.ChunkedContent.ChunkOrders[ScavengedOrderOffset + ScavengedOrderIndex]; - const IoHash& ScavengedChunkHash = ScavengedContent.ChunkedContent.ChunkHashes[ScavengedChunkIndex]; - const uint64_t ScavengedChunkRawSize = ScavengedContent.ChunkedContent.ChunkRawSizes[ScavengedChunkIndex]; - - if (auto RemoteChunkIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ScavengedChunkHash); - RemoteChunkIt != m_RemoteLookup.ChunkHashToChunkIndex.end()) - { - const uint32_t RemoteChunkIndex = RemoteChunkIt->second; - if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) - { - std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = - GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex); - - if (!ChunkTargetPtrs.empty()) - { - CopyChunkData::ChunkTarget Target = { - .TargetChunkLocationCount = gsl::narrow<uint32_t>(ChunkTargetPtrs.size()), - .RemoteChunkIndex = RemoteChunkIndex, - .CacheFileOffset = SourceOffset}; - if (auto CopySourceIt = RawHashToCopyChunkDataIndex.find(ScavengedSequenceRawHash); - CopySourceIt != RawHashToCopyChunkDataIndex.end()) - { - CopyChunkData& Data = CopyChunkDatas[CopySourceIt->second]; - if (Data.TargetChunkLocationPtrs.size() > 1024) - { - RawHashToCopyChunkDataIndex.insert_or_assign(ScavengedSequenceRawHash, - CopyChunkDatas.size()); - CopyChunkDatas.push_back( - CopyChunkData{.ScavengeSourceIndex = ScavengedContentIndex, - .SourceSequenceIndex = ScavengedSequenceIndex, - .TargetChunkLocationPtrs = ChunkTargetPtrs, - .ChunkTargets = std::vector<CopyChunkData::ChunkTarget>{Target}}); - } - else - { - Data.TargetChunkLocationPtrs.insert(Data.TargetChunkLocationPtrs.end(), - ChunkTargetPtrs.begin(), - ChunkTargetPtrs.end()); - Data.ChunkTargets.push_back(Target); - } - } - else - { - RawHashToCopyChunkDataIndex.insert_or_assign(ScavengedSequenceRawHash, CopyChunkDatas.size()); - CopyChunkDatas.push_back( - CopyChunkData{.ScavengeSourceIndex = ScavengedContentIndex, - .SourceSequenceIndex = ScavengedSequenceIndex, - .TargetChunkLocationPtrs = ChunkTargetPtrs, - .ChunkTargets = std::vector<CopyChunkData::ChunkTarget>{Target}}); - } - m_CacheMappingStats.ScavengedChunkMatchingRemoteCount++; - m_CacheMappingStats.ScavengedChunkMatchingRemoteByteCount += ScavengedChunkRawSize; - RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true; - RemainingChunkCount--; - } - } - } - SourceOffset += ScavengedChunkRawSize; - } - } - } + const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengedContentIndex]; + + ScavengeSourceForChunks(RemainingChunkCount, + RemoteChunkIndexNeedsCopyFromLocalFileFlags, + RawHashToCopyChunkDataIndex, + SequenceIndexChunksLeftToWriteCounters, + ScavengedContent, + ScavengedLookup, + CopyChunkDatas, + ScavengedContentIndex, + m_CacheMappingStats.ScavengedChunkMatchingRemoteCount, + m_CacheMappingStats.ScavengedChunkMatchingRemoteByteCount); } m_CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs(); } + if (!m_Options.IsQuiet) { if (m_CacheMappingStats.CacheSequenceHashesCount > 0 || m_CacheMappingStats.CacheChunkCount > 0 || @@ -1418,37 +1303,39 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) } } - Work.ScheduleWork(m_IOWorkerPool, - [this, - &SequenceIndexChunksLeftToWriteCounters, - &Work, - &ExistsResult, - &WritePartsComplete, - &LooseChunkHashWorks, - LooseChunkHashWorkIndex, - TotalRequestCount, - TotalPartWriteCount, - &WriteCache, - &FilteredDownloadedBytesPerSecond, - &FilteredWrittenBytesPerSecond](std::atomic<bool>&) mutable { - ZEN_TRACE_CPU("Async_ReadPreDownloadedChunk"); - if (!m_AbortFlag) - { - LooseChunkHashWorkData& LooseChunkHashWork = LooseChunkHashWorks[LooseChunkHashWorkIndex]; - const uint32_t RemoteChunkIndex = LooseChunkHashWorks[LooseChunkHashWorkIndex].RemoteChunkIndex; - WriteLooseChunk(RemoteChunkIndex, - ExistsResult, - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - std::move(LooseChunkHashWork.ChunkTargetPtrs), - WriteCache, - Work, - TotalRequestCount, - TotalPartWriteCount, - FilteredDownloadedBytesPerSecond, - FilteredWrittenBytesPerSecond); - } - }); + Work.ScheduleWork( + m_IOWorkerPool, + [this, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &ExistsResult, + &WritePartsComplete, + &LooseChunkHashWorks, + LooseChunkHashWorkIndex, + TotalRequestCount, + TotalPartWriteCount, + &WriteCache, + &FilteredDownloadedBytesPerSecond, + &FilteredWrittenBytesPerSecond](std::atomic<bool>&) mutable { + ZEN_TRACE_CPU("Async_ReadPreDownloadedChunk"); + if (!m_AbortFlag) + { + LooseChunkHashWorkData& LooseChunkHashWork = LooseChunkHashWorks[LooseChunkHashWorkIndex]; + const uint32_t RemoteChunkIndex = LooseChunkHashWorks[LooseChunkHashWorkIndex].RemoteChunkIndex; + WriteLooseChunk(RemoteChunkIndex, + ExistsResult, + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + std::move(LooseChunkHashWork.ChunkTargetPtrs), + WriteCache, + Work, + TotalRequestCount, + TotalPartWriteCount, + FilteredDownloadedBytesPerSecond, + FilteredWrittenBytesPerSecond); + } + }, + WorkerThreadPool::EMode::EnableBacklog); } std::unique_ptr<CloneQueryInterface> CloneQuery; @@ -1708,7 +1595,9 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) FilteredWrittenBytesPerSecond.Stop(); } } - }); + }, + OnDiskPath.empty() ? WorkerThreadPool::EMode::DisableBacklog + : WorkerThreadPool::EMode::EnableBacklog); } }); } @@ -1896,7 +1785,9 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) FilteredWrittenBytesPerSecond.Stop(); } } - }); + }, + BlockChunkPath.empty() ? WorkerThreadPool::EMode::DisableBacklog + : WorkerThreadPool::EMode::EnableBacklog); } } } @@ -2078,13 +1969,14 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) { if (!m_Options.WipeTargetFolder) { + // Check if it is already in the correct place if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string()); RemotePathIt != RemotePathToRemoteIndex.end()) { const uint32_t RemotePathIndex = RemotePathIt->second; if (m_RemoteContent.RawHashes[RemotePathIndex] == RawHash) { - // It is already in it's desired place + // It is already in it's correct place RemotePathIndexToLocalPathIndex[RemotePathIndex] = LocalPathIndex; SequenceHashToLocalPathIndex.insert({RawHash, LocalPathIndex}); MatchCount++; @@ -2100,31 +1992,27 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) PathMismatchCount++; } } + + // Do we need it? if (m_RemoteLookup.RawHashToSequenceIndex.contains(RawHash)) { if (!CachedRemoteSequences.contains(RawHash)) { - ZEN_TRACE_CPU("MoveToCache"); - // We need it + // We need it, make sure we move it to the cache FilesToCache.push_back(LocalPathIndex); CachedRemoteSequences.insert(RawHash); + continue; } else { - // We already have it SkippedCount++; } } - else if (!m_Options.WipeTargetFolder) - { - // We don't need it - RemoveLocalPathIndexes.push_back(LocalPathIndex); - DeleteCount++; - } } - else if (!m_Options.WipeTargetFolder) + + if (!m_Options.WipeTargetFolder) { - // Delete local file as we did not scavenge the folder + // Explicitly delete the unneeded local file RemoveLocalPathIndexes.push_back(LocalPathIndex); DeleteCount++; } @@ -2941,6 +2829,81 @@ BuildsOperationUpdateFolder::FindScavengeContent(const ScavengeSource& Source, return true; } +void +BuildsOperationUpdateFolder::ScavengeSourceForChunks(uint32_t& InOutRemainingChunkCount, + std::vector<bool>& InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags, + tsl::robin_map<IoHash, size_t, IoHash::Hasher>& InOutRawHashToCopyChunkDataIndex, + const std::vector<std::atomic<uint32_t>>& SequenceIndexChunksLeftToWriteCounters, + const ChunkedFolderContent& ScavengedContent, + const ChunkedContentLookup& ScavengedLookup, + std::vector<CopyChunkData>& InOutCopyChunkDatas, + uint32_t ScavengedContentIndex, + uint64_t& InOutChunkMatchingRemoteCount, + uint64_t& InOutChunkMatchingRemoteByteCount) +{ + for (uint32_t RemoteChunkIndex = 0; + RemoteChunkIndex < m_RemoteContent.ChunkedContent.ChunkHashes.size() && (InOutRemainingChunkCount > 0); + RemoteChunkIndex++) + { + if (!InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) + { + const IoHash& RemoteChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + if (auto It = ScavengedLookup.ChunkHashToChunkIndex.find(RemoteChunkHash); It != ScavengedLookup.ChunkHashToChunkIndex.end()) + { + std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = + GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex); + + if (!ChunkTargetPtrs.empty()) + { + const uint32_t ScavengedChunkIndex = It->second; + const uint64_t ScavengedChunkRawSize = ScavengedContent.ChunkedContent.ChunkRawSizes[ScavengedChunkIndex]; + const size_t ChunkSequenceLocationOffset = ScavengedLookup.ChunkSequenceLocationOffset[ScavengedChunkIndex]; + const ChunkedContentLookup::ChunkSequenceLocation& ScavengeLocation = + ScavengedLookup.ChunkSequenceLocations[ChunkSequenceLocationOffset]; + const IoHash& ScavengedSequenceRawHash = + ScavengedContent.ChunkedContent.SequenceRawHashes[ScavengeLocation.SequenceIndex]; + + CopyChunkData::ChunkTarget Target = {.TargetChunkLocationCount = gsl::narrow<uint32_t>(ChunkTargetPtrs.size()), + .RemoteChunkIndex = RemoteChunkIndex, + .CacheFileOffset = ScavengeLocation.Offset}; + if (auto CopySourceIt = InOutRawHashToCopyChunkDataIndex.find(ScavengedSequenceRawHash); + CopySourceIt != InOutRawHashToCopyChunkDataIndex.end()) + { + CopyChunkData& Data = InOutCopyChunkDatas[CopySourceIt->second]; + if (Data.TargetChunkLocationPtrs.size() > 1024) + { + InOutRawHashToCopyChunkDataIndex.insert_or_assign(ScavengedSequenceRawHash, InOutCopyChunkDatas.size()); + InOutCopyChunkDatas.push_back(CopyChunkData{.ScavengeSourceIndex = ScavengedContentIndex, + .SourceSequenceIndex = ScavengeLocation.SequenceIndex, + .TargetChunkLocationPtrs = ChunkTargetPtrs, + .ChunkTargets = std::vector<CopyChunkData::ChunkTarget>{Target}}); + } + else + { + Data.TargetChunkLocationPtrs.insert(Data.TargetChunkLocationPtrs.end(), + ChunkTargetPtrs.begin(), + ChunkTargetPtrs.end()); + Data.ChunkTargets.push_back(Target); + } + } + else + { + InOutRawHashToCopyChunkDataIndex.insert_or_assign(ScavengedSequenceRawHash, InOutCopyChunkDatas.size()); + InOutCopyChunkDatas.push_back(CopyChunkData{.ScavengeSourceIndex = ScavengedContentIndex, + .SourceSequenceIndex = ScavengeLocation.SequenceIndex, + .TargetChunkLocationPtrs = ChunkTargetPtrs, + .ChunkTargets = std::vector<CopyChunkData::ChunkTarget>{Target}}); + } + InOutChunkMatchingRemoteCount++; + InOutChunkMatchingRemoteByteCount += ScavengedChunkRawSize; + InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true; + InOutRemainingChunkCount--; + } + } + } + } +} + std::filesystem::path BuildsOperationUpdateFolder::FindDownloadedChunk(const IoHash& ChunkHash) { @@ -3225,6 +3188,8 @@ BuildsOperationUpdateFolder::WriteLooseChunk(const uint32_t RemoteChunkInd { FilteredDownloadedBytesPerSecond.Stop(); } + IoBufferFileReference FileRef; + bool EnableBacklog = Payload.GetFileReference(FileRef); AsyncWriteDownloadedChunk(m_Options.ZenFolderPath, RemoteChunkIndex, std::move(ChunkTargetPtrs), @@ -3234,7 +3199,8 @@ BuildsOperationUpdateFolder::WriteLooseChunk(const uint32_t RemoteChunkInd SequenceIndexChunksLeftToWriteCounters, WritePartsComplete, TotalPartWriteCount, - FilteredWrittenBytesPerSecond); + FilteredWrittenBytesPerSecond, + EnableBacklog); }); } }); @@ -3791,7 +3757,7 @@ BuildsOperationUpdateFolder::WriteLocalChunkToCache(CloneQueryInterface* C break; } const uint64_t NextChunkLength = m_RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex]; - if (ReadLength + NextChunkLength > m_Options.MaximumInMemoryPayloadSize) + if (ReadLength + NextChunkLength > BufferedOpenFile::BlockSize) { break; } @@ -4345,7 +4311,8 @@ BuildsOperationUpdateFolder::AsyncWriteDownloadedChunk(const std::filesystem::pa std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters, std::atomic<uint64_t>& WritePartsComplete, const uint64_t TotalPartWriteCount, - FilteredRate& FilteredWrittenBytesPerSecond) + FilteredRate& FilteredWrittenBytesPerSecond, + bool EnableBacklog) { ZEN_TRACE_CPU("AsyncWriteDownloadedChunk"); @@ -4462,7 +4429,8 @@ BuildsOperationUpdateFolder::AsyncWriteDownloadedChunk(const std::filesystem::pa } } } - }); + }, + EnableBacklog ? WorkerThreadPool::EMode::EnableBacklog : WorkerThreadPool::EMode::DisableBacklog); } void @@ -4604,10 +4572,7 @@ BuildsOperationUploadFolder::BuildsOperationUploadFolder(OperationLogOutput& WorkerThreadPool& IOWorkerPool, WorkerThreadPool& NetworkPool, const Oid& BuildId, - const Oid& BuildPartId, - const std::string_view BuildPartName, const std::filesystem::path& Path, - const std::filesystem::path& ManifestPath, bool CreateBuild, const CbObject& MetaData, const Options& Options) @@ -4618,10 +4583,7 @@ BuildsOperationUploadFolder::BuildsOperationUploadFolder(OperationLogOutput& , m_IOWorkerPool(IOWorkerPool) , m_NetworkPool(NetworkPool) , m_BuildId(BuildId) -, m_BuildPartId(BuildPartId) -, m_BuildPartName(BuildPartName) , m_Path(Path) -, m_ManifestPath(ManifestPath) , m_CreateBuild(CreateBuild) , m_MetaData(MetaData) , m_Options(Options) @@ -4633,739 +4595,259 @@ BuildsOperationUploadFolder::BuildsOperationUploadFolder(OperationLogOutput& } } -void -BuildsOperationUploadFolder::Execute() +BuildsOperationUploadFolder::PrepareBuildResult +BuildsOperationUploadFolder::PrepareBuild() { - ZEN_TRACE_CPU("BuildsOperationUploadFolder::Execute"); - try - { - enum class TaskSteps : uint32_t - { - PrepareBuild, - CalculateDelta, - GenerateBlocks, - BuildPartManifest, - UploadBuildPart, - UploadAttachments, - FinalizeBuild, - PutBuildPartStats, - Cleanup, - StepCount - }; - - auto EndProgress = - MakeGuard([&]() { m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::StepCount, (uint32_t)TaskSteps::StepCount); }); - - Stopwatch ProcessTimer; - - CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir); - CreateDirectories(m_Options.TempDir); - auto _ = MakeGuard([&]() { CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir); }); - - m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::PrepareBuild, (uint32_t)TaskSteps::StepCount); + ZEN_TRACE_CPU("PrepareBuild"); - std::uint64_t TotalRawSize = 0; - - CbObject ChunkerParameters; + PrepareBuildResult Result; + Result.PreferredMultipartChunkSize = m_Options.PreferredMultipartChunkSize; + Stopwatch Timer; + if (m_CreateBuild) + { + ZEN_TRACE_CPU("CreateBuild"); - struct PrepareBuildResult + Stopwatch PutBuildTimer; + CbObject PutBuildResult = m_Storage.BuildStorage->PutBuild(m_BuildId, m_MetaData); + Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs(); + if (auto ChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(); ChunkSize != 0) { - std::vector<ChunkBlockDescription> KnownBlocks; - uint64_t PreferredMultipartChunkSize = 0; - uint64_t PayloadSize = 0; - uint64_t PrepareBuildTimeMs = 0; - uint64_t FindBlocksTimeMs = 0; - uint64_t ElapsedTimeMs = 0; - }; - - std::future<PrepareBuildResult> PrepBuildResultFuture = m_NetworkPool.EnqueueTask( - std::packaged_task<PrepareBuildResult()>{[this] { - ZEN_TRACE_CPU("PrepareBuild"); - - PrepareBuildResult Result; - Result.PreferredMultipartChunkSize = m_Options.PreferredMultipartChunkSize; - Stopwatch Timer; - if (m_CreateBuild) - { - ZEN_TRACE_CPU("CreateBuild"); - - Stopwatch PutBuildTimer; - CbObject PutBuildResult = m_Storage.BuildStorage->PutBuild(m_BuildId, m_MetaData); - Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs(); - Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize); - Result.PayloadSize = m_MetaData.GetSize(); - } - else - { - ZEN_TRACE_CPU("PutBuild"); - Stopwatch GetBuildTimer; - CbObject Build = m_Storage.BuildStorage->GetBuild(m_BuildId); - Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs(); - Result.PayloadSize = Build.GetSize(); - if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) - { - Result.PreferredMultipartChunkSize = ChunkSize; - } - else if (m_Options.AllowMultiparts) - { - ZEN_OPERATION_LOG_WARN(m_LogOutput, - "PreferredMultipartChunkSize is unknown. Defaulting to '{}'", - NiceBytes(Result.PreferredMultipartChunkSize)); - } - } - - if (!m_Options.IgnoreExistingBlocks) - { - ZEN_TRACE_CPU("FindBlocks"); - Stopwatch KnownBlocksTimer; - CbObject BlockDescriptionList = m_Storage.BuildStorage->FindBlocks(m_BuildId, m_Options.FindBlockMaxCount); - if (BlockDescriptionList) - { - Result.KnownBlocks = ParseChunkBlockDescriptionList(BlockDescriptionList); - } - m_FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); - m_FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size(); - Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); - } - Result.ElapsedTimeMs = Timer.GetElapsedTimeMs(); - return Result; - }}, - WorkerThreadPool::EMode::EnableBacklog); + Result.PreferredMultipartChunkSize = ChunkSize; + } + Result.PayloadSize = m_MetaData.GetSize(); + } + else + { + ZEN_TRACE_CPU("PutBuild"); + Stopwatch GetBuildTimer; + CbObject Build = m_Storage.BuildStorage->GetBuild(m_BuildId); + Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs(); + Result.PayloadSize = Build.GetSize(); + if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) + { + Result.PreferredMultipartChunkSize = ChunkSize; + } + else if (m_Options.AllowMultiparts) + { + ZEN_OPERATION_LOG_WARN(m_LogOutput, + "PreferredMultipartChunkSize is unknown. Defaulting to '{}'", + NiceBytes(Result.PreferredMultipartChunkSize)); + } + } - ChunkedFolderContent LocalContent; + if (!m_Options.IgnoreExistingBlocks) + { + ZEN_TRACE_CPU("FindBlocks"); + Stopwatch KnownBlocksTimer; + CbObject BlockDescriptionList = m_Storage.BuildStorage->FindBlocks(m_BuildId, m_Options.FindBlockMaxCount); + if (BlockDescriptionList) + { + Result.KnownBlocks = ParseChunkBlockDescriptionList(BlockDescriptionList); + } + Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); + } + Result.ElapsedTimeMs = Timer.GetElapsedTimeMs(); + return Result; +} +std::vector<BuildsOperationUploadFolder::UploadPart> +BuildsOperationUploadFolder::ReadFolder() +{ + std::vector<UploadPart> UploadParts; + std::filesystem::path ExcludeManifestPath = m_Path / m_Options.ZenExcludeManifestName; + tsl::robin_set<std::string> ExcludeAssetPaths; + if (IsFile(ExcludeManifestPath)) + { + std::filesystem::path AbsoluteExcludeManifestPath = + MakeSafeAbsolutePath(ExcludeManifestPath.is_absolute() ? ExcludeManifestPath : m_Path / ExcludeManifestPath); + BuildManifest Manifest = ParseBuildManifest(AbsoluteExcludeManifestPath); + const std::vector<std::filesystem::path>& AssetPaths = Manifest.Parts.front().Files; + ExcludeAssetPaths.reserve(AssetPaths.size()); + for (const std::filesystem::path& AssetPath : AssetPaths) { - Stopwatch ScanTimer; - FolderContent Content; - if (m_ManifestPath.empty()) - { - std::filesystem::path ExcludeManifestPath = m_Path / m_Options.ZenExcludeManifestName; - tsl::robin_set<std::string> ExcludeAssetPaths; - if (IsFile(ExcludeManifestPath)) - { - std::vector<std::filesystem::path> AssetPaths = ParseManifest(m_Path, ExcludeManifestPath); - ExcludeAssetPaths.reserve(AssetPaths.size()); - for (const std::filesystem::path& AssetPath : AssetPaths) - { - ExcludeAssetPaths.insert(AssetPath.generic_string()); - } - } - Content = GetFolderContent( - m_LocalFolderScanStats, - m_Path, - [this](const std::string_view& RelativePath) { return IsAcceptedFolder(RelativePath); }, - [this, &ExcludeAssetPaths](const std::string_view& RelativePath, uint64_t Size, uint32_t Attributes) -> bool { - ZEN_UNUSED(Size, Attributes); - if (!IsAcceptedFile(RelativePath)) - { - return false; - } - if (ExcludeAssetPaths.contains(std::filesystem::path(RelativePath).generic_string())) - { - return false; - } - return true; - }, - m_IOWorkerPool, - m_LogOutput.GetProgressUpdateDelayMS(), - [&](bool, std::ptrdiff_t) { - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "Found {} files in '{}'...", - m_LocalFolderScanStats.AcceptedFileCount.load(), - m_Path); - }, - m_AbortFlag); - } - else - { - Stopwatch ManifestParseTimer; - std::vector<std::filesystem::path> AssetPaths = ParseManifest(m_Path, m_ManifestPath); - for (const std::filesystem::path& AssetPath : AssetPaths) - { - Content.Paths.push_back(AssetPath); - const std::filesystem::path AssetFilePath = (m_Path / AssetPath).make_preferred(); - Content.RawSizes.push_back(FileSizeFromPath(AssetFilePath)); -#if ZEN_PLATFORM_WINDOWS - Content.Attributes.push_back(GetFileAttributesFromPath(AssetFilePath)); -#endif // ZEN_PLATFORM_WINDOWS -#if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX - Content.Attributes.push_back(GetFileMode(AssetFilePath)); -#endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX - m_LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back(); - m_LocalFolderScanStats.AcceptedFileCount++; - } - if (m_ManifestPath.is_relative()) - { - Content.Paths.push_back(m_ManifestPath); - const std::filesystem::path ManifestFilePath = (m_Path / m_ManifestPath).make_preferred(); - Content.RawSizes.push_back(FileSizeFromPath(ManifestFilePath)); -#if ZEN_PLATFORM_WINDOWS - Content.Attributes.push_back(GetFileAttributesFromPath(ManifestFilePath)); -#endif // ZEN_PLATFORM_WINDOWS -#if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX - Content.Attributes.push_back(GetFileMode(ManifestFilePath)); -#endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX - - m_LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back(); - m_LocalFolderScanStats.AcceptedFileCount++; - } - m_LocalFolderScanStats.FoundFileByteCount.store(m_LocalFolderScanStats.AcceptedFileByteCount); - m_LocalFolderScanStats.FoundFileCount.store(m_LocalFolderScanStats.AcceptedFileCount); - m_LocalFolderScanStats.ElapsedWallTimeUS = ManifestParseTimer.GetElapsedTimeUs(); - } + ExcludeAssetPaths.insert(AssetPath.generic_string()); + } + } - std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); - { - CbObjectWriter ChunkParametersWriter; - ChunkParametersWriter.AddString("name"sv, ChunkController->GetName()); - ChunkParametersWriter.AddObject("parameters"sv, ChunkController->GetParameters()); - ChunkerParameters = ChunkParametersWriter.Save(); - } + UploadParts.resize(1); - TotalRawSize = std::accumulate(Content.RawSizes.begin(), Content.RawSizes.end(), std::uint64_t(0)); + UploadPart& Part = UploadParts.front(); + GetFolderContentStatistics& LocalFolderScanStats = Part.LocalFolderScanStats; + Part.Content = GetFolderContent( + Part.LocalFolderScanStats, + m_Path, + [this](const std::string_view& RelativePath) { return IsAcceptedFolder(RelativePath); }, + [this, &ExcludeAssetPaths](const std::string_view& RelativePath, uint64_t Size, uint32_t Attributes) -> bool { + ZEN_UNUSED(Size, Attributes); + if (!IsAcceptedFile(RelativePath)) { - std::unique_ptr<OperationLogOutput::ProgressBar> ProgressBarPtr(m_LogOutput.CreateProgressBar("Scan Folder")); - OperationLogOutput::ProgressBar& Progress(*ProgressBarPtr); - - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - LocalContent = ChunkFolderContent( - m_ChunkingStats, - m_IOWorkerPool, - m_Path, - Content, - *ChunkController, - m_LogOutput.GetProgressUpdateDelayMS(), - [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { - FilteredBytesHashed.Update(m_ChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - m_ChunkingStats.FilesProcessed.load(), - Content.Paths.size(), - NiceBytes(m_ChunkingStats.BytesHashed.load()), - NiceBytes(TotalRawSize), - NiceNum(FilteredBytesHashed.GetCurrent()), - m_ChunkingStats.UniqueChunksFound.load(), - NiceBytes(m_ChunkingStats.UniqueBytesFound.load())); - Progress.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = TotalRawSize, - .RemainingCount = TotalRawSize - m_ChunkingStats.BytesHashed.load(), - .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, - false); - }, - m_AbortFlag, - m_PauseFlag); - FilteredBytesHashed.Stop(); - Progress.Finish(); - if (m_AbortFlag) - { - return; - } + return false; } - - if (!m_Options.IsQuiet) + if (ExcludeAssetPaths.contains(std::filesystem::path(RelativePath).generic_string())) { - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", - LocalContent.Paths.size(), - NiceBytes(TotalRawSize), - m_ChunkingStats.UniqueChunksFound.load(), - NiceBytes(m_ChunkingStats.UniqueBytesFound.load()), - m_Path, - NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()), - NiceNum(GetBytesPerSecond(m_ChunkingStats.ElapsedWallTimeUS, m_ChunkingStats.BytesHashed))); + return false; } - } - - const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent); - - std::vector<size_t> ReuseBlockIndexes; - std::vector<uint32_t> NewBlockChunkIndexes; + return true; + }, + m_IOWorkerPool, + m_LogOutput.GetProgressUpdateDelayMS(), + [&](bool, std::ptrdiff_t) { + ZEN_OPERATION_LOG_INFO(m_LogOutput, "Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), m_Path); + }, + m_AbortFlag); + Part.TotalRawSize = std::accumulate(Part.Content.RawSizes.begin(), Part.Content.RawSizes.end(), std::uint64_t(0)); + + return UploadParts; +} - PrepareBuildResult PrepBuildResult = PrepBuildResultFuture.get(); +std::vector<BuildsOperationUploadFolder::UploadPart> +BuildsOperationUploadFolder::ReadManifestParts(const std::filesystem::path& ManifestPath) +{ + std::vector<UploadPart> UploadParts; + Stopwatch ManifestParseTimer; + std::filesystem::path AbsoluteManifestPath = MakeSafeAbsolutePath(ManifestPath.is_absolute() ? ManifestPath : m_Path / ManifestPath); + BuildManifest Manifest = ParseBuildManifest(AbsoluteManifestPath); + if (Manifest.Parts.empty()) + { + throw std::runtime_error(fmt::format("Manifest file at '{}' is invalid", ManifestPath)); + } - if (!m_Options.IsQuiet) + UploadParts.resize(Manifest.Parts.size()); + for (size_t PartIndex = 0; PartIndex < Manifest.Parts.size(); PartIndex++) + { + BuildManifest::Part& PartManifest = Manifest.Parts[PartIndex]; + if (ManifestPath.is_relative()) { - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "Build prepare took {}. {} took {}, payload size {}{}", - NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs), - m_CreateBuild ? "PutBuild" : "GetBuild", - NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs), - NiceBytes(PrepBuildResult.PayloadSize), - m_Options.IgnoreExistingBlocks ? "" - : fmt::format(". Found {} blocks in {}", - PrepBuildResult.KnownBlocks.size(), - NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs))); + PartManifest.Files.push_back(ManifestPath); } - m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::CalculateDelta, (uint32_t)TaskSteps::StepCount); + UploadPart& Part = UploadParts[PartIndex]; + FolderContent& Content = Part.Content; - const std::uint64_t LargeAttachmentSize = - m_Options.AllowMultiparts ? PrepBuildResult.PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; + GetFolderContentStatistics& LocalFolderScanStats = Part.LocalFolderScanStats; - Stopwatch BlockArrangeTimer; + const std::vector<std::filesystem::path>& AssetPaths = PartManifest.Files; + Content = GetValidFolderContent( + m_IOWorkerPool, + LocalFolderScanStats, + m_Path, + AssetPaths, + [](uint64_t PathCount, uint64_t CompletedPathCount) { ZEN_UNUSED(PathCount, CompletedPathCount); }, + 1000, + m_AbortFlag, + m_PauseFlag); - std::vector<std::uint32_t> LooseChunkIndexes; + if (Content.Paths.size() != AssetPaths.size()) { - bool EnableBlocks = true; - std::vector<std::uint32_t> BlockChunkIndexes; - for (uint32_t ChunkIndex = 0; ChunkIndex < LocalContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++) - { - const uint64_t ChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - if (!EnableBlocks || ChunkRawSize == 0 || ChunkRawSize > m_Options.BlockParameters.MaxChunkEmbedSize) - { - LooseChunkIndexes.push_back(ChunkIndex); - m_LooseChunksStats.ChunkByteCount += ChunkRawSize; - } - else - { - BlockChunkIndexes.push_back(ChunkIndex); - m_FindBlocksStats.PotentialChunkByteCount += ChunkRawSize; - } - } - m_FindBlocksStats.PotentialChunkCount = BlockChunkIndexes.size(); - m_LooseChunksStats.ChunkCount = LooseChunkIndexes.size(); - - if (m_Options.IgnoreExistingBlocks) + const tsl::robin_set<std::filesystem::path> FoundPaths(Content.Paths.begin(), Content.Paths.end()); + ExtendableStringBuilder<1024> SB; + for (const std::filesystem::path& AssetPath : AssetPaths) { - if (!m_Options.IsQuiet) + if (!FoundPaths.contains(AssetPath)) { - ZEN_OPERATION_LOG_INFO(m_LogOutput, "Ignoring any existing blocks in store"); - } - NewBlockChunkIndexes = std::move(BlockChunkIndexes); - } - else - { - ReuseBlockIndexes = FindReuseBlocks(m_LogOutput, - m_Options.BlockReuseMinPercentLimit, - m_Options.IsVerbose, - m_ReuseBlocksStats, - PrepBuildResult.KnownBlocks, - LocalContent.ChunkedContent.ChunkHashes, - BlockChunkIndexes, - NewBlockChunkIndexes); - m_FindBlocksStats.AcceptedBlockCount = ReuseBlockIndexes.size(); - - for (const ChunkBlockDescription& Description : PrepBuildResult.KnownBlocks) - { - for (uint32_t ChunkRawLength : Description.ChunkRawLengths) - { - m_FindBlocksStats.FoundBlockByteCount += ChunkRawLength; - } - m_FindBlocksStats.FoundBlockChunkCount += Description.ChunkRawHashes.size(); + SB << "\n " << AssetPath.generic_string(); } } + throw std::runtime_error( + fmt::format("Manifest file at '{}' references files that does not exist{}", ManifestPath, SB.ToView())); } - std::vector<std::vector<uint32_t>> NewBlockChunks; - ArrangeChunksIntoBlocks(LocalContent, LocalLookup, NewBlockChunkIndexes, NewBlockChunks); + Part.PartId = PartManifest.PartId; + Part.PartName = PartManifest.PartName; + Part.TotalRawSize = std::accumulate(Part.Content.RawSizes.begin(), Part.Content.RawSizes.end(), std::uint64_t(0)); + } - m_FindBlocksStats.NewBlocksCount = NewBlockChunks.size(); - for (uint32_t ChunkIndex : NewBlockChunkIndexes) - { - m_FindBlocksStats.NewBlocksChunkByteCount += LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - } - m_FindBlocksStats.NewBlocksChunkCount = NewBlockChunkIndexes.size(); + return UploadParts; +} - const double AcceptedByteCountPercent = - m_FindBlocksStats.PotentialChunkByteCount > 0 - ? (100.0 * m_ReuseBlocksStats.AcceptedRawByteCount / m_FindBlocksStats.PotentialChunkByteCount) - : 0.0; +std::vector<std::pair<Oid, std::string>> +BuildsOperationUploadFolder::Execute(const Oid& BuildPartId, + const std::string_view BuildPartName, + const std::filesystem::path& ManifestPath, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache) +{ + ZEN_TRACE_CPU("BuildsOperationUploadFolder::Execute"); + try + { + Stopwatch ReadPartsTimer; + std::vector<UploadPart> UploadParts = ManifestPath.empty() ? ReadFolder() : ReadManifestParts(ManifestPath); - const double AcceptedReduntantByteCountPercent = - m_ReuseBlocksStats.AcceptedByteCount > 0 - ? (100.0 * m_ReuseBlocksStats.AcceptedReduntantByteCount) / - (m_ReuseBlocksStats.AcceptedByteCount + m_ReuseBlocksStats.AcceptedReduntantByteCount) - : 0.0; - if (!m_Options.IsQuiet) + for (UploadPart& Part : UploadParts) { - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n" - " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n" - " Accepting {} ({}) redundant chunks ({:.1f}%)\n" - " Rejected {} ({}) chunks in {} blocks\n" - " Arranged {} ({}) chunks in {} new blocks\n" - " Keeping {} ({}) chunks as loose chunks\n" - " Discovery completed in {}", - m_FindBlocksStats.FoundBlockChunkCount, - m_FindBlocksStats.FoundBlockCount, - NiceBytes(m_FindBlocksStats.FoundBlockByteCount), - NiceTimeSpanMs(m_FindBlocksStats.FindBlockTimeMS), - - m_ReuseBlocksStats.AcceptedChunkCount, - NiceBytes(m_ReuseBlocksStats.AcceptedRawByteCount), - m_FindBlocksStats.AcceptedBlockCount, - AcceptedByteCountPercent, - - m_ReuseBlocksStats.AcceptedReduntantChunkCount, - NiceBytes(m_ReuseBlocksStats.AcceptedReduntantByteCount), - AcceptedReduntantByteCountPercent, - - m_ReuseBlocksStats.RejectedChunkCount, - NiceBytes(m_ReuseBlocksStats.RejectedByteCount), - m_ReuseBlocksStats.RejectedBlockCount, - - m_FindBlocksStats.NewBlocksChunkCount, - NiceBytes(m_FindBlocksStats.NewBlocksChunkByteCount), - m_FindBlocksStats.NewBlocksCount, - - m_LooseChunksStats.ChunkCount, - NiceBytes(m_LooseChunksStats.ChunkByteCount), - - NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs())); - } - - m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::GenerateBlocks, (uint32_t)TaskSteps::StepCount); - GeneratedBlocks NewBlocks; - - if (!NewBlockChunks.empty()) - { - Stopwatch GenerateBuildBlocksTimer; - auto __ = MakeGuard([&]() { - uint64_t BlockGenerateTimeUs = GenerateBuildBlocksTimer.GetElapsedTimeUs(); - if (!m_Options.IsQuiet) - { - ZEN_OPERATION_LOG_INFO( - m_LogOutput, - "Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.", - m_GenerateBlocksStats.GeneratedBlockCount.load(), - NiceBytes(m_GenerateBlocksStats.GeneratedBlockByteCount), - m_UploadStats.BlockCount.load(), - NiceBytes(m_UploadStats.BlocksBytes.load()), - NiceTimeSpanMs(BlockGenerateTimeUs / 1000), - NiceNum(GetBytesPerSecond(m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, - m_GenerateBlocksStats.GeneratedBlockByteCount)), - NiceNum(GetBytesPerSecond(m_UploadStats.ElapsedWallTimeUS, m_UploadStats.BlocksBytes * 8))); - } - }); - GenerateBuildBlocks(LocalContent, LocalLookup, NewBlockChunks, NewBlocks); - } - - m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::BuildPartManifest, (uint32_t)TaskSteps::StepCount); - - CbObject PartManifest; - { - CbObjectWriter PartManifestWriter; - Stopwatch ManifestGenerationTimer; - auto __ = MakeGuard([&]() { - if (!m_Options.IsQuiet) - { - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "Generated build part manifest in {} ({})", - NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()), - NiceBytes(PartManifestWriter.GetSaveSize())); - } - }); - PartManifestWriter.AddObject("chunker"sv, ChunkerParameters); - - std::vector<IoHash> AllChunkBlockHashes; - std::vector<ChunkBlockDescription> AllChunkBlockDescriptions; - AllChunkBlockHashes.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size()); - AllChunkBlockDescriptions.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size()); - for (size_t ReuseBlockIndex : ReuseBlockIndexes) - { - AllChunkBlockDescriptions.push_back(PrepBuildResult.KnownBlocks[ReuseBlockIndex]); - AllChunkBlockHashes.push_back(PrepBuildResult.KnownBlocks[ReuseBlockIndex].BlockHash); - } - AllChunkBlockDescriptions.insert(AllChunkBlockDescriptions.end(), - NewBlocks.BlockDescriptions.begin(), - NewBlocks.BlockDescriptions.end()); - for (const ChunkBlockDescription& BlockDescription : NewBlocks.BlockDescriptions) - { - AllChunkBlockHashes.push_back(BlockDescription.BlockHash); - } - std::vector<IoHash> AbsoluteChunkHashes; - if (m_Options.DoExtraContentValidation) + if (Part.PartId == Oid::Zero) { - tsl::robin_map<IoHash, size_t, IoHash::Hasher> ChunkHashToAbsoluteChunkIndex; - AbsoluteChunkHashes.reserve(LocalContent.ChunkedContent.ChunkHashes.size()); - for (uint32_t ChunkIndex : LooseChunkIndexes) - { - ChunkHashToAbsoluteChunkIndex.insert({LocalContent.ChunkedContent.ChunkHashes[ChunkIndex], AbsoluteChunkHashes.size()}); - AbsoluteChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); - } - for (const ChunkBlockDescription& Block : AllChunkBlockDescriptions) - { - for (const IoHash& ChunkHash : Block.ChunkRawHashes) - { - ChunkHashToAbsoluteChunkIndex.insert({ChunkHash, AbsoluteChunkHashes.size()}); - AbsoluteChunkHashes.push_back(ChunkHash); - } - } - for (const IoHash& ChunkHash : LocalContent.ChunkedContent.ChunkHashes) + if (UploadParts.size() != 1) { - ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(ChunkHash)] == ChunkHash); - ZEN_ASSERT(LocalContent.ChunkedContent.ChunkHashes[LocalLookup.ChunkHashToChunkIndex.at(ChunkHash)] == ChunkHash); + throw std::runtime_error(fmt::format("Multi part upload manifest '{}' must contains build part id", ManifestPath)); } - for (const uint32_t ChunkIndex : LocalContent.ChunkedContent.ChunkOrders) + + if (BuildPartId == Oid::Zero) { - ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex])] == - LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); - ZEN_ASSERT(LocalLookup.ChunkHashToChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]) == ChunkIndex); + Part.PartId = Oid::NewOid(); } - } - std::vector<uint32_t> AbsoluteChunkOrders = CalculateAbsoluteChunkOrders(LocalContent.ChunkedContent.ChunkHashes, - LocalContent.ChunkedContent.ChunkOrders, - LocalLookup.ChunkHashToChunkIndex, - LooseChunkIndexes, - AllChunkBlockDescriptions); - - if (m_Options.DoExtraContentValidation) - { - for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); ChunkOrderIndex++) + else { - uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndex]; - uint32_t AbsoluteChunkIndex = AbsoluteChunkOrders[ChunkOrderIndex]; - const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; - const IoHash& AbsoluteChunkHash = AbsoluteChunkHashes[AbsoluteChunkIndex]; - ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash); + Part.PartId = BuildPartId; } } - - WriteBuildContentToCompactBinary(PartManifestWriter, - LocalContent.Platform, - LocalContent.Paths, - LocalContent.RawHashes, - LocalContent.RawSizes, - LocalContent.Attributes, - LocalContent.ChunkedContent.SequenceRawHashes, - LocalContent.ChunkedContent.ChunkCounts, - LocalContent.ChunkedContent.ChunkHashes, - LocalContent.ChunkedContent.ChunkRawSizes, - AbsoluteChunkOrders, - LooseChunkIndexes, - AllChunkBlockHashes); - - if (m_Options.DoExtraContentValidation) + if (Part.PartName.empty()) { - ChunkedFolderContent VerifyFolderContent; - - std::vector<uint32_t> OutAbsoluteChunkOrders; - std::vector<IoHash> OutLooseChunkHashes; - std::vector<uint64_t> OutLooseChunkRawSizes; - std::vector<IoHash> OutBlockRawHashes; - ReadBuildContentFromCompactBinary(PartManifestWriter.Save(), - VerifyFolderContent.Platform, - VerifyFolderContent.Paths, - VerifyFolderContent.RawHashes, - VerifyFolderContent.RawSizes, - VerifyFolderContent.Attributes, - VerifyFolderContent.ChunkedContent.SequenceRawHashes, - VerifyFolderContent.ChunkedContent.ChunkCounts, - OutAbsoluteChunkOrders, - OutLooseChunkHashes, - OutLooseChunkRawSizes, - OutBlockRawHashes); - ZEN_ASSERT(OutBlockRawHashes == AllChunkBlockHashes); - - for (uint32_t OrderIndex = 0; OrderIndex < OutAbsoluteChunkOrders.size(); OrderIndex++) + if (UploadParts.size() != 1) { - uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex]; - const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; - - uint32_t VerifyChunkIndex = OutAbsoluteChunkOrders[OrderIndex]; - const IoHash VerifyChunkHash = AbsoluteChunkHashes[VerifyChunkIndex]; - - ZEN_ASSERT(LocalChunkHash == VerifyChunkHash); + throw std::runtime_error(fmt::format("Multi part upload manifest '{}' must contains build part name", ManifestPath)); } - - CalculateLocalChunkOrders(OutAbsoluteChunkOrders, - OutLooseChunkHashes, - OutLooseChunkRawSizes, - AllChunkBlockDescriptions, - VerifyFolderContent.ChunkedContent.ChunkHashes, - VerifyFolderContent.ChunkedContent.ChunkRawSizes, - VerifyFolderContent.ChunkedContent.ChunkOrders, - m_Options.DoExtraContentValidation); - - ZEN_ASSERT(LocalContent.Paths == VerifyFolderContent.Paths); - ZEN_ASSERT(LocalContent.RawHashes == VerifyFolderContent.RawHashes); - ZEN_ASSERT(LocalContent.RawSizes == VerifyFolderContent.RawSizes); - ZEN_ASSERT(LocalContent.Attributes == VerifyFolderContent.Attributes); - ZEN_ASSERT(LocalContent.ChunkedContent.SequenceRawHashes == VerifyFolderContent.ChunkedContent.SequenceRawHashes); - ZEN_ASSERT(LocalContent.ChunkedContent.ChunkCounts == VerifyFolderContent.ChunkedContent.ChunkCounts); - - for (uint32_t OrderIndex = 0; OrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); OrderIndex++) + if (BuildPartName.empty()) { - uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex]; - const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; - uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex]; - - uint32_t VerifyChunkIndex = VerifyFolderContent.ChunkedContent.ChunkOrders[OrderIndex]; - const IoHash VerifyChunkHash = VerifyFolderContent.ChunkedContent.ChunkHashes[VerifyChunkIndex]; - uint64_t VerifyChunkRawSize = VerifyFolderContent.ChunkedContent.ChunkRawSizes[VerifyChunkIndex]; - - ZEN_ASSERT(LocalChunkHash == VerifyChunkHash); - ZEN_ASSERT(LocalChunkRawSize == VerifyChunkRawSize); + throw std::runtime_error("Build part name must be set"); } + Part.PartName = std::string(BuildPartName); } - PartManifest = PartManifestWriter.Save(); } - m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::UploadBuildPart, (uint32_t)TaskSteps::StepCount); - - Stopwatch PutBuildPartResultTimer; - std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult = - m_Storage.BuildStorage->PutBuildPart(m_BuildId, m_BuildPartId, m_BuildPartName, PartManifest); if (!m_Options.IsQuiet) { ZEN_OPERATION_LOG_INFO(m_LogOutput, - "PutBuildPart took {}, payload size {}. {} attachments are needed.", - NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()), - NiceBytes(PartManifest.GetSize()), - PutBuildPartResult.second.size()); + "Reading {} parts took {}", + UploadParts.size(), + NiceTimeSpanMs(ReadPartsTimer.GetElapsedTimeMs())); } - IoHash PartHash = PutBuildPartResult.first; - auto UploadAttachments = [this, &LocalContent, &LocalLookup, &NewBlockChunks, &NewBlocks, &LooseChunkIndexes, &LargeAttachmentSize]( - std::span<IoHash> RawHashes, - std::vector<IoHash>& OutUnknownChunks) { - if (!m_AbortFlag) - { - UploadStatistics TempUploadStats; - LooseChunksStatistics TempLooseChunksStats; + const uint32_t PartsUploadStepCount = gsl::narrow<uint32_t>(uint32_t(PartTaskSteps::StepCount) * UploadParts.size()); - Stopwatch TempUploadTimer; - auto __ = MakeGuard([&]() { - if (!m_Options.IsQuiet) - { - uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs(); - ZEN_OPERATION_LOG_INFO( - m_LogOutput, - "Uploaded {} ({}) blocks. " - "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. " - "Transferred {} ({}bits/s) in {}", - TempUploadStats.BlockCount.load(), - NiceBytes(TempUploadStats.BlocksBytes), + const uint32_t PrepareBuildStep = 0; + const uint32_t UploadPartsStep = 1; + const uint32_t FinalizeBuildStep = UploadPartsStep + PartsUploadStepCount; + const uint32_t CleanupStep = FinalizeBuildStep + 1; + const uint32_t StepCount = CleanupStep + 1; - TempLooseChunksStats.CompressedChunkCount.load(), - NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()), - NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, - TempLooseChunksStats.ChunkByteCount)), - TempUploadStats.ChunkCount.load(), - NiceBytes(TempUploadStats.ChunksBytes), + auto EndProgress = MakeGuard([&]() { m_LogOutput.SetLogOperationProgress(StepCount, StepCount); }); - NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes), - NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)), - NiceTimeSpanMs(TempChunkUploadTimeUs / 1000)); - } - }); - UploadPartBlobs(LocalContent, - LocalLookup, - RawHashes, - NewBlockChunks, - NewBlocks, - LooseChunkIndexes, - LargeAttachmentSize, - TempUploadStats, - TempLooseChunksStats, - OutUnknownChunks); - m_UploadStats += TempUploadStats; - m_LooseChunksStats += TempLooseChunksStats; - } - }; - - m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::UploadAttachments, (uint32_t)TaskSteps::StepCount); - - std::vector<IoHash> UnknownChunks; - if (m_Options.IgnoreExistingBlocks) - { - if (m_Options.IsVerbose) - { - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "PutBuildPart uploading all attachments, needs are: {}", - FormatArray<IoHash>(PutBuildPartResult.second, "\n "sv)); - } - - std::vector<IoHash> ForceUploadChunkHashes; - ForceUploadChunkHashes.reserve(LooseChunkIndexes.size()); - - for (uint32_t ChunkIndex : LooseChunkIndexes) - { - ForceUploadChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); - } + Stopwatch ProcessTimer; - for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockHeaders.size(); BlockIndex++) - { - if (NewBlocks.BlockHeaders[BlockIndex]) - { - // Block was not uploaded during generation - ForceUploadChunkHashes.push_back(NewBlocks.BlockDescriptions[BlockIndex].BlockHash); - } - } - UploadAttachments(ForceUploadChunkHashes, UnknownChunks); - } - else if (!PutBuildPartResult.second.empty()) - { - if (m_Options.IsVerbose) - { - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "PutBuildPart needs attachments: {}", - FormatArray<IoHash>(PutBuildPartResult.second, "\n "sv)); - } - UploadAttachments(PutBuildPartResult.second, UnknownChunks); - } + CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir); + CreateDirectories(m_Options.TempDir); + auto _ = MakeGuard([&]() { CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir); }); - auto BuildUnkownChunksResponse = [](const std::vector<IoHash>& UnknownChunks, bool WillRetry) { - return fmt::format( - "The following build blobs was reported as needed for upload but was reported as existing at the start of the " - "operation.{}{}", - WillRetry ? " Treating this as a transient inconsistency issue and will attempt to retry finalization."sv : ""sv, - FormatArray<IoHash>(UnknownChunks, "\n "sv)); - }; + m_LogOutput.SetLogOperationProgress(PrepareBuildStep, StepCount); - if (!UnknownChunks.empty()) - { - ZEN_OPERATION_LOG_WARN(m_LogOutput, "{}", BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ true)); - } + m_PrepBuildResultFuture = m_NetworkPool.EnqueueTask(std::packaged_task<PrepareBuildResult()>{[this] { return PrepareBuild(); }}, + WorkerThreadPool::EMode::EnableBacklog); - uint32_t FinalizeBuildPartRetryCount = 5; - while (!m_AbortFlag && (FinalizeBuildPartRetryCount--) > 0) + for (uint32_t PartIndex = 0; PartIndex < UploadParts.size(); PartIndex++) { - Stopwatch FinalizeBuildPartTimer; - std::vector<IoHash> Needs = m_Storage.BuildStorage->FinalizeBuildPart(m_BuildId, m_BuildPartId, PartHash); - if (!m_Options.IsQuiet) - { - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "FinalizeBuildPart took {}. {} attachments are missing.", - NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()), - Needs.size()); - } - if (Needs.empty()) - { - break; - } - if (m_Options.IsVerbose) - { - ZEN_OPERATION_LOG_INFO(m_LogOutput, "FinalizeBuildPart needs attachments: {}", FormatArray<IoHash>(Needs, "\n "sv)); - } + const uint32_t PartStepOffset = UploadPartsStep + (PartIndex * uint32_t(PartTaskSteps::StepCount)); - std::vector<IoHash> RetryUnknownChunks; - UploadAttachments(Needs, RetryUnknownChunks); - if (RetryUnknownChunks == UnknownChunks) - { - if (FinalizeBuildPartRetryCount > 0) - { - // Back off a bit - Sleep(1000); - } - } - else + const UploadPart& Part = UploadParts[PartIndex]; + UploadBuildPart(ChunkController, ChunkCache, PartIndex, Part, PartStepOffset, StepCount); + if (m_AbortFlag) { - UnknownChunks = RetryUnknownChunks; - ZEN_OPERATION_LOG_WARN(m_LogOutput, - "{}", - BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ FinalizeBuildPartRetryCount != 0)); + return {}; } } - if (!UnknownChunks.empty()) - { - throw std::runtime_error(BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ false)); - } - - m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::FinalizeBuild, (uint32_t)TaskSteps::StepCount); + m_LogOutput.SetLogOperationProgress(FinalizeBuildStep, StepCount); if (m_CreateBuild && !m_AbortFlag) { @@ -5377,79 +4859,15 @@ BuildsOperationUploadFolder::Execute() } } - if (!NewBlocks.BlockDescriptions.empty() && !m_AbortFlag) - { - uint64_t UploadBlockMetadataCount = 0; - Stopwatch UploadBlockMetadataTimer; + m_LogOutput.SetLogOperationProgress(CleanupStep, StepCount); - uint32_t FailedMetadataUploadCount = 1; - int32_t MetadataUploadRetryCount = 3; - while ((MetadataUploadRetryCount-- > 0) && (FailedMetadataUploadCount > 0)) - { - FailedMetadataUploadCount = 0; - for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++) - { - if (m_AbortFlag) - { - break; - } - const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; - if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex]) - { - const CbObject BlockMetaData = - BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); - if (m_Storage.BuildCacheStorage && m_Options.PopulateCache) - { - m_Storage.BuildCacheStorage->PutBlobMetadatas(m_BuildId, - std::vector<IoHash>({BlockHash}), - std::vector<CbObject>({BlockMetaData})); - } - bool MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData); - if (MetadataSucceeded) - { - m_UploadStats.BlocksBytes += BlockMetaData.GetSize(); - NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; - UploadBlockMetadataCount++; - } - else - { - FailedMetadataUploadCount++; - } - } - } - } - if (UploadBlockMetadataCount > 0) - { - uint64_t ElapsedUS = UploadBlockMetadataTimer.GetElapsedTimeUs(); - m_UploadStats.ElapsedWallTimeUS += ElapsedUS; - if (!m_Options.IsQuiet) - { - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "Uploaded metadata for {} blocks in {}", - UploadBlockMetadataCount, - NiceTimeSpanMs(ElapsedUS / 1000)); - } - } + std::vector<std::pair<Oid, std::string>> Result; + Result.reserve(UploadParts.size()); + for (UploadPart& Part : UploadParts) + { + Result.push_back(std::make_pair(Part.PartId, Part.PartName)); } - - m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::PutBuildPartStats, (uint32_t)TaskSteps::StepCount); - - m_Storage.BuildStorage->PutBuildPartStats( - m_BuildId, - m_BuildPartId, - {{"totalSize", double(m_LocalFolderScanStats.FoundFileByteCount.load())}, - {"reusedRatio", AcceptedByteCountPercent / 100.0}, - {"reusedBlockCount", double(m_FindBlocksStats.AcceptedBlockCount)}, - {"reusedBlockByteCount", double(m_ReuseBlocksStats.AcceptedRawByteCount)}, - {"newBlockCount", double(m_FindBlocksStats.NewBlocksCount)}, - {"newBlockByteCount", double(m_FindBlocksStats.NewBlocksChunkByteCount)}, - {"uploadedCount", double(m_UploadStats.BlockCount.load() + m_UploadStats.ChunkCount.load())}, - {"uploadedByteCount", double(m_UploadStats.BlocksBytes.load() + m_UploadStats.ChunksBytes.load())}, - {"uploadedBytesPerSec", - double(GetBytesPerSecond(m_UploadStats.ElapsedWallTimeUS, m_UploadStats.ChunksBytes + m_UploadStats.BlocksBytes))}, - {"elapsedTimeSec", double(ProcessTimer.GetElapsedTimeMs() / 1000.0)}}); - - m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::Cleanup, (uint32_t)TaskSteps::StepCount); + return Result; } catch (const std::exception&) { @@ -5458,43 +4876,6 @@ BuildsOperationUploadFolder::Execute() } } -std::vector<std::filesystem::path> -BuildsOperationUploadFolder::ParseManifest(const std::filesystem::path& Path, const std::filesystem::path& ManifestPath) -{ - std::vector<std::filesystem::path> AssetPaths; - std::filesystem::path AbsoluteManifestPath = MakeSafeAbsolutePath(ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath); - IoBuffer ManifestContent = ReadFile(AbsoluteManifestPath).Flatten(); - std::string_view ManifestString((const char*)ManifestContent.GetView().GetData(), ManifestContent.GetSize()); - std::string_view::size_type Offset = 0; - while (Offset < ManifestContent.GetSize()) - { - size_t PathBreakOffset = ManifestString.find_first_of("\t\r\n", Offset); - if (PathBreakOffset == std::string_view::npos) - { - PathBreakOffset = ManifestContent.GetSize(); - } - std::string_view AssetPath = ManifestString.substr(Offset, PathBreakOffset - Offset); - if (!AssetPath.empty()) - { - AssetPaths.emplace_back(std::filesystem::path(AssetPath)); - } - Offset = PathBreakOffset; - size_t EolOffset = ManifestString.find_first_of("\r\n", Offset); - if (EolOffset == std::string_view::npos) - { - break; - } - Offset = EolOffset; - size_t LineBreakOffset = ManifestString.find_first_not_of("\t\r\n", Offset); - if (LineBreakOffset == std::string_view::npos) - { - break; - } - Offset = LineBreakOffset; - } - return AssetPaths; -} - bool BuildsOperationUploadFolder::IsAcceptedFolder(const std::string_view& RelativePath) const { @@ -5630,7 +5011,9 @@ void BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, const std::vector<std::vector<uint32_t>>& NewBlockChunks, - GeneratedBlocks& OutBlocks) + GeneratedBlocks& OutBlocks, + GenerateBlocksStatistics& GenerateBlocksStats, + UploadStatistics& UploadStats) { ZEN_TRACE_CPU("GenerateBuildBlocks"); const std::size_t NewBlockCount = NewBlockChunks.size(); @@ -5676,6 +5059,8 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& ChunksInBlock, &Lock, &OutBlocks, + &GenerateBlocksStats, + &UploadStats, &FilteredGeneratedBytesPerSecond, &QueuedPendingBlocksForUpload, &FilteredUploadedBytesPerSecond, @@ -5705,8 +5090,8 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& Writer.AddString("createdBy", "zen"); OutBlocks.BlockMetaDatas[BlockIndex] = Writer.Save(); } - m_GenerateBlocksStats.GeneratedBlockByteCount += OutBlocks.BlockSizes[BlockIndex]; - m_GenerateBlocksStats.GeneratedBlockCount++; + GenerateBlocksStats.GeneratedBlockByteCount += OutBlocks.BlockSizes[BlockIndex]; + GenerateBlocksStats.GeneratedBlockCount++; Lock.WithExclusiveLock([&]() { OutBlocks.BlockHashToBlockIndex.insert_or_assign(OutBlocks.BlockDescriptions[BlockIndex].BlockHash, BlockIndex); @@ -5718,7 +5103,7 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); } - if (m_GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) + if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) { FilteredGeneratedBytesPerSecond.Stop(); } @@ -5739,6 +5124,8 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& UploadBlocksPool, [this, NewBlockCount, + &GenerateBlocksStats, + &UploadStats, &FilteredUploadedBytesPerSecond, &QueuedPendingBlocksForUpload, &OutBlocks, @@ -5747,7 +5134,7 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& auto _ = MakeGuard([&QueuedPendingBlocksForUpload] { QueuedPendingBlocksForUpload--; }); if (!m_AbortFlag) { - if (m_GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) + if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) { ZEN_TRACE_CPU("GenerateBuildBlocks_Save"); @@ -5781,7 +5168,7 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& BlockHash, ZenContentType::kCompressedBinary, std::move(Payload).GetCompressed()); - m_UploadStats.BlocksBytes += CompressedBlockSize; + UploadStats.BlocksBytes += CompressedBlockSize; if (m_Options.IsVerbose) { @@ -5812,11 +5199,11 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& } OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; - m_UploadStats.BlocksBytes += BlockMetaData.GetSize(); + UploadStats.BlocksBytes += BlockMetaData.GetSize(); } - m_UploadStats.BlockCount++; - if (m_UploadStats.BlockCount == NewBlockCount) + UploadStats.BlockCount++; + if (UploadStats.BlockCount == NewBlockCount) { FilteredUploadedBytesPerSecond.Stop(); } @@ -5832,23 +5219,23 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& Work.Wait(m_LogOutput.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { ZEN_UNUSED(PendingWork); - FilteredGeneratedBytesPerSecond.Update(m_GenerateBlocksStats.GeneratedBlockByteCount.load()); - FilteredUploadedBytesPerSecond.Update(m_UploadStats.BlocksBytes.load()); + FilteredGeneratedBytesPerSecond.Update(GenerateBlocksStats.GeneratedBlockByteCount.load()); + FilteredUploadedBytesPerSecond.Update(UploadStats.BlocksBytes.load()); std::string Details = fmt::format("Generated {}/{} ({}, {}B/s). Uploaded {}/{} ({}, {}bits/s)", - m_GenerateBlocksStats.GeneratedBlockCount.load(), + GenerateBlocksStats.GeneratedBlockCount.load(), NewBlockCount, - NiceBytes(m_GenerateBlocksStats.GeneratedBlockByteCount.load()), + NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()), NiceNum(FilteredGeneratedBytesPerSecond.GetCurrent()), - m_UploadStats.BlockCount.load(), + UploadStats.BlockCount.load(), NewBlockCount, - NiceBytes(m_UploadStats.BlocksBytes.load()), + NiceBytes(UploadStats.BlocksBytes.load()), NiceNum(FilteredUploadedBytesPerSecond.GetCurrent() * 8)); Progress.UpdateState({.Task = "Generating blocks", .Details = Details, .TotalCount = gsl::narrow<uint64_t>(NewBlockCount), - .RemainingCount = gsl::narrow<uint64_t>(NewBlockCount - m_GenerateBlocksStats.GeneratedBlockCount.load()), + .RemainingCount = gsl::narrow<uint64_t>(NewBlockCount - GenerateBlocksStats.GeneratedBlockCount.load()), .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }); @@ -5857,8 +5244,8 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& Progress.Finish(); - m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGeneratedBytesPerSecond.GetElapsedTimeUS(); - m_UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTimeUS(); + GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGeneratedBytesPerSecond.GetElapsedTimeUS(); + UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTimeUS(); } } @@ -6035,6 +5422,671 @@ BuildsOperationUploadFolder::RebuildBlock(const ChunkedFolderContent& Content, }; void +BuildsOperationUploadFolder::UploadBuildPart(ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + uint32_t PartIndex, + const UploadPart& Part, + uint32_t PartStepOffset, + uint32_t StepCount) +{ + Stopwatch UploadTimer; + + ChunkingStatistics ChunkingStats; + FindBlocksStatistics FindBlocksStats; + ReuseBlocksStatistics ReuseBlocksStats; + UploadStatistics UploadStats; + GenerateBlocksStatistics GenerateBlocksStats; + + LooseChunksStatistics LooseChunksStats; + ChunkedFolderContent LocalContent; + + m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::ChunkPartContent, StepCount); + + Stopwatch ScanTimer; + { + std::unique_ptr<OperationLogOutput::ProgressBar> ProgressBarPtr(m_LogOutput.CreateProgressBar("Scan Folder")); + OperationLogOutput::ProgressBar& Progress(*ProgressBarPtr); + + FilteredRate FilteredBytesHashed; + FilteredBytesHashed.Start(); + LocalContent = ChunkFolderContent( + ChunkingStats, + m_IOWorkerPool, + m_Path, + Part.Content, + ChunkController, + ChunkCache, + m_LogOutput.GetProgressUpdateDelayMS(), + [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { + FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", + ChunkingStats.FilesProcessed.load(), + Part.Content.Paths.size(), + NiceBytes(ChunkingStats.BytesHashed.load()), + NiceBytes(Part.TotalRawSize), + NiceNum(FilteredBytesHashed.GetCurrent()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load())); + Progress.UpdateState({.Task = "Scanning files ", + .Details = Details, + .TotalCount = Part.TotalRawSize, + .RemainingCount = Part.TotalRawSize - ChunkingStats.BytesHashed.load(), + .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, + false); + }, + m_AbortFlag, + m_PauseFlag); + FilteredBytesHashed.Stop(); + Progress.Finish(); + if (m_AbortFlag) + { + return; + } + } + + if (!m_Options.IsQuiet) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", + Part.Content.Paths.size(), + NiceBytes(Part.TotalRawSize), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load()), + m_Path, + NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()), + NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + } + + const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent); + + std::vector<size_t> ReuseBlockIndexes; + std::vector<uint32_t> NewBlockChunkIndexes; + + if (PartIndex == 0) + { + const PrepareBuildResult PrepBuildResult = m_PrepBuildResultFuture.get(); + + m_FindBlocksStats.FindBlockTimeMS = PrepBuildResult.ElapsedTimeMs; + m_FindBlocksStats.FoundBlockCount = PrepBuildResult.KnownBlocks.size(); + + if (!m_Options.IsQuiet) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Build prepare took {}. {} took {}, payload size {}{}", + NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs), + m_CreateBuild ? "PutBuild" : "GetBuild", + NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs), + NiceBytes(PrepBuildResult.PayloadSize), + m_Options.IgnoreExistingBlocks ? "" + : fmt::format(". Found {} blocks in {}", + PrepBuildResult.KnownBlocks.size(), + NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs))); + } + + m_PreferredMultipartChunkSize = PrepBuildResult.PreferredMultipartChunkSize; + + m_LargeAttachmentSize = m_Options.AllowMultiparts ? m_PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; + + m_KnownBlocks = std::move(PrepBuildResult.KnownBlocks); + } + + ZEN_ASSERT(m_PreferredMultipartChunkSize != 0); + ZEN_ASSERT(m_LargeAttachmentSize != 0); + + m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::CalculateDelta, StepCount); + + Stopwatch BlockArrangeTimer; + + std::vector<std::uint32_t> LooseChunkIndexes; + { + bool EnableBlocks = true; + std::vector<std::uint32_t> BlockChunkIndexes; + for (uint32_t ChunkIndex = 0; ChunkIndex < LocalContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++) + { + const uint64_t ChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + if (!EnableBlocks || ChunkRawSize == 0 || ChunkRawSize > m_Options.BlockParameters.MaxChunkEmbedSize) + { + LooseChunkIndexes.push_back(ChunkIndex); + LooseChunksStats.ChunkByteCount += ChunkRawSize; + } + else + { + BlockChunkIndexes.push_back(ChunkIndex); + FindBlocksStats.PotentialChunkByteCount += ChunkRawSize; + } + } + FindBlocksStats.PotentialChunkCount += BlockChunkIndexes.size(); + LooseChunksStats.ChunkCount = LooseChunkIndexes.size(); + + if (m_Options.IgnoreExistingBlocks) + { + if (!m_Options.IsQuiet) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, "Ignoring any existing blocks in store"); + } + NewBlockChunkIndexes = std::move(BlockChunkIndexes); + } + else + { + ReuseBlockIndexes = FindReuseBlocks(m_LogOutput, + m_Options.BlockReuseMinPercentLimit, + m_Options.IsVerbose, + ReuseBlocksStats, + m_KnownBlocks, + LocalContent.ChunkedContent.ChunkHashes, + BlockChunkIndexes, + NewBlockChunkIndexes); + FindBlocksStats.AcceptedBlockCount += ReuseBlockIndexes.size(); + + for (const ChunkBlockDescription& Description : m_KnownBlocks) + { + for (uint32_t ChunkRawLength : Description.ChunkRawLengths) + { + FindBlocksStats.FoundBlockByteCount += ChunkRawLength; + } + FindBlocksStats.FoundBlockChunkCount += Description.ChunkRawHashes.size(); + } + } + } + + std::vector<std::vector<uint32_t>> NewBlockChunks; + ArrangeChunksIntoBlocks(LocalContent, LocalLookup, NewBlockChunkIndexes, NewBlockChunks); + + FindBlocksStats.NewBlocksCount += NewBlockChunks.size(); + for (uint32_t ChunkIndex : NewBlockChunkIndexes) + { + FindBlocksStats.NewBlocksChunkByteCount += LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + } + FindBlocksStats.NewBlocksChunkCount += NewBlockChunkIndexes.size(); + + const double AcceptedByteCountPercent = FindBlocksStats.PotentialChunkByteCount > 0 + ? (100.0 * ReuseBlocksStats.AcceptedRawByteCount / FindBlocksStats.PotentialChunkByteCount) + : 0.0; + + const double AcceptedReduntantByteCountPercent = + ReuseBlocksStats.AcceptedByteCount > 0 ? (100.0 * ReuseBlocksStats.AcceptedReduntantByteCount) / + (ReuseBlocksStats.AcceptedByteCount + ReuseBlocksStats.AcceptedReduntantByteCount) + : 0.0; + if (!m_Options.IsQuiet) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n" + " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n" + " Accepting {} ({}) redundant chunks ({:.1f}%)\n" + " Rejected {} ({}) chunks in {} blocks\n" + " Arranged {} ({}) chunks in {} new blocks\n" + " Keeping {} ({}) chunks as loose chunks\n" + " Discovery completed in {}", + FindBlocksStats.FoundBlockChunkCount, + FindBlocksStats.FoundBlockCount, + NiceBytes(FindBlocksStats.FoundBlockByteCount), + NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS), + + ReuseBlocksStats.AcceptedChunkCount, + NiceBytes(ReuseBlocksStats.AcceptedRawByteCount), + FindBlocksStats.AcceptedBlockCount, + AcceptedByteCountPercent, + + ReuseBlocksStats.AcceptedReduntantChunkCount, + NiceBytes(ReuseBlocksStats.AcceptedReduntantByteCount), + AcceptedReduntantByteCountPercent, + + ReuseBlocksStats.RejectedChunkCount, + NiceBytes(ReuseBlocksStats.RejectedByteCount), + ReuseBlocksStats.RejectedBlockCount, + + FindBlocksStats.NewBlocksChunkCount, + NiceBytes(FindBlocksStats.NewBlocksChunkByteCount), + FindBlocksStats.NewBlocksCount, + + LooseChunksStats.ChunkCount, + NiceBytes(LooseChunksStats.ChunkByteCount), + + NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs())); + } + + m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::GenerateBlocks, StepCount); + GeneratedBlocks NewBlocks; + + if (!NewBlockChunks.empty()) + { + Stopwatch GenerateBuildBlocksTimer; + auto __ = MakeGuard([&]() { + uint64_t BlockGenerateTimeUs = GenerateBuildBlocksTimer.GetElapsedTimeUs(); + if (!m_Options.IsQuiet) + { + ZEN_OPERATION_LOG_INFO( + m_LogOutput, + "Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.", + GenerateBlocksStats.GeneratedBlockCount.load(), + NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount), + UploadStats.BlockCount.load(), + NiceBytes(UploadStats.BlocksBytes.load()), + NiceTimeSpanMs(BlockGenerateTimeUs / 1000), + NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, + GenerateBlocksStats.GeneratedBlockByteCount)), + NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8))); + } + }); + GenerateBuildBlocks(LocalContent, LocalLookup, NewBlockChunks, NewBlocks, GenerateBlocksStats, UploadStats); + } + + m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::BuildPartManifest, StepCount); + + CbObject PartManifest; + { + CbObjectWriter PartManifestWriter; + Stopwatch ManifestGenerationTimer; + auto __ = MakeGuard([&]() { + if (!m_Options.IsQuiet) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Generated build part manifest in {} ({})", + NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()), + NiceBytes(PartManifestWriter.GetSaveSize())); + } + }); + + PartManifestWriter.BeginObject("chunker"sv); + { + PartManifestWriter.AddString("name"sv, ChunkController.GetName()); + PartManifestWriter.AddObject("parameters"sv, ChunkController.GetParameters()); + } + PartManifestWriter.EndObject(); // chunker + + std::vector<IoHash> AllChunkBlockHashes; + std::vector<ChunkBlockDescription> AllChunkBlockDescriptions; + AllChunkBlockHashes.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size()); + AllChunkBlockDescriptions.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size()); + for (size_t ReuseBlockIndex : ReuseBlockIndexes) + { + AllChunkBlockDescriptions.push_back(m_KnownBlocks[ReuseBlockIndex]); + AllChunkBlockHashes.push_back(m_KnownBlocks[ReuseBlockIndex].BlockHash); + } + AllChunkBlockDescriptions.insert(AllChunkBlockDescriptions.end(), + NewBlocks.BlockDescriptions.begin(), + NewBlocks.BlockDescriptions.end()); + for (const ChunkBlockDescription& BlockDescription : NewBlocks.BlockDescriptions) + { + AllChunkBlockHashes.push_back(BlockDescription.BlockHash); + } + + std::vector<IoHash> AbsoluteChunkHashes; + if (m_Options.DoExtraContentValidation) + { + tsl::robin_map<IoHash, size_t, IoHash::Hasher> ChunkHashToAbsoluteChunkIndex; + AbsoluteChunkHashes.reserve(LocalContent.ChunkedContent.ChunkHashes.size()); + for (uint32_t ChunkIndex : LooseChunkIndexes) + { + ChunkHashToAbsoluteChunkIndex.insert({LocalContent.ChunkedContent.ChunkHashes[ChunkIndex], AbsoluteChunkHashes.size()}); + AbsoluteChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); + } + for (const ChunkBlockDescription& Block : AllChunkBlockDescriptions) + { + for (const IoHash& ChunkHash : Block.ChunkRawHashes) + { + ChunkHashToAbsoluteChunkIndex.insert({ChunkHash, AbsoluteChunkHashes.size()}); + AbsoluteChunkHashes.push_back(ChunkHash); + } + } + for (const IoHash& ChunkHash : LocalContent.ChunkedContent.ChunkHashes) + { + ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(ChunkHash)] == ChunkHash); + ZEN_ASSERT(LocalContent.ChunkedContent.ChunkHashes[LocalLookup.ChunkHashToChunkIndex.at(ChunkHash)] == ChunkHash); + } + for (const uint32_t ChunkIndex : LocalContent.ChunkedContent.ChunkOrders) + { + ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex])] == + LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); + ZEN_ASSERT(LocalLookup.ChunkHashToChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]) == ChunkIndex); + } + } + std::vector<uint32_t> AbsoluteChunkOrders = CalculateAbsoluteChunkOrders(LocalContent.ChunkedContent.ChunkHashes, + LocalContent.ChunkedContent.ChunkOrders, + LocalLookup.ChunkHashToChunkIndex, + LooseChunkIndexes, + AllChunkBlockDescriptions); + + if (m_Options.DoExtraContentValidation) + { + for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); ChunkOrderIndex++) + { + uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndex]; + uint32_t AbsoluteChunkIndex = AbsoluteChunkOrders[ChunkOrderIndex]; + const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; + const IoHash& AbsoluteChunkHash = AbsoluteChunkHashes[AbsoluteChunkIndex]; + ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash); + } + } + + WriteBuildContentToCompactBinary(PartManifestWriter, + LocalContent.Platform, + LocalContent.Paths, + LocalContent.RawHashes, + LocalContent.RawSizes, + LocalContent.Attributes, + LocalContent.ChunkedContent.SequenceRawHashes, + LocalContent.ChunkedContent.ChunkCounts, + LocalContent.ChunkedContent.ChunkHashes, + LocalContent.ChunkedContent.ChunkRawSizes, + AbsoluteChunkOrders, + LooseChunkIndexes, + AllChunkBlockHashes); + + if (m_Options.DoExtraContentValidation) + { + ChunkedFolderContent VerifyFolderContent; + + std::vector<uint32_t> OutAbsoluteChunkOrders; + std::vector<IoHash> OutLooseChunkHashes; + std::vector<uint64_t> OutLooseChunkRawSizes; + std::vector<IoHash> OutBlockRawHashes; + ReadBuildContentFromCompactBinary(PartManifestWriter.Save(), + VerifyFolderContent.Platform, + VerifyFolderContent.Paths, + VerifyFolderContent.RawHashes, + VerifyFolderContent.RawSizes, + VerifyFolderContent.Attributes, + VerifyFolderContent.ChunkedContent.SequenceRawHashes, + VerifyFolderContent.ChunkedContent.ChunkCounts, + OutAbsoluteChunkOrders, + OutLooseChunkHashes, + OutLooseChunkRawSizes, + OutBlockRawHashes); + ZEN_ASSERT(OutBlockRawHashes == AllChunkBlockHashes); + + for (uint32_t OrderIndex = 0; OrderIndex < OutAbsoluteChunkOrders.size(); OrderIndex++) + { + uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex]; + const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; + + uint32_t VerifyChunkIndex = OutAbsoluteChunkOrders[OrderIndex]; + const IoHash VerifyChunkHash = AbsoluteChunkHashes[VerifyChunkIndex]; + + ZEN_ASSERT(LocalChunkHash == VerifyChunkHash); + } + + CalculateLocalChunkOrders(OutAbsoluteChunkOrders, + OutLooseChunkHashes, + OutLooseChunkRawSizes, + AllChunkBlockDescriptions, + VerifyFolderContent.ChunkedContent.ChunkHashes, + VerifyFolderContent.ChunkedContent.ChunkRawSizes, + VerifyFolderContent.ChunkedContent.ChunkOrders, + m_Options.DoExtraContentValidation); + + ZEN_ASSERT(LocalContent.Paths == VerifyFolderContent.Paths); + ZEN_ASSERT(LocalContent.RawHashes == VerifyFolderContent.RawHashes); + ZEN_ASSERT(LocalContent.RawSizes == VerifyFolderContent.RawSizes); + ZEN_ASSERT(LocalContent.Attributes == VerifyFolderContent.Attributes); + ZEN_ASSERT(LocalContent.ChunkedContent.SequenceRawHashes == VerifyFolderContent.ChunkedContent.SequenceRawHashes); + ZEN_ASSERT(LocalContent.ChunkedContent.ChunkCounts == VerifyFolderContent.ChunkedContent.ChunkCounts); + + for (uint32_t OrderIndex = 0; OrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); OrderIndex++) + { + uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex]; + const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; + uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex]; + + uint32_t VerifyChunkIndex = VerifyFolderContent.ChunkedContent.ChunkOrders[OrderIndex]; + const IoHash VerifyChunkHash = VerifyFolderContent.ChunkedContent.ChunkHashes[VerifyChunkIndex]; + uint64_t VerifyChunkRawSize = VerifyFolderContent.ChunkedContent.ChunkRawSizes[VerifyChunkIndex]; + + ZEN_ASSERT(LocalChunkHash == VerifyChunkHash); + ZEN_ASSERT(LocalChunkRawSize == VerifyChunkRawSize); + } + } + PartManifest = PartManifestWriter.Save(); + } + + m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::UploadBuildPart, StepCount); + + Stopwatch PutBuildPartResultTimer; + std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult = + m_Storage.BuildStorage->PutBuildPart(m_BuildId, Part.PartId, Part.PartName, PartManifest); + if (!m_Options.IsQuiet) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "PutBuildPart took {}, payload size {}. {} attachments are needed.", + NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()), + NiceBytes(PartManifest.GetSize()), + PutBuildPartResult.second.size()); + } + IoHash PartHash = PutBuildPartResult.first; + + auto UploadAttachments = + [this, &LooseChunksStats, &UploadStats, &LocalContent, &LocalLookup, &NewBlockChunks, &NewBlocks, &LooseChunkIndexes]( + std::span<IoHash> RawHashes, + std::vector<IoHash>& OutUnknownChunks) { + if (!m_AbortFlag) + { + UploadStatistics TempUploadStats; + LooseChunksStatistics TempLooseChunksStats; + + Stopwatch TempUploadTimer; + auto __ = MakeGuard([&]() { + if (!m_Options.IsQuiet) + { + uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs(); + ZEN_OPERATION_LOG_INFO( + m_LogOutput, + "Uploaded {} ({}) blocks. " + "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. " + "Transferred {} ({}bits/s) in {}", + TempUploadStats.BlockCount.load(), + NiceBytes(TempUploadStats.BlocksBytes), + + TempLooseChunksStats.CompressedChunkCount.load(), + NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()), + NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, + TempLooseChunksStats.ChunkByteCount)), + TempUploadStats.ChunkCount.load(), + NiceBytes(TempUploadStats.ChunksBytes), + + NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes), + NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)), + NiceTimeSpanMs(TempChunkUploadTimeUs / 1000)); + } + }); + UploadPartBlobs(LocalContent, + LocalLookup, + RawHashes, + NewBlockChunks, + NewBlocks, + LooseChunkIndexes, + m_LargeAttachmentSize, + TempUploadStats, + TempLooseChunksStats, + OutUnknownChunks); + UploadStats += TempUploadStats; + LooseChunksStats += TempLooseChunksStats; + } + }; + + m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::UploadAttachments, StepCount); + + std::vector<IoHash> UnknownChunks; + if (m_Options.IgnoreExistingBlocks) + { + if (m_Options.IsVerbose) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "PutBuildPart uploading all attachments, needs are: {}", + FormatArray<IoHash>(PutBuildPartResult.second, "\n "sv)); + } + + std::vector<IoHash> ForceUploadChunkHashes; + ForceUploadChunkHashes.reserve(LooseChunkIndexes.size()); + + for (uint32_t ChunkIndex : LooseChunkIndexes) + { + ForceUploadChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); + } + + for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockHeaders.size(); BlockIndex++) + { + if (NewBlocks.BlockHeaders[BlockIndex]) + { + // Block was not uploaded during generation + ForceUploadChunkHashes.push_back(NewBlocks.BlockDescriptions[BlockIndex].BlockHash); + } + } + UploadAttachments(ForceUploadChunkHashes, UnknownChunks); + } + else if (!PutBuildPartResult.second.empty()) + { + if (m_Options.IsVerbose) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "PutBuildPart needs attachments: {}", + FormatArray<IoHash>(PutBuildPartResult.second, "\n "sv)); + } + UploadAttachments(PutBuildPartResult.second, UnknownChunks); + } + + auto BuildUnkownChunksResponse = [](const std::vector<IoHash>& UnknownChunks, bool WillRetry) { + return fmt::format( + "The following build blobs was reported as needed for upload but was reported as existing at the start of the " + "operation.{}{}", + WillRetry ? " Treating this as a transient inconsistency issue and will attempt to retry finalization."sv : ""sv, + FormatArray<IoHash>(UnknownChunks, "\n "sv)); + }; + + if (!UnknownChunks.empty()) + { + ZEN_OPERATION_LOG_WARN(m_LogOutput, "{}", BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ true)); + } + + uint32_t FinalizeBuildPartRetryCount = 5; + while (!m_AbortFlag && (FinalizeBuildPartRetryCount--) > 0) + { + Stopwatch FinalizeBuildPartTimer; + std::vector<IoHash> Needs = m_Storage.BuildStorage->FinalizeBuildPart(m_BuildId, Part.PartId, PartHash); + if (!m_Options.IsQuiet) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "FinalizeBuildPart took {}. {} attachments are missing.", + NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()), + Needs.size()); + } + if (Needs.empty()) + { + break; + } + if (m_Options.IsVerbose) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, "FinalizeBuildPart needs attachments: {}", FormatArray<IoHash>(Needs, "\n "sv)); + } + + std::vector<IoHash> RetryUnknownChunks; + UploadAttachments(Needs, RetryUnknownChunks); + if (RetryUnknownChunks == UnknownChunks) + { + if (FinalizeBuildPartRetryCount > 0) + { + // Back off a bit + Sleep(1000); + } + } + else + { + UnknownChunks = RetryUnknownChunks; + ZEN_OPERATION_LOG_WARN(m_LogOutput, + "{}", + BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ FinalizeBuildPartRetryCount != 0)); + } + } + + if (!UnknownChunks.empty()) + { + throw std::runtime_error(BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ false)); + } + + if (!NewBlocks.BlockDescriptions.empty() && !m_AbortFlag) + { + uint64_t UploadBlockMetadataCount = 0; + Stopwatch UploadBlockMetadataTimer; + + uint32_t FailedMetadataUploadCount = 1; + int32_t MetadataUploadRetryCount = 3; + while ((MetadataUploadRetryCount-- > 0) && (FailedMetadataUploadCount > 0)) + { + FailedMetadataUploadCount = 0; + for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++) + { + if (m_AbortFlag) + { + break; + } + const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; + if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex]) + { + const CbObject BlockMetaData = + BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); + if (m_Storage.BuildCacheStorage && m_Options.PopulateCache) + { + m_Storage.BuildCacheStorage->PutBlobMetadatas(m_BuildId, + std::vector<IoHash>({BlockHash}), + std::vector<CbObject>({BlockMetaData})); + } + bool MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData); + if (MetadataSucceeded) + { + UploadStats.BlocksBytes += BlockMetaData.GetSize(); + NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + UploadBlockMetadataCount++; + } + else + { + FailedMetadataUploadCount++; + } + } + } + } + if (UploadBlockMetadataCount > 0) + { + uint64_t ElapsedUS = UploadBlockMetadataTimer.GetElapsedTimeUs(); + UploadStats.ElapsedWallTimeUS += ElapsedUS; + if (!m_Options.IsQuiet) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Uploaded metadata for {} blocks in {}", + UploadBlockMetadataCount, + NiceTimeSpanMs(ElapsedUS / 1000)); + } + } + + // The newly generated blocks are now known blocks so the next part upload can use those blocks as well + m_KnownBlocks.insert(m_KnownBlocks.end(), NewBlocks.BlockDescriptions.begin(), NewBlocks.BlockDescriptions.end()); + } + + m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::PutBuildPartStats, StepCount); + + m_Storage.BuildStorage->PutBuildPartStats( + m_BuildId, + Part.PartId, + {{"totalSize", double(Part.LocalFolderScanStats.FoundFileByteCount.load())}, + {"reusedRatio", AcceptedByteCountPercent / 100.0}, + {"reusedBlockCount", double(FindBlocksStats.AcceptedBlockCount)}, + {"reusedBlockByteCount", double(ReuseBlocksStats.AcceptedRawByteCount)}, + {"newBlockCount", double(FindBlocksStats.NewBlocksCount)}, + {"newBlockByteCount", double(FindBlocksStats.NewBlocksChunkByteCount)}, + {"uploadedCount", double(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load())}, + {"uploadedByteCount", double(UploadStats.BlocksBytes.load() + UploadStats.ChunksBytes.load())}, + {"uploadedBytesPerSec", + double(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.ChunksBytes + UploadStats.BlocksBytes))}, + {"elapsedTimeSec", double(UploadTimer.GetElapsedTimeMs() / 1000.0)}}); + + m_LocalFolderScanStats += Part.LocalFolderScanStats; + m_ChunkingStats += ChunkingStats; + m_FindBlocksStats += FindBlocksStats; + m_ReuseBlocksStats += ReuseBlocksStats; + m_UploadStats += UploadStats; + m_GenerateBlocksStats += GenerateBlocksStats; + m_LooseChunksStats += LooseChunksStats; +} + +void BuildsOperationUploadFolder::UploadPartBlobs(const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, std::span<IoHash> RawHashes, @@ -7229,4 +7281,1007 @@ BuildsOperationValidateBuildPart::ValidateChunkBlock(IoBuffer&& Payload, return GetChunkBlockDescription(BlockBuffer.Flatten(), BlobHash); } +std::vector<std::pair<Oid, std::string>> +ResolveBuildPartNames(CbObjectView BuildObject, + const Oid& BuildId, + const std::vector<Oid>& BuildPartIds, + std::span<const std::string> BuildPartNames, + std::uint64_t& OutPreferredMultipartChunkSize) +{ + std::vector<std::pair<Oid, std::string>> Result; + { + CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); + if (!PartsObject) + { + throw std::runtime_error("Build object does not have a 'parts' object"); + } + + OutPreferredMultipartChunkSize = BuildObject["chunkSize"sv].AsUInt64(OutPreferredMultipartChunkSize); + + std::vector<std::pair<Oid, std::string>> AvailableParts; + + for (CbFieldView PartView : PartsObject) + { + const std::string BuildPartName = std::string(PartView.GetName()); + const Oid BuildPartId = PartView.AsObjectId(); + if (BuildPartId == Oid::Zero) + { + ExtendableStringBuilder<128> SB; + for (CbFieldView ScanPartView : PartsObject) + { + SB.Append(fmt::format("\n {}: {}", ScanPartView.GetName(), ScanPartView.AsObjectId())); + } + throw std::runtime_error(fmt::format("Build object parts does not have a '{}' object id{}", BuildPartName, SB.ToView())); + } + AvailableParts.push_back({BuildPartId, BuildPartName}); + } + + if (BuildPartIds.empty() && BuildPartNames.empty()) + { + Result = AvailableParts; + } + else + { + for (const std::string& BuildPartName : BuildPartNames) + { + if (auto It = std::find_if(AvailableParts.begin(), + AvailableParts.end(), + [&BuildPartName](const auto& Part) { return Part.second == BuildPartName; }); + It != AvailableParts.end()) + { + Result.push_back(*It); + } + else + { + throw std::runtime_error(fmt::format("Build {} object does not have a part named '{}'", BuildId, BuildPartName)); + } + } + for (const Oid& BuildPartId : BuildPartIds) + { + if (auto It = std::find_if(AvailableParts.begin(), + AvailableParts.end(), + [&BuildPartId](const auto& Part) { return Part.first == BuildPartId; }); + It != AvailableParts.end()) + { + Result.push_back(*It); + } + else + { + throw std::runtime_error(fmt::format("Build {} object does not have a part with id '{}'", BuildId, BuildPartId)); + } + } + } + + if (Result.empty()) + { + throw std::runtime_error(fmt::format("Build object does not have any parts", BuildId)); + } + } + return Result; +} + +ChunkedFolderContent +GetRemoteContent(OperationLogOutput& Output, + StorageInstance& Storage, + const Oid& BuildId, + const std::vector<std::pair<Oid, std::string>>& BuildParts, + const BuildManifest& Manifest, + std::span<const std::string> IncludeWildcards, + std::span<const std::string> ExcludeWildcards, + std::unique_ptr<ChunkingController>& OutChunkController, + std::vector<ChunkedFolderContent>& OutPartContents, + std::vector<ChunkBlockDescription>& OutBlockDescriptions, + std::vector<IoHash>& OutLooseChunkHashes, + bool IsQuiet, + bool IsVerbose, + bool DoExtraContentVerify) +{ + ZEN_TRACE_CPU("GetRemoteContent"); + + Stopwatch GetBuildPartTimer; + const Oid BuildPartId = BuildParts[0].first; + const std::string_view BuildPartName = BuildParts[0].second; + CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); + if (!IsQuiet) + { + ZEN_OPERATION_LOG_INFO(Output, + "GetBuildPart {} ('{}') took {}. Payload size: {}", + BuildPartId, + BuildPartName, + NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), + NiceBytes(BuildPartManifest.GetSize())); + ZEN_OPERATION_LOG_INFO(Output, "{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv)); + } + + { + CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView(); + std::string_view ChunkerName = Chunker["name"sv].AsString(); + CbObjectView Parameters = Chunker["parameters"sv].AsObjectView(); + OutChunkController = CreateChunkingController(ChunkerName, Parameters); + } + + auto ParseBuildPartManifest = [&Output, IsQuiet, IsVerbose, DoExtraContentVerify]( + StorageInstance& Storage, + const Oid& BuildId, + const Oid& BuildPartId, + CbObject BuildPartManifest, + std::span<const std::string> IncludeWildcards, + std::span<const std::string> ExcludeWildcards, + const BuildManifest::Part* OptionalManifest, + ChunkedFolderContent& OutRemoteContent, + std::vector<ChunkBlockDescription>& OutBlockDescriptions, + std::vector<IoHash>& OutLooseChunkHashes) { + std::vector<uint32_t> AbsoluteChunkOrders; + std::vector<uint64_t> LooseChunkRawSizes; + std::vector<IoHash> BlockRawHashes; + + ReadBuildContentFromCompactBinary(BuildPartManifest, + OutRemoteContent.Platform, + OutRemoteContent.Paths, + OutRemoteContent.RawHashes, + OutRemoteContent.RawSizes, + OutRemoteContent.Attributes, + OutRemoteContent.ChunkedContent.SequenceRawHashes, + OutRemoteContent.ChunkedContent.ChunkCounts, + AbsoluteChunkOrders, + OutLooseChunkHashes, + LooseChunkRawSizes, + BlockRawHashes); + + // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them + + { + bool AttemptFallback = false; + OutBlockDescriptions = GetBlockDescriptions(Output, + *Storage.BuildStorage, + Storage.BuildCacheStorage.get(), + BuildId, + BuildPartId, + BlockRawHashes, + AttemptFallback, + IsQuiet, + IsVerbose); + } + + CalculateLocalChunkOrders(AbsoluteChunkOrders, + OutLooseChunkHashes, + LooseChunkRawSizes, + OutBlockDescriptions, + OutRemoteContent.ChunkedContent.ChunkHashes, + OutRemoteContent.ChunkedContent.ChunkRawSizes, + OutRemoteContent.ChunkedContent.ChunkOrders, + DoExtraContentVerify); + + std::vector<std::filesystem::path> DeletedPaths; + + if (OptionalManifest) + { + tsl::robin_set<std::string> PathsInManifest; + PathsInManifest.reserve(OptionalManifest->Files.size()); + for (const std::filesystem::path& ManifestPath : OptionalManifest->Files) + { + PathsInManifest.insert(ToLower(ManifestPath.generic_string())); + } + for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths) + { + if (!PathsInManifest.contains(ToLower(RemotePath.generic_string()))) + { + DeletedPaths.push_back(RemotePath); + } + } + } + + if (!IncludeWildcards.empty() || !ExcludeWildcards.empty()) + { + for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths) + { + if (!IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(RemotePath.generic_string()), /*CaseSensitive*/ true)) + { + DeletedPaths.push_back(RemotePath); + } + } + } + + if (!DeletedPaths.empty()) + { + OutRemoteContent = DeletePathsFromChunkedContent(OutRemoteContent, DeletedPaths); + InlineRemoveUnusedHashes(OutLooseChunkHashes, OutRemoteContent.ChunkedContent.ChunkHashes); + } + +#if ZEN_BUILD_DEBUG + ValidateChunkedFolderContent(OutRemoteContent, OutBlockDescriptions, OutLooseChunkHashes, IncludeWildcards, ExcludeWildcards); +#endif // ZEN_BUILD_DEBUG + }; + + auto FindManifest = [&Manifest](const Oid& BuildPartId, std::string_view BuildPartName) -> const BuildManifest::Part* { + if (Manifest.Parts.empty()) + { + return nullptr; + } + if (Manifest.Parts.size() == 1) + { + if (Manifest.Parts[0].PartId == Oid::Zero && Manifest.Parts[0].PartName.empty()) + { + return &Manifest.Parts[0]; + } + } + + auto It = std::find_if(Manifest.Parts.begin(), Manifest.Parts.end(), [BuildPartId, BuildPartName](const BuildManifest::Part& Part) { + if (Part.PartId != Oid::Zero) + { + return Part.PartId == BuildPartId; + } + if (!Part.PartName.empty()) + { + return Part.PartName == BuildPartName; + } + return false; + }); + if (It != Manifest.Parts.end()) + { + return &(*It); + } + return nullptr; + }; + + OutPartContents.resize(1); + ParseBuildPartManifest(Storage, + BuildId, + BuildPartId, + BuildPartManifest, + IncludeWildcards, + ExcludeWildcards, + FindManifest(BuildPartId, BuildPartName), + OutPartContents[0], + OutBlockDescriptions, + OutLooseChunkHashes); + ChunkedFolderContent RemoteContent; + if (BuildParts.size() > 1) + { + std::vector<ChunkBlockDescription> OverlayBlockDescriptions; + std::vector<IoHash> OverlayLooseChunkHashes; + for (size_t PartIndex = 1; PartIndex < BuildParts.size(); PartIndex++) + { + const Oid& OverlayBuildPartId = BuildParts[PartIndex].first; + const std::string& OverlayBuildPartName = BuildParts[PartIndex].second; + Stopwatch GetOverlayBuildPartTimer; + CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId); + if (!IsQuiet) + { + ZEN_OPERATION_LOG_INFO(Output, + "GetBuildPart {} ('{}') took {}. Payload size: {}", + OverlayBuildPartId, + OverlayBuildPartName, + NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()), + NiceBytes(OverlayBuildPartManifest.GetSize())); + } + + ChunkedFolderContent OverlayPartContent; + std::vector<ChunkBlockDescription> OverlayPartBlockDescriptions; + std::vector<IoHash> OverlayPartLooseChunkHashes; + + ParseBuildPartManifest(Storage, + BuildId, + OverlayBuildPartId, + OverlayBuildPartManifest, + IncludeWildcards, + ExcludeWildcards, + FindManifest(OverlayBuildPartId, OverlayBuildPartName), + OverlayPartContent, + OverlayPartBlockDescriptions, + OverlayPartLooseChunkHashes); + OutPartContents.push_back(OverlayPartContent); + OverlayBlockDescriptions.insert(OverlayBlockDescriptions.end(), + OverlayPartBlockDescriptions.begin(), + OverlayPartBlockDescriptions.end()); + OverlayLooseChunkHashes.insert(OverlayLooseChunkHashes.end(), + OverlayPartLooseChunkHashes.begin(), + OverlayPartLooseChunkHashes.end()); + } + + RemoteContent = MergeChunkedFolderContents(OutPartContents[0], std::span<const ChunkedFolderContent>(OutPartContents).subspan(1)); + { + tsl::robin_set<IoHash> AllBlockHashes; + for (const ChunkBlockDescription& Description : OutBlockDescriptions) + { + AllBlockHashes.insert(Description.BlockHash); + } + for (const ChunkBlockDescription& Description : OverlayBlockDescriptions) + { + if (!AllBlockHashes.contains(Description.BlockHash)) + { + AllBlockHashes.insert(Description.BlockHash); + OutBlockDescriptions.push_back(Description); + } + } + } + { + tsl::robin_set<IoHash> AllLooseChunkHashes(OutLooseChunkHashes.begin(), OutLooseChunkHashes.end()); + for (const IoHash& OverlayLooseChunkHash : OverlayLooseChunkHashes) + { + if (!AllLooseChunkHashes.contains(OverlayLooseChunkHash)) + { + AllLooseChunkHashes.insert(OverlayLooseChunkHash); + OutLooseChunkHashes.push_back(OverlayLooseChunkHash); + } + } + } + } + else + { + RemoteContent = OutPartContents[0]; + } + return RemoteContent; +} + +std::string +GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix) +{ + ExtendableStringBuilder<512> SB; + std::vector<std::pair<std::string, std::string>> NameStringValuePairs; + for (CbFieldView Field : Object) + { + std::string_view Name = Field.GetName(); + switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + { + case CbFieldType::String: + NameStringValuePairs.push_back({std::string(Name), std::string(Accessor.AsString())}); + break; + case CbFieldType::IntegerPositive: + NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerPositive())}); + break; + case CbFieldType::IntegerNegative: + NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerNegative())}); + break; + case CbFieldType::Float32: + { + const float Value = Accessor.AsFloat32(); + if (std::isfinite(Value)) + { + NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.9g}", Value)}); + } + else + { + NameStringValuePairs.push_back({std::string(Name), "null"}); + } + } + break; + case CbFieldType::Float64: + { + const double Value = Accessor.AsFloat64(); + if (std::isfinite(Value)) + { + NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.17g}", Value)}); + } + else + { + NameStringValuePairs.push_back({std::string(Name), "null"}); + } + } + break; + case CbFieldType::BoolFalse: + NameStringValuePairs.push_back({std::string(Name), "false"}); + break; + case CbFieldType::BoolTrue: + NameStringValuePairs.push_back({std::string(Name), "true"}); + break; + case CbFieldType::Hash: + { + NameStringValuePairs.push_back({std::string(Name), Accessor.AsHash().ToHexString()}); + } + break; + case CbFieldType::Uuid: + { + StringBuilder<Oid::StringLength + 1> Builder; + Accessor.AsUuid().ToString(Builder); + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + } + break; + case CbFieldType::DateTime: + { + ExtendableStringBuilder<64> Builder; + Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + } + break; + case CbFieldType::TimeSpan: + { + ExtendableStringBuilder<64> Builder; + const TimeSpan Span(Accessor.AsTimeSpanTicks()); + if (Span.GetDays() == 0) + { + Builder << Span.ToString("%h:%m:%s.%n"); + } + else + { + Builder << Span.ToString("%d.%h:%m:%s.%n"); + } + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + break; + } + case CbFieldType::ObjectId: + NameStringValuePairs.push_back({std::string(Name), Accessor.AsObjectId().ToString()}); + break; + } + } + std::string::size_type LongestKey = 0; + for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs) + { + LongestKey = Max(KeyValue.first.length(), LongestKey); + } + for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs) + { + SB.Append(fmt::format("{}{:<{}}: {}{}", Prefix, KeyValue.first, LongestKey, KeyValue.second, Suffix)); + } + return SB.ToString(); +} + +#if ZEN_WITH_TESTS + +namespace buildstorageoperations_testutils { + struct TestState + { + TestState(const std::filesystem::path& InRootPath) + : RootPath(InRootPath) + , LogOutput(CreateStandardLogOutput(Log)) + , ChunkController(CreateStandardChunkingController(StandardChunkingControllerSettings{})) + , ChunkCache(CreateMemoryChunkingCache()) + , WorkerPool(2) + , NetworkPool(2) + { + } + + void Initialize() + { + StoragePath = RootPath / "storage"; + TempPath = RootPath / "temp"; + SystemRootDir = RootPath / "sysroot"; + ZenFolderPath = RootPath / ".zen"; + + CreateDirectories(TempPath); + CreateDirectories(StoragePath); + + Storage.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false); + } + + void CreateSourceData(const std::filesystem::path& Source, std::span<const std::string> Paths, std::span<const uint64_t> Sizes) + { + const std::filesystem::path SourcePath = RootPath / Source; + CreateDirectories(SourcePath); + for (size_t FileIndex = 0; FileIndex < Paths.size(); FileIndex++) + { + const std::string& FilePath = Paths[FileIndex]; + const uint64_t FileSize = Sizes[FileIndex]; + IoBuffer FileData = FileSize > 0 ? CreateSemiRandomBlob(FileSize) : IoBuffer{}; + WriteFile(SourcePath / FilePath, FileData); + } + } + + std::vector<std::pair<Oid, std::string>> Upload(const Oid& BuildId, + const Oid& BuildPartId, + const std::string_view BuildPartName, + const std::filesystem::path& Source, + const std::filesystem::path& ManifestPath) + { + const std::filesystem::path SourcePath = RootPath / Source; + CbObject MetaData; + BuildsOperationUploadFolder Upload(*LogOutput, + Storage, + AbortFlag, + PauseFlag, + WorkerPool, + NetworkPool, + BuildId, + SourcePath, + true, + MetaData, + BuildsOperationUploadFolder::Options{.TempDir = TempPath}); + return Upload.Execute(BuildPartId, BuildPartName, ManifestPath, *ChunkController, *ChunkCache); + } + + void ValidateUpload(const Oid& BuildId, const std::vector<std::pair<Oid, std::string>>& Parts) + { + for (auto Part : Parts) + { + BuildsOperationValidateBuildPart Validate(*LogOutput, + *Storage.BuildStorage, + AbortFlag, + PauseFlag, + WorkerPool, + NetworkPool, + BuildId, + Part.first, + Part.second, + BuildsOperationValidateBuildPart::Options{}); + Validate.Execute(); + } + } + + FolderContent Download(const Oid& BuildId, + const Oid& BuildPartId, + const std::string_view BuildPartName, + const std::filesystem::path& Target, + bool Append) + { + const std::filesystem::path TargetPath = RootPath / Target; + + CreateDirectories(TargetPath); + + uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; + CbObject BuildObject = Storage.BuildStorage->GetBuild(BuildId); + std::vector<Oid> PartIds; + if (BuildPartId != Oid::Zero) + { + PartIds.push_back(BuildPartId); + } + std::vector<std::string> PartNames; + if (!BuildPartName.empty()) + { + PartNames.push_back(std::string(BuildPartName)); + } + std::vector<std::pair<Oid, std::string>> AllBuildParts = + ResolveBuildPartNames(BuildObject, BuildId, PartIds, PartNames, PreferredMultipartChunkSize); + + std::vector<ChunkedFolderContent> PartContents; + + std::vector<ChunkBlockDescription> BlockDescriptions; + std::vector<IoHash> LooseChunkHashes; + + ChunkedFolderContent RemoteContent = GetRemoteContent(*LogOutput, + Storage, + BuildId, + AllBuildParts, + {}, + {}, + {}, + ChunkController, + PartContents, + BlockDescriptions, + LooseChunkHashes, + /*IsQuiet*/ false, + /*IsVerbose*/ false, + /*DoExtraContentVerify*/ true); + + GetFolderContentStatistics LocalFolderScanStats; + + struct ContentVisitor : public GetDirectoryContentVisitor + { + virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) + { + RwLock::ExclusiveLockScope _(ExistingPathsLock); + for (const std::filesystem::path& FileName : Content.FileNames) + { + if (RelativeRoot.empty()) + { + ExistingPaths.push_back(FileName); + } + else + { + ExistingPaths.push_back(RelativeRoot / FileName); + } + } + } + + RwLock ExistingPathsLock; + std::vector<std::filesystem::path> ExistingPaths; + } Visitor; + + Latch PendingWorkCount(1); + + GetDirectoryContent(TargetPath, + DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive, + Visitor, + WorkerPool, + PendingWorkCount); + + PendingWorkCount.CountDown(); + PendingWorkCount.Wait(); + + FolderContent CurrentLocalFolderState = GetValidFolderContent( + WorkerPool, + LocalFolderScanStats, + TargetPath, + Visitor.ExistingPaths, + [](uint64_t PathCount, uint64_t CompletedPathCount) { ZEN_UNUSED(PathCount, CompletedPathCount); }, + 1000, + AbortFlag, + PauseFlag); + + ChunkingStatistics LocalChunkingStats; + ChunkedFolderContent LocalContent = ChunkFolderContent( + LocalChunkingStats, + WorkerPool, + TargetPath, + CurrentLocalFolderState, + *ChunkController, + *ChunkCache, + 1000, + [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { ZEN_UNUSED(IsAborted, IsPaused); }, + AbortFlag, + PauseFlag); + + if (Append) + { + RemoteContent = ApplyChunkedContentOverlay(LocalContent, RemoteContent, {}, {}); + } + + const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent); + const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); + + BuildsOperationUpdateFolder Download(*LogOutput, + Storage, + AbortFlag, + PauseFlag, + WorkerPool, + NetworkPool, + BuildId, + TargetPath, + LocalContent, + LocalLookup, + RemoteContent, + RemoteLookup, + BlockDescriptions, + LooseChunkHashes, + BuildsOperationUpdateFolder::Options{.SystemRootDir = SystemRootDir, + .ZenFolderPath = ZenFolderPath, + .ValidateCompletedSequences = true}); + FolderContent ResultingState; + Download.Execute(ResultingState); + + return ResultingState; + } + + void ValidateDownload(std::span<const std::string> Paths, + std::span<const uint64_t> Sizes, + const std::filesystem::path& Source, + const std::filesystem::path& Target, + const FolderContent& DownloadContent) + { + const std::filesystem::path SourcePath = RootPath / Source; + const std::filesystem::path TargetPath = RootPath / Target; + + CHECK_EQ(Paths.size(), DownloadContent.Paths.size()); + tsl::robin_map<std::string, uint64_t> ExpectedSizes; + tsl::robin_map<std::string, IoHash> ExpectedHashes; + for (size_t Index = 0; Index < Paths.size(); Index++) + { + const std::string LookupString = std::filesystem::path(Paths[Index]).generic_string(); + ExpectedSizes.insert_or_assign(LookupString, Sizes[Index]); + std::filesystem::path FilePath = SourcePath / Paths[Index]; + const IoHash SourceHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(FilePath.make_preferred())); + ExpectedHashes.insert_or_assign(LookupString, SourceHash); + } + for (size_t Index = 0; Index < DownloadContent.Paths.size(); Index++) + { + const std::string LookupString = std::filesystem::path(DownloadContent.Paths[Index]).generic_string(); + auto SizeIt = ExpectedSizes.find(LookupString); + CHECK_NE(SizeIt, ExpectedSizes.end()); + CHECK_EQ(SizeIt->second, DownloadContent.RawSizes[Index]); + std::filesystem::path FilePath = TargetPath / DownloadContent.Paths[Index]; + const IoHash DownloadedHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(FilePath.make_preferred())); + auto HashIt = ExpectedHashes.find(LookupString); + CHECK_NE(HashIt, ExpectedHashes.end()); + CHECK_EQ(HashIt->second, DownloadedHash); + } + } + + const std::filesystem::path RootPath; + std::filesystem::path StoragePath; + std::filesystem::path TempPath; + std::filesystem::path SystemRootDir; + std::filesystem::path ZenFolderPath; + + LoggerRef Log = ConsoleLog(); + std::unique_ptr<OperationLogOutput> LogOutput; + + std::unique_ptr<ChunkingController> ChunkController; + std::unique_ptr<ChunkingCache> ChunkCache; + + StorageInstance Storage; + BuildStorageBase::Statistics StorageStats; + + WorkerThreadPool WorkerPool; + WorkerThreadPool NetworkPool; + + std::atomic<bool> AbortFlag; + std::atomic<bool> PauseFlag; + }; + +} // namespace buildstorageoperations_testutils + +TEST_CASE("buildstorageoperations.upload.folder") +{ + using namespace buildstorageoperations_testutils; + + FastRandom BaseRandom; + + const size_t FileCount = 11; + + const std::string Paths[FileCount] = {{"file_1"}, + {"file_2.exe"}, + {"file_3.txt"}, + {"dir_1/dir1_file_1.exe"}, + {"dir_1/dir1_file_2.pdb"}, + {"dir_1/dir1_file_3.txt"}, + {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"}, + {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"}, + {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"}, + {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"}, + {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}}; + const uint64_t Sizes[FileCount] = + {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u}; + + ScopedTemporaryDirectory SourceFolder; + TestState State(SourceFolder.Path()); + State.Initialize(); + State.CreateSourceData("source", Paths, Sizes); + + const Oid BuildId = Oid::NewOid(); + const Oid BuildPartId = Oid::NewOid(); + const std::string BuildPartName = "default"; + + auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", {}); + + CHECK_EQ(Result.size(), 1u); + CHECK_EQ(Result[0].first, BuildPartId); + CHECK_EQ(Result[0].second, BuildPartName); + State.ValidateUpload(BuildId, Result); + + FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false); + CHECK_EQ(DownloadContent.Paths.size(), FileCount); + State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent); +} + +TEST_CASE("buildstorageoperations.upload.manifest") +{ + using namespace buildstorageoperations_testutils; + + FastRandom BaseRandom; + + const size_t FileCount = 11; + + const std::string Paths[FileCount] = {{"file_1"}, + {"file_2.exe"}, + {"file_3.txt"}, + {"dir_1/dir1_file_1.exe"}, + {"dir_1/dir1_file_2.pdb"}, + {"dir_1/dir1_file_3.txt"}, + {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"}, + {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"}, + {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"}, + {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"}, + {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}}; + const uint64_t Sizes[FileCount] = + {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u}; + + ScopedTemporaryDirectory SourceFolder; + TestState State(SourceFolder.Path()); + State.Initialize(); + State.CreateSourceData("source", Paths, Sizes); + + std::span<const std::string> ManifestFiles(Paths); + ManifestFiles = ManifestFiles.subspan(0, FileCount / 2); + + std::span<const uint64_t> ManifestSizes(Sizes); + ManifestSizes = ManifestSizes.subspan(0, FileCount / 2); + + ExtendableStringBuilder<1024> Manifest; + for (const std::string& FilePath : ManifestFiles) + { + Manifest << FilePath << "\n"; + } + + WriteFile(State.RootPath / "manifest.txt", IoBuffer(IoBuffer::Wrap, Manifest.Data(), Manifest.Size())); + + const Oid BuildId = Oid::NewOid(); + const Oid BuildPartId = Oid::NewOid(); + const std::string BuildPartName = "default"; + + auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", State.RootPath / "manifest.txt"); + + CHECK_EQ(Result.size(), 1u); + CHECK_EQ(Result[0].first, BuildPartId); + CHECK_EQ(Result[0].second, BuildPartName); + State.ValidateUpload(BuildId, Result); + + FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false); + State.ValidateDownload(ManifestFiles, ManifestSizes, "source", "download", DownloadContent); +} + +TEST_CASE("buildstorageoperations.memorychunkingcache") +{ + using namespace buildstorageoperations_testutils; + + FastRandom BaseRandom; + + const size_t FileCount = 11; + + const std::string Paths[FileCount] = {{"file_1"}, + {"file_2.exe"}, + {"file_3.txt"}, + {"dir_1/dir1_file_1.exe"}, + {"dir_1/dir1_file_2.pdb"}, + {"dir_1/dir1_file_3.txt"}, + {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"}, + {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"}, + {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"}, + {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"}, + {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}}; + const uint64_t Sizes[FileCount] = + {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u}; + + ScopedTemporaryDirectory SourceFolder; + TestState State(SourceFolder.Path()); + State.Initialize(); + State.CreateSourceData("source", Paths, Sizes); + + const Oid BuildId = Oid::NewOid(); + const Oid BuildPartId = Oid::NewOid(); + const std::string BuildPartName = "default"; + + { + const std::filesystem::path SourcePath = SourceFolder.Path() / "source"; + CbObject MetaData; + BuildsOperationUploadFolder Upload(*State.LogOutput, + State.Storage, + State.AbortFlag, + State.PauseFlag, + State.WorkerPool, + State.NetworkPool, + BuildId, + SourcePath, + true, + MetaData, + BuildsOperationUploadFolder::Options{.TempDir = State.TempPath}); + auto Result = Upload.Execute(BuildPartId, BuildPartName, {}, *State.ChunkController, *State.ChunkCache); + + CHECK_EQ(Upload.m_ChunkingStats.FilesStoredInCache.load(), FileCount - 1); // Zero size files are not stored in cache + CHECK_EQ(Upload.m_ChunkingStats.BytesStoredInCache.load(), std::accumulate(&Sizes[0], &Sizes[FileCount], uint64_t(0))); + CHECK(Upload.m_ChunkingStats.ChunksStoredInCache.load() >= FileCount - 1); // Zero size files are not stored in cache + + CHECK_EQ(Result.size(), 1u); + CHECK_EQ(Result[0].first, BuildPartId); + CHECK_EQ(Result[0].second, BuildPartName); + } + + auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", {}); + + const Oid BuildId2 = Oid::NewOid(); + const Oid BuildPartId2 = Oid::NewOid(); + + { + const std::filesystem::path SourcePath = SourceFolder.Path() / "source"; + CbObject MetaData; + BuildsOperationUploadFolder Upload(*State.LogOutput, + State.Storage, + State.AbortFlag, + State.PauseFlag, + State.WorkerPool, + State.NetworkPool, + BuildId2, + SourcePath, + true, + MetaData, + BuildsOperationUploadFolder::Options{.TempDir = State.TempPath}); + Upload.Execute(BuildPartId2, BuildPartName, {}, *State.ChunkController, *State.ChunkCache); + + CHECK_EQ(Upload.m_ChunkingStats.FilesFoundInCache.load(), FileCount - 1); // Zero size files are not stored in cache + CHECK_EQ(Upload.m_ChunkingStats.BytesFoundInCache.load(), std::accumulate(&Sizes[0], &Sizes[FileCount], uint64_t(0))); + CHECK(Upload.m_ChunkingStats.ChunksFoundInCache.load() >= FileCount - 1); // Zero size files are not stored in cache + } + + FolderContent DownloadContent = State.Download(BuildId2, BuildPartId2, {}, "download", /* Append */ false); + State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent); +} + +TEST_CASE("buildstorageoperations.upload.multipart") +{ + using namespace buildstorageoperations_testutils; + + FastRandom BaseRandom; + + const size_t FileCount = 11; + + const std::string Paths[FileCount] = {{"file_1"}, + {"file_2.exe"}, + {"file_3.txt"}, + {"dir_1/dir1_file_1.exe"}, + {"dir_1/dir1_file_2.pdb"}, + {"dir_1/dir1_file_3.txt"}, + {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"}, + {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"}, + {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"}, + {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"}, + {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}}; + const uint64_t Sizes[FileCount] = + {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u}; + + ScopedTemporaryDirectory SourceFolder; + TestState State(SourceFolder.Path()); + State.Initialize(); + State.CreateSourceData("source", Paths, Sizes); + + std::span<const std::string> ManifestFiles1(Paths); + ManifestFiles1 = ManifestFiles1.subspan(0, FileCount / 2); + + std::span<const uint64_t> ManifestSizes1(Sizes); + ManifestSizes1 = ManifestSizes1.subspan(0, FileCount / 2); + + std::span<const std::string> ManifestFiles2(Paths); + ManifestFiles2 = ManifestFiles2.subspan(FileCount / 2 - 1); + + std::span<const uint64_t> ManifestSizes2(Sizes); + ManifestSizes2 = ManifestSizes2.subspan(FileCount / 2 - 1); + + const Oid BuildPart1Id = Oid::NewOid(); + const std::string BuildPart1Name = "part1"; + const Oid BuildPart2Id = Oid::NewOid(); + const std::string BuildPart2Name = "part2"; + { + CbObjectWriter Writer; + Writer.BeginObject("parts"sv); + { + Writer.BeginObject(BuildPart1Name); + { + Writer.AddObjectId("partId"sv, BuildPart1Id); + Writer.BeginArray("files"sv); + for (const std::string& ManifestFile : ManifestFiles1) + { + Writer.AddString(ManifestFile); + } + Writer.EndArray(); // files + } + Writer.EndObject(); // part1 + + Writer.BeginObject(BuildPart2Name); + { + Writer.AddObjectId("partId"sv, BuildPart2Id); + Writer.BeginArray("files"sv); + for (const std::string& ManifestFile : ManifestFiles2) + { + Writer.AddString(ManifestFile); + } + Writer.EndArray(); // files + } + Writer.EndObject(); // part2 + } + Writer.EndObject(); // parts + + ExtendableStringBuilder<1024> Manifest; + CompactBinaryToJson(Writer.Save(), Manifest); + WriteFile(State.RootPath / "manifest.json", IoBuffer(IoBuffer::Wrap, Manifest.Data(), Manifest.Size())); + } + + const Oid BuildId = Oid::NewOid(); + + auto Result = State.Upload(BuildId, {}, {}, "source", State.RootPath / "manifest.json"); + + CHECK_EQ(Result.size(), 2u); + CHECK_EQ(Result[0].first, BuildPart1Id); + CHECK_EQ(Result[0].second, BuildPart1Name); + CHECK_EQ(Result[1].first, BuildPart2Id); + CHECK_EQ(Result[1].second, BuildPart2Name); + State.ValidateUpload(BuildId, Result); + + FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false); + State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent); + + FolderContent Part1DownloadContent = State.Download(BuildId, BuildPart1Id, {}, "download_part1", /* Append */ false); + State.ValidateDownload(ManifestFiles1, ManifestSizes1, "source", "download_part1", Part1DownloadContent); + + FolderContent Part2DownloadContent = State.Download(BuildId, Oid::Zero, BuildPart2Name, "download_part2", /* Append */ false); + State.ValidateDownload(ManifestFiles2, ManifestSizes2, "source", "download_part2", Part2DownloadContent); + + (void)State.Download(BuildId, BuildPart1Id, BuildPart1Name, "download_part1+2", /* Append */ false); + FolderContent Part1And2DownloadContent = State.Download(BuildId, BuildPart2Id, {}, "download_part1+2", /* Append */ true); + State.ValidateDownload(Paths, Sizes, "source", "download_part1+2", Part1And2DownloadContent); +} + +void +buildstorageoperations_forcelink() +{ +} + +#endif // ZEN_WITH_TESTS + } // namespace zen diff --git a/src/zenremotestore/builds/buildstorageutil.cpp b/src/zenremotestore/builds/buildstorageutil.cpp index 15ece2edd..36b45e800 100644 --- a/src/zenremotestore/builds/buildstorageutil.cpp +++ b/src/zenremotestore/builds/buildstorageutil.cpp @@ -129,7 +129,10 @@ ResolveBuildStorage(OperationLogOutput& Output, TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2, ClientSettings.Verbose); TestResult.Success) { - ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl); + if (Verbose) + { + ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl); + } HostUrl = ServerEndpoint.BaseUrl; HostAssumeHttp2 = ServerEndpoint.AssumeHttp2; @@ -172,7 +175,10 @@ ResolveBuildStorage(OperationLogOutput& Output, TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2, ClientSettings.Verbose); TestResult.Success) { - ZEN_OPERATION_LOG_INFO(Output, "Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl); + if (Verbose) + { + ZEN_OPERATION_LOG_INFO(Output, "Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl); + } CacheUrl = CacheEndpoint.BaseUrl; CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2; @@ -391,7 +397,10 @@ GetBlockDescriptions(OperationLogOutput& Output, [BlockHash](const ChunkBlockDescription& Description) { return Description.BlockHash == BlockHash; }); ListBlocksIt != FoundBlocks.end()) { - ZEN_OPERATION_LOG_INFO(Output, "Found block {} via context find successfully", BlockHash); + if (!IsQuiet) + { + ZEN_OPERATION_LOG_INFO(Output, "Found block {} via context find successfully", BlockHash); + } AugmentedBlockDescriptions.emplace_back(std::move(*ListBlocksIt)); } else diff --git a/src/zenremotestore/builds/filebuildstorage.cpp b/src/zenremotestore/builds/filebuildstorage.cpp index 1474fd819..55e69de61 100644 --- a/src/zenremotestore/builds/filebuildstorage.cpp +++ b/src/zenremotestore/builds/filebuildstorage.cpp @@ -61,13 +61,12 @@ public: return Writer.Save(); } - virtual CbObject ListBuilds(CbObject Query) override + virtual CbObject ListBuilds(std::string_view JsonQuery) override { ZEN_TRACE_CPU("FileBuildStorage::ListBuilds"); - ZEN_UNUSED(Query); uint64_t ReceivedBytes = 0; - uint64_t SentBytes = Query.GetSize(); + uint64_t SentBytes = JsonQuery.size(); SimulateLatency(SentBytes, 0); auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); diff --git a/src/zenremotestore/builds/jupiterbuildstorage.cpp b/src/zenremotestore/builds/jupiterbuildstorage.cpp index 962ffaaed..23d0ddd4c 100644 --- a/src/zenremotestore/builds/jupiterbuildstorage.cpp +++ b/src/zenremotestore/builds/jupiterbuildstorage.cpp @@ -104,15 +104,13 @@ public: return Response.Save(); } - virtual CbObject ListBuilds(CbObject Query) override + virtual CbObject ListBuilds(std::string_view JsonQuery) override { ZEN_TRACE_CPU("Jupiter::ListBuilds"); - Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - IoBuffer Payload = Query.GetBuffer().AsIoBuffer(); - Payload.SetContentType(ZenContentType::kCbObject); - JupiterResult ListResult = m_Session.ListBuilds(m_Namespace, m_Bucket, Payload); + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + JupiterResult ListResult = m_Session.ListBuilds(m_Namespace, m_Bucket, JsonQuery); AddStatistic(ListResult); if (!ListResult.Success) { diff --git a/src/zenremotestore/chunking/chunkblock.cpp b/src/zenremotestore/chunking/chunkblock.cpp index a5d0db205..c4d8653f4 100644 --- a/src/zenremotestore/chunking/chunkblock.cpp +++ b/src/zenremotestore/chunking/chunkblock.cpp @@ -297,6 +297,7 @@ FindReuseBlocks(OperationLogOutput& Output, if (ChunkCount > 0) { + size_t AcceptedChunkCount = 0; if (!KnownBlocks.empty()) { Stopwatch ReuseTimer; @@ -420,6 +421,7 @@ FindReuseBlocks(OperationLogOutput& Output, { ChunkFound[ChunkIndex] = true; } + AcceptedChunkCount += FoundChunkIndexes.size(); Stats.AcceptedChunkCount += FoundChunkIndexes.size(); Stats.AcceptedByteCount += AdjustedReuseSize; Stats.AcceptedRawByteCount += AdjustedRawReuseSize; @@ -440,7 +442,8 @@ FindReuseBlocks(OperationLogOutput& Output, } } } - OutUnusedChunkIndexes.reserve(ChunkIndexes.size() - Stats.AcceptedChunkCount); + + OutUnusedChunkIndexes.reserve(ChunkIndexes.size() - AcceptedChunkCount); for (uint32_t ChunkIndex : ChunkIndexes) { if (!ChunkFound[ChunkIndex]) diff --git a/src/zenremotestore/chunking/chunkedcontent.cpp b/src/zenremotestore/chunking/chunkedcontent.cpp index e8187d348..26d179f14 100644 --- a/src/zenremotestore/chunking/chunkedcontent.cpp +++ b/src/zenremotestore/chunking/chunkedcontent.cpp @@ -13,6 +13,7 @@ #include <zencore/trace.h> #include <zenremotestore/chunking/chunkblock.h> #include <zenremotestore/chunking/chunkedfile.h> +#include <zenremotestore/chunking/chunkingcache.h> #include <zenremotestore/chunking/chunkingcontroller.h> #include <zenutil/wildcard.h> @@ -100,6 +101,8 @@ namespace { IoHash HashOneFile(ChunkingStatistics& Stats, const ChunkingController& InChunkingController, + ChunkingCache& InChunkingCache, + std::span<const uint64_t> ModificationTicks, ChunkedFolderContent& OutChunkedContent, tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToChunkIndex, tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& RawHashToSequenceRawHashIndex, @@ -108,10 +111,11 @@ namespace { uint32_t PathIndex, std::atomic<bool>& AbortFlag) { - ZEN_TRACE_CPU("ChunkFolderContent"); + ZEN_TRACE_CPU("HashOneFile"); - const uint64_t RawSize = OutChunkedContent.RawSizes[PathIndex]; - const std::filesystem::path& Path = OutChunkedContent.Paths[PathIndex]; + const std::filesystem::path& Path = OutChunkedContent.Paths[PathIndex]; + const uint64_t RawSize = OutChunkedContent.RawSizes[PathIndex]; + const uint64_t ModificationTick = ModificationTicks[PathIndex]; if (RawSize == 0) { @@ -119,16 +123,53 @@ namespace { } else { + std::filesystem::path FullPath = FolderPath / Path; + FullPath.make_preferred(); + ChunkedInfoWithSource Chunked; - const bool DidChunking = - InChunkingController.ProcessFile((FolderPath / Path).make_preferred(), RawSize, Chunked, Stats.BytesHashed, AbortFlag); - if (DidChunking) + + if (!InChunkingCache.GetCachedFile(FullPath, RawSize, ModificationTick, Chunked)) { - Lock.WithExclusiveLock([&]() { - if (!RawHashToSequenceRawHashIndex.contains(Chunked.Info.RawHash)) + const bool DidChunking = InChunkingController.ProcessFile(FullPath, RawSize, Chunked, Stats.BytesHashed, AbortFlag); + if (!DidChunking) + { + ZEN_TRACE_CPU("HashOnly"); + + IoBuffer Buffer = IoBufferBuilder::MakeFromFile(FullPath); + if (Buffer.GetSize() != RawSize) + { + throw std::runtime_error(fmt::format("Failed opening file '{}' for hashing", FolderPath / Path)); + } + + Chunked.Info.RawSize = RawSize; + Chunked.Info.RawHash = IoHash::HashBuffer(Buffer, &Stats.BytesHashed); + } + if (InChunkingCache.PutCachedFile(FullPath, ModificationTick, Chunked)) + { + Stats.FilesStoredInCache++; + Stats.ChunksStoredInCache += Chunked.Info.ChunkSequence.empty() ? 1 : Chunked.Info.ChunkHashes.size(); + Stats.BytesStoredInCache += RawSize; + } + } + else + { + Stats.FilesFoundInCache++; + Stats.ChunksFoundInCache += Chunked.Info.ChunkSequence.empty() ? 1 : Chunked.Info.ChunkHashes.size(); + Stats.BytesFoundInCache += RawSize; + } + Lock.WithExclusiveLock([&]() { + if (!RawHashToSequenceRawHashIndex.contains(Chunked.Info.RawHash)) + { + RawHashToSequenceRawHashIndex.insert( + {Chunked.Info.RawHash, gsl::narrow<uint32_t>(OutChunkedContent.ChunkedContent.SequenceRawHashes.size())}); + + if (Chunked.Info.ChunkSequence.empty()) + { + AddChunkSequence(Stats, OutChunkedContent.ChunkedContent, ChunkHashToChunkIndex, Chunked.Info.RawHash, RawSize); + Stats.UniqueSequencesFound++; + } + else { - RawHashToSequenceRawHashIndex.insert( - {Chunked.Info.RawHash, gsl::narrow<uint32_t>(OutChunkedContent.ChunkedContent.SequenceRawHashes.size())}); std::vector<uint64_t> ChunkSizes; ChunkSizes.reserve(Chunked.ChunkSources.size()); for (const ChunkSource& Source : Chunked.ChunkSources) @@ -144,34 +185,12 @@ namespace { Chunked.Info.ChunkSequence, Chunked.Info.ChunkHashes, ChunkSizes); - Stats.UniqueSequencesFound++; } - }); - Stats.FilesChunked++; - return Chunked.Info.RawHash; - } - else - { - ZEN_TRACE_CPU("HashOnly"); - - IoBuffer Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred()); - if (Buffer.GetSize() != RawSize) - { - throw std::runtime_error(fmt::format("Failed opening file '{}' for hashing", FolderPath / Path)); + Stats.UniqueSequencesFound++; } - const IoHash Hash = IoHash::HashBuffer(Buffer, &Stats.BytesHashed); - - Lock.WithExclusiveLock([&]() { - if (!RawHashToSequenceRawHashIndex.contains(Hash)) - { - RawHashToSequenceRawHashIndex.insert( - {Hash, gsl::narrow<uint32_t>(OutChunkedContent.ChunkedContent.SequenceRawHashes.size())}); - AddChunkSequence(Stats, OutChunkedContent.ChunkedContent, ChunkHashToChunkIndex, Hash, RawSize); - Stats.UniqueSequencesFound++; - } - }); - return Hash; - } + }); + Stats.FilesChunked++; + return Chunked.Info.RawHash; } } @@ -1113,6 +1132,7 @@ ChunkFolderContent(ChunkingStatistics& Stats, const std::filesystem::path& RootPath, const FolderContent& Content, const ChunkingController& InChunkingController, + ChunkingCache& InChunkingCache, int32_t UpdateIntervalMS, std::function<void(bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork)>&& UpdateCallback, std::atomic<bool>& AbortFlag, @@ -1123,6 +1143,10 @@ ChunkFolderContent(ChunkingStatistics& Stats, Stopwatch Timer; auto _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); }); + ZEN_ASSERT(Content.ModificationTicks.size() == Content.Paths.size()); + ZEN_ASSERT(Content.RawSizes.size() == Content.Paths.size()); + ZEN_ASSERT(Content.Attributes.size() == Content.Paths.size()); + ChunkedFolderContent Result = {.Platform = Content.Platform, .Paths = Content.Paths, .RawSizes = Content.RawSizes, @@ -1163,12 +1187,15 @@ ChunkFolderContent(ChunkingStatistics& Stats, { break; } + Work.ScheduleWork(WorkerPool, // GetSyncWorkerPool() [&, PathIndex](std::atomic<bool>& AbortFlag) { if (!AbortFlag) { IoHash RawHash = HashOneFile(Stats, InChunkingController, + InChunkingCache, + Content.ModificationTicks, Result, ChunkHashToChunkIndex, RawHashToSequenceRawHashIndex, diff --git a/src/zenremotestore/chunking/chunkingcache.cpp b/src/zenremotestore/chunking/chunkingcache.cpp new file mode 100644 index 000000000..7f0a26330 --- /dev/null +++ b/src/zenremotestore/chunking/chunkingcache.cpp @@ -0,0 +1,627 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zenremotestore/chunking/chunkingcache.h> + +#include <zenbase/zenbase.h> +#include <zencore/basicfile.h> +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryutil.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zenremotestore/chunking/chunkedfile.h> +#include <zenremotestore/chunking/chunkingcontroller.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <tsl/robin_map.h> +#include <xxhash.h> +#include <gsl/gsl-lite.hpp> +ZEN_THIRD_PARTY_INCLUDES_END + +#if ZEN_WITH_TESTS +# include <zencore/testing.h> +# include <zencore/testutils.h> +# include <algorithm> +#endif // ZEN_WITH_TESTS + +namespace zen { + +class NullChunkingCache : public ChunkingCache +{ +public: + NullChunkingCache() {} + + virtual bool GetCachedFile(const std::filesystem::path& InputPath, + uint64_t RawSize, + uint64_t ModificationTick, + ChunkedInfoWithSource& OutChunked) override + { + ZEN_UNUSED(InputPath, RawSize, OutChunked, ModificationTick); + return false; + } + + virtual bool PutCachedFile(const std::filesystem::path& InputPath, + uint64_t ModificationTick, + const ChunkedInfoWithSource& Chunked) override + { + ZEN_UNUSED(InputPath, Chunked, ModificationTick); + return false; + } +}; + +class MemoryChunkingCache : public ChunkingCache +{ +public: + MemoryChunkingCache() {} + + virtual bool GetCachedFile(const std::filesystem::path& InputPath, + uint64_t RawSize, + uint64_t ModificationTick, + ChunkedInfoWithSource& OutChunked) override + { + const std::u8string PathString = InputPath.generic_u8string(); + const IoHash PathHash = IoHash::HashBuffer(PathString.data(), PathString.length()); + + RwLock::SharedLockScope Lock(m_Lock); + if (auto It = m_PathHashToEntry.find(PathHash); It != m_PathHashToEntry.end()) + { + const CachedEntry& Entry = m_Entries[It->second]; + if (ModificationTick == Entry.ModificationTick && RawSize == Entry.Chunked.Info.RawSize) + { + OutChunked = Entry.Chunked; + return true; + } + else + { + Lock.ReleaseNow(); + RwLock::ExclusiveLockScope EditLock(m_Lock); + if (auto RemoveIt = m_PathHashToEntry.find(PathHash); It != m_PathHashToEntry.end()) + { + CachedEntry& DeleteEntry = m_Entries[It->second]; + DeleteEntry.Chunked = {}; + DeleteEntry.ModificationTick = 0; + m_FreeEntryIndexes.push_back(It->second); + m_PathHashToEntry.erase(It); + } + } + } + return false; + } + + virtual bool PutCachedFile(const std::filesystem::path& InputPath, + uint64_t ModificationTick, + const ChunkedInfoWithSource& Chunked) override + { + const std::u8string PathString = InputPath.generic_u8string(); + const IoHash PathHash = IoHash::HashBuffer(PathString.data(), PathString.length()); + + RwLock::ExclusiveLockScope _(m_Lock); + if (auto It = m_PathHashToEntry.find(PathHash); It != m_PathHashToEntry.end()) + { + CachedEntry& Entry = m_Entries[It->second]; + if (ModificationTick != Entry.ModificationTick || Chunked.Info.RawSize != Entry.Chunked.Info.RawSize) + { + Entry.Chunked = Chunked; + Entry.ModificationTick = ModificationTick; + } + } + else + { + uint32_t EntryIndex = gsl::narrow<uint32_t>(m_Entries.size()); + if (!m_FreeEntryIndexes.empty()) + { + EntryIndex = m_FreeEntryIndexes.back(); + m_FreeEntryIndexes.pop_back(); + m_Entries[EntryIndex] = CachedEntry{.Chunked = Chunked, .ModificationTick = ModificationTick}; + } + else + { + m_Entries.emplace_back(CachedEntry{.Chunked = Chunked, .ModificationTick = ModificationTick}); + } + m_PathHashToEntry.insert_or_assign(PathHash, EntryIndex); + } + return true; + } + + RwLock m_Lock; + + tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> m_PathHashToEntry; + std::vector<uint32_t> m_FreeEntryIndexes; + + struct CachedEntry + { + ChunkedInfoWithSource Chunked; + uint64_t ModificationTick = 0; + }; + + std::vector<CachedEntry> m_Entries; +}; + +class DiskChunkingCache : public ChunkingCache +{ +public: + DiskChunkingCache(const std::filesystem::path& RootPath, ChunkingController& ChunkController, uint64_t MinimumRawSizeForCaching) + : m_RootPath(RootPath) + , m_ChunkerId(GetChunkerIdentity(ChunkController)) + , m_MinimumRawSizeForCaching(MinimumRawSizeForCaching) + { + } + + virtual bool GetCachedFile(const std::filesystem::path& InputPath, + uint64_t RawSize, + uint64_t ModificationTick, + ChunkedInfoWithSource& OutChunked) override + { + if (RawSize < m_MinimumRawSizeForCaching) + { + return false; + } + + const std::filesystem::path CachePath = GetCachePath(InputPath); + + return ReadChunkedInfo(CachePath, RawSize, ModificationTick, OutChunked); + } + + virtual bool PutCachedFile(const std::filesystem::path& InputPath, + uint64_t ModificationTick, + const ChunkedInfoWithSource& Chunked) override + { + if (Chunked.Info.RawSize < m_MinimumRawSizeForCaching) + { + return false; + } + + const std::filesystem::path CachePath = GetCachePath(InputPath); + + return WriteChunkedInfo(CachePath, ModificationTick, Chunked); + } + +private: + static constexpr uint32_t ImplementationRevision = 1; + +#pragma pack(push) +#pragma pack(1) + struct ChunkedInfoHeader + { + static constexpr uint32_t ExpectedMagic = 0x75636368; // 'ucch'; + static constexpr uint32_t CurrentVersion = 1; + + uint32_t Magic = ExpectedMagic; + uint32_t Version = CurrentVersion; + uint64_t SequenceCount = 0; + uint64_t ChunkCount = 0; + uint64_t RawSize = 0; + IoHash RawHash = IoHash::Zero; + uint64_t ModificationTick = 0; + uint32_t Checksum = 0; + + static uint32_t ComputeChecksum(const ChunkedInfoHeader& Header) + { + return XXH32(&Header.Magic, sizeof(Header) - sizeof(uint32_t), 0xC0C0'BABA); + } + }; +#pragma pack(pop) + static_assert(sizeof(ChunkedInfoHeader) == 64); + static_assert(sizeof(ChunkSource) == 12); + + std::filesystem::path GetCachePath(const std::filesystem::path& InputPath) + { + const std::string IdentityString = fmt::format("{}_{}_{}", ImplementationRevision, m_ChunkerId, InputPath.generic_string()); + const IoHash IdentityHash = IoHash::HashBuffer(IdentityString.data(), IdentityString.length()); + std::filesystem::path CachePath = m_RootPath / fmt::format("{}.chunked_content", IdentityHash); + return CachePath; + } + + bool WriteChunkedInfo(const std::filesystem::path& CachePath, uint64_t ModificationTick, const ChunkedInfoWithSource& Chunked) + { + CreateDirectories(CachePath.parent_path()); + + TemporaryFile OutputFile; + std::error_code Ec; + OutputFile.CreateTemporary(CachePath.parent_path(), Ec); + if (Ec) + { + ZEN_DEBUG("Failed to create temp file for cached chunked data at '{}'", CachePath); + return false; + } + ChunkedInfoHeader Header = {.SequenceCount = Chunked.Info.ChunkSequence.size(), + .ChunkCount = Chunked.Info.ChunkHashes.size(), + .RawSize = Chunked.Info.RawSize, + .RawHash = Chunked.Info.RawHash, + .ModificationTick = ModificationTick}; + + Header.Checksum = ChunkedInfoHeader::ComputeChecksum(Header); + + try + { + uint64_t Offset = 0; + + OutputFile.Write(&Header, sizeof(ChunkedInfoHeader), Offset); + Offset += sizeof(ChunkedInfoHeader); + + if (Header.SequenceCount > 0) + { + OutputFile.Write(Chunked.Info.ChunkSequence.data(), Header.SequenceCount * sizeof(uint32_t), Offset); + Offset += Header.SequenceCount * sizeof(uint32_t); + } + + if (Header.ChunkCount > 0) + { + OutputFile.Write(Chunked.Info.ChunkHashes.data(), Header.ChunkCount * sizeof(IoHash), Offset); + Offset += Header.ChunkCount * sizeof(IoHash); + + OutputFile.Write(Chunked.ChunkSources.data(), Header.ChunkCount * sizeof(ChunkSource), Offset); + Offset += Header.ChunkCount * sizeof(ChunkSource); + } + + OutputFile.Flush(); + } + catch (const std::exception& Ex) + { + ZEN_DEBUG("Failed to write cached file {}. Reason: {}", CachePath, Ex.what()); + return false; + } + OutputFile.MoveTemporaryIntoPlace(CachePath, Ec); + if (Ec) + { + ZEN_DEBUG("Failed to move temporary file {} to {}. Reason: {}", OutputFile.GetPath(), CachePath, Ec.message()); + return false; + } + + return true; + } + + bool ReadChunkedInfo(const std::filesystem::path& CachePath, + uint64_t RawSize, + uint64_t ModificationTick, + ChunkedInfoWithSource& OutChunked) + { + BasicFile InputFile; + std::error_code Ec; + InputFile.Open(CachePath, BasicFile::Mode::kRead, Ec); + if (Ec) + { + return false; + } + try + { + uint64_t Size = InputFile.FileSize(); + if (Size < sizeof(ChunkedInfoHeader)) + { + throw std::runtime_error(fmt::format("Expected size >= {}, file has size {}", sizeof(ChunkedInfoHeader), Size)); + } + + uint64_t Offset = 0; + ChunkedInfoHeader Header; + InputFile.Read(&Header, sizeof(ChunkedInfoHeader), Offset); + Offset += sizeof(Header); + + if (Header.Magic != ChunkedInfoHeader::ExpectedMagic) + { + throw std::runtime_error( + fmt::format("Expected magic 0x{:04x}, file has magic 0x{:04x}", ChunkedInfoHeader::ExpectedMagic, Header.Magic)); + } + if (Header.Version != ChunkedInfoHeader::CurrentVersion) + { + throw std::runtime_error( + fmt::format("Expected version {}, file has version {}", ChunkedInfoHeader::CurrentVersion, Header.Version)); + } + if (Header.Checksum != ChunkedInfoHeader::ComputeChecksum(Header)) + { + throw std::runtime_error(fmt::format("Expected checksum 0x{:04x}, file has checksum 0x{:04x}", + Header.Checksum, + ChunkedInfoHeader::ComputeChecksum(Header))); + } + + uint64_t ExpectedSize = sizeof(ChunkedInfoHeader) + Header.SequenceCount * sizeof(uint32_t) + + Header.ChunkCount * sizeof(IoHash) + Header.ChunkCount * sizeof(ChunkSource); + + if (ExpectedSize != Size) + { + throw std::runtime_error(fmt::format("Expected size {}, file has size {}", ExpectedSize, Size)); + } + + if (Header.RawSize != RawSize) + { + InputFile.Close(); + RemoveFile(CachePath, Ec); + return false; + } + + if (Header.ModificationTick != ModificationTick) + { + InputFile.Close(); + RemoveFile(CachePath, Ec); + return false; + } + + OutChunked.Info.RawSize = Header.RawSize; + OutChunked.Info.RawHash = Header.RawHash; + + if (Header.SequenceCount > 0) + { + OutChunked.Info.ChunkSequence.resize(Header.SequenceCount); + InputFile.Read(OutChunked.Info.ChunkSequence.data(), Header.SequenceCount * sizeof(uint32_t), Offset); + Offset += Header.SequenceCount * sizeof(uint32_t); + } + + if (Header.ChunkCount > 0) + { + OutChunked.Info.ChunkHashes.resize(Header.ChunkCount); + OutChunked.ChunkSources.resize(Header.ChunkCount); + + InputFile.Read(OutChunked.Info.ChunkHashes.data(), Header.ChunkCount * sizeof(IoHash), Offset); + Offset += Header.ChunkCount * sizeof(IoHash); + + InputFile.Read(OutChunked.ChunkSources.data(), Header.ChunkCount * sizeof(ChunkSource), Offset); + Offset += Header.ChunkCount * sizeof(ChunkSource); + } + } + catch (const std::exception& Ex) + { + ZEN_DEBUG("Failed to read cached file {}. Reason: {}", CachePath, Ex.what()); + InputFile.Close(); + RemoveFile(CachePath, Ec); + return false; + } + + return true; + } + + const std::filesystem::path m_RootPath; + const IoHash m_ChunkerId; + const uint64_t m_MinimumRawSizeForCaching; + + static IoHash GetChunkerIdentity(ChunkingController& ChunkController) + { + IoHashStream ChunkerIdStream; + std::string_view ChunkerName = ChunkController.GetName(); + ChunkerIdStream.Append(ChunkerName.data(), ChunkerName.length()); + const CbObject ChunkerParameters = ChunkController.GetParameters(); + ChunkerParameters.GetHash(ChunkerIdStream); + return ChunkerIdStream.GetHash(); + } +}; + +std::unique_ptr<ChunkingCache> +CreateNullChunkingCache() +{ + return std::make_unique<NullChunkingCache>(); +} + +std::unique_ptr<ChunkingCache> +CreateMemoryChunkingCache() +{ + return std::make_unique<MemoryChunkingCache>(); +} + +std::unique_ptr<ChunkingCache> +CreateDiskChunkingCache(const std::filesystem::path& RootPath, ChunkingController& ChunkController, uint64_t MinimumRawSizeForCaching) +{ + return std::make_unique<DiskChunkingCache>(RootPath, ChunkController, MinimumRawSizeForCaching); +} + +#if ZEN_WITH_TESTS + +namespace chunkingcache_testutils { + ChunkedInfoWithSource CreateChunked(const std::string_view Data, uint32_t SplitSize) + { + std::vector<uint32_t> ChunkSequence; + std::vector<IoHash> ChunkHashes; + std::vector<ChunkSource> ChunkSources; + + if (SplitSize > 0) + { + std::string_view::size_type SplitOffset = 0; + while (SplitOffset < Data.length()) + { + std::string_view DataPart(Data.substr(SplitOffset, SplitSize)); + + ChunkSequence.push_back(gsl::narrow<uint32_t>(ChunkSequence.size())); + ChunkHashes.push_back(IoHash::HashBuffer(DataPart.data(), DataPart.length())); + ChunkSources.push_back({.Offset = SplitOffset, .Size = gsl::narrow<uint32_t>(DataPart.length())}); + SplitOffset += DataPart.length(); + } + } + + return ChunkedInfoWithSource{.Info = {.RawSize = Data.length(), + .RawHash = IoHash::HashBuffer(Data.data(), Data.length()), + .ChunkSequence = std::move(ChunkSequence), + .ChunkHashes = std::move(ChunkHashes)}, + .ChunkSources = std::move(ChunkSources)}; + } + + bool Equals(const ChunkedInfoWithSource& Lhs, const ChunkedInfoWithSource& Rhs) + { + if (Lhs.ChunkSources.size() != Rhs.ChunkSources.size()) + { + return false; + } + if (std::mismatch(Lhs.ChunkSources.begin(), + Lhs.ChunkSources.end(), + Rhs.ChunkSources.begin(), + [](const ChunkSource& Lhs, const ChunkSource& Rhs) { return Lhs.Offset == Rhs.Offset && Lhs.Size == Rhs.Size; }) + .first != Lhs.ChunkSources.end()) + { + return false; + } + if (Lhs.Info.RawSize != Rhs.Info.RawSize) + { + return false; + } + if (Lhs.Info.ChunkSequence != Rhs.Info.ChunkSequence) + { + return false; + } + if (Lhs.Info.ChunkHashes != Rhs.Info.ChunkHashes) + { + return false; + } + return true; + } +} // namespace chunkingcache_testutils + +TEST_CASE("chunkingcache.nullchunkingcache") +{ + using namespace chunkingcache_testutils; + + std::unique_ptr<ChunkingCache> Cache = CreateNullChunkingCache(); + ChunkedInfoWithSource Result; + CHECK(!Cache->GetCachedFile("dummy-path", 495, 91283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + ChunkedInfoWithSource Chunked = CreateChunked("my data string", 4); + CHECK(!Cache->PutCachedFile("dummy-path", 91283, Chunked)); + + CHECK(!Cache->GetCachedFile("dummy-path", 495, 91283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); +} + +TEST_CASE("chunkingcache.memorychunkingcache") +{ + using namespace chunkingcache_testutils; + + std::unique_ptr<ChunkingCache> Cache = CreateMemoryChunkingCache(); + ChunkedInfoWithSource Result; + CHECK(!Cache->GetCachedFile("file/A/Path", 495, 91283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + CHECK(!Cache->GetCachedFile("file/B/Path", 395, 671283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + ChunkedInfoWithSource ChunkedAV1 = CreateChunked("File A data string", 4); + ChunkedInfoWithSource ChunkedAV2 = CreateChunked("File A updated data string", 4); + ChunkedInfoWithSource ChunkedBV1 = CreateChunked("File B data string", 4); + + CHECK(Cache->PutCachedFile("file/A/Path", 91283, ChunkedAV1)); + CHECK(Cache->PutCachedFile("file/B/Path", 51283, ChunkedBV1)); + + CHECK(Cache->GetCachedFile("file/A/Path", ChunkedAV1.Info.RawSize, 91283, Result)); + CHECK(Equals(Result, ChunkedAV1)); + Result = ChunkedInfoWithSource{}; + + CHECK(Cache->GetCachedFile("file/B/Path", ChunkedBV1.Info.RawSize, 51283, Result)); + CHECK(Equals(Result, ChunkedBV1)); + Result = ChunkedInfoWithSource{}; + + CHECK(!Cache->GetCachedFile("file/A/Path-wrong", 495, 91283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + CHECK(!Cache->GetCachedFile("file/A/Path", 493, 91283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + // Asking a path that exists but without a match will remove that path + CHECK(!Cache->GetCachedFile("file/A/Path", 495, 91283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + CHECK(!Cache->GetCachedFile("file/A/Path", 495, 9283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + CHECK(Cache->PutCachedFile("file/A/Path", 91283, ChunkedAV1)); + CHECK(Cache->GetCachedFile("file/A/Path", ChunkedAV1.Info.RawSize, 91283, Result)); + CHECK(Equals(Result, ChunkedAV1)); + Result = ChunkedInfoWithSource{}; + + CHECK(Cache->PutCachedFile("file/A/Path", 91483, ChunkedAV2)); + CHECK(Cache->GetCachedFile("file/A/Path", ChunkedAV2.Info.RawSize, 91483, Result)); + CHECK(Equals(Result, ChunkedAV2)); + Result = ChunkedInfoWithSource{}; + + CHECK(!Cache->GetCachedFile("file/A/Path", ChunkedAV1.Info.RawSize, 91283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + CHECK(Cache->GetCachedFile("file/B/Path", ChunkedBV1.Info.RawSize, 51283, Result)); + CHECK(Equals(Result, ChunkedBV1)); + Result = ChunkedInfoWithSource{}; + + CHECK(!Cache->GetCachedFile("file/B/Path", ChunkedBV1.Info.RawSize + 1, 51283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); +} + +TEST_CASE("chunkingcache.diskchunkingcache") +{ + using namespace chunkingcache_testutils; + + ScopedTemporaryDirectory TmpDir; + + std::unique_ptr<ChunkingController> ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{}); + + ChunkedInfoWithSource ChunkedAV1 = CreateChunked("File A data string", 4); + ChunkedInfoWithSource ChunkedAV2 = CreateChunked("File A updated data string", 4); + ChunkedInfoWithSource ChunkedBV1 = CreateChunked("File B data string", 4); + + { + std::unique_ptr<ChunkingCache> Cache = CreateDiskChunkingCache(TmpDir.Path(), *ChunkController, 0); + ChunkedInfoWithSource Result; + CHECK(!Cache->GetCachedFile("file/A/Path", 495, 91283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + CHECK(!Cache->GetCachedFile("file/B/Path", 395, 671283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + CHECK(Cache->PutCachedFile("file/A/Path", 91283, ChunkedAV1)); + CHECK(Cache->PutCachedFile("file/B/Path", 51283, ChunkedBV1)); + + CHECK(Cache->GetCachedFile("file/A/Path", ChunkedAV1.Info.RawSize, 91283, Result)); + CHECK(Equals(Result, ChunkedAV1)); + Result = ChunkedInfoWithSource{}; + + CHECK(Cache->GetCachedFile("file/B/Path", ChunkedBV1.Info.RawSize, 51283, Result)); + CHECK(Equals(Result, ChunkedBV1)); + Result = ChunkedInfoWithSource{}; + + CHECK(!Cache->GetCachedFile("file/A/Path-wrong", 495, 91283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + CHECK(!Cache->GetCachedFile("file/A/Path", 493, 91283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + // Asking a path that exists but without a match will remove that path + CHECK(!Cache->GetCachedFile("file/A/Path", 495, 91283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + CHECK(!Cache->GetCachedFile("file/A/Path", 495, 9283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + CHECK(Cache->PutCachedFile("file/A/Path", 91283, ChunkedAV1)); + CHECK(Cache->GetCachedFile("file/A/Path", ChunkedAV1.Info.RawSize, 91283, Result)); + CHECK(Equals(Result, ChunkedAV1)); + Result = ChunkedInfoWithSource{}; + + CHECK(Cache->PutCachedFile("file/A/Path", 91483, ChunkedAV2)); + CHECK(Cache->GetCachedFile("file/A/Path", ChunkedAV2.Info.RawSize, 91483, Result)); + CHECK(Equals(Result, ChunkedAV2)); + Result = ChunkedInfoWithSource{}; + } + { + std::unique_ptr<ChunkingCache> Cache = CreateDiskChunkingCache(TmpDir.Path(), *ChunkController, 0); + ChunkedInfoWithSource Result; + + CHECK(Cache->GetCachedFile("file/A/Path", ChunkedAV2.Info.RawSize, 91483, Result)); + CHECK(Equals(Result, ChunkedAV2)); + Result = ChunkedInfoWithSource{}; + + CHECK(!Cache->GetCachedFile("file/A/Path", ChunkedAV2.Info.RawSize, 91283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + CHECK(!Cache->GetCachedFile("file/A/Path", ChunkedAV2.Info.RawSize, 91483, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + CHECK(Cache->GetCachedFile("file/B/Path", ChunkedBV1.Info.RawSize, 51283, Result)); + CHECK(Equals(Result, ChunkedBV1)); + Result = ChunkedInfoWithSource{}; + + CHECK(!Cache->GetCachedFile("file/B/Path", ChunkedBV1.Info.RawSize + 1, 51283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + + CHECK(!Cache->GetCachedFile("file/B/Path", ChunkedBV1.Info.RawSize, 51283, Result)); + CHECK(Result.Info.ChunkHashes.empty()); + } +} + +void +chunkingcache_forcelink() +{ +} + +#endif // ZEN_WITH_TESTS + +} // namespace zen diff --git a/src/zenremotestore/filesystemutils.cpp b/src/zenremotestore/filesystemutils.cpp index 8dff05c6b..fa1ce6f78 100644 --- a/src/zenremotestore/filesystemutils.cpp +++ b/src/zenremotestore/filesystemutils.cpp @@ -11,6 +11,11 @@ #include <zencore/timer.h> #include <zencore/trace.h> +#if ZEN_WITH_TESTS +# include <zencore/testing.h> +# include <zencore/testutils.h> +#endif // ZEN_WITH_TESTS + namespace zen { BufferedOpenFile::BufferedOpenFile(const std::filesystem::path Path, @@ -349,13 +354,14 @@ CleanDirectory( std::atomic<uint64_t> DeletedItemCount = 0; std::atomic<uint64_t> DeletedByteCount = 0; - CleanDirectoryResult Result; - RwLock ResultLock; - auto _ = MakeGuard([&]() { - Result.DeletedCount = DeletedItemCount.load(); - Result.DeletedByteCount = DeletedByteCount.load(); - Result.FoundCount = DiscoveredItemCount.load(); - }); + std::vector<std::filesystem::path> DirectoriesToDelete; + CleanDirectoryResult Result; + RwLock ResultLock; + auto _ = MakeGuard([&]() { + Result.DeletedCount = DeletedItemCount.load(); + Result.DeletedByteCount = DeletedByteCount.load(); + Result.FoundCount = DiscoveredItemCount.load(); + }); ParallelWork Work(AbortFlag, PauseFlag, @@ -363,119 +369,133 @@ CleanDirectory( struct AsyncVisitor : public GetDirectoryContentVisitor { - AsyncVisitor(const std::filesystem::path& InPath, - std::atomic<bool>& InAbortFlag, - std::atomic<uint64_t>& InDiscoveredItemCount, - std::atomic<uint64_t>& InDeletedItemCount, - std::atomic<uint64_t>& InDeletedByteCount, - std::span<const std::string> InExcludeDirectories, - CleanDirectoryResult& InResult, - RwLock& InResultLock) + AsyncVisitor(const std::filesystem::path& InPath, + std::atomic<bool>& InAbortFlag, + std::atomic<uint64_t>& InDiscoveredItemCount, + std::atomic<uint64_t>& InDeletedItemCount, + std::atomic<uint64_t>& InDeletedByteCount, + std::span<const std::string> InExcludeDirectories, + std::vector<std::filesystem::path>& OutDirectoriesToDelete, + CleanDirectoryResult& InResult, + RwLock& InResultLock) : Path(InPath) , AbortFlag(InAbortFlag) , DiscoveredItemCount(InDiscoveredItemCount) , DeletedItemCount(InDeletedItemCount) , DeletedByteCount(InDeletedByteCount) , ExcludeDirectories(InExcludeDirectories) + , DirectoriesToDelete(OutDirectoriesToDelete) , Result(InResult) , ResultLock(InResultLock) { } + + virtual bool AsyncAllowDirectory(const std::filesystem::path& Parent, const std::filesystem::path& DirectoryName) const override + { + ZEN_UNUSED(Parent); + + if (AbortFlag) + { + return false; + } + const std::string DirectoryString = DirectoryName.string(); + for (const std::string_view ExcludeDirectory : ExcludeDirectories) + { + if (DirectoryString == ExcludeDirectory) + { + return false; + } + } + return true; + } + virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override { ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory"); if (!AbortFlag) { - if (!Content.FileNames.empty()) + DiscoveredItemCount += Content.FileNames.size(); + + ZEN_TRACE_CPU("DeleteFiles"); + std::vector<std::pair<std::filesystem::path, std::error_code>> FailedRemovePaths; + for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++) { - DiscoveredItemCount += Content.FileNames.size(); + const std::filesystem::path& FileName = Content.FileNames[FileIndex]; + const std::filesystem::path FilePath = (Path / RelativeRoot / FileName).make_preferred(); - const std::string RelativeRootString = RelativeRoot.generic_string(); - bool RemoveContent = true; - for (const std::string_view ExcludeDirectory : ExcludeDirectories) + bool IsRemoved = false; + std::error_code Ec; + (void)SetFileReadOnly(FilePath, false, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) { - if (RelativeRootString.starts_with(ExcludeDirectory)) + if (!IsFileWithRetry(FilePath)) { - if (RelativeRootString.length() > ExcludeDirectory.length()) - { - const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()]; - if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' || - MaybePathDelimiter == std::filesystem::path::preferred_separator) - { - RemoveContent = false; - break; - } - } - else - { - RemoveContent = false; - break; - } + IsRemoved = true; + Ec.clear(); + break; } + Sleep(100 + int(Retries * 50)); + Ec.clear(); + (void)SetFileReadOnly(FilePath, false, Ec); } - if (RemoveContent) + if (!IsRemoved && !Ec) { - ZEN_TRACE_CPU("DeleteFiles"); - for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++) + (void)RemoveFile(FilePath, Ec); + for (size_t Retries = 0; Ec && Retries < 6; Retries++) { - const std::filesystem::path& FileName = Content.FileNames[FileIndex]; - const std::filesystem::path FilePath = (Path / RelativeRoot / FileName).make_preferred(); - - bool IsRemoved = false; - std::error_code Ec; - (void)SetFileReadOnly(FilePath, false, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) + if (!IsFileWithRetry(FilePath)) { - if (!IsFileWithRetry(FilePath)) - { - IsRemoved = true; - Ec.clear(); - break; - } - Sleep(100 + int(Retries * 50)); + IsRemoved = true; Ec.clear(); - (void)SetFileReadOnly(FilePath, false, Ec); - } - if (!IsRemoved && !Ec) - { - (void)RemoveFile(FilePath, Ec); - for (size_t Retries = 0; Ec && Retries < 6; Retries++) - { - if (!IsFileWithRetry(FilePath)) - { - IsRemoved = true; - Ec.clear(); - return; - } - Sleep(100 + int(Retries * 50)); - Ec.clear(); - (void)RemoveFile(FilePath, Ec); - } - } - if (!IsRemoved && Ec) - { - RwLock::ExclusiveLockScope _(ResultLock); - Result.FailedRemovePaths.push_back(std::make_pair(FilePath, Ec)); - } - else - { - DeletedItemCount++; - DeletedByteCount += Content.FileSizes[FileIndex]; + break; } + Sleep(100 + int(Retries * 50)); + Ec.clear(); + (void)RemoveFile(FilePath, Ec); } } + if (!IsRemoved && Ec) + { + FailedRemovePaths.push_back(std::make_pair(FilePath, Ec)); + } + else + { + DeletedItemCount++; + DeletedByteCount += Content.FileSizes[FileIndex]; + } + } + + if (!FailedRemovePaths.empty()) + { + RwLock::ExclusiveLockScope _(ResultLock); + FailedRemovePaths.insert(FailedRemovePaths.end(), FailedRemovePaths.begin(), FailedRemovePaths.end()); + } + else if (!RelativeRoot.empty()) + { + DiscoveredItemCount++; + RwLock::ExclusiveLockScope _(ResultLock); + DirectoriesToDelete.push_back(RelativeRoot); } } } - const std::filesystem::path& Path; - std::atomic<bool>& AbortFlag; - std::atomic<uint64_t>& DiscoveredItemCount; - std::atomic<uint64_t>& DeletedItemCount; - std::atomic<uint64_t>& DeletedByteCount; - std::span<const std::string> ExcludeDirectories; - CleanDirectoryResult& Result; - RwLock& ResultLock; - } Visitor(Path, AbortFlag, DiscoveredItemCount, DeletedItemCount, DeletedByteCount, ExcludeDirectories, Result, ResultLock); + const std::filesystem::path& Path; + std::atomic<bool>& AbortFlag; + std::atomic<uint64_t>& DiscoveredItemCount; + std::atomic<uint64_t>& DeletedItemCount; + std::atomic<uint64_t>& DeletedByteCount; + std::span<const std::string> ExcludeDirectories; + std::vector<std::filesystem::path>& DirectoriesToDelete; + CleanDirectoryResult& Result; + RwLock& ResultLock; + } Visitor(Path, + AbortFlag, + DiscoveredItemCount, + DeletedItemCount, + DeletedByteCount, + ExcludeDirectories, + DirectoriesToDelete, + Result, + ResultLock); GetDirectoryContent(Path, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes, @@ -483,29 +503,6 @@ CleanDirectory( IOWorkerPool, Work.PendingWork()); - DirectoryContent LocalDirectoryContent; - GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); - DiscoveredItemCount += LocalDirectoryContent.Directories.size(); - std::vector<std::filesystem::path> DirectoriesToDelete; - DirectoriesToDelete.reserve(LocalDirectoryContent.Directories.size()); - for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) - { - bool Leave = false; - for (const std::string_view ExcludeDirectory : ExcludeDirectories) - { - if (LocalDirPath == (Path / ExcludeDirectory)) - { - Leave = true; - break; - } - } - if (!Leave) - { - DirectoriesToDelete.emplace_back(std::move(LocalDirPath)); - DiscoveredItemCount++; - } - } - uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs(); if (ProgressFunc && ProgressUpdateDelayMS != 0) @@ -528,6 +525,15 @@ CleanDirectory( { ZEN_TRACE_CPU("DeleteDirs"); + + std::sort(DirectoriesToDelete.begin(), + DirectoriesToDelete.end(), + [](const std::filesystem::path& Lhs, const std::filesystem::path& Rhs) { + auto DistanceLhs = std::distance(Lhs.begin(), Lhs.end()); + auto DistanceRhs = std::distance(Rhs.begin(), Rhs.end()); + return DistanceLhs > DistanceRhs; + }); + for (const std::filesystem::path& DirectoryToDelete : DirectoriesToDelete) { if (AbortFlag) @@ -542,53 +548,48 @@ CleanDirectory( } } - { - std::error_code Ec; - zen::CleanDirectory(DirectoryToDelete, /*ForceRemoveReadOnlyFiles*/ true, Ec); - if (Ec) - { - Sleep(200); - Ec.clear(); - zen::CleanDirectory(DirectoryToDelete, /*ForceRemoveReadOnlyFiles*/ true, Ec); - } + const std::filesystem::path FullPath = Path / DirectoryToDelete; - if (!Ec) + std::error_code Ec; + RemoveDir(FullPath, Ec); + if (Ec) + { + for (size_t Retries = 0; Ec && Retries < 3; Retries++) { - RemoveDir(DirectoryToDelete, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) + if (!IsDir(FullPath)) { - if (!IsDir(DirectoryToDelete)) - { - Ec.clear(); - break; - } - Sleep(100 + int(Retries * 50)); Ec.clear(); - RemoveDir(DirectoryToDelete, Ec); + break; } - } - if (Ec) - { - RwLock::ExclusiveLockScope __(ResultLock); - Result.FailedRemovePaths.push_back(std::make_pair(DirectoryToDelete, Ec)); - } - else - { - DeletedItemCount++; + Sleep(100 + int(Retries * 50)); + Ec.clear(); + RemoveDir(FullPath, Ec); } } + if (Ec) + { + RwLock::ExclusiveLockScope __(ResultLock); + Result.FailedRemovePaths.push_back(std::make_pair(DirectoryToDelete, Ec)); + } + else + { + DeletedItemCount++; + } - uint64_t NowMs = Timer.GetElapsedTimeMs(); - - if ((NowMs - LastUpdateTimeMs) >= ProgressUpdateDelayMS) + if (ProgressFunc) { - LastUpdateTimeMs = NowMs; + uint64_t NowMs = Timer.GetElapsedTimeMs(); + + if ((NowMs - LastUpdateTimeMs) > 0) + { + LastUpdateTimeMs = NowMs; - uint64_t Deleted = DeletedItemCount.load(); - uint64_t DeletedBytes = DeletedByteCount.load(); - uint64_t Discovered = DiscoveredItemCount.load(); - std::string Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)); - ProgressFunc(Details, Discovered, Discovered - Deleted, PauseFlag, AbortFlag); + uint64_t Deleted = DeletedItemCount.load(); + uint64_t DeletedBytes = DeletedByteCount.load(); + uint64_t Discovered = DiscoveredItemCount.load(); + std::string Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)); + ProgressFunc(Details, Discovered, Discovered - Deleted, PauseFlag, AbortFlag); + } } } } @@ -625,4 +626,72 @@ CleanAndRemoveDirectory(WorkerThreadPool& WorkerPool, return false; } +#if ZEN_WITH_TESTS + +void +filesystemutils_forcelink() +{ +} + +namespace { + void GenerateFile(const std::filesystem::path& Path) { BasicFile _(Path, BasicFile::Mode::kTruncate); } +} // namespace + +TEST_CASE("filesystemutils.CleanDirectory") +{ + ScopedTemporaryDirectory TmpDir; + + CreateDirectories(TmpDir.Path() / ".keepme"); + GenerateFile(TmpDir.Path() / ".keepme" / "keep"); + GenerateFile(TmpDir.Path() / "deleteme1"); + GenerateFile(TmpDir.Path() / "deleteme2"); + GenerateFile(TmpDir.Path() / "deleteme3"); + CreateDirectories(TmpDir.Path() / ".keepmenot"); + CreateDirectories(TmpDir.Path() / "no.keepme"); + + CreateDirectories(TmpDir.Path() / "DeleteMe"); + GenerateFile(TmpDir.Path() / "DeleteMe" / "delete1"); + CreateDirectories(TmpDir.Path() / "CantDeleteMe"); + GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete1"); + GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete2"); + GenerateFile(TmpDir.Path() / "CantDeleteMe" / "delete3"); + CreateDirectories(TmpDir.Path() / "CantDeleteMe" / ".keepme"); + CreateDirectories(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2"); + GenerateFile(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2" / "delete2"); + GenerateFile(TmpDir.Path() / "CantDeleteMe" / "DeleteMe2" / "delete3"); + CreateDirectories(TmpDir.Path() / "CantDeleteMe2" / ".keepme"); + CreateDirectories(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept"); + GenerateFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept1"); + GenerateFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept2"); + GenerateFile(TmpDir.Path() / "CantDeleteMe2" / "deleteme"); + + WorkerThreadPool Pool(4); + std::atomic<bool> AbortFlag; + std::atomic<bool> PauseFlag; + + CleanDirectory(Pool, AbortFlag, PauseFlag, TmpDir.Path(), std::vector<std::string>{".keepme"}, {}, 0); + + CHECK(IsDir(TmpDir.Path() / ".keepme")); + CHECK(IsFile(TmpDir.Path() / ".keepme" / "keep")); + CHECK(!IsFile(TmpDir.Path() / "deleteme1")); + CHECK(!IsFile(TmpDir.Path() / "deleteme2")); + CHECK(!IsFile(TmpDir.Path() / "deleteme3")); + CHECK(!IsFile(TmpDir.Path() / ".keepmenot")); + CHECK(!IsFile(TmpDir.Path() / "no.keepme")); + + CHECK(!IsDir(TmpDir.Path() / "DeleteMe")); + CHECK(!IsDir(TmpDir.Path() / "DeleteMe2")); + + CHECK(IsDir(TmpDir.Path() / "CantDeleteMe")); + CHECK(IsDir(TmpDir.Path() / "CantDeleteMe" / ".keepme")); + CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2")); + CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2" / ".keepme")); + CHECK(IsDir(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept")); + CHECK(IsFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept1")); + CHECK(IsFile(TmpDir.Path() / "CantDeleteMe2" / ".keepme" / "Kept" / "kept2")); + CHECK(!IsFile(TmpDir.Path() / "CantDeleteMe2" / "deleteme")); +} + +#endif + } // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/builds/buildmanifest.h b/src/zenremotestore/include/zenremotestore/builds/buildmanifest.h new file mode 100644 index 000000000..a0d9a7691 --- /dev/null +++ b/src/zenremotestore/include/zenremotestore/builds/buildmanifest.h @@ -0,0 +1,27 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/filesystem.h> +#include <zencore/uid.h> + +namespace zen { + +struct BuildManifest +{ + struct Part + { + Oid PartId = Oid::Zero; + std::string PartName; + std::vector<std::filesystem::path> Files; + }; + std::vector<Part> Parts; +}; + +BuildManifest ParseBuildManifest(const std::filesystem::path& ManifestPath); + +#if ZEN_WITH_TESTS +void buildmanifest_forcelink(); +#endif // ZEN_WITH_TESTS + +} // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorage.h b/src/zenremotestore/include/zenremotestore/builds/buildstorage.h index 4b7e54d85..85dabc59f 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorage.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorage.h @@ -34,7 +34,7 @@ public: virtual ~BuildStorageBase() {} virtual CbObject ListNamespaces(bool bRecursive = false) = 0; - virtual CbObject ListBuilds(CbObject Query) = 0; + virtual CbObject ListBuilds(std::string_view JsonQuery) = 0; virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) = 0; virtual CbObject GetBuild(const Oid& BuildId) = 0; virtual void FinalizeBuild(const Oid& BuildId) = 0; diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h index d78ee29c1..6304159ae 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h @@ -11,6 +11,7 @@ #include <zenutil/bufferedwritefilecache.h> #include <atomic> +#include <future> #include <memory> ZEN_THIRD_PARTY_INCLUDES_START @@ -141,7 +142,6 @@ public: bool EnableTargetFolderScavenging = true; bool ValidateCompletedSequences = true; std::vector<std::string> ExcludeFolders; - std::vector<std::string> ExcludeExtensions; uint64_t MaximumInMemoryPayloadSize = 512u * 1024u; bool PopulateCache = true; }; @@ -257,6 +257,17 @@ private: ChunkedFolderContent& OutScavengedLocalContent, ChunkedContentLookup& OutScavengedLookup); + void ScavengeSourceForChunks(uint32_t& InOutRemainingChunkCount, + std::vector<bool>& InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags, + tsl::robin_map<IoHash, size_t, IoHash::Hasher>& InOutRawHashToCopyChunkDataIndex, + const std::vector<std::atomic<uint32_t>>& SequenceIndexChunksLeftToWriteCounters, + const ChunkedFolderContent& ScavengedContent, + const ChunkedContentLookup& ScavengedLookup, + std::vector<CopyChunkData>& InOutCopyChunkDatas, + uint32_t ScavengedContentIndex, + uint64_t& InOutChunkMatchingRemoteCount, + uint64_t& InOutChunkMatchingRemoteByteCount); + std::filesystem::path FindDownloadedChunk(const IoHash& ChunkHash); std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> GetRemainingChunkTargets( @@ -367,7 +378,8 @@ private: std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters, std::atomic<uint64_t>& WritePartsComplete, const uint64_t TotalPartWriteCount, - FilteredRate& FilteredWrittenBytesPerSecond); + FilteredRate& FilteredWrittenBytesPerSecond, + bool EnableBacklog); void VerifyAndCompleteChunkSequencesAsync(std::span<const uint32_t> RemoteSequenceIndexes, ParallelWork& Work); bool CompleteSequenceChunk(uint32_t RemoteSequenceIndex, std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters); @@ -411,6 +423,21 @@ struct FindBlocksStatistics uint64_t NewBlocksCount = 0; uint64_t NewBlocksChunkCount = 0; uint64_t NewBlocksChunkByteCount = 0; + + FindBlocksStatistics& operator+=(const FindBlocksStatistics& Rhs) + { + FindBlockTimeMS += Rhs.FindBlockTimeMS; + PotentialChunkCount += Rhs.PotentialChunkCount; + PotentialChunkByteCount += Rhs.PotentialChunkByteCount; + FoundBlockCount += Rhs.FoundBlockCount; + FoundBlockChunkCount += Rhs.FoundBlockChunkCount; + FoundBlockByteCount += Rhs.FoundBlockByteCount; + AcceptedBlockCount += Rhs.AcceptedBlockCount; + NewBlocksCount += Rhs.NewBlocksCount; + NewBlocksChunkCount += Rhs.NewBlocksChunkCount; + NewBlocksChunkByteCount += Rhs.NewBlocksChunkByteCount; + return *this; + } }; struct UploadStatistics @@ -518,15 +545,16 @@ public: WorkerThreadPool& IOWorkerPool, WorkerThreadPool& NetworkPool, const Oid& BuildId, - const Oid& BuildPartId, - const std::string_view BuildPartName, const std::filesystem::path& Path, - const std::filesystem::path& ManifestPath, bool CreateBuild, const CbObject& MetaData, const Options& Options); - void Execute(); + std::vector<std::pair<Oid, std::string>> Execute(const Oid& BuildPartId, + const std::string_view BuildPartName, + const std::filesystem::path& ManifestPath, + ChunkingController& ChunkController, + ChunkingCache& ChunkCache); DiskStatistics m_DiskStats; GetFolderContentStatistics m_LocalFolderScanStats; @@ -538,7 +566,29 @@ public: LooseChunksStatistics m_LooseChunksStats; private: - std::vector<std::filesystem::path> ParseManifest(const std::filesystem::path& Path, const std::filesystem::path& ManifestPath); + struct PrepareBuildResult + { + std::vector<ChunkBlockDescription> KnownBlocks; + uint64_t PreferredMultipartChunkSize = 0; + uint64_t PayloadSize = 0; + uint64_t PrepareBuildTimeMs = 0; + uint64_t FindBlocksTimeMs = 0; + uint64_t ElapsedTimeMs = 0; + }; + + PrepareBuildResult PrepareBuild(); + + struct UploadPart + { + Oid PartId = Oid::Zero; + std::string PartName; + FolderContent Content; + uint64_t TotalRawSize = 0; + GetFolderContentStatistics LocalFolderScanStats; + }; + + std::vector<BuildsOperationUploadFolder::UploadPart> ReadFolder(); + std::vector<UploadPart> ReadManifestParts(const std::filesystem::path& ManifestPath); bool IsAcceptedFolder(const std::string_view& RelativePath) const; bool IsAcceptedFile(const std::string_view& RelativePath) const; @@ -561,7 +611,9 @@ private: void GenerateBuildBlocks(const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, const std::vector<std::vector<uint32_t>>& NewBlockChunks, - GeneratedBlocks& OutBlocks); + GeneratedBlocks& OutBlocks, + GenerateBlocksStatistics& GenerateBlocksStats, + UploadStatistics& UploadStats); std::vector<uint32_t> CalculateAbsoluteChunkOrders(const std::span<const IoHash> LocalChunkHashes, const std::span<const uint32_t> LocalChunkOrder, @@ -584,6 +636,25 @@ private: CompositeBuffer&& HeaderBuffer, const std::vector<uint32_t>& ChunksInBlock); + enum class PartTaskSteps : uint32_t + { + ChunkPartContent = 0, + CalculateDelta, + GenerateBlocks, + BuildPartManifest, + UploadBuildPart, + UploadAttachments, + PutBuildPartStats, + StepCount + }; + + void UploadBuildPart(ChunkingController& ChunkController, + ChunkingCache& ChunkCache, + uint32_t PartIndex, + const UploadPart& Part, + uint32_t PartStepOffset, + uint32_t StepCount); + void UploadPartBlobs(const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, std::span<IoHash> RawHashes, @@ -607,16 +678,18 @@ private: WorkerThreadPool& m_IOWorkerPool; WorkerThreadPool& m_NetworkPool; const Oid m_BuildId; - const Oid m_BuildPartId; - const std::string m_BuildPartName; const std::filesystem::path m_Path; - const std::filesystem::path m_ManifestPath; const bool m_CreateBuild; // ?? Member? const CbObject m_MetaData; // ?? Member const Options m_Options; tsl::robin_set<uint32_t> m_NonCompressableExtensionHashes; + + std::future<PrepareBuildResult> m_PrepBuildResultFuture; + std::vector<ChunkBlockDescription> m_KnownBlocks; + uint64_t m_PreferredMultipartChunkSize = 0; + uint64_t m_LargeAttachmentSize = 0; }; struct ValidateStatistics @@ -720,4 +793,33 @@ CompositeBuffer ValidateBlob(std::atomic<bool>& AbortFlag, uint64_t& OutCompressedSize, uint64_t& OutDecompressedSize); +std::vector<std::pair<Oid, std::string>> ResolveBuildPartNames(CbObjectView BuildObject, + const Oid& BuildId, + const std::vector<Oid>& BuildPartIds, + std::span<const std::string> BuildPartNames, + std::uint64_t& OutPreferredMultipartChunkSize); + +struct BuildManifest; + +ChunkedFolderContent GetRemoteContent(OperationLogOutput& Output, + StorageInstance& Storage, + const Oid& BuildId, + const std::vector<std::pair<Oid, std::string>>& BuildParts, + const BuildManifest& Manifest, + std::span<const std::string> IncludeWildcards, + std::span<const std::string> ExcludeWildcards, + std::unique_ptr<ChunkingController>& OutChunkController, + std::vector<ChunkedFolderContent>& OutPartContents, + std::vector<ChunkBlockDescription>& OutBlockDescriptions, + std::vector<IoHash>& OutLooseChunkHashes, + bool IsQuiet, + bool IsVerbose, + bool DoExtraContentVerify); + +std::string GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix); + +#if ZEN_WITH_TESTS +void buildstorageoperations_forcelink(); +#endif // ZEN_WITH_TESTS + } // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h index 295d275d1..d339b0f94 100644 --- a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h +++ b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h @@ -47,6 +47,19 @@ struct ReuseBlocksStatistics uint64_t RejectedByteCount = 0; uint64_t AcceptedReduntantChunkCount = 0; uint64_t AcceptedReduntantByteCount = 0; + + ReuseBlocksStatistics& operator+=(const ReuseBlocksStatistics& Rhs) + { + AcceptedChunkCount += Rhs.AcceptedChunkCount; + AcceptedByteCount += Rhs.AcceptedByteCount; + AcceptedRawByteCount += Rhs.AcceptedRawByteCount; + RejectedBlockCount += Rhs.RejectedBlockCount; + RejectedChunkCount += Rhs.RejectedChunkCount; + RejectedByteCount += Rhs.RejectedByteCount; + AcceptedReduntantChunkCount += Rhs.AcceptedReduntantChunkCount; + AcceptedReduntantByteCount += Rhs.AcceptedReduntantByteCount; + return *this; + } }; class OperationLogOutput; diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h index 78f20a727..d402bd3f0 100644 --- a/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h +++ b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h @@ -17,6 +17,7 @@ namespace zen { class CbWriter; class ChunkingController; +class ChunkingCache; class WorkerThreadPool; enum class SourcePlatform @@ -55,11 +56,30 @@ FolderContent LoadFolderContentToCompactBinary(CbObjectView Input); struct GetFolderContentStatistics { + GetFolderContentStatistics() {} + GetFolderContentStatistics(GetFolderContentStatistics&& Rhs) + : FoundFileCount(Rhs.FoundFileCount.load()) + , FoundFileByteCount(Rhs.FoundFileByteCount.load()) + , AcceptedFileCount(Rhs.AcceptedFileCount.load()) + , AcceptedFileByteCount(Rhs.AcceptedFileByteCount.load()) + , ElapsedWallTimeUS(Rhs.ElapsedWallTimeUS) + { + } std::atomic<uint64_t> FoundFileCount = 0; std::atomic<uint64_t> FoundFileByteCount = 0; std::atomic<uint64_t> AcceptedFileCount = 0; std::atomic<uint64_t> AcceptedFileByteCount = 0; uint64_t ElapsedWallTimeUS = 0; + + inline GetFolderContentStatistics& operator+=(const GetFolderContentStatistics& Rhs) + { + FoundFileCount += Rhs.FoundFileCount; + FoundFileByteCount += Rhs.FoundFileByteCount; + AcceptedFileCount += Rhs.AcceptedFileCount; + AcceptedFileByteCount += Rhs.AcceptedFileByteCount; + ElapsedWallTimeUS += Rhs.ElapsedWallTimeUS; + return *this; + } }; FolderContent GetFolderContent(GetFolderContentStatistics& Stats, @@ -146,6 +166,12 @@ struct ChunkingStatistics std::atomic<uint64_t> UniqueChunksFound = 0; std::atomic<uint64_t> UniqueSequencesFound = 0; std::atomic<uint64_t> UniqueBytesFound = 0; + std::atomic<uint64_t> FilesFoundInCache = 0; + std::atomic<uint64_t> ChunksFoundInCache = 0; + std::atomic<uint64_t> BytesFoundInCache = 0; + std::atomic<uint64_t> FilesStoredInCache = 0; + std::atomic<uint64_t> ChunksStoredInCache = 0; + std::atomic<uint64_t> BytesStoredInCache = 0; uint64_t ElapsedWallTimeUS = 0; inline ChunkingStatistics& operator+=(const ChunkingStatistics& Rhs) @@ -157,6 +183,12 @@ struct ChunkingStatistics UniqueSequencesFound += Rhs.UniqueSequencesFound; UniqueBytesFound += Rhs.UniqueBytesFound; ElapsedWallTimeUS += Rhs.ElapsedWallTimeUS; + FilesFoundInCache += Rhs.FilesFoundInCache; + ChunksFoundInCache += Rhs.ChunksFoundInCache; + BytesFoundInCache += Rhs.BytesFoundInCache; + FilesStoredInCache += Rhs.FilesStoredInCache; + ChunksStoredInCache += Rhs.ChunksStoredInCache; + BytesStoredInCache += Rhs.BytesStoredInCache; return *this; } }; @@ -166,6 +198,7 @@ ChunkedFolderContent ChunkFolderContent(ChunkingStatistics& Stats, const std::filesystem::path& RootPath, const FolderContent& Content, const ChunkingController& InChunkingController, + ChunkingCache& InChunkingCache, int32_t UpdateIntervalMS, std::function<void(bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork)>&& UpdateCallback, std::atomic<bool>& AbortFlag, diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkedfile.h b/src/zenremotestore/include/zenremotestore/chunking/chunkedfile.h index 4cec80fdb..64e2c9c29 100644 --- a/src/zenremotestore/include/zenremotestore/chunking/chunkedfile.h +++ b/src/zenremotestore/include/zenremotestore/chunking/chunkedfile.h @@ -21,11 +21,14 @@ struct ChunkedInfo std::vector<IoHash> ChunkHashes; }; +#pragma pack(push) +#pragma pack(4) struct ChunkSource { uint64_t Offset; // 8 uint32_t Size; // 4 }; +#pragma pack(pop) struct ChunkedInfoWithSource { diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkingcache.h b/src/zenremotestore/include/zenremotestore/chunking/chunkingcache.h new file mode 100644 index 000000000..e213bc41b --- /dev/null +++ b/src/zenremotestore/include/zenremotestore/chunking/chunkingcache.h @@ -0,0 +1,44 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <filesystem> + +namespace zen { + +struct ChunkedInfoWithSource; +class ChunkingController; + +class ChunkingCache +{ +public: + virtual ~ChunkingCache() {} + + /* + * Attempting to fetch a cached file with mismatching RawSize of ModificationTick will delete any existing cached data for that + * InputPath + * + * If GetCachedFile returns false, OutChunked is untouched + */ + virtual bool GetCachedFile(const std::filesystem::path& InputPath, + uint64_t RawSize, + uint64_t ModificationTick, + ChunkedInfoWithSource& OutChunked) = 0; + + /* + * Putting a cached entry with an existing InputPath will overwrite it with the new ModificationTick and Chunked data + */ + virtual bool PutCachedFile(const std::filesystem::path& InputPath, uint64_t ModificationTick, const ChunkedInfoWithSource& Chunked) = 0; +}; + +std::unique_ptr<ChunkingCache> CreateNullChunkingCache(); +std::unique_ptr<ChunkingCache> CreateMemoryChunkingCache(); +std::unique_ptr<ChunkingCache> CreateDiskChunkingCache(const std::filesystem::path& RootPath, + ChunkingController& ChunkController, + uint64_t MinimumRawSizeForCaching); + +#if ZEN_WITH_TESTS +void chunkingcache_forcelink(); +#endif // ZEN_WITH_TESTS + +} // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/filesystemutils.h b/src/zenremotestore/include/zenremotestore/filesystemutils.h index a6c88e5cb..cb2d718f7 100644 --- a/src/zenremotestore/include/zenremotestore/filesystemutils.h +++ b/src/zenremotestore/include/zenremotestore/filesystemutils.h @@ -12,6 +12,8 @@ class CompositeBuffer; class BufferedOpenFile { public: + static constexpr uint64_t BlockSize = 256u * 1024u; + BufferedOpenFile(const std::filesystem::path Path, std::atomic<uint64_t>& OpenReadCount, std::atomic<uint64_t>& CurrentOpenFileCount, @@ -30,8 +32,6 @@ public: void* Handle() { return m_Source.Handle(); } private: - const uint64_t BlockSize = 256u * 1024u; - BasicFile m_Source; const uint64_t m_SourceSize; std::atomic<uint64_t>& m_OpenReadCount; @@ -116,4 +116,6 @@ bool CleanAndRemoveDirectory(WorkerThreadPool& WorkerPool, std::atomic<bool>& PauseFlag, const std::filesystem::path& Directory); +void filesystemutils_forcelink(); // internal + } // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/jupiter/jupitersession.h b/src/zenremotestore/include/zenremotestore/jupiter/jupitersession.h index 15077376c..eaf6962fd 100644 --- a/src/zenremotestore/include/zenremotestore/jupiter/jupitersession.h +++ b/src/zenremotestore/include/zenremotestore/jupiter/jupitersession.h @@ -110,7 +110,7 @@ public: JupiterResult ListBuildNamespaces(); JupiterResult ListBuildBuckets(std::string_view Namespace); - JupiterResult ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload); + JupiterResult ListBuilds(std::string_view Namespace, std::string_view BucketId, std::string_view JsonQuery); JupiterResult PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload); JupiterResult GetBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); JupiterResult FinalizeBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); diff --git a/src/zenremotestore/include/zenremotestore/projectstore/projectstoreoperations.h b/src/zenremotestore/include/zenremotestore/projectstore/projectstoreoperations.h index 044436509..a07ede6f6 100644 --- a/src/zenremotestore/include/zenremotestore/projectstore/projectstoreoperations.h +++ b/src/zenremotestore/include/zenremotestore/projectstore/projectstoreoperations.h @@ -56,7 +56,7 @@ private: const Oid m_BuildId; const Options m_Options; - Oid m_BuildPartId; + Oid m_BuildPartId = Oid::Zero; CbObject m_BuildObject; CbObject m_BuildPartsObject; CbObject m_OpsSectionObject; diff --git a/src/zenremotestore/jupiter/jupitersession.cpp b/src/zenremotestore/jupiter/jupitersession.cpp index dd0e5ad1f..1bc6564ce 100644 --- a/src/zenremotestore/jupiter/jupitersession.cpp +++ b/src/zenremotestore/jupiter/jupitersession.cpp @@ -430,9 +430,10 @@ JupiterSession::ListBuildBuckets(std::string_view Namespace) } JupiterResult -JupiterSession::ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload) +JupiterSession::ListBuilds(std::string_view Namespace, std::string_view BucketId, std::string_view JsonQuery) { - ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject); + IoBuffer Payload(IoBuffer::Wrap, JsonQuery.data(), JsonQuery.size()); + Payload.SetContentType(ZenContentType::kJSON); std::string OptionalBucketPath = BucketId.empty() ? "" : fmt::format("/{}", BucketId); HttpClient::Response Response = m_HttpClient.Post(fmt::format("/api/v2/builds/{}{}/search", Namespace, OptionalBucketPath), Payload, diff --git a/src/zenremotestore/projectstore/remoteprojectstore.cpp b/src/zenremotestore/projectstore/remoteprojectstore.cpp index b566e5bed..8be8eb0df 100644 --- a/src/zenremotestore/projectstore/remoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/remoteprojectstore.cpp @@ -11,6 +11,7 @@ #include <zencore/scopeguard.h> #include <zencore/stream.h> #include <zencore/timer.h> +#include <zencore/trace.h> #include <zencore/workthreadpool.h> #include <zenhttp/httpcommon.h> #include <zenremotestore/chunking/chunkedfile.h> @@ -266,6 +267,8 @@ namespace remotestore_impl { &DownloadStartMS, IgnoreMissingAttachments, OptionalContext]() { + ZEN_TRACE_CPU("DownloadBlockChunks"); + auto _ = MakeGuard([&AttachmentsDownloadLatch] { AttachmentsDownloadLatch.CountDown(); }); if (RemoteResult.IsError()) { @@ -386,7 +389,9 @@ namespace remotestore_impl { IgnoreMissingAttachments, OptionalContext, RetriesLeft, - Chunks = Chunks]() { + Chunks = std::vector<IoHash>(Chunks)]() { + ZEN_TRACE_CPU("DownloadBlock"); + auto _ = MakeGuard([&AttachmentsDownloadLatch] { AttachmentsDownloadLatch.CountDown(); }); if (RemoteResult.IsError()) { @@ -439,7 +444,7 @@ namespace remotestore_impl { IgnoreMissingAttachments, OptionalContext, RetriesLeft, - Chunks = Chunks, + Chunks = std::move(Chunks), Bytes = std::move(BlockResult.Bytes)]() { auto _ = MakeGuard([&AttachmentsWriteLatch] { AttachmentsWriteLatch.CountDown(); }); if (RemoteResult.IsError()) @@ -492,7 +497,7 @@ namespace remotestore_impl { {}); return; } - SharedBuffer BlockPayload = Compressed.Decompress(); + CompositeBuffer BlockPayload = Compressed.DecompressToComposite(); if (!BlockPayload) { if (RetriesLeft > 0) @@ -542,7 +547,7 @@ namespace remotestore_impl { uint64_t BlockHeaderSize = 0; bool StoreChunksOK = IterateChunkBlock( - BlockPayload, + BlockPayload.Flatten(), [&WantedChunks, &WriteAttachmentBuffers, &WriteRawHashes, &Info, &PotentialSize]( CompressedBuffer&& Chunk, const IoHash& AttachmentRawHash) { @@ -648,6 +653,8 @@ namespace remotestore_impl { &Info, IgnoreMissingAttachments, OptionalContext]() { + ZEN_TRACE_CPU("DownloadAttachment"); + auto _ = MakeGuard([&AttachmentsDownloadLatch] { AttachmentsDownloadLatch.CountDown(); }); if (RemoteResult.IsError()) { @@ -694,6 +701,8 @@ namespace remotestore_impl { AttachmentSize, Bytes = std::move(AttachmentResult.Bytes), OptionalContext]() { + ZEN_TRACE_CPU("WriteAttachment"); + auto _ = MakeGuard([&AttachmentsWriteLatch] { AttachmentsWriteLatch.CountDown(); }); if (RemoteResult.IsError()) { @@ -745,6 +754,8 @@ namespace remotestore_impl { Chunks = std::move(ChunksInBlock), &AsyncOnBlock, &RemoteResult]() mutable { + ZEN_TRACE_CPU("CreateBlock"); + auto _ = MakeGuard([&OpSectionsLatch] { OpSectionsLatch.CountDown(); }); if (RemoteResult.IsError()) { @@ -917,6 +928,8 @@ namespace remotestore_impl { &LooseFileAttachments, &Info, OptionalContext]() { + ZEN_TRACE_CPU("UploadAttachment"); + auto _ = MakeGuard([&SaveAttachmentsLatch] { SaveAttachmentsLatch.CountDown(); }); if (RemoteResult.IsError()) { @@ -1039,6 +1052,8 @@ namespace remotestore_impl { &BulkBlockAttachmentsToUpload, &Info, OptionalContext]() { + ZEN_TRACE_CPU("UploadChunk"); + auto _ = MakeGuard([&SaveAttachmentsLatch] { SaveAttachmentsLatch.CountDown(); }); if (RemoteResult.IsError()) { @@ -1587,6 +1602,8 @@ BuildContainer(CidStore& ChunkStore, AllowChunking, &RemoteResult, OptionalContext]() { + ZEN_TRACE_CPU("PrepareChunk"); + auto _ = MakeGuard([&ResolveAttachmentsLatch] { ResolveAttachmentsLatch.CountDown(); }); if (remotestore_impl::IsCancelled(OptionalContext)) { @@ -1972,10 +1989,16 @@ BuildContainer(CidStore& ChunkStore, try { uint64_t FetchAttachmentsStartMS = Timer.GetElapsedTimeMs(); - std::unordered_set<IoHash, IoHash::Hasher> BlockAttachmentHashes; + std::unordered_set<IoHash, IoHash::Hasher> AddedAttachmentHashes; auto NewBlock = [&]() { - size_t BlockIndex = remotestore_impl::AddBlock(BlocksLock, Blocks); - size_t ChunkCount = ChunksInBlock.size(); + size_t BlockIndex = remotestore_impl::AddBlock(BlocksLock, Blocks); + size_t ChunkCount = ChunksInBlock.size(); + std::vector<IoHash> ChunkRawHashes; + ChunkRawHashes.reserve(ChunkCount); + for (const std::pair<IoHash, FetchChunkFunc>& Chunk : ChunksInBlock) + { + ChunkRawHashes.push_back(Chunk.first); + } if (BuildBlocks) { remotestore_impl::CreateBlock(WorkerPool, @@ -1990,15 +2013,13 @@ BuildContainer(CidStore& ChunkStore, } else { - ZEN_INFO("Bulk group {} attachments", BlockAttachmentHashes.size()); + ZEN_INFO("Bulk group {} attachments", ChunkCount); OnBlockChunks(std::move(ChunksInBlock)); } { // We can share the lock as we are not resizing the vector and only touch BlockHash at our own index RwLock::SharedLockScope _(BlocksLock); - Blocks[BlockIndex].ChunkRawHashes.insert(Blocks[BlockIndex].ChunkRawHashes.end(), - BlockAttachmentHashes.begin(), - BlockAttachmentHashes.end()); + Blocks[BlockIndex].ChunkRawHashes = std::move(ChunkRawHashes); } uint64_t NowMS = Timer.GetElapsedTimeMs(); ZEN_INFO("Assembled block {} with {} chunks in {} ({})", @@ -2007,7 +2028,6 @@ BuildContainer(CidStore& ChunkStore, NiceTimeSpanMs(NowMS - FetchAttachmentsStartMS), NiceBytes(BlockSize)); FetchAttachmentsStartMS = NowMS; - BlockAttachmentHashes.clear(); ChunksInBlock.clear(); BlockSize = 0; GeneratedBlockCount++; @@ -2039,8 +2059,17 @@ BuildContainer(CidStore& ChunkStore, ZEN_ASSERT(InfoIt != UploadAttachments.end()); uint64_t PayloadSize = InfoIt->second.Size; - if (BlockAttachmentHashes.insert(AttachmentHash).second) + if (AddedAttachmentHashes.insert(AttachmentHash).second) { + if (BuildBlocks && ChunksInBlock.size() > 0) + { + if (((BlockSize + PayloadSize) > MaxBlockSize || (ChunksInBlock.size() + 1) > MaxChunksPerBlock) && + (CurrentOpKey != LastOpKey)) + { + NewBlock(); + } + } + if (auto It = LooseUploadAttachments.find(RawHash); It != LooseUploadAttachments.end()) { ChunksInBlock.emplace_back(std::make_pair( @@ -2079,10 +2108,6 @@ BuildContainer(CidStore& ChunkStore, } BlockSize += PayloadSize; - if ((BlockSize >= MaxBlockSize || ChunksInBlock.size() > MaxChunksPerBlock) && (CurrentOpKey != LastOpKey)) - { - NewBlock(); - } LastOpKey = CurrentOpKey; ChunksAssembled++; } @@ -2123,9 +2148,17 @@ BuildContainer(CidStore& ChunkStore, const IoHash& ChunkHash = ChunkedFile.Chunked.Info.ChunkHashes[ChunkIndex]; if (auto FindIt = ChunkedHashes.find(ChunkHash); FindIt != ChunkedHashes.end()) { - if (BlockAttachmentHashes.insert(ChunkHash).second) + if (AddedAttachmentHashes.insert(ChunkHash).second) { const ChunkSource& Source = Chunked.ChunkSources[ChunkIndex]; + uint32_t ChunkSize = gsl::narrow<uint32_t>(CompressedBuffer::GetHeaderSizeForNoneEncoder() + Source.Size); + if (BuildBlocks && ChunksInBlock.size() > 0) + { + if ((BlockSize + ChunkSize) > MaxBlockSize || (ChunksInBlock.size() + 1) > MaxChunksPerBlock) + { + NewBlock(); + } + } ChunksInBlock.emplace_back( std::make_pair(ChunkHash, [Source = ChunkedFile.Source, Offset = Source.Offset, Size = Source.Size]( @@ -2136,13 +2169,6 @@ BuildContainer(CidStore& ChunkStore, OodleCompressionLevel::None)}; })); BlockSize += CompressedBuffer::GetHeaderSizeForNoneEncoder() + Source.Size; - if (BuildBlocks) - { - if (BlockSize >= MaxBlockSize || ChunksInBlock.size() > MaxChunksPerBlock) - { - NewBlock(); - } - } ChunksAssembled++; } ChunkedHashes.erase(FindIt); @@ -2781,12 +2807,26 @@ ParseOplogContainer(const CbObject& ContainerObject, for (CbFieldView OpEntry : OpsArray) { OpEntry.IterateAttachments([&](CbFieldView FieldView) { OpsAttachments.insert(FieldView.AsAttachment()); }); + if (remotestore_impl::IsCancelled(OptionalContext)) + { + return RemoteProjectStore::Result{.ErrorCode = gsl::narrow<int>(HttpResponseCode::OK), + .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0, + .Reason = "Operation cancelled"}; + } } } { std::vector<IoHash> ReferencedAttachments(OpsAttachments.begin(), OpsAttachments.end()); OnReferencedAttachments(ReferencedAttachments); } + + if (remotestore_impl::IsCancelled(OptionalContext)) + { + return RemoteProjectStore::Result{.ErrorCode = gsl::narrow<int>(HttpResponseCode::OK), + .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0, + .Reason = "Operation cancelled"}; + } + remotestore_impl::ReportMessage(OptionalContext, fmt::format("Oplog references {} attachments", OpsAttachments.size())); CbArrayView ChunkedFilesArray = ContainerObject["chunkedfiles"sv].AsArrayView(); @@ -3206,6 +3246,8 @@ LoadOplog(CidStore& ChunkStore, IgnoreMissingAttachments, &Info, OptionalContext]() { + ZEN_TRACE_CPU("DechunkAttachment"); + auto _ = MakeGuard([&DechunkLatch, &TempFileName] { std::error_code Ec; if (IsFile(TempFileName, Ec)) @@ -3232,7 +3274,7 @@ LoadOplog(CidStore& ChunkStore, { BasicFileWriter TmpWriter(TmpFile, 64u * 1024u); - uint64_t Offset = CompressedBuffer::GetHeaderSizeForNoneEncoder(); + uint64_t ChunkOffset = CompressedBuffer::GetHeaderSizeForNoneEncoder(); BLAKE3Stream HashingStream; for (std::uint32_t SequenceIndex : Chunked.ChunkSequence) { @@ -3255,15 +3297,80 @@ LoadOplog(CidStore& ChunkStore, } return; } - CompositeBuffer Decompressed = - CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)).DecompressToComposite(); - for (const SharedBuffer& Segment : Decompressed.GetSegments()) + + IoHash RawHash; + uint64_t RawSize; + + CompressedBuffer Compressed = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunk)), RawHash, RawSize); + if (RawHash != ChunkHash) { - MemoryView SegmentData = Segment.GetView(); - HashingStream.Append(SegmentData); - TmpWriter.Write(SegmentData.GetData(), SegmentData.GetSize(), Offset); - Offset += SegmentData.GetSize(); + remotestore_impl::ReportMessage( + OptionalContext, + fmt::format("Mismatching raw hash {} for chunk {} for chunked attachment {}", + RawHash, + ChunkHash, + Chunked.RawHash)); + + // We only add 1 as the resulting missing count will be 1 for the dechunked file + Info.MissingAttachmentCount.fetch_add(1); + if (!IgnoreMissingAttachments) + { + RemoteResult.SetError( + gsl::narrow<int>(HttpResponseCode::NotFound), + "Missing chunk", + fmt::format("Mismatching raw hash {} for chunk {} for chunked attachment {}", + RawHash, + ChunkHash, + Chunked.RawHash)); + } + return; + } + + { + ZEN_TRACE_CPU("DecompressChunk"); + + if (!Compressed.DecompressToStream(0, + RawSize, + [&](uint64_t SourceOffset, + uint64_t SourceSize, + uint64_t Offset, + const CompositeBuffer& RangeBuffer) { + ZEN_UNUSED(SourceOffset, SourceSize, Offset); + + for (const SharedBuffer& Segment : + RangeBuffer.GetSegments()) + { + MemoryView SegmentData = Segment.GetView(); + HashingStream.Append(SegmentData); + TmpWriter.Write(SegmentData.GetData(), + SegmentData.GetSize(), + ChunkOffset + Offset); + } + return true; + })) + { + remotestore_impl::ReportMessage( + OptionalContext, + fmt::format("Failed to decompress chunk {} for chunked attachment {}", + ChunkHash, + Chunked.RawHash)); + + // We only add 1 as the resulting missing count will be 1 for the dechunked file + Info.MissingAttachmentCount.fetch_add(1); + if (!IgnoreMissingAttachments) + { + RemoteResult.SetError( + gsl::narrow<int>(HttpResponseCode::NotFound), + "Missing chunk", + fmt::format("Failed to decompress chunk {} for chunked attachment {}", + ChunkHash, + Chunked.RawHash)); + } + return; + } } + ChunkOffset += RawSize; } BLAKE3 RawHash = HashingStream.GetHash(); ZEN_ASSERT(Chunked.RawHash == IoHash::FromBLAKE3(RawHash)); diff --git a/src/zenremotestore/zenremotestore.cpp b/src/zenremotestore/zenremotestore.cpp index e074455b3..a0bb17260 100644 --- a/src/zenremotestore/zenremotestore.cpp +++ b/src/zenremotestore/zenremotestore.cpp @@ -2,9 +2,13 @@ #include <zenremotestore/zenremotestore.h> +#include <zenremotestore/builds/buildmanifest.h> #include <zenremotestore/builds/buildsavedstate.h> +#include <zenremotestore/builds/buildstorageoperations.h> #include <zenremotestore/chunking/chunkedcontent.h> #include <zenremotestore/chunking/chunkedfile.h> +#include <zenremotestore/chunking/chunkingcache.h> +#include <zenremotestore/filesystemutils.h> #include <zenremotestore/projectstore/remoteprojectstore.h> #if ZEN_WITH_TESTS @@ -14,11 +18,14 @@ namespace zen { void zenremotestore_forcelinktests() { + buildmanifest_forcelink(); buildsavedstate_forcelink(); + buildstorageoperations_forcelink(); chunkblock_forcelink(); chunkedcontent_forcelink(); chunkedfile_forcelink(); - chunkedcontent_forcelink(); + chunkingcache_forcelink(); + filesystemutils_forcelink(); remoteprojectstore_forcelink(); } diff --git a/src/zenserver-test/cache-tests.cpp b/src/zenserver-test/cache-tests.cpp index 854590987..0272d3797 100644 --- a/src/zenserver-test/cache-tests.cpp +++ b/src/zenserver-test/cache-tests.cpp @@ -35,7 +35,7 @@ TEST_CASE("zcache.basic") { ZenServerInstance Instance1(TestEnv); - Instance1.SetTestDir(TestDir); + Instance1.SetDataDir(TestDir); const uint16_t PortNumber = Instance1.SpawnServerAndWaitUntilReady(); const std::string BaseUri = fmt::format("http://localhost:{}/z$", PortNumber); @@ -91,7 +91,7 @@ TEST_CASE("zcache.basic") { ZenServerInstance Instance1(TestEnv); - Instance1.SetTestDir(TestDir); + Instance1.SetDataDir(TestDir); const uint16_t PortNumber = Instance1.SpawnServerAndWaitUntilReady(); const std::string BaseUri = fmt::format("http://localhost:{}/z$", PortNumber); @@ -167,7 +167,7 @@ TEST_CASE("zcache.cbpackage") std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); ZenServerInstance Instance1(TestEnv); - Instance1.SetTestDir(TestDir); + Instance1.SetDataDir(TestDir); const uint16_t PortNumber = Instance1.SpawnServerAndWaitUntilReady(); const std::string BaseUri = fmt::format("http://localhost:{}/z$", PortNumber); @@ -203,11 +203,11 @@ TEST_CASE("zcache.cbpackage") std::filesystem::path RemoteDataDir = TestEnv.CreateNewTestDir(); ZenServerInstance RemoteInstance(TestEnv); - RemoteInstance.SetTestDir(RemoteDataDir); + RemoteInstance.SetDataDir(RemoteDataDir); const uint16_t RemotePortNumber = RemoteInstance.SpawnServerAndWaitUntilReady(); ZenServerInstance LocalInstance(TestEnv); - LocalInstance.SetTestDir(LocalDataDir); + LocalInstance.SetDataDir(LocalDataDir); LocalInstance.SpawnServer(TestEnv.GetNewPortNumber(), fmt::format("--upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", RemotePortNumber)); const uint16_t LocalPortNumber = LocalInstance.WaitUntilReady(); @@ -261,11 +261,11 @@ TEST_CASE("zcache.cbpackage") std::filesystem::path RemoteDataDir = TestEnv.CreateNewTestDir(); ZenServerInstance RemoteInstance(TestEnv); - RemoteInstance.SetTestDir(RemoteDataDir); + RemoteInstance.SetDataDir(RemoteDataDir); const uint16_t RemotePortNumber = RemoteInstance.SpawnServerAndWaitUntilReady(); ZenServerInstance LocalInstance(TestEnv); - LocalInstance.SetTestDir(LocalDataDir); + LocalInstance.SetDataDir(LocalDataDir); LocalInstance.SpawnServer(TestEnv.GetNewPortNumber(), fmt::format("--upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", RemotePortNumber)); const uint16_t LocalPortNumber = LocalInstance.WaitUntilReady(); @@ -756,7 +756,7 @@ TEST_CASE("zcache.rpc") std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); ZenServerInstance Inst(TestEnv); - Inst.SetTestDir(TestDir); + Inst.SetDataDir(TestDir); const uint16_t BasePort = Inst.SpawnServerAndWaitUntilReady(); const std::string BaseUri = fmt::format("http://localhost:{}/z$", BasePort); @@ -786,7 +786,7 @@ TEST_CASE("zcache.rpc") std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); ZenServerInstance Inst(TestEnv); - Inst.SetTestDir(TestDir); + Inst.SetDataDir(TestDir); const uint16_t BasePort = Inst.SpawnServerAndWaitUntilReady(); const std::string BaseUri = fmt::format("http://localhost:{}/z$", BasePort); @@ -1239,7 +1239,7 @@ TEST_CASE("zcache.rpc") std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); ZenServerInstance Inst(TestEnv); - Inst.SetTestDir(TestDir); + Inst.SetDataDir(TestDir); const uint16_t BasePort = Inst.SpawnServerAndWaitUntilReady(); const std::string BaseUri = fmt::format("http://localhost:{}/z$", BasePort); diff --git a/src/zenserver-test/hub-tests.cpp b/src/zenserver-test/hub-tests.cpp new file mode 100644 index 000000000..42a5dcae4 --- /dev/null +++ b/src/zenserver-test/hub-tests.cpp @@ -0,0 +1,252 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#if ZEN_WITH_TESTS +# include "zenserver-test.h" +# include <zencore/testing.h> +# include <zencore/testutils.h> +# include <zencore/workthreadpool.h> +# include <zencore/compactbinarybuilder.h> +# include <zencore/compactbinarypackage.h> +# include <zencore/compress.h> +# include <zencore/filesystem.h> +# include <zencore/stream.h> +# include <zencore/string.h> +# include <zencore/fmtutils.h> +# include <zencore/scopeguard.h> +# include <zenhttp/packageformat.h> +# include <zenremotestore/builds/buildstoragecache.h> +# include <zenutil/workerpools.h> +# include <zenutil/zenserverprocess.h> +# include <zenhttp/httpclient.h> +# include <zenutil/consul.h> + +namespace zen::tests::hub { + +using namespace std::literals; + +TEST_SUITE_BEGIN("hub.lifecycle"); + +TEST_CASE("hub.lifecycle.basic") +{ + { + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kHubServer); + + const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri() + "/hub/"); + + HttpClient::Response Result = Client.Get("status"); + CHECK(Result); + } +} + +TEST_CASE("hub.lifecycle.children") +{ + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kHubServer); + + const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(); + REQUIRE(PortNumber != 0); + + SUBCASE("spawn") + { + HttpClient Client(Instance.GetBaseUri() + "/hub/"); + + HttpClient::Response Result = Client.Get("status"); + REQUIRE(Result); + + { + Result = Client.Post("modules/abc/provision"); + REQUIRE(Result); + + CbObject AbcResult = Result.AsObject(); + CHECK(AbcResult["moduleId"].AsString() == "abc"sv); + const uint16_t AbcPort = AbcResult["port"].AsUInt16(0); + CHECK_NE(AbcPort, 0); + + // This should be a fresh instance with no contents + + HttpClient AbcClient(fmt::format("http://localhost:{}", AbcPort)); + + Result = AbcClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567"); + CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound); + + Result = AbcClient.Put("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567", + IoBufferBuilder::MakeFromMemory(MakeMemoryView("abcdef"sv))); + CHECK_EQ(Result.StatusCode, HttpResponseCode::Created); + } + + { + Result = Client.Post("modules/def/provision"); + REQUIRE(Result); + + CbObject DefResult = Result.AsObject(); + CHECK(DefResult["moduleId"].AsString() == "def"sv); + const uint16_t DefPort = DefResult["port"].AsUInt16(0); + REQUIRE_NE(DefPort, 0); + + // This should be a fresh instance with no contents + + HttpClient DefClient(fmt::format("http://localhost:{}", DefPort)); + + Result = DefClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567"); + CHECK_EQ(Result.StatusCode, HttpResponseCode::NotFound); + + Result = DefClient.Put("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567", + IoBufferBuilder::MakeFromMemory(MakeMemoryView("AbcDef"sv))); + CHECK_EQ(Result.StatusCode, HttpResponseCode::Created); + } + + // this should be rejected because of the invalid module id + Result = Client.Post("modules/!!!!!/provision"); + CHECK(!Result); + + Result = Client.Post("modules/ghi/provision"); + REQUIRE(Result); + + // Tear down instances + + Result = Client.Post("modules/abc/deprovision"); + REQUIRE(Result); + + Result = Client.Post("modules/def/deprovision"); + REQUIRE(Result); + + Result = Client.Post("modules/ghi/deprovision"); + REQUIRE(Result); + + // re-provision to verify that (de)hydration preserved state + { + Result = Client.Post("modules/abc/provision"); + REQUIRE(Result); + + CbObject AbcResult = Result.AsObject(); + CHECK(AbcResult["moduleId"].AsString() == "abc"sv); + const uint16_t AbcPort = AbcResult["port"].AsUInt16(0); + REQUIRE_NE(AbcPort, 0); + + // This should contain the content from the previous run + + HttpClient AbcClient(fmt::format("http://localhost:{}", AbcPort)); + + Result = AbcClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567"); + CHECK_EQ(Result.StatusCode, HttpResponseCode::OK); + + CHECK_EQ(Result.AsText(), "abcdef"sv); + + Result = AbcClient.Put("/z$/ns1/b/1123456789abcdef0123456789abcdef01234567", + IoBufferBuilder::MakeFromMemory(MakeMemoryView("ghijklmnop"sv))); + CHECK_EQ(Result.StatusCode, HttpResponseCode::Created); + } + + { + Result = Client.Post("modules/def/provision"); + REQUIRE(Result); + + CbObject DefResult = Result.AsObject(); + CHECK(DefResult["moduleId"].AsString() == "def"sv); + const uint16_t DefPort = DefResult["port"].AsUInt16(0); + REQUIRE_NE(DefPort, 0); + + // This should contain the content from the previous run + + HttpClient DefClient(fmt::format("http://localhost:{}", DefPort)); + + Result = DefClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567"); + CHECK_EQ(Result.StatusCode, HttpResponseCode::OK); + + CHECK_EQ(Result.AsText(), "AbcDef"sv); + + Result = DefClient.Put("/z$/ns1/b/1123456789abcdef0123456789abcdef01234567", + IoBufferBuilder::MakeFromMemory(MakeMemoryView("GhijklmNop"sv))); + CHECK_EQ(Result.StatusCode, HttpResponseCode::Created); + } + + Result = Client.Post("modules/abc/deprovision"); + REQUIRE(Result); + + Result = Client.Post("modules/def/deprovision"); + REQUIRE(Result); + + // re-provision to verify that (de)hydration preserved state, including + // state which was generated after the very first dehydration + { + Result = Client.Post("modules/abc/provision"); + REQUIRE(Result); + + CbObject AbcResult = Result.AsObject(); + CHECK(AbcResult["moduleId"].AsString() == "abc"sv); + const uint16_t AbcPort = AbcResult["port"].AsUInt16(0); + REQUIRE_NE(AbcPort, 0); + + // This should contain the content from the previous two runs + + HttpClient AbcClient(fmt::format("http://localhost:{}", AbcPort)); + + Result = AbcClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567"); + CHECK_EQ(Result.StatusCode, HttpResponseCode::OK); + + CHECK_EQ(Result.AsText(), "abcdef"sv); + + Result = AbcClient.Get("/z$/ns1/b/1123456789abcdef0123456789abcdef01234567"); + CHECK_EQ(Result.StatusCode, HttpResponseCode::OK); + + CHECK_EQ(Result.AsText(), "ghijklmnop"sv); + } + + { + Result = Client.Post("modules/def/provision"); + REQUIRE(Result); + + CbObject DefResult = Result.AsObject(); + REQUIRE(DefResult["moduleId"].AsString() == "def"sv); + const uint16_t DefPort = DefResult["port"].AsUInt16(0); + REQUIRE_NE(DefPort, 0); + + // This should contain the content from the previous two runs + + HttpClient DefClient(fmt::format("http://localhost:{}", DefPort)); + + Result = DefClient.Get("/z$/ns1/b/0123456789abcdef0123456789abcdef01234567"); + CHECK_EQ(Result.StatusCode, HttpResponseCode::OK); + + CHECK_EQ(Result.AsText(), "AbcDef"sv); + + Result = DefClient.Get("/z$/ns1/b/1123456789abcdef0123456789abcdef01234567"); + CHECK_EQ(Result.StatusCode, HttpResponseCode::OK); + + CHECK_EQ(Result.AsText(), "GhijklmNop"sv); + } + + Result = Client.Post("modules/abc/deprovision"); + REQUIRE(Result); + + Result = Client.Post("modules/def/deprovision"); + REQUIRE(Result); + + // final sanity check that the hub is still responsive + Result = Client.Get("status"); + CHECK(Result); + } +} + +TEST_SUITE_END(); + +TEST_CASE("hub.consul.lifecycle") +{ + zen::consul::ConsulProcess ConsulProc; + ConsulProc.SpawnConsulAgent(); + + zen::consul::ConsulClient Client("http://localhost:8500/"); + Client.SetKeyValue("zen/hub/testkey", "testvalue"); + + std::string RetrievedValue = Client.GetKeyValue("zen/hub/testkey"); + CHECK_EQ(RetrievedValue, "testvalue"); + + Client.DeleteKey("zen/hub/testkey"); + + ConsulProc.StopConsulAgent(); +} + +} // namespace zen::tests::hub +#endif diff --git a/src/zenserver-test/projectstore-tests.cpp b/src/zenserver-test/projectstore-tests.cpp index c8c96dbbb..735aef159 100644 --- a/src/zenserver-test/projectstore-tests.cpp +++ b/src/zenserver-test/projectstore-tests.cpp @@ -34,7 +34,7 @@ TEST_CASE("project.basic") std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); ZenServerInstance Instance1(TestEnv); - Instance1.SetTestDir(TestDir); + Instance1.SetDataDir(TestDir); const uint16_t PortNumber = Instance1.SpawnServerAndWaitUntilReady(); diff --git a/src/zenserver-test/workspace-tests.cpp b/src/zenserver-test/workspace-tests.cpp index f299b6dcf..7595d790a 100644 --- a/src/zenserver-test/workspace-tests.cpp +++ b/src/zenserver-test/workspace-tests.cpp @@ -81,7 +81,7 @@ TEST_CASE("workspaces.create") std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); ZenServerInstance Instance(TestEnv); - Instance.SetTestDir(TestDir); + Instance.SetDataDir(TestDir); const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady( fmt::format("--workspaces-enabled --workspaces-allow-changes --system-dir {}", SystemRootPath)); CHECK(PortNumber != 0); @@ -214,7 +214,7 @@ TEST_CASE("workspaces.restricted") std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); ZenServerInstance Instance(TestEnv); - Instance.SetTestDir(TestDir); + Instance.SetDataDir(TestDir); const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(fmt::format("--workspaces-enabled --system-dir {}", SystemRootPath)); CHECK(PortNumber != 0); @@ -319,7 +319,7 @@ TEST_CASE("workspaces.lifetimes") { std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); ZenServerInstance Instance(TestEnv); - Instance.SetTestDir(TestDir); + Instance.SetDataDir(TestDir); const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady( fmt::format("--workspaces-enabled --workspaces-allow-changes --system-dir {}", SystemRootPath)); CHECK(PortNumber != 0); @@ -343,7 +343,7 @@ TEST_CASE("workspaces.lifetimes") { std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); ZenServerInstance Instance(TestEnv); - Instance.SetTestDir(TestDir); + Instance.SetDataDir(TestDir); const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(fmt::format("--workspaces-enabled --system-dir {}", SystemRootPath)); CHECK(PortNumber != 0); @@ -362,7 +362,7 @@ TEST_CASE("workspaces.lifetimes") { std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); ZenServerInstance Instance(TestEnv); - Instance.SetTestDir(TestDir); + Instance.SetDataDir(TestDir); const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(fmt::format("--workspaces-enabled --system-dir {}", SystemRootPath)); CHECK(PortNumber != 0); diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp index 42296cbe1..9a42bb73d 100644 --- a/src/zenserver-test/zenserver-test.cpp +++ b/src/zenserver-test/zenserver-test.cpp @@ -17,6 +17,7 @@ # include <zencore/timer.h> # include <zenhttp/httpclient.h> # include <zenhttp/packageformat.h> +# include <zenutil/commandlineoptions.h> # include <zenutil/logging/testformatter.h> # include <zenutil/zenserverprocess.h> @@ -61,9 +62,15 @@ zen::ZenServerEnvironment TestEnv; int main(int argc, char** argv) { +# if ZEN_PLATFORM_WINDOWS + setlocale(LC_ALL, "en_us.UTF8"); +# endif // ZEN_PLATFORM_WINDOWS + using namespace std::literals; using namespace zen; + zen::CommandLineConverter ArgConverter(argc, argv); + # if ZEN_PLATFORM_LINUX IgnoreChildSignals(); # endif @@ -118,7 +125,7 @@ TEST_CASE("default.single") { std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); ZenServerInstance Instance(TestEnv); - Instance.SetTestDir(TestDir); + Instance.SetDataDir(TestDir); const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(); std::atomic<uint64_t> RequestCount{0}; @@ -168,7 +175,7 @@ TEST_CASE("default.loopback") std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); ZenServerInstance Instance(TestEnv); - Instance.SetTestDir(TestDir); + Instance.SetDataDir(TestDir); const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady("--http-forceloopback"); ZEN_INFO("Running loopback server test..."); @@ -196,12 +203,12 @@ TEST_CASE("multi.basic") { ZenServerInstance Instance1(TestEnv); std::filesystem::path TestDir1 = TestEnv.CreateNewTestDir(); - Instance1.SetTestDir(TestDir1); + Instance1.SetDataDir(TestDir1); Instance1.SpawnServer(); ZenServerInstance Instance2(TestEnv); std::filesystem::path TestDir2 = TestEnv.CreateNewTestDir(); - Instance2.SetTestDir(TestDir2); + Instance2.SetDataDir(TestDir2); Instance2.SpawnServer(); ZEN_INFO("Waiting..."); @@ -332,14 +339,14 @@ TEST_CASE("lifetime.owner") ZenServerInstance Zen1(TestEnv); std::filesystem::path TestDir1 = TestEnv.CreateNewTestDir(); - Zen1.SetTestDir(TestDir1); + Zen1.SetDataDir(TestDir1); Zen1.SpawnServer(PortNumber); Zen1.WaitUntilReady(); Zen1.Detach(); ZenServerInstance Zen2(TestEnv); std::filesystem::path TestDir2 = TestEnv.CreateNewTestDir(); - Zen2.SetTestDir(TestDir2); + Zen2.SetDataDir(TestDir2); Zen2.SpawnServer(PortNumber); Zen2.WaitUntilReady(); Zen2.Detach(); @@ -358,24 +365,24 @@ TEST_CASE("lifetime.owner.2") std::filesystem::path TestDir2 = TestEnv.CreateNewTestDir(); ZenServerInstance Zen1(TestEnv); - Zen1.SetTestDir(TestDir1); + Zen1.SetDataDir(TestDir1); Zen1.SpawnServer(PortNumber); Zen1.WaitUntilReady(); ZenServerInstance Zen2(TestEnv); - Zen2.SetTestDir(TestDir2); + Zen2.SetDataDir(TestDir2); Zen2.SetOwnerPid(Zen1.GetPid()); Zen2.SpawnServer(PortNumber + 1); Zen2.Detach(); ZenServerInstance Zen3(TestEnv); - Zen3.SetTestDir(TestDir2); + Zen3.SetDataDir(TestDir2); Zen3.SetOwnerPid(Zen1.GetPid()); Zen3.SpawnServer(PortNumber + 1); Zen3.Detach(); ZenServerInstance Zen4(TestEnv); - Zen4.SetTestDir(TestDir2); + Zen4.SetDataDir(TestDir2); Zen4.SetOwnerPid(Zen1.GetPid()); Zen4.SpawnServer(PortNumber + 1); Zen4.Detach(); diff --git a/src/zenserver-test/zenserver-test.h b/src/zenserver-test/zenserver-test.h index e7cee3f94..8d83285e6 100644 --- a/src/zenserver-test/zenserver-test.h +++ b/src/zenserver-test/zenserver-test.h @@ -82,7 +82,7 @@ namespace utils { void Spawn(ZenServerInstance& Inst) { - Inst.SetTestDir(DataDir); + Inst.SetDataDir(DataDir); Inst.SpawnServer(Port, Args); const uint16_t InstancePort = Inst.WaitUntilReady(); CHECK_MESSAGE(InstancePort != 0, Inst.GetLogOutput()); @@ -163,7 +163,7 @@ public: { auto& Instance = m_Instances[i]; Instance = std::make_unique<ZenServerInstance>(TestEnv); - Instance->SetTestDir(TestEnv.CreateNewTestDir()); + Instance->SetDataDir(TestEnv.CreateNewTestDir()); } for (int i = 0; i < m_ServerCount; ++i) diff --git a/src/zenserver/config/config.cpp b/src/zenserver/config/config.cpp index 18187711b..07913e891 100644 --- a/src/zenserver/config/config.cpp +++ b/src/zenserver/config/config.cpp @@ -132,12 +132,14 @@ ZenServerConfiguratorBase::AddCommonConfigOptions(LuaConfig::Options& LuaOptions LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv); LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv); + LuaOptions.AddOption("server.otlpendpoint"sv, ServerOptions.OtelEndpointUri, "otlp-endpoint"sv); LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv); LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv); LuaOptions.AddOption("server.quiet"sv, ServerOptions.QuietConsole, "quiet"sv); LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "noconsole"sv); ////// network + LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpConfig.ServerClass, "http"sv); LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpConfig.ThreadCount, "http-threads"sv); LuaOptions.AddOption("network.port"sv, ServerOptions.BasePort, "port"sv); @@ -249,17 +251,18 @@ ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerConfi // clang-format off options.add_options("logging") - ("abslog", "Path to log file", cxxopts::value<std::string>(AbsLogFile)) - ("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId)) - ("quiet", "Configure console logger output to level WARN", cxxopts::value<bool>(ServerOptions.QuietConsole)->default_value("false")) - ("noconsole", "Disable console logging", cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false")) - ("log-trace", "Change selected loggers to level TRACE", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace])) - ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug])) - ("log-info", "Change selected loggers to level INFO", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info])) - ("log-warn", "Change selected loggers to level WARN", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn])) - ("log-error", "Change selected loggers to level ERROR", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err])) - ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical])) - ("log-off", "Change selected loggers to level OFF", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off])) + ("abslog", "Path to log file", cxxopts::value<std::string>(AbsLogFile)) + ("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId)) + ("quiet", "Configure console logger output to level WARN", cxxopts::value<bool>(ServerOptions.QuietConsole)->default_value("false")) + ("noconsole", "Disable console logging", cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false")) + ("log-trace", "Change selected loggers to level TRACE", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace])) + ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug])) + ("log-info", "Change selected loggers to level INFO", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info])) + ("log-warn", "Change selected loggers to level WARN", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn])) + ("log-error", "Change selected loggers to level ERROR", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err])) + ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical])) + ("log-off", "Change selected loggers to level OFF", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off])) + ("otlp-endpoint", "OpenTelemetry endpoint URI (e.g http://localhost:4318)", cxxopts::value<std::string>(ServerOptions.OtelEndpointUri)) ; // clang-format on diff --git a/src/zenserver/config/config.h b/src/zenserver/config/config.h index 40639da13..7c3192a1f 100644 --- a/src/zenserver/config/config.h +++ b/src/zenserver/config/config.h @@ -64,6 +64,8 @@ struct ZenServerConfig std::string ChildId; // Id assigned by parent process (used for lifetime management) std::string LogId; // Id for tagging log output std::string Loggers[zen::logging::level::LogLevelCount]; + std::string OtelEndpointUri; // OpenTelemetry endpoint URI + #if ZEN_WITH_TRACE bool HasTraceCommandlineOptions = false; TraceOptions TraceCmdLineOptions; diff --git a/src/zenserver/diag/diagsvcs.cpp b/src/zenserver/diag/diagsvcs.cpp index 8abf6e8a3..d8d53b0e3 100644 --- a/src/zenserver/diag/diagsvcs.cpp +++ b/src/zenserver/diag/diagsvcs.cpp @@ -28,30 +28,6 @@ GetHealthTag() using namespace std::literals; -static bool -ReadLogFile(const std::string& Path, StringBuilderBase& Out) -{ - try - { - constexpr auto ReadSize = std::size_t{4096}; - auto FileStream = std::ifstream{Path}; - - std::string Buf(ReadSize, '\0'); - while (FileStream.read(&Buf[0], ReadSize)) - { - Out.Append(std::string_view(&Buf[0], FileStream.gcount())); - } - Out.Append(std::string_view(&Buf[0], FileStream.gcount())); - - return true; - } - catch (const std::exception&) - { - Out.Reset(); - return false; - } -} - HttpHealthService::HttpHealthService() { ZEN_MEMSCOPE(GetHealthTag()); @@ -95,10 +71,9 @@ HttpHealthService::HttpHealthService() return m_HealthInfo.AbsLogPath.empty() ? m_HealthInfo.DataRoot / "logs/zenserver.log" : m_HealthInfo.AbsLogPath; }(); - ExtendableStringBuilder<4096> Sb; - if (ReadLogFile(Path.string(), Sb) && Sb.Size() > 0) + if (IoBuffer LogBuffer = IoBufferBuilder::MakeFromFile(Path)) { - HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Sb.ToView()); + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, LogBuffer); } else { diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp index 80da240e8..4962b9006 100644 --- a/src/zenserver/diag/logging.cpp +++ b/src/zenserver/diag/logging.cpp @@ -78,12 +78,11 @@ InitializeServerLogging(const ZenServerConfig& InOptions, bool WithCacheService) spdlog::register_logger(ZenClientLogger); } - // - #if ZEN_WITH_OTEL - if (false) + if (!InOptions.OtelEndpointUri.empty()) { - auto OtelSink = std::make_shared<zen::logging::OtelHttpProtobufSink>("http://signoz.localdomain:4318"); + // TODO: Should sanity check that endpoint is reachable? Also, a valid URI? + auto OtelSink = std::make_shared<zen::logging::OtelHttpProtobufSink>(InOptions.OtelEndpointUri); zen::logging::Default().SpdLogger->sinks().push_back(std::move(OtelSink)); } #endif diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip Binary files differindex f9fc8a8ef..5d33302dd 100644 --- a/src/zenserver/frontend/html.zip +++ b/src/zenserver/frontend/html.zip diff --git a/src/zenserver/frontend/html/pages/oplog.js b/src/zenserver/frontend/html/pages/oplog.js index bef5bacce..879fc4c97 100644 --- a/src/zenserver/frontend/html/pages/oplog.js +++ b/src/zenserver/frontend/html/pages/oplog.js @@ -58,12 +58,12 @@ export class Page extends ZenPage { const nav = section.add_widget(Toolbar); const left = nav.left(); - left.add("|<") .on_click(() => this._on_next_prev(-10e10)); - left.add("<<").on_click(() => this._on_next_prev(-10)); - left.add("prev") .on_click(() => this._on_next_prev( -1)); - left.add("next") .on_click(() => this._on_next_prev( 1)); - left.add(">>").on_click(() => this._on_next_prev( 10)); - left.add(">|") .on_click(() => this._on_next_prev( 10e10)); + left.add("|<") .on_click(() => this._on_next_prev(-10e10)); + left.add("<<") .on_click(() => this._on_next_prev(-10)); + left.add("prev").on_click(() => this._on_next_prev( -1)); + left.add("next").on_click(() => this._on_next_prev( 1)); + left.add(">>") .on_click(() => this._on_next_prev( 10)); + left.add(">|") .on_click(() => this._on_next_prev( 10e10)); left.sep(); for (var count of [10, 25, 50, 100]) diff --git a/src/zenserver/frontend/html/pages/stat.js b/src/zenserver/frontend/html/pages/stat.js index c7902d5ed..d6c7fa8e8 100644 --- a/src/zenserver/frontend/html/pages/stat.js +++ b/src/zenserver/frontend/html/pages/stat.js @@ -38,7 +38,6 @@ class TemporalStat var content = ""; for (var i = 0; i < columns.length; ++i) { - content += "<pre>"; const column = columns[i]; for (var key in column) { @@ -51,13 +50,17 @@ class TemporalStat } else content += friendly(value); - content += "\n"; + content += "\r\n"; } - content += "</pre>"; } return content; } + + tag() + { + return "pre"; + } } //////////////////////////////////////////////////////////////////////////////// diff --git a/src/zenserver/frontend/html/util/component.js b/src/zenserver/frontend/html/util/component.js index 205aa038e..830c6989a 100644 --- a/src/zenserver/frontend/html/util/component.js +++ b/src/zenserver/frontend/html/util/component.js @@ -71,8 +71,11 @@ class ComponentDom extends ComponentBase text(value) { + if (value != undefined && typeof value.tag === "function") + this.tag(value.tag()) + value = (value == undefined) ? "undefined" : value.toString(); - this._element.innerHTML = (value != "") ? value : ""; + this._element.textContent = (value != "") ? value : ""; return this; } diff --git a/src/zenserver/frontend/html/zen.css b/src/zenserver/frontend/html/zen.css index c52609f52..cc53c0519 100644 --- a/src/zenserver/frontend/html/zen.css +++ b/src/zenserver/frontend/html/zen.css @@ -168,6 +168,7 @@ a { overflow: auto; overflow-wrap: break-word; background-color: inherit; + white-space: pre-wrap; } } diff --git a/src/zenserver/hub/README.md b/src/zenserver/hub/README.md new file mode 100644 index 000000000..322be3649 --- /dev/null +++ b/src/zenserver/hub/README.md @@ -0,0 +1,28 @@ +# Zen Server Hub + +The Zen Server can act in a "hub" mode. In this mode, the only services offered are the basic health +and diagnostic services alongside an API to provision and deprovision Storage server instances. + +## Generic Server API + +GET `/health` - returns an `OK!` payload when all enabled services are up and responding + +## Hub API + +GET `{moduleid}` - alphanumeric identifier to identify a dataset (typically associated with a content plug-in module) + +GET `/hub/status` - obtain a summary of the currently live instances + +GET `/hub/modules/{moduleid}` - retrieve information about a module + +POST `/hub/modules/{moduleid}/provision` - provision service for module + +POST `/hub/modules/{moduleid}/deprovision` - deprovision service for module + +GET `/hub/stats` - retrieve stats for service + +## Hub Configuration + +The hub service can use Consul to provide status updates + +The hub service can emit telemetry to an Open Telemetry collector diff --git a/src/zenserver/hub/hubservice.cpp b/src/zenserver/hub/hubservice.cpp new file mode 100644 index 000000000..4d9da3a57 --- /dev/null +++ b/src/zenserver/hub/hubservice.cpp @@ -0,0 +1,867 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "hubservice.h" + +#include "hydration.h" + +#include <zencore/compactbinarybuilder.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/scopeguard.h> +#include <zencore/system.h> +#include <zenutil/zenserverprocess.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <EASTL/fixed_vector.h> +#include <asio.hpp> +ZEN_THIRD_PARTY_INCLUDES_END + +#include <unordered_map> +#include <unordered_set> + +namespace zen { + +/////////////////////////////////////////////////////////////////////////// + +/** + * A timeline of events with sequence IDs and timestamps. Used to + * track significant events for broadcasting to listeners. + */ +class EventTimeline +{ +public: + EventTimeline() { m_Events.reserve(1024); } + + ~EventTimeline() {} + + EventTimeline(const EventTimeline&) = delete; + EventTimeline& operator=(const EventTimeline&) = delete; + + void RecordEvent(std::string_view EventTag, CbObject EventMetadata) + { + const uint64_t SequenceId = m_NextEventId++; + const auto Now = std::chrono::steady_clock::now(); + RwLock::ExclusiveLockScope _(m_Lock); + m_Events.emplace_back(SequenceId, EventTag, Now, std::move(EventMetadata)); + } + + struct EventRecord + { + uint64_t SequenceId; + std::string Tag; + std::chrono::steady_clock::time_point Timestamp; + CbObject EventMetadata; + + EventRecord(uint64_t InSequenceId, + std::string_view InTag, + std::chrono::steady_clock::time_point InTimestamp, + CbObject InEventMetadata = CbObject()) + : SequenceId(InSequenceId) + , Tag(InTag) + , Timestamp(InTimestamp) + , EventMetadata(InEventMetadata) + { + } + }; + + /** + * Iterate over events that have a SequenceId greater than SinceEventId + * + * @param Callback A callable that takes a const EventRecord& + * @param SinceEventId The SequenceId to compare against + */ + void IterateEventsSince(auto&& Callback, uint64_t SinceEventId) + { + // Hold the lock for as short a time as possible + eastl::fixed_vector<EventRecord, 128> EventsToProcess; + m_Lock.WithSharedLock([&] { + for (auto& Event : m_Events) + { + if (Event.SequenceId > SinceEventId) + { + EventsToProcess.push_back(Event); + } + } + }); + + // Now invoke the callback outside the lock + for (auto& Event : EventsToProcess) + { + Callback(Event); + } + } + + /** + * Trim events up to (and including) the given SequenceId. Intended + * to be used for cleaning up events which are not longer interesting. + * + * @param UpToEventId The SequenceId up to which events should be removed + */ + void TrimEventsUpTo(uint64_t UpToEventId) + { + RwLock::ExclusiveLockScope _(m_Lock); + auto It = std::remove_if(m_Events.begin(), m_Events.end(), [UpToEventId](const EventRecord& Event) { + return Event.SequenceId <= UpToEventId; + }); + m_Events.erase(It, m_Events.end()); + } + +private: + std::atomic<uint64_t> m_NextEventId{0}; + + RwLock m_Lock; + std::vector<EventRecord> m_Events; +}; + +////////////////////////////////////////////////////////////////////////// + +struct ResourceMetrics +{ + uint64_t DiskUsageBytes = 0; + uint64_t MemoryUsageBytes = 0; +}; + +/** + * Storage Server Instance + * + * This class manages the lifecycle of a storage server instance, and + * provides functions to query its state. There should be one instance + * per module ID. + */ +struct StorageServerInstance +{ + StorageServerInstance(ZenServerEnvironment& RunEnvironment, + std::string_view ModuleId, + std::filesystem::path FileHydrationPath, + std::filesystem::path HydrationTempPath); + ~StorageServerInstance(); + + void Provision(); + void Deprovision(); + + void Hibernate(); + void Wake(); + + const ResourceMetrics& GetResourceMetrics() const { return m_ResourceMetrics; } + + inline std::string_view GetModuleId() const { return m_ModuleId; } + inline bool IsProvisioned() const { return m_IsProvisioned.load(); } + + inline uint16_t GetBasePort() const { return m_ServerInstance.GetBasePort(); } + +private: + RwLock m_Lock; + std::string m_ModuleId; + std::atomic<bool> m_IsProvisioned{false}; + std::atomic<bool> m_IsHibernated{false}; + ZenServerInstance m_ServerInstance; + std::filesystem::path m_BaseDir; + std::filesystem::path m_TempDir; + std::filesystem::path m_HydrationPath; + ResourceMetrics m_ResourceMetrics; + + void SpawnServerProcess(); + + void Hydrate(); + void Dehydrate(); +}; + +StorageServerInstance::StorageServerInstance(ZenServerEnvironment& RunEnvironment, + std::string_view ModuleId, + std::filesystem::path FileHydrationPath, + std::filesystem::path HydrationTempPath) +: m_ModuleId(ModuleId) +, m_ServerInstance(RunEnvironment, ZenServerInstance::ServerMode::kStorageServer) +, m_HydrationPath(FileHydrationPath) +{ + m_BaseDir = RunEnvironment.CreateChildDir(ModuleId); + m_TempDir = HydrationTempPath / ModuleId; +} + +StorageServerInstance::~StorageServerInstance() +{ +} + +void +StorageServerInstance::SpawnServerProcess() +{ + ZEN_ASSERT(!m_ServerInstance.IsRunning(), "Storage server instance for module '{}' is already running", m_ModuleId); + + m_ServerInstance.SetServerExecutablePath(GetRunningExecutablePath()); + m_ServerInstance.SetDataDir(m_BaseDir); + const uint16_t BasePort = m_ServerInstance.SpawnServerAndWaitUntilReady(); + + ZEN_DEBUG("Storage server instance for module '{}' started, listening on port {}", m_ModuleId, BasePort); + + m_ServerInstance.EnableShutdownOnDestroy(); +} + +void +StorageServerInstance::Provision() +{ + RwLock::ExclusiveLockScope _(m_Lock); + + if (m_IsProvisioned) + { + ZEN_WARN("Storage server instance for module '{}' is already provisioned", m_ModuleId); + + return; + } + + if (m_IsHibernated) + { + Wake(); + } + else + { + ZEN_INFO("Provisioning storage server instance for module '{}', at '{}'", m_ModuleId, m_BaseDir); + + Hydrate(); + + SpawnServerProcess(); + } + + m_IsProvisioned = true; +} + +void +StorageServerInstance::Deprovision() +{ + RwLock::ExclusiveLockScope _(m_Lock); + + if (!m_IsProvisioned) + { + ZEN_WARN("Attempted to deprovision storage server instance for module '{}' which is not provisioned", m_ModuleId); + + return; + } + + ZEN_INFO("Deprovisioning storage server instance for module '{}'", m_ModuleId); + + m_ServerInstance.Shutdown(); + + Dehydrate(); + + m_IsProvisioned = false; +} + +void +StorageServerInstance::Hibernate() +{ + // Signal server to shut down, but keep data around for later wake + + RwLock::ExclusiveLockScope _(m_Lock); + + if (!m_IsProvisioned) + { + ZEN_WARN("Attempted to hibernate storage server instance for module '{}' which is not provisioned", m_ModuleId); + + return; + } + + if (m_IsHibernated) + { + ZEN_WARN("Storage server instance for module '{}' is already hibernated", m_ModuleId); + + return; + } + + if (!m_ServerInstance.IsRunning()) + { + ZEN_WARN("Attempted to hibernate storage server instance for module '{}' which is not running", m_ModuleId); + + // This is an unexpected state. Should consider the instance invalid? + + return; + } + + try + { + m_ServerInstance.Shutdown(); + + m_IsHibernated = true; + m_IsProvisioned = false; + + return; + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Failed to hibernate storage server instance for module '{}': {}", m_ModuleId, Ex.what()); + } +} + +void +StorageServerInstance::Wake() +{ + // Start server in-place using existing data + + RwLock::ExclusiveLockScope _(m_Lock); + + if (!m_IsHibernated) + { + ZEN_WARN("Attempted to wake storage server instance for module '{}' which is not hibernated", m_ModuleId); + + return; + } + + ZEN_ASSERT(!m_ServerInstance.IsRunning(), "Storage server instance for module '{}' is already running", m_ModuleId); + + try + { + SpawnServerProcess(); + m_IsHibernated = false; + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Failed to wake storage server instance for module '{}': {}", m_ModuleId, Ex.what()); + + // TODO: this instance should be marked as invalid + } +} + +void +StorageServerInstance::Hydrate() +{ + HydrationConfig Config{.ServerStateDir = m_BaseDir, + .TempDir = m_TempDir, + .ModuleId = m_ModuleId, + .TargetSpecification = WideToUtf8(m_HydrationPath.native())}; + + std::unique_ptr<HydrationStrategyBase> Hydrator = CreateFileHydrator(); + + Hydrator->Configure(Config); + Hydrator->Hydrate(); +} + +void +StorageServerInstance::Dehydrate() +{ + HydrationConfig Config{.ServerStateDir = m_BaseDir, + .TempDir = m_TempDir, + .ModuleId = m_ModuleId, + .TargetSpecification = WideToUtf8(m_HydrationPath.native())}; + + std::unique_ptr<HydrationStrategyBase> Hydrator = CreateFileHydrator(); + + Hydrator->Configure(Config); + Hydrator->Dehydrate(); +} + +////////////////////////////////////////////////////////////////////////// + +struct HttpHubService::Impl +{ + Impl(const Impl&) = delete; + Impl& operator=(const Impl&) = delete; + + Impl(); + ~Impl(); + + void Initialize(std::filesystem::path HubBaseDir, std::filesystem::path ChildBaseDir) + { + m_RunEnvironment.InitializeForHub(HubBaseDir, ChildBaseDir); + m_FileHydrationPath = m_RunEnvironment.CreateChildDir("hydration_storage"); + ZEN_INFO("using file hydration path: '{}'", m_FileHydrationPath); + + m_HydrationTempPath = m_RunEnvironment.CreateChildDir("hydration_temp"); + ZEN_INFO("using hydration temp path: '{}'", m_HydrationTempPath); + + // This is necessary to ensure the hub assigns a distinct port range. + // We need to do this primarily because otherwise automated tests will + // fail as the test runner will create processes in the default range. + // We should probably make this configurable or dynamic for maximum + // flexibility, and to allow running multiple hubs on the same host if + // necessary. + m_RunEnvironment.SetNextPortNumber(21000); + } + + void Cleanup() + { + RwLock::ExclusiveLockScope _(m_Lock); + m_Instances.clear(); + } + + struct ProvisionedInstanceInfo + { + std::string BaseUri; + uint16_t Port; + }; + + /** + * Provision a storage server instance for the given module ID. + * + * @param ModuleId The ID of the module to provision. + * @param OutInfo If successful, information about the provisioned instance will be returned here. + * @param OutReason If unsuccessful, the reason will be returned here. + */ + bool Provision(std::string_view ModuleId, ProvisionedInstanceInfo& OutInfo, std::string& OutReason) + { + StorageServerInstance* Instance = nullptr; + bool IsNewInstance = false; + { + RwLock::ExclusiveLockScope _(m_Lock); + if (auto It = m_Instances.find(std::string(ModuleId)); It == m_Instances.end()) + { + std::string Reason; + if (!CanProvisionInstance(ModuleId, /* out */ Reason)) + { + ZEN_WARN("Cannot provision new storage server instance for module '{}': {}", ModuleId, Reason); + + OutReason = Reason; + + return false; + } + + IsNewInstance = true; + auto NewInstance = + std::make_unique<StorageServerInstance>(m_RunEnvironment, ModuleId, m_FileHydrationPath, m_HydrationTempPath); + Instance = NewInstance.get(); + m_Instances.emplace(std::string(ModuleId), std::move(NewInstance)); + + ZEN_INFO("Created new storage server instance for module '{}'", ModuleId); + } + else + { + Instance = It->second.get(); + } + + m_ProvisioningModules.emplace(std::string(ModuleId)); + } + + ZEN_ASSERT(Instance != nullptr); + + auto RemoveProvisioningModule = MakeGuard([&] { + RwLock::ExclusiveLockScope _(m_Lock); + m_ProvisioningModules.erase(std::string(ModuleId)); + }); + + // NOTE: this is done while not holding the lock, as provisioning may take time + // and we don't want to block other operations. We track which modules are being + // provisioned using m_ProvisioningModules, and reject attempts to provision/deprovision + // those modules while in this state. + + UpdateStats(); + + try + { + Instance->Provision(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Failed to provision storage server instance for module '{}': {}", ModuleId, Ex.what()); + if (IsNewInstance) + { + // Clean up + RwLock::ExclusiveLockScope _(m_Lock); + m_Instances.erase(std::string(ModuleId)); + } + return false; + } + + OutInfo.Port = Instance->GetBasePort(); + + // TODO: base URI? Would need to know what host name / IP to use + + return true; + } + + /** + * Deprovision a storage server instance for the given module ID. + * + * @param ModuleId The ID of the module to deprovision. + * @param OutReason If unsuccessful, the reason will be returned here. + * @return true if the instance was found and deprovisioned, false otherwise. + */ + bool Deprovision(const std::string& ModuleId, std::string& OutReason) + { + std::unique_ptr<StorageServerInstance> Instance; + + { + RwLock::ExclusiveLockScope _(m_Lock); + + if (auto It = m_ProvisioningModules.find(ModuleId); It != m_ProvisioningModules.end()) + { + OutReason = fmt::format("Module '{}' is currently being provisioned", ModuleId); + + ZEN_WARN("Attempted to deprovision module '{}' which is currently being provisioned", ModuleId); + + return false; + } + + if (auto It = m_Instances.find(ModuleId); It == m_Instances.end()) + { + ZEN_WARN("Attempted to deprovision non-existent module '{}'", ModuleId); + + // Not found, OutReason should be empty + return false; + } + else + { + Instance = std::move(It->second); + m_Instances.erase(It); + m_DeprovisioningModules.emplace(ModuleId); + } + } + + // The module is deprovisioned outside the lock to avoid blocking other operations. + // + // To ensure that no new provisioning can occur while we're deprovisioning, + // we add the module ID to m_DeprovisioningModules and remove it once + // deprovisioning is complete. + + auto _ = MakeGuard([&] { + RwLock::ExclusiveLockScope _(m_Lock); + m_DeprovisioningModules.erase(ModuleId); + }); + + Instance->Deprovision(); + + return true; + } + + /** + * Find a storage server instance for the given module ID. + * + * Beware that as this returns a raw pointer to the instance, the caller must ensure + * that the instance is not deprovisioned while in use. + * + * @param ModuleId The ID of the module to find. + * @param OutInstance If found, the instance will be returned here. + * @return true if the instance was found, false otherwise. + */ + bool Find(std::string_view ModuleId, StorageServerInstance** OutInstance = nullptr) + { + RwLock::SharedLockScope _(m_Lock); + if (auto It = m_Instances.find(std::string(ModuleId)); It != m_Instances.end()) + { + if (OutInstance) + { + *OutInstance = It->second.get(); + } + return true; + } + else if (OutInstance) + { + *OutInstance = nullptr; + } + return false; + } + + /** + * Enumerate all storage server instances. + * + * @param Callback The callback to invoke for each instance. Note that you should + * not do anything heavyweight in the callback as it is invoked while holding + * a shared lock. + */ + void EnumerateModules(auto&& Callback) + { + RwLock::SharedLockScope _(m_Lock); + for (auto& It : m_Instances) + { + Callback(*It.second); + } + } + + int GetInstanceCount() + { + RwLock::SharedLockScope _(m_Lock); + return gsl::narrow_cast<int>(m_Instances.size()); + } + + inline int GetInstanceLimit() { return m_InstanceLimit; } + inline int GetMaxInstanceCount() { return m_MaxInstanceCount; } + +private: + ZenServerEnvironment m_RunEnvironment; + std::filesystem::path m_FileHydrationPath; + std::filesystem::path m_HydrationTempPath; + RwLock m_Lock; + std::unordered_map<std::string, std::unique_ptr<StorageServerInstance>> m_Instances; + std::unordered_set<std::string> m_DeprovisioningModules; + std::unordered_set<std::string> m_ProvisioningModules; + int m_MaxInstanceCount = 0; + void UpdateStats(); + + // Capacity tracking + + int m_InstanceLimit = 1000; + ResourceMetrics m_ResourceLimits; + SystemMetrics m_HostMetrics; + + void UpdateCapacityMetrics(); + bool CanProvisionInstance(std::string_view ModuleId, std::string& OutReason); +}; + +HttpHubService::Impl::Impl() +{ + m_HostMetrics = zen::GetSystemMetrics(); + m_ResourceLimits.DiskUsageBytes = 1000ull * 1024 * 1024 * 1024; + m_ResourceLimits.MemoryUsageBytes = 16ull * 1024 * 1024 * 1024; +} + +HttpHubService::Impl::~Impl() +{ + try + { + ZEN_INFO("Hub service shutting down, deprovisioning any current instances"); + + m_Lock.WithExclusiveLock([this] { + for (auto& [ModuleId, Instance] : m_Instances) + { + Instance->Deprovision(); + } + m_Instances.clear(); + }); + } + catch (const std::exception& e) + { + ZEN_WARN("Exception during hub service shutdown: {}", e.what()); + } +} + +void +HttpHubService::Impl::UpdateCapacityMetrics() +{ + m_HostMetrics = zen::GetSystemMetrics(); + + // Update per-instance metrics +} + +void +HttpHubService::Impl::UpdateStats() +{ + m_Lock.WithSharedLock([this] { m_MaxInstanceCount = Max(m_MaxInstanceCount, gsl::narrow_cast<int>(m_Instances.size())); }); +} + +bool +HttpHubService::Impl::CanProvisionInstance(std::string_view ModuleId, std::string& OutReason) +{ + if (m_DeprovisioningModules.find(std::string(ModuleId)) != m_DeprovisioningModules.end()) + { + OutReason = fmt::format("module '{}' is currently being deprovisioned", ModuleId); + + return false; + } + + if (m_ProvisioningModules.find(std::string(ModuleId)) != m_ProvisioningModules.end()) + { + OutReason = fmt::format("module '{}' is currently being provisioned", ModuleId); + + return false; + } + + if (gsl::narrow_cast<int>(m_Instances.size()) >= m_InstanceLimit) + { + OutReason = fmt::format("instance limit exceeded ({})", m_InstanceLimit); + + return false; + } + + // TODO: handle additional resource metrics + + return true; +} + +/////////////////////////////////////////////////////////////////////////// + +HttpHubService::HttpHubService(std::filesystem::path HubBaseDir, std::filesystem::path ChildBaseDir) : m_Impl(std::make_unique<Impl>()) +{ + using namespace std::literals; + + m_Impl->Initialize(HubBaseDir, ChildBaseDir); + + m_Router.AddMatcher("moduleid", [](std::string_view Str) -> bool { + for (const auto C : Str) + { + if (std::isalnum(C) || C == '-') + { + // fine + } + else + { + // not fine + return false; + } + } + + return true; + }); + + m_Router.RegisterRoute( + "status", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + Obj.BeginArray("modules"); + m_Impl->EnumerateModules([&Obj](StorageServerInstance& Instance) { + Obj.BeginObject(); + Obj << "moduleId" << Instance.GetModuleId(); + Obj << "provisioned" << Instance.IsProvisioned(); + Obj.EndObject(); + }); + Obj.EndArray(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "modules/{moduleid}", + [this](HttpRouterRequest& Req) { + std::string_view ModuleId = Req.GetCapture(1); + + if (Req.ServerRequest().RequestVerb() == HttpVerb::kDelete) + { + HandleModuleDelete(Req.ServerRequest(), ModuleId); + } + else + { + HandleModuleGet(Req.ServerRequest(), ModuleId); + } + }, + HttpVerb::kGet | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "modules/{moduleid}/provision", + [this](HttpRouterRequest& Req) { + std::string_view ModuleId = Req.GetCapture(1); + + std::string FailureReason = "unknown"; + HttpResponseCode ResponseCode = HttpResponseCode::OK; + + try + { + Impl::ProvisionedInstanceInfo Info; + if (m_Impl->Provision(ModuleId, /* out */ Info, /* out */ FailureReason)) + { + CbObjectWriter Obj; + Obj << "moduleId" << ModuleId; + Obj << "baseUri" << Info.BaseUri; + Obj << "port" << Info.Port; + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + + return; + } + else + { + ResponseCode = HttpResponseCode::BadRequest; + } + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Exception while provisioning module '{}': {}", ModuleId, Ex.what()); + + FailureReason = Ex.what(); + ResponseCode = HttpResponseCode::InternalServerError; + } + + Req.ServerRequest().WriteResponse(ResponseCode, HttpContentType::kText, FailureReason); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "modules/{moduleid}/deprovision", + [this](HttpRouterRequest& Req) { + std::string_view ModuleId = Req.GetCapture(1); + std::string FailureReason = "unknown"; + + try + { + if (!m_Impl->Deprovision(std::string(ModuleId), /* out */ FailureReason)) + { + if (FailureReason.empty()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound); + } + else + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, FailureReason); + } + } + + CbObjectWriter Obj; + Obj << "moduleId" << ModuleId; + + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Exception while deprovisioning module '{}': {}", ModuleId, Ex.what()); + + FailureReason = Ex.what(); + } + + Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, FailureReason); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "stats", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + Obj << "currentInstanceCount" << m_Impl->GetInstanceCount(); + Obj << "maxInstanceCount" << m_Impl->GetMaxInstanceCount(); + Obj << "instanceLimit" << m_Impl->GetInstanceLimit(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK); + }, + HttpVerb::kGet); +} + +HttpHubService::~HttpHubService() +{ +} + +const char* +HttpHubService::BaseUri() const +{ + return "/hub/"; +} + +void +HttpHubService::SetNotificationEndpoint(std::string_view UpstreamNotificationEndpoint, std::string_view InstanceId) +{ + ZEN_UNUSED(UpstreamNotificationEndpoint, InstanceId); + // TODO: store these for use in notifications, on some interval/criteria which is currently TBD +} + +void +HttpHubService::HandleRequest(zen::HttpServerRequest& Request) +{ + m_Router.HandleRequest(Request); +} + +void +HttpHubService::HandleModuleGet(HttpServerRequest& Request, std::string_view ModuleId) +{ + StorageServerInstance* Instance = nullptr; + if (!m_Impl->Find(ModuleId, &Instance)) + { + Request.WriteResponse(HttpResponseCode::NotFound); + return; + } + + CbObjectWriter Obj; + Obj << "moduleId" << Instance->GetModuleId(); + Obj << "provisioned" << Instance->IsProvisioned(); + Request.WriteResponse(HttpResponseCode::OK, Obj.Save()); +} + +void +HttpHubService::HandleModuleDelete(HttpServerRequest& Request, std::string_view ModuleId) +{ + StorageServerInstance* Instance = nullptr; + if (!m_Impl->Find(ModuleId, &Instance)) + { + Request.WriteResponse(HttpResponseCode::NotFound); + return; + } + + // TODO: deprovision and nuke all related storage + + CbObjectWriter Obj; + Obj << "moduleId" << Instance->GetModuleId(); + Obj << "provisioned" << Instance->IsProvisioned(); + Request.WriteResponse(HttpResponseCode::OK, Obj.Save()); +} + +} // namespace zen diff --git a/src/zenserver/hub/hubservice.h b/src/zenserver/hub/hubservice.h new file mode 100644 index 000000000..1a5a8c57c --- /dev/null +++ b/src/zenserver/hub/hubservice.h @@ -0,0 +1,42 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenhttp/httpserver.h> + +#include "hydration.h" + +namespace zen { + +/** ZenServer Hub Service + * + * Manages a set of storage servers on the behalf of external clients. For + * use in UEFN content worker style scenarios. + * + */ +class HttpHubService : public zen::HttpService +{ +public: + HttpHubService(std::filesystem::path HubBaseDir, std::filesystem::path ChildBaseDir); + ~HttpHubService(); + + HttpHubService(const HttpHubService&) = delete; + HttpHubService& operator=(const HttpHubService&) = delete; + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + + void SetNotificationEndpoint(std::string_view UpstreamNotificationEndpoint, std::string_view InstanceId); + +private: + HttpRequestRouter m_Router; + + struct Impl; + + std::unique_ptr<Impl> m_Impl; + + void HandleModuleGet(HttpServerRequest& Request, std::string_view ModuleId); + void HandleModuleDelete(HttpServerRequest& Request, std::string_view ModuleId); +}; + +} // namespace zen diff --git a/src/zenserver/hub/hydration.cpp b/src/zenserver/hub/hydration.cpp new file mode 100644 index 000000000..52c17fe1a --- /dev/null +++ b/src/zenserver/hub/hydration.cpp @@ -0,0 +1,119 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "hydration.h" + +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> + +namespace zen { + +/////////////////////////////////////////////////////////////////////////// + +struct FileHydrator : public HydrationStrategyBase +{ + virtual void Configure(const HydrationConfig& Config) override; + virtual void Hydrate() override; + virtual void Dehydrate() override; + +private: + HydrationConfig m_Config; + std::filesystem::path m_StorageModuleRootDir; +}; + +void +FileHydrator::Configure(const HydrationConfig& Config) +{ + m_Config = Config; + + std::filesystem::path ConfigPath(Utf8ToWide(m_Config.TargetSpecification)); + + if (!std::filesystem::exists(ConfigPath)) + { + throw std::invalid_argument(fmt::format("Target does not exist: '{}'", ConfigPath.string())); + } + + m_StorageModuleRootDir = ConfigPath / m_Config.ModuleId; + + CreateDirectories(m_StorageModuleRootDir); +} + +void +FileHydrator::Hydrate() +{ + ZEN_INFO("Hydrating state from '{}' to '{}'", m_StorageModuleRootDir, m_Config.ServerStateDir); + + // Ensure target is clean + ZEN_DEBUG("Wiping server state at '{}'", m_Config.ServerStateDir); + const bool ForceRemoveReadOnlyFiles = true; + CleanDirectory(m_Config.ServerStateDir, ForceRemoveReadOnlyFiles); + + bool WipeServerState = false; + + try + { + ZEN_DEBUG("Copying '{}' to '{}'", m_StorageModuleRootDir, m_Config.ServerStateDir); + CopyTree(m_StorageModuleRootDir, m_Config.ServerStateDir, {.EnableClone = true}); + } + catch (std::exception& Ex) + { + ZEN_WARN("Copy failed: {}. Will wipe any partially copied state from '{}'", Ex.what(), m_Config.ServerStateDir); + + // We don't do the clean right here to avoid potentially running into double-throws + WipeServerState = true; + } + + if (WipeServerState) + { + ZEN_DEBUG("Cleaning server state '{}'", m_Config.ServerStateDir); + CleanDirectory(m_Config.ServerStateDir, ForceRemoveReadOnlyFiles); + } + + // Note that we leave the storage state intact until next dehydration replaces the content +} + +void +FileHydrator::Dehydrate() +{ + ZEN_INFO("Dehydrating state from '{}' to '{}'", m_Config.ServerStateDir, m_StorageModuleRootDir); + + const std::filesystem::path TargetDir = m_StorageModuleRootDir; + + // Ensure target is clean. This could be replaced with an atomic copy at a later date + // (i.e copy into a temporary directory name and rename it once complete) + + ZEN_DEBUG("Cleaning storage root '{}'", TargetDir); + const bool ForceRemoveReadOnlyFiles = true; + CleanDirectory(TargetDir, ForceRemoveReadOnlyFiles); + + bool CopySuccess = true; + + try + { + ZEN_DEBUG("Copying '{}' to '{}'", m_Config.ServerStateDir, TargetDir); + CopyTree(m_Config.ServerStateDir, TargetDir, {.EnableClone = true}); + } + catch (std::exception& Ex) + { + ZEN_WARN("Copy failed: {}. Will wipe any partially copied state from '{}'", Ex.what(), m_StorageModuleRootDir); + + // We don't do the clean right here to avoid potentially running into double-throws + CopySuccess = false; + } + + if (!CopySuccess) + { + ZEN_DEBUG("Removing partially copied state from '{}'", TargetDir); + CleanDirectory(TargetDir, ForceRemoveReadOnlyFiles); + } + + ZEN_DEBUG("Wiping server state '{}'", m_Config.ServerStateDir); + CleanDirectory(m_Config.ServerStateDir, ForceRemoveReadOnlyFiles); +} + +std::unique_ptr<HydrationStrategyBase> +CreateFileHydrator() +{ + return std::make_unique<FileHydrator>(); +} + +} // namespace zen diff --git a/src/zenserver/hub/hydration.h b/src/zenserver/hub/hydration.h new file mode 100644 index 000000000..f86f2accf --- /dev/null +++ b/src/zenserver/hub/hydration.h @@ -0,0 +1,40 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zenhubserver.h" + +namespace zen { + +struct HydrationConfig +{ + // Location of server state to hydrate/dehydrate + std::filesystem::path ServerStateDir; + // Temporary directory available for use during hydration/dehydration + std::filesystem::path TempDir; + // Module ID of the server state being hydrated/dehydrated + std::string ModuleId; + // Back-end specific target specification (e.g. S3 bucket, file path, etc) + std::string TargetSpecification; +}; + +/** + * @brief State hydration strategy interface + * + * An instance of this interface is used to perform hydration OR + * dehydration of server state. It's expected to be used only once + * and not reused. + * + */ +struct HydrationStrategyBase +{ + virtual ~HydrationStrategyBase() = default; + + virtual void Dehydrate() = 0; + virtual void Hydrate() = 0; + virtual void Configure(const HydrationConfig& Config) = 0; +}; + +std::unique_ptr<HydrationStrategyBase> CreateFileHydrator(); + +} // namespace zen diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp new file mode 100644 index 000000000..7a4ba951d --- /dev/null +++ b/src/zenserver/hub/zenhubserver.cpp @@ -0,0 +1,303 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zenhubserver.h" +#include "hubservice.h" + +#include <zencore/fmtutils.h> +#include <zencore/memory/llm.h> +#include <zencore/memory/memorytrace.h> +#include <zencore/memory/tagtrace.h> +#include <zencore/scopeguard.h> +#include <zencore/sentryintegration.h> +#include <zencore/system.h> +#include <zencore/windows.h> +#include <zenhttp/httpapiservice.h> +#include <zenutil/service.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <cxxopts.hpp> +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +void +ZenHubServerConfigurator::AddCliOptions(cxxopts::Options& Options) +{ + Options.add_option("hub", + "", + "upstream-notification-endpoint", + "Endpoint URL for upstream notifications", + cxxopts::value<std::string>(m_ServerOptions.UpstreamNotificationEndpoint)->default_value(""), + ""); + + Options.add_option("hub", + "", + "instance-id", + "Instance ID for use in notifications", + cxxopts::value<std::string>(m_ServerOptions.InstanceId)->default_value(""), + ""); +} + +void +ZenHubServerConfigurator::AddConfigOptions(LuaConfig::Options& Options) +{ + ZEN_UNUSED(Options); +} + +void +ZenHubServerConfigurator::ApplyOptions(cxxopts::Options& Options) +{ + ZEN_UNUSED(Options); +} + +void +ZenHubServerConfigurator::OnConfigFileParsed(LuaConfig::Options& LuaOptions) +{ + ZEN_UNUSED(LuaOptions); +} + +void +ZenHubServerConfigurator::ValidateOptions() +{ +} + +/////////////////////////////////////////////////////////////////////////// + +ZenHubServer::ZenHubServer() +{ +} + +ZenHubServer::~ZenHubServer() +{ + Cleanup(); +} + +int +ZenHubServer::Initialize(const ZenHubServerConfig& ServerConfig, ZenServerState::ZenServerEntry* ServerEntry) +{ + ZEN_TRACE_CPU("ZenHubServer::Initialize"); + ZEN_MEMSCOPE(GetZenserverTag()); + + ZEN_INFO(ZEN_APP_NAME " initializing in HUB server mode"); + + const int EffectiveBasePort = ZenServerBase::Initialize(ServerConfig, ServerEntry); + if (EffectiveBasePort < 0) + { + return EffectiveBasePort; + } + + // This is a workaround to make sure we can have automated tests. Without + // this the ranges for different child zen hub processes could overlap with + // the main test range. + ZenServerEnvironment::SetBaseChildId(1000); + + m_DebugOptionForcedCrash = ServerConfig.ShouldCrash; + + InitializeState(ServerConfig); + InitializeServices(ServerConfig); + RegisterServices(ServerConfig); + + ZenServerBase::Finalize(); + + return EffectiveBasePort; +} + +void +ZenHubServer::Cleanup() +{ + ZEN_TRACE_CPU("ZenStorageServer::Cleanup"); + ZEN_INFO(ZEN_APP_NAME " cleaning up"); + try + { + m_IoContext.stop(); + if (m_IoRunner.joinable()) + { + m_IoRunner.join(); + } + + if (m_Http) + { + m_Http->Close(); + } + } + catch (const std::exception& Ex) + { + ZEN_ERROR("exception thrown during Cleanup() in {}: '{}'", ZEN_APP_NAME, Ex.what()); + } +} + +void +ZenHubServer::InitializeState(const ZenHubServerConfig& ServerConfig) +{ + ZEN_UNUSED(ServerConfig); +} + +void +ZenHubServer::InitializeServices(const ZenHubServerConfig& ServerConfig) +{ + ZEN_UNUSED(ServerConfig); + + ZEN_INFO("instantiating API service"); + m_ApiService = std::make_unique<zen::HttpApiService>(*m_Http); + + ZEN_INFO("instantiating hub service"); + m_HubService = std::make_unique<HttpHubService>(ServerConfig.DataDir / "hub", ServerConfig.DataDir / "servers"); + m_HubService->SetNotificationEndpoint(ServerConfig.UpstreamNotificationEndpoint, ServerConfig.InstanceId); +} + +void +ZenHubServer::RegisterServices(const ZenHubServerConfig& ServerConfig) +{ + ZEN_UNUSED(ServerConfig); + + if (m_HubService) + { + m_Http->RegisterService(*m_HubService); + } + + if (m_ApiService) + { + m_Http->RegisterService(*m_ApiService); + } +} + +void +ZenHubServer::Run() +{ + if (m_ProcessMonitor.IsActive()) + { + CheckOwnerPid(); + } + + if (!m_TestMode) + { + // clang-format off + ZEN_INFO(R"(__________ ___ ___ ___. )" "\n" + R"(\____ /____ ____ / | \ __ _\_ |__ )" "\n" + R"( / // __ \ / \ / ~ \ | \ __ \ )" "\n" + R"( / /\ ___/| | \ \ Y / | / \_\ \)" "\n" + R"(/_______ \___ >___| / \___|_ /|____/|___ /)" "\n" + R"( \/ \/ \/ \/ \/ )"); + // clang-format on + + ExtendableStringBuilder<256> BuildOptions; + GetBuildOptions(BuildOptions, '\n'); + ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions); + } + + ZEN_INFO(ZEN_APP_NAME " now running as HUB (pid: {})", GetCurrentProcessId()); + +#if ZEN_PLATFORM_WINDOWS + if (zen::windows::IsRunningOnWine()) + { + ZEN_INFO("detected Wine session - " ZEN_APP_NAME " is not formally tested on Wine and may therefore not work or perform well"); + } +#endif + +#if ZEN_USE_SENTRY + ZEN_INFO("sentry crash handler {}", m_UseSentry ? "ENABLED" : "DISABLED"); + if (m_UseSentry) + { + SentryIntegration::ClearCaches(); + } +#endif + + if (m_DebugOptionForcedCrash) + { + ZEN_DEBUG_BREAK(); + } + + const bool IsInteractiveMode = IsInteractiveSession(); // &&!m_TestMode; + + SetNewState(kRunning); + + OnReady(); + + m_Http->Run(IsInteractiveMode); + + SetNewState(kShuttingDown); + + ZEN_INFO(ZEN_APP_NAME " exiting"); +} + +////////////////////////////////////////////////////////////////////////////////// + +ZenHubServerMain::ZenHubServerMain(ZenHubServerConfig& ServerOptions) : ZenServerMain(ServerOptions), m_ServerOptions(ServerOptions) +{ +} + +void +ZenHubServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) +{ + ZenHubServer Server; + Server.SetDataRoot(m_ServerOptions.DataDir); + Server.SetContentRoot(m_ServerOptions.ContentDir); + Server.SetTestMode(m_ServerOptions.IsTest); + Server.SetDedicatedMode(m_ServerOptions.IsDedicated); + + const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry); + if (EffectiveBasePort == -1) + { + // Server.Initialize has already logged what the issue is - just exit with failure code here. + std::exit(1); + } + + Entry->EffectiveListenPort = uint16_t(EffectiveBasePort); + if (EffectiveBasePort != m_ServerOptions.BasePort) + { + ZEN_INFO(ZEN_APP_NAME " - relocated to base port {}", EffectiveBasePort); + m_ServerOptions.BasePort = EffectiveBasePort; + } + + std::unique_ptr<std::thread> ShutdownThread; + std::unique_ptr<NamedEvent> ShutdownEvent; + + ExtendableStringBuilder<64> ShutdownEventName; + ShutdownEventName << "Zen_" << m_ServerOptions.BasePort << "_Shutdown"; + ShutdownEvent.reset(new NamedEvent{ShutdownEventName}); + + // Monitor shutdown signals + + ShutdownThread.reset(new std::thread{[&] { + SetCurrentThreadName("shutdown_mon"); + + ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}' for process {}", ShutdownEventName, zen::GetCurrentProcessId()); + + if (ShutdownEvent->Wait()) + { + ZEN_INFO("shutdown signal for pid {} received", zen::GetCurrentProcessId()); + Server.RequestExit(0); + } + else + { + ZEN_INFO("shutdown signal wait() failed"); + } + }}); + + auto CleanupShutdown = MakeGuard([&ShutdownEvent, &ShutdownThread] { + ReportServiceStatus(ServiceStatus::Stopping); + + if (ShutdownEvent) + { + ShutdownEvent->Set(); + } + if (ShutdownThread && ShutdownThread->joinable()) + { + ShutdownThread->join(); + } + }); + + // If we have a parent process, establish the mechanisms we need + // to be able to communicate readiness with the parent + + Server.SetIsReadyFunc([&] { + std::error_code Ec; + m_LockFile.Update(MakeLockData(true), Ec); + ReportServiceStatus(ServiceStatus::Running); + NotifyReady(); + }); + + Server.Run(); +} + +} // namespace zen diff --git a/src/zenserver/hub/zenhubserver.h b/src/zenserver/hub/zenhubserver.h new file mode 100644 index 000000000..ac14362f0 --- /dev/null +++ b/src/zenserver/hub/zenhubserver.h @@ -0,0 +1,92 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zenserver.h" + +namespace cxxopts { +class Options; +} +namespace zen::LuaConfig { +struct Options; +} + +namespace zen { + +class HttpApiService; +class HttpHubService; + +struct ZenHubServerConfig : public ZenServerConfig +{ + std::string UpstreamNotificationEndpoint; + std::string InstanceId; // For use in notifications +}; + +struct ZenHubServerConfigurator : public ZenServerConfiguratorBase +{ + ZenHubServerConfigurator(ZenHubServerConfig& ServerOptions) : ZenServerConfiguratorBase(ServerOptions), m_ServerOptions(ServerOptions) + { + } + + ~ZenHubServerConfigurator() = default; + +private: + virtual void AddCliOptions(cxxopts::Options& Options) override; + virtual void AddConfigOptions(LuaConfig::Options& Options) override; + virtual void ApplyOptions(cxxopts::Options& Options) override; + virtual void OnConfigFileParsed(LuaConfig::Options& LuaOptions) override; + virtual void ValidateOptions() override; + + ZenHubServerConfig& m_ServerOptions; +}; + +class ZenHubServerMain : public ZenServerMain +{ +public: + ZenHubServerMain(ZenHubServerConfig& ServerOptions); + virtual void DoRun(ZenServerState::ZenServerEntry* Entry) override; + + ZenHubServerMain(const ZenHubServerMain&) = delete; + ZenHubServerMain& operator=(const ZenHubServerMain&) = delete; + + typedef ZenHubServerConfig Config; + typedef ZenHubServerConfigurator Configurator; + +private: + ZenHubServerConfig& m_ServerOptions; +}; + +class ZenHubServer : public ZenServerBase +{ + ZenHubServer& operator=(ZenHubServer&&) = delete; + ZenHubServer(ZenHubServer&&) = delete; + +public: + ZenHubServer(); + ~ZenHubServer(); + + int Initialize(const ZenHubServerConfig& ServerConfig, ZenServerState::ZenServerEntry* ServerEntry); + void Run(); + void Cleanup(); + + void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; } + void SetTestMode(bool State) { m_TestMode = State; } + void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; } + void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; } + +private: + bool m_IsDedicatedMode = false; + bool m_TestMode = false; + std::filesystem::path m_DataRoot; + std::filesystem::path m_ContentRoot; + bool m_DebugOptionForcedCrash = false; + + std::unique_ptr<HttpHubService> m_HubService; + std::unique_ptr<HttpApiService> m_ApiService; + + void InitializeState(const ZenHubServerConfig& ServerConfig); + void InitializeServices(const ZenHubServerConfig& ServerConfig); + void RegisterServices(const ZenHubServerConfig& ServerConfig); +}; + +} // namespace zen diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 34848c831..3a58d1f4a 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -19,12 +19,15 @@ #include <zencore/thread.h> #include <zencore/trace.h> #include <zentelemetry/otlptrace.h> +#include <zenutil/commandlineoptions.h> #include <zenutil/service.h> #include "diag/logging.h" #include "storage/storageconfig.h" #include "storage/zenstorageserver.h" +#include "hub/zenhubserver.h" + #if ZEN_PLATFORM_WINDOWS # include <zencore/windows.h> # include <zenutil/windows/windowsservice.h> @@ -54,8 +57,6 @@ SignalCallbackHandler(int SigNum) namespace zen { -using namespace std::literals; - ////////////////////////////////////////////////////////////////////////// #if ZEN_PLATFORM_WINDOWS @@ -89,6 +90,13 @@ AppMain(int argc, char* argv[]) { using namespace std::literals; + signal(SIGINT, utils::SignalCallbackHandler); + signal(SIGTERM, utils::SignalCallbackHandler); + +#if ZEN_PLATFORM_LINUX + IgnoreChildSignals(); +#endif + try { typename Main::Config ServerOptions; @@ -188,12 +196,12 @@ AppMain(int argc, char* argv[]) } catch (const AssertException& AssertEx) { - fprintf(stderr, ZEN_APP_NAME " ERROR: Caught assert exception in main: '%s'", AssertEx.FullDescription().c_str()); + fprintf(stderr, ZEN_APP_NAME " ERROR: Caught assert exception in main: '%s'\n", AssertEx.FullDescription().c_str()); return 1; } catch (const std::exception& Ex) { - fprintf(stderr, ZEN_APP_NAME " ERROR: Caught exception in main: '%s'", Ex.what()); + fprintf(stderr, ZEN_APP_NAME " ERROR: Caught exception in main: '%s'\n", Ex.what()); return 1; } @@ -206,6 +214,10 @@ AppMain(int argc, char* argv[]) int test_main(int argc, char** argv) { +# if ZEN_PLATFORM_WINDOWS + setlocale(LC_ALL, "en_us.UTF8"); +# endif // ZEN_PLATFORM_WINDOWS + zen::logging::InitializeLogging(); zen::logging::SetLogLevel(zen::logging::level::Debug); @@ -218,27 +230,58 @@ test_main(int argc, char** argv) int main(int argc, char* argv[]) { +#if ZEN_PLATFORM_WINDOWS + setlocale(LC_ALL, "en_us.UTF8"); +#endif // ZEN_PLATFORM_WINDOWS + + zen::CommandLineConverter ArgConverter(argc, argv); + using namespace zen; + using namespace std::literals; + + auto _ = zen::MakeGuard([] { + // Allow some time for worker threads to unravel, in an effort + // to prevent shutdown races in TLS object destruction + WaitForThreads(1000); + }); + + enum + { + kHub, + kStore, + kTest + } ServerMode = kStore; if (argc >= 2) { - if (argv[1] == "test"sv) + if (argv[1] == "hub"sv) { + ServerMode = kHub; + } + else if (argv[1] == "store"sv) + { + ServerMode = kStore; + } + else if (argv[1] == "test"sv) + { + ServerMode = kTest; + } + } + + switch (ServerMode) + { + case kTest: #if ZEN_WITH_TESTS return test_main(argc, argv); #else fprintf(stderr, "test option not available in release mode!\n"); exit(5); #endif - } + break; + case kHub: + return AppMain<ZenHubServerMain>(argc, argv); + default: + case kStore: + return AppMain<ZenStorageServerMain>(argc, argv); } - - signal(SIGINT, utils::SignalCallbackHandler); - signal(SIGTERM, utils::SignalCallbackHandler); - -#if ZEN_PLATFORM_LINUX - IgnoreChildSignals(); -#endif - - return AppMain<ZenStorageServerMain>(argc, argv); } diff --git a/src/zenserver/storage/admin/admin.cpp b/src/zenserver/storage/admin/admin.cpp index 04f43d33a..19155e02b 100644 --- a/src/zenserver/storage/admin/admin.cpp +++ b/src/zenserver/storage/admin/admin.cpp @@ -121,7 +121,10 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, }, HttpVerb::kGet); - m_Router.AddPattern("jobid", "([[:digit:]]+?)"); + static constexpr AsciiSet ValidNumberCharactersSet{"0123456789"}; + + m_Router.AddMatcher("jobid", + [](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidNumberCharactersSet); }); m_Router.RegisterRoute( "jobs", @@ -539,7 +542,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); GcScheduler::TriggerScrubParams ScrubParams; - ScrubParams.MaxTimeslice = std::chrono::seconds(100); + ScrubParams.MaxTimeslice = std::chrono::seconds(300); if (auto Param = Params.GetValue("skipdelete"); Param.empty() == false) { @@ -556,6 +559,14 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, ScrubParams.SkipCas = (Param == "true"sv); } + if (auto Param = Params.GetValue("maxtimeslice"); Param.empty() == false) + { + if (auto Value = ParseInt<uint64_t>(Param)) + { + ScrubParams.MaxTimeslice = std::chrono::seconds(Value.value()); + } + } + m_GcScheduler.TriggerScrub(ScrubParams); CbObjectWriter Response; diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp index 18fae7027..f5ba30616 100644 --- a/src/zenserver/storage/buildstore/httpbuildstore.cpp +++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp @@ -48,10 +48,20 @@ HttpBuildStoreService::Initialize() { ZEN_LOG_INFO(LogBuilds, "Initializing Builds Service"); - m_Router.AddPattern("namespace", "([[:alnum:]\\-_.]+)"); - m_Router.AddPattern("bucket", "([[:alnum:]\\-_.]+)"); - m_Router.AddPattern("buildid", "([[:xdigit:]]{24})"); - m_Router.AddPattern("hash", "([[:xdigit:]]{40})"); + static constexpr AsciiSet ValidNamespaceCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; + static constexpr AsciiSet ValidBucketCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; + static constexpr AsciiSet ValidHexCharactersSet{"0123456789abcdefABCDEF"}; + + m_Router.AddMatcher("namespace", + [](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidNamespaceCharactersSet); }); + m_Router.AddMatcher("bucket", + [](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidBucketCharactersSet); }); + m_Router.AddMatcher("buildid", [](std::string_view Str) -> bool { + return Str.length() == Oid::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet); + }); + m_Router.AddMatcher("hash", [](std::string_view Str) -> bool { + return Str.length() == IoHash::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet); + }); m_Router.RegisterRoute( "{namespace}/{bucket}/{buildid}/blobs/{hash}", diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp index d8ad40621..052c3d630 100644 --- a/src/zenserver/storage/objectstore/objectstore.cpp +++ b/src/zenserver/storage/objectstore/objectstore.cpp @@ -271,8 +271,13 @@ HttpObjectStoreService::Inititalize() CreateDirectories(BucketsPath); } - m_Router.AddPattern("path", "([[:alnum:]/_.,;$~\\{\\}\\+\\-\\[\\]\\%\\(\\)]+)"); - m_Router.AddPattern("bucket", "([[:alnum:]\\-_.]+)"); + static constexpr AsciiSet ValidPathCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789/_.,;$~{}+-[]%()]ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; + static constexpr AsciiSet ValidBucketCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; + + m_Router.AddMatcher("path", + [](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidPathCharactersSet); }); + m_Router.AddMatcher("bucket", + [](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidBucketCharactersSet); }); m_Router.RegisterRoute( "bucket", diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp index 91c0a8af1..e3dd52ca7 100644 --- a/src/zenserver/storage/projectstore/httpprojectstore.cpp +++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp @@ -565,11 +565,23 @@ HttpProjectService::HttpProjectService(CidStore& Store, using namespace std::literals; - m_Router.AddPattern("project", "([[:alnum:]_.]+)"); - m_Router.AddPattern("log", "([[:alnum:]_.]+)"); - m_Router.AddPattern("op", "([[:digit:]]+?)"); - m_Router.AddPattern("chunk", "([[:xdigit:]]{24})"); - m_Router.AddPattern("hash", "([[:xdigit:]]{40})"); + static constexpr AsciiSet ValidProjectCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; + static constexpr AsciiSet ValidOplogCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; + static constexpr AsciiSet ValidNumberCharactersSet{"0123456789"}; + static constexpr AsciiSet ValidHexCharactersSet{"0123456789abcdefABCDEF"}; + + m_Router.AddMatcher("project", + [](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidProjectCharactersSet); }); + m_Router.AddMatcher("log", + [](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidOplogCharactersSet); }); + m_Router.AddMatcher("op", + [](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidNumberCharactersSet); }); + m_Router.AddMatcher("chunk", [](std::string_view Str) -> bool { + return Str.length() == Oid::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet); + }); + m_Router.AddMatcher("hash", [](std::string_view Str) -> bool { + return Str.length() == IoHash::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet); + }); m_Router.RegisterRoute( "", @@ -2900,6 +2912,8 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) }; tsl::robin_map<IoHash, AddedChunk, IoHash::Hasher> AddedChunks; + const std::filesystem::path CanonicalRoot = std::filesystem::canonical(Project->RootDir); + Oplog->IterateOplog( [&](CbObjectView Op) { bool OpRewritten = false; @@ -2918,10 +2932,36 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) if (DataHash == IoHash::Zero) { - std::string_view ServerPath = View["serverpath"sv].AsString(); - std::filesystem::path FilePath = Project->RootDir / ServerPath; - BasicFile DataFile; - std::error_code Ec; + std::string_view ServerPath = View["serverpath"sv].AsString(); + if (CanonicalRoot.empty()) + { + ZEN_WARN("Attempting to load file '{}' from project with unset project root", ServerPath); + AllOk = false; + continue; + } + + std::error_code Ec; + const std::filesystem::path FilePath = std::filesystem::canonical(Project->RootDir / ServerPath, Ec); + + if (Ec) + { + ZEN_WARN("Failed to find file '{}' in project root '{}' for 'snapshot'. Reason: '{}'", + ServerPath, + Project->RootDir, + Ec.message()); + AllOk = false; + continue; + } + + if (std::mismatch(CanonicalRoot.begin(), CanonicalRoot.end(), FilePath.begin()).first != + CanonicalRoot.end()) + { + ZEN_WARN("Unable to read file '{}' outside of project root '{}'", FilePath, CanonicalRoot); + AllOk = false; + continue; + } + + BasicFile DataFile; DataFile.Open(FilePath, BasicFile::Mode::kRead, Ec); if (Ec) diff --git a/src/zenserver/storage/workspaces/httpworkspaces.cpp b/src/zenserver/storage/workspaces/httpworkspaces.cpp index 3fea46b2f..dc4cc7e69 100644 --- a/src/zenserver/storage/workspaces/httpworkspaces.cpp +++ b/src/zenserver/storage/workspaces/httpworkspaces.cpp @@ -169,10 +169,20 @@ HttpWorkspacesService::Initialize() ZEN_LOG_INFO(LogFs, "Initializing Workspaces Service"); - m_Router.AddPattern("workspace_id", "([[:xdigit:]]{24})"); - m_Router.AddPattern("share_id", "([[:xdigit:]]{24})"); - m_Router.AddPattern("chunk", "([[:xdigit:]]{24})"); - m_Router.AddPattern("share_alias", "([[:alnum:]_.\\+\\-\\[\\]]+)"); + static constexpr AsciiSet ValidHexCharactersSet{"0123456789abcdefABCDEF"}; + + m_Router.AddMatcher("workspace_id", [](std::string_view Str) -> bool { + return Str.length() == Oid::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet); + }); + m_Router.AddMatcher("share_id", [](std::string_view Str) -> bool { + return Str.length() == Oid::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet); + }); + m_Router.AddMatcher("chunk", [](std::string_view Str) -> bool { + return Str.length() == Oid::StringLength && AsciiSet::HasOnly(Str, ValidHexCharactersSet); + }); + m_Router.AddMatcher("share_alias", [](std::string_view Str) -> bool { + return !Str.empty() && AsciiSet::HasOnly(Str, Workspaces::ValidAliasCharactersSet); + }); m_Router.RegisterRoute( "{workspace_id}/{share_id}/files", diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp index cf4936f6f..edce75d2a 100644 --- a/src/zenserver/storage/zenstorageserver.cpp +++ b/src/zenserver/storage/zenstorageserver.cpp @@ -17,6 +17,7 @@ #include <zencore/sentryintegration.h> #include <zencore/session.h> #include <zencore/string.h> +#include <zencore/system.h> #include <zencore/thread.h> #include <zencore/timer.h> #include <zencore/trace.h> @@ -683,8 +684,8 @@ ZenStorageServer::Run() " \\/ \\/ \\/ \\/ \\/ \n"); ExtendableStringBuilder<256> BuildOptions; - GetBuildOptions(BuildOptions); - ZEN_INFO("Build options: {}", BuildOptions); + GetBuildOptions(BuildOptions, '\n'); + ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions); } ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId()); @@ -904,6 +905,16 @@ ZenStorageServerMain::ZenStorageServerMain(ZenStorageServerConfig& ServerOptions { } +ZenStorageServerMain::~ZenStorageServerMain() +{ +} + +void +ZenStorageServerMain::InitializeLogging() +{ + InitializeServerLogging(m_ServerOptions, /* WithCacheService */ true); +} + void ZenStorageServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) { diff --git a/src/zenserver/storage/zenstorageserver.h b/src/zenserver/storage/zenstorageserver.h index f79c55bc8..5ccb587d6 100644 --- a/src/zenserver/storage/zenstorageserver.h +++ b/src/zenserver/storage/zenstorageserver.h @@ -103,7 +103,10 @@ class ZenStorageServerMain : public ZenServerMain { public: ZenStorageServerMain(ZenStorageServerConfig& ServerOptions); + ~ZenStorageServerMain(); + virtual void DoRun(ZenServerState::ZenServerEntry* Entry) override; + virtual void InitializeLogging() override; ZenStorageServerMain(const ZenStorageServerMain&) = delete; ZenStorageServerMain& operator=(const ZenStorageServerMain&) = delete; diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua index fb65fa949..6ee80dc62 100644 --- a/src/zenserver/xmake.lua +++ b/src/zenserver/xmake.lua @@ -22,6 +22,7 @@ target("zenserver") add_deps("sol2") add_packages("json11") add_packages("lua") + add_packages("consul") if has_config("zenmimalloc") then add_packages("mimalloc") @@ -55,10 +56,7 @@ target("zenserver") add_ldflags("-framework Security") add_ldflags("-framework SystemConfiguration") end - - add_options("compute") - add_options("exec") - + -- to work around some unfortunate Ctrl-C behaviour on Linux/Mac due to -- our use of setsid() at startup we pass in `--no-detach` to zenserver -- ensure that it recieves signals when the user requests termination @@ -87,17 +85,60 @@ target("zenserver") end end) + after_build(function (target) - if has_config("zensentry") then - local crashpad_handler = "crashpad_handler" - if is_plat("windows") then - crashpad_handler = "crashpad_handler.exe" + local function copy_if_newer(src_file, dst_file, file_description) + if not os.exists(src_file) then + print("Source file '" .. src_file .. "' does not exist, cannot copy " .. file_description) + return end + local should_copy = false + if not os.exists(dst_file) then + should_copy = true + else + local src_size = os.filesize(src_file) + local dst_size = os.filesize(dst_file) + local src_mtime = os.mtime(src_file) + local dst_mtime = os.mtime(dst_file) + + if src_size ~= dst_size or src_mtime > dst_mtime then + should_copy = true + end + end + + if should_copy then + os.cp(src_file, dst_file) + print("Copied '" .. file_description .. "' to output directory") + end + end + + if has_config("zensentry") then local pkg = target:pkg("sentry-native") if pkg then local installdir = pkg:installdir() - os.cp(path.join(installdir, "bin/" .. crashpad_handler), target:targetdir()) - print("Copied " .. crashpad_handler .. " to output directory") + + local crashpad_handler = "crashpad_handler" + if is_plat("windows") then + crashpad_handler = "crashpad_handler.exe" + end + + local crashpad_handler_path = path.join(installdir, "bin/" .. crashpad_handler) + copy_if_newer(crashpad_handler_path, path.join(target:targetdir(), crashpad_handler), crashpad_handler) + + if is_plat("windows") then + local crashpad_wer_path = path.join(installdir, "bin/crashpad_wer.dll") + copy_if_newer(crashpad_wer_path, path.join(target:targetdir(), "crashpad_wer.dll"), "crashpad_wer.dll") + end + end + end + + local consul_pkg = target:pkg("consul") + if consul_pkg then + local installdir = consul_pkg:installdir() + local consul_bin = "consul" + if is_plat("windows") then + consul_bin = "consul.exe" end + copy_if_newer(path.join(installdir, "bin", consul_bin), path.join(target:targetdir(), consul_bin), consul_bin) end end) diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 08be5475a..04f60c73e 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -123,7 +123,9 @@ ZenServerBase::Initialize(const ZenServerConfig& ServerOptions, ZenServerState:: if (m_ServerMutex.Create(MutexName) == false) { - ThrowLastError(fmt::format("Failed to create mutex '{}'", MutexName).c_str()); + std::error_code Ec = MakeErrorCodeFromLastError(); + ZEN_WARN("Failed to create server mutex '{}'. Reason: '{}' ({})", MutexName, Ec.message(), Ec.value()); + return -1; } EnqueueSigIntTimer(); @@ -150,6 +152,13 @@ ZenServerBase::Initialize(const ZenServerConfig& ServerOptions, ZenServerState:: EnqueueStatsReportingTimer(); } + m_HealthService.SetHealthInfo({.DataRoot = ServerOptions.DataDir, + .AbsLogPath = ServerOptions.AbsLogFile, + .HttpServerClass = std::string(ServerOptions.HttpConfig.ServerClass), + .BuildVersion = std::string(ZEN_CFG_VERSION_BUILD_STRING_FULL)}); + + LogSettingsSummary(ServerOptions); + return EffectiveBasePort; } @@ -162,24 +171,24 @@ ZenServerBase::Finalize() } void -ZenServerBase::GetBuildOptions(StringBuilderBase& OutOptions) +ZenServerBase::GetBuildOptions(StringBuilderBase& OutOptions, char Separator) const { ZEN_MEMSCOPE(GetZenserverTag()); OutOptions << "ZEN_ADDRESS_SANITIZER=" << (ZEN_ADDRESS_SANITIZER ? "1" : "0"); - OutOptions << ","; + OutOptions << Separator; OutOptions << "ZEN_USE_SENTRY=" << (ZEN_USE_SENTRY ? "1" : "0"); - OutOptions << ","; + OutOptions << Separator; OutOptions << "ZEN_WITH_TESTS=" << (ZEN_WITH_TESTS ? "1" : "0"); - OutOptions << ","; + OutOptions << Separator; OutOptions << "ZEN_USE_MIMALLOC=" << (ZEN_USE_MIMALLOC ? "1" : "0"); - OutOptions << ","; + OutOptions << Separator; OutOptions << "ZEN_USE_RPMALLOC=" << (ZEN_USE_RPMALLOC ? "1" : "0"); - OutOptions << ","; + OutOptions << Separator; OutOptions << "ZEN_WITH_HTTPSYS=" << (ZEN_WITH_HTTPSYS ? "1" : "0"); - OutOptions << ","; + OutOptions << Separator; OutOptions << "ZEN_WITH_MEMTRACK=" << (ZEN_WITH_MEMTRACK ? "1" : "0"); - OutOptions << ","; + OutOptions << Separator; OutOptions << "ZEN_WITH_TRACE=" << (ZEN_WITH_TRACE ? "1" : "0"); } @@ -373,6 +382,57 @@ ZenServerBase::HandleStatusRequest(HttpServerRequest& Request) Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } +void +ZenServerBase::LogSettingsSummary(const ZenServerConfig& ServerConfig) +{ + // clang-format off + std::list<std::pair<std::string_view, std::string>> Settings = { + {"DataDir"sv, ServerConfig.DataDir.string()}, + {"AbsLogFile"sv, ServerConfig.AbsLogFile.string()}, + {"SystemRootDir"sv, ServerConfig.SystemRootDir.string()}, + {"ContentDir"sv, ServerConfig.ContentDir.string()}, + {"BasePort"sv, fmt::to_string(ServerConfig.BasePort)}, + {"IsDebug"sv, fmt::to_string(ServerConfig.IsDebug)}, + {"IsCleanStart"sv, fmt::to_string(ServerConfig.IsCleanStart)}, + {"IsPowerCycle"sv, fmt::to_string(ServerConfig.IsPowerCycle)}, + {"IsTest"sv, fmt::to_string(ServerConfig.IsTest)}, + {"Detach"sv, fmt::to_string(ServerConfig.Detach)}, + {"NoConsoleOutput"sv, fmt::to_string(ServerConfig.NoConsoleOutput)}, + {"QuietConsole"sv, fmt::to_string(ServerConfig.QuietConsole)}, + {"CoreLimit"sv, fmt::to_string(ServerConfig.CoreLimit)}, + {"IsDedicated"sv, fmt::to_string(ServerConfig.IsDedicated)}, + {"ShouldCrash"sv, fmt::to_string(ServerConfig.ShouldCrash)}, + {"ChildId"sv, ServerConfig.ChildId}, + {"LogId"sv, ServerConfig.LogId}, + {"Sentry DSN"sv, ServerConfig.SentryConfig.Dsn.empty() ? "not set" : ServerConfig.SentryConfig.Dsn}, + {"Sentry Environment"sv, ServerConfig.SentryConfig.Environment}, + {"Statsd Enabled"sv, fmt::to_string(ServerConfig.StatsConfig.Enabled)}, + }; + // clang-format on + + if (ServerConfig.StatsConfig.Enabled) + { + Settings.emplace_back("Statsd Host", ServerConfig.StatsConfig.StatsdHost); + Settings.emplace_back("Statsd Port", fmt::to_string(ServerConfig.StatsConfig.StatsdPort)); + } + + size_t MaxWidth = 0; + for (const auto& Setting : Settings) + { + MaxWidth = std::max(MaxWidth, Setting.first.size()); + } + + ZEN_INFO("Server settings summary:"); + + for (const auto& Setting : Settings) + { + if (!Setting.second.empty()) + { + ZEN_INFO(" {:<{}} : {}", Setting.first, MaxWidth, Setting.second); + } + } +} + ////////////////////////////////////////////////////////////////////////// ZenServerMain::ZenServerMain(ZenServerConfig& ServerOptions) : m_ServerOptions(ServerOptions) @@ -383,6 +443,12 @@ ZenServerMain::~ZenServerMain() { } +void +ZenServerMain::InitializeLogging() +{ + InitializeServerLogging(m_ServerOptions, /* WithCacheService */ false); +} + int ZenServerMain::Run() { @@ -510,7 +576,7 @@ ZenServerMain::Run() } } - InitializeServerLogging(m_ServerOptions, /* WithCacheService */ true); + InitializeLogging(); ZEN_INFO("Command line: {}", m_ServerOptions.CommandLine); diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h index 4de865a5f..ab7122fcc 100644 --- a/src/zenserver/zenserver.h +++ b/src/zenserver/zenserver.h @@ -10,6 +10,7 @@ #include <memory> #include <string_view> +#include "config/config.h" ZEN_THIRD_PARTY_INCLUDES_START #include <asio.hpp> @@ -45,7 +46,8 @@ public: protected: int Initialize(const ZenServerConfig& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry); void Finalize(); - void GetBuildOptions(StringBuilderBase& OutOptions); + void GetBuildOptions(StringBuilderBase& OutOptions, char Separator = ',') const; + void LogSettingsSummary(const ZenServerConfig& ServerConfig); protected: NamedMutex m_ServerMutex; @@ -122,6 +124,7 @@ protected: ZenServerConfig& m_ServerOptions; LockFile m_LockFile; + virtual void InitializeLogging(); virtual void DoRun(ZenServerState::ZenServerEntry* Entry) = 0; void NotifyReady(); diff --git a/src/zenstore-test/zenstore-test.cpp b/src/zenstore-test/zenstore-test.cpp index 6df7162fd..c055dbb64 100644 --- a/src/zenstore-test/zenstore-test.cpp +++ b/src/zenstore-test/zenstore-test.cpp @@ -16,6 +16,10 @@ int main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) { +#if ZEN_PLATFORM_WINDOWS + setlocale(LC_ALL, "en_us.UTF8"); +#endif // ZEN_PLATFORM_WINDOWS + #if ZEN_WITH_TESTS zen::zenstore_forcelinktests(); diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index f97c98e08..3ea91ead6 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -374,7 +374,7 @@ BlockStoreFile::GetMetaPath() const //////////////////////////////////////////////////////// -constexpr uint64_t DefaultIterateSmallChunkWindowSize = 2 * 1024 * 1024; +constexpr uint64_t DefaultIterateSmallChunkWindowSize = 512u * 1024u; BlockStore::BlockStore() { @@ -762,7 +762,7 @@ BlockStore::WriteChunks(std::span<const IoBuffer> Datas, uint32_t Alignment, con LargestSize = Max(LargestSize, Size); } - const uint64_t MinSize = Max(LargestSize, 8u * 1024u * 1024u); + const uint64_t MinSize = Max(LargestSize, 512u * 1024u); const uint64_t BufferSize = Min(TotalSize, MinSize); std::vector<uint8_t> Buffer(BufferSize); @@ -815,7 +815,12 @@ BlockStore::WriteChunks(std::span<const IoBuffer> Datas, uint32_t Alignment, con auto _ = MakeGuard([this, WriteBlockIndex]() { RemoveActiveWriteBlock(WriteBlockIndex); }); + if (Count > 1) { + if (Buffer.empty()) + { + Buffer.resize(BufferSize); + } MutableMemoryView WriteBuffer(Buffer.data(), RangeSize); for (size_t Index = 0; Index < Count; Index++) { @@ -824,9 +829,14 @@ BlockStore::WriteChunks(std::span<const IoBuffer> Datas, uint32_t Alignment, con WriteBuffer.MidInline(RoundUp(SourceBuffer.GetSize(), Alignment)); } WriteBlock->Write(Buffer.data(), RangeSize, AlignedInsertOffset); + m_TotalSize.fetch_add(RangeSize, std::memory_order::relaxed); + } + else + { + MemoryView SourceBuffer = Datas[Offset]; + WriteBlock->Write(SourceBuffer.GetData(), SourceBuffer.GetSize(), AlignedInsertOffset); + m_TotalSize.fetch_add(SourceBuffer.GetSize(), std::memory_order::relaxed); } - - m_TotalSize.fetch_add(RangeSize, std::memory_order::relaxed); uint32_t ChunkOffset = AlignedInsertOffset; std::vector<BlockStoreLocation> Locations(Count); @@ -845,11 +855,11 @@ BlockStore::WriteChunks(std::span<const IoBuffer> Datas, uint32_t Alignment, con bool BlockStore::HasChunk(const BlockStoreLocation& Location) const { - ZEN_TRACE_CPU("BlockStore::TryGetChunk"); + ZEN_TRACE_CPU("BlockStore::HasChunk"); RwLock::SharedLockScope InsertLock(m_InsertLock); if (auto BlockIt = m_ChunkBlocks.find(Location.BlockIndex); BlockIt != m_ChunkBlocks.end()) { - if (const Ref<BlockStoreFile>& Block = BlockIt->second; Block) + if (Ref<BlockStoreFile> Block = BlockIt->second; Block) { InsertLock.ReleaseNow(); @@ -878,8 +888,10 @@ BlockStore::TryGetChunk(const BlockStoreLocation& Location) const RwLock::SharedLockScope InsertLock(m_InsertLock); if (auto BlockIt = m_ChunkBlocks.find(Location.BlockIndex); BlockIt != m_ChunkBlocks.end()) { - if (const Ref<BlockStoreFile>& Block = BlockIt->second; Block) + if (Ref<BlockStoreFile> Block = BlockIt->second; Block) { + InsertLock.ReleaseNow(); + IoBuffer Chunk = Block->GetChunk(Location.Offset, Location.Size); if (Chunk.GetSize() == Location.Size) { @@ -941,7 +953,7 @@ BlockStore::IterateBlock(std::span<const BlockStoreLocation> ChunkLocations, uint64_t IterateSmallChunkWindowSize = Max(DefaultIterateSmallChunkWindowSize, LargeSizeLimit); - const uint64_t IterateSmallChunkMaxGapSize = Max(2048u, IterateSmallChunkWindowSize / 512u); + const uint64_t IterateSmallChunkMaxGapSize = Max(2048u, IterateSmallChunkWindowSize / 256u); IterateSmallChunkWindowSize = Min((LargeSizeLimit + IterateSmallChunkMaxGapSize) * ChunkLocations.size(), IterateSmallChunkWindowSize); diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index b2e045632..ead7e4f3a 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -2410,74 +2410,95 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) try { - std::vector<BlockStoreLocation> ChunkLocations; - std::vector<IoHash> ChunkIndexToChunkHash; + std::vector<DiskLocation> ChunkLocations; + std::vector<IoHash> ChunkIndexToChunkHash; + std::vector<DiskLocation> StandaloneLocations; + std::vector<IoHash> StandaloneIndexToKeysHash; - RwLock::SharedLockScope _(m_IndexLock); + { + RwLock::SharedLockScope _(m_IndexLock); - const size_t BlockChunkInitialCount = m_Index.size() / 4; - ChunkLocations.reserve(BlockChunkInitialCount); - ChunkIndexToChunkHash.reserve(BlockChunkInitialCount); + const size_t InitialCount = m_Index.size() / 4; + ChunkLocations.reserve(InitialCount); + ChunkIndexToChunkHash.reserve(InitialCount); + StandaloneLocations.reserve(InitialCount); + StandaloneIndexToKeysHash.reserve(InitialCount); - // Do a pass over the index and verify any standalone file values straight away - // all other storage classes are gathered and verified in bulk in order to enable - // more efficient I/O scheduling + for (auto& Kv : m_Index) + { + const IoHash& HashKey = Kv.first; + const BucketPayload& Payload = m_Payloads[Kv.second]; + const DiskLocation& Loc = Payload.Location; - for (auto& Kv : m_Index) - { - const IoHash& HashKey = Kv.first; - const BucketPayload& Payload = m_Payloads[Kv.second]; - const DiskLocation& Loc = Payload.Location; + Ctx.ThrowIfDeadlineExpired(); + if (Loc.IsFlagSet(DiskLocation::kStandaloneFile)) + { + StandaloneLocations.push_back(Loc); + StandaloneIndexToKeysHash.push_back(HashKey); + } + else + { + ChunkLocations.push_back(Loc); + ChunkIndexToChunkHash.push_back(HashKey); + } + } + } + + for (size_t StandaloneKeyIndex = 0; StandaloneKeyIndex < StandaloneIndexToKeysHash.size(); StandaloneKeyIndex++) + { Ctx.ThrowIfDeadlineExpired(); - if (Loc.IsFlagSet(DiskLocation::kStandaloneFile)) - { - ChunkCount.fetch_add(1); - VerifiedChunkBytes.fetch_add(Loc.Size()); + const IoHash& HashKey = StandaloneIndexToKeysHash[StandaloneKeyIndex]; + const DiskLocation& Loc = StandaloneLocations[StandaloneKeyIndex]; - if (Loc.GetContentType() == ZenContentType::kBinary) - { - // Blob cache value, not much we can do about data integrity checking - // here since there's no hash available - ExtendablePathBuilder<256> DataFilePath; - BuildPath(DataFilePath, HashKey); + ChunkCount.fetch_add(1); + VerifiedChunkBytes.fetch_add(Loc.Size()); - RwLock::SharedLockScope ValueLock(LockForHash(HashKey)); + if (Loc.GetContentType() == ZenContentType::kBinary) + { + // Blob cache value, not much we can do about data integrity checking + // here since there's no hash available + ExtendablePathBuilder<256> DataFilePath; + BuildPath(DataFilePath, HashKey); - std::error_code Ec; - uintmax_t size = FileSizeFromPath(DataFilePath.ToPath(), Ec); - if (Ec) - { - ReportBadKey(HashKey); - } - if (size != Loc.Size()) - { - ReportBadKey(HashKey); - } - continue; + RwLock::SharedLockScope ValueLock(LockForHash(HashKey)); + + std::error_code Ec; + uintmax_t Size = FileSizeFromPath(DataFilePath.ToPath(), Ec); + if (Ec) + { + ReportBadKey(HashKey); } - else + ValueLock.ReleaseNow(); + + if (Size != Loc.Size()) { - // Structured cache value - IoBuffer Buffer = GetStandaloneCacheValue(Loc, HashKey); - if (!Buffer) + // Make sure we verify that size hasn't changed behind our back... + RwLock::SharedLockScope _(m_IndexLock); + if (auto It = m_Index.find(HashKey); It != m_Index.end()) { - ReportBadKey(HashKey); - continue; - } - if (!ValidateIoBuffer(Loc.GetContentType(), std::move(Buffer))) - { - ReportBadKey(HashKey); - continue; + const BucketPayload& Payload = m_Payloads[It->second]; + const DiskLocation& CurrentLoc = Payload.Location; + if (Size != CurrentLoc.Size()) + { + ReportBadKey(HashKey); + } } } } else { - ChunkLocations.emplace_back(Loc.GetBlockLocation(m_Configuration.PayloadAlignment)); - ChunkIndexToChunkHash.push_back(HashKey); - continue; + // Structured cache value + IoBuffer Buffer = GetStandaloneCacheValue(Loc, HashKey); + if (!Buffer) + { + ReportBadKey(HashKey); + } + else if (!ValidateIoBuffer(Loc.GetContentType(), std::move(Buffer))) + { + ReportBadKey(HashKey); + } } } @@ -2502,8 +2523,9 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) ReportBadKey(Hash); return true; } - const BucketPayload& Payload = m_Payloads[m_Index.at(Hash)]; - ZenContentType ContentType = Payload.Location.GetContentType(); + + const DiskLocation& Loc = ChunkLocations[ChunkIndex]; + ZenContentType ContentType = Loc.GetContentType(); Buffer.SetContentType(ContentType); if (!ValidateIoBuffer(ContentType, std::move(Buffer))) { @@ -2525,8 +2547,8 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) ReportBadKey(Hash); return true; } - const BucketPayload& Payload = m_Payloads[m_Index.at(Hash)]; - ZenContentType ContentType = Payload.Location.GetContentType(); + const DiskLocation& Loc = ChunkLocations[ChunkIndex]; + ZenContentType ContentType = Loc.GetContentType(); Buffer.SetContentType(ContentType); if (!ValidateIoBuffer(ContentType, std::move(Buffer))) { @@ -2536,8 +2558,16 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) return true; }; - m_BlockStore.IterateChunks(ChunkLocations, [&](uint32_t, std::span<const size_t> ChunkIndexes) { - return m_BlockStore.IterateBlock(ChunkLocations, ChunkIndexes, ValidateSmallChunk, ValidateLargeChunk, 0); + std::vector<BlockStoreLocation> ChunkBlockLocations; + ChunkBlockLocations.reserve(ChunkLocations.size()); + + for (const DiskLocation& Loc : ChunkLocations) + { + ChunkBlockLocations.push_back(Loc.GetBlockLocation(m_Configuration.PayloadAlignment)); + } + + m_BlockStore.IterateChunks(ChunkBlockLocations, [&](uint32_t, std::span<const size_t> ChunkIndexes) { + return m_BlockStore.IterateBlock(ChunkBlockLocations, ChunkIndexes, ValidateSmallChunk, ValidateLargeChunk, 0); }); } catch (ScrubDeadlineExpiredException&) diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index 660c66b9a..94abcf547 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -594,16 +594,16 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb { FoundLocalInvalid = true; } - else if (CbValidateError Error = ValidateCompactBinary(Request.RecordCacheValue.GetView(), CbValidateMode::Default); - Error != CbValidateError::None) + else if (CbObjectView RecordObject = CbObjectView(Request.RecordCacheValue.GetData()); + RecordObject.GetSize() != Request.RecordCacheValue.GetSize()) { ZEN_WARN("HandleRpcGetCacheRecords stored record is corrupt, compact binary format validation failed. Reason: '{}'", - ToString(Error)); + "Object size does not match payload size"); FoundLocalInvalid = true; } else { - Request.RecordObject = CbObjectView(Request.RecordCacheValue.GetData()); + Request.RecordObject = std::move(RecordObject); ParseValues(Request); Request.Complete = true; @@ -1710,16 +1710,15 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, Record.ValuesRead = true; if (Record.CacheValue && Record.CacheValue.GetContentType() == ZenContentType::kCbObject) { - if (CbValidateError Error = ValidateCompactBinary(Record.CacheValue.GetView(), CbValidateMode::Default); - Error != CbValidateError::None) + if (CbObjectView RecordObject = CbObjectView(Record.CacheValue.GetData()); + RecordObject.GetSize() != Record.CacheValue.GetSize()) { - ZEN_WARN("GetLocalCacheRecords stored record for is corrupt, compact binary format validation failed. Reason: '{}'", - ToString(Error)); + ZEN_WARN("GetLocalCacheRecords stored record is corrupt, compact binary format validation failed. Reason: '{}'", + "Object size does not match payload size"); } else { - CbObjectView RecordObject = CbObjectView(Record.CacheValue.GetData()); - CbArrayView ValuesArray = RecordObject["Values"sv].AsArrayView(); + CbArrayView ValuesArray = RecordObject["Values"sv].AsArrayView(); Record.Values.reserve(ValuesArray.Num()); for (CbFieldView ValueField : ValuesArray) { diff --git a/src/zenstore/cas.h b/src/zenstore/cas.h index 0f6e2ba9d..47b6e63cc 100644 --- a/src/zenstore/cas.h +++ b/src/zenstore/cas.h @@ -59,7 +59,7 @@ protected: CidStoreConfiguration m_Config; }; -ZENCORE_API std::unique_ptr<CasStore> CreateCasStore(GcManager& Gc); +std::unique_ptr<CasStore> CreateCasStore(GcManager& Gc); void CAS_forcelink(); diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index a5de5c448..5d8f95c9e 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -301,13 +301,14 @@ CasContainerStrategy::FindChunk(const IoHash& ChunkHash) { ZEN_TRACE_CPU("CasContainer::FindChunk"); - RwLock::SharedLockScope _(m_LocationMapLock); + RwLock::SharedLockScope Lock(m_LocationMapLock); auto KeyIt = m_LocationMap.find(ChunkHash); if (KeyIt == m_LocationMap.end()) { return IoBuffer(); } - const BlockStoreLocation& Location = m_Locations[KeyIt->second].Get(m_PayloadAlignment); + const BlockStoreLocation Location = m_Locations[KeyIt->second].Get(m_PayloadAlignment); + Lock.ReleaseNow(); IoBuffer Chunk = m_BlockStore.TryGetChunk(Location); return Chunk; @@ -316,10 +317,11 @@ CasContainerStrategy::FindChunk(const IoHash& ChunkHash) bool CasContainerStrategy::HaveChunk(const IoHash& ChunkHash) { - RwLock::SharedLockScope _(m_LocationMapLock); + RwLock::SharedLockScope Lock(m_LocationMapLock); if (auto KeyIt = m_LocationMap.find(ChunkHash); KeyIt != m_LocationMap.end()) { - const BlockStoreLocation& Location = m_Locations[KeyIt->second].Get(m_PayloadAlignment); + const BlockStoreLocation Location = m_Locations[KeyIt->second].Get(m_PayloadAlignment); + Lock.ReleaseNow(); return m_BlockStore.HasChunk(Location); } return false; @@ -545,11 +547,11 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx) if (Ctx.IsSkipCas()) { - ZEN_INFO("SKIPPED scrubbing: '{}'", m_BlocksBasePath); + ZEN_INFO("SKIPPED scrubbing: '{}'", m_RootDirectory); return; } - ZEN_INFO("scrubbing '{}'", m_BlocksBasePath); + ZEN_INFO("scrubbing '{}'", m_RootDirectory); RwLock BadKeysLock; std::vector<IoHash> BadKeys; @@ -563,20 +565,21 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx) try { - RwLock::SharedLockScope _(m_LocationMapLock); - - uint64_t TotalChunkCount = m_LocationMap.size(); - ChunkLocations.reserve(TotalChunkCount); - ChunkIndexToChunkHash.reserve(TotalChunkCount); { - for (const auto& Entry : m_LocationMap) + uint64_t TotalChunkCount = m_LocationMap.size(); + ChunkLocations.reserve(TotalChunkCount); + ChunkIndexToChunkHash.reserve(TotalChunkCount); + RwLock::SharedLockScope _(m_LocationMapLock); { - const IoHash& ChunkHash = Entry.first; - const BlockStoreDiskLocation& DiskLocation = m_Locations[Entry.second]; - BlockStoreLocation Location = DiskLocation.Get(m_PayloadAlignment); + for (const auto& Entry : m_LocationMap) + { + const IoHash& ChunkHash = Entry.first; + const BlockStoreDiskLocation& DiskLocation = m_Locations[Entry.second]; + BlockStoreLocation Location = DiskLocation.Get(m_PayloadAlignment); - ChunkLocations.push_back(Location); - ChunkIndexToChunkHash.push_back(ChunkHash); + ChunkLocations.push_back(Location); + ChunkIndexToChunkHash.push_back(ChunkHash); + } } } diff --git a/src/zenstore/include/zenstore/caslog.h b/src/zenstore/include/zenstore/caslog.h index 3d95c9c90..f3dd32fb1 100644 --- a/src/zenstore/include/zenstore/caslog.h +++ b/src/zenstore/include/zenstore/caslog.h @@ -41,8 +41,8 @@ private: static const inline uint8_t MagicSequence[16] = {'.', '-', '=', ' ', 'C', 'A', 'S', 'L', 'O', 'G', 'v', '1', ' ', '=', '-', '.'}; - ZENCORE_API uint32_t ComputeChecksum(); - void Finalize() { Checksum = ComputeChecksum(); } + uint32_t ComputeChecksum(); + void Finalize() { Checksum = ComputeChecksum(); } }; static_assert(sizeof(FileHeader) == 64); diff --git a/src/zenstore/include/zenstore/projectstore.h b/src/zenstore/include/zenstore/projectstore.h index 09c3096ad..33ef996db 100644 --- a/src/zenstore/include/zenstore/projectstore.h +++ b/src/zenstore/include/zenstore/projectstore.h @@ -238,6 +238,16 @@ public: std::atomic_bool& IsCancelledFlag, WorkerThreadPool* OptionalWorkerPool); + struct OplogSnapshot + { + std::vector<CbObjectView> Ops; + std::vector<Oid> Keys; + std::vector<LogSequenceNumber> LSNs; + std::vector<IoBuffer> PayloadBuffers; + }; + + OplogSnapshot GetSnapshotLocked(); + private: struct FileMapEntry { diff --git a/src/zenstore/projectstore.cpp b/src/zenstore/projectstore.cpp index f1001f665..1ab2b317a 100644 --- a/src/zenstore/projectstore.cpp +++ b/src/zenstore/projectstore.cpp @@ -22,6 +22,8 @@ #include "referencemetadata.h" +#include <numeric> + ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_set.h> #include <xxh3.h> @@ -861,9 +863,9 @@ struct ProjectStore::OplogStorage : public RefCounted } } - void ReplayLogEntries(const std::span<const Oplog::OplogPayload> Entries, - const std::span<const Oplog::PayloadIndex> Order, - std::function<void(LogSequenceNumber Lsn, const IoBuffer& Buffer)>&& Handler) + void ReplayLogEntries(const std::span<const Oplog::OplogPayload> Entries, + const std::span<const Oplog::PayloadIndex> Order, + std::function<void(LogSequenceNumber Lsn, IoBuffer&& Buffer)>&& Handler) { ZEN_MEMSCOPE(GetProjectstoreTag()); ZEN_TRACE_CPU("Store::OplogStorage::ReplayLogEntries"); @@ -886,13 +888,13 @@ struct ProjectStore::OplogStorage : public RefCounted if (OpBufferView.GetSize() == Entry.Address.Size) { IoBuffer Buffer = IoBuffer(IoBuffer::Wrap, OpBufferView.GetData(), OpBufferView.GetSize()); - Handler(Entry.Lsn, Buffer); + Handler(Entry.Lsn, std::move(Buffer)); } else { IoBuffer OpBuffer(Entry.Address.Size); OpBlobsBuffer.Read((void*)OpBuffer.Data(), Entry.Address.Size, OpFileOffset); - Handler(Entry.Lsn, OpBuffer); + Handler(Entry.Lsn, std::move(OpBuffer)); } } } @@ -1645,18 +1647,47 @@ ProjectStore::Oplog::Validate(const std::filesystem::path& ProjectRootDir, Keys.reserve(OpCount); Mappings.reserve(OpCount); - IterateOplogWithKey([&](LogSequenceNumber LSN, const Oid& Key, CbObjectView OpView) { - Result.LSNLow = Min(Result.LSNLow, LSN); - Result.LSNHigh = Max(Result.LSNHigh, LSN); - KeyHashes.push_back(Key); - Keys.emplace_back(std::string(OpView["key"sv].AsString())); + { + Stopwatch SnapshotTimer; + RwLock::SharedLockScope OplogLock(m_OplogLock); + ProjectStore::Oplog::OplogSnapshot Snapshot = GetSnapshotLocked(); + OplogLock.ReleaseNow(); - std::vector<IoHash> OpAttachments; - OpView.IterateAttachments([&OpAttachments](CbFieldView Attachment) { OpAttachments.push_back(Attachment.AsAttachment()); }); - Attachments.emplace_back(std::move(OpAttachments)); + uint64_t AllocatedSize = std::accumulate(Snapshot.PayloadBuffers.begin(), + Snapshot.PayloadBuffers.end(), + uint64_t(0), + [](uint64_t Current, const IoBuffer& Buffer) { return Current + Buffer.GetSize(); }); + uint64_t UsedSize = + std::accumulate(Snapshot.Ops.begin(), Snapshot.Ops.end(), uint64_t(0), [](uint64_t Current, const CbObjectView& Object) { + return Current + Object.GetSize(); + }); - Mappings.push_back(GetMapping(OpView)); - }); + ZEN_INFO("Oplog snapshot fetched {} ops from {} op data using {} memory from oplog '{}/{}' in {}", + Snapshot.Ops.size(), + NiceBytes(UsedSize), + NiceBytes(AllocatedSize), + m_OuterProjectId, + m_OplogId, + NiceTimeSpanMs(SnapshotTimer.GetElapsedTimeMs())); + + for (size_t Index = 0; Index < Snapshot.Ops.size(); Index++) + { + CbObjectView& OpView = Snapshot.Ops[Index]; + LogSequenceNumber LSN = Snapshot.LSNs[Index]; + const Oid& Key = Snapshot.Keys[Index]; + + Result.LSNLow = Min(Result.LSNLow, LSN); + Result.LSNHigh = Max(Result.LSNHigh, LSN); + KeyHashes.push_back(Key); + Keys.emplace_back(std::string(OpView["key"sv].AsString())); + + std::vector<IoHash> OpAttachments; + OpView.IterateAttachments([&OpAttachments](CbFieldView Attachment) { OpAttachments.push_back(Attachment.AsAttachment()); }); + Attachments.emplace_back(std::move(OpAttachments)); + + Mappings.push_back(GetMapping(OpView)); + } + } Result.OpCount = gsl::narrow<uint32_t>(Keys.size()); @@ -2644,6 +2675,104 @@ ProjectStore::Oplog::GetSortedOpPayloadRangeLocked(const Paging& Entry return ReplayOrder; } +ProjectStore::Oplog::OplogSnapshot +ProjectStore::Oplog::GetSnapshotLocked() +{ + ZEN_MEMSCOPE(GetProjectstoreTag()); + ZEN_TRACE_CPU("Store::Oplog::GetSnapshotLocked"); + if (!m_Storage) + { + return {}; + } + + uint64_t WriteOffset = 0; + OplogSnapshot Snapshot; + + const uint64_t PayloadBufferPageSize = 64u * 1024u; + + size_t OpCount = GetOplogEntryCount(); + Snapshot.Ops.reserve(OpCount); + Snapshot.Keys.reserve(OpCount); + Snapshot.LSNs.reserve(OpCount); + + tsl::robin_map<PayloadIndex, Oid, PayloadIndex::Hasher> ReverseKeyMap; + std::vector<PayloadIndex> ReplayOrder = GetSortedOpPayloadRangeLocked(Paging{}, &ReverseKeyMap); + if (!ReplayOrder.empty()) + { + uint32_t EntryIndex = 0; + m_Storage->ReplayLogEntries(m_OpLogPayloads, ReplayOrder, [&](LogSequenceNumber LSN, IoBuffer&& Buffer) { + const PayloadIndex PayloadOffset = ReplayOrder[EntryIndex]; + + Snapshot.Keys.push_back(ReverseKeyMap.at(PayloadOffset)); + Snapshot.LSNs.push_back(LSN); + + uint64_t Size = Buffer.GetSize(); + if (Buffer.IsOwned()) + { + CbObjectView CopyView(Buffer.GetData()); + if (CopyView.GetSize() == Size) + { + Snapshot.Ops.emplace_back(std::move(CopyView)); + } + else + { + Snapshot.Ops.emplace_back(CbObjectView{}); + } + if (Snapshot.PayloadBuffers.empty()) + { + Snapshot.PayloadBuffers.emplace_back(std::move(Buffer)); + WriteOffset = Snapshot.PayloadBuffers.back().Size(); + } + else + { + Snapshot.PayloadBuffers.insert(Snapshot.PayloadBuffers.end() - 1, std::move(Buffer)); + } + } + else + { + uint64_t AvailableSize = Snapshot.PayloadBuffers.empty() ? 0 : Snapshot.PayloadBuffers.back().GetSize() - WriteOffset; + MutableMemoryView WriteBuffer; + if (Size > AvailableSize) + { + if (Size >= PayloadBufferPageSize && !Snapshot.PayloadBuffers.empty()) + { + // Insert the large payload before the current payload buffer so we can continue to use that + IoBuffer PayloadBuffer(Size); + WriteBuffer = PayloadBuffer.GetMutableView(); + Snapshot.PayloadBuffers.insert(Snapshot.PayloadBuffers.end() - 1, std::move(PayloadBuffer)); + } + else + { + IoBuffer PayloadBuffer(Max(Size, PayloadBufferPageSize)); + WriteBuffer = PayloadBuffer.GetMutableView().Mid(0, Size); + Snapshot.PayloadBuffers.emplace_back(std::move(PayloadBuffer)); + WriteOffset = Size; + } + } + else + { + WriteBuffer = Snapshot.PayloadBuffers.back().GetMutableView().Mid(WriteOffset, Size); + WriteOffset += Size; + } + WriteBuffer.CopyFrom(Buffer.GetView()); + CbObjectView CopyView(WriteBuffer.GetData()); + + if (CopyView.GetSize() == Size) + { + Snapshot.Ops.emplace_back(std::move(CopyView)); + } + else + { + Snapshot.Ops.emplace_back(CbObjectView{}); + } + } + + EntryIndex++; + }); + } + return Snapshot; +} + void ProjectStore::Oplog::IterateOplogLocked(std::function<void(CbObjectView)>&& Handler, const Paging& EntryPaging) { @@ -2710,7 +2839,7 @@ ProjectStore::Oplog::IterateOplogWithKeyRaw(std::function<void(LogSequenceNumber if (!ReplayOrder.empty()) { uint32_t EntryIndex = 0; - m_Storage->ReplayLogEntries(m_OpLogPayloads, ReplayOrder, [&](LogSequenceNumber Lsn, const IoBuffer& Buffer) { + m_Storage->ReplayLogEntries(m_OpLogPayloads, ReplayOrder, [&](LogSequenceNumber Lsn, IoBuffer&& Buffer) { const PayloadIndex PayloadOffset = ReplayOrder[EntryIndex]; Handler(Lsn, ReverseKeyMap.at(PayloadOffset), Buffer); EntryIndex++; @@ -3917,7 +4046,7 @@ ProjectStore::Project::Scrub(ScrubContext& Ctx) { ZEN_MEMSCOPE(GetProjectstoreTag()); - ZEN_INFO("scrubbing '{}'", ProjectRootDir); + ZEN_INFO("scrubbing '{}'", m_OplogStoragePath); // Scrubbing needs to check all existing oplogs std::vector<std::string> OpLogs = ScanForOplogs(); @@ -3932,6 +4061,7 @@ ProjectStore::Project::Scrub(ScrubContext& Ctx) { for (const std::string& OpLogId : OpLogs) { + Ctx.ThrowIfDeadlineExpired(); Ref<ProjectStore::Oplog> OpLog; { if (auto OpIt = m_Oplogs.find(OpLogId); OpIt != m_Oplogs.end()) @@ -4358,6 +4488,7 @@ ProjectStore::ScrubStorage(ScrubContext& Ctx) } for (const Ref<Project>& Project : Projects) { + Ctx.ThrowIfDeadlineExpired(); Project->Scrub(Ctx); } } @@ -6033,39 +6164,41 @@ public: { Ref<ProjectStore::Oplog> Oplog; - RwLock::SharedLockScope __(m_Project->m_ProjectLock); - if (auto It = m_Project->m_Oplogs.find(m_OplogId); It != m_Project->m_Oplogs.end()) { - Oplog = It->second; - Oplog->EnableUpdateCapture(); - m_OplogHasUpdateCapture = true; - } - else if (ProjectStore::Oplog::ExistsAt(m_OplogBasePath)) - { - Stopwatch OplogTimer; - Oplog = new ProjectStore::Oplog(m_Project->Log(), - m_Project->Identifier, - m_OplogId, - m_Project->m_CidStore, - m_OplogBasePath, - std::filesystem::path{}, - ProjectStore::Oplog::EMode::kBasicReadOnly); - Oplog->Read(); - if (Ctx.Settings.Verbose) + RwLock::SharedLockScope __(m_Project->m_ProjectLock); + if (auto It = m_Project->m_Oplogs.find(m_OplogId); It != m_Project->m_Oplogs.end()) { - ZEN_INFO("GCV2: projectstore [PRECACHE] '{}': read oplog '{}/{}' in {}", - m_OplogBasePath, - m_Project->Identifier, - m_OplogId, - NiceTimeSpanMs(OplogTimer.GetElapsedTimeMs())); + Oplog = It->second; + Oplog->EnableUpdateCapture(); + m_OplogHasUpdateCapture = true; + } + else if (ProjectStore::Oplog::ExistsAt(m_OplogBasePath)) + { + Stopwatch OplogTimer; + Oplog = new ProjectStore::Oplog(m_Project->Log(), + m_Project->Identifier, + m_OplogId, + m_Project->m_CidStore, + m_OplogBasePath, + std::filesystem::path{}, + ProjectStore::Oplog::EMode::kBasicReadOnly); + Oplog->Read(); + if (Ctx.Settings.Verbose) + { + ZEN_INFO("GCV2: projectstore [PRECACHE] '{}': read oplog '{}/{}' in {}", + m_OplogBasePath, + m_Project->Identifier, + m_OplogId, + NiceTimeSpanMs(OplogTimer.GetElapsedTimeMs())); + } + } + else + { + return; } - } - else - { - return; } - RwLock::SharedLockScope ___(Oplog->m_OplogLock); + RwLock::SharedLockScope OplogLock(Oplog->m_OplogLock); if (Ctx.IsCancelledFlag) { return; @@ -6081,7 +6214,45 @@ public: } } - Oplog->GetAttachmentsLocked(m_References, Ctx.Settings.StoreProjectAttachmentMetaData); + if (Ctx.Settings.StoreProjectAttachmentMetaData) + { + Oplog->GetAttachmentsLocked(m_References, /*StoreMetaDataOnDisk*/ true); + } + else + { + Stopwatch SnapshotTimer; + ProjectStore::Oplog::OplogSnapshot Snapshot = Oplog->GetSnapshotLocked(); + OplogLock.ReleaseNow(); + + uint64_t AllocatedSize = + std::accumulate(Snapshot.PayloadBuffers.begin(), + Snapshot.PayloadBuffers.end(), + uint64_t(0), + [](uint64_t Current, const IoBuffer& Buffer) { return Current + Buffer.GetSize(); }); + uint64_t UsedSize = + std::accumulate(Snapshot.Ops.begin(), + Snapshot.Ops.end(), + uint64_t(0), + [](uint64_t Current, const CbObjectView& Object) { return Current + Object.GetSize(); }); + + ZEN_INFO( + "GCV2: projectstore [PRECACHE] '{}': Oplog snapshot fetched {} ops from {} op data using {} memory from oplog '{}/{}' " + "in {}", + m_OplogBasePath, + Snapshot.Ops.size(), + NiceBytes(UsedSize), + NiceBytes(AllocatedSize), + m_Project->Identifier, + m_OplogId, + NiceTimeSpanMs(SnapshotTimer.GetElapsedTimeMs())); + + for (size_t Index = 0; Index < Snapshot.Ops.size(); Index++) + { + CbObjectView& Op = Snapshot.Ops[Index]; + Op.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); }); + } + } + m_OplogAccessTime = m_Project->LastOplogAccessTime(m_OplogId); } FilterReferences(Ctx, fmt::format("projectstore [PRECACHE] '{}'", m_OplogBasePath), m_References); @@ -6145,7 +6316,43 @@ public: OplogTimer.Reset(); - Oplog->GetAttachmentsLocked(m_AddedReferences, Ctx.Settings.StoreProjectAttachmentMetaData); + if (Ctx.Settings.StoreProjectAttachmentMetaData) + { + Oplog->GetAttachmentsLocked(m_References, /*StoreMetaDataOnDisk*/ true); + } + else + { + Stopwatch SnapshotTimer; + ProjectStore::Oplog::OplogSnapshot Snapshot = Oplog->GetSnapshotLocked(); + + uint64_t AllocatedSize = + std::accumulate(Snapshot.PayloadBuffers.begin(), + Snapshot.PayloadBuffers.end(), + uint64_t(0), + [](uint64_t Current, const IoBuffer& Buffer) { return Current + Buffer.GetSize(); }); + uint64_t UsedSize = + std::accumulate(Snapshot.Ops.begin(), + Snapshot.Ops.end(), + uint64_t(0), + [](uint64_t Current, const CbObjectView& Object) { return Current + Object.GetSize(); }); + + ZEN_INFO( + "GCV2: projectstore [LOCKSTATE] '{}': Oplog snapshot fetched {} ops from {} op data using {} memory from oplog " + "'{}/{}' in {}", + m_OplogBasePath, + Snapshot.Ops.size(), + NiceBytes(UsedSize), + NiceBytes(AllocatedSize), + m_Project->Identifier, + m_OplogId, + NiceTimeSpanMs(SnapshotTimer.GetElapsedTimeMs())); + + for (size_t Index = 0; Index < Snapshot.Ops.size(); Index++) + { + CbObjectView& Op = Snapshot.Ops[Index]; + Op.IterateAttachments([&](CbFieldView Visitor) { m_AddedReferences.emplace_back(Visitor.AsAttachment()); }); + } + } } if (Ctx.Settings.Verbose) { diff --git a/src/zentelemetry-test/zentelemetry-test.cpp b/src/zentelemetry-test/zentelemetry-test.cpp index c8b067226..83fd549db 100644 --- a/src/zentelemetry-test/zentelemetry-test.cpp +++ b/src/zentelemetry-test/zentelemetry-test.cpp @@ -16,6 +16,10 @@ int main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) { +#if ZEN_PLATFORM_WINDOWS + setlocale(LC_ALL, "en_us.UTF8"); +#endif // ZEN_PLATFORM_WINDOWS + #if ZEN_WITH_TESTS zen::zentelemetry_forcelinktests(); diff --git a/src/zentelemetry/include/zentelemetry/otlptrace.h b/src/zentelemetry/include/zentelemetry/otlptrace.h index f191241ed..49dd90358 100644 --- a/src/zentelemetry/include/zentelemetry/otlptrace.h +++ b/src/zentelemetry/include/zentelemetry/otlptrace.h @@ -4,6 +4,7 @@ #include <zenbase/refcount.h> #include <zencore/memcmp.h> +#include <zencore/string.h> #include <zencore/uid.h> #include <span> @@ -27,12 +28,26 @@ class MemoryArena; namespace zen::otel { -using AttributeList = std::span<std::pair<std::string, std::string>>; +using AttributeList = std::span<std::pair<std::string_view, std::string_view>>; class Tracer; -// OLTP Span ID +/** Check if OTLP tracing is enabled + * + * This can be used to avoid unnecessary work when tracing is disabled. + * + * In many cases it is preferable and more convenient to use the ScopedSpan + * constructor which takes a naming function to avoid string formatting work + * when tracing is disabled. + */ +bool IsOtlpTraceEnabled(); +/** OTLP Span ID + * + * A SpanId is an 8-byte identifier for a span within a trace. It's not something + * that should be interpreted or manipulated directly, but it can be used + * for correlating spans during analysis. + */ struct SpanId { constexpr static size_t kSize = 8; @@ -60,7 +75,12 @@ private: uint8_t Id[kSize]; }; -// OLTP Trace ID +/** OTLP Trace ID + * + * A TraceId is a 16-byte identifier for a trace. It's not something + * that should be interpreted or manipulated directly, but it can be used + * for correlating traces during analysis. + */ struct TraceId { @@ -68,23 +88,35 @@ struct TraceId std::span<const uint8_t> GetBytes() const { return std::span<const uint8_t>(Id, kSize); } - inline TraceId() noexcept : Id{0} {} + inline TraceId() noexcept { memset(Id, 0, kSize); } explicit TraceId(const std::span<const uint8_t, kSize> Bytes) noexcept { std::copy(Bytes.begin(), Bytes.end(), Id); } std::strong_ordering operator<=>(const TraceId& Rhs) const noexcept { - int cmp = MemCmpFixed<kSize>(Id, Rhs.Id); - if (cmp < 0) + if (int Diff = MemCmpFixed<kSize>(Id, Rhs.Id); Diff < 0) + { return std::strong_ordering::less; - if (cmp > 0) + } + else if (Diff > 0) + { return std::strong_ordering::greater; - return std::strong_ordering::equal; + } + else + { + return std::strong_ordering::equal; + } } + inline operator bool() const { return !(reinterpret_cast<const uint64_t*>(Id)[0] == 0 && reinterpret_cast<const uint64_t*>(Id)[1] == 0); } + // Generates a new TraceId. The current scheme uses the session ID + // as the first 12 bytes, and a counter for the last 4 bytes. This makes + // it more likely that traces from the same session can be correlated, and + // should make indexes more efficient since the identifiers will be emitted + // in a mostly (lexically) increasing order. static TraceId NewTraceId(); inline const char* GetData() const { return reinterpret_cast<const char*>(Id); } @@ -93,6 +125,17 @@ private: uint8_t Id[kSize]; }; +/** OTEL attribute key-value pair + * + * An AttributePair is a key-value pair that provides additional information + * about a span or event. This class is intended to support a variety of + * value types, but currently only supports string and integer values. + * + * This is an internal structure used by the Span class, which encodes + * the key-value pair efficiently. Each instance is allocated within + * a MemoryArena associated with the trace and will be freed without + * explicit deallocation when the arena is destroyed. + */ struct AttributePair { const char* Key = nullptr; @@ -129,6 +172,12 @@ struct AttributePair AttributePair* Next = nullptr; }; +/** OTEL event + * + * An event represents a time-stamped annotation of the span, consisting + * of a name and optional attributes. + * + */ struct Event { Event* NextEvent = nullptr; @@ -142,6 +191,10 @@ struct Event * A span represents a single operation within a trace. Spans can be nested * to form a trace tree. * + * This class is reference counted in order to support spans which may + * cross thread boundaries. A single span may be referenced by multiple threads + * and will be finalized when the last reference is released. + * */ struct Span final : public TRefCounted<Span> @@ -225,7 +278,21 @@ private: class ScopedSpan final { public: + /** Create a new scoped span + * + * @param Name Name of the span + */ ScopedSpan(std::string_view Name); + + /** Create a new scoped span with an initializer function + * + * This allows initializing the span (e.g. adding attributes etc) in a safe way + * that avoids unnecessary work when OTLP tracing is disabled. + * + * @param Name Name of the span + * @param InitializerFunction Function which is called with the newly created span + * so that it can be initialized (e.g. adding attributes etc) + */ ScopedSpan(std::string_view Name, std::invocable<Span&> auto&& InitializerFunction) : ScopedSpan(Name) { if (m_Span) @@ -233,6 +300,43 @@ public: InitializerFunction(*m_Span); } } + + /** Construct a new span with a naming function + * + * The naming function will only be called if OTLP tracing is enabled. This can be + * used to avoid unnecessary string formatting when tracing is disabled. + * + * @param NamingFunction Function which is called with a string builder to create the span name + */ + ScopedSpan(std::invocable<StringBuilderBase&> auto&& NamingFunction) + { + if (!IsOtlpTraceEnabled()) + { + return; + } + + ExtendableStringBuilder<128> NameBuilder; + NamingFunction(NameBuilder); + } + + /** Construct a new span with a naming function AND initializer function + * + * Both functions will only be called if OTLP tracing is enabled. This can be + * used to avoid unnecessary string formatting and span initialization related work + * when tracing. + * + * @param NamingFunction Function which is called with a string builder to create the span name + * @param InitializerFunction Function which is called with the newly created span + * so that it can be initialized (e.g. adding attributes etc) + */ + ScopedSpan(std::invocable<StringBuilderBase&> auto&& NamingFunction, std::invocable<Span&> auto&& InitializerFunction) + : ScopedSpan(NamingFunction) + { + if (m_Span) + { + InitializerFunction(*m_Span); + } + } ScopedSpan() = delete; ~ScopedSpan(); @@ -241,11 +345,15 @@ public: ScopedSpan(ScopedSpan&& Rhs) = default; ScopedSpan(const ScopedSpan& Rhs) = default; - operator bool() const { return !!m_Span; } - void WithSpan(auto Func) const { Func(*m_Span); } + // Check if the span is valid (i.e OTLP tracing is enabled) + inline explicit operator bool() const { return !!m_Span; } + + // Execute a function with the span pointer if valid. This can + // be used to add attributes or events to the span after creation + inline void WithSpan(auto Func) const { Func(*m_Span); } private: - ScopedSpan(Span* InSpan, Tracer* InTracer); + void Initialize(std::string_view Name); Ref<Tracer> m_Tracer; // This needs to precede the span ref to ensure proper destruction order Ref<Span> m_Span; diff --git a/src/zentelemetry/otlptrace.cpp b/src/zentelemetry/otlptrace.cpp index f987afcfe..6a095cfeb 100644 --- a/src/zentelemetry/otlptrace.cpp +++ b/src/zentelemetry/otlptrace.cpp @@ -273,7 +273,7 @@ IsRecording() std::atomic<bool> g_OtlpTraceEnabled{false}; -inline bool +bool IsOtlpTraceEnabled() { return g_OtlpTraceEnabled.load(); @@ -346,6 +346,12 @@ ScopedSpan::ScopedSpan(std::string_view Name) return; } + Initialize(Name); +} + +void +ScopedSpan::Initialize(std::string_view Name) +{ Tracer* TracerPtr = Tracer::GetTracer(); Tracer::Impl* const ImplPtr = TracerPtr->m_Impl; @@ -359,12 +365,9 @@ ScopedSpan::ScopedSpan(std::string_view Name) m_Span = NewSpan; } -ScopedSpan::ScopedSpan(Span* InSpan, Tracer* InTracer) : m_Tracer(InTracer), m_Span(InSpan) -{ -} - ScopedSpan::~ScopedSpan() { + // this is not inline to avoid code bloat on every use site } } // namespace zen::otel diff --git a/src/zenutil-test/zenutil-test.cpp b/src/zenutil-test/zenutil-test.cpp index 3e3a11a01..f5cfd5a72 100644 --- a/src/zenutil-test/zenutil-test.cpp +++ b/src/zenutil-test/zenutil-test.cpp @@ -16,6 +16,10 @@ int main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) { +#if ZEN_PLATFORM_WINDOWS + setlocale(LC_ALL, "en_us.UTF8"); +#endif // ZEN_PLATFORM_WINDOWS + #if ZEN_WITH_TESTS zen::zenutil_forcelinktests(); diff --git a/src/zenutil/commandlineoptions.cpp b/src/zenutil/commandlineoptions.cpp index 81699361b..d94564843 100644 --- a/src/zenutil/commandlineoptions.cpp +++ b/src/zenutil/commandlineoptions.cpp @@ -2,7 +2,11 @@ #include <zenutil/commandlineoptions.h> +#include <zencore/string.h> #include <filesystem> + +#include <zencore/windows.h> + #if ZEN_WITH_TESTS # include <zencore/testing.h> #endif // ZEN_WITH_TESTS @@ -160,6 +164,29 @@ RemoveQuotes(const std::string_view& Arg) return Arg; } +CommandLineConverter::CommandLineConverter(int& argc, char**& argv) +{ +#if ZEN_PLATFORM_WINDOWS + LPWSTR RawCommandLine = GetCommandLineW(); + std::string CommandLine = WideToUtf8(RawCommandLine); + Args = ParseCommandLine(CommandLine); +#else + Args.reserve(argc); + for (int I = 0; I < argc; I++) + { + std::string Arg(argv[I]); + if ((!Arg.empty()) && (Arg != " ")) + { + Args.emplace_back(std::move(Arg)); + } + } +#endif + RawArgs = StripCommandlineQuotes(Args); + + argc = static_cast<int>(RawArgs.size()); + argv = RawArgs.data(); +} + #if ZEN_WITH_TESTS void diff --git a/src/zenutil/consul/consul.cpp b/src/zenutil/consul/consul.cpp new file mode 100644 index 000000000..6ddebf97a --- /dev/null +++ b/src/zenutil/consul/consul.cpp @@ -0,0 +1,140 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zenutil/consul.h> + +#include <zencore/except_fmt.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/process.h> +#include <zencore/string.h> +#include <zencore/timer.h> + +#include <fmt/format.h> + +namespace zen::consul { + +////////////////////////////////////////////////////////////////////////// + +struct ConsulProcess::Impl +{ + Impl(std::string_view BaseUri) : m_HttpClient(BaseUri) {} + ~Impl() = default; + + void SpawnConsulAgent() + { + if (m_ProcessHandle.IsValid()) + { + return; + } + + CreateProcOptions Options; + Options.Flags |= CreateProcOptions::Flag_Windows_NewProcessGroup; + + CreateProcResult Result = CreateProc("consul" ZEN_EXE_SUFFIX_LITERAL, "consul" ZEN_EXE_SUFFIX_LITERAL " agent -dev", Options); + + if (Result) + { + m_ProcessHandle.Initialize(Result); + + Stopwatch Timer; + + // Poll to check when the agent is ready + + do + { + Sleep(100); + HttpClient::Response Resp = m_HttpClient.Get("v1/status/leader"); + if (Resp) + { + ZEN_INFO("Consul agent started successfully (waited {})", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + + return; + } + } while (Timer.GetElapsedTimeMs() < 10000); + } + + // Report failure! + + ZEN_WARN("Consul agent failed to start within timeout period"); + } + + void StopConsulAgent() + { + if (!m_ProcessHandle.IsValid()) + { + return; + } + + // This waits for the process to exit and also resets the handle + m_ProcessHandle.Kill(); + } + +private: + ProcessHandle m_ProcessHandle; + HttpClient m_HttpClient; +}; + +ConsulProcess::ConsulProcess() : m_Impl(std::make_unique<Impl>("http://localhost:8500/")) +{ +} + +ConsulProcess::~ConsulProcess() +{ +} + +void +ConsulProcess::SpawnConsulAgent() +{ + m_Impl->SpawnConsulAgent(); +} + +void +ConsulProcess::StopConsulAgent() +{ + m_Impl->StopConsulAgent(); +} + +////////////////////////////////////////////////////////////////////////// + +ConsulClient::ConsulClient(std::string_view BaseUri) : m_HttpClient(BaseUri) +{ +} + +ConsulClient::~ConsulClient() +{ +} + +void +ConsulClient::SetKeyValue(std::string_view Key, std::string_view Value) +{ + IoBuffer ValueBuffer = IoBufferBuilder::MakeFromMemory(MakeMemoryView(Value)); + HttpClient::Response Result = + m_HttpClient.Put(fmt::format("v1/kv/{}", Key), ValueBuffer, {{"Content-Type", "text/plain"}, {"Accept", "application/json"}}); + if (!Result) + { + throw runtime_error("ConsulClient::SetKeyValue() failed to set key '{}' ({})", Key, Result.ErrorMessage("")); + } +} + +std::string +ConsulClient::GetKeyValue(std::string_view Key) +{ + HttpClient::Response Result = m_HttpClient.Get(fmt::format("v1/kv/{}?raw", Key)); + if (!Result) + { + throw runtime_error("ConsulClient::GetKeyValue() failed to get key '{}' ({})", Key, Result.ErrorMessage("")); + } + return Result.ToText(); +} + +void +ConsulClient::DeleteKey(std::string_view Key) +{ + HttpClient::Response Result = m_HttpClient.Delete(fmt::format("v1/kv/{}", Key)); + if (!Result) + { + throw runtime_error("ConsulClient::DeleteKey() failed to delete key '{}' ({})", Key, Result.ErrorMessage("")); + } +} + +} // namespace zen::consul diff --git a/src/zenutil/include/zenutil/commandlineoptions.h b/src/zenutil/include/zenutil/commandlineoptions.h index d6a171242..01cceedb1 100644 --- a/src/zenutil/include/zenutil/commandlineoptions.h +++ b/src/zenutil/include/zenutil/commandlineoptions.h @@ -22,6 +22,19 @@ std::vector<char*> StripCommandlineQuotes(std::vector<std::string>& InOutArgs) std::filesystem::path StringToPath(const std::string_view& Path); std::string_view RemoveQuotes(const std::string_view& Arg); +class CommandLineConverter +{ +public: + CommandLineConverter(int& argc, char**& argv); + + int ArgC = 0; + char** ArgV = nullptr; + +private: + std::vector<std::string> Args; + std::vector<char*> RawArgs; +}; + void commandlineoptions_forcelink(); // internal } // namespace zen diff --git a/src/zenutil/include/zenutil/consul.h b/src/zenutil/include/zenutil/consul.h new file mode 100644 index 000000000..08871fa66 --- /dev/null +++ b/src/zenutil/include/zenutil/consul.h @@ -0,0 +1,47 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenbase/zenbase.h> +#include <zenhttp/httpclient.h> + +#include <string> +#include <string_view> + +namespace zen::consul { + +class ConsulClient +{ +public: + ConsulClient(std::string_view BaseUri); + ~ConsulClient(); + + ConsulClient(const ConsulClient&) = delete; + ConsulClient& operator=(const ConsulClient&) = delete; + + void SetKeyValue(std::string_view Key, std::string_view Value); + std::string GetKeyValue(std::string_view Key); + void DeleteKey(std::string_view Key); + +private: + HttpClient m_HttpClient; +}; + +class ConsulProcess +{ +public: + ConsulProcess(); + ~ConsulProcess(); + + ConsulProcess(const ConsulProcess&) = delete; + ConsulProcess& operator=(const ConsulProcess&) = delete; + + void SpawnConsulAgent(); + void StopConsulAgent(); + +private: + struct Impl; + std::unique_ptr<Impl> m_Impl; +}; + +} // namespace zen::consul diff --git a/src/zenutil/include/zenutil/logging/rotatingfilesink.h b/src/zenutil/include/zenutil/logging/rotatingfilesink.h index 4d10f3794..8901b7779 100644 --- a/src/zenutil/include/zenutil/logging/rotatingfilesink.h +++ b/src/zenutil/include/zenutil/logging/rotatingfilesink.h @@ -11,6 +11,7 @@ ZEN_THIRD_PARTY_INCLUDES_START #include <spdlog/sinks/sink.h> ZEN_THIRD_PARTY_INCLUDES_END +#include <atomic> #include <filesystem> namespace zen::logging { @@ -248,7 +249,7 @@ private: const std::size_t m_MaxSize; const std::size_t m_MaxFiles; BasicFile m_CurrentFile; - bool m_NeedFlush = false; + std::atomic<bool> m_NeedFlush = false; }; } // namespace zen::logging diff --git a/src/zenutil/include/zenutil/zenserverprocess.h b/src/zenutil/include/zenutil/zenserverprocess.h index 0da63285b..d0402640b 100644 --- a/src/zenutil/include/zenutil/zenserverprocess.h +++ b/src/zenutil/include/zenutil/zenserverprocess.h @@ -34,8 +34,10 @@ public: void Initialize(std::filesystem::path ProgramBaseDir); void InitializeForTest(std::filesystem::path ProgramBaseDir, std::filesystem::path TestBaseDir, std::string_view ServerClass = ""); + void InitializeForHub(std::filesystem::path ProgramBaseDir, std::filesystem::path TestBaseDir, std::string_view ServerClass = ""); std::filesystem::path CreateNewTestDir(); + std::filesystem::path CreateChildDir(std::string_view ChildName); std::filesystem::path ProgramBaseDir() const { return m_ProgramBaseDir; } std::filesystem::path GetTestRootDir(std::string_view Path); inline bool IsInitialized() const { return m_IsInitialized; } @@ -43,11 +45,18 @@ public: inline std::string_view GetServerClass() const { return m_ServerClass; } inline uint16_t GetNewPortNumber() { return m_NextPortNumber.fetch_add(1); } + // The defaults will work for a single root process only. For hierarchical + // setups (e.g., hub managing storage servers), we need to be able to + // allocate distinct child IDs and ports to avoid overlap/conflicts. + static void SetBaseChildId(int InitialValue); + void SetNextPortNumber(uint16_t NewValue) { m_NextPortNumber = NewValue; } + private: std::filesystem::path m_ProgramBaseDir; - std::filesystem::path m_TestBaseDir; + std::filesystem::path m_ChildProcessBaseDir; bool m_IsInitialized = false; bool m_IsTestInstance = false; + bool m_IsHubInstance = false; std::string m_ServerClass; std::atomic_uint16_t m_NextPortNumber{20000}; }; @@ -60,10 +69,19 @@ private: Especially useful for automated testing but can also be used for management tools. + This is also used by zenserver in hub mode, for managing storage + server instances. + */ struct ZenServerInstance { - ZenServerInstance(ZenServerEnvironment& TestEnvironment); + enum class ServerMode + { + kStorageServer, // default + kHubServer, + }; + + ZenServerInstance(ZenServerEnvironment& TestEnvironment, ServerMode Mode = ServerMode::kStorageServer); ~ZenServerInstance(); int Shutdown(); @@ -72,6 +90,7 @@ struct ZenServerInstance [[nodiscard]] bool WaitUntilReady(int Timeout); [[nodiscard]] bool WaitUntilExited(int Timeout, std::error_code& OutEc); void EnableTermination() { m_Terminate = true; } + void EnableShutdownOnDestroy() { m_ShutdownOnDestroy = true; } void DisableShutdownOnDestroy() { m_ShutdownOnDestroy = false; } void Detach(); inline int GetPid() const { return m_Process.Pid(); } @@ -81,7 +100,11 @@ struct ZenServerInstance bool Terminate(); std::string GetLogOutput() const; - void SetTestDir(std::filesystem::path TestDir); + inline ServerMode GetServerMode() const { return m_ServerMode; } + + inline void SetServerExecutablePath(std::filesystem::path ExecutablePath) { m_ServerExecutablePath = ExecutablePath; } + + void SetDataDir(std::filesystem::path TestDir); inline void SpawnServer(std::string_view AdditionalServerArgs = std::string_view()) { @@ -117,11 +140,13 @@ private: std::unique_ptr<NamedEvent> m_ShutdownEvent; bool m_Terminate = false; bool m_ShutdownOnDestroy = true; - std::filesystem::path m_TestDir; - uint16_t m_BasePort = 0; + std::filesystem::path m_DataDir; + uint16_t m_BasePort = 0; + ServerMode m_ServerMode = ServerMode::kStorageServer; std::optional<int> m_OwnerPid; std::string m_Name; std::filesystem::path m_OutputCapturePath; + std::filesystem::path m_ServerExecutablePath; void CreateShutdownEvent(int BasePort); void SpawnServer(int BasePort, std::string_view AdditionalServerArgs, int WaitTimeoutMs); diff --git a/src/zenutil/service.cpp b/src/zenutil/service.cpp index 103fdaa2f..f2a925e61 100644 --- a/src/zenutil/service.cpp +++ b/src/zenutil/service.cpp @@ -229,12 +229,13 @@ namespace { std::filesystem::path GetPListPath(const std::string& DaemonName) { - const std::filesystem::path PListFolder = "/Library/LaunchDaemons"; + char* HomeDir = getenv("HOME"); + std::filesystem::path PListFolder = HomeDir; + PListFolder /= "Library/LaunchAgents"; return PListFolder / (DaemonName + ".plist"); } - std::string BuildPlist(std::string_view ServiceName, - const std::filesystem::path& ExecutablePath, + std::string BuildPlist(const std::filesystem::path& ExecutablePath, std::string_view CommandLineOptions, std::string_view DaemonName, bool Debug) @@ -265,14 +266,8 @@ namespace { " <key>RunAtLoad</key>\n" " <true/>\n" " \n" - // " <key>KeepAlive</key>\n" - // " <true/>\n" - // " \n" - " <key>StandardOutPath</key>\n" - " <string>/var/log/{}.log</string>\n" - " \n" - " <key>StandardErrorPath</key>\n" - " <string>/var/log/{}.err.log</string>\n" + " <key>KeepAlive</key>\n" + " <true/>\n" " \n" " <key>Debug</key>\n" " <{}/>\n" @@ -282,22 +277,7 @@ namespace { DaemonName, ExecutablePath, ProgramArguments.ToView(), - ServiceName, - ServiceName, Debug ? "true"sv : "false"sv); - - // "<key>Sockets</key>" - // "<dict>" - // "<key>Listeners</key>" - // "<dict>" - // "<key>SockServiceName</key>" - // "<string>{}</string>" // Listen socket - // "<key>SockType</key>" - // "<string>tcp</string>" - // "<key>SockFamily</key>" - // "<string>IPv4</string>" - // "</dict>" - // "</dict>" } #endif // ZEN_PLATFORM_MAC @@ -752,9 +732,8 @@ StopService(std::string_view ServiceName) std::error_code InstallService(std::string_view ServiceName, const ServiceSpec& Spec) { - // TODO: Do we need to create a separate user for the service or is running as the default service user OK? const std::string DaemonName = GetDaemonName(ServiceName); - std::string PList = BuildPlist(ServiceName, Spec.ExecutablePath, Spec.CommandLineOptions, DaemonName, true); + std::string PList = BuildPlist(Spec.ExecutablePath, Spec.CommandLineOptions, DaemonName, true); const std::filesystem::path PListPath = GetPListPath(DaemonName); ZEN_INFO("Writing launchd plist to {}", PListPath.string()); @@ -910,8 +889,14 @@ StartService(std::string_view ServiceName) { const std::string DaemonName = GetDaemonName(ServiceName); const std::filesystem::path PListPath = GetPListPath(DaemonName); + std::pair<int, std::string> User = ExecuteProgram("id -u"); + + if (User.first != 0) + { + return MakeErrorCode(User.first); + } - std::pair<int, std::string> Res = ExecuteProgram(fmt::format("launchctl bootstrap system {}", PListPath)); + std::pair<int, std::string> Res = ExecuteProgram(fmt::format("launchctl bootstrap gui/{} {}", User.second, PListPath)); if (Res.first != 0) { return MakeErrorCode(Res.first); @@ -926,13 +911,18 @@ StopService(std::string_view ServiceName) const std::string DaemonName = GetDaemonName(ServiceName); const std::filesystem::path PListPath = GetPListPath(DaemonName); - /* - std::pair<int, std::string> Res = ExecuteProgram(fmt::format("launchctl bootout system ", PListPath.)); + std::pair<int, std::string> User = ExecuteProgram("id -u"); + + if (User.first != 0) + { + return MakeErrorCode(User.first); + } + + std::pair<int, std::string> Res = ExecuteProgram(fmt::format("launchctl bootout gui/{} {}", User.second, PListPath)); if (Res.first != 0) { return MakeErrorCode(Res.first); } - */ return {}; } diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index 6a93f0c63..ef2a4fda5 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -31,6 +31,19 @@ namespace zen { +// this needs to key off the current process child-id, in order to avoid conflicts +// in situations where we have a tree of zenserver child processes (such as in hub +// tests) + +std::atomic<int> ChildIdCounter{0}; + +void +ZenServerEnvironment::SetBaseChildId(int InitialValue) +{ + ZEN_ASSERT(ChildIdCounter == 0); + ChildIdCounter = InitialValue; +} + namespace zenutil { #if ZEN_PLATFORM_WINDOWS class SecurityAttributes @@ -507,8 +520,8 @@ ZenServerEnvironment::InitializeForTest(std::filesystem::path ProgramBaseDir, { using namespace std::literals; - m_ProgramBaseDir = ProgramBaseDir; - m_TestBaseDir = TestBaseDir; + m_ProgramBaseDir = ProgramBaseDir; + m_ChildProcessBaseDir = TestBaseDir; ZEN_INFO("Program base dir is '{}'", ProgramBaseDir); ZEN_INFO("Cleaning test base dir '{}'", TestBaseDir); @@ -536,6 +549,59 @@ ZenServerEnvironment::InitializeForTest(std::filesystem::path ProgramBaseDir, } } +void +ZenServerEnvironment::InitializeForHub(std::filesystem::path ProgramBaseDir, + std::filesystem::path ChildBaseDir, + std::string_view ServerClass) +{ + using namespace std::literals; + + m_ProgramBaseDir = ProgramBaseDir; + m_ChildProcessBaseDir = ChildBaseDir; + + ZEN_INFO("Program base dir is '{}'", m_ProgramBaseDir); + ZEN_INFO("Cleaning child base dir '{}'", m_ChildProcessBaseDir); + DeleteDirectories(m_ChildProcessBaseDir.c_str()); + + m_IsHubInstance = true; + m_IsInitialized = true; + + if (ServerClass.empty()) + { +#if ZEN_WITH_HTTPSYS + if (!zen::windows::IsRunningOnWine()) + { + m_ServerClass = "httpsys"sv; + + return; + } +#endif + + m_ServerClass = "asio"sv; + } + else + { + m_ServerClass = ServerClass; + } +} + +std::filesystem::path +ZenServerEnvironment::CreateChildDir(std::string_view ChildName) +{ + using namespace std::literals; + + std::filesystem::path ChildPath = m_ChildProcessBaseDir / ChildName; + + if (!IsDir(ChildPath)) + { + ZEN_INFO("Creating new test dir @ '{}'", ChildPath); + + CreateDirectories(ChildPath); + } + + return ChildPath; +} + std::filesystem::path ZenServerEnvironment::CreateNewTestDir() { @@ -544,7 +610,7 @@ ZenServerEnvironment::CreateNewTestDir() ExtendableWideStringBuilder<256> TestDir; TestDir << "test"sv << int64_t(ZenServerTestCounter.fetch_add(1)); - std::filesystem::path TestPath = m_TestBaseDir / TestDir.c_str(); + std::filesystem::path TestPath = m_ChildProcessBaseDir / TestDir.c_str(); ZEN_ASSERT(!IsDir(TestPath)); ZEN_INFO("Creating new test dir @ '{}'", TestPath); @@ -566,11 +632,11 @@ ZenServerEnvironment::GetTestRootDir(std::string_view Path) ////////////////////////////////////////////////////////////////////////// -std::atomic<int> ChildIdCounter{0}; - -ZenServerInstance::ZenServerInstance(ZenServerEnvironment& TestEnvironment) : m_Env(TestEnvironment) +ZenServerInstance::ZenServerInstance(ZenServerEnvironment& TestEnvironment, ServerMode Mode) : m_Env(TestEnvironment), m_ServerMode(Mode) { ZEN_ASSERT(TestEnvironment.IsInitialized()); + + m_ServerMode = Mode; } ZenServerInstance::~ZenServerInstance() @@ -632,7 +698,7 @@ ZenServerInstance::Shutdown() { Stopwatch Timer; ZEN_DEBUG("Waiting for zenserver process {} ({}) to shut down", m_Name, m_Process.Pid()); - while (!m_Process.Wait(1000)) + while (!m_Process.Wait(2000)) { if (!m_Process.IsValid()) { @@ -710,6 +776,22 @@ ZenServerInstance::SpawnServer(std::string_view ServerArgs, bool OpenConsole, in SpawnServerInternal(ChildId, ServerArgs, OpenConsole, WaitTimeoutMs); } +std::string_view +ToString(ZenServerInstance::ServerMode Mode) +{ + using namespace std::literals; + + switch (Mode) + { + case ZenServerInstance::ServerMode::kStorageServer: + return "storage"sv; + case ZenServerInstance::ServerMode::kHubServer: + return "hub"sv; + default: + return "invalid"sv; + } +} + void ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, bool OpenConsole, int WaitTimeoutMs) { @@ -721,6 +803,12 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, ExtendableStringBuilder<512> CommandLine; CommandLine << "zenserver" ZEN_EXE_SUFFIX_LITERAL; // see CreateProc() call for actual binary path + + if (m_ServerMode == ServerMode::kHubServer) + { + CommandLine << " hub"; + } + CommandLine << " --child-id " << ChildEventName; if (!ServerArgs.empty()) @@ -730,7 +818,7 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, std::filesystem::path CurrentDirectory = std::filesystem::current_path(); - ZEN_DEBUG("Spawning server '{}'", m_Name); + ZEN_DEBUG("Spawning {} server '{}'", ToString(m_ServerMode), m_Name); uint32_t CreationFlags = 0; if (OpenConsole) @@ -738,8 +826,9 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, CreationFlags |= CreateProcOptions::Flag_NewConsole; } - const std::filesystem::path BaseDir = m_Env.ProgramBaseDir(); - const std::filesystem::path Executable = BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL; + const std::filesystem::path BaseDir = m_Env.ProgramBaseDir(); + const std::filesystem::path Executable = + m_ServerExecutablePath.empty() ? (BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL) : m_ServerExecutablePath; const std::filesystem::path OutputPath = OpenConsole ? std::filesystem::path{} : std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"); CreateProcOptions CreateOptions = {.WorkingDirectory = &CurrentDirectory, .Flags = CreationFlags, .StdoutFile = OutputPath}; @@ -799,8 +888,7 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr const int ChildId = AssignName(); ExtendableStringBuilder<512> CommandLine; - - const bool IsTest = m_Env.IsTestEnvironment(); + const bool IsTest = m_Env.IsTestEnvironment(); if (IsTest) { @@ -835,10 +923,10 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr m_BasePort = gsl::narrow_cast<uint16_t>(BasePort); } - if (!m_TestDir.empty()) + if (!m_DataDir.empty()) { CommandLine << " --data-dir "; - PathToUtf8(m_TestDir.c_str(), CommandLine); + PathToUtf8(m_DataDir.c_str(), CommandLine); } if (!AdditionalServerArgs.empty()) @@ -1028,10 +1116,10 @@ ZenServerInstance::GetBaseUri() const } void -ZenServerInstance::SetTestDir(std::filesystem::path TestDir) +ZenServerInstance::SetDataDir(std::filesystem::path TestDir) { ZEN_ASSERT(!m_Process.IsValid()); - m_TestDir = TestDir; + m_DataDir = TestDir; } bool |