diff options
Diffstat (limited to 'src')
29 files changed, 954 insertions, 381 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index abb4bfe0c..cd27daa9e 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -806,6 +806,50 @@ namespace { return CleanWipe; } + bool ParseCloudUrl(std::string_view InUrl, + std::string& OutHost, + std::string& OutNamespace, + std::string& OutBucket, + std::string& OutBuildId) + { + std::string Url(RemoveQuotes(InUrl)); + const std::string_view ExtendedApiString = "api/v2/builds/"; + if (auto ApiString = ToLower(Url).find(ExtendedApiString); ApiString != std::string::npos) + { + Url.erase(ApiString, ExtendedApiString.length()); + } + + const std::string ArtifactURLRegExString = R"((http[s]?:\/\/.*?)\/(.*?)\/(.*?)\/(.*))"; + const std::regex ArtifactURLRegEx(ArtifactURLRegExString, std::regex::ECMAScript | std::regex::icase); + std::match_results<std::string_view::const_iterator> MatchResults; + std::string_view UrlToParse(Url); + if (regex_match(begin(UrlToParse), end(UrlToParse), MatchResults, ArtifactURLRegEx) && MatchResults.size() == 5) + { + auto GetMatch = [&MatchResults](uint32_t Index) -> std::string_view { + ZEN_ASSERT(Index < MatchResults.size()); + + const auto& Match = MatchResults[Index]; + + return std::string_view(&*Match.first, Match.second - Match.first); + }; + + const std::string_view Host = GetMatch(1); + const std::string_view Namespace = GetMatch(2); + const std::string_view Bucket = GetMatch(3); + const std::string_view BuildId = GetMatch(4); + + OutHost = Host; + OutNamespace = Namespace; + OutBucket = Bucket; + OutBuildId = BuildId; + return true; + } + else + { + return false; + } + } + std::string ReadAccessTokenFromFile(const std::filesystem::path& Path) { if (!IsFile(Path)) @@ -2735,14 +2779,18 @@ namespace { std::vector<CbObject>({BlockMetaData})); } - Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); - ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", - BlockHash, - NiceBytes(BlockMetaData.GetSize())); + bool MetadataSucceeded = + Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + if (MetadataSucceeded) + { + ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", + BlockHash, + NiceBytes(BlockMetaData.GetSize())); - OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + UploadStats.BlocksBytes += BlockMetaData.GetSize(); + } - UploadStats.BlocksBytes += BlockMetaData.GetSize(); UploadStats.BlockCount++; if (UploadStats.BlockCount == NewBlockCount) { @@ -2944,13 +2992,16 @@ namespace { std::vector<IoHash>({BlockHash}), std::vector<CbObject>({BlockMetaData})); } - Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); - ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize())); + bool MetadataSucceeded = Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + if (MetadataSucceeded) + { + ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize())); - NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + UploadStats.BlocksBytes += BlockMetaData.GetSize(); + } UploadStats.BlockCount++; - UploadStats.BlocksBytes += BlockMetaData.GetSize(); UploadedBlockCount++; if (UploadedBlockCount == UploadBlockCount && UploadedChunkCount == UploadChunkCount) @@ -3503,7 +3554,11 @@ namespace { { ZEN_TRACE_CPU("FindBlocks"); Stopwatch KnownBlocksTimer; - Result.KnownBlocks = ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId, FindBlockMaxCount)); + CbObject BlockDescriptionList = Storage.BuildStorage->FindBlocks(BuildId, FindBlockMaxCount); + if (BlockDescriptionList) + { + Result.KnownBlocks = ParseChunkBlockDescriptionList(BlockDescriptionList); + } FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size(); Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); @@ -4156,29 +4211,44 @@ namespace { if (!NewBlocks.BlockDescriptions.empty() && !AbortFlag) { - uint64_t UploadBlockMetadataCount = 0; - std::vector<IoHash> BlockHashes; - BlockHashes.reserve(NewBlocks.BlockDescriptions.size()); + uint64_t UploadBlockMetadataCount = 0; Stopwatch UploadBlockMetadataTimer; - for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++) + + uint32_t FailedMetadataUploadCount = 1; + int32_t MetadataUploadRetryCount = 3; + while ((MetadataUploadRetryCount-- > 0) && (FailedMetadataUploadCount > 0)) { - const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; - if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex]) + FailedMetadataUploadCount = 0; + for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++) { - const CbObject BlockMetaData = - BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); - if (Storage.BuildCacheStorage) + if (AbortFlag) { - Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, - std::vector<IoHash>({BlockHash}), - std::vector<CbObject>({BlockMetaData})); + break; + } + const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; + if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex]) + { + const CbObject BlockMetaData = + BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, + std::vector<IoHash>({BlockHash}), + std::vector<CbObject>({BlockMetaData})); + } + bool MetadataSucceeded = Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + if (MetadataSucceeded) + { + UploadStats.BlocksBytes += BlockMetaData.GetSize(); + NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + UploadBlockMetadataCount++; + } + else + { + FailedMetadataUploadCount++; + } } - Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); - UploadStats.BlocksBytes += BlockMetaData.GetSize(); - NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; - UploadBlockMetadataCount++; } - BlockHashes.push_back(BlockHash); } if (UploadBlockMetadataCount > 0) { @@ -7440,7 +7510,10 @@ namespace { } if (!BlockBuffer) { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + throw std::runtime_error(fmt::format("Block {} is missing when fetching range {} -> {}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeStart + BlockRange.RangeLength)); } if (!AbortFlag) { @@ -10076,31 +10149,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("buildid is not compatible with the url option\n{}", SubOption->help())); } - const std::string ArtifactURLRegExString = R"((.*?:\/\/.*?)\/api\/v2\/builds\/(.*?)\/(.*?)\/(.*))"; - const std::regex ArtifactURLRegEx(ArtifactURLRegExString, std::regex::ECMAScript); - std::match_results<std::string_view::const_iterator> MatchResults; - const std::string_view Url(RemoveQuotes(m_Url)); - if (regex_match(begin(Url), end(Url), MatchResults, ArtifactURLRegEx) && MatchResults.size() == 5) - { - auto GetMatch = [&MatchResults](uint32_t Index) -> std::string_view { - ZEN_ASSERT(Index < MatchResults.size()); - - const auto& Match = MatchResults[Index]; - - return std::string_view(&*Match.first, Match.second - Match.first); - }; - - const std::string_view Host = GetMatch(1); - const std::string_view Namespace = GetMatch(2); - const std::string_view Bucket = GetMatch(3); - const std::string_view BuildId = GetMatch(4); - - m_Host = Host; - m_Namespace = Namespace; - m_Bucket = Bucket; - m_BuildId = BuildId; - } - else + if (!ParseCloudUrl(m_Url, m_Host, m_Namespace, m_Bucket, m_BuildId)) { throw zen::OptionParseException(fmt::format("url does not match the Cloud Artifact URL format\n{}", SubOption->help())); } diff --git a/src/zen/cmds/version_cmd.cpp b/src/zen/cmds/version_cmd.cpp index 41042533d..7dfa125e4 100644 --- a/src/zen/cmds/version_cmd.cpp +++ b/src/zen/cmds/version_cmd.cpp @@ -2,10 +2,12 @@ #include "version_cmd.h" +#include <zencore/basicfile.h> #include <zencore/config.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> +#include <zenhttp/httpclient.h> #include <zenhttp/httpcommon.h> #include <zenutil/zenserverprocess.h> @@ -17,11 +19,14 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { +using namespace std::literals; + VersionCommand::VersionCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName), "[hosturl]"); m_Options.add_option("", "d", "detailed", "Detailed Version", cxxopts::value(m_DetailedVersion), "[detailedversion]"); + m_Options.add_option("", "o", "output-path", "Path for output", cxxopts::value(m_OutputPath), "[outputpath]"); m_Options.parse_positional({"hosturl"}); } @@ -51,28 +56,40 @@ VersionCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - const std::string UrlBase = fmt::format("{}/health", m_HostName); - cpr::Session Session; - std::string VersionRequest = fmt::format("{}/version{}", UrlBase, m_DetailedVersion ? "?detailed=true" : ""); - Session.SetUrl(VersionRequest); - cpr::Response Response = Session.Get(); - if (!zen::IsHttpSuccessCode(Response.status_code)) + if (!m_OutputPath.empty()) { - if (Response.status_code) - { - ZEN_ERROR("{} failed: {}: {} ({})", VersionRequest, Response.status_code, Response.reason, Response.text); - } - else - { - ZEN_ERROR("{} failed: {}", VersionRequest, Response.error.message); - } + ZEN_CONSOLE("Querying host {}", m_HostName); + } + HttpClient Client(m_HostName, HttpClientSettings{.Timeout = std::chrono::milliseconds(5000)}); + + HttpClient::KeyValueMap Parameters; + if (m_DetailedVersion) + { + Parameters.Entries.insert_or_assign("detailed", "true"); + } + const std::string_view VersionRequest("/health/version"sv); + HttpClient::Response Response = Client.Get(VersionRequest, {}, Parameters); + if (!Response.IsSuccess()) + { + ZEN_ERROR("{} failed: {}", VersionRequest, Response.ErrorMessage(""sv)); return 1; } - Version = Response.text; + Version = Response.AsText(); } - ZEN_CONSOLE("{}", Version); + if (m_OutputPath.empty()) + { + ZEN_CONSOLE("{}", Version); + } + else + { + ZEN_CONSOLE("Writing version '{}' to '{}'", Version, m_OutputPath); + + BasicFile OutputFile(m_OutputPath, BasicFile::Mode::kTruncate); + OutputFile.Write(Version.data(), Version.length(), 0); + OutputFile.Close(); + } return 0; } diff --git a/src/zen/cmds/version_cmd.h b/src/zen/cmds/version_cmd.h index f8d16fb96..7a910e463 100644 --- a/src/zen/cmds/version_cmd.h +++ b/src/zen/cmds/version_cmd.h @@ -9,6 +9,9 @@ namespace zen { class VersionCommand : public ZenCmdBase { public: + static constexpr char Name[] = "version"; + static constexpr char Description[] = "Get zen service version"; + VersionCommand(); ~VersionCommand(); @@ -16,9 +19,10 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"version", "Get zen service version"}; - std::string m_HostName; - bool m_DetailedVersion = false; + cxxopts::Options m_Options{Name, Description}; + std::string m_HostName; + bool m_DetailedVersion = false; + std::filesystem::path m_OutputPath; }; } // namespace zen diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 614112235..24bd545e2 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -37,6 +37,7 @@ #include <zencore/trace.h> #include <zencore/windows.h> #include <zenhttp/httpcommon.h> +#include <zenutil/environmentoptions.h> #include <zenutil/logging.h> #include <zenutil/zenserverprocess.h> @@ -737,7 +738,7 @@ main(int argc, char** argv) {"top", &TopCmd, "Monitor zen server activity"}, {"trace", &TraceCmd, "Control zen realtime tracing"}, {"up", &UpCmd, "Bring zen server up"}, - {"version", &VersionCmd, "Get zen server version"}, + {VersionCommand::Name, &VersionCmd, VersionCommand::Description}, {"vfs", &VfsCmd, "Manage virtual file system"}, {"flush", &FlushCmd, "Flush storage"}, {WipeCommand::Name, &WipeCmd, WipeCommand::Description}, @@ -874,18 +875,22 @@ main(int argc, char** argv) #endif // ZEN_WITH_TRACE #if ZEN_USE_SENTRY - bool NoSentry = false; - bool SentryAllowPII = false; - std::string SentryDsn; + + SentryIntegration::Config SentryConfig; + + bool NoSentry = false; + Options .add_option("sentry", "", "no-sentry", "Disable Sentry crash handler", cxxopts::value<bool>(NoSentry)->default_value("false"), ""); Options.add_option("sentry", "", "sentry-allow-personal-info", "Allow personally identifiable information in sentry crash reports", - cxxopts::value<bool>(SentryAllowPII)->default_value("false"), + cxxopts::value<bool>(SentryConfig.AllowPII)->default_value("false"), ""); - Options.add_option("sentry", "", "sentry-dsn", "Sentry DSN to send events to", cxxopts::value<std::string>(SentryDsn), ""); + Options.add_option("sentry", "", "sentry-dsn", "Sentry DSN to send events to", cxxopts::value<std::string>(SentryConfig.Dsn), ""); + Options.add_option("sentry", "", "sentry-environment", "Sentry environment", cxxopts::value<std::string>(SentryConfig.Environment), ""); + Options.add_options()("sentry-debug", "Enable debug mode for Sentry", cxxopts::value<bool>(SentryConfig.Debug)->default_value("false")); #endif Options.parse_positional({"command"}); @@ -930,6 +935,27 @@ main(int argc, char** argv) } #if ZEN_USE_SENTRY + + { + EnvironmentOptions EnvOptions; + + EnvOptions.AddOption("UE_ZEN_SENTRY_DSN"sv, SentryConfig.Dsn, "sentry-dsn"sv); + EnvOptions.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, SentryConfig.AllowPII, "sentry-allow-personal-info"sv); + EnvOptions.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, SentryConfig.Environment, "sentry-environment"sv); + + bool EnvEnableSentry = !NoSentry; + EnvOptions.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv); + + EnvOptions.AddOption("UE_ZEN_SENTRY_DEBUG"sv, SentryConfig.Debug, "sentry-debug"sv); + + EnvOptions.Parse(ParseResult); + + if (EnvEnableSentry != !NoSentry) + { + NoSentry = !EnvEnableSentry; + } + } + SentryIntegration Sentry; if (NoSentry == false) @@ -947,7 +973,9 @@ main(int argc, char** argv) SB.Append(argv[i]); } - Sentry.Initialize(SentryDatabasePath, {}, SentryDsn, SentryAllowPII, SB.ToString()); + SentryConfig.DatabasePath = SentryDatabasePath; + + Sentry.Initialize(SentryConfig, SB.ToString()); SentryIntegration::ClearCaches(); } diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h index d14c1c275..faf1238b7 100644 --- a/src/zencore/include/zencore/sentryintegration.h +++ b/src/zencore/include/zencore/sentryintegration.h @@ -31,11 +31,17 @@ public: SentryIntegration(); ~SentryIntegration(); - void Initialize(std::string SentryDatabasePath, - std::string SentryAttachmentsPath, - std::string SentryDsn, - bool AllowPII, - const std::string& CommandLine); + struct Config + { + std::string DatabasePath; + std::string AttachmentsPath; + std::string Dsn; + std::string Environment; + bool AllowPII = false; + bool Debug = false; + }; + + void Initialize(const Config& Conf, const std::string& CommandLine); void LogStartupInformation(); static void ClearCaches(); diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h index 8fb781571..d9fb5c023 100644 --- a/src/zencore/include/zencore/thread.h +++ b/src/zencore/include/zencore/thread.h @@ -183,6 +183,7 @@ public: void CountDown() { std::ptrdiff_t Old = Counter.fetch_sub(1); + ZEN_ASSERT(Old > 0); if (Old == 1) { Complete.Set(); @@ -197,8 +198,7 @@ public: void AddCount(std::ptrdiff_t Count) { std::atomic_ptrdiff_t Old = Counter.fetch_add(Count); - ZEN_UNUSED(Old); - ZEN_ASSERT_SLOW(Old > 0); + ZEN_ASSERT(Old > 0); } bool Wait(int TimeoutMs = -1) diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index 520d5162e..118c4158a 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -196,22 +196,23 @@ SentryIntegration::~SentryIntegration() } void -SentryIntegration::Initialize(std::string SentryDatabasePath, - std::string SentryAttachmentsPath, - std::string SentryDsn, - bool AllowPII, - const std::string& CommandLine) +SentryIntegration::Initialize(const Config& Conf, const std::string& CommandLine) { - m_AllowPII = AllowPII; + m_AllowPII = Conf.AllowPII; + std::string SentryDatabasePath = Conf.DatabasePath; if (SentryDatabasePath.starts_with("\\\\?\\")) { SentryDatabasePath = SentryDatabasePath.substr(4); } sentry_options_t* SentryOptions = sentry_options_new(); - sentry_options_set_dsn(SentryOptions, SentryDsn.empty() ? sentry::DefaultDsn.c_str() : SentryDsn.c_str()); + + sentry_options_set_dsn(SentryOptions, Conf.Dsn.empty() ? sentry::DefaultDsn.c_str() : Conf.Dsn.c_str()); sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str()); sentry_options_set_logger(SentryOptions, SentryLogFunction, this); + sentry_options_set_environment(SentryOptions, Conf.Environment.empty() ? "production" : Conf.Environment.c_str()); + + std::string SentryAttachmentsPath = Conf.AttachmentsPath; if (!SentryAttachmentsPath.empty()) { if (SentryAttachmentsPath.starts_with("\\\\?\\")) @@ -222,7 +223,10 @@ SentryIntegration::Initialize(std::string SentryDatabasePath, } sentry_options_set_release(SentryOptions, ZEN_CFG_VERSION); - // sentry_options_set_debug(SentryOptions, 1); + if (Conf.Debug) + { + sentry_options_set_debug(SentryOptions, 1); + } m_SentryErrorCode = sentry_init(SentryOptions); diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp index 75a333687..bcec74ce6 100644 --- a/src/zenserver/buildstore/httpbuildstore.cpp +++ b/src/zenserver/buildstore/httpbuildstore.cpp @@ -177,7 +177,7 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) const uint64_t BlobSize = Blob.GetSize(); const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0; const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); - if (Range.Start + RangeSize >= BlobSize) + if (Range.Start + RangeSize > BlobSize) { return ServerRequest.WriteResponse(HttpResponseCode::NoContent); } diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp index 9f2e826d6..bb0c55618 100644 --- a/src/zenserver/cache/httpstructuredcache.cpp +++ b/src/zenserver/cache/httpstructuredcache.cpp @@ -1593,10 +1593,19 @@ HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext& Co ZEN_INFO("Replaying {} requests", RequestCount); for (uint64_t RequestIndex = 0; RequestIndex < RequestCount; ++RequestIndex) { - Work.ScheduleWork(WorkerPool, [this, &Context, &Replayer, RequestIndex](std::atomic<bool>&) { + if (AbortFlag) + { + break; + } + Work.ScheduleWork(WorkerPool, [this, &Context, &Replayer, RequestIndex](std::atomic<bool>& AbortFlag) { IoBuffer Body; zen::cache::RecordedRequestInfo RequestInfo = Replayer.GetRequest(RequestIndex, /* out */ Body); + if (AbortFlag) + { + return; + } + if (Body) { uint32_t AcceptMagic = 0; diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 055376b5c..1f9ae5fb6 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -16,6 +16,7 @@ #include <zencore/string.h> #include <zenhttp/zenhttp.h> #include <zenutil/commandlineoptions.h> +#include <zenutil/environmentoptions.h> ZEN_THIRD_PARTY_INCLUDES_START #include <fmt/format.h> @@ -429,6 +430,29 @@ MakeOption(std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& }; void +ParseEnvVariables(ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult) +{ + using namespace std::literals; + + EnvironmentOptions Options; + Options.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); + Options.AddOption("UE_ZEN_SENTRY_DSN"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); + Options.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv); + + bool EnvEnableSentry = !ServerOptions.SentryConfig.Disable; + Options.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv); + + Options.AddOption("UE_ZEN_SENTRY_DEBUG"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv); + + Options.Parse(CmdLineResult); + + if (EnvEnableSentry != !ServerOptions.SentryConfig.Disable) + { + ServerOptions.SentryConfig.Disable = !EnvEnableSentry; + } +} + +void ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult, @@ -441,9 +465,11 @@ ParseConfigFile(const std::filesystem::path& Path, ////// server LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv); LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv); - LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.NoSentry, "no-sentry"sv); - LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryAllowPII, "sentry-allow-personal-info"sv); - LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryDsn, "sentry-dsn"sv); + LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv); + LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); + LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); + LuaOptions.AddOption("server.sentry.environment"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv); + LuaOptions.AddOption("server.sentry.debug"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv); LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv); LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv); LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); @@ -759,11 +785,15 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) options.add_options()("write-config", "Path to output Lua config file", cxxopts::value<std::string>(OutputConfigFile)); options.add_options()("no-sentry", "Disable Sentry crash handler", - cxxopts::value<bool>(ServerOptions.NoSentry)->default_value("false")); + cxxopts::value<bool>(ServerOptions.SentryConfig.Disable)->default_value("false")); options.add_options()("sentry-allow-personal-info", "Allow personally identifiable information in sentry crash reports", - cxxopts::value<bool>(ServerOptions.SentryAllowPII)->default_value("false")); - options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value<std::string>(ServerOptions.SentryDsn)); + cxxopts::value<bool>(ServerOptions.SentryConfig.AllowPII)->default_value("false")); + options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value<std::string>(ServerOptions.SentryConfig.Dsn)); + options.add_options()("sentry-environment", "Sentry environment", cxxopts::value<std::string>(ServerOptions.SentryConfig.Environment)); + options.add_options()("sentry-debug", + "Enable debug mode for Sentry", + cxxopts::value<bool>(ServerOptions.SentryConfig.Debug)->default_value("false")); options.add_options()("detach", "Indicate whether zenserver should detach from parent process group", cxxopts::value<bool>(ServerOptions.Detach)->default_value("true")); @@ -1336,6 +1366,8 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs); + ParseEnvVariables(ServerOptions, Result); + if (!ServerOptions.ConfigFile.empty()) { ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, OutputConfigFile); diff --git a/src/zenserver/config.h b/src/zenserver/config.h index 1a1793b8d..9753e3ae2 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -157,6 +157,15 @@ struct ZenWorkspacesConfig bool AllowConfigurationChanges = false; }; +struct ZenSentryConfig +{ + bool Disable = false; + bool AllowPII = false; // Allow personally identifiable information in sentry crash reports + std::string Dsn; + std::string Environment; + bool Debug = false; // Enable debug mode for Sentry +}; + struct ZenServerOptions { ZenUpstreamCacheConfig UpstreamCacheConfig; @@ -169,31 +178,29 @@ struct ZenServerOptions ZenBuildStoreConfig BuildStoreConfig; ZenStatsConfig StatsConfig; ZenWorkspacesConfig WorksSpacesConfig; - std::filesystem::path SystemRootDir; // System root directory (used for machine level config) - std::filesystem::path DataDir; // Root directory for state (used for testing) - std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) - std::filesystem::path AbsLogFile; // Absolute path to main log file - std::filesystem::path ConfigFile; // Path to Lua config file - std::filesystem::path PluginsConfigFile; // Path to plugins config file - std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) - std::string ChildId; // Id assigned by parent process (used for lifetime management) - std::string LogId; // Id for tagging log output - std::string EncryptionKey; // 256 bit AES encryption key - std::string EncryptionIV; // 128 bit AES initialization vector - int BasePort = 8558; // Service listen port (used for both UDP and TCP) - int OwnerPid = 0; // Parent process id (zero for standalone) - bool InstallService = false; // Flag used to initiate service install (temporary) - bool UninstallService = false; // Flag used to initiate service uninstall (temporary) - bool IsDebug = false; - bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not - bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization - bool IsTest = false; - bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements - bool ShouldCrash = false; // Option for testing crash handling - bool IsFirstRun = false; - bool NoSentry = false; - bool SentryAllowPII = false; // Allow personally identifiable information in sentry crash reports - std::string SentryDsn; + ZenSentryConfig SentryConfig; + std::filesystem::path SystemRootDir; // System root directory (used for machine level config) + std::filesystem::path DataDir; // Root directory for state (used for testing) + std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) + std::filesystem::path AbsLogFile; // Absolute path to main log file + std::filesystem::path ConfigFile; // Path to Lua config file + std::filesystem::path PluginsConfigFile; // Path to plugins config file + std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) + std::string ChildId; // Id assigned by parent process (used for lifetime management) + std::string LogId; // Id for tagging log output + std::string EncryptionKey; // 256 bit AES encryption key + std::string EncryptionIV; // 128 bit AES initialization vector + int BasePort = 8558; // Service listen port (used for both UDP and TCP) + int OwnerPid = 0; // Parent process id (zero for standalone) + bool InstallService = false; // Flag used to initiate service install (temporary) + bool UninstallService = false; // Flag used to initiate service uninstall (temporary) + bool IsDebug = false; + bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not + bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization + bool IsTest = false; + bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements + bool ShouldCrash = false; // Option for testing crash handling + bool IsFirstRun = false; bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux) bool ObjectStoreEnabled = false; bool NoConsoleOutput = false; // Control default use of stdout for diagnostics diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip Binary files differindex eadfcf84c..5778fa3d2 100644 --- a/src/zenserver/frontend/html.zip +++ b/src/zenserver/frontend/html.zip diff --git a/src/zenserver/frontend/html/pages/start.js b/src/zenserver/frontend/html/pages/start.js index 472bb27ae..d1c13ccc7 100644 --- a/src/zenserver/frontend/html/pages/start.js +++ b/src/zenserver/frontend/html/pages/start.js @@ -41,6 +41,41 @@ export class Page extends ZenPage action_tb.left().add("drop").on_click((x) => this.drop_project(x), project.Id); } + // cache + var section = this.add_section("z$"); + columns = [ + "namespace", + "dir", + "buckets", + "entries", + "size disk", + "size mem", + "actions", + ] + var zcache_info = new Fetcher().resource("/z$/").json(); + const cache_table = section.add_widget(Table, columns, Table.Flag_FitLeft|Table.Flag_PackRight); + for (const namespace of (await zcache_info)["Namespaces"]) + { + new Fetcher().resource(`/z$/${namespace}/`).json().then((data) => { + const row = cache_table.add_row( + "", + data["Configuration"]["RootDir"], + data["Buckets"].length, + data["EntryCount"], + Friendly.kib(data["StorageSize"].DiskSize), + Friendly.kib(data["StorageSize"].MemorySize) + ); + var cell = row.get_cell(0); + cell.tag().text(namespace).on_click(() => this.view_zcache(namespace)); + row.get_cell(1).tag().text(namespace); + + cell = row.get_cell(-1); + const action_tb = new Toolbar(cell, true); + action_tb.left().add("view").on_click(() => this.view_zcache(namespace)); + action_tb.left().add("drop").on_click(() => this.drop_zcache(namespace)); + }); + } + // stats section = this.add_section("stats"); columns = [ @@ -92,4 +127,23 @@ export class Page extends ZenPage .option("Yes", () => drop()) .option("No"); } + + view_zcache(namespace) + { + window.location = "?page=zcache&namespace=" + namespace; + } + + drop_zcache(namespace) + { + const drop = async () => { + await new Fetcher().resource("z$", namespace).delete(); + this.reload(); + }; + + new Modal() + .title("Confirmation") + .message(`Drop zcache '${namespace}'?`) + .option("Yes", () => drop()) + .option("No"); + } } diff --git a/src/zenserver/frontend/html/pages/zcache.js b/src/zenserver/frontend/html/pages/zcache.js new file mode 100644 index 000000000..974893b21 --- /dev/null +++ b/src/zenserver/frontend/html/pages/zcache.js @@ -0,0 +1,70 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { ZenPage } from "./page.js" +import { Fetcher } from "../util/fetcher.js" +import { Friendly } from "../util/friendly.js" +import { Modal } from "../util/modal.js" +import { Table, PropTable, Toolbar } from "../util/widgets.js" + +//////////////////////////////////////////////////////////////////////////////// +export class Page extends ZenPage +{ + async main() + { + const namespace = this.get_param("namespace"); + + var info = new Fetcher().resource(`/z$/${namespace}/`).json(); + + this.set_title("cache - " + namespace); + + var section = this.add_section("info"); + var cfg_table = section.add_section("config").add_widget(PropTable); + var storage_table = section.add_section("storage").add_widget(PropTable); + + info = await info; + + cfg_table.add_object(info["Configuration"], true); + + storage_table.add_property("disk", Friendly.kib(info["StorageSize"]["DiskSize"])); + storage_table.add_property("mem", Friendly.kib(info["StorageSize"]["MemorySize"])); + storage_table.add_property("entries", Friendly.sep(info["EntryCount"])); + + var column_names = ["name", "disk", "mem", "entries", "actions"]; + var bucket_table = this.add_section("buckets").add_widget( + Table, + column_names, + Table.Flag_BiasLeft + ); + for (const bucket of info["Buckets"]) + { + const row = bucket_table.add_row(bucket); + new Fetcher().resource(`/z$/${namespace}/${bucket}`).json().then((data) => { + row.get_cell(1).text(Friendly.kib(data["StorageSize"]["DiskSize"])); + row.get_cell(2).text(Friendly.kib(data["StorageSize"]["MemorySize"])); + row.get_cell(3).text(Friendly.sep(data["DiskEntryCount"])); + + const cell = row.get_cell(-1); + const action_tb = new Toolbar(cell, true); + action_tb.left().add("view") + action_tb.left().add("drop").on_click(() => this.drop_bucket(bucket)); + }); + } + } + + drop_bucket(bucket) + { + const drop = async () => { + const namespace = this.get_param("namespace"); + await new Fetcher().resource("z$", namespace, bucket).delete(); + this.reload(); + }; + + new Modal() + .title("Confirmation") + .message(`Drop bucket '${bucket}'?`) + .option("Yes", () => drop()) + .option("No"); + } +} diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 868126533..b0d945814 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -96,15 +96,17 @@ ZenEntryPoint::Run() #if ZEN_USE_SENTRY SentryIntegration Sentry; - if (m_ServerOptions.NoSentry == false) + if (m_ServerOptions.SentryConfig.Disable == false) { std::string SentryDatabasePath = (m_ServerOptions.DataDir / ".sentry-native").string(); std::string SentryAttachmentPath = m_ServerOptions.AbsLogFile.string(); - Sentry.Initialize(SentryDatabasePath, - SentryAttachmentPath, - m_ServerOptions.SentryDsn, - m_ServerOptions.SentryAllowPII, + Sentry.Initialize({.DatabasePath = SentryDatabasePath, + .AttachmentsPath = SentryAttachmentPath, + .Dsn = m_ServerOptions.SentryConfig.Dsn, + .Environment = m_ServerOptions.SentryConfig.Environment, + .AllowPII = m_ServerOptions.SentryConfig.AllowPII, + .Debug = m_ServerOptions.SentryConfig.Debug}, m_ServerOptions.CommandLine); } #endif diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 6359b9db9..53e687983 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -1599,20 +1599,36 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo std::atomic<bool> AbortFlag; std::atomic<bool> PauseFlag; ParallelWork Work(AbortFlag, PauseFlag); - for (uint32_t OpIndex = 0; !IsCancelledFlag && OpIndex < Result.OpCount; OpIndex++) + try { - if (OptionalWorkerPool) - { - Work.ScheduleWork(*OptionalWorkerPool, [&, Index = OpIndex](std::atomic<bool>&) { - ZEN_MEMSCOPE(GetProjectstoreTag()); - ValidateOne(Index); - }); - } - else + for (uint32_t OpIndex = 0; !IsCancelledFlag && OpIndex < Result.OpCount; OpIndex++) { - ValidateOne(OpIndex); + if (AbortFlag) + { + break; + } + if (OptionalWorkerPool) + { + Work.ScheduleWork(*OptionalWorkerPool, [&ValidateOne, Index = OpIndex](std::atomic<bool>& AbortFlag) { + ZEN_MEMSCOPE(GetProjectstoreTag()); + if (AbortFlag) + { + return; + } + ValidateOne(Index); + }); + } + else + { + ValidateOne(OpIndex); + } } } + catch (const std::exception& Ex) + { + AbortFlag.store(true); + ZEN_WARN("Failed validating oplogs in {}. Reason: '{}'", m_BasePath, Ex.what()); + } Work.Wait(); { @@ -2110,67 +2126,81 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds, } if (OptionalWorkerPool) { - std::atomic_bool Result = true; std::atomic<bool> AbortFlag; std::atomic<bool> PauseFlag; ParallelWork Work(AbortFlag, PauseFlag); - - for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++) + try { - if (Result.load() == false) + for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++) { - break; - } - Work.ScheduleWork( - *OptionalWorkerPool, - [this, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result]( - std::atomic<bool>&) { - if (Result.load() == false) - { - return; - } - size_t FileChunkIndex = FileChunkIndexes[ChunkIndex]; - const std::filesystem::path& FilePath = FileChunkPaths[ChunkIndex]; - try - { - IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath); - if (!Payload) + if (AbortFlag) + { + break; + } + Work.ScheduleWork( + *OptionalWorkerPool, + [this, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback]( + std::atomic<bool>& AbortFlag) { + if (AbortFlag) { - ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[FileChunkIndex], FilePath); + return; } + size_t FileChunkIndex = FileChunkIndexes[ChunkIndex]; + const std::filesystem::path& FilePath = FileChunkPaths[ChunkIndex]; + try + { + IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath); + if (!Payload) + { + ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[FileChunkIndex], FilePath); + } - if (!AsyncCallback(FileChunkIndex, Payload, IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0)) + if (!AsyncCallback(FileChunkIndex, + Payload, + IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0)) + { + AbortFlag.store(true); + } + } + catch (const std::exception& Ex) { - Result.store(false); + ZEN_WARN("oplog '{}/{}': exception caught when iterating file chunk {}, path '{}'. Reason: '{}'", + m_OuterProject->Identifier, + m_OplogId, + FileChunkIndex, + FilePath, + Ex.what()); } - } - catch (const std::exception& Ex) - { - ZEN_WARN("oplog '{}/{}': exception caught when iterating file chunk {}, path '{}'. Reason: '{}'", - m_OuterProject->Identifier, - m_OplogId, - FileChunkIndex, - FilePath, - Ex.what()); - } - }); - } + }); + } - if (!CidChunkHashes.empty()) + if (!CidChunkHashes.empty() && !AbortFlag) + { + m_CidStore.IterateChunks( + CidChunkHashes, + [&](size_t Index, const IoBuffer& Payload) { + size_t CidChunkIndex = CidChunkIndexes[Index]; + if (AbortFlag) + { + return false; + } + return AsyncCallback(CidChunkIndex, + Payload, + IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0); + }, + OptionalWorkerPool, + LargeSizeLimit); + } + } + catch (const std::exception& Ex) { - m_CidStore.IterateChunks( - CidChunkHashes, - [&](size_t Index, const IoBuffer& Payload) { - size_t CidChunkIndex = CidChunkIndexes[Index]; - return AsyncCallback(CidChunkIndex, Payload, IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0); - }, - OptionalWorkerPool, - LargeSizeLimit); + AbortFlag.store(true); + ZEN_WARN("Failed iterating oplog chunks in {}. Reason: '{}'", m_BasePath, Ex.what()); } Work.Wait(); - return Result.load(); + return !AbortFlag; } else { @@ -3894,19 +3924,26 @@ ProjectStore::Flush() std::atomic<bool> AbortFlag; std::atomic<bool> PauseFlag; ParallelWork Work(AbortFlag, PauseFlag); - - for (const Ref<Project>& Project : Projects) + try { - Work.ScheduleWork(WorkerPool, [this, Project](std::atomic<bool>&) { - try - { - Project->Flush(); - } - catch (const std::exception& Ex) - { - ZEN_WARN("Exception while flushing project {}: {}", Project->Identifier, Ex.what()); - } - }); + for (const Ref<Project>& Project : Projects) + { + Work.ScheduleWork(WorkerPool, [this, Project](std::atomic<bool>&) { + try + { + Project->Flush(); + } + catch (const std::exception& Ex) + { + ZEN_WARN("Exception while flushing project {}: {}", Project->Identifier, Ex.what()); + } + }); + } + } + catch (const std::exception& Ex) + { + AbortFlag.store(true); + ZEN_WARN("Failed projects in {}. Reason: '{}'", m_ProjectBasePath, Ex.what()); } Work.Wait(); diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 1ad94ed63..27ec4c690 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -137,7 +137,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen return -1; } - m_UseSentry = ServerOptions.NoSentry == false; + m_UseSentry = ServerOptions.SentryConfig.Disable == false; m_ServerEntry = ServerEntry; m_DebugOptionForcedCrash = ServerOptions.ShouldCrash; m_IsPowerCycle = ServerOptions.IsPowerCycle; diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index c25f762f5..20dc55bca 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -528,43 +528,55 @@ BuildStore::GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* O std::atomic<bool> PauseFlag; ParallelWork Work(AbortFlag, PauseFlag); - m_MetadataBlockStore.IterateChunks( - MetaLocations, - [this, OptionalWorkerPool, &Work, &Result, &MetaLocations, &MetaLocationResultIndexes, &ReferencedBlocks, DoOneBlock]( - uint32_t BlockIndex, - std::span<const size_t> ChunkIndexes) -> bool { - ZEN_UNUSED(BlockIndex); - if (ChunkIndexes.size() == MetaLocations.size() || OptionalWorkerPool == nullptr || ReferencedBlocks.size() == 1) - { - return DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result); - } - else - { - ZEN_ASSERT(OptionalWorkerPool != nullptr); - std::vector<size_t> TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end()); - Work.ScheduleWork( - *OptionalWorkerPool, - [this, &Result, &MetaLocations, &MetaLocationResultIndexes, DoOneBlock, ChunkIndexes = std::move(TmpChunkIndexes)]( - std::atomic<bool>& AbortFlag) { - if (AbortFlag) - { - return; - } - try - { - if (!DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result)) + try + { + m_MetadataBlockStore.IterateChunks( + MetaLocations, + [this, OptionalWorkerPool, &Work, &Result, &MetaLocations, &MetaLocationResultIndexes, &ReferencedBlocks, DoOneBlock]( + uint32_t BlockIndex, + std::span<const size_t> ChunkIndexes) -> bool { + ZEN_UNUSED(BlockIndex); + if (ChunkIndexes.size() == MetaLocations.size() || OptionalWorkerPool == nullptr || ReferencedBlocks.size() == 1) + { + return DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result); + } + else + { + ZEN_ASSERT(OptionalWorkerPool != nullptr); + std::vector<size_t> TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end()); + Work.ScheduleWork( + *OptionalWorkerPool, + [this, + &Result, + &MetaLocations, + &MetaLocationResultIndexes, + DoOneBlock, + ChunkIndexes = std::move(TmpChunkIndexes)](std::atomic<bool>& AbortFlag) { + if (AbortFlag) { - AbortFlag.store(true); + return; } - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed getting metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); - } - }); - return !Work.IsAborted(); - } - }); + try + { + if (!DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result)) + { + AbortFlag.store(true); + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed getting metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); + } + }); + return !Work.IsAborted(); + } + }); + } + catch (const std::exception& Ex) + { + AbortFlag.store(true); + ZEN_WARN("Failed iterating block metadata chunks in {}. Reason: '{}'", m_Config.RootDirectory, Ex.what()); + } Work.Wait(); } diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 0ee70890c..15a1c9650 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -2004,6 +2004,8 @@ ZenCacheDiskLayer::CacheBucket::Drop() { ZEN_TRACE_CPU("Z$::Bucket::Drop"); + m_Gc.RemoveGcReferencer(*this); + RwLock::ExclusiveLockScope _(m_IndexLock); std::vector<std::unique_ptr<RwLock::ExclusiveLockScope>> ShardLocks; @@ -3165,7 +3167,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) return nullptr; } - if (Ctx.Settings.IsDeleteMode) + if (Ctx.Settings.IsDeleteMode && !ExpiredEntries.empty()) { for (const DiskIndexEntry& Entry : ExpiredEntries) { @@ -4036,50 +4038,58 @@ ZenCacheDiskLayer::DiscoverBuckets() std::atomic<bool> AbortFlag; std::atomic<bool> PauseFlag; ParallelWork Work(AbortFlag, PauseFlag); - for (auto& BucketPath : FoundBucketDirectories) + try { - Work.ScheduleWork(Pool, [this, &SyncLock, BucketPath](std::atomic<bool>&) { - ZEN_MEMSCOPE(GetCacheDiskTag()); - - const std::string BucketName = PathToUtf8(BucketPath.stem()); - try - { - BucketConfiguration* BucketConfig = &m_Configuration.BucketConfig; - if (auto It = m_Configuration.BucketConfigMap.find_as(std::string_view(BucketName), - std::hash<std::string_view>(), - eastl::equal_to_2<std::string, std::string_view>()); - It != m_Configuration.BucketConfigMap.end()) - { - BucketConfig = &It->second; - } - - std::unique_ptr<CacheBucket> NewBucket = - std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, BucketName, *BucketConfig); + for (auto& BucketPath : FoundBucketDirectories) + { + Work.ScheduleWork(Pool, [this, &SyncLock, BucketPath](std::atomic<bool>&) { + ZEN_MEMSCOPE(GetCacheDiskTag()); - CacheBucket* Bucket = nullptr; + const std::string BucketName = PathToUtf8(BucketPath.stem()); + try { - RwLock::ExclusiveLockScope __(SyncLock); - auto InsertResult = m_Buckets.emplace(BucketName, std::move(NewBucket)); - Bucket = InsertResult.first->second.get(); - } - ZEN_ASSERT(Bucket); + BucketConfiguration* BucketConfig = &m_Configuration.BucketConfig; + if (auto It = m_Configuration.BucketConfigMap.find_as(std::string_view(BucketName), + std::hash<std::string_view>(), + eastl::equal_to_2<std::string, std::string_view>()); + It != m_Configuration.BucketConfigMap.end()) + { + BucketConfig = &It->second; + } - if (!Bucket->OpenOrCreate(BucketPath, /* AllowCreate */ false)) - { - ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir); + std::unique_ptr<CacheBucket> NewBucket = + std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, BucketName, *BucketConfig); + CacheBucket* Bucket = nullptr; { RwLock::ExclusiveLockScope __(SyncLock); - m_Buckets.erase(BucketName); + auto InsertResult = m_Buckets.emplace(BucketName, std::move(NewBucket)); + Bucket = InsertResult.first->second.get(); + } + ZEN_ASSERT(Bucket); + + if (!Bucket->OpenOrCreate(BucketPath, /* AllowCreate */ false)) + { + ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir); + + { + RwLock::ExclusiveLockScope __(SyncLock); + m_Buckets.erase(BucketName); + } } } - } - catch (const std::exception& Err) - { - ZEN_ERROR("Opening bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what()); - return; - } - }); + catch (const std::exception& Err) + { + ZEN_ERROR("Opening bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what()); + return; + } + }); + } + } + catch (const std::exception& Ex) + { + AbortFlag.store(true); + ZEN_WARN("Failed discovering buckets in {}. Reason: '{}'", m_RootDir, Ex.what()); } Work.Wait(); } @@ -4220,8 +4230,10 @@ ZenCacheDiskLayer::Flush() } catch (const std::exception& Ex) { + AbortFlag.store(true); ZEN_ERROR("Failed to flush buckets at '{}'. Reason: '{}'", m_RootDir, Ex.what()); } + Work.Wait(1000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t RemainingWork) { ZEN_UNUSED(IsAborted, IsPaused); ZEN_DEBUG("Waiting for {} buckets at '{}' to flush", RemainingWork, m_RootDir); diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 5ce254fac..d956384ca 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -297,6 +297,7 @@ ZenCacheNamespace::EnumerateBucketContents(std::string_view std::function<void()> ZenCacheNamespace::Drop() { + m_Gc.RemoveGcStorage(this); return m_DiskLayer.Drop(); } diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 2ab5752ff..b00abb2cb 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -396,89 +396,96 @@ CasContainerStrategy::IterateChunks(std::span<const IoHash> ChunkHas return m_BlockStore.IterateBlock( FoundChunkLocations, ChunkIndexes, - [AsyncCallback, FoundChunkIndexes, LargeSizeLimit](size_t ChunkIndex, const void* Data, uint64_t Size) { + [AsyncCallback, FoundChunkIndexes](size_t ChunkIndex, const void* Data, uint64_t Size) { if (Data == nullptr) { return AsyncCallback(FoundChunkIndexes[ChunkIndex], IoBuffer()); } return AsyncCallback(FoundChunkIndexes[ChunkIndex], IoBuffer(IoBuffer::Wrap, Data, Size)); }, - [&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) { + [AsyncCallback, FoundChunkIndexes](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) { return AsyncCallback(FoundChunkIndexes[ChunkIndex], File.GetChunk(Offset, Size)); }, LargeSizeLimit); }; - std::atomic<bool> AsyncContinue = true; + std::atomic<bool> AbortFlag; { - std::atomic<bool> AbortFlag; std::atomic<bool> PauseFlag; ParallelWork Work(AbortFlag, PauseFlag); - const bool Continue = m_BlockStore.IterateChunks( - FoundChunkLocations, - [this, - &Work, - &AsyncContinue, - &AsyncCallback, - LargeSizeLimit, - DoOneBlock, - &FoundChunkIndexes, - &FoundChunkLocations, - OptionalWorkerPool](uint32_t BlockIndex, std::span<const size_t> ChunkIndexes) { - if (OptionalWorkerPool && (ChunkIndexes.size() > 3)) - { - std::vector<size_t> TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end()); - Work.ScheduleWork( - *OptionalWorkerPool, - [this, - &AsyncContinue, - &AsyncCallback, - LargeSizeLimit, - DoOneBlock, - BlockIndex, - &FoundChunkIndexes, - &FoundChunkLocations, - ChunkIndexes = std::move(TmpChunkIndexes)](std::atomic<bool>& AbortFlag) { - if (AbortFlag) - { - AsyncContinue.store(false); - } - if (!AsyncContinue) - { - return; - } - try - { - bool Continue = - DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes); - if (!Continue) - { - AsyncContinue.store(false); - } - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed iterating chunks for cas root path {}, block {}. Reason: '{}'", - m_RootDirectory, - BlockIndex, - Ex.what()); - AsyncContinue.store(false); - } - }); - return AsyncContinue.load(); - } - else - { - return DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes); - } - }); - if (!Continue) + try { - AsyncContinue.store(false); + const bool Continue = m_BlockStore.IterateChunks( + FoundChunkLocations, + [this, + &Work, + &AbortFlag, + &AsyncCallback, + LargeSizeLimit, + DoOneBlock, + &FoundChunkIndexes, + &FoundChunkLocations, + OptionalWorkerPool](uint32_t BlockIndex, std::span<const size_t> ChunkIndexes) { + if (OptionalWorkerPool && (ChunkIndexes.size() > 3)) + { + std::vector<size_t> TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end()); + Work.ScheduleWork( + *OptionalWorkerPool, + [this, + &AsyncCallback, + LargeSizeLimit, + DoOneBlock, + BlockIndex, + &FoundChunkIndexes, + &FoundChunkLocations, + ChunkIndexes = std::move(TmpChunkIndexes)](std::atomic<bool>& AbortFlag) { + if (AbortFlag) + { + return; + } + try + { + bool Continue = + DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes); + if (!Continue) + { + AbortFlag.store(true); + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed iterating chunks for cas root path {}, block {}. Reason: '{}'", + m_RootDirectory, + BlockIndex, + Ex.what()); + AbortFlag.store(true); + } + }); + return !AbortFlag.load(); + } + else + { + if (!DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes)) + { + AbortFlag.store(true); + } + return !AbortFlag.load(); + } + }); + if (!Continue) + { + AbortFlag.store(true); + } } + catch (const std::exception& Ex) + { + AbortFlag.store(true); + ZEN_WARN("Failed iterating chunks for cas root path {}. Reason: '{}'", m_RootDirectory, Ex.what()); + } + Work.Wait(); } - return AsyncContinue.load(); + return !AbortFlag.load(); } void diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 11a266f1c..68644be2d 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -666,52 +666,64 @@ FileCasStrategy::IterateChunks(std::span<IoHash> ChunkHashes, std::atomic<bool> AbortFlag; std::atomic<bool> PauseFlag; ParallelWork Work(AbortFlag, PauseFlag); - for (size_t Index = 0; Index < FoundChunkIndexes.size(); Index++) + try { - if (!AsyncContinue) + for (size_t Index = 0; Index < FoundChunkIndexes.size(); Index++) { - break; - } - size_t ChunkIndex = FoundChunkIndexes[Index]; - uint64_t ExpectedSize = FoundChunkExpectedSizes[Index]; - if (OptionalWorkerPool) - { - Work.ScheduleWork( - *OptionalWorkerPool, - [this, &ProcessOne, &ChunkHashes, ChunkIndex, ExpectedSize, &AsyncContinue](std::atomic<bool>& AbortFlag) { - if (AbortFlag) - { - AsyncContinue.store(false); - } - if (!AsyncContinue) - { - return; - } - try - { - if (!ProcessOne(ChunkIndex, ExpectedSize)) + if (AbortFlag) + { + AsyncContinue.store(false); + } + if (!AsyncContinue) + { + break; + } + size_t ChunkIndex = FoundChunkIndexes[Index]; + uint64_t ExpectedSize = FoundChunkExpectedSizes[Index]; + if (OptionalWorkerPool) + { + Work.ScheduleWork( + *OptionalWorkerPool, + [this, &ProcessOne, &ChunkHashes, ChunkIndex, ExpectedSize, &AsyncContinue](std::atomic<bool>& AbortFlag) { + if (AbortFlag) { AsyncContinue.store(false); } - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed iterating chunks for cas root path {}, chunk {}. Reason: '{}'", - m_RootDirectory, - ChunkHashes[ChunkIndex], - Ex.what()); - AsyncContinue.store(false); - } - }); - } - else - { - if (!ProcessOne(ChunkIndex, ExpectedSize)) + if (!AsyncContinue) + { + return; + } + try + { + if (!ProcessOne(ChunkIndex, ExpectedSize)) + { + AsyncContinue.store(false); + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed iterating chunks for cas root path {}, chunk {}. Reason: '{}'", + m_RootDirectory, + ChunkHashes[ChunkIndex], + Ex.what()); + AsyncContinue.store(false); + } + }); + } + else { - AsyncContinue.store(false); + if (!ProcessOne(ChunkIndex, ExpectedSize)) + { + AsyncContinue.store(false); + } } } } + catch (const std::exception& Ex) + { + AbortFlag.store(true); + ZEN_WARN("Failed iterating chunks in {}. Reason: '{}'", this->m_RootDirectory, Ex.what()); + } Work.Wait(); } return AsyncContinue.load(); diff --git a/src/zenutil/environmentoptions.cpp b/src/zenutil/environmentoptions.cpp new file mode 100644 index 000000000..1b7ce8029 --- /dev/null +++ b/src/zenutil/environmentoptions.cpp @@ -0,0 +1,84 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zenutil/environmentoptions.h> + +#include <zencore/filesystem.h> + +namespace zen { + +EnvironmentOptions::StringOption::StringOption(std::string& Value) : RefValue(Value) +{ +} +void +EnvironmentOptions::StringOption::Parse(std::string_view Value) +{ + RefValue = std::string(Value); +} + +EnvironmentOptions::FilePathOption::FilePathOption(std::filesystem::path& Value) : RefValue(Value) +{ +} +void +EnvironmentOptions::FilePathOption::Parse(std::string_view Value) +{ + RefValue = MakeSafeAbsolutePath(Value); +} + +EnvironmentOptions::BoolOption::BoolOption(bool& Value) : RefValue(Value) +{ +} +void +EnvironmentOptions::BoolOption::Parse(std::string_view Value) +{ + const std::string Lower = ToLower(Value); + if (Lower == "true" || Lower == "y" || Lower == "yes") + { + RefValue = true; + } + else if (Lower == "false" || Lower == "n" || Lower == "no") + { + RefValue = false; + } +} + +std::shared_ptr<EnvironmentOptions::OptionValue> +EnvironmentOptions::MakeOption(std::string& Value) +{ + return std::make_shared<StringOption>(Value); +} + +std::shared_ptr<EnvironmentOptions::OptionValue> +EnvironmentOptions::MakeOption(std::filesystem::path& Value) +{ + return std::make_shared<FilePathOption>(Value); +} + +std::shared_ptr<EnvironmentOptions::OptionValue> +EnvironmentOptions::MakeOption(bool& Value) +{ + return std::make_shared<BoolOption>(Value); +} + +EnvironmentOptions::EnvironmentOptions() +{ +} + +void +EnvironmentOptions::Parse(const cxxopts::ParseResult& CmdLineResult) +{ + for (auto& It : OptionMap) + { + std::string_view EnvName = It.first; + const Option& Opt = It.second; + if (CmdLineResult.count(Opt.CommandLineOptionName) == 0) + { + std::string EnvValue = GetEnvVariable(It.first); + if (!EnvValue.empty()) + { + Opt.Value->Parse(EnvValue); + } + } + } +} + +} // namespace zen diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index c389d16c5..c2cc5ab3c 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -450,7 +450,7 @@ public: return {}; } - virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override + virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override { ZEN_TRACE_CPU("FileBuildStorage::PutBlockMetadata"); ZEN_UNUSED(BuildId); @@ -467,6 +467,7 @@ public: m_Stats.TotalBytesWritten += MetaData.GetSize(); WriteAsJson(BlockMetaDataPath, MetaData); SimulateLatency(0, 0); + return true; } virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 5422c837c..f49d4b42a 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -55,9 +55,9 @@ public: std::function<void(uint64_t Offset, const IoBuffer& Chunk)>&& OnReceive, std::function<void()>&& OnComplete) = 0; - virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; - virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; - virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes) = 0; + [[nodiscard]] virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; + virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; + virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes) = 0; virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map<std::string, double>& FloatStats) = 0; }; diff --git a/src/zenutil/include/zenutil/environmentoptions.h b/src/zenutil/include/zenutil/environmentoptions.h new file mode 100644 index 000000000..7418608e4 --- /dev/null +++ b/src/zenutil/include/zenutil/environmentoptions.h @@ -0,0 +1,92 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/string.h> +#include <zenutil/commandlineoptions.h> + +namespace zen { + +class EnvironmentOptions +{ +public: + class OptionValue + { + public: + virtual void Parse(std::string_view Value) = 0; + + virtual ~OptionValue() {} + }; + + class StringOption : public OptionValue + { + public: + explicit StringOption(std::string& Value); + virtual void Parse(std::string_view Value) override; + std::string& RefValue; + }; + + class FilePathOption : public OptionValue + { + public: + explicit FilePathOption(std::filesystem::path& Value); + virtual void Parse(std::string_view Value) override; + std::filesystem::path& RefValue; + }; + + class BoolOption : public OptionValue + { + public: + explicit BoolOption(bool& Value); + virtual void Parse(std::string_view Value); + bool& RefValue; + }; + + template<Integral T> + class NumberOption : public OptionValue + { + public: + explicit NumberOption(T& Value) : RefValue(Value) {} + virtual void Parse(std::string_view Value) override + { + if (std::optional<T> OptionalValue = ParseInt<T>(Value); OptionalValue.has_value()) + { + RefValue = OptionalValue.value(); + } + } + T& RefValue; + }; + + struct Option + { + std::string CommandLineOptionName; + std::shared_ptr<OptionValue> Value; + }; + + std::shared_ptr<OptionValue> MakeOption(std::string& Value); + std::shared_ptr<OptionValue> MakeOption(std::filesystem::path& Value); + + template<Integral T> + std::shared_ptr<OptionValue> MakeOption(T& Value) + { + return std::make_shared<NumberOption<T>>(Value); + }; + + std::shared_ptr<OptionValue> MakeOption(bool& Value); + + template<typename T> + void AddOption(std::string_view EnvName, T& Value, std::string_view CommandLineOptionName = "") + { + OptionMap.insert_or_assign(std::string(EnvName), + Option{.CommandLineOptionName = std::string(CommandLineOptionName), .Value = MakeOption(Value)}); + }; + + EnvironmentOptions(); + + void Parse(const cxxopts::ParseResult& CmdLineResult); + +private: + std::unordered_map<std::string, Option> OptionMap; +}; + +} // namespace zen diff --git a/src/zenutil/include/zenutil/parallelwork.h b/src/zenutil/include/zenutil/parallelwork.h index d7e986551..639c6968c 100644 --- a/src/zenutil/include/zenutil/parallelwork.h +++ b/src/zenutil/include/zenutil/parallelwork.h @@ -2,6 +2,7 @@ #pragma once +#include <zencore/scopeguard.h> #include <zencore/thread.h> #include <zencore/workthreadpool.h> @@ -26,6 +27,7 @@ public: try { WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = OnError ? std::move(OnError) : DefaultErrorFunction()] { + auto _ = MakeGuard([this]() { m_PendingWork.CountDown(); }); try { while (m_PauseFlag && !m_AbortFlag) @@ -38,7 +40,6 @@ public: { OnError(std::current_exception(), m_AbortFlag); } - m_PendingWork.CountDown(); }); } catch (const std::exception&) diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index 53cad78da..9974725ff 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -334,7 +334,7 @@ public: return WorkList; } - virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override + virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override { ZEN_TRACE_CPU("Jupiter::PutBlockMetadata"); @@ -346,9 +346,14 @@ public: AddStatistic(PutMetaResult); if (!PutMetaResult.Success) { + if (PutMetaResult.ErrorCode == int32_t(HttpResponseCode::NotFound)) + { + return false; + } throw std::runtime_error( fmt::format("Failed putting build block metadata: {} ({})", PutMetaResult.Reason, PutMetaResult.ErrorCode)); } + return true; } virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override diff --git a/src/zenutil/parallelwork.cpp b/src/zenutil/parallelwork.cpp index 67fc03c04..aa806438b 100644 --- a/src/zenutil/parallelwork.cpp +++ b/src/zenutil/parallelwork.cpp @@ -2,6 +2,7 @@ #include <zenutil/parallelwork.h> +#include <zencore/callstack.h> #include <zencore/except.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> @@ -34,7 +35,33 @@ ParallelWork::~ParallelWork() m_PendingWork.CountDown(); } m_PendingWork.Wait(); - ZEN_ASSERT(m_PendingWork.Remaining() == 0); + ptrdiff_t RemainingWork = m_PendingWork.Remaining(); + if (RemainingWork != 0) + { + void* Frames[8]; + uint32_t FrameCount = GetCallstack(2, 8, Frames); + CallstackFrames* Callstack = CreateCallstack(FrameCount, Frames); + ZEN_ERROR("ParallelWork destructor waited for outstanding work but pending work count is {} instead of 0\n{}", + RemainingWork, + CallstackToString(Callstack, " ")); + FreeCallstack(Callstack); + + uint32_t WaitedMs = 0; + while (m_PendingWork.Remaining() > 0 && WaitedMs < 2000) + { + Sleep(50); + WaitedMs += 50; + } + RemainingWork = m_PendingWork.Remaining(); + if (RemainingWork != 0) + { + ZEN_WARN("ParallelWork destructor safety wait failed, pending work count at {}", RemainingWork) + } + else + { + ZEN_INFO("ParallelWork destructor safety wait succeeded"); + } + } } catch (const std::exception& Ex) { |