diff options
Diffstat (limited to 'src/zenremotestore/builds/jupiterbuildstorage.cpp')
| -rw-r--r-- | src/zenremotestore/builds/jupiterbuildstorage.cpp | 206 |
1 files changed, 166 insertions, 40 deletions
diff --git a/src/zenremotestore/builds/jupiterbuildstorage.cpp b/src/zenremotestore/builds/jupiterbuildstorage.cpp index c3f7b9e71..c0cc16941 100644 --- a/src/zenremotestore/builds/jupiterbuildstorage.cpp +++ b/src/zenremotestore/builds/jupiterbuildstorage.cpp @@ -4,6 +4,7 @@ #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinaryutil.h> +#include <zencore/compress.h> #include <zencore/fmtutils.h> #include <zencore/scopeguard.h> #include <zencore/timer.h> @@ -14,7 +15,7 @@ ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_map.h> ZEN_THIRD_PARTY_INCLUDES_END -#include <regex> +#include <string_view> namespace zen { @@ -263,7 +264,7 @@ public: std::vector<std::function<void()>> WorkList; for (auto& WorkItem : WorkItems) { - WorkList.emplace_back([this, WorkItem = std::move(WorkItem), OnSentBytes]() { + WorkList.emplace_back([this, WorkItem = std::move(WorkItem), OnSentBytes = std::move(OnSentBytes)]() { Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); bool IsComplete = false; @@ -285,7 +286,10 @@ public: Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - CreateDirectories(m_TempFolderPath); + if (!m_TempFolderPath.empty()) + { + CreateDirectories(m_TempFolderPath); + } JupiterResult GetBuildBlobResult = m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath, RangeOffset, RangeBytes); AddStatistic(GetBuildBlobResult); @@ -304,7 +308,10 @@ public: Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - CreateDirectories(m_TempFolderPath); + if (!m_TempFolderPath.empty()) + { + CreateDirectories(m_TempFolderPath); + } BuildBlobRangesResult GetBuildBlobResult = m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath, Ranges); @@ -444,11 +451,13 @@ public: virtual bool GetExtendedStatistics(ExtendedStatistics& OutStats) override { - OutStats.ReceivedBytesPerSource.reserve(m_ReceivedBytesPerSource.size()); - for (auto& It : m_ReceivedBytesPerSource) - { - OutStats.ReceivedBytesPerSource.insert_or_assign(It.first, m_SourceBytes[It.second]); - } + m_SourceLock.WithSharedLock([this, &OutStats]() { + OutStats.ReceivedBytesPerSource.reserve(m_ReceivedBytesPerSource.size()); + for (auto& It : m_ReceivedBytesPerSource) + { + OutStats.ReceivedBytesPerSource.insert_or_assign(It.first, m_SourceBytes[It.second].load(std::memory_order_relaxed)); + } + }); return true; } @@ -521,15 +530,29 @@ private: } if (!Result.Source.empty()) { - if (tsl::robin_map<std::string, uint32_t>::const_iterator It = m_ReceivedBytesPerSource.find(Result.Source); - It != m_ReceivedBytesPerSource.end()) - { - m_SourceBytes[It->second] += Result.ReceivedBytes; - } - else + if (!m_SourceLock.WithSharedLock([&]() { + if (tsl::robin_map<std::string, uint32_t>::const_iterator It = m_ReceivedBytesPerSource.find(Result.Source); + It != m_ReceivedBytesPerSource.end()) + { + m_SourceBytes[It->second] += Result.ReceivedBytes; + return true; + } + return false; + })) { - m_ReceivedBytesPerSource.insert_or_assign(Result.Source, m_SourceBytes.size()); - m_SourceBytes.push_back(Result.ReceivedBytes); + m_SourceLock.WithExclusiveLock([&]() { + if (tsl::robin_map<std::string, uint32_t>::const_iterator It = m_ReceivedBytesPerSource.find(Result.Source); + It != m_ReceivedBytesPerSource.end()) + { + m_SourceBytes[It->second] += Result.ReceivedBytes; + } + else if (m_SourceCount < MaxSourceCount) + { + size_t Index = m_SourceCount++; + m_ReceivedBytesPerSource.insert_or_assign(Result.Source, Index); + m_SourceBytes[Index] += Result.ReceivedBytes; + } + }); } } } @@ -540,8 +563,11 @@ private: const std::string m_Bucket; const std::filesystem::path m_TempFolderPath; - tsl::robin_map<std::string, uint32_t> m_ReceivedBytesPerSource; - std::vector<uint64_t> m_SourceBytes; + RwLock m_SourceLock; + tsl::robin_map<std::string, uint32_t> m_ReceivedBytesPerSource; + static constexpr size_t MaxSourceCount = 8u; + std::array<std::atomic<uint64_t>, MaxSourceCount> m_SourceBytes; + size_t m_SourceCount = 0; }; std::unique_ptr<BuildStorageBase> @@ -572,35 +598,135 @@ ParseBuildStorageUrl(std::string_view InUrl, Url.erase(ApiString, ExtendedApiString.length()); } - const std::string ArtifactURLRegExString = R"((http[s]?:\/\/.*?)\/(.*?)\/(.*?)\/(.*))"; - const std::regex ArtifactURLRegEx(ArtifactURLRegExString, std::regex::ECMAScript | std::regex::icase); - std::match_results<std::string_view::const_iterator> MatchResults; - std::string_view UrlToParse(Url); - if (regex_match(begin(UrlToParse), end(UrlToParse), MatchResults, ArtifactURLRegEx) && MatchResults.size() == 5) - { - auto GetMatch = [&MatchResults](uint32_t Index) -> std::string_view { - ZEN_ASSERT(Index < MatchResults.size()); + // Parse URL of the form: http[s]://host/namespace/bucket/buildid + std::string_view Remaining(Url); - const auto& Match = MatchResults[Index]; + // Find the end of the scheme (e.g. "http://" or "https://") + size_t SchemeEnd = Remaining.find("://"); + if (SchemeEnd == std::string_view::npos) + { + return false; + } + SchemeEnd += 3; // skip past "://" - return std::string_view(&*Match.first, Match.second - Match.first); - }; + // Find the first '/' after the host + size_t HostEnd = Remaining.find('/', SchemeEnd); + if (HostEnd == std::string_view::npos) + { + return false; + } - const std::string_view Host = GetMatch(1); - const std::string_view Namespace = GetMatch(2); - const std::string_view Bucket = GetMatch(3); - const std::string_view BuildId = GetMatch(4); + // Find the '/' after namespace + size_t NamespaceEnd = Remaining.find('/', HostEnd + 1); + if (NamespaceEnd == std::string_view::npos) + { + return false; + } - OutHost = Host; - OutNamespace = Namespace; - OutBucket = Bucket; - OutBuildId = BuildId; - return true; + // Find the '/' after bucket + size_t BucketEnd = Remaining.find('/', NamespaceEnd + 1); + if (BucketEnd == std::string_view::npos) + { + return false; } - else + + // BuildId must be non-empty + if (BucketEnd + 1 >= Remaining.size()) { return false; } + + OutHost = Remaining.substr(0, HostEnd); + OutNamespace = Remaining.substr(HostEnd + 1, NamespaceEnd - HostEnd - 1); + OutBucket = Remaining.substr(NamespaceEnd + 1, BucketEnd - NamespaceEnd - 1); + OutBuildId = Remaining.substr(BucketEnd + 1); + return true; } } // namespace zen + +#if ZEN_WITH_TESTS + +# include <zencore/testing.h> + +namespace zen { + +void +jupiterbuildstorage_forcelink() +{ +} + +} // namespace zen + +TEST_SUITE_BEGIN("remotestore.jupiterbuildstorage"); + +TEST_CASE("ParseBuildStorageUrl.ValidUrl") +{ + std::string Host, Namespace, Bucket, BuildId; + bool Result = + zen::ParseBuildStorageUrl("https://horde.devtools.epicgames.com/mynamespace/mybucket/mybuildid", Host, Namespace, Bucket, BuildId); + CHECK(Result); + CHECK(Host == "https://horde.devtools.epicgames.com"); + CHECK(Namespace == "mynamespace"); + CHECK(Bucket == "mybucket"); + CHECK(BuildId == "mybuildid"); +} + +TEST_CASE("ParseBuildStorageUrl.ValidUrlWithApiPrefix") +{ + std::string Host, Namespace, Bucket, BuildId; + bool Result = zen::ParseBuildStorageUrl("https://horde.devtools.epicgames.com/api/v2/builds/mynamespace/mybucket/mybuildid", + Host, + Namespace, + Bucket, + BuildId); + CHECK(Result); + CHECK(Host == "https://horde.devtools.epicgames.com"); + CHECK(Namespace == "mynamespace"); + CHECK(Bucket == "mybucket"); + CHECK(BuildId == "mybuildid"); +} + +TEST_CASE("ParseBuildStorageUrl.HttpScheme") +{ + std::string Host, Namespace, Bucket, BuildId; + bool Result = zen::ParseBuildStorageUrl("http://localhost/ns/bucket/build123", Host, Namespace, Bucket, BuildId); + CHECK(Result); + CHECK(Host == "http://localhost"); + CHECK(Namespace == "ns"); + CHECK(Bucket == "bucket"); + CHECK(BuildId == "build123"); +} + +TEST_CASE("ParseBuildStorageUrl.BuildIdWithSlashes") +{ + std::string Host, Namespace, Bucket, BuildId; + bool Result = zen::ParseBuildStorageUrl("https://host/ns/bucket/build/with/slashes", Host, Namespace, Bucket, BuildId); + CHECK(Result); + CHECK(Host == "https://host"); + CHECK(Namespace == "ns"); + CHECK(Bucket == "bucket"); + CHECK(BuildId == "build/with/slashes"); +} + +TEST_CASE("ParseBuildStorageUrl.MissingBuildId") +{ + std::string Host, Namespace, Bucket, BuildId; + CHECK_FALSE(zen::ParseBuildStorageUrl("https://host/ns/bucket/", Host, Namespace, Bucket, BuildId)); +} + +TEST_CASE("ParseBuildStorageUrl.MissingBucket") +{ + std::string Host, Namespace, Bucket, BuildId; + CHECK_FALSE(zen::ParseBuildStorageUrl("https://host/ns", Host, Namespace, Bucket, BuildId)); +} + +TEST_CASE("ParseBuildStorageUrl.NoScheme") +{ + std::string Host, Namespace, Bucket, BuildId; + CHECK_FALSE(zen::ParseBuildStorageUrl("host/ns/bucket/buildid", Host, Namespace, Bucket, BuildId)); +} + +TEST_SUITE_END(); + +#endif // ZEN_WITH_TESTS |