aboutsummaryrefslogtreecommitdiff
path: root/src/zenutil/jupiter/jupiterbuildstorage.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenutil/jupiter/jupiterbuildstorage.cpp')
-rw-r--r--src/zenutil/jupiter/jupiterbuildstorage.cpp141
1 files changed, 108 insertions, 33 deletions
diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
index bf89ce785..9974725ff 100644
--- a/src/zenutil/jupiter/jupiterbuildstorage.cpp
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -25,8 +25,9 @@ public:
Statistics& Stats,
std::string_view Namespace,
std::string_view Bucket,
+ bool AllowRedirect,
const std::filesystem::path& TempFolderPath)
- : m_Session(InLog, InHttpClient)
+ : m_Session(InLog, InHttpClient, AllowRedirect)
, m_Stats(Stats)
, m_Namespace(Namespace)
, m_Bucket(Bucket)
@@ -35,6 +36,61 @@ public:
}
virtual ~JupiterBuildStorage() {}
+ virtual CbObject ListNamespaces(bool bRecursive) override
+ {
+ ZEN_TRACE_CPU("Jupiter::ListNamespaces");
+
+ Stopwatch ExecutionTimer;
+ auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+ JupiterResult ListResult = m_Session.ListBuildNamespaces();
+ AddStatistic(ListResult);
+ if (!ListResult.Success)
+ {
+ throw std::runtime_error(fmt::format("Failed listing namespaces: {} ({})", ListResult.Reason, ListResult.ErrorCode));
+ }
+ CbObject NamespaceResponse = PayloadToCbObject("Failed listing namespaces"sv, ListResult.Response);
+
+ CbObjectWriter Response;
+ Response.BeginArray("results"sv);
+ for (CbFieldView NamespaceField : NamespaceResponse["namespaces"])
+ {
+ std::string_view Namespace = NamespaceField.AsString();
+ if (!Namespace.empty())
+ {
+ Response.BeginObject();
+ Response.AddString("name", Namespace);
+
+ if (bRecursive)
+ {
+ JupiterResult BucketsResult = m_Session.ListBuildBuckets(Namespace);
+ AddStatistic(BucketsResult);
+ if (!BucketsResult.Success)
+ {
+ throw std::runtime_error(
+ fmt::format("Failed listing namespaces: {} ({})", BucketsResult.Reason, BucketsResult.ErrorCode));
+ }
+ CbObject BucketResponse = PayloadToCbObject("Failed listing namespaces"sv, BucketsResult.Response);
+
+ Response.BeginArray("items");
+ for (CbFieldView BucketField : BucketResponse["buckets"])
+ {
+ std::string_view Bucket = BucketField.AsString();
+ if (!Bucket.empty())
+ {
+ Response.AddString(Bucket);
+ }
+ }
+ Response.EndArray();
+ }
+
+ Response.EndObject();
+ }
+ }
+ Response.EndArray();
+
+ return Response.Save();
+ }
+
virtual CbObject ListBuilds(CbObject Query) override
{
ZEN_TRACE_CPU("Jupiter::ListBuilds");
@@ -49,7 +105,7 @@ public:
{
throw std::runtime_error(fmt::format("Failed listing builds: {} ({})", ListResult.Reason, ListResult.ErrorCode));
}
- return PayloadToJson("Failed listing builds"sv, ListResult.Response);
+ return PayloadToCbObject("Failed listing builds"sv, ListResult.Response);
}
virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override
@@ -66,7 +122,7 @@ public:
{
throw std::runtime_error(fmt::format("Failed creating build: {} ({})", PutResult.Reason, PutResult.ErrorCode));
}
- return PayloadToJson(fmt::format("Failed creating build: {}", BuildId), PutResult.Response);
+ return PayloadToCbObject(fmt::format("Failed creating build: {}", BuildId), PutResult.Response);
}
virtual CbObject GetBuild(const Oid& BuildId) override
@@ -81,7 +137,7 @@ public:
{
throw std::runtime_error(fmt::format("Failed fetching build: {} ({})", GetBuildResult.Reason, GetBuildResult.ErrorCode));
}
- return PayloadToJson(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response);
+ return PayloadToCbObject(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response);
}
virtual void FinalizeBuild(const Oid& BuildId) override
@@ -134,7 +190,7 @@ public:
GetBuildPartResult.Reason,
GetBuildPartResult.ErrorCode));
}
- return PayloadToJson(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response);
+ return PayloadToCbObject(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response);
}
virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override
@@ -235,19 +291,25 @@ public:
return std::move(GetBuildBlobResult.Response);
}
- virtual std::vector<std::function<void()>> GetLargeBuildBlob(
- const Oid& BuildId,
- const IoHash& RawHash,
- uint64_t ChunkSize,
- std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver) override
+ virtual std::vector<std::function<void()>> GetLargeBuildBlob(const Oid& BuildId,
+ const IoHash& RawHash,
+ uint64_t ChunkSize,
+ std::function<void(uint64_t Offset, const IoBuffer& Chunk)>&& OnReceive,
+ std::function<void()>&& OnComplete) override
{
ZEN_TRACE_CPU("Jupiter::GetLargeBuildBlob");
Stopwatch ExecutionTimer;
auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
std::vector<std::function<JupiterResult()>> WorkItems;
- JupiterResult GetMultipartBlobResult =
- m_Session.GetMultipartBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, ChunkSize, std::move(Receiver), WorkItems);
+ JupiterResult GetMultipartBlobResult = m_Session.GetMultipartBuildBlob(m_Namespace,
+ m_Bucket,
+ BuildId,
+ RawHash,
+ ChunkSize,
+ std::move(OnReceive),
+ std::move(OnComplete),
+ WorkItems);
AddStatistic(GetMultipartBlobResult);
if (!GetMultipartBlobResult.Success)
@@ -272,7 +334,7 @@ public:
return WorkList;
}
- virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
+ virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
{
ZEN_TRACE_CPU("Jupiter::PutBlockMetadata");
@@ -284,27 +346,32 @@ public:
AddStatistic(PutMetaResult);
if (!PutMetaResult.Success)
{
+ if (PutMetaResult.ErrorCode == int32_t(HttpResponseCode::NotFound))
+ {
+ return false;
+ }
throw std::runtime_error(
fmt::format("Failed putting build block metadata: {} ({})", PutMetaResult.Reason, PutMetaResult.ErrorCode));
}
+ return true;
}
- virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override
+ virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override
{
ZEN_TRACE_CPU("Jupiter::FindBlocks");
Stopwatch ExecutionTimer;
auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId);
+ JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId, MaxBlockCount);
AddStatistic(FindResult);
if (!FindResult.Success)
{
throw std::runtime_error(fmt::format("Failed fetching known blocks: {} ({})", FindResult.Reason, FindResult.ErrorCode));
}
- return ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching known blocks"sv, FindResult.Response));
+ return PayloadToCbObject("Failed fetching known blocks"sv, FindResult.Response);
}
- virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
+ virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
{
ZEN_TRACE_CPU("Jupiter::GetBlockMetadata");
@@ -328,28 +395,35 @@ public:
throw std::runtime_error(
fmt::format("Failed fetching block metadatas: {} ({})", GetBlockMetadataResult.Reason, GetBlockMetadataResult.ErrorCode));
}
- std::vector<ChunkBlockDescription> UnorderedList =
- ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching block metadatas", GetBlockMetadataResult.Response));
- tsl::robin_map<IoHash, size_t, IoHash::Hasher> BlockDescriptionLookup;
- for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++)
+ return PayloadToCbObject("Failed fetching block metadatas", GetBlockMetadataResult.Response);
+ }
+
+ virtual void PutBuildPartStats(const Oid& BuildId,
+ const Oid& BuildPartId,
+ const tsl::robin_map<std::string, double>& FloatStats) override
+ {
+ ZEN_UNUSED(BuildId, BuildPartId, FloatStats);
+ CbObjectWriter Request;
+ Request.BeginObject("floatStats"sv);
+ for (auto It : FloatStats)
{
- const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex];
- BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex);
+ Request.AddFloat(It.first, It.second);
}
- std::vector<ChunkBlockDescription> SortedBlockDescriptions;
- SortedBlockDescriptions.reserve(BlockDescriptionLookup.size());
- for (const IoHash& BlockHash : BlockHashes)
+ Request.EndObject();
+ IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer();
+ Payload.SetContentType(ZenContentType::kCbObject);
+ JupiterResult PutBuildPartStatsResult = m_Session.PutBuildPartStats(m_Namespace, m_Bucket, BuildId, BuildPartId, Payload);
+ AddStatistic(PutBuildPartStatsResult);
+ if (!PutBuildPartStatsResult.Success)
{
- if (auto It = BlockDescriptionLookup.find(BlockHash); It != BlockDescriptionLookup.end())
- {
- SortedBlockDescriptions.push_back(std::move(UnorderedList[It->second]));
- }
+ throw std::runtime_error(fmt::format("Failed posting build part statistics: {} ({})",
+ PutBuildPartStatsResult.Reason,
+ PutBuildPartStatsResult.ErrorCode));
}
- return SortedBlockDescriptions;
}
private:
- static CbObject PayloadToJson(std::string_view Context, const IoBuffer& Payload)
+ static CbObject PayloadToCbObject(std::string_view Context, const IoBuffer& Payload)
{
if (Payload.GetContentType() == ZenContentType::kJSON)
{
@@ -394,11 +468,12 @@ CreateJupiterBuildStorage(LoggerRef InLog,
BuildStorage::Statistics& Stats,
std::string_view Namespace,
std::string_view Bucket,
+ bool AllowRedirect,
const std::filesystem::path& TempFolderPath)
{
ZEN_TRACE_CPU("CreateJupiterBuildStorage");
- return std::make_unique<JupiterBuildStorage>(InLog, InHttpClient, Stats, Namespace, Bucket, TempFolderPath);
+ return std::make_unique<JupiterBuildStorage>(InLog, InHttpClient, Stats, Namespace, Bucket, AllowRedirect, TempFolderPath);
}
} // namespace zen
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-20 01:02:49 +0000