aboutsummaryrefslogtreecommitdiff
path: root/src/zenutil
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenutil')
-rw-r--r--src/zenutil/chunkedfile.cpp2
-rw-r--r--src/zenutil/include/zenutil/jupiter/jupitersession.h13
-rw-r--r--src/zenutil/jupiter/jupitersession.cpp85
-rw-r--r--src/zenutil/logging.cpp9
4 files changed, 40 insertions, 69 deletions
diff --git a/src/zenutil/chunkedfile.cpp b/src/zenutil/chunkedfile.cpp
index c08492eb0..3f3a6661c 100644
--- a/src/zenutil/chunkedfile.cpp
+++ b/src/zenutil/chunkedfile.cpp
@@ -121,7 +121,7 @@ ChunkData(BasicFile& RawData, uint64_t Offset, uint64_t Size, ChunkedParams Para
Chunker.SetUseThreshold(Params.UseThreshold);
Chunker.SetChunkSize(Params.MinSize, Params.MaxSize, Params.AvgSize);
size_t End = Offset + Size;
- const size_t ScanBufferSize = 1u * 1024 * 1024; // (Params.MaxSize * 9) / 3;//1 * 1024 * 1024;
+ const size_t ScanBufferSize = Max(1u * 1024 * 1024, Params.MaxSize);
BasicFileBuffer RawBuffer(RawData, ScanBufferSize);
MemoryView SliceView = RawBuffer.MakeView(Min(End - Offset, ScanBufferSize), Offset);
ZEN_ASSERT(!SliceView.IsEmpty());
diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h
index 075c35b40..852271868 100644
--- a/src/zenutil/include/zenutil/jupiter/jupitersession.h
+++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h
@@ -116,21 +116,18 @@ public:
JupiterResult PutBuildBlob(std::string_view Namespace,
std::string_view BucketId,
const Oid& BuildId,
- const Oid& PartId,
const IoHash& Hash,
ZenContentType ContentType,
const CompositeBuffer& Payload);
JupiterResult GetBuildBlob(std::string_view Namespace,
std::string_view BucketId,
const Oid& BuildId,
- const Oid& PartId,
const IoHash& Hash,
std::filesystem::path TempFolderPath);
JupiterResult PutMultipartBuildBlob(std::string_view Namespace,
std::string_view BucketId,
const Oid& BuildId,
- const Oid& PartId,
const IoHash& Hash,
ZenContentType ContentType,
uint64_t PayloadSize,
@@ -139,7 +136,6 @@ public:
JupiterResult GetMultipartBuildBlob(std::string_view Namespace,
std::string_view BucketId,
const Oid& BuildId,
- const Oid& PartId,
const IoHash& Hash,
uint64_t ChunkSize,
std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver,
@@ -147,7 +143,6 @@ public:
JupiterResult PutBlockMetadata(std::string_view Namespace,
std::string_view BucketId,
const Oid& BuildId,
- const Oid& PartId,
const IoHash& Hash,
const IoBuffer& Payload);
FinalizeBuildPartResult FinalizeBuildPart(std::string_view Namespace,
@@ -155,12 +150,8 @@ public:
const Oid& BuildId,
const Oid& PartId,
const IoHash& RawHash);
- JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId);
- JupiterResult GetBlockMetadata(std::string_view Namespace,
- std::string_view BucketId,
- const Oid& BuildId,
- const Oid& PartId,
- IoBuffer Payload);
+ JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId);
+ JupiterResult GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload);
private:
inline LoggerRef Log() { return m_Log; }
diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp
index d56927b44..06ac6ae36 100644
--- a/src/zenutil/jupiter/jupitersession.cpp
+++ b/src/zenutil/jupiter/jupitersession.cpp
@@ -436,15 +436,14 @@ JupiterResult
JupiterSession::PutBuildBlob(std::string_view Namespace,
std::string_view BucketId,
const Oid& BuildId,
- const Oid& PartId,
const IoHash& Hash,
ZenContentType ContentType,
const CompositeBuffer& Payload)
{
- HttpClient::Response Response = m_HttpClient.Upload(
- fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
- Payload,
- ContentType);
+ HttpClient::Response Response =
+ m_HttpClient.Upload(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()),
+ Payload,
+ ContentType);
return detail::ConvertResponse(Response, "JupiterSession::PutBuildBlob"sv);
}
@@ -452,7 +451,6 @@ JupiterResult
JupiterSession::PutMultipartBuildBlob(std::string_view Namespace,
std::string_view BucketId,
const Oid& BuildId,
- const Oid& PartId,
const IoHash& Hash,
ZenContentType ContentType,
uint64_t PayloadSize,
@@ -498,12 +496,8 @@ JupiterSession::PutMultipartBuildBlob(std::string_view Namespace,
StartMultipartPayloadWriter.AddInteger("blobLength"sv, PayloadSize);
CbObject StartMultipartPayload = StartMultipartPayloadWriter.Save();
- std::string StartMultipartResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/startMultipartUpload",
- Namespace,
- BucketId,
- BuildId,
- PartId,
- Hash.ToHexString());
+ std::string StartMultipartResponseRequestString =
+ fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/startMultipartUpload", Namespace, BucketId, BuildId, Hash.ToHexString());
// ZEN_INFO("POST: {}", StartMultipartResponseRequestString);
HttpClient::Response StartMultipartResponse =
m_HttpClient.Post(StartMultipartResponseRequestString, StartMultipartPayload, HttpClient::Accept(ZenContentType::kCbObject));
@@ -529,15 +523,14 @@ JupiterSession::PutMultipartBuildBlob(std::string_view Namespace,
for (size_t PartIndex = 0; PartIndex < Workload->PartDescription.Parts.size(); PartIndex++)
{
- OutWorkItems.emplace_back([this, Namespace, BucketId, BuildId, PartId, Hash, ContentType, Workload, PartIndex](
+ OutWorkItems.emplace_back([this, Namespace, BucketId, BuildId, Hash, ContentType, Workload, PartIndex](
bool& OutIsComplete) -> JupiterResult {
const MultipartUploadResponse::Part& Part = Workload->PartDescription.Parts[PartIndex];
IoBuffer PartPayload = Workload->Transmitter(Part.FirstByte, Part.LastByte - Part.FirstByte);
- std::string MultipartUploadResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/uploadMultipart{}",
+ std::string MultipartUploadResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}",
Namespace,
BucketId,
BuildId,
- PartId,
Hash.ToHexString(),
Part.QueryString);
// ZEN_INFO("PUT: {}", MultipartUploadResponseRequestString);
@@ -571,12 +564,7 @@ JupiterSession::PutMultipartBuildBlob(std::string_view Namespace,
CbObject CompletePayload = CompletePayloadWriter.Save();
std::string MultipartEndResponseRequestString =
- fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/completeMultipart",
- Namespace,
- BucketId,
- BuildId,
- PartId,
- Hash.ToHexString());
+ fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/completeMultipart", Namespace, BucketId, BuildId, Hash.ToHexString());
MultipartEndResponse = m_HttpClient.Post(MultipartEndResponseRequestString,
CompletePayload,
@@ -600,13 +588,12 @@ JupiterSession::PutMultipartBuildBlob(std::string_view Namespace,
size_t RetryPartIndex = PartNameToIndex.at(RetryPartId);
const MultipartUploadResponse::Part& RetryPart = Workload->PartDescription.Parts[RetryPartIndex];
IoBuffer RetryPartPayload =
- Workload->Transmitter(RetryPart.FirstByte, RetryPart.LastByte - RetryPart.FirstByte);
+ Workload->Transmitter(RetryPart.FirstByte, RetryPart.LastByte - RetryPart.FirstByte - 1);
std::string RetryMultipartUploadResponseRequestString =
- fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/uploadMultipart{}",
+ fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}",
Namespace,
BucketId,
BuildId,
- RetryPartId,
Hash.ToHexString(),
RetryPart.QueryString);
@@ -642,14 +629,12 @@ JupiterResult
JupiterSession::GetMultipartBuildBlob(std::string_view Namespace,
std::string_view BucketId,
const Oid& BuildId,
- const Oid& PartId,
const IoHash& Hash,
uint64_t ChunkSize,
std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver,
std::vector<std::function<JupiterResult()>>& OutWorkItems)
{
- std::string RequestUrl =
- fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString());
+ std::string RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString());
HttpClient::Response Response =
m_HttpClient.Get(RequestUrl, HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", 0, ChunkSize - 1)}}));
if (Response.IsSuccess())
@@ -683,17 +668,12 @@ JupiterSession::GetMultipartBuildBlob(std::string_view Namespa
{
uint64_t PartSize = Min(ChunkSize, TotalSize - Offset);
OutWorkItems.emplace_back(
- [this, Namespace, BucketId, BuildId, PartId, Hash, TotalSize, Workload, Offset, PartSize]()
- -> JupiterResult {
- std::string RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}",
- Namespace,
- BucketId,
- BuildId,
- PartId,
- Hash.ToHexString());
- HttpClient::Response Response = m_HttpClient.Get(
- RequestUrl,
- HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}}));
+ [this, Namespace, BucketId, BuildId, Hash, TotalSize, Workload, Offset, PartSize]() -> JupiterResult {
+ std::string RequestUrl =
+ fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString());
+ HttpClient::Response Response = m_HttpClient.Get(
+ RequestUrl,
+ HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}}));
if (Response.IsSuccess())
{
uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Response.ResponsePayload.GetSize());
@@ -717,13 +697,12 @@ JupiterResult
JupiterSession::GetBuildBlob(std::string_view Namespace,
std::string_view BucketId,
const Oid& BuildId,
- const Oid& PartId,
const IoHash& Hash,
std::filesystem::path TempFolderPath)
{
- HttpClient::Response Response = m_HttpClient.Download(
- fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
- TempFolderPath);
+ HttpClient::Response Response =
+ m_HttpClient.Download(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()),
+ TempFolderPath);
return detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv);
}
@@ -732,14 +711,13 @@ JupiterResult
JupiterSession::PutBlockMetadata(std::string_view Namespace,
std::string_view BucketId,
const Oid& BuildId,
- const Oid& PartId,
const IoHash& Hash,
const IoBuffer& Payload)
{
ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
- HttpClient::Response Response = m_HttpClient.Put(
- fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/{}/metadata", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()),
- Payload);
+ HttpClient::Response Response =
+ m_HttpClient.Put(fmt::format("/api/v2/builds/{}/{}/{}/blocks/{}/metadata", Namespace, BucketId, BuildId, Hash.ToHexString()),
+ Payload);
return detail::ConvertResponse(Response, "JupiterSession::PutBlockMetadata"sv);
}
@@ -772,24 +750,19 @@ JupiterSession::FinalizeBuildPart(std::string_view Namespace,
}
JupiterResult
-JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId)
+JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId)
{
- HttpClient::Response Response =
- m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/listBlocks", Namespace, BucketId, BuildId, PartId),
- HttpClient::Accept(ZenContentType::kCbObject));
+ HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/blocks/listBlocks", Namespace, BucketId, BuildId),
+ HttpClient::Accept(ZenContentType::kCbObject));
return detail::ConvertResponse(Response, "JupiterSession::FindBlocks"sv);
}
JupiterResult
-JupiterSession::GetBlockMetadata(std::string_view Namespace,
- std::string_view BucketId,
- const Oid& BuildId,
- const Oid& PartId,
- IoBuffer Payload)
+JupiterSession::GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload)
{
ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject);
HttpClient::Response Response =
- m_HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/getBlockMetadata", Namespace, BucketId, BuildId, PartId),
+ m_HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/blocks/getBlockMetadata", Namespace, BucketId, BuildId),
Payload,
HttpClient::Accept(ZenContentType::kCbObject));
return detail::ConvertResponse(Response, "JupiterSession::GetBlockMetadata"sv);
diff --git a/src/zenutil/logging.cpp b/src/zenutil/logging.cpp
index 6314c407f..0444fa2c4 100644
--- a/src/zenutil/logging.cpp
+++ b/src/zenutil/logging.cpp
@@ -10,6 +10,7 @@ ZEN_THIRD_PARTY_INCLUDES_START
#include <spdlog/spdlog.h>
ZEN_THIRD_PARTY_INCLUDES_END
+#include <zencore/callstack.h>
#include <zencore/compactbinary.h>
#include <zencore/filesystem.h>
#include <zencore/logging.h>
@@ -97,7 +98,13 @@ BeginInitializeLogging(const LoggingOptions& LogOptions)
}
}
- std::set_terminate([]() { ZEN_CRITICAL("Program exited abnormally via std::terminate()"); });
+ std::set_terminate([]() {
+ void* Frames[8];
+ uint32_t FrameCount = GetCallstack(2, 8, Frames);
+ CallstackFrames* Callstack = CreateCallstack(FrameCount, Frames);
+ ZEN_CRITICAL("Program exited abnormally via std::terminate()\n{}", CallstackToString(Callstack, " "));
+ FreeCallstack(Callstack);
+ });
// Default
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-24 20:08:18 +0000