aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver/storage
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenserver/storage')
-rw-r--r--src/zenserver/storage/admin/admin.cpp4
-rw-r--r--src/zenserver/storage/buildstore/httpbuildstore.cpp144
-rw-r--r--src/zenserver/storage/cache/httpstructuredcache.cpp27
-rw-r--r--src/zenserver/storage/objectstore/objectstore.cpp63
-rw-r--r--src/zenserver/storage/projectstore/httpprojectstore.cpp30
-rw-r--r--src/zenserver/storage/storageconfig.cpp18
-rw-r--r--src/zenserver/storage/storageconfig.h5
-rw-r--r--src/zenserver/storage/upstream/upstreamcache.cpp22
-rw-r--r--src/zenserver/storage/zenstorageserver.cpp68
-rw-r--r--src/zenserver/storage/zenstorageserver.h6
10 files changed, 186 insertions, 201 deletions
diff --git a/src/zenserver/storage/admin/admin.cpp b/src/zenserver/storage/admin/admin.cpp
index f1c2daea4..34d9e570e 100644
--- a/src/zenserver/storage/admin/admin.cpp
+++ b/src/zenserver/storage/admin/admin.cpp
@@ -720,7 +720,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
"logs",
[this](HttpRouterRequest& Req) {
CbObjectWriter Obj;
- auto LogLevel = logging::ToStringView(logging::GetLogLevel());
+ auto LogLevel = logging::ToString(logging::GetLogLevel());
Obj.AddString("loglevel", std::string_view(LogLevel.data(), LogLevel.size()));
Obj.AddString("Logfile", PathToUtf8(m_LogPaths.AbsLogPath));
Obj.BeginObject("cache");
@@ -772,7 +772,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler,
if (std::string Param(Params.GetValue("loglevel")); Param.empty() == false)
{
logging::LogLevel NewLevel = logging::ParseLogLevelString(Param);
- std::string_view LogLevel = logging::ToStringView(NewLevel);
+ std::string_view LogLevel = logging::ToString(NewLevel);
if (LogLevel != Param)
{
return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest,
diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp
index bbbb0c37b..f935e2c6b 100644
--- a/src/zenserver/storage/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp
@@ -162,96 +162,81 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
fmt::format("Invalid blob hash '{}'", Hash));
}
- std::vector<std::pair<uint64_t, uint64_t>> OffsetAndLengthPairs;
+ m_BuildStoreStats.BlobReadCount++;
+ IoBuffer Blob = m_BuildStore.GetBlob(BlobHash);
+ if (!Blob)
+ {
+ return ServerRequest.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, fmt::format("Blob {} not found", Hash));
+ }
+ m_BuildStoreStats.BlobHitCount++;
+
if (ServerRequest.RequestVerb() == HttpVerb::kPost)
{
+ if (ServerRequest.AcceptContentType() != HttpContentType::kCbPackage)
+ {
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Accept type '{}' is not supported for blob {}, expected '{}'",
+ ToString(ServerRequest.AcceptContentType()),
+ Hash,
+ ToString(HttpContentType::kCbPackage)));
+ }
+
CbObject RangePayload = ServerRequest.ReadPayloadObject();
- if (RangePayload)
+ if (!RangePayload)
{
- CbArrayView RangesArray = RangePayload["ranges"sv].AsArrayView();
- OffsetAndLengthPairs.reserve(RangesArray.Num());
- for (CbFieldView FieldView : RangesArray)
- {
- CbObjectView RangeView = FieldView.AsObjectView();
- uint64_t RangeOffset = RangeView["offset"sv].AsUInt64();
- uint64_t RangeLength = RangeView["length"sv].AsUInt64();
- OffsetAndLengthPairs.push_back(std::make_pair(RangeOffset, RangeLength));
- }
- if (OffsetAndLengthPairs.size() > MaxRangeCountPerRequestSupported)
- {
- return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
- HttpContentType::kText,
- fmt::format("Number of ranges ({}) for blob request exceeds maximum range count {}",
- OffsetAndLengthPairs.size(),
- MaxRangeCountPerRequestSupported));
- }
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Missing payload for range request on blob {}", BlobHash));
}
- if (OffsetAndLengthPairs.empty())
+
+ CbArrayView RangesArray = RangePayload["ranges"sv].AsArrayView();
+ const uint64_t RangeCount = RangesArray.Num();
+ if (RangeCount == 0)
{
m_BuildStoreStats.BadRequestCount++;
return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
HttpContentType::kText,
- "Fetching blob without ranges must be done with the GET verb");
+ "POST request must include a non-empty 'ranges' array");
}
- }
- else
- {
- HttpRanges Ranges;
- bool HasRange = ServerRequest.TryGetRanges(Ranges);
- if (HasRange)
+ if (RangeCount > MaxRangeCountPerRequestSupported)
{
- if (Ranges.size() > 1)
- {
- // Only a single http range is supported, we have limited support for http multirange responses
- m_BuildStoreStats.BadRequestCount++;
- return ServerRequest.WriteResponse(HttpResponseCode::BadRequest,
- HttpContentType::kText,
- fmt::format("Multiple ranges in blob request is only supported for {} accept type",
- ToString(HttpContentType::kCbPackage)));
- }
- const HttpRange& FirstRange = Ranges.front();
- OffsetAndLengthPairs.push_back(std::make_pair<uint64_t, uint64_t>(FirstRange.Start, FirstRange.End - FirstRange.Start + 1));
+ m_BuildStoreStats.BadRequestCount++;
+ return ServerRequest.WriteResponse(
+ HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Range count {} exceeds maximum of {}", RangeCount, MaxRangeCountPerRequestSupported));
}
- }
-
- m_BuildStoreStats.BlobReadCount++;
- IoBuffer Blob = m_BuildStore.GetBlob(BlobHash);
- if (!Blob)
- {
- return ServerRequest.WriteResponse(HttpResponseCode::NotFound,
- HttpContentType::kText,
- fmt::format("Blob with hash '{}' could not be found", Hash));
- }
- m_BuildStoreStats.BlobHitCount++;
- if (OffsetAndLengthPairs.empty())
- {
- return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob);
- }
+ const uint64_t BlobSize = Blob.GetSize();
+ std::vector<IoBuffer> RangeBuffers;
+ RangeBuffers.reserve(RangeCount);
- if (ServerRequest.AcceptContentType() == HttpContentType::kCbPackage)
- {
- const uint64_t BlobSize = Blob.GetSize();
+ CbPackage ResponsePackage;
+ CbObjectWriter Writer;
- CbPackage ResponsePackage;
- std::vector<IoBuffer> RangeBuffers;
- CbObjectWriter Writer;
Writer.BeginArray("ranges"sv);
- for (const std::pair<uint64_t, uint64_t>& Range : OffsetAndLengthPairs)
+ for (CbFieldView FieldView : RangesArray)
{
- const uint64_t MaxBlobSize = Range.first < BlobSize ? BlobSize - Range.first : 0;
- const uint64_t RangeSize = Min(Range.second, MaxBlobSize);
+ CbObjectView RangeView = FieldView.AsObjectView();
+ uint64_t RangeOffset = RangeView["offset"sv].AsUInt64();
+ uint64_t RangeLength = RangeView["length"sv].AsUInt64();
+
+ const uint64_t MaxBlobSize = RangeOffset < BlobSize ? BlobSize - RangeOffset : 0;
+ const uint64_t RangeSize = Min(RangeLength, MaxBlobSize);
Writer.BeginObject();
{
- if (Range.first + RangeSize <= BlobSize)
+ if (RangeOffset + RangeSize <= BlobSize)
{
- RangeBuffers.push_back(IoBuffer(Blob, Range.first, RangeSize));
- Writer.AddInteger("offset"sv, Range.first);
+ RangeBuffers.push_back(IoBuffer(Blob, RangeOffset, RangeSize));
+ Writer.AddInteger("offset"sv, RangeOffset);
Writer.AddInteger("length"sv, RangeSize);
}
else
{
- Writer.AddInteger("offset"sv, Range.first);
+ Writer.AddInteger("offset"sv, RangeOffset);
Writer.AddInteger("length"sv, 0);
}
}
@@ -259,7 +244,7 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
}
Writer.EndArray();
- CompositeBuffer Ranges(RangeBuffers);
+ CompositeBuffer Ranges(std::move(RangeBuffers));
CbAttachment PayloadAttachment(std::move(Ranges), BlobHash);
Writer.AddAttachment("payload", PayloadAttachment);
@@ -269,32 +254,21 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
ResponsePackage.SetObject(HeaderObject);
CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage);
- uint64_t ResponseSize = RpcResponseBuffer.GetSize();
- ZEN_UNUSED(ResponseSize);
return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer);
}
else
{
- if (OffsetAndLengthPairs.size() != 1)
+ HttpRanges RequestedRangeHeader;
+ bool HasRange = ServerRequest.TryGetRanges(RequestedRangeHeader);
+ if (HasRange)
{
- // Only a single http range is supported, we have limited support for http multirange responses
- m_BuildStoreStats.BadRequestCount++;
- return ServerRequest.WriteResponse(
- HttpResponseCode::BadRequest,
- HttpContentType::kText,
- fmt::format("Multiple ranges in blob request is only supported for {} accept type", ToString(HttpContentType::kCbPackage)));
+ // Standard HTTP GET with Range header: framework handles 206, Content-Range, and 416 on OOB.
+ return ServerRequest.WriteResponse(HttpContentType::kBinary, Blob, RequestedRangeHeader);
}
-
- const std::pair<uint64_t, uint64_t>& OffsetAndLength = OffsetAndLengthPairs.front();
- const uint64_t BlobSize = Blob.GetSize();
- const uint64_t MaxBlobSize = OffsetAndLength.first < BlobSize ? BlobSize - OffsetAndLength.first : 0;
- const uint64_t RangeSize = Min(OffsetAndLength.second, MaxBlobSize);
- if (OffsetAndLength.first + RangeSize > BlobSize)
+ else
{
- return ServerRequest.WriteResponse(HttpResponseCode::NoContent);
+ return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob);
}
- Blob = IoBuffer(Blob, OffsetAndLength.first, RangeSize);
- return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob);
}
}
diff --git a/src/zenserver/storage/cache/httpstructuredcache.cpp b/src/zenserver/storage/cache/httpstructuredcache.cpp
index 8ad48225b..4d3673e70 100644
--- a/src/zenserver/storage/cache/httpstructuredcache.cpp
+++ b/src/zenserver/storage/cache/httpstructuredcache.cpp
@@ -437,19 +437,22 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
std::string RecordPath = UrlDecode(Params.GetValue("path"));
- uint32_t ThreadCount = GetHardwareConcurrency();
+ const uint32_t HardwareConcurrency = GetHardwareConcurrency();
+ const uint32_t MaxThreadCount = std::max<uint32_t>(HardwareConcurrency, 16u);
+ uint32_t ThreadCount = HardwareConcurrency;
if (auto Param = Params.GetValue("thread_count"); Param.empty() == false)
{
if (auto Value = ParseInt<uint64_t>(Param))
{
- ThreadCount = gsl::narrow<uint32_t>(Value.value());
+ ThreadCount = gsl::narrow_cast<uint32_t>(std::min<uint64_t>(Value.value(), MaxThreadCount));
}
}
+ ThreadCount = std::clamp<uint32_t>(ThreadCount, 1u, MaxThreadCount);
ZEN_INFO("initiating cache RPC replay using {} threads, from '{}'", ThreadCount, RecordPath);
std::unique_ptr<cache::IRpcRequestReplayer> Replayer(cache::MakeDiskRequestReplayer(RecordPath, false));
- ReplayRequestRecorder(RequestContext, *Replayer, ThreadCount < 1 ? 1 : ThreadCount);
+ ReplayRequestRecorder(RequestContext, *Replayer, ThreadCount);
ZEN_INFO("cache RPC replay COMPLETED");
@@ -557,7 +560,7 @@ HttpStructuredCacheService::HandleCacheRequest(HttpServerRequest& Request)
break;
default:
m_CacheStats.BadRequestCount++;
- break;
+ return Request.WriteResponse(HttpResponseCode::MethodNotAllowed);
}
}
@@ -707,7 +710,8 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque
break;
default:
- break;
+ m_CacheStats.BadRequestCount++;
+ return Request.WriteResponse(HttpResponseCode::MethodNotAllowed);
}
}
@@ -797,7 +801,8 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request,
break;
default:
- break;
+ m_CacheStats.BadRequestCount++;
+ return Request.WriteResponse(HttpResponseCode::MethodNotAllowed);
}
}
@@ -816,7 +821,8 @@ HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request,
break;
default:
- break;
+ m_CacheStats.BadRequestCount++;
+ return Request.WriteResponse(HttpResponseCode::MethodNotAllowed);
}
}
@@ -1216,8 +1222,6 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
}
auto WriteFailureResponse = [&Request](const ZenCacheStore::PutResult& PutResult) {
- ZEN_UNUSED(PutResult);
-
HttpResponseCode ResponseCode = HttpResponseCode::InternalServerError;
switch (PutResult.Status)
{
@@ -1231,7 +1235,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
if (PutResult.Details)
{
- Request.WriteResponse(ResponseCode, PutResult.Details);
+ return Request.WriteResponse(ResponseCode, PutResult.Details);
}
return Request.WriteResponse(ResponseCode);
};
@@ -1507,7 +1511,8 @@ HttpStructuredCacheService::HandleCacheChunkRequest(HttpServerRequest& Request,
HandlePutCacheChunk(Request, Ref, PolicyFromUrl);
break;
default:
- break;
+ m_CacheStats.BadRequestCount++;
+ return Request.WriteResponse(HttpResponseCode::MethodNotAllowed);
}
}
diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp
index d6516fa1a..1115c1cd6 100644
--- a/src/zenserver/storage/objectstore/objectstore.cpp
+++ b/src/zenserver/storage/objectstore/objectstore.cpp
@@ -637,11 +637,7 @@ HttpObjectStoreService::GetObject(HttpRouterRequest& Request, const std::string_
}
HttpRanges Ranges;
- if (Request.ServerRequest().TryGetRanges(Ranges); Ranges.size() > 1)
- {
- // Only a single range is supported
- return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
- }
+ Request.ServerRequest().TryGetRanges(Ranges);
FileContents File;
{
@@ -665,42 +661,49 @@ HttpObjectStoreService::GetObject(HttpRouterRequest& Request, const std::string_
if (Ranges.empty())
{
- const uint64_t TotalServed = m_TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size();
-
+ const uint64_t TotalServed = m_TotalBytesServed.fetch_add(FileBuf.GetSize()) + FileBuf.GetSize();
ZEN_LOG_DEBUG(LogObj,
"GET - '{}/{}' ({}) [OK] (Served: {})",
BucketName,
RelativeBucketPath,
- NiceBytes(FileBuf.Size()),
+ NiceBytes(FileBuf.GetSize()),
NiceBytes(TotalServed));
-
- Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, FileBuf);
}
else
{
- const auto Range = Ranges[0];
- const uint64_t RangeSize = 1 + (Range.End - Range.Start);
- const uint64_t TotalServed = m_TotalBytesServed.fetch_add(RangeSize) + RangeSize;
-
- ZEN_LOG_DEBUG(LogObj,
- "GET - '{}/{}' (Range: {}-{}) ({}/{}) [OK] (Served: {})",
- BucketName,
- RelativeBucketPath,
- Range.Start,
- Range.End,
- NiceBytes(RangeSize),
- NiceBytes(FileBuf.Size()),
- NiceBytes(TotalServed));
-
- MemoryView RangeView = FileBuf.GetView().Mid(Range.Start, RangeSize);
- if (RangeView.GetSize() != RangeSize)
+ const uint64_t TotalSize = FileBuf.GetSize();
+ uint64_t ServedBytes = 0;
+ for (const HttpRange& Range : Ranges)
{
- return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+ const uint64_t RangeEnd = (Range.End != ~uint64_t(0)) ? Range.End : TotalSize - 1;
+ if (RangeEnd < TotalSize && Range.Start <= RangeEnd)
+ {
+ ServedBytes += 1 + (RangeEnd - Range.Start);
+ }
+ }
+ if (ServedBytes > 0)
+ {
+ const uint64_t TotalServed = m_TotalBytesServed.fetch_add(ServedBytes) + ServedBytes;
+ ZEN_LOG_DEBUG(LogObj,
+ "GET - '{}/{}' (Ranges: {}) ({}/{}) [OK] (Served: {})",
+ BucketName,
+ RelativeBucketPath,
+ Ranges.size(),
+ NiceBytes(ServedBytes),
+ NiceBytes(TotalSize),
+ NiceBytes(TotalServed));
+ }
+ else
+ {
+ ZEN_LOG_DEBUG(LogObj,
+ "GET - '{}/{}' (Ranges: {}) [416] ({})",
+ BucketName,
+ RelativeBucketPath,
+ Ranges.size(),
+ NiceBytes(TotalSize));
}
-
- IoBuffer RangeBuf = IoBuffer(IoBuffer::Wrap, RangeView.GetData(), RangeView.GetSize());
- Request.ServerRequest().WriteResponse(HttpResponseCode::PartialContent, HttpContentType::kBinary, RangeBuf);
}
+ Request.ServerRequest().WriteResponse(HttpContentType::kBinary, FileBuf, Ranges);
}
void
diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp
index afd0d8f82..9844d02f0 100644
--- a/src/zenserver/storage/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp
@@ -16,9 +16,9 @@
#include <zenhttp/httpclientauth.h>
#include <zenhttp/packageformat.h>
#include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/builds/buildstorageresolve.h>
#include <zenremotestore/builds/buildstorageutil.h>
#include <zenremotestore/jupiter/jupiterhost.h>
-#include <zenremotestore/operationlogoutput.h>
#include <zenremotestore/projectstore/buildsremoteprojectstore.h>
#include <zenremotestore/projectstore/fileremoteprojectstore.h>
#include <zenremotestore/projectstore/jupiterremoteprojectstore.h>
@@ -279,7 +279,7 @@ namespace {
{
ZEN_MEMSCOPE(GetProjectHttpTag());
- auto Log = [InLog]() { return InLog; };
+ ZEN_SCOPED_LOG(InLog);
using namespace std::literals;
@@ -566,11 +566,9 @@ namespace {
.AllowResume = true,
.RetryCount = 2};
- std::unique_ptr<OperationLogOutput> Output(CreateStandardLogOutput(Log()));
-
try
{
- ResolveResult = ResolveBuildStorage(*Output,
+ ResolveResult = ResolveBuildStorage(Log(),
ClientSettings,
Host,
OverrideHost,
@@ -636,11 +634,6 @@ namespace {
return Result;
}
- static uint64_t GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory)
- {
- return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u;
- }
-
} // namespace
//////////////////////////////////////////////////////////////////////////
@@ -787,22 +780,22 @@ HttpProjectService::HttpProjectService(CidStore& Store,
HttpVerb::kPost);
m_Router.RegisterRoute(
- "details\\$",
+ "details$",
[this](HttpRouterRequest& Req) { HandleDetailsRequest(Req); },
HttpVerb::kGet);
m_Router.RegisterRoute(
- "details\\$/{project}",
+ "details$/{project}",
[this](HttpRouterRequest& Req) { HandleProjectDetailsRequest(Req); },
HttpVerb::kGet);
m_Router.RegisterRoute(
- "details\\$/{project}/{log}",
+ "details$/{project}/{log}",
[this](HttpRouterRequest& Req) { HandleOplogDetailsRequest(Req); },
HttpVerb::kGet);
m_Router.RegisterRoute(
- "details\\$/{project}/{log}/{chunk}",
+ "details$/{project}/{log}/{chunk}",
[this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); },
HttpVerb::kGet);
@@ -1264,7 +1257,7 @@ HttpProjectService::HandleChunkInfoRequest(HttpRouterRequest& Req)
const Oid Obj = Oid::FromHexString(ChunkId);
- CbObject ResponsePayload = ProjectStore::GetChunkInfo(Log(), *Project, *FoundLog, Obj);
+ CbObject ResponsePayload = ProjectStore::GetChunkInfo(*Project, *FoundLog, Obj);
if (ResponsePayload)
{
m_ProjectStats.ChunkHitCount++;
@@ -1353,7 +1346,7 @@ HttpProjectService::HandleChunkByIdRequest(HttpRouterRequest& Req)
HttpContentType AcceptType = HttpReq.AcceptContentType();
ProjectStore::GetChunkRangeResult Result =
- ProjectStore::GetChunkRange(Log(), *Project, *FoundLog, Obj, Offset, Size, AcceptType, /*OptionalInOutModificationTag*/ nullptr);
+ ProjectStore::GetChunkRange(*Project, *FoundLog, Obj, Offset, Size, AcceptType, /*OptionalInOutModificationTag*/ nullptr);
switch (Result.Error)
{
@@ -2691,6 +2684,7 @@ HttpProjectService::HandleOplogLoadRequest(HttpRouterRequest& Req)
try
{
CbObject ContainerObject = BuildContainer(
+ Log(),
m_CidStore,
*Project,
*Oplog,
@@ -2891,6 +2885,7 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
try
{
LoadOplog(LoadOplogContext{
+ .Log = Log(),
.ChunkStore = m_CidStore,
.RemoteStore = *RemoteStoreResult->Store,
.OptionalCache = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Cache.get() : nullptr,
@@ -3016,7 +3011,8 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req)
try
{
- SaveOplog(m_CidStore,
+ SaveOplog(Log(),
+ m_CidStore,
*ActualRemoteStore,
*Project,
*Oplog,
diff --git a/src/zenserver/storage/storageconfig.cpp b/src/zenserver/storage/storageconfig.cpp
index 0dbb45164..bb4f053e4 100644
--- a/src/zenserver/storage/storageconfig.cpp
+++ b/src/zenserver/storage/storageconfig.cpp
@@ -57,6 +57,12 @@ ZenStorageServerConfigurator::ValidateOptions()
ZEN_WARN("'--gc-v2=false' is deprecated, reverting to '--gc-v2=true'");
ServerOptions.GcConfig.UseGCV2 = true;
}
+ if (ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent > 100)
+ {
+ throw OptionParseException(fmt::format("'--buildstore-disksizelimit-percent' ('{}') is invalid, must be between 1 and 100.",
+ ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent),
+ {});
+ }
}
class ZenStructuredCacheBucketsConfigOption : public LuaConfig::OptionValue
@@ -382,6 +388,9 @@ ZenStorageServerConfigurator::AddConfigOptions(LuaConfig::Options& LuaOptions)
////// buildsstore
LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv);
LuaOptions.AddOption("server.buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit");
+ LuaOptions.AddOption("server.buildstore.disksizelimitpercent"sv,
+ ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent,
+ "buildstore-disksizelimit-percent");
////// cache
LuaOptions.AddOption("cache.enable"sv, ServerOptions.StructuredCacheConfig.Enabled);
@@ -477,7 +486,7 @@ ZenStorageServerConfigurator::AddConfigOptions(LuaConfig::Options& LuaOptions)
ServerOptions.GcConfig.CompactBlockUsageThresholdPercent,
"gc-compactblock-threshold"sv);
LuaOptions.AddOption("gc.verbose"sv, ServerOptions.GcConfig.Verbose, "gc-verbose"sv);
- LuaOptions.AddOption("gc.single-threaded"sv, ServerOptions.GcConfig.SingleThreaded, "gc-single-threaded"sv);
+ LuaOptions.AddOption("gc.singlethreaded"sv, ServerOptions.GcConfig.SingleThreaded, "gc-single-threaded"sv);
LuaOptions.AddOption("gc.cache.attachment.store"sv, ServerOptions.GcConfig.StoreCacheAttachmentMetaData, "gc-cache-attachment-store");
LuaOptions.AddOption("gc.projectstore.attachment.store"sv,
ServerOptions.GcConfig.StoreProjectAttachmentMetaData,
@@ -1035,6 +1044,13 @@ ZenStorageServerCmdLineOptions::AddBuildStoreOptions(cxxopts::Options& options,
"Max number of bytes before build store entries get evicted. Default set to 1099511627776 (1TB week)",
cxxopts::value<uint64_t>(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit)->default_value("1099511627776"),
"");
+ options.add_option("buildstore",
+ "",
+ "buildstore-disksizelimit-percent",
+ "Max percentage (1-100) of total drive capacity (of --data-dir drive) before build store entries get evicted. "
+ "0 (default) disables this limit. When combined with --buildstore-disksizelimit, the lower value wins.",
+ cxxopts::value<uint32_t>(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent)->default_value("0"),
+ "");
}
void
diff --git a/src/zenserver/storage/storageconfig.h b/src/zenserver/storage/storageconfig.h
index 18af4f096..fec8fd70b 100644
--- a/src/zenserver/storage/storageconfig.h
+++ b/src/zenserver/storage/storageconfig.h
@@ -135,8 +135,9 @@ struct ZenProjectStoreConfig
struct ZenBuildStoreConfig
{
- bool Enabled = false;
- uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB
+ bool Enabled = false;
+ uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB
+ uint32_t MaxDiskSpaceLimitPercent = 0;
};
struct ZenWorkspacesConfig
diff --git a/src/zenserver/storage/upstream/upstreamcache.cpp b/src/zenserver/storage/upstream/upstreamcache.cpp
index b26c57414..a516c452c 100644
--- a/src/zenserver/storage/upstream/upstreamcache.cpp
+++ b/src/zenserver/storage/upstream/upstreamcache.cpp
@@ -772,7 +772,7 @@ namespace detail {
UpstreamEndpointInfo m_Info;
UpstreamStatus m_Status;
UpstreamEndpointStats m_Stats;
- RefPtr<JupiterClient> m_Client;
+ Ref<JupiterClient> m_Client;
const bool m_AllowRedirect = false;
};
@@ -1446,7 +1446,7 @@ namespace detail {
// Make sure we safely bump the refcount inside a scope lock
RwLock::SharedLockScope _(m_ClientLock);
ZEN_ASSERT(m_Client);
- Ref<ZenStructuredCacheClient> ClientRef(m_Client);
+ Ref<ZenStructuredCacheClient> ClientRef(m_Client.Get());
_.ReleaseNow();
return ClientRef;
}
@@ -1485,15 +1485,15 @@ namespace detail {
LoggerRef Log() { return m_Log; }
- LoggerRef m_Log;
- UpstreamEndpointInfo m_Info;
- UpstreamStatus m_Status;
- UpstreamEndpointStats m_Stats;
- std::vector<ZenEndpoint> m_Endpoints;
- std::chrono::milliseconds m_ConnectTimeout;
- std::chrono::milliseconds m_Timeout;
- RwLock m_ClientLock;
- RefPtr<ZenStructuredCacheClient> m_Client;
+ LoggerRef m_Log;
+ UpstreamEndpointInfo m_Info;
+ UpstreamStatus m_Status;
+ UpstreamEndpointStats m_Stats;
+ std::vector<ZenEndpoint> m_Endpoints;
+ std::chrono::milliseconds m_ConnectTimeout;
+ std::chrono::milliseconds m_Timeout;
+ RwLock m_ClientLock;
+ Ref<ZenStructuredCacheClient> m_Client;
};
} // namespace detail
diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp
index 6b1da5f12..44291395a 100644
--- a/src/zenserver/storage/zenstorageserver.cpp
+++ b/src/zenserver/storage/zenstorageserver.cpp
@@ -37,8 +37,6 @@
#include <zenutil/sessionsclient.h>
#include <zenutil/workerpools.h>
#include <zenutil/zenserverprocess.h>
-#include "sessions/inprocsessionlogsink.h"
-#include "sessions/sessions.h"
#if ZEN_PLATFORM_WINDOWS
# include <zencore/windows.h>
@@ -165,11 +163,6 @@ ZenStorageServer::RegisterServices()
m_Http->RegisterService(*m_HttpWorkspacesService);
}
- if (m_HttpSessionsService)
- {
- m_Http->RegisterService(*m_HttpSessionsService);
- }
-
m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatsService, m_StatusService);
if (m_FrontendService)
@@ -229,7 +222,7 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
m_ProjectStore = new ProjectStore(*m_CidStore, m_DataRoot / "projects", m_GcManager, ProjectStore::Configuration{});
m_HttpProjectService.reset(new HttpProjectService{*m_CidStore,
- m_ProjectStore,
+ m_ProjectStore.Get(),
m_StatusService,
m_StatsService,
*m_AuthMgr,
@@ -253,16 +246,6 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
*m_Workspaces));
}
- {
- m_SessionsService = std::make_unique<SessionsService>();
- m_HttpSessionsService = std::make_unique<HttpSessionsService>(m_StatusService, m_StatsService, *m_SessionsService, m_IoContext);
- m_HttpSessionsService->SetSelfSessionId(GetSessionId());
-
- m_InProcSessionLogSink = logging::SinkPtr(new InProcSessionLogSink(*m_SessionsService));
- m_InProcSessionLogSink->SetLevel(logging::Info);
- GetDefaultBroadcastSink()->AddSink(m_InProcSessionLogSink);
- }
-
if (!ServerOptions.SessionsTargetUrl.empty())
{
m_SessionsClient = std::make_unique<SessionsServiceClient>(SessionsServiceClient::Options{
@@ -283,7 +266,31 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
BuildStoreConfig BuildsCfg;
BuildsCfg.RootDirectory = m_DataRoot / "builds";
BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit;
- m_BuildStore = std::make_unique<BuildStore>(std::move(BuildsCfg), m_GcManager, *m_BuildCidStore);
+
+ if (ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent > 0)
+ {
+ DiskSpace Space;
+ if (DiskSpaceInfo(m_DataRoot, Space) && Space.Total > 0)
+ {
+ uint64_t PercentLimit = Space.Total * ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent / 100;
+ BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit > 0
+ ? std::min(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, PercentLimit)
+ : PercentLimit;
+ ZEN_INFO("buildstore disk limit: {}% of {} = {} (effective limit: {})",
+ ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent,
+ NiceBytes(Space.Total),
+ NiceBytes(PercentLimit),
+ NiceBytes(BuildsCfg.MaxDiskSpaceLimit));
+ }
+ else
+ {
+ ZEN_WARN("buildstore-disksizelimit-percent: failed to query disk space for {}, using absolute limit {}",
+ m_DataRoot.string(),
+ NiceBytes(BuildsCfg.MaxDiskSpaceLimit));
+ }
+ }
+
+ m_BuildStore = std::make_unique<BuildStore>(std::move(BuildsCfg), m_GcManager, *m_BuildCidStore);
}
if (ServerOptions.StructuredCacheConfig.Enabled)
@@ -325,13 +332,13 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions
ZEN_OTEL_SPAN("InitializeComputeService");
m_HttpComputeService =
- std::make_unique<compute::HttpComputeService>(*m_CidStore, m_StatsService, ServerOptions.DataDir / "functions");
+ std::make_unique<compute::HttpComputeService>(*m_CidStore, *m_CidStore, m_StatsService, ServerOptions.DataDir / "functions");
}
#endif
#if ZEN_WITH_VFS
m_VfsServiceImpl = std::make_unique<VfsServiceImpl>();
- m_VfsServiceImpl->AddService(Ref<ProjectStore>(m_ProjectStore));
+ m_VfsServiceImpl->AddService(Ref<ProjectStore>(m_ProjectStore.Get()));
m_VfsServiceImpl->AddService(Ref<ZenCacheStore>(m_CacheStore));
m_VfsService = std::make_unique<VfsService>(m_StatusService, m_VfsServiceImpl.get());
@@ -841,11 +848,11 @@ ZenStorageServer::Run()
OnReady();
- m_SessionsService->RegisterSession(GetSessionId(), "zenserver", GetServerMode(), Oid::Zero, {});
+ StartSelfSession("zenserver");
if (m_SessionsClient)
{
- (void)m_SessionsClient->Announce();
+ m_SessionsClient->Announce();
EnqueueSessionAnnounceTimer();
m_SessionLogSink = m_SessionsClient->CreateLogSink();
@@ -891,11 +898,6 @@ ZenStorageServer::Cleanup()
m_Http->Close();
}
- if (m_InProcSessionLogSink)
- {
- GetDefaultBroadcastSink()->RemoveSink(m_InProcSessionLogSink);
- m_InProcSessionLogSink = {};
- }
if (m_SessionLogSink)
{
GetDefaultBroadcastSink()->RemoveSink(m_SessionLogSink);
@@ -903,15 +905,9 @@ ZenStorageServer::Cleanup()
}
if (m_SessionsClient)
{
- (void)m_SessionsClient->Remove();
m_SessionsClient.reset();
}
- if (m_SessionsService)
- {
- m_SessionsService->RemoveSession(GetSessionId());
- }
-
ShutdownServices();
if (m_JobQueue)
@@ -943,8 +939,6 @@ ZenStorageServer::Cleanup()
m_UpstreamCache.reset();
m_CacheStore = {};
- m_HttpSessionsService.reset();
- m_SessionsService.reset();
m_HttpWorkspacesService.reset();
m_Workspaces.reset();
m_HttpProjectService.reset();
@@ -1004,7 +998,7 @@ ZenStorageServer::EnqueueSessionAnnounceTimer()
m_SessionAnnounceTimer.async_wait([this](const asio::error_code& Ec) {
if (!Ec && m_SessionsClient)
{
- (void)m_SessionsClient->Announce();
+ m_SessionsClient->Announce();
EnqueueSessionAnnounceTimer();
}
});
diff --git a/src/zenserver/storage/zenstorageserver.h b/src/zenserver/storage/zenstorageserver.h
index e3c6248e6..9fa46ba9b 100644
--- a/src/zenserver/storage/zenstorageserver.h
+++ b/src/zenserver/storage/zenstorageserver.h
@@ -20,7 +20,6 @@
#include "frontend/frontend.h"
#include "objectstore/objectstore.h"
#include "projectstore/httpprojectstore.h"
-#include "sessions/httpsessions.h"
#include "stats/statsreporter.h"
#include "upstream/upstream.h"
#include "vfs/vfsservice.h"
@@ -81,13 +80,11 @@ private:
HttpTestingService m_TestingService;
#endif
- RefPtr<ProjectStore> m_ProjectStore;
+ Ref<ProjectStore> m_ProjectStore;
std::unique_ptr<VfsServiceImpl> m_VfsServiceImpl;
std::unique_ptr<HttpProjectService> m_HttpProjectService;
std::unique_ptr<Workspaces> m_Workspaces;
std::unique_ptr<HttpWorkspacesService> m_HttpWorkspacesService;
- std::unique_ptr<SessionsService> m_SessionsService;
- std::unique_ptr<HttpSessionsService> m_HttpSessionsService;
std::unique_ptr<UpstreamCache> m_UpstreamCache;
std::unique_ptr<HttpUpstreamService> m_UpstreamService;
std::unique_ptr<HttpStructuredCacheService> m_StructuredCacheService;
@@ -100,7 +97,6 @@ private:
std::unique_ptr<SessionsServiceClient> m_SessionsClient;
logging::SinkPtr m_SessionLogSink;
- logging::SinkPtr m_InProcSessionLogSink;
asio::steady_timer m_SessionAnnounceTimer{m_IoContext};
void EnqueueSessionAnnounceTimer();
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-27 19:34:25 +0000