aboutsummaryrefslogtreecommitdiff
path: root/src/zenserver/projectstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenserver/projectstore')
-rw-r--r--src/zenserver/projectstore/buildsremoteprojectstore.cpp146
-rw-r--r--src/zenserver/projectstore/buildsremoteprojectstore.h23
-rw-r--r--src/zenserver/projectstore/fileremoteprojectstore.cpp14
-rw-r--r--src/zenserver/projectstore/httpprojectstore.cpp112
-rw-r--r--src/zenserver/projectstore/httpprojectstore.h11
-rw-r--r--src/zenserver/projectstore/jupiterremoteprojectstore.cpp41
-rw-r--r--src/zenserver/projectstore/jupiterremoteprojectstore.h23
-rw-r--r--src/zenserver/projectstore/projectstore.cpp515
-rw-r--r--src/zenserver/projectstore/projectstore.h1
-rw-r--r--src/zenserver/projectstore/remoteprojectstore.cpp359
-rw-r--r--src/zenserver/projectstore/remoteprojectstore.h29
-rw-r--r--src/zenserver/projectstore/zenremoteprojectstore.cpp2
12 files changed, 597 insertions, 679 deletions
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
index 302b81729..ab96ae92d 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp
@@ -3,6 +3,7 @@
#include "buildsremoteprojectstore.h"
#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryvalidation.h>
#include <zencore/compress.h>
#include <zencore/fmtutils.h>
@@ -34,16 +35,10 @@ public:
, m_BuildId(BuildId)
, m_MetaData(MetaData)
, m_TempFilePath(TempFilePath)
+ , m_EnableBlocks(!ForceDisableBlocks)
+ , m_UseTempBlocks(!ForceDisableTempBlocks)
{
m_MetaData.MakeOwned();
- if (ForceDisableBlocks)
- {
- m_EnableBlocks = false;
- }
- if (ForceDisableTempBlocks)
- {
- m_UseTempBlocks = false;
- }
}
virtual RemoteStoreInfo GetInfo() const override
@@ -70,7 +65,7 @@ public:
{
ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
IoBuffer Payload = m_MetaData;
Payload.SetContentType(ZenContentType::kCbObject);
@@ -94,7 +89,7 @@ public:
virtual SaveResult SaveContainer(const IoBuffer& Payload) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
PutBuildPartResult PutResult =
Session.PutBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, OplogContainerPartName, Payload);
AddStats(PutResult);
@@ -114,24 +109,25 @@ public:
return Result;
}
- virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&& Block) override
+ virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload,
+ const IoHash& RawHash,
+ ChunkBlockDescription&& Block) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
JupiterResult PutResult =
- Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, ZenContentType::kCompressedBinary, Payload);
+ Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
AddStats(PutResult);
SaveAttachmentResult Result{ConvertResult(PutResult)};
if (Result.ErrorCode)
{
- Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}/{}/{}. Reason: '{}'",
+ Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}/{}. Reason: '{}'",
m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
- m_OplogBuildPartId,
RawHash,
Result.Reason);
return Result;
@@ -139,57 +135,21 @@ public:
if (Block.BlockHash == RawHash)
{
- ZEN_ASSERT(Block.ChunkLengths.size() == Block.ChunkHashes.size());
- CbObjectWriter Writer;
- Writer.AddHash("rawHash"sv, RawHash);
- Writer.BeginArray("rawHashes"sv);
- {
- for (const IoHash& ChunkHash : Block.ChunkHashes)
- {
- Writer.AddHash(ChunkHash);
- }
- }
- Writer.EndArray();
- Writer.BeginArray("chunkLengths");
- {
- for (uint32_t ChunkSize : Block.ChunkLengths)
- {
- Writer.AddInteger(ChunkSize);
- }
- }
- Writer.EndArray();
- Writer.BeginArray("chunkOffsets");
- {
- ZEN_ASSERT(Block.FirstChunkOffset != (uint32_t)-1);
- uint32_t Offset = Block.FirstChunkOffset;
- for (uint32_t ChunkSize : Block.ChunkLengths)
- {
- Writer.AddInteger(Offset);
- Offset += ChunkSize;
- }
- }
- Writer.EndArray();
+ CbObjectWriter BlockMetaData;
+ BlockMetaData.AddString("createdBy", GetRunningExecutablePath().stem().string());
- Writer.BeginObject("metadata"sv);
- {
- Writer.AddString("createdBy", "zenserver");
- }
- Writer.EndObject();
-
- IoBuffer MetaPayload = Writer.Save().GetBuffer().AsIoBuffer();
+ IoBuffer MetaPayload = BuildChunkBlockDescription(Block, BlockMetaData.Save()).GetBuffer().AsIoBuffer();
MetaPayload.SetContentType(ZenContentType::kCbObject);
- JupiterResult PutMetaResult =
- Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, MetaPayload);
+ JupiterResult PutMetaResult = Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, RawHash, MetaPayload);
AddStats(PutMetaResult);
RemoteProjectStore::Result MetaDataResult = ConvertResult(PutMetaResult);
if (MetaDataResult.ErrorCode)
{
- ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}/{}. Reason: '{}'",
+ ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}. Reason: '{}'",
m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
- m_OplogBuildPartId,
RawHash,
MetaDataResult.Reason);
}
@@ -217,7 +177,7 @@ public:
ZEN_UNUSED(RawHash);
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
FinalizeBuildPartResult FinalizeRefResult =
Session.FinalizeBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash);
AddStats(FinalizeRefResult);
@@ -256,7 +216,7 @@ public:
{
ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
JupiterResult GetBuildResult = Session.GetBuild(m_Namespace, m_Bucket, m_BuildId);
AddStats(GetBuildResult);
LoadContainerResult Result{ConvertResult(GetBuildResult)};
@@ -341,52 +301,48 @@ public:
virtual GetKnownBlocksResult GetKnownBlocks() override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
- JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId);
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
+ JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, (uint64_t)-1);
AddStats(FindResult);
GetKnownBlocksResult Result{ConvertResult(FindResult)};
if (Result.ErrorCode)
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}/{}. Reason: '{}'",
+ Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'",
m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
- m_OplogBuildPartId,
Result.Reason);
return Result;
}
- CbObject BlocksObject = LoadCompactBinaryObject(FindResult.Response);
- if (!BlocksObject)
+ if (ValidateCompactBinary(FindResult.Response.GetView(), CbValidateMode::Default) != CbValidateError::None)
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The block list {}/{}/{}/{} is not formatted as a compact binary object"sv,
+ Result.Reason = fmt::format("The block list {}/{}/{} is not formatted as a compact binary object"sv,
m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
- m_BuildId,
- m_OplogBuildPartId);
+ m_BuildId);
return Result;
}
-
- CbArrayView Blocks = BlocksObject["blocks"].AsArrayView();
- Result.Blocks.reserve(Blocks.Num());
- for (CbFieldView BlockView : Blocks)
+ std::optional<std::vector<ChunkBlockDescription>> Blocks =
+ ParseChunkBlockDescriptionList(LoadCompactBinaryObject(FindResult.Response));
+ if (!Blocks)
{
- CbObjectView BlockObject = BlockView.AsObjectView();
- IoHash BlockHash = BlockObject["rawHash"sv].AsHash();
- if (BlockHash != IoHash::Zero)
- {
- CbArrayView ChunksArray = BlockObject["rawHashes"sv].AsArrayView();
- std::vector<IoHash> ChunkHashes;
- ChunkHashes.reserve(ChunksArray.Num());
- for (CbFieldView ChunkView : ChunksArray)
- {
- ChunkHashes.push_back(ChunkView.AsHash());
- }
- Result.Blocks.emplace_back(Block{.BlockHash = BlockHash, .ChunkHashes = ChunkHashes});
- }
+ Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
+ Result.Reason = fmt::format("The block list {}/{}/{} is not formatted as a list of blocks"sv,
+ m_JupiterClient->ServiceUrl(),
+ m_Namespace,
+ m_Bucket,
+ m_BuildId);
+ return Result;
+ }
+ Result.Blocks.reserve(Blocks.value().size());
+ for (ChunkBlockDescription& BlockDescription : Blocks.value())
+ {
+ Result.Blocks.push_back(ThinChunkBlockDescription{.BlockHash = BlockDescription.BlockHash,
+ .ChunkRawHashes = std::move(BlockDescription.ChunkRawHashes)});
}
return Result;
}
@@ -394,19 +350,18 @@ public:
virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
{
ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero);
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
- JupiterResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, m_TempFilePath);
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
+ JupiterResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, m_TempFilePath);
AddStats(GetResult);
LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)};
if (GetResult.ErrorCode)
{
- Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}&{}/{}/{}. Reason: '{}'",
+ Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}/{}. Reason: '{}'",
m_JupiterClient->ServiceUrl(),
m_Namespace,
m_Bucket,
m_BuildId,
- m_OplogBuildPartId,
RawHash,
Result.Reason);
}
@@ -491,8 +446,9 @@ private:
IoBuffer m_MetaData;
Oid m_OplogBuildPartId = Oid::Zero;
std::filesystem::path m_TempFilePath;
- bool m_EnableBlocks = true;
- bool m_UseTempBlocks = true;
+ const bool m_EnableBlocks = true;
+ const bool m_UseTempBlocks = true;
+ const bool m_AllowRedirect = false;
std::atomic_uint64_t m_SentBytes = {};
std::atomic_uint64_t m_ReceivedBytes = {};
@@ -533,7 +489,15 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file
{
TokenProvider = httpclientauth::CreateFromStaticToken(Options.AccessToken);
}
- else
+ else if (!Options.OidcExePath.empty())
+ {
+ if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe)
+ {
+ TokenProvider = TokenProviderMaybe.value();
+ }
+ }
+
+ if (!TokenProvider)
{
TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager);
}
diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.h b/src/zenserver/projectstore/buildsremoteprojectstore.h
index 8b2c6c8c8..c52b13886 100644
--- a/src/zenserver/projectstore/buildsremoteprojectstore.h
+++ b/src/zenserver/projectstore/buildsremoteprojectstore.h
@@ -10,17 +10,18 @@ class AuthMgr;
struct BuildsRemoteStoreOptions : RemoteStoreOptions
{
- std::string Url;
- std::string Namespace;
- std::string Bucket;
- Oid BuildId;
- std::string OpenIdProvider;
- std::string AccessToken;
- AuthMgr& AuthManager;
- bool ForceDisableBlocks = false;
- bool ForceDisableTempBlocks = false;
- bool AssumeHttp2 = false;
- IoBuffer MetaData;
+ std::string Url;
+ std::string Namespace;
+ std::string Bucket;
+ Oid BuildId;
+ std::string OpenIdProvider;
+ std::string AccessToken;
+ AuthMgr& AuthManager;
+ std::filesystem::path OidcExePath;
+ bool ForceDisableBlocks = false;
+ bool ForceDisableTempBlocks = false;
+ bool AssumeHttp2 = false;
+ IoBuffer MetaData;
};
std::shared_ptr<RemoteProjectStore> CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options,
diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp
index 0fe739a12..375e44e59 100644
--- a/src/zenserver/projectstore/fileremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp
@@ -73,7 +73,7 @@ public:
ContainerObject.IterateAttachments([&](CbFieldView FieldView) {
IoHash AttachmentHash = FieldView.AsBinaryAttachment();
std::filesystem::path AttachmentPath = GetAttachmentPath(AttachmentHash);
- if (!std::filesystem::exists(AttachmentPath))
+ if (!IsFile(AttachmentPath))
{
Result.Needs.insert(AttachmentHash);
}
@@ -106,12 +106,12 @@ public:
return Result;
}
- virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&&) override
+ virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&&) override
{
Stopwatch Timer;
SaveAttachmentResult Result;
std::filesystem::path ChunkPath = GetAttachmentPath(RawHash);
- if (!std::filesystem::exists(ChunkPath))
+ if (!IsFile(ChunkPath))
{
try
{
@@ -182,7 +182,7 @@ public:
for (const IoHash& RawHash : BlockHashes)
{
std::filesystem::path ChunkPath = GetAttachmentPath(RawHash);
- if (std::filesystem::is_regular_file(ChunkPath))
+ if (IsFile(ChunkPath))
{
ExistingBlockHashes.push_back(RawHash);
}
@@ -192,7 +192,7 @@ public:
return GetKnownBlocksResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent),
.ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}};
}
- std::vector<RemoteProjectStore::Block> KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes);
+ std::vector<ThinChunkBlockDescription> KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes);
GetKnownBlocksResult Result{{.ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}};
Result.Blocks = std::move(KnownBlocks);
return Result;
@@ -203,7 +203,7 @@ public:
Stopwatch Timer;
LoadAttachmentResult Result;
std::filesystem::path ChunkPath = GetAttachmentPath(RawHash);
- if (!std::filesystem::is_regular_file(ChunkPath))
+ if (!IsFile(ChunkPath))
{
Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound);
Result.Reason = fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string());
@@ -246,7 +246,7 @@ private:
LoadContainerResult Result;
std::filesystem::path SourcePath = m_OutputPath;
SourcePath.append(Name);
- if (!std::filesystem::is_regular_file(SourcePath))
+ if (!IsFile(SourcePath))
{
Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound);
Result.Reason = fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", SourcePath.string());
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 0b8e5f13b..317a419eb 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -8,6 +8,7 @@
#include <zencore/compactbinarybuilder.h>
#include <zencore/compactbinarypackage.h>
#include <zencore/compactbinaryutil.h>
+#include <zencore/compactbinaryvalidation.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
@@ -234,10 +235,15 @@ namespace {
//////////////////////////////////////////////////////////////////////////
-HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr)
+HttpProjectService::HttpProjectService(CidStore& Store,
+ ProjectStore* Projects,
+ HttpStatusService& StatusService,
+ HttpStatsService& StatsService,
+ AuthMgr& AuthMgr)
: m_Log(logging::Get("project"))
, m_CidStore(Store)
, m_ProjectStore(Projects)
+, m_StatusService(StatusService)
, m_StatsService(StatsService)
, m_AuthMgr(AuthMgr)
{
@@ -245,8 +251,6 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
using namespace std::literals;
- m_StatsService.RegisterHandler("prj", *this);
-
m_Router.AddPattern("project", "([[:alnum:]_.]+)");
m_Router.AddPattern("log", "([[:alnum:]_.]+)");
m_Router.AddPattern("op", "([[:digit:]]+?)");
@@ -365,11 +369,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
"details\\$/{project}/{log}/{chunk}",
[this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); },
HttpVerb::kGet);
+
+ m_StatusService.RegisterHandler("prj", *this);
+ m_StatsService.RegisterHandler("prj", *this);
}
HttpProjectService::~HttpProjectService()
{
m_StatsService.UnregisterHandler("prj", *this);
+ m_StatusService.UnregisterHandler("prj", *this);
}
const char*
@@ -465,6 +473,15 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq)
}
void
+HttpProjectService::HandleStatusRequest(HttpServerRequest& Request)
+{
+ ZEN_TRACE_CPU("HttpProjectService::Status");
+ CbObjectWriter Cbo;
+ Cbo << "ok" << true;
+ Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
HttpProjectService::HandleProjectListRequest(HttpRouterRequest& Req)
{
ZEN_TRACE_CPU("ProjectService::ProjectList");
@@ -885,10 +902,63 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req)
case HttpVerb::kGet:
{
IoBuffer Value;
- std::pair<HttpResponseCode, std::string> Result =
- m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value, nullptr);
+ std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, Value, nullptr);
if (Result.first == HttpResponseCode::OK)
{
+ if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary ||
+ AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML ||
+ AcceptType == ZenContentType::kCbObject)
+ {
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value));
+ IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer();
+
+ if (DecompressedBuffer)
+ {
+ if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML ||
+ AcceptType == ZenContentType::kCbObject)
+ {
+ CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default);
+ if (!!CbErr)
+ {
+ m_ProjectStats.BadRequestCount++;
+ ZEN_DEBUG(
+ "chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not convert to object`",
+ ProjectId,
+ OplogId,
+ Cid,
+ ToString(AcceptType));
+ return HttpReq.WriteResponse(
+ HttpResponseCode::NotAcceptable,
+ HttpContentType::kText,
+ fmt::format("Content format not supported, requested {} format, but could not convert to object",
+ ToString(AcceptType)));
+ }
+
+ m_ProjectStats.ChunkHitCount++;
+ CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer);
+ return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject);
+ }
+ else
+ {
+ Value = DecompressedBuffer;
+ Value.SetContentType(ZenContentType::kBinary);
+ }
+ }
+ else
+ {
+ m_ProjectStats.BadRequestCount++;
+ ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not decompress stored data`",
+ ProjectId,
+ OplogId,
+ Cid,
+ ToString(AcceptType));
+ return HttpReq.WriteResponse(
+ HttpResponseCode::NotAcceptable,
+ HttpContentType::kText,
+ fmt::format("Content format not supported, requested {} format, but could not decompress stored data",
+ ToString(AcceptType)));
+ }
+ }
m_ProjectStats.ChunkHitCount++;
return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value);
}
@@ -983,15 +1053,19 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req)
IoBuffer Payload = HttpReq.ReadPayload();
CbObject RequestObject = LoadCompactBinaryObject(Payload);
- std::vector<IoHash> ChunkList;
- CbArrayView HaveList = RequestObject["have"sv].AsArrayView();
- ChunkList.reserve(HaveList.Num());
- for (auto& Entry : HaveList)
+ std::vector<IoHash> NeedList;
+
{
- ChunkList.push_back(Entry.AsHash());
- }
+ eastl::fixed_vector<IoHash, 16> ChunkList;
+ CbArrayView HaveList = RequestObject["have"sv].AsArrayView();
+ ChunkList.reserve(HaveList.Num());
+ for (auto& Entry : HaveList)
+ {
+ ChunkList.push_back(Entry.AsHash());
+ }
- std::vector<IoHash> NeedList = FoundLog->CheckPendingChunkReferences(ChunkList, std::chrono::minutes(2));
+ NeedList = FoundLog->CheckPendingChunkReferences(std::span(begin(ChunkList), end(ChunkList)), std::chrono::minutes(2));
+ }
CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1);
Cbo.BeginArray("need");
@@ -1151,7 +1225,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified");
}
- std::vector<IoHash> ReferencedChunks;
+ eastl::fixed_vector<IoHash, 16> ReferencedChunks;
Core.IterateAttachments([&ReferencedChunks](CbFieldView View) { ReferencedChunks.push_back(View.AsAttachment()); });
// Write core to oplog
@@ -1169,7 +1243,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
// Once we stored the op, we no longer need to retain any chunks this op references
if (!ReferencedChunks.empty())
{
- FoundLog->RemovePendingChunkReferences(ReferencedChunks);
+ FoundLog->RemovePendingChunkReferences(std::span(begin(ReferencedChunks), end(ReferencedChunks)));
}
m_ProjectStats.OpWriteCount++;
@@ -1301,9 +1375,9 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
HttpServerRequest& HttpReq = Req.ServerRequest();
- const std::string& ProjectId = Req.GetCapture(1);
- const std::string& OplogId = Req.GetCapture(2);
- const std::string& OpIdString = Req.GetCapture(3);
+ const std::string_view ProjectId = Req.GetCapture(1);
+ const std::string_view OplogId = Req.GetCapture(2);
+ const std::string_view OpIdString = Req.GetCapture(3);
Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
if (!Project)
@@ -1690,8 +1764,8 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req)
using namespace std::literals;
- HttpServerRequest& HttpReq = Req.ServerRequest();
- const std::string ProjectId = Req.GetCapture(1);
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+ const std::string_view ProjectId = Req.GetCapture(1);
switch (HttpReq.RequestVerb())
{
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
index 8e74c57a5..295defa5c 100644
--- a/src/zenserver/projectstore/httpprojectstore.h
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -5,6 +5,7 @@
#include <zencore/stats.h>
#include <zenhttp/httpserver.h>
#include <zenhttp/httpstats.h>
+#include <zenhttp/httpstatus.h>
#include <zenstore/cidstore.h>
namespace zen {
@@ -31,16 +32,21 @@ class ProjectStore;
// refs:
//
-class HttpProjectService : public HttpService, public IHttpStatsProvider
+class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider
{
public:
- HttpProjectService(CidStore& Store, ProjectStore* InProjectStore, HttpStatsService& StatsService, AuthMgr& AuthMgr);
+ HttpProjectService(CidStore& Store,
+ ProjectStore* InProjectStore,
+ HttpStatusService& StatusService,
+ HttpStatsService& StatsService,
+ AuthMgr& AuthMgr);
~HttpProjectService();
virtual const char* BaseUri() const override;
virtual void HandleRequest(HttpServerRequest& Request) override;
virtual void HandleStatsRequest(HttpServerRequest& Request) override;
+ virtual void HandleStatusRequest(HttpServerRequest& Request) override;
private:
struct ProjectStats
@@ -89,6 +95,7 @@ private:
CidStore& m_CidStore;
HttpRequestRouter m_Router;
Ref<ProjectStore> m_ProjectStore;
+ HttpStatusService& m_StatusService;
HttpStatsService& m_StatsService;
AuthMgr& m_AuthMgr;
ProjectStats m_ProjectStats;
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index e906127ff..3728babb5 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -31,15 +31,9 @@ public:
, m_Key(Key)
, m_OptionalBaseKey(OptionalBaseKey)
, m_TempFilePath(TempFilePath)
+ , m_EnableBlocks(!ForceDisableBlocks)
+ , m_UseTempBlocks(!ForceDisableTempBlocks)
{
- if (ForceDisableBlocks)
- {
- m_EnableBlocks = false;
- }
- if (ForceDisableTempBlocks)
- {
- m_UseTempBlocks = false;
- }
}
virtual RemoteStoreInfo GetInfo() const override
@@ -75,7 +69,7 @@ public:
virtual SaveResult SaveContainer(const IoBuffer& Payload) override
{
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
PutRefResult PutResult = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject);
AddStats(PutResult);
@@ -92,9 +86,9 @@ public:
return Result;
}
- virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&&) override
+ virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&&) override
{
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
JupiterResult PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload);
AddStats(PutResult);
@@ -127,7 +121,7 @@ public:
virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) override
{
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
FinalizeRefResult FinalizeRefResult = Session.FinalizeRef(m_Namespace, m_Bucket, m_Key, RawHash);
AddStats(FinalizeRefResult);
@@ -164,7 +158,7 @@ public:
{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent), .ElapsedSeconds = LoadResult.ElapsedSeconds}};
}
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
JupiterExistsResult ExistsResult =
Session.CompressedBlobExists(m_Namespace, std::set<IoHash>(BlockHashes.begin(), BlockHashes.end()));
AddStats(ExistsResult);
@@ -193,7 +187,7 @@ public:
return GetKnownBlocksResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent),
.ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds}};
}
- std::vector<RemoteProjectStore::Block> KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes);
+ std::vector<ThinChunkBlockDescription> KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes);
GetKnownBlocksResult Result{
{.ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000.0}};
@@ -203,7 +197,7 @@ public:
virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override
{
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
JupiterResult GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath);
AddStats(GetResult);
@@ -239,7 +233,7 @@ public:
private:
LoadContainerResult LoadContainer(const IoHash& Key)
{
- JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client());
+ JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect);
JupiterResult GetResult = Session.GetRef(m_Namespace, m_Bucket, Key, ZenContentType::kCbObject);
AddStats(GetResult);
if (GetResult.ErrorCode || !GetResult.Success)
@@ -329,8 +323,9 @@ private:
const IoHash m_Key;
const IoHash m_OptionalBaseKey;
std::filesystem::path m_TempFilePath;
- bool m_EnableBlocks = true;
- bool m_UseTempBlocks = true;
+ const bool m_EnableBlocks = true;
+ const bool m_UseTempBlocks = true;
+ const bool m_AllowRedirect = false;
std::atomic_uint64_t m_SentBytes = {};
std::atomic_uint64_t m_ReceivedBytes = {};
@@ -371,7 +366,15 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi
{
TokenProvider = httpclientauth::CreateFromStaticToken(Options.AccessToken);
}
- else
+ else if (!Options.OidcExePath.empty())
+ {
+ if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe)
+ {
+ TokenProvider = TokenProviderMaybe.value();
+ }
+ }
+
+ if (!TokenProvider)
{
TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager);
}
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.h b/src/zenserver/projectstore/jupiterremoteprojectstore.h
index 27f3d9b73..8bf79d563 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.h
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.h
@@ -10,17 +10,18 @@ class AuthMgr;
struct JupiterRemoteStoreOptions : RemoteStoreOptions
{
- std::string Url;
- std::string Namespace;
- std::string Bucket;
- IoHash Key;
- IoHash OptionalBaseKey;
- std::string OpenIdProvider;
- std::string AccessToken;
- AuthMgr& AuthManager;
- bool ForceDisableBlocks = false;
- bool ForceDisableTempBlocks = false;
- bool AssumeHttp2 = false;
+ std::string Url;
+ std::string Namespace;
+ std::string Bucket;
+ IoHash Key;
+ IoHash OptionalBaseKey;
+ std::string OpenIdProvider;
+ std::string AccessToken;
+ AuthMgr& AuthManager;
+ std::filesystem::path OidcExePath;
+ bool ForceDisableBlocks = false;
+ bool ForceDisableTempBlocks = false;
+ bool AssumeHttp2 = false;
};
std::shared_ptr<RemoteProjectStore> CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options,
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 46a236af9..53e687983 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -22,6 +22,7 @@
#include <zenstore/scrubcontext.h>
#include <zenutil/cache/rpcrecording.h>
#include <zenutil/openprocesscache.h>
+#include <zenutil/parallelwork.h>
#include <zenutil/referencemetadata.h>
#include <zenutil/workerpools.h>
@@ -58,7 +59,7 @@ namespace {
std::filesystem::path DroppedBucketPath;
do
{
- if (!std::filesystem::exists(Dir))
+ if (!IsDir(Dir))
{
return true;
}
@@ -68,7 +69,7 @@ namespace {
std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), MovedId);
DroppedBucketPath = Dir.parent_path() / DroppedName;
- if (std::filesystem::exists(DroppedBucketPath))
+ if (IsDir(DroppedBucketPath))
{
if (!DeleteDirectories(DroppedBucketPath))
{
@@ -77,7 +78,7 @@ namespace {
Dir);
continue;
}
- if (std::filesystem::exists(DroppedBucketPath))
+ if (IsDir(DroppedBucketPath))
{
ZEN_INFO("Drop directory '{}' for '{}' still exists after remove, attempting different name.", DroppedBucketPath, Dir);
continue;
@@ -88,13 +89,13 @@ namespace {
do
{
std::error_code Ec;
- std::filesystem::rename(Dir, DroppedBucketPath, Ec);
+ RenameDirectory(Dir, DroppedBucketPath, Ec);
if (!Ec)
{
OutDeleteDir = DroppedBucketPath;
return true;
}
- if (std::filesystem::exists(DroppedBucketPath))
+ if (IsDir(DroppedBucketPath))
{
ZEN_INFO("Can't rename '{}' to still existing drop directory '{}'. Reason: '{}'. Attempting different name.",
Dir,
@@ -210,6 +211,16 @@ namespace {
AccessToken = GetEnvVariable(AccessTokenEnvVariable);
}
}
+ std::filesystem::path OidcExePath;
+ if (std::string_view OidcExePathString = Cloud["oidc-exe-path"].AsString(); !OidcExePathString.empty())
+ {
+ std::filesystem::path OidcExePathMaybe(OidcExePathString);
+ if (!IsFile(OidcExePathMaybe))
+ {
+ ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString);
+ OidcExePath = std::move(OidcExePathMaybe);
+ }
+ }
std::string_view KeyParam = Cloud["key"sv].AsString();
if (KeyParam.empty())
{
@@ -252,6 +263,7 @@ namespace {
std::string(OpenIdProvider),
AccessToken,
AuthManager,
+ OidcExePath,
ForceDisableBlocks,
ForceDisableTempBlocks,
AssumeHttp2};
@@ -307,6 +319,16 @@ namespace {
AccessToken = GetEnvVariable(AccessTokenEnvVariable);
}
}
+ std::filesystem::path OidcExePath;
+ if (std::string_view OidcExePathString = Builds["oidc-exe-path"].AsString(); !OidcExePathString.empty())
+ {
+ std::filesystem::path OidcExePathMaybe(OidcExePathString);
+ if (!IsFile(OidcExePathMaybe))
+ {
+ ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString);
+ OidcExePath = std::move(OidcExePathMaybe);
+ }
+ }
std::string_view BuildIdParam = Builds["buildsid"sv].AsString();
if (BuildIdParam.empty())
{
@@ -337,6 +359,7 @@ namespace {
std::string(OpenIdProvider),
AccessToken,
AuthManager,
+ OidcExePath,
ForceDisableBlocks,
ForceDisableTempBlocks,
AssumeHttp2,
@@ -423,9 +446,13 @@ ComputeOpKey(const CbObjectView& Op)
{
using namespace std::literals;
- BinaryWriter KeyStream;
+ eastl::fixed_vector<uint8_t, 256> KeyData;
- Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyStream.Write(Data, Size); });
+ Op["key"sv].WriteToStream([&](const void* Data, size_t Size) {
+ auto Begin = reinterpret_cast<const uint8_t*>(Data);
+ auto End = Begin + Size;
+ KeyData.insert(KeyData.end(), Begin, End);
+ });
XXH3_128 KeyHash128;
@@ -434,15 +461,15 @@ ComputeOpKey(const CbObjectView& Op)
// path but longer paths are evaluated properly. In the future all key lengths
// should be evaluated using the proper path, this is a temporary workaround to
// maintain compatibility with existing disk state.
- if (KeyStream.GetSize() < 240)
+ if (KeyData.size() < 240)
{
XXH3_128Stream_deprecated KeyHasher;
- KeyHasher.Append(KeyStream.Data(), KeyStream.Size());
+ KeyHasher.Append(KeyData.data(), KeyData.size());
KeyHash128 = KeyHasher.GetHash();
}
else
{
- KeyHash128 = XXH3_128::HashMemory(KeyStream.GetView());
+ KeyHash128 = XXH3_128::HashMemory(KeyData.data(), KeyData.size());
}
Oid KeyHash;
@@ -482,7 +509,7 @@ struct ProjectStore::OplogStorage : public RefCounted
[[nodiscard]] bool Exists() const { return Exists(m_OplogStoragePath); }
[[nodiscard]] static bool Exists(const std::filesystem::path& BasePath)
{
- return std::filesystem::exists(GetLogPath(BasePath)) && std::filesystem::exists(GetBlobsPath(BasePath));
+ return IsFile(GetLogPath(BasePath)) && IsFile(GetBlobsPath(BasePath));
}
[[nodiscard]] bool IsValid() const { return IsValid(m_OplogStoragePath); }
[[nodiscard]] static bool IsValid(const std::filesystem::path& BasePath)
@@ -492,13 +519,13 @@ struct ProjectStore::OplogStorage : public RefCounted
void WipeState() const
{
std::error_code Ec;
- std::filesystem::remove(GetLogPath(), Ec);
- std::filesystem::remove(GetBlobsPath(), Ec);
+ RemoveFile(GetLogPath(), Ec);
+ RemoveFile(GetBlobsPath(), Ec);
}
static bool Delete(const std::filesystem::path& BasePath) { return DeleteDirectories(BasePath); }
- uint64_t OpBlobsSize() const { return std::filesystem::file_size(GetBlobsPath()); }
+ uint64_t OpBlobsSize() const { return FileSizeFromPath(GetBlobsPath()); }
uint64_t OpsSize() const { return OpsSize(m_OplogStoragePath); }
static uint64_t OpsSize(const std::filesystem::path& BasePath)
@@ -506,7 +533,7 @@ struct ProjectStore::OplogStorage : public RefCounted
if (Exists(BasePath))
{
std::error_code DummyEc;
- return std::filesystem::file_size(GetLogPath(BasePath)) + std::filesystem::file_size(GetBlobsPath(BasePath));
+ return FileSizeFromPath(GetLogPath(BasePath)) + FileSizeFromPath(GetBlobsPath(BasePath));
}
return 0;
}
@@ -685,7 +712,7 @@ struct ProjectStore::OplogStorage : public RefCounted
m_OpBlobs.Close();
Oplog.Close();
- std::filesystem::rename(OplogPath, GetLogPath(), Ec);
+ RenameFile(OplogPath, GetLogPath(), Ec);
if (Ec)
{
throw std::system_error(
@@ -698,9 +725,9 @@ struct ProjectStore::OplogStorage : public RefCounted
if (Ec)
{
// We failed late - clean everything up as best we can
- std::filesystem::remove(OpBlobs.GetPath(), Ec);
- std::filesystem::remove(GetLogPath(), Ec);
- std::filesystem::remove(GetBlobsPath(), Ec);
+ RemoveFile(OpBlobs.GetPath(), Ec);
+ RemoveFile(GetLogPath(), Ec);
+ RemoveFile(GetBlobsPath(), Ec);
throw std::system_error(Ec,
fmt::format("Oplog::Compact failed to rename temporary oplog file from '{}' to '{}'",
OpBlobs.GetPath(),
@@ -735,7 +762,7 @@ struct ProjectStore::OplogStorage : public RefCounted
}
catch (const std::exception& /*Ex*/)
{
- std::filesystem::remove(OpBlobs.GetPath(), Ec);
+ RemoveFile(OpBlobs.GetPath(), Ec);
throw;
}
}
@@ -983,8 +1010,8 @@ struct ProjectStore::OplogStorage : public RefCounted
.OpCoreHash = OpData.OpCoreHash,
.OpKeyHash = OpData.KeyHash};
- m_Oplog.Append(Entry);
m_OpBlobs.Write(OpData.Buffer.GetData(), WriteSize, WriteOffset);
+ m_Oplog.Append(Entry);
return Entry;
}
@@ -1104,7 +1131,7 @@ ProjectStore::Oplog::Oplog(std::string_view Id,
ZEN_WARN("Invalid oplog found at '{}'. Wiping state for oplog.", m_BasePath);
m_Storage->WipeState();
std::error_code DummyEc;
- std::filesystem::remove(m_MetaPath, DummyEc);
+ RemoveFile(m_MetaPath, DummyEc);
}
}
m_Storage->Open(/* IsCreate */ !StoreExists);
@@ -1112,7 +1139,7 @@ ProjectStore::Oplog::Oplog(std::string_view Id,
m_MetaPath = m_BasePath / "ops.meta"sv;
m_MetaValid = !IsFileOlderThan(m_MetaPath, m_Storage->GetBlobsPath());
- CleanDirectory(m_TempPath);
+ CleanDirectory(m_TempPath, /*ForceRemoveReadOnlyFiles*/ false);
}
ProjectStore::Oplog::~Oplog()
@@ -1138,7 +1165,7 @@ ProjectStore::Oplog::Flush()
if (!m_MetaValid)
{
std::error_code DummyEc;
- std::filesystem::remove(m_MetaPath, DummyEc);
+ RemoveFile(m_MetaPath, DummyEc);
}
uint64_t LogCount = m_Storage->LogCount();
@@ -1234,19 +1261,19 @@ ProjectStore::Oplog::TotalSize(const std::filesystem::path& BasePath)
uint64_t Size = OplogStorage::OpsSize(BasePath);
std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv;
- if (std::filesystem::exists(StateFilePath))
+ if (IsFile(StateFilePath))
{
- Size += std::filesystem::file_size(StateFilePath);
+ Size += FileSizeFromPath(StateFilePath);
}
std::filesystem::path MetaFilePath = BasePath / "ops.meta"sv;
- if (std::filesystem::exists(MetaFilePath))
+ if (IsFile(MetaFilePath))
{
- Size += std::filesystem::file_size(MetaFilePath);
+ Size += FileSizeFromPath(MetaFilePath);
}
std::filesystem::path IndexFilePath = BasePath / "ops.zidx"sv;
- if (std::filesystem::exists(IndexFilePath))
+ if (IsFile(IndexFilePath))
{
- Size += std::filesystem::file_size(IndexFilePath);
+ Size += FileSizeFromPath(IndexFilePath);
}
return Size;
@@ -1299,7 +1326,7 @@ ProjectStore::Oplog::ExistsAt(const std::filesystem::path& BasePath)
using namespace std::literals;
std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv;
- return std::filesystem::is_regular_file(StateFilePath);
+ return IsFile(StateFilePath);
}
bool
@@ -1333,7 +1360,7 @@ ProjectStore::Oplog::Read()
if (!m_MetaValid)
{
std::error_code DummyEc;
- std::filesystem::remove(m_MetaPath, DummyEc);
+ RemoveFile(m_MetaPath, DummyEc);
}
ReadIndexSnapshot();
@@ -1434,7 +1461,7 @@ ProjectStore::Oplog::Reset()
m_Storage = new OplogStorage(this, m_BasePath);
m_Storage->Open(true);
m_MetaValid = false;
- CleanDirectory(m_TempPath);
+ CleanDirectory(m_TempPath, /*ForceRemoveReadOnlyFiles*/ false);
Write();
}
// Erase content on disk
@@ -1453,7 +1480,7 @@ ProjectStore::Oplog::ReadStateFile(const std::filesystem::path& BasePath, std::f
using namespace std::literals;
std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv;
- if (std::filesystem::is_regular_file(StateFilePath))
+ if (IsFile(StateFilePath))
{
// ZEN_INFO("oplog '{}/{}': config read from '{}'", m_OuterProject->Identifier, m_OplogId, StateFilePath);
@@ -1484,11 +1511,17 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo
ValidationResult Result;
+ const size_t OpCount = OplogCount();
+
std::vector<Oid> KeyHashes;
std::vector<std::string> Keys;
std::vector<std::vector<IoHash>> Attachments;
std::vector<OplogEntryMapping> Mappings;
+ KeyHashes.reserve(OpCount);
+ Keys.reserve(OpCount);
+ Mappings.reserve(OpCount);
+
IterateOplogWithKey([&](uint32_t LSN, const Oid& Key, CbObjectView OpView) {
Result.LSNLow = Min(Result.LSNLow, LSN);
Result.LSNHigh = Max(Result.LSNHigh, LSN);
@@ -1513,77 +1546,90 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo
bool HasMissingEntries = false;
for (const ChunkMapping& Chunk : Mapping.Chunks)
{
- if (IoBuffer Payload = m_CidStore.FindChunkByCid(Chunk.Hash); !Payload)
+ if (!m_CidStore.ContainsChunk(Chunk.Hash))
{
ResultLock.WithExclusiveLock([&]() { Result.MissingChunks.push_back({KeyHash, Chunk}); });
HasMissingEntries = true;
}
}
+
for (const ChunkMapping& Meta : Mapping.Meta)
{
- if (IoBuffer Payload = m_CidStore.FindChunkByCid(Meta.Hash); !Payload)
+ if (!m_CidStore.ContainsChunk(Meta.Hash))
{
ResultLock.WithExclusiveLock([&]() { Result.MissingMetas.push_back({KeyHash, Meta}); });
HasMissingEntries = true;
}
}
+
for (const FileMapping& File : Mapping.Files)
{
if (File.Hash == IoHash::Zero)
{
std::filesystem::path FilePath = m_OuterProject->RootDir / File.ServerPath;
- if (!std::filesystem::is_regular_file(FilePath))
+ if (!IsFile(FilePath))
{
ResultLock.WithExclusiveLock([&]() { Result.MissingFiles.push_back({KeyHash, File}); });
HasMissingEntries = true;
}
}
- else
+ else if (!m_CidStore.ContainsChunk(File.Hash))
{
- if (IoBuffer Payload = m_CidStore.FindChunkByCid(File.Hash); !Payload)
- {
- ResultLock.WithExclusiveLock([&]() { Result.MissingFiles.push_back({KeyHash, File}); });
- HasMissingEntries = true;
- }
+ ResultLock.WithExclusiveLock([&]() { Result.MissingFiles.push_back({KeyHash, File}); });
+ HasMissingEntries = true;
}
}
+
const std::vector<IoHash>& OpAttachments = Attachments[OpIndex];
for (const IoHash& Attachment : OpAttachments)
{
- if (IoBuffer Payload = m_CidStore.FindChunkByCid(Attachment); !Payload)
+ if (!m_CidStore.ContainsChunk(Attachment))
{
ResultLock.WithExclusiveLock([&]() { Result.MissingAttachments.push_back({KeyHash, Attachment}); });
HasMissingEntries = true;
}
}
+
if (HasMissingEntries)
{
ResultLock.WithExclusiveLock([&]() { Result.OpKeys.push_back({KeyHash, Key}); });
}
};
- Latch WorkLatch(1);
-
- for (uint32_t OpIndex = 0; !IsCancelledFlag && OpIndex < Result.OpCount; OpIndex++)
+ std::atomic<bool> AbortFlag;
+ std::atomic<bool> PauseFlag;
+ ParallelWork Work(AbortFlag, PauseFlag);
+ try
{
- if (OptionalWorkerPool)
- {
- WorkLatch.AddCount(1);
- OptionalWorkerPool->ScheduleWork([&, Index = OpIndex]() {
- ZEN_MEMSCOPE(GetProjectstoreTag());
-
- auto _ = MakeGuard([&WorkLatch] { WorkLatch.CountDown(); });
- ValidateOne(Index);
- });
- }
- else
+ for (uint32_t OpIndex = 0; !IsCancelledFlag && OpIndex < Result.OpCount; OpIndex++)
{
- ValidateOne(OpIndex);
+ if (AbortFlag)
+ {
+ break;
+ }
+ if (OptionalWorkerPool)
+ {
+ Work.ScheduleWork(*OptionalWorkerPool, [&ValidateOne, Index = OpIndex](std::atomic<bool>& AbortFlag) {
+ ZEN_MEMSCOPE(GetProjectstoreTag());
+ if (AbortFlag)
+ {
+ return;
+ }
+ ValidateOne(Index);
+ });
+ }
+ else
+ {
+ ValidateOne(OpIndex);
+ }
}
}
-
- WorkLatch.CountDown();
- WorkLatch.Wait();
+ catch (const std::exception& Ex)
+ {
+ AbortFlag.store(true);
+ ZEN_WARN("Failed validating oplogs in {}. Reason: '{}'", m_BasePath, Ex.what());
+ }
+ Work.Wait();
{
// Check if we were deleted while we were checking the references without a lock...
@@ -1617,31 +1663,9 @@ ProjectStore::Oplog::WriteIndexSnapshot()
namespace fs = std::filesystem;
- fs::path IndexPath = m_BasePath / "ops.zidx";
- fs::path TempIndexPath = m_BasePath / "ops.zidx.tmp";
-
- // Move index away, we keep it if something goes wrong
- if (fs::is_regular_file(TempIndexPath))
- {
- std::error_code Ec;
- if (!fs::remove(TempIndexPath, Ec) || Ec)
- {
- ZEN_WARN("oplog '{}/{}': snapshot failed to clean up temp snapshot at {}, reason: '{}'",
- GetOuterProject()->Identifier,
- m_OplogId,
- TempIndexPath,
- Ec.message());
- return;
- }
- }
-
+ const fs::path IndexPath = m_BasePath / "ops.zidx";
try
{
- if (fs::is_regular_file(IndexPath))
- {
- fs::rename(IndexPath, TempIndexPath);
- }
-
// Write the current state of the location map to a new index state
std::vector<uint32_t> LSNEntries;
std::vector<Oid> Keys;
@@ -1722,36 +1746,28 @@ ProjectStore::Oplog::WriteIndexSnapshot()
uint64_t Offset = 0;
IndexFile.Write(&Header, sizeof(OplogIndexHeader), Offset);
- Offset += sizeof(OplogIndexHeader);
- Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+ Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
IndexFile.Write(LSNEntries.data(), LSNEntries.size() * sizeof(uint32_t), Offset);
- Offset += LSNEntries.size() * sizeof(uint32_t);
- Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+ Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
IndexFile.Write(Keys.data(), Keys.size() * sizeof(Oid), Offset);
- Offset += Keys.size() * sizeof(Oid);
- Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+ Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
IndexFile.Write(AddressMapEntries.data(), AddressMapEntries.size() * sizeof(OplogEntryAddress), Offset);
- Offset += AddressMapEntries.size() * sizeof(OplogEntryAddress);
- Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+ Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
IndexFile.Write(LatestOpMapEntries.data(), LatestOpMapEntries.size() * sizeof(uint32_t), Offset);
- Offset += LatestOpMapEntries.size() * sizeof(uint32_t);
- Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+ Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
IndexFile.Write(ChunkMapEntries.data(), ChunkMapEntries.size() * sizeof(IoHash), Offset);
- Offset += ChunkMapEntries.size() * sizeof(IoHash);
- Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+ Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
IndexFile.Write(MetaMapEntries.data(), MetaMapEntries.size() * sizeof(IoHash), Offset);
- Offset += MetaMapEntries.size() * sizeof(IoHash);
- Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+ Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
IndexFile.Write(FilePathLengths.data(), FilePathLengths.size() * sizeof(uint32_t), Offset);
- Offset += FilePathLengths.size() * sizeof(uint32_t);
- Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment);
+ Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment);
for (const auto& FilePath : FilePaths)
{
@@ -1763,7 +1779,11 @@ ProjectStore::Oplog::WriteIndexSnapshot()
ObjectIndexFile.MoveTemporaryIntoPlace(IndexPath, Ec);
if (Ec)
{
- throw std::system_error(Ec, fmt::format("Failed to move temp file '{}' to '{}'", ObjectIndexFile.GetPath(), IndexPath));
+ throw std::system_error(Ec,
+ fmt::format("Snapshot failed to rename new snapshot '{}' to '{}', reason: '{}'",
+ ObjectIndexFile.GetPath(),
+ IndexPath,
+ Ec.message()));
}
EntryCount = LSNEntries.size();
m_LogFlushPosition = IndexLogPosition;
@@ -1771,35 +1791,6 @@ ProjectStore::Oplog::WriteIndexSnapshot()
catch (const std::exception& Err)
{
ZEN_WARN("oplog '{}/{}': snapshot FAILED, reason: '{}'", m_OuterProject->Identifier, m_OplogId, Err.what());
-
- // Restore any previous snapshot
-
- if (fs::is_regular_file(TempIndexPath))
- {
- std::error_code Ec;
- fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless
- fs::rename(TempIndexPath, IndexPath, Ec);
- if (Ec)
- {
- ZEN_WARN("oplog '{}/{}': snapshot failed to restore old snapshot from {}, reason: '{}'",
- m_OuterProject->Identifier,
- m_OplogId,
- TempIndexPath,
- Ec.message());
- }
- }
- }
- if (fs::is_regular_file(TempIndexPath))
- {
- std::error_code Ec;
- if (!fs::remove(TempIndexPath, Ec) || Ec)
- {
- ZEN_WARN("oplog '{}/{}': snapshot failed to remove temporary file {}, reason: '{}'",
- m_OuterProject->Identifier,
- m_OplogId,
- TempIndexPath,
- Ec.message());
- }
}
}
@@ -1809,8 +1800,8 @@ ProjectStore::Oplog::ReadIndexSnapshot()
ZEN_MEMSCOPE(GetProjectstoreTag());
ZEN_TRACE_CPU("Oplog::ReadIndexSnapshot");
- std::filesystem::path IndexPath = m_BasePath / "ops.zidx";
- if (std::filesystem::is_regular_file(IndexPath))
+ const std::filesystem::path IndexPath = m_BasePath / "ops.zidx";
+ if (IsFile(IndexPath))
{
uint64_t EntryCount = 0;
Stopwatch Timer;
@@ -2135,66 +2126,81 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid> ChunkIds,
}
if (OptionalWorkerPool)
{
- std::atomic_bool Result = true;
- Latch WorkLatch(1);
-
- for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++)
+ std::atomic<bool> AbortFlag;
+ std::atomic<bool> PauseFlag;
+ ParallelWork Work(AbortFlag, PauseFlag);
+ try
{
- if (Result.load() == false)
+ for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++)
{
- break;
- }
- WorkLatch.AddCount(1);
- OptionalWorkerPool->ScheduleWork(
- [this, &WorkLatch, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result]() {
- auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
- if (Result.load() == false)
- {
- return;
- }
- size_t FileChunkIndex = FileChunkIndexes[ChunkIndex];
- const std::filesystem::path& FilePath = FileChunkPaths[ChunkIndex];
- try
- {
- IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath);
- if (!Payload)
+ if (AbortFlag)
+ {
+ break;
+ }
+ Work.ScheduleWork(
+ *OptionalWorkerPool,
+ [this, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback](
+ std::atomic<bool>& AbortFlag) {
+ if (AbortFlag)
{
- ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[ChunkIndex], FilePath);
+ return;
}
+ size_t FileChunkIndex = FileChunkIndexes[ChunkIndex];
+ const std::filesystem::path& FilePath = FileChunkPaths[ChunkIndex];
+ try
+ {
+ IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath);
+ if (!Payload)
+ {
+ ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[FileChunkIndex], FilePath);
+ }
- if (!AsyncCallback(FileChunkIndex, Payload, IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0))
+ if (!AsyncCallback(FileChunkIndex,
+ Payload,
+ IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0))
+ {
+ AbortFlag.store(true);
+ }
+ }
+ catch (const std::exception& Ex)
{
- Result.store(false);
+ ZEN_WARN("oplog '{}/{}': exception caught when iterating file chunk {}, path '{}'. Reason: '{}'",
+ m_OuterProject->Identifier,
+ m_OplogId,
+ FileChunkIndex,
+ FilePath,
+ Ex.what());
}
- }
- catch (const std::exception& Ex)
- {
- ZEN_WARN("oplog '{}/{}': exception caught when iterating file chunk {}, path '{}'. Reason: '{}'",
- m_OuterProject->Identifier,
- m_OplogId,
- FileChunkIndex,
- FilePath,
- Ex.what());
- }
- });
- }
+ });
+ }
- if (!CidChunkHashes.empty())
+ if (!CidChunkHashes.empty() && !AbortFlag)
+ {
+ m_CidStore.IterateChunks(
+ CidChunkHashes,
+ [&](size_t Index, const IoBuffer& Payload) {
+ size_t CidChunkIndex = CidChunkIndexes[Index];
+ if (AbortFlag)
+ {
+ return false;
+ }
+ return AsyncCallback(CidChunkIndex,
+ Payload,
+ IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0);
+ },
+ OptionalWorkerPool,
+ LargeSizeLimit);
+ }
+ }
+ catch (const std::exception& Ex)
{
- m_CidStore.IterateChunks(
- CidChunkHashes,
- [&](size_t Index, const IoBuffer& Payload) {
- size_t CidChunkIndex = CidChunkIndexes[Index];
- return AsyncCallback(CidChunkIndex, Payload, IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0);
- },
- OptionalWorkerPool,
- LargeSizeLimit);
+ AbortFlag.store(true);
+ ZEN_WARN("Failed iterating oplog chunks in {}. Reason: '{}'", m_BasePath, Ex.what());
}
- WorkLatch.CountDown();
- WorkLatch.Wait();
+ Work.Wait();
- return Result.load();
+ return !AbortFlag;
}
else
{
@@ -2735,7 +2741,7 @@ ProjectStore::Oplog::CheckPendingChunkReferences(std::span<const IoHash> ChunkHa
MissingChunks.reserve(ChunkHashes.size());
for (const IoHash& FileHash : ChunkHashes)
{
- if (IoBuffer Payload = m_CidStore.FindChunkByCid(FileHash); !Payload)
+ if (!m_CidStore.ContainsChunk(FileHash))
{
MissingChunks.push_back(FileHash);
}
@@ -3129,7 +3135,7 @@ ProjectStore::Project::~Project()
bool
ProjectStore::Project::Exists(const std::filesystem::path& BasePath)
{
- return std::filesystem::exists(BasePath / "Project.zcb");
+ return IsFile(BasePath / "Project.zcb");
}
void
@@ -3203,7 +3209,7 @@ ProjectStore::Project::ReadAccessTimes()
using namespace std::literals;
std::filesystem::path ProjectAccessTimesFilePath = m_OplogStoragePath / "AccessTimes.zcb"sv;
- if (!std::filesystem::exists(ProjectAccessTimesFilePath))
+ if (!IsFile(ProjectAccessTimesFilePath))
{
return;
}
@@ -3359,7 +3365,6 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact, bo
ZEN_MEMSCOPE(GetProjectstoreTag());
ZEN_TRACE_CPU("Store::OpenOplog");
- std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
{
RwLock::SharedLockScope ProjectLock(m_ProjectLock);
@@ -3367,21 +3372,35 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact, bo
if (OplogIt != m_Oplogs.end())
{
- if (!VerifyPathOnDisk || Oplog::ExistsAt(OplogBasePath))
+ bool ReOpen = false;
+
+ if (VerifyPathOnDisk)
{
- return OplogIt->second.get();
+ std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
+
+ if (!Oplog::ExistsAt(OplogBasePath))
+ {
+ // Somebody deleted the oplog on disk behind our back
+ ProjectLock.ReleaseNow();
+ std::filesystem::path DeletePath;
+ if (!RemoveOplog(OplogId, DeletePath))
+ {
+ ZEN_WARN("Failed to clean up deleted oplog {}/{}", Identifier, OplogId, OplogBasePath);
+ }
+
+ ReOpen = true;
+ }
}
- // Somebody deleted the oplog on disk behind our back
- ProjectLock.ReleaseNow();
- std::filesystem::path DeletePath;
- if (!RemoveOplog(OplogId, DeletePath))
+ if (!ReOpen)
{
- ZEN_WARN("Failed to clean up deleted oplog {}/{}", Identifier, OplogId, OplogBasePath);
+ return OplogIt->second.get();
}
}
}
+ std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
+
RwLock::ExclusiveLockScope Lock(m_ProjectLock);
if (auto It = m_Oplogs.find(std::string{OplogId}); It != m_Oplogs.end())
{
@@ -3581,14 +3600,14 @@ ProjectStore::Project::TotalSize(const std::filesystem::path& BasePath)
uint64_t Size = 0;
std::filesystem::path AccessTimesFilePath = BasePath / "AccessTimes.zcb"sv;
- if (std::filesystem::exists(AccessTimesFilePath))
+ if (IsFile(AccessTimesFilePath))
{
- Size += std::filesystem::file_size(AccessTimesFilePath);
+ Size += FileSizeFromPath(AccessTimesFilePath);
}
std::filesystem::path ProjectFilePath = BasePath / "Project.zcb"sv;
- if (std::filesystem::exists(ProjectFilePath))
+ if (IsFile(ProjectFilePath))
{
- Size += std::filesystem::file_size(ProjectFilePath);
+ Size += FileSizeFromPath(ProjectFilePath);
}
return Size;
@@ -3700,7 +3719,7 @@ ProjectStore::Project::IsExpired(const std::string& EntryName,
if (!MarkerPath.empty())
{
std::error_code Ec;
- if (std::filesystem::exists(MarkerPath, Ec))
+ if (IsFile(MarkerPath, Ec))
{
if (Ec)
{
@@ -3853,7 +3872,7 @@ void
ProjectStore::DiscoverProjects()
{
ZEN_MEMSCOPE(GetProjectstoreTag());
- if (!std::filesystem::exists(m_ProjectBasePath))
+ if (!IsDir(m_ProjectBasePath))
{
return;
}
@@ -3902,26 +3921,32 @@ ProjectStore::Flush()
}
WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst);
- Latch WorkLatch(1);
-
- for (const Ref<Project>& Project : Projects)
+ std::atomic<bool> AbortFlag;
+ std::atomic<bool> PauseFlag;
+ ParallelWork Work(AbortFlag, PauseFlag);
+ try
{
- WorkLatch.AddCount(1);
- WorkerPool.ScheduleWork([this, &WorkLatch, Project]() {
- auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); });
- try
- {
- Project->Flush();
- }
- catch (const std::exception& Ex)
- {
- ZEN_WARN("Exception while flushing project {}: {}", Project->Identifier, Ex.what());
- }
- });
+ for (const Ref<Project>& Project : Projects)
+ {
+ Work.ScheduleWork(WorkerPool, [this, Project](std::atomic<bool>&) {
+ try
+ {
+ Project->Flush();
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_WARN("Exception while flushing project {}: {}", Project->Identifier, Ex.what());
+ }
+ });
+ }
+ }
+ catch (const std::exception& Ex)
+ {
+ AbortFlag.store(true);
+ ZEN_WARN("Failed projects in {}. Reason: '{}'", m_ProjectBasePath, Ex.what());
}
- WorkLatch.CountDown();
- WorkLatch.Wait();
+ Work.Wait();
}
void
@@ -3962,7 +3987,7 @@ ProjectStore::StorageSize() const
GcStorageSize Result;
{
- if (std::filesystem::exists(m_ProjectBasePath))
+ if (IsDir(m_ProjectBasePath))
{
DirectoryContent ProjectsFolderContent;
GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, ProjectsFolderContent);
@@ -3970,7 +3995,7 @@ ProjectStore::StorageSize() const
for (const std::filesystem::path& ProjectBasePath : ProjectsFolderContent.Directories)
{
std::filesystem::path ProjectStateFilePath = ProjectBasePath / "Project.zcb"sv;
- if (std::filesystem::exists(ProjectStateFilePath))
+ if (IsFile(ProjectStateFilePath))
{
Result.DiskSize += Project::TotalSize(ProjectBasePath);
DirectoryContent DirContent;
@@ -4753,7 +4778,6 @@ std::pair<HttpResponseCode, std::string>
ProjectStore::GetChunk(const std::string_view ProjectId,
const std::string_view OplogId,
const std::string_view Cid,
- ZenContentType AcceptType,
IoBuffer& OutChunk,
uint64_t* OptionalInOutModificationTag)
{
@@ -4795,16 +4819,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId,
}
}
- if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary)
- {
- CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk));
- OutChunk = Compressed.Decompress().AsIoBuffer();
- OutChunk.SetContentType(ZenContentType::kBinary);
- }
- else
- {
- OutChunk.SetContentType(ZenContentType::kCompressedBinary);
- }
+ OutChunk.SetContentType(ZenContentType::kCompressedBinary);
return {HttpResponseCode::OK, {}};
}
@@ -5347,7 +5362,7 @@ ProjectStore::ReadOplog(const std::string_view ProjectId,
/* BuildBlocks */ false,
/* IgnoreMissingAttachments */ false,
/* AllowChunking*/ false,
- [](CompressedBuffer&&, RemoteProjectStore::Block&&) {},
+ [](CompressedBuffer&&, ChunkBlockDescription&&) {},
[](const IoHash&, TGetAttachmentBufferFunc&&) {},
[](std::vector<std::pair<IoHash, FetchChunkFunc>>&&) {},
/* EmbedLooseFiles*/ false);
@@ -7236,7 +7251,7 @@ TEST_CASE("project.store.gc")
CHECK(ProjectStore.OpenProject("proj2"sv));
}
- std::filesystem::remove(Project1FilePath);
+ RemoveFile(Project1FilePath);
{
GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24),
@@ -7265,7 +7280,7 @@ TEST_CASE("project.store.gc")
CHECK(ProjectStore.OpenProject("proj2"sv));
}
- std::filesystem::remove(Project2Oplog1Path);
+ RemoveFile(Project2Oplog1Path);
{
GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24),
.ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24),
@@ -7294,7 +7309,7 @@ TEST_CASE("project.store.gc")
CHECK(ProjectStore.OpenProject("proj2"sv));
}
- std::filesystem::remove(Project2FilePath);
+ RemoveFile(Project2FilePath);
{
GcSettings Settings = {.CacheExpireTime = GcClock::Now() + std::chrono::hours(24),
.ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24),
@@ -8018,7 +8033,7 @@ TEST_CASE("project.store.rpc.getchunks")
CompositeBuffer Buffer = Attachment->AsCompositeBinary();
CHECK_EQ(IoHash::HashBuffer(IoBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten(), 81823, 5434)),
IoHash::HashBuffer(Buffer));
- CHECK_EQ(Chunk["Size"sv].AsUInt64(), std::filesystem::file_size(FilesOpIdAttachments[0].second));
+ CHECK_EQ(Chunk["Size"sv].AsUInt64(), FileSizeFromPath(FilesOpIdAttachments[0].second));
CHECK(!Chunk.FindView("RawSize"));
}
{
@@ -8499,12 +8514,7 @@ TEST_CASE("project.store.partial.read")
uint64_t ModificationTag = 0;
IoBuffer Chunk;
CHECK(ProjectStore
- .GetChunk("proj1"sv,
- "oplog1"sv,
- Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(),
- HttpContentType::kCompressedBinary,
- Chunk,
- &ModificationTag)
+ .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag)
.first == HttpResponseCode::OK);
IoHash RawHash;
uint64_t RawSize;
@@ -8513,12 +8523,7 @@ TEST_CASE("project.store.partial.read")
CHECK(ModificationTag != 0);
CHECK(ProjectStore
- .GetChunk("proj1"sv,
- "oplog1"sv,
- Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(),
- HttpContentType::kCompressedBinary,
- Chunk,
- &ModificationTag)
+ .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag)
.first == HttpResponseCode::NotModified);
}
@@ -8621,14 +8626,18 @@ TEST_CASE("project.store.block")
Chunks.reserve(AttachmentSizes.size());
for (const auto& It : AttachmentsWithId)
{
- Chunks.push_back(std::make_pair(It.second.DecodeRawHash(),
- [Buffer = It.second.GetCompressed().Flatten().AsIoBuffer()](const IoHash&) -> CompositeBuffer {
- return CompositeBuffer(SharedBuffer(Buffer));
- }));
- }
- RemoteProjectStore::Block Block;
- CompressedBuffer BlockBuffer = GenerateBlock(std::move(Chunks), Block);
- CHECK(IterateBlock(BlockBuffer.Decompress(), [](CompressedBuffer&&, const IoHash&) {}));
+ Chunks.push_back(
+ std::make_pair(It.second.DecodeRawHash(), [Buffer = It.second](const IoHash&) -> std::pair<uint64_t, CompressedBuffer> {
+ return {Buffer.DecodeRawSize(), Buffer};
+ }));
+ }
+ ChunkBlockDescription Block;
+ CompressedBuffer BlockBuffer = GenerateChunkBlock(std::move(Chunks), Block);
+ uint64_t HeaderSize;
+ CHECK(IterateChunkBlock(
+ BlockBuffer.Decompress(),
+ [](CompressedBuffer&&, const IoHash&) {},
+ HeaderSize));
}
TEST_CASE("project.store.iterateoplog")
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 8f2d3ce0d..368da5ea4 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -449,7 +449,6 @@ public:
std::pair<HttpResponseCode, std::string> GetChunk(const std::string_view ProjectId,
const std::string_view OplogId,
const std::string_view Cid,
- ZenContentType AcceptType,
IoBuffer& OutChunk,
uint64_t* OptionalInOutModificationTag);
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index 0589fdc5f..f96b3e185 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -12,8 +12,8 @@
#include <zencore/stream.h>
#include <zencore/timer.h>
#include <zencore/workthreadpool.h>
-#include <zenstore/chunkedfile.h>
#include <zenstore/cidstore.h>
+#include <zenutil/chunkedfile.h>
#include <zenutil/workerpools.h>
#include <unordered_map>
@@ -143,7 +143,7 @@ namespace remotestore_impl {
NiceBytes(Stats.m_PeakReceivedBytes));
}
- size_t AddBlock(RwLock& BlocksLock, std::vector<RemoteProjectStore::Block>& Blocks)
+ size_t AddBlock(RwLock& BlocksLock, std::vector<ChunkBlockDescription>& Blocks)
{
size_t BlockIndex;
{
@@ -154,63 +154,6 @@ namespace remotestore_impl {
return BlockIndex;
}
- IoBuffer WriteToTempFile(CompressedBuffer&& CompressedBuffer, std::filesystem::path Path)
- {
- if (std::filesystem::is_regular_file(Path))
- {
- IoBuffer ExistingTempFile = IoBuffer(IoBufferBuilder::MakeFromFile(Path));
- if (ExistingTempFile && ExistingTempFile.GetSize() == CompressedBuffer.GetCompressedSize())
- {
- ExistingTempFile.SetDeleteOnClose(true);
- return ExistingTempFile;
- }
- }
- IoBuffer BlockBuffer;
- BasicFile BlockFile;
- uint32_t RetriesLeft = 3;
- BlockFile.Open(Path, BasicFile::Mode::kTruncateDelete, [&](std::error_code& Ec) {
- if (RetriesLeft == 0)
- {
- return false;
- }
- ZEN_WARN("Failed to create temporary oplog block '{}': '{}', retries left: {}.", Path, Ec.message(), RetriesLeft);
- Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms
- RetriesLeft--;
- return true;
- });
- uint64_t Offset = 0;
- {
- CompositeBuffer Compressed = std::move(CompressedBuffer).GetCompressed();
- for (const SharedBuffer& Segment : Compressed.GetSegments())
- {
- size_t SegmentSize = Segment.GetSize();
- static const uint64_t BufferingSize = 256u * 1024u;
-
- IoBufferFileReference FileRef;
- if (SegmentSize >= (BufferingSize + BufferingSize / 2) && Segment.GetFileReference(FileRef))
- {
- ScanFile(FileRef.FileHandle,
- FileRef.FileChunkOffset,
- FileRef.FileChunkSize,
- BufferingSize,
- [&BlockFile, &Offset](const void* Data, size_t Size) {
- BlockFile.Write(Data, Size, Offset);
- Offset += Size;
- });
- }
- else
- {
- BlockFile.Write(Segment.GetData(), SegmentSize, Offset);
- Offset += SegmentSize;
- }
- }
- }
- void* FileHandle = BlockFile.Detach();
- BlockBuffer = IoBuffer(IoBuffer::File, FileHandle, 0, Offset, /*IsWholeFile*/ true);
- BlockBuffer.SetDeleteOnClose(true);
- return BlockBuffer;
- }
-
RemoteProjectStore::Result WriteOplogSection(ProjectStore::Oplog& Oplog, const CbObjectView& SectionObject, JobContext* OptionalContext)
{
using namespace std::literals;
@@ -573,21 +516,23 @@ namespace remotestore_impl {
return;
}
- bool StoreChunksOK = IterateBlock(
- BlockPayload,
- [&WantedChunks, &WriteAttachmentBuffers, &WriteRawHashes, &Info](CompressedBuffer&& Chunk,
- const IoHash& AttachmentRawHash) {
- if (WantedChunks.contains(AttachmentRawHash))
- {
- WriteAttachmentBuffers.emplace_back(Chunk.GetCompressed().Flatten().AsIoBuffer());
- IoHash RawHash;
- uint64_t RawSize;
- ZEN_ASSERT(CompressedBuffer::ValidateCompressedHeader(WriteAttachmentBuffers.back(), RawHash, RawSize));
- ZEN_ASSERT(RawHash == AttachmentRawHash);
- WriteRawHashes.emplace_back(AttachmentRawHash);
- WantedChunks.erase(AttachmentRawHash);
- }
- });
+ uint64_t BlockHeaderSize = 0;
+ bool StoreChunksOK = IterateChunkBlock(
+ BlockPayload,
+ [&WantedChunks, &WriteAttachmentBuffers, &WriteRawHashes, &Info](CompressedBuffer&& Chunk,
+ const IoHash& AttachmentRawHash) {
+ if (WantedChunks.contains(AttachmentRawHash))
+ {
+ WriteAttachmentBuffers.emplace_back(Chunk.GetCompressed().Flatten().AsIoBuffer());
+ IoHash RawHash;
+ uint64_t RawSize;
+ ZEN_ASSERT(CompressedBuffer::ValidateCompressedHeader(WriteAttachmentBuffers.back(), RawHash, RawSize));
+ ZEN_ASSERT(RawHash == AttachmentRawHash);
+ WriteRawHashes.emplace_back(AttachmentRawHash);
+ WantedChunks.erase(AttachmentRawHash);
+ }
+ },
+ BlockHeaderSize);
if (!StoreChunksOK)
{
@@ -738,14 +683,14 @@ namespace remotestore_impl {
});
};
- void CreateBlock(WorkerThreadPool& WorkerPool,
- Latch& OpSectionsLatch,
- std::vector<std::pair<IoHash, FetchChunkFunc>>&& ChunksInBlock,
- RwLock& SectionsLock,
- std::vector<RemoteProjectStore::Block>& Blocks,
- size_t BlockIndex,
- const std::function<void(CompressedBuffer&&, RemoteProjectStore::Block&&)>& AsyncOnBlock,
- AsyncRemoteResult& RemoteResult)
+ void CreateBlock(WorkerThreadPool& WorkerPool,
+ Latch& OpSectionsLatch,
+ std::vector<std::pair<IoHash, FetchChunkFunc>>&& ChunksInBlock,
+ RwLock& SectionsLock,
+ std::vector<ChunkBlockDescription>& Blocks,
+ size_t BlockIndex,
+ const std::function<void(CompressedBuffer&&, ChunkBlockDescription&&)>& AsyncOnBlock,
+ AsyncRemoteResult& RemoteResult)
{
OpSectionsLatch.AddCount(1);
WorkerPool.ScheduleWork([&Blocks,
@@ -764,10 +709,10 @@ namespace remotestore_impl {
try
{
ZEN_ASSERT(ChunkCount > 0);
- Stopwatch Timer;
- RemoteProjectStore::Block Block;
- CompressedBuffer CompressedBlock = GenerateBlock(std::move(Chunks), Block);
- IoHash BlockHash = CompressedBlock.DecodeRawHash();
+ Stopwatch Timer;
+ ChunkBlockDescription Block;
+ CompressedBuffer CompressedBlock = GenerateChunkBlock(std::move(Chunks), Block);
+ IoHash BlockHash = CompressedBlock.DecodeRawHash();
{
// We can share the lock as we are not resizing the vector and only touch BlockHash at our own index
RwLock::SharedLockScope __(SectionsLock);
@@ -800,8 +745,8 @@ namespace remotestore_impl {
struct CreatedBlock
{
- IoBuffer Payload;
- RemoteProjectStore::Block Block;
+ IoBuffer Payload;
+ ChunkBlockDescription Block;
};
void UploadAttachments(WorkerThreadPool& WorkerPool,
@@ -931,8 +876,8 @@ namespace remotestore_impl {
}
try
{
- IoBuffer Payload;
- RemoteProjectStore::Block Block;
+ IoBuffer Payload;
+ ChunkBlockDescription Block;
if (auto BlockIt = CreatedBlocks.find(RawHash); BlockIt != CreatedBlocks.end())
{
Payload = BlockIt->second.Payload;
@@ -1058,7 +1003,7 @@ namespace remotestore_impl {
{
auto It = BulkBlockAttachmentsToUpload.find(Chunk);
ZEN_ASSERT(It != BulkBlockAttachmentsToUpload.end());
- CompositeBuffer ChunkPayload = It->second(It->first);
+ CompressedBuffer ChunkPayload = It->second(It->first).second;
if (!ChunkPayload)
{
RemoteResult.SetError(static_cast<int32_t>(HttpResponseCode::NotFound),
@@ -1067,8 +1012,8 @@ namespace remotestore_impl {
ChunkBuffers.clear();
break;
}
- ChunksSize += ChunkPayload.GetSize();
- ChunkBuffers.emplace_back(SharedBuffer(std::move(ChunkPayload).Flatten().AsIoBuffer()));
+ ChunksSize += ChunkPayload.GetCompressedSize();
+ ChunkBuffers.emplace_back(SharedBuffer(std::move(ChunkPayload).GetCompressed().Flatten().AsIoBuffer()));
}
RemoteProjectStore::SaveAttachmentsResult Result = RemoteStore.SaveAttachments(ChunkBuffers);
if (Result.ErrorCode)
@@ -1139,54 +1084,13 @@ namespace remotestore_impl {
}
} // namespace remotestore_impl
-bool
-IterateBlock(const SharedBuffer& BlockPayload, std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor)
-{
- ZEN_ASSERT(BlockPayload);
- if (BlockPayload.GetSize() < 1)
- {
- return false;
- }
-
- MemoryView BlockView = BlockPayload.GetView();
- const uint8_t* ReadPtr = reinterpret_cast<const uint8_t*>(BlockView.GetData());
- uint32_t NumberSize;
- uint64_t ChunkCount = ReadVarUInt(ReadPtr, NumberSize);
- ReadPtr += NumberSize;
- std::vector<uint64_t> ChunkSizes;
- ChunkSizes.reserve(ChunkCount);
- while (ChunkCount--)
- {
- ChunkSizes.push_back(ReadVarUInt(ReadPtr, NumberSize));
- ReadPtr += NumberSize;
- }
- ptrdiff_t TempBufferLength = std::distance(reinterpret_cast<const uint8_t*>(BlockView.GetData()), ReadPtr);
- ZEN_ASSERT(TempBufferLength > 0);
- for (uint64_t ChunkSize : ChunkSizes)
- {
- IoBuffer Chunk(IoBuffer::Wrap, ReadPtr, ChunkSize);
- IoHash AttachmentRawHash;
- uint64_t AttachmentRawSize;
- CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), AttachmentRawHash, AttachmentRawSize);
-
- if (!CompressedChunk)
- {
- ZEN_ERROR("Invalid chunk in block");
- return false;
- }
- Visitor(std::move(CompressedChunk), AttachmentRawHash);
- ReadPtr += ChunkSize;
- ZEN_ASSERT(ReadPtr <= BlockView.GetDataEnd());
- }
- return true;
-};
std::vector<IoHash>
GetBlockHashesFromOplog(CbObjectView ContainerObject)
{
using namespace std::literals;
- std::vector<RemoteProjectStore::Block> Result;
- CbArrayView BlocksArray = ContainerObject["blocks"sv].AsArrayView();
+ std::vector<ChunkBlockDescription> Result;
+ CbArrayView BlocksArray = ContainerObject["blocks"sv].AsArrayView();
std::vector<IoHash> BlockHashes;
BlockHashes.reserve(BlocksArray.Num());
@@ -1199,11 +1103,11 @@ GetBlockHashesFromOplog(CbObjectView ContainerObject)
return BlockHashes;
}
-std::vector<RemoteProjectStore::Block>
+std::vector<ThinChunkBlockDescription>
GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> IncludeBlockHashes)
{
using namespace std::literals;
- std::vector<RemoteProjectStore::Block> Result;
+ std::vector<ThinChunkBlockDescription> Result;
CbArrayView BlocksArray = ContainerObject["blocks"sv].AsArrayView();
tsl::robin_set<IoHash, IoHash::Hasher> IncludeSet;
IncludeSet.insert(IncludeBlockHashes.begin(), IncludeBlockHashes.end());
@@ -1226,53 +1130,12 @@ GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> Include
{
ChunkHashes.push_back(ChunkField.AsHash());
}
- Result.push_back({.BlockHash = BlockHash, .ChunkHashes = std::move(ChunkHashes)});
+ Result.push_back(ThinChunkBlockDescription{.BlockHash = BlockHash, .ChunkRawHashes = std::move(ChunkHashes)});
}
}
return Result;
}
-CompressedBuffer
-GenerateBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, RemoteProjectStore::Block& OutBlock)
-{
- const size_t ChunkCount = FetchChunks.size();
-
- std::vector<SharedBuffer> ChunkSegments;
- ChunkSegments.resize(1);
- ChunkSegments.reserve(1 + ChunkCount);
- OutBlock.ChunkHashes.reserve(ChunkCount);
- OutBlock.ChunkLengths.reserve(ChunkCount);
- {
- IoBuffer TempBuffer(ChunkCount * 9);
- MutableMemoryView View = TempBuffer.GetMutableView();
- uint8_t* BufferStartPtr = reinterpret_cast<uint8_t*>(View.GetData());
- uint8_t* BufferEndPtr = BufferStartPtr;
- BufferEndPtr += WriteVarUInt(gsl::narrow<uint64_t>(ChunkCount), BufferEndPtr);
- for (const auto& It : FetchChunks)
- {
- CompositeBuffer Chunk = It.second(It.first);
- uint64_t ChunkSize = 0;
- std::span<const SharedBuffer> Segments = Chunk.GetSegments();
- for (const SharedBuffer& Segment : Segments)
- {
- ChunkSize += Segment.GetSize();
- ChunkSegments.push_back(Segment);
- }
- BufferEndPtr += WriteVarUInt(ChunkSize, BufferEndPtr);
- OutBlock.ChunkHashes.push_back(It.first);
- OutBlock.ChunkLengths.push_back(gsl::narrow<uint32_t>(ChunkSize));
- }
- ZEN_ASSERT(BufferEndPtr <= View.GetDataEnd());
- ptrdiff_t TempBufferLength = std::distance(BufferStartPtr, BufferEndPtr);
- ChunkSegments[0] = SharedBuffer(IoBuffer(TempBuffer, 0, gsl::narrow<size_t>(TempBufferLength)));
- }
- CompressedBuffer CompressedBlock =
- CompressedBuffer::Compress(CompositeBuffer(std::move(ChunkSegments)), OodleCompressor::Mermaid, OodleCompressionLevel::None);
- OutBlock.BlockHash = CompressedBlock.DecodeRawHash();
- OutBlock.FirstChunkOffset = gsl::narrow<uint32_t>(CompressedBuffer::GetHeaderSizeForNoneEncoder() + ChunkSegments[0].GetSize());
- return CompressedBlock;
-}
-
CbObject
BuildContainer(CidStore& ChunkStore,
ProjectStore::Project& Project,
@@ -1283,9 +1146,9 @@ BuildContainer(CidStore& ChunkStore,
bool BuildBlocks,
bool IgnoreMissingAttachments,
bool AllowChunking,
- const std::vector<RemoteProjectStore::Block>& KnownBlocks,
+ const std::vector<ThinChunkBlockDescription>& KnownBlocks,
WorkerThreadPool& WorkerPool,
- const std::function<void(CompressedBuffer&&, RemoteProjectStore::Block&&)>& AsyncOnBlock,
+ const std::function<void(CompressedBuffer&&, ChunkBlockDescription&&)>& AsyncOnBlock,
const std::function<void(const IoHash&, TGetAttachmentBufferFunc&&)>& OnLargeAttachment,
const std::function<void(std::vector<std::pair<IoHash, FetchChunkFunc>>&&)>& OnBlockChunks,
bool EmbedLooseFiles,
@@ -1307,9 +1170,9 @@ BuildContainer(CidStore& ChunkStore,
std::unordered_map<IoHash, FoundAttachment, IoHash::Hasher> UploadAttachments;
- RwLock BlocksLock;
- std::vector<RemoteProjectStore::Block> Blocks;
- CompressedBuffer OpsBuffer;
+ RwLock BlocksLock;
+ std::vector<ChunkBlockDescription> Blocks;
+ CompressedBuffer OpsBuffer;
std::filesystem::path AttachmentTempPath = Oplog.TempPath();
AttachmentTempPath.append(".pending");
@@ -1349,7 +1212,7 @@ BuildContainer(CidStore& ChunkStore,
{
std::string_view ServerPath = View["serverpath"sv].AsString();
std::filesystem::path FilePath = Project.RootDir / ServerPath;
- if (!std::filesystem::is_regular_file(FilePath))
+ if (!IsFile(FilePath))
{
remotestore_impl::ReportMessage(
OptionalContext,
@@ -1525,7 +1388,7 @@ BuildContainer(CidStore& ChunkStore,
return {};
}
- auto FindReuseBlocks = [](const std::vector<RemoteProjectStore::Block>& KnownBlocks,
+ auto FindReuseBlocks = [](const std::vector<ThinChunkBlockDescription>& KnownBlocks,
const std::unordered_set<IoHash, IoHash::Hasher>& Attachments,
JobContext* OptionalContext) -> std::vector<size_t> {
std::vector<size_t> ReuseBlockIndexes;
@@ -1538,14 +1401,14 @@ BuildContainer(CidStore& ChunkStore,
for (size_t KnownBlockIndex = 0; KnownBlockIndex < KnownBlocks.size(); KnownBlockIndex++)
{
- const RemoteProjectStore::Block& KnownBlock = KnownBlocks[KnownBlockIndex];
- size_t BlockAttachmentCount = KnownBlock.ChunkHashes.size();
+ const ThinChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex];
+ size_t BlockAttachmentCount = KnownBlock.ChunkRawHashes.size();
if (BlockAttachmentCount == 0)
{
continue;
}
size_t FoundAttachmentCount = 0;
- for (const IoHash& KnownHash : KnownBlock.ChunkHashes)
+ for (const IoHash& KnownHash : KnownBlock.ChunkRawHashes)
{
if (Attachments.contains(KnownHash))
{
@@ -1586,8 +1449,8 @@ BuildContainer(CidStore& ChunkStore,
std::vector<size_t> ReusedBlockIndexes = FindReuseBlocks(KnownBlocks, FoundHashes, OptionalContext);
for (size_t KnownBlockIndex : ReusedBlockIndexes)
{
- const RemoteProjectStore::Block& KnownBlock = KnownBlocks[KnownBlockIndex];
- for (const IoHash& KnownHash : KnownBlock.ChunkHashes)
+ const ThinChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex];
+ for (const IoHash& KnownHash : KnownBlock.ChunkRawHashes)
{
if (UploadAttachments.erase(KnownHash) == 1)
{
@@ -1605,10 +1468,7 @@ BuildContainer(CidStore& ChunkStore,
};
std::vector<ChunkedFile> ChunkedFiles;
- auto ChunkFile = [AttachmentTempPath](const IoHash& RawHash,
- IoBuffer& RawData,
- const IoBufferFileReference& FileRef,
- JobContext*) -> ChunkedFile {
+ auto ChunkFile = [](const IoHash& RawHash, IoBuffer& RawData, const IoBufferFileReference& FileRef, JobContext*) -> ChunkedFile {
ChunkedFile Chunked;
Stopwatch Timer;
@@ -1632,12 +1492,12 @@ BuildContainer(CidStore& ChunkStore,
return Chunked;
};
- RwLock ResolveLock;
- std::unordered_set<IoHash, IoHash::Hasher> ChunkedHashes;
- std::unordered_set<IoHash, IoHash::Hasher> LargeChunkHashes;
- std::unordered_map<IoHash, size_t, IoHash::Hasher> ChunkedUploadAttachments;
- std::unordered_map<IoHash, IoBuffer, IoHash::Hasher> LooseUploadAttachments;
- std::unordered_set<IoHash, IoHash::Hasher> MissingHashes;
+ RwLock ResolveLock;
+ std::unordered_set<IoHash, IoHash::Hasher> ChunkedHashes;
+ std::unordered_set<IoHash, IoHash::Hasher> LargeChunkHashes;
+ std::unordered_map<IoHash, size_t, IoHash::Hasher> ChunkedUploadAttachments;
+ std::unordered_map<IoHash, std::pair<uint64_t, IoBuffer>, IoHash::Hasher> LooseUploadAttachments;
+ std::unordered_set<IoHash, IoHash::Hasher> MissingHashes;
remotestore_impl::ReportMessage(OptionalContext,
fmt::format("Resolving {} attachments from {} ops", UploadAttachments.size(), TotalOpCount));
@@ -1717,9 +1577,7 @@ BuildContainer(CidStore& ChunkStore,
std::filesystem::path AttachmentPath = AttachmentTempPath;
AttachmentPath.append(RawHash.ToHexString());
-
- IoBuffer TempAttachmentBuffer =
- remotestore_impl::WriteToTempFile(std::move(Compressed), AttachmentPath);
+ IoBuffer TempAttachmentBuffer = WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath);
ZEN_INFO("Saved temp attachment to '{}', {} ({})",
AttachmentPath,
NiceBytes(RawSize),
@@ -1730,7 +1588,7 @@ BuildContainer(CidStore& ChunkStore,
}
else
{
- size_t RawSize = RawData.GetSize();
+ uint64_t RawSize = RawData.GetSize();
CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(RawData),
OodleCompressor::Mermaid,
OodleCompressionLevel::VeryFast);
@@ -1738,23 +1596,24 @@ BuildContainer(CidStore& ChunkStore,
std::filesystem::path AttachmentPath = AttachmentTempPath;
AttachmentPath.append(RawHash.ToHexString());
- IoBuffer TempAttachmentBuffer = remotestore_impl::WriteToTempFile(std::move(Compressed), AttachmentPath);
+ uint64_t CompressedSize = Compressed.GetCompressedSize();
+ IoBuffer TempAttachmentBuffer = WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath);
ZEN_INFO("Saved temp attachment to '{}', {} ({})",
AttachmentPath,
NiceBytes(RawSize),
NiceBytes(TempAttachmentBuffer.GetSize()));
- if (Compressed.GetCompressedSize() > MaxChunkEmbedSize)
+ if (CompressedSize > MaxChunkEmbedSize)
{
OnLargeAttachment(RawHash, [Data = std::move(TempAttachmentBuffer)](const IoHash&) { return Data; });
ResolveLock.WithExclusiveLock([RawHash, &LargeChunkHashes]() { LargeChunkHashes.insert(RawHash); });
}
else
{
- UploadAttachment->Size = Compressed.GetCompressedSize();
+ UploadAttachment->Size = CompressedSize;
ResolveLock.WithExclusiveLock(
- [RawHash, &LooseUploadAttachments, Data = std::move(TempAttachmentBuffer)]() {
- LooseUploadAttachments.insert_or_assign(RawHash, std::move(Data));
+ [RawHash, RawSize, &LooseUploadAttachments, Data = std::move(TempAttachmentBuffer)]() {
+ LooseUploadAttachments.insert_or_assign(RawHash, std::make_pair(RawSize, std::move(Data)));
});
}
}
@@ -1927,8 +1786,8 @@ BuildContainer(CidStore& ChunkStore,
std::vector<size_t> ReusedBlockFromChunking = FindReuseBlocks(KnownBlocks, ChunkedHashes, OptionalContext);
for (size_t KnownBlockIndex : ReusedBlockIndexes)
{
- const RemoteProjectStore::Block& KnownBlock = KnownBlocks[KnownBlockIndex];
- for (const IoHash& KnownHash : KnownBlock.ChunkHashes)
+ const ThinChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex];
+ for (const IoHash& KnownHash : KnownBlock.ChunkRawHashes)
{
if (ChunkedHashes.erase(KnownHash) == 1)
{
@@ -1946,7 +1805,7 @@ BuildContainer(CidStore& ChunkStore,
Blocks.reserve(ReuseBlockCount);
for (auto It = ReusedBlockIndexes.begin(); It != UniqueKnownBlocksEnd; It++)
{
- Blocks.push_back(KnownBlocks[*It]);
+ Blocks.push_back({KnownBlocks[*It]});
}
remotestore_impl::ReportMessage(OptionalContext,
fmt::format("Reused {} attachments from {} blocks", ReusedAttachmentCount, ReuseBlockCount));
@@ -2062,9 +1921,9 @@ BuildContainer(CidStore& ChunkStore,
{
// We can share the lock as we are not resizing the vector and only touch BlockHash at our own index
RwLock::SharedLockScope _(BlocksLock);
- Blocks[BlockIndex].ChunkHashes.insert(Blocks[BlockIndex].ChunkHashes.end(),
- BlockAttachmentHashes.begin(),
- BlockAttachmentHashes.end());
+ Blocks[BlockIndex].ChunkRawHashes.insert(Blocks[BlockIndex].ChunkRawHashes.end(),
+ BlockAttachmentHashes.begin(),
+ BlockAttachmentHashes.end());
}
uint64_t NowMS = Timer.GetElapsedTimeMs();
ZEN_INFO("Assembled block {} with {} chunks in {} ({})",
@@ -2109,16 +1968,25 @@ BuildContainer(CidStore& ChunkStore,
{
if (auto It = LooseUploadAttachments.find(RawHash); It != LooseUploadAttachments.end())
{
- ChunksInBlock.emplace_back(std::make_pair(RawHash, [IoBuffer = SharedBuffer(It->second)](const IoHash&) {
- return CompositeBuffer(IoBuffer);
- }));
+ ChunksInBlock.emplace_back(std::make_pair(
+ RawHash,
+ [RawSize = It->second.first,
+ IoBuffer = SharedBuffer(It->second.second)](const IoHash&) -> std::pair<uint64_t, CompressedBuffer> {
+ return std::make_pair(RawSize, CompressedBuffer::FromCompressedNoValidate(IoBuffer.AsIoBuffer()));
+ }));
LooseUploadAttachments.erase(It);
}
else
{
- ChunksInBlock.emplace_back(std::make_pair(RawHash, [&ChunkStore](const IoHash& RawHash) {
- return CompositeBuffer(SharedBuffer(ChunkStore.FindChunkByCid(RawHash)));
- }));
+ ChunksInBlock.emplace_back(
+ std::make_pair(RawHash, [&ChunkStore](const IoHash& RawHash) -> std::pair<uint64_t, CompressedBuffer> {
+ IoBuffer Chunk = ChunkStore.FindChunkByCid(RawHash);
+ IoHash _;
+ uint64_t RawSize = 0;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunk)), _, RawSize);
+ ZEN_ASSERT(Compressed);
+ return {RawSize, Compressed};
+ }));
}
BlockSize += PayloadSize;
@@ -2169,14 +2037,15 @@ BuildContainer(CidStore& ChunkStore,
if (BlockAttachmentHashes.insert(ChunkHash).second)
{
const ChunkSource& Source = Chunked.ChunkSources[ChunkIndex];
- ChunksInBlock.emplace_back(std::make_pair(
- ChunkHash,
- [Source = ChunkedFile.Source, Offset = Source.Offset, Size = Source.Size](const IoHash&) {
- return CompressedBuffer::Compress(SharedBuffer(IoBuffer(Source, Offset, Size)),
- OodleCompressor::Mermaid,
- OodleCompressionLevel::None)
- .GetCompressed();
- }));
+ ChunksInBlock.emplace_back(
+ std::make_pair(ChunkHash,
+ [Source = ChunkedFile.Source, Offset = Source.Offset, Size = Source.Size](
+ const IoHash&) -> std::pair<uint64_t, CompressedBuffer> {
+ return {Size,
+ CompressedBuffer::Compress(SharedBuffer(IoBuffer(Source, Offset, Size)),
+ OodleCompressor::Mermaid,
+ OodleCompressionLevel::None)};
+ }));
BlockSize += CompressedBuffer::GetHeaderSizeForNoneEncoder() + Source.Size;
if (BuildBlocks)
{
@@ -2298,9 +2167,9 @@ BuildContainer(CidStore& ChunkStore,
OplogContinerWriter.AddBinary("ops"sv, CompressedOpsSection.GetCompressed().Flatten().AsIoBuffer());
OplogContinerWriter.BeginArray("blocks"sv);
{
- for (const RemoteProjectStore::Block& B : Blocks)
+ for (const ChunkBlockDescription& B : Blocks)
{
- ZEN_ASSERT(!B.ChunkHashes.empty());
+ ZEN_ASSERT(!B.ChunkRawHashes.empty());
if (BuildBlocks)
{
ZEN_ASSERT(B.BlockHash != IoHash::Zero);
@@ -2310,7 +2179,7 @@ BuildContainer(CidStore& ChunkStore,
OplogContinerWriter.AddBinaryAttachment("rawhash"sv, B.BlockHash);
OplogContinerWriter.BeginArray("chunks"sv);
{
- for (const IoHash& RawHash : B.ChunkHashes)
+ for (const IoHash& RawHash : B.ChunkRawHashes)
{
OplogContinerWriter.AddHash(RawHash);
}
@@ -2326,7 +2195,7 @@ BuildContainer(CidStore& ChunkStore,
{
OplogContinerWriter.BeginArray("chunks"sv);
{
- for (const IoHash& RawHash : B.ChunkHashes)
+ for (const IoHash& RawHash : B.ChunkRawHashes)
{
OplogContinerWriter.AddBinaryAttachment(RawHash);
}
@@ -2392,7 +2261,7 @@ BuildContainer(CidStore& ChunkStore,
bool BuildBlocks,
bool IgnoreMissingAttachments,
bool AllowChunking,
- const std::function<void(CompressedBuffer&&, RemoteProjectStore::Block&&)>& AsyncOnBlock,
+ const std::function<void(CompressedBuffer&&, ChunkBlockDescription&&)>& AsyncOnBlock,
const std::function<void(const IoHash&, TGetAttachmentBufferFunc&&)>& OnLargeAttachment,
const std::function<void(std::vector<std::pair<IoHash, FetchChunkFunc>>&&)>& OnBlockChunks,
bool EmbedLooseFiles)
@@ -2458,13 +2327,13 @@ SaveOplog(CidStore& ChunkStore,
std::unordered_map<IoHash, remotestore_impl::CreatedBlock, IoHash::Hasher> CreatedBlocks;
tsl::robin_map<IoHash, TGetAttachmentBufferFunc, IoHash::Hasher> LooseLargeFiles;
- auto MakeTempBlock = [AttachmentTempPath, &RemoteResult, &AttachmentsLock, &CreatedBlocks](CompressedBuffer&& CompressedBlock,
- RemoteProjectStore::Block&& Block) {
+ auto MakeTempBlock = [AttachmentTempPath, &RemoteResult, &AttachmentsLock, &CreatedBlocks](CompressedBuffer&& CompressedBlock,
+ ChunkBlockDescription&& Block) {
std::filesystem::path BlockPath = AttachmentTempPath;
BlockPath.append(Block.BlockHash.ToHexString());
try
{
- IoBuffer BlockBuffer = remotestore_impl::WriteToTempFile(std::move(CompressedBlock), BlockPath);
+ IoBuffer BlockBuffer = WriteToTempFile(std::move(CompressedBlock).GetCompressed(), BlockPath);
RwLock::ExclusiveLockScope __(AttachmentsLock);
CreatedBlocks.insert({Block.BlockHash, {.Payload = std::move(BlockBuffer), .Block = std::move(Block)}});
ZEN_DEBUG("Saved temp block to '{}', {}", AttachmentTempPath, NiceBytes(BlockBuffer.GetSize()));
@@ -2478,8 +2347,8 @@ SaveOplog(CidStore& ChunkStore,
}
};
- auto UploadBlock = [&RemoteStore, &RemoteResult, &Info, OptionalContext](CompressedBuffer&& CompressedBlock,
- RemoteProjectStore::Block&& Block) {
+ auto UploadBlock = [&RemoteStore, &RemoteResult, &Info, OptionalContext](CompressedBuffer&& CompressedBlock,
+ ChunkBlockDescription&& Block) {
IoHash BlockHash = Block.BlockHash;
RemoteProjectStore::SaveAttachmentResult Result =
RemoteStore.SaveAttachment(CompressedBlock.GetCompressed(), BlockHash, std::move(Block));
@@ -2512,7 +2381,7 @@ SaveOplog(CidStore& ChunkStore,
ZEN_DEBUG("Found attachment {}", AttachmentHash);
};
- std::function<void(CompressedBuffer&&, RemoteProjectStore::Block &&)> OnBlock;
+ std::function<void(CompressedBuffer&&, ChunkBlockDescription &&)> OnBlock;
if (RemoteStoreInfo.UseTempBlockFiles)
{
OnBlock = MakeTempBlock;
@@ -2522,7 +2391,7 @@ SaveOplog(CidStore& ChunkStore,
OnBlock = UploadBlock;
}
- std::vector<RemoteProjectStore::Block> KnownBlocks;
+ std::vector<ThinChunkBlockDescription> KnownBlocks;
uint64_t TransferWallTimeMS = 0;
@@ -3214,9 +3083,9 @@ LoadOplog(CidStore& ChunkStore,
OptionalContext]() {
auto _ = MakeGuard([&DechunkLatch, &TempFileName] {
std::error_code Ec;
- if (std::filesystem::exists(TempFileName, Ec))
+ if (IsFile(TempFileName, Ec))
{
- std::filesystem::remove(TempFileName, Ec);
+ RemoveFile(TempFileName, Ec);
if (Ec)
{
ZEN_INFO("Failed to remove temporary file '{}'. Reason: {}", TempFileName, Ec.message());
diff --git a/src/zenserver/projectstore/remoteprojectstore.h b/src/zenserver/projectstore/remoteprojectstore.h
index e05cb9923..1210afc7c 100644
--- a/src/zenserver/projectstore/remoteprojectstore.h
+++ b/src/zenserver/projectstore/remoteprojectstore.h
@@ -5,6 +5,8 @@
#include <zencore/jobqueue.h>
#include "projectstore.h"
+#include <zenutil/chunkblock.h>
+
#include <unordered_set>
namespace zen {
@@ -16,14 +18,6 @@ struct ChunkedInfo;
class RemoteProjectStore
{
public:
- struct Block
- {
- IoHash BlockHash;
- std::vector<IoHash> ChunkHashes;
- std::vector<uint32_t> ChunkLengths;
- uint32_t FirstChunkOffset = (uint32_t)-1;
- };
-
struct Result
{
int32_t ErrorCode{};
@@ -72,7 +66,7 @@ public:
struct GetKnownBlocksResult : public Result
{
- std::vector<Block> Blocks;
+ std::vector<ThinChunkBlockDescription> Blocks;
};
struct RemoteStoreInfo
@@ -101,11 +95,11 @@ public:
virtual RemoteStoreInfo GetInfo() const = 0;
virtual Stats GetStats() const = 0;
- virtual CreateContainerResult CreateContainer() = 0;
- virtual SaveResult SaveContainer(const IoBuffer& Payload) = 0;
- virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&& Block) = 0;
- virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) = 0;
- virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Payloads) = 0;
+ virtual CreateContainerResult CreateContainer() = 0;
+ virtual SaveResult SaveContainer(const IoBuffer& Payload) = 0;
+ virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&& Block) = 0;
+ virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) = 0;
+ virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Payloads) = 0;
virtual LoadContainerResult LoadContainer() = 0;
virtual GetKnownBlocksResult GetKnownBlocks() = 0;
@@ -125,7 +119,6 @@ struct RemoteStoreOptions
};
typedef std::function<IoBuffer(const IoHash& AttachmentHash)> TGetAttachmentBufferFunc;
-typedef std::function<CompositeBuffer(const IoHash& RawHash)> FetchChunkFunc;
RemoteProjectStore::LoadContainerResult BuildContainer(
CidStore& ChunkStore,
@@ -137,7 +130,7 @@ RemoteProjectStore::LoadContainerResult BuildContainer(
bool BuildBlocks,
bool IgnoreMissingAttachments,
bool AllowChunking,
- const std::function<void(CompressedBuffer&&, RemoteProjectStore::Block&&)>& AsyncOnBlock,
+ const std::function<void(CompressedBuffer&&, ChunkBlockDescription&&)>& AsyncOnBlock,
const std::function<void(const IoHash&, TGetAttachmentBufferFunc&&)>& OnLargeAttachment,
const std::function<void(std::vector<std::pair<IoHash, FetchChunkFunc>>&&)>& OnBlockChunks,
bool EmbedLooseFiles);
@@ -173,9 +166,7 @@ RemoteProjectStore::Result LoadOplog(CidStore& ChunkStore,
bool CleanOplog,
JobContext* OptionalContext);
-CompressedBuffer GenerateBlock(std::vector<std::pair<IoHash, FetchChunkFunc>>&& FetchChunks, RemoteProjectStore::Block& OutBlock);
-bool IterateBlock(const SharedBuffer& BlockPayload, std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor);
std::vector<IoHash> GetBlockHashesFromOplog(CbObjectView ContainerObject);
-std::vector<RemoteProjectStore::Block> GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> IncludeBlockHashes);
+std::vector<ThinChunkBlockDescription> GetBlocksFromOplog(CbObjectView ContainerObject, std::span<const IoHash> IncludeBlockHashes);
} // namespace zen
diff --git a/src/zenserver/projectstore/zenremoteprojectstore.cpp b/src/zenserver/projectstore/zenremoteprojectstore.cpp
index 42519b108..2ebf58a5d 100644
--- a/src/zenserver/projectstore/zenremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/zenremoteprojectstore.cpp
@@ -93,7 +93,7 @@ public:
return Result;
}
- virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&&) override
+ virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&&) override
{
std::string SaveRequest = fmt::format("/{}/oplog/{}/{}"sv, m_Project, m_Oplog, RawHash);
HttpClient::Response Response = m_Client.Post(SaveRequest, Payload, ZenContentType::kCompressedBinary);
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-22 19:10:42 +0000