aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2025-09-04 13:17:25 +0200
committerGitHub Enterprise <[email protected]>2025-09-04 13:17:25 +0200
commit9f575bd416e1f7afbd11d4b221074f34bb89605c (patch)
tree07c87ccdbc01cdaf13015f46dddfaa71fa791d5b /src
parentoplog memory usage reduction (#482) (diff)
downloadzen-9f575bd416e1f7afbd11d4b221074f34bb89605c.tar.xz
zen-9f575bd416e1f7afbd11d4b221074f34bb89605c.zip
add validation of compact binary payloads before reading them (#483)
* add validation of compact binary payloads before reading them
Diffstat (limited to 'src')
-rw-r--r--src/zen/cmds/status_cmd.cpp22
-rw-r--r--src/zen/cmds/up_cmd.cpp43
-rw-r--r--src/zencore/compactbinaryfile.cpp10
-rw-r--r--src/zencore/compactbinaryutil.cpp33
-rw-r--r--src/zencore/include/zencore/compactbinaryutil.h9
-rw-r--r--src/zenhttp/auth/authmgr.cpp13
-rw-r--r--src/zenhttp/httpclient.cpp25
-rw-r--r--src/zenhttp/httpserver.cpp64
-rw-r--r--src/zenhttp/packageformat.cpp47
-rw-r--r--src/zenserver/cache/httpstructuredcache.cpp67
-rw-r--r--src/zenserver/config.cpp12
-rw-r--r--src/zenserver/projectstore/fileremoteprojectstore.cpp10
-rw-r--r--src/zenserver/projectstore/httpprojectstore.cpp204
-rw-r--r--src/zenserver/projectstore/jupiterremoteprojectstore.cpp11
-rw-r--r--src/zenserver/projectstore/projectstore.cpp138
-rw-r--r--src/zenserver/projectstore/remoteprojectstore.cpp23
-rw-r--r--src/zenstore/buildstore/buildstore.cpp38
-rw-r--r--src/zenstore/cache/cacherpc.cpp8
-rw-r--r--src/zenutil/jupiter/jupiterbuildstorage.cpp37
-rw-r--r--src/zenutil/jupiter/jupitersession.cpp11
20 files changed, 536 insertions, 289 deletions
diff --git a/src/zen/cmds/status_cmd.cpp b/src/zen/cmds/status_cmd.cpp
index b5764af44..5b172f1a5 100644
--- a/src/zen/cmds/status_cmd.cpp
+++ b/src/zen/cmds/status_cmd.cpp
@@ -3,6 +3,7 @@
#include "status_cmd.h"
#include <zencore/compactbinary.h>
+#include <zencore/compactbinaryutil.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
#include <zencore/string.h>
@@ -37,14 +38,25 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZEN_CONSOLE_ERROR("Lock file does not exist in directory '{}'", m_DataDir);
return 1;
}
- LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock")));
- std::string Reason;
- if (!ValidateLockFileInfo(Info, Reason))
+ CbValidateError ValidateResult = CbValidateError::None;
+ if (CbObject LockFileObject =
+ ValidateAndReadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"), ValidateResult);
+ ValidateResult == CbValidateError::None)
{
- ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
+ LockFileInfo Info = ReadLockFilePayload(LockFileObject);
+ std::string Reason;
+ if (!ValidateLockFileInfo(Info, Reason))
+ {
+ ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
+ return 1;
+ }
+ EffectivePort = Info.EffectiveListenPort;
+ }
+ else
+ {
+ ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, ToString(ValidateResult));
return 1;
}
- EffectivePort = Info.EffectiveListenPort;
}
ZenServerState State;
diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp
index fd330f616..8deb6c70e 100644
--- a/src/zen/cmds/up_cmd.cpp
+++ b/src/zen/cmds/up_cmd.cpp
@@ -3,6 +3,7 @@
#include "up_cmd.h"
#include <zencore/compactbinary.h>
+#include <zencore/compactbinaryutil.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
@@ -160,14 +161,25 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZEN_CONSOLE("Lock file does not exist in directory '{}'", m_DataDir);
return 1;
}
- LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock")));
- std::string Reason;
- if (!ValidateLockFileInfo(Info, Reason))
+ CbValidateError ValidateResult = CbValidateError::None;
+ if (CbObject LockFileObject =
+ ValidateAndReadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"), ValidateResult);
+ ValidateResult == CbValidateError::None && LockFileObject)
{
- ZEN_CONSOLE("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
+ LockFileInfo Info = ReadLockFilePayload(LockFileObject);
+ std::string Reason;
+ if (!ValidateLockFileInfo(Info, Reason))
+ {
+ ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
+ return 1;
+ }
+ Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort);
+ }
+ else
+ {
+ ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, ToString(ValidateResult));
return 1;
}
- Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort);
}
if (!Entry)
@@ -227,14 +239,25 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZEN_CONSOLE_ERROR("Lock file does not exist in directory '{}'", m_DataDir);
return 1;
}
- LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock")));
- std::string Reason;
- if (!ValidateLockFileInfo(Info, Reason))
+ CbValidateError ValidateResult = CbValidateError::None;
+ if (CbObject LockFileObject =
+ ValidateAndReadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"), ValidateResult);
+ ValidateResult == CbValidateError::None && LockFileObject)
+ {
+ LockFileInfo Info = ReadLockFilePayload(LockFileObject);
+ std::string Reason;
+ if (!ValidateLockFileInfo(Info, Reason))
+ {
+ ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
+ return 1;
+ }
+ Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort);
+ }
+ else
{
- ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason);
+ ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, ToString(ValidateResult));
return 1;
}
- Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort);
}
if (Entry)
diff --git a/src/zencore/compactbinaryfile.cpp b/src/zencore/compactbinaryfile.cpp
index 1526c21d5..ec2fc3cd5 100644
--- a/src/zencore/compactbinaryfile.cpp
+++ b/src/zencore/compactbinaryfile.cpp
@@ -1,7 +1,7 @@
// Copyright Epic Games, Inc. All Rights Reserved.
#include "zencore/compactbinaryfile.h"
-#include "zencore/compactbinaryvalidation.h"
+#include "zencore/compactbinaryutil.h"
#include <zencore/filesystem.h>
@@ -19,12 +19,12 @@ LoadCompactBinaryObject(const std::filesystem::path& FilePath)
IoBuffer ObjectBuffer = ObjectFile.Flatten();
- if (CbValidateError Result = ValidateCompactBinary(ObjectBuffer, CbValidateMode::Default); Result == CbValidateError::None)
+ CbValidateError ValidateResult;
+ CbObject Object = ValidateAndReadCompactBinaryObject(IoBuffer(ObjectBuffer), ValidateResult);
+ if (ValidateResult == CbValidateError::None)
{
- CbObject Object = LoadCompactBinaryObject(ObjectBuffer);
const IoHash WorkerId = IoHash::HashBuffer(ObjectBuffer);
-
- return {.Object = Object, .Hash = WorkerId};
+ return {.Object = std::move(Object), .Hash = WorkerId};
}
return {.Hash = IoHash::Zero};
diff --git a/src/zencore/compactbinaryutil.cpp b/src/zencore/compactbinaryutil.cpp
new file mode 100644
index 000000000..c8cde21c3
--- /dev/null
+++ b/src/zencore/compactbinaryutil.cpp
@@ -0,0 +1,33 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/compactbinaryutil.h>
+
+#include <zencore/compress.h>
+#include <zencore/filesystem.h>
+
+namespace zen {
+
+CbObject
+ValidateAndReadCompactBinaryObject(const SharedBuffer&& Payload, CbValidateError& OutError)
+{
+ if (Payload.GetSize() > 0)
+ {
+ if (OutError = ValidateCompactBinary(Payload.GetView(), CbValidateMode::Default); OutError == CbValidateError::None)
+ {
+ return CbObject(std::move(Payload));
+ }
+ }
+ return CbObject();
+}
+
+CbObject
+ValidateAndReadCompactBinaryObject(const CompressedBuffer&& Payload, CbValidateError& OutError)
+{
+ if (CompositeBuffer Decompressed = Payload.DecompressToComposite())
+ {
+ return ValidateAndReadCompactBinaryObject(std::move(Decompressed).Flatten(), OutError);
+ }
+ return CbObject();
+}
+
+} // namespace zen
diff --git a/src/zencore/include/zencore/compactbinaryutil.h b/src/zencore/include/zencore/compactbinaryutil.h
index 9524d1fc4..d750c6492 100644
--- a/src/zencore/include/zencore/compactbinaryutil.h
+++ b/src/zencore/include/zencore/compactbinaryutil.h
@@ -6,6 +6,7 @@
#include <zencore/compactbinary.h>
#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryvalidation.h>
namespace zen {
@@ -43,4 +44,12 @@ RewriteCbObject(CbObjectView InObj, Invocable<CbObjectWriter&, CbFieldView&> aut
return Writer.Save();
}
+CbObject ValidateAndReadCompactBinaryObject(const SharedBuffer&& Payload, CbValidateError& OutError);
+inline CbObject
+ValidateAndReadCompactBinaryObject(const IoBuffer&& Payload, CbValidateError& OutError)
+{
+ return ValidateAndReadCompactBinaryObject(SharedBuffer(std::move(Payload)), OutError);
+}
+CbObject ValidateAndReadCompactBinaryObject(const CompressedBuffer&& Payload, CbValidateError& OutError);
+
} // namespace zen
diff --git a/src/zenhttp/auth/authmgr.cpp b/src/zenhttp/auth/authmgr.cpp
index 6c1a66a99..209276621 100644
--- a/src/zenhttp/auth/authmgr.cpp
+++ b/src/zenhttp/auth/authmgr.cpp
@@ -5,7 +5,7 @@
#include <zencore/basicfile.h>
#include <zencore/compactbinary.h>
#include <zencore/compactbinarybuilder.h>
-#include <zencore/compactbinaryvalidation.h>
+#include <zencore/compactbinaryutil.h>
#include <zencore/crypto.h>
#include <zencore/filesystem.h>
#include <zencore/logging.h>
@@ -297,15 +297,14 @@ private:
return;
}
- const CbValidateError ValidationError = ValidateCompactBinary(Buffer, CbValidateMode::All);
-
- if (ValidationError != CbValidateError::None)
+ CbValidateError ValidationError;
+ if (CbObject AuthState = ValidateAndReadCompactBinaryObject(std::move(Buffer), ValidationError);
+ ValidationError != CbValidateError::None)
{
- ZEN_WARN("load serialized state FAILED, reason 'Invalid compact binary'");
+ ZEN_WARN("load serialized state FAILED, reason '{}'", ToString(ValidationError));
return;
}
-
- if (CbObject AuthState = LoadCompactBinaryObject(Buffer))
+ else
{
for (CbFieldView ProviderView : AuthState["OpenIdProviders"sv])
{
diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp
index 30a2bfc65..9ee8cc05a 100644
--- a/src/zenhttp/httpclient.cpp
+++ b/src/zenhttp/httpclient.cpp
@@ -7,6 +7,7 @@
#include <zencore/compactbinarybuilder.h>
#include <zencore/compactbinarypackage.h>
+#include <zencore/compactbinaryutil.h>
#include <zencore/compositebuffer.h>
#include <zencore/except.h>
#include <zencore/filesystem.h>
@@ -984,13 +985,16 @@ HttpClient::TransactPackage(std::string_view Url, CbPackage Package, const KeyVa
if (FilterResponse.status_code == 200)
{
- IoBuffer ResponseBuffer(IoBuffer::Wrap, FilterResponse.text.data(), FilterResponse.text.size());
- CbObject ResponseObject = LoadCompactBinaryObject(ResponseBuffer);
-
- for (CbFieldView& Entry : ResponseObject["need"])
+ IoBuffer ResponseBuffer(IoBuffer::Wrap, FilterResponse.text.data(), FilterResponse.text.size());
+ CbValidateError ValidationError = CbValidateError::None;
+ if (CbObject ResponseObject = ValidateAndReadCompactBinaryObject(std::move(ResponseBuffer), ValidationError);
+ ValidationError == CbValidateError::None)
{
- ZEN_ASSERT(Entry.IsHash());
- AttachmentsToSend.push_back(Entry.AsHash());
+ for (CbFieldView& Entry : ResponseObject["need"])
+ {
+ ZEN_ASSERT(Entry.IsHash());
+ AttachmentsToSend.push_back(Entry.AsHash());
+ }
}
}
}
@@ -1550,11 +1554,14 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold
CbObject
HttpClient::Response::AsObject() const
{
- // TODO: sanity check the payload format etc
-
if (ResponsePayload)
{
- return LoadCompactBinaryObject(ResponsePayload);
+ CbValidateError ValidationError = CbValidateError::None;
+ if (CbObject ResponseObject = ValidateAndReadCompactBinaryObject(IoBuffer(ResponsePayload), ValidationError);
+ ValidationError == CbValidateError::None)
+ {
+ return ResponseObject;
+ }
}
return {};
diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp
index 764f2a2a7..2c063d646 100644
--- a/src/zenhttp/httpserver.cpp
+++ b/src/zenhttp/httpserver.cpp
@@ -18,6 +18,7 @@
#include <zencore/compactbinary.h>
#include <zencore/compactbinarybuilder.h>
#include <zencore/compactbinarypackage.h>
+#include <zencore/compactbinaryutil.h>
#include <zencore/iobuffer.h>
#include <zencore/logging.h>
#include <zencore/stream.h>
@@ -665,9 +666,13 @@ HttpServerRequest::ReadPayloadObject()
}
return CbObject();
}
- return LoadCompactBinaryObject(std::move(Payload));
+ CbValidateError ValidationError = CbValidateError::None;
+ if (CbObject ResponseObject = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidationError);
+ ValidationError == CbValidateError::None)
+ {
+ return ResponseObject;
+ }
}
-
return {};
}
@@ -931,42 +936,51 @@ HandlePackageOffers(HttpService& Service, HttpServerRequest& Request, Ref<IHttpP
if (PackageHandlerRef)
{
- CbObject OfferMessage = LoadCompactBinaryObject(Request.ReadPayload());
-
- std::vector<IoHash> OfferCids;
-
- for (auto& CidEntry : OfferMessage["offer"])
+ CbValidateError ValidationError = CbValidateError::None;
+ if (CbObject OfferMessage = ValidateAndReadCompactBinaryObject(IoBuffer(Request.ReadPayload()), ValidationError);
+ ValidationError == CbValidateError::None)
{
- if (!CidEntry.IsHash())
+ std::vector<IoHash> OfferCids;
+
+ for (auto& CidEntry : OfferMessage["offer"])
{
- // Should yield bad request response?
+ if (!CidEntry.IsHash())
+ {
+ // Should yield bad request response?
+
+ ZEN_WARN("found invalid entry in offer");
- ZEN_WARN("found invalid entry in offer");
+ continue;
+ }
- continue;
+ OfferCids.push_back(CidEntry.AsHash());
}
- OfferCids.push_back(CidEntry.AsHash());
- }
+ ZEN_TRACE("request #{} -> filtering offer of {} entries", Request.RequestId(), OfferCids.size());
+
+ PackageHandlerRef->FilterOffer(OfferCids);
- ZEN_TRACE("request #{} -> filtering offer of {} entries", Request.RequestId(), OfferCids.size());
+ ZEN_TRACE("request #{} -> filtered to {} entries", Request.RequestId(), OfferCids.size());
- PackageHandlerRef->FilterOffer(OfferCids);
+ CbObjectWriter ResponseWriter;
+ ResponseWriter.BeginArray("need");
- ZEN_TRACE("request #{} -> filtered to {} entries", Request.RequestId(), OfferCids.size());
+ for (const IoHash& Cid : OfferCids)
+ {
+ ResponseWriter.AddHash(Cid);
+ }
- CbObjectWriter ResponseWriter;
- ResponseWriter.BeginArray("need");
+ ResponseWriter.EndArray();
- for (const IoHash& Cid : OfferCids)
+ // Emit filter response
+ Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save());
+ }
+ else
{
- ResponseWriter.AddHash(Cid);
+ Request.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Invalid request payload: '{}'", ToString(ValidationError)));
}
-
- ResponseWriter.EndArray();
-
- // Emit filter response
- Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save());
return true;
}
}
diff --git a/src/zenhttp/packageformat.cpp b/src/zenhttp/packageformat.cpp
index 0b7848f79..f622b93ea 100644
--- a/src/zenhttp/packageformat.cpp
+++ b/src/zenhttp/packageformat.cpp
@@ -4,6 +4,7 @@
#include <zencore/compactbinarybuilder.h>
#include <zencore/compactbinarypackage.h>
+#include <zencore/compactbinaryutil.h>
#include <zencore/compositebuffer.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
@@ -499,6 +500,8 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
{
if (Entry.Flags & CbAttachmentEntry::kIsObject)
{
+ CbObject AttachmentObject;
+
CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer)));
if (!CompBuf)
{
@@ -509,7 +512,18 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
AttachmentBuffer.GetSize(),
Entry.AttachmentHash)));
}
- CbObject AttachmentObject = LoadCompactBinaryObject(std::move(CompBuf));
+ else
+ {
+ CbValidateError ValidationError = CbValidateError::None;
+ AttachmentObject = ValidateAndReadCompactBinaryObject(std::move(CompBuf), ValidationError);
+ if (ValidationError != CbValidateError::None)
+ {
+ MalformedAttachments.push_back(std::make_pair(
+ i,
+ fmt::format("Invalid format, CbObject for {}. Reason '{}'", Entry.AttachmentHash, ToString(ValidationError))));
+ }
+ }
+
if (i == 0)
{
// First payload is always a compact binary object
@@ -541,7 +555,15 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
{
if (Entry.Flags & CbAttachmentEntry::kIsObject)
{
- CbObject AttachmentObject = LoadCompactBinaryObject(AttachmentBuffer);
+ CbValidateError ValidationError = CbValidateError::None;
+ CbObject AttachmentObject = ValidateAndReadCompactBinaryObject(std::move(AttachmentBuffer), ValidationError);
+ if (ValidationError != CbValidateError::None)
+ {
+ MalformedAttachments.push_back(std::make_pair(
+ i,
+ fmt::format("Invalid format, CbObject for {}. Reason '{}'", Entry.AttachmentHash, ToString(ValidationError))));
+ }
+
if (i == 0)
{
Package.SetObject(AttachmentObject);
@@ -709,7 +731,12 @@ CbPackageReader::Finalize()
{
if (Entry.Flags & CbAttachmentEntry::kIsLocalRef)
{
- m_RootObject = LoadCompactBinaryObject(MarshalLocalChunkReference(AttachmentBuffer));
+ CbValidateError ValidateError = CbValidateError::None;
+ m_RootObject = ValidateAndReadCompactBinaryObject(MarshalLocalChunkReference(AttachmentBuffer), ValidateError);
+ if (ValidateError != CbValidateError::None)
+ {
+ throw std::runtime_error(fmt::format("Root object format is invalid, reason: '{}'", ToString(ValidateError)));
+ }
}
else if (Entry.Flags & CbAttachmentEntry::kIsCompressed)
{
@@ -718,12 +745,22 @@ CbPackageReader::Finalize()
CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer), RawHash, RawSize);
if (RawHash == Entry.AttachmentHash)
{
- m_RootObject = LoadCompactBinaryObject(Compressed);
+ CbValidateError ValidateError = CbValidateError::None;
+ m_RootObject = ValidateAndReadCompactBinaryObject(std::move(Compressed), ValidateError);
+ if (ValidateError != CbValidateError::None)
+ {
+ throw std::runtime_error(fmt::format("Root object format is invalid, reason: '{}'", ToString(ValidateError)));
+ }
}
}
else
{
- m_RootObject = LoadCompactBinaryObject(std::move(AttachmentBuffer));
+ CbValidateError ValidateError = CbValidateError::None;
+ m_RootObject = ValidateAndReadCompactBinaryObject(std::move(AttachmentBuffer), ValidateError);
+ if (ValidateError != CbValidateError::None)
+ {
+ throw std::runtime_error(fmt::format("Root object format is invalid, reason: '{}'", ToString(ValidateError)));
+ }
}
}
else
diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index 68f1c602e..08d0b12a7 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -5,6 +5,7 @@
#include <zencore/compactbinary.h>
#include <zencore/compactbinarybuilder.h>
#include <zencore/compactbinarypackage.h>
+#include <zencore/compactbinaryutil.h>
#include <zencore/compactbinaryvalidation.h>
#include <zencore/compress.h>
#include <zencore/enumflags.h>
@@ -842,45 +843,61 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con
{
if (ContentType == ZenContentType::kCbObject)
{
- CbPackage Package;
- uint32_t MissingCount = 0;
-
- CbObjectView CacheRecord(ClientResultValue.Value.Data());
- CacheRecord.IterateAttachments([this, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) {
- if (SkipData)
- {
- if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash()))
+ CbPackage Package;
+ uint32_t MissingCount = 0;
+ CbValidateError ValidateError = CbValidateError::None;
+ if (CbObject PackageObject = ValidateAndReadCompactBinaryObject(std::move(ClientResultValue.Value), ValidateError);
+ ValidateError == CbValidateError::None)
+ {
+ CbObjectView CacheRecord(ClientResultValue.Value.Data());
+ CacheRecord.IterateAttachments([this, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) {
+ if (SkipData)
{
- MissingCount++;
+ if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash()))
+ {
+ MissingCount++;
+ }
}
- }
- else
- {
- if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash()))
+ else
{
- CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk));
- if (Compressed)
+ if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash()))
{
- Package.AddAttachment(CbAttachment(Compressed, AttachmentHash.AsHash()));
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk));
+ if (Compressed)
+ {
+ Package.AddAttachment(CbAttachment(Compressed, AttachmentHash.AsHash()));
+ }
+ else
+ {
+ ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash.AsHash());
+ MissingCount++;
+ }
}
else
{
- ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash.AsHash());
MissingCount++;
}
}
- else
- {
- MissingCount++;
- }
- }
- });
+ });
- Success = MissingCount == 0 || PartialRecord;
+ Success = MissingCount == 0 || PartialRecord;
+ }
+ else
+ {
+ ZEN_WARN("Invalid compact binary payload returned for {}/{}/{} ({}). Reason: '{}'",
+ Ref.Namespace,
+ Ref.BucketSegment,
+ Ref.HashKey,
+ Ref.ValueContentId,
+ ToString(ValidateError));
+ Success = false;
+ }
if (Success)
{
- Package.SetObject(LoadCompactBinaryObject(ClientResultValue.Value));
+ CbObject PackageObject = LoadCompactBinaryObject(std::move(ClientResultValue.Value));
+
+ Package.SetObject(std::move(PackageObject));
BinaryWriter MemStream;
Package.Save(MemStream);
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index fb2d9b7f4..0397677b9 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -7,6 +7,7 @@
#include <zencore/basicfile.h>
#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryutil.h>
#include <zencore/compactbinaryvalidation.h>
#include <zencore/crypto.h>
#include <zencore/except.h>
@@ -72,13 +73,12 @@ ReadAllCentralManifests(const std::filesystem::path& SystemRoot)
{
try
{
- FileContents FileData = ReadFile(File);
- IoBuffer DataBuffer = FileData.Flatten();
- CbValidateError ValidateError = ValidateCompactBinary(DataBuffer, CbValidateMode::All);
-
- if (ValidateError == CbValidateError::None)
+ FileContents FileData = ReadFile(File);
+ CbValidateError ValidateError;
+ if (CbObject Manifest = ValidateAndReadCompactBinaryObject(FileData.Flatten(), ValidateError);
+ ValidateError == CbValidateError::None)
{
- Manifests.push_back(LoadCompactBinaryObject(DataBuffer));
+ Manifests.emplace_back(std::move(Manifest));
}
else
{
diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp
index 375e44e59..e550222fd 100644
--- a/src/zenserver/projectstore/fileremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp
@@ -2,6 +2,7 @@
#include "fileremoteprojectstore.h"
+#include <zencore/compactbinaryutil.h>
#include <zencore/compress.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
@@ -260,11 +261,14 @@ private:
ContainerPayload = ContainerFile.ReadAll();
}
AddStats(0, ContainerPayload.GetSize(), Timer.GetElapsedTimeUs() * 1000);
- Result.ContainerObject = LoadCompactBinaryObject(ContainerPayload);
- if (!Result.ContainerObject)
+ CbValidateError ValidateResult = CbValidateError::None;
+ if (Result.ContainerObject = ValidateAndReadCompactBinaryObject(std::move(ContainerPayload), ValidateResult);
+ ValidateResult != CbValidateError::None || !Result.ContainerObject)
{
Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError);
- Result.Reason = fmt::format("The file {} is not formatted as a compact binary object", SourcePath.string());
+ Result.Reason = fmt::format("The file {} is not formatted as a compact binary object ('{}')",
+ SourcePath.string(),
+ ToString(ValidateResult));
Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0;
return Result;
}
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index feeec3e37..237dc097e 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -1050,36 +1050,44 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req)
// chunks are not present on this server. This list is then returned in
// the "need" list in the response
- IoBuffer Payload = HttpReq.ReadPayload();
- CbObject RequestObject = LoadCompactBinaryObject(Payload);
+ CbValidateError ValidateResult;
+ if (CbObject RequestObject = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult);
+ ValidateResult == CbValidateError::None)
+ {
+ std::vector<IoHash> NeedList;
- std::vector<IoHash> NeedList;
+ {
+ eastl::fixed_vector<IoHash, 16> ChunkList;
+ CbArrayView HaveList = RequestObject["have"sv].AsArrayView();
+ ChunkList.reserve(HaveList.Num());
+ for (auto& Entry : HaveList)
+ {
+ ChunkList.push_back(Entry.AsHash());
+ }
- {
- eastl::fixed_vector<IoHash, 16> ChunkList;
- CbArrayView HaveList = RequestObject["have"sv].AsArrayView();
- ChunkList.reserve(HaveList.Num());
- for (auto& Entry : HaveList)
+ NeedList = FoundLog->CheckPendingChunkReferences(std::span(begin(ChunkList), end(ChunkList)), std::chrono::minutes(2));
+ }
+
+ CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1);
+ Cbo.BeginArray("need");
{
- ChunkList.push_back(Entry.AsHash());
+ for (const IoHash& Hash : NeedList)
+ {
+ ZEN_DEBUG("prep - NEED: {}", Hash);
+ Cbo << Hash;
+ }
}
+ Cbo.EndArray();
+ CbObject Response = Cbo.Save();
- NeedList = FoundLog->CheckPendingChunkReferences(std::span(begin(ChunkList), end(ChunkList)), std::chrono::minutes(2));
+ return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
}
-
- CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1);
- Cbo.BeginArray("need");
+ else
{
- for (const IoHash& Hash : NeedList)
- {
- ZEN_DEBUG("prep - NEED: {}", Hash);
- Cbo << Hash;
- }
+ return HttpReq.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Invalid compact binary format: '{}'", ToString(ValidateResult)));
}
- Cbo.EndArray();
- CbObject Response = Cbo.Save();
-
- return HttpReq.WriteResponse(HttpResponseCode::OK, Response);
}
void
@@ -1173,7 +1181,9 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver))
{
- if (CbObject Core = LoadCompactBinaryObject(Payload))
+ CbValidateError ValidateResult;
+ if (CbObject Core = ValidateAndReadCompactBinaryObject(IoBuffer(Payload), ValidateResult);
+ ValidateResult == CbValidateError::None && Core)
{
Package.SetObject(Core);
}
@@ -1182,7 +1192,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req)
std::filesystem::path BadPackagePath =
Oplog.TempPath() / "bad_packages"sv / fmt::format("session{}_request{}"sv, HttpReq.SessionId(), HttpReq.RequestId());
- ZEN_WARN("Received malformed package! Saving payload to '{}'", BadPackagePath);
+ ZEN_WARN("Received malformed package ('{}')! Saving payload to '{}'", ToString(ValidateResult), BadPackagePath);
WriteFile(BadPackagePath, Payload);
@@ -1413,15 +1423,18 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req)
switch (Payload.GetContentType())
{
case ZenContentType::kCbObject:
- if (CbObject Object = LoadCompactBinaryObject(Payload))
{
- Package.AddAttachment(CbAttachment(Object));
- }
- else
- {
- // Error - malformed object
-
- ZEN_WARN("malformed object returned for {}", AttachmentHash);
+ CbValidateError ValidateResult;
+ if (CbObject Object = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult);
+ ValidateResult == CbValidateError::None && Object)
+ {
+ Package.AddAttachment(CbAttachment(Object));
+ }
+ else
+ {
+ // Error - malformed object
+ ZEN_WARN("malformed object returned for {} ('{}')", AttachmentHash, ToString(ValidateResult));
+ }
}
break;
@@ -1774,66 +1787,20 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req)
return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
}
- IoBuffer Payload = HttpReq.ReadPayload();
- CbObject Params = LoadCompactBinaryObject(Payload);
- std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`)
- std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`)
- std::filesystem::path ProjectRoot =
- Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`)
- std::filesystem::path ProjectFilePath =
- Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`)
-
- const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId;
- m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath);
-
- ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})",
- ProjectId,
- Root,
- EngineRoot,
- ProjectRoot,
- ProjectFilePath,
- ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : "");
-
- m_ProjectStats.ProjectWriteCount++;
- HttpReq.WriteResponse(HttpResponseCode::Created);
- }
- break;
-
- case HttpVerb::kPut:
- {
- if (!m_ProjectStore->AreDiskWritesAllowed())
+ CbValidateError ValidateResult;
+ if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult);
+ ValidateResult == CbValidateError::None)
{
- return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
- }
+ std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`)
+ std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`)
+ std::filesystem::path ProjectRoot =
+ Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`)
+ std::filesystem::path ProjectFilePath =
+ Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`)
- IoBuffer Payload = HttpReq.ReadPayload();
- CbObject Params = LoadCompactBinaryObject(Payload);
- std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`)
- std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`)
- std::filesystem::path ProjectRoot =
- Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`)
- std::filesystem::path ProjectFilePath =
- Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`)
-
- if (m_ProjectStore->UpdateProject(ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath))
- {
- m_ProjectStats.ProjectWriteCount++;
- ZEN_INFO("updated project (id: '{}', roots: '{}', '{}', '{}', '{}'{})",
- ProjectId,
- Root,
- EngineRoot,
- ProjectRoot,
- ProjectFilePath,
- ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : "");
-
- HttpReq.WriteResponse(HttpResponseCode::OK);
- }
- else
- {
const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId;
m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath);
- m_ProjectStats.ProjectWriteCount++;
ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})",
ProjectId,
Root,
@@ -1842,8 +1809,71 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req)
ProjectFilePath,
ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : "");
+ m_ProjectStats.ProjectWriteCount++;
HttpReq.WriteResponse(HttpResponseCode::Created);
}
+ else
+ {
+ HttpReq.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Malformed compact binary object: '{}'", ToString(ValidateResult)));
+ }
+ }
+ break;
+
+ case HttpVerb::kPut:
+ {
+ if (!m_ProjectStore->AreDiskWritesAllowed())
+ {
+ return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage);
+ }
+ CbValidateError ValidateResult;
+ if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult);
+ ValidateResult == CbValidateError::None)
+ {
+ std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`)
+ std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`)
+ std::filesystem::path ProjectRoot =
+ Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`)
+ std::filesystem::path ProjectFilePath =
+ Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`)
+
+ if (m_ProjectStore->UpdateProject(ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath))
+ {
+ m_ProjectStats.ProjectWriteCount++;
+ ZEN_INFO("updated project (id: '{}', roots: '{}', '{}', '{}', '{}'{})",
+ ProjectId,
+ Root,
+ EngineRoot,
+ ProjectRoot,
+ ProjectFilePath,
+ ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : "");
+
+ HttpReq.WriteResponse(HttpResponseCode::OK);
+ }
+ else
+ {
+ const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId;
+ m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath);
+
+ m_ProjectStats.ProjectWriteCount++;
+ ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})",
+ ProjectId,
+ Root,
+ EngineRoot,
+ ProjectRoot,
+ ProjectFilePath,
+ ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : "");
+
+ HttpReq.WriteResponse(HttpResponseCode::Created);
+ }
+ }
+ else
+ {
+ HttpReq.WriteResponse(HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Malformed compact binary object: '{}'", ToString(ValidateResult)));
+ }
}
break;
diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
index d79ad3cb7..a8d486e2a 100644
--- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
+++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp
@@ -2,6 +2,7 @@
#include "jupiterremoteprojectstore.h"
+#include <zencore/compactbinaryutil.h>
#include <zencore/compress.h>
#include <zencore/fmtutils.h>
@@ -249,8 +250,9 @@ private:
return Result;
}
- CbObject ContainerObject = LoadCompactBinaryObject(GetResult.Response);
- if (!ContainerObject)
+ CbValidateError ValidateResult = CbValidateError::None;
+ if (CbObject ContainerObject = ValidateAndReadCompactBinaryObject(IoBuffer(GetResult.Response), ValidateResult);
+ ValidateResult != CbValidateError::None || !ContainerObject)
{
return LoadContainerResult{
RemoteProjectStore::Result{.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError),
@@ -262,7 +264,10 @@ private:
Key)},
{}};
}
- return LoadContainerResult{ConvertResult(GetResult), std::move(ContainerObject)};
+ else
+ {
+ return LoadContainerResult{ConvertResult(GetResult), std::move(ContainerObject)};
+ }
}
void AddStats(const JupiterResult& Result)
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 56b64cee4..5277d689f 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -5439,67 +5439,70 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie
}
Project->TouchOplog(OplogId);
- CbObject ContainerObject = LoadCompactBinaryObject(Payload);
- if (!ContainerObject)
- {
- return {HttpResponseCode::BadRequest, "Invalid payload format"};
- }
-
- CidStore& ChunkStore = m_CidStore;
- RwLock AttachmentsLock;
- tsl::robin_set<IoHash, IoHash::Hasher> Attachments;
-
- auto HasAttachment = [&ChunkStore](const IoHash& RawHash) { return ChunkStore.ContainsChunk(RawHash); };
- auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector<IoHash>&& ChunkHashes) {
- RwLock::ExclusiveLockScope _(AttachmentsLock);
- if (BlockHash != IoHash::Zero)
- {
- Attachments.insert(BlockHash);
- }
- else
- {
- Attachments.insert(ChunkHashes.begin(), ChunkHashes.end());
- }
- };
- auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) {
- RwLock::ExclusiveLockScope _(AttachmentsLock);
- Attachments.insert(RawHash);
- };
+ CbValidateError ValidateResult;
+ if (CbObject ContainerObject = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult);
+ ValidateResult == CbValidateError::None && ContainerObject)
+ {
+ CidStore& ChunkStore = m_CidStore;
+ RwLock AttachmentsLock;
+ tsl::robin_set<IoHash, IoHash::Hasher> Attachments;
+
+ auto HasAttachment = [&ChunkStore](const IoHash& RawHash) { return ChunkStore.ContainsChunk(RawHash); };
+ auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector<IoHash>&& ChunkHashes) {
+ RwLock::ExclusiveLockScope _(AttachmentsLock);
+ if (BlockHash != IoHash::Zero)
+ {
+ Attachments.insert(BlockHash);
+ }
+ else
+ {
+ Attachments.insert(ChunkHashes.begin(), ChunkHashes.end());
+ }
+ };
+ auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) {
+ RwLock::ExclusiveLockScope _(AttachmentsLock);
+ Attachments.insert(RawHash);
+ };
- auto OnChunkedAttachment = [](const ChunkedInfo&) {};
+ auto OnChunkedAttachment = [](const ChunkedInfo&) {};
- auto OnReferencedAttachments = [&Oplog](std::span<IoHash> RawHashes) { Oplog->CaptureAddedAttachments(RawHashes); };
- // Make sure we retain any attachments we download before writing the oplog
- Oplog->EnableUpdateCapture();
- auto _ = MakeGuard([&Oplog]() { Oplog->DisableUpdateCapture(); });
+ auto OnReferencedAttachments = [&Oplog](std::span<IoHash> RawHashes) { Oplog->CaptureAddedAttachments(RawHashes); };
+ // Make sure we retain any attachments we download before writing the oplog
+ Oplog->EnableUpdateCapture();
+ auto _ = MakeGuard([&Oplog]() { Oplog->DisableUpdateCapture(); });
- RemoteProjectStore::Result RemoteResult = SaveOplogContainer(*Oplog,
- ContainerObject,
- OnReferencedAttachments,
- HasAttachment,
- OnNeedBlock,
- OnNeedAttachment,
- OnChunkedAttachment,
- nullptr);
+ RemoteProjectStore::Result RemoteResult = SaveOplogContainer(*Oplog,
+ ContainerObject,
+ OnReferencedAttachments,
+ HasAttachment,
+ OnNeedBlock,
+ OnNeedAttachment,
+ OnChunkedAttachment,
+ nullptr);
- if (RemoteResult.ErrorCode)
- {
- return ConvertResult(RemoteResult);
- }
+ if (RemoteResult.ErrorCode)
+ {
+ return ConvertResult(RemoteResult);
+ }
- CbObjectWriter Cbo(1 + 1 + 5 + Attachments.size() * (1 + sizeof(IoHash::Hash)) + 1);
- Cbo.BeginArray("need");
- {
- for (const IoHash& Hash : Attachments)
+ CbObjectWriter Cbo(1 + 1 + 5 + Attachments.size() * (1 + sizeof(IoHash::Hash)) + 1);
+ Cbo.BeginArray("need");
{
- ZEN_DEBUG("Need attachment {}", Hash);
- Cbo << Hash;
+ for (const IoHash& Hash : Attachments)
+ {
+ ZEN_DEBUG("Need attachment {}", Hash);
+ Cbo << Hash;
+ }
}
- }
- Cbo.EndArray(); // "need"
+ Cbo.EndArray(); // "need"
- OutResponse = Cbo.Save();
- return {HttpResponseCode::OK, {}};
+ OutResponse = Cbo.Save();
+ return {HttpResponseCode::OK, {}};
+ }
+ else
+ {
+ return {HttpResponseCode::BadRequest, fmt::format("Invalid payload format ('{}')", ToString(ValidateResult))};
+ }
}
std::pair<HttpResponseCode, std::string>
@@ -5604,15 +5607,19 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
}
break;
case HttpContentType::kCbObject:
- Cb = LoadCompactBinaryObject(Payload);
- if (!Cb)
{
- HttpReq.WriteResponse(HttpResponseCode::BadRequest,
- HttpContentType::kText,
- "Content format not supported, expected compact binary format");
- return false;
+ CbValidateError ValidateResult;
+ if (Cb = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult);
+ ValidateResult != CbValidateError::None || !Cb)
+ {
+ HttpReq.WriteResponse(
+ HttpResponseCode::BadRequest,
+ HttpContentType::kText,
+ fmt::format("Content format not supported, expected compact binary format ('{}')", ToString(ValidateResult)));
+ return false;
+ }
+ break;
}
- break;
case HttpContentType::kCbPackage:
try
{
@@ -5917,8 +5924,8 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq,
ResponseObj.EndArray();
}
- // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the new
- // chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have
+ // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the
+ // new chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have
// unreferenced chunks.
for (auto It : AddedChunks)
{
@@ -7768,7 +7775,8 @@ TEST_CASE("project.store.gc.prep")
Project1->DeleteOplog("oplog1"sv);
}
- // Caution - putting breakpoints and stepping through this part of the test likely makes it fails due to expiry time of pending chunks
+ // Caution - putting breakpoints and stepping through this part of the test likely makes it fails due to expiry time of pending
+ // chunks
{
Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv);
Ref<ProjectStore::Oplog> Oplog = Project1->NewOplog("oplog1"sv, Project1OplogPath);
@@ -7790,8 +7798,8 @@ TEST_CASE("project.store.gc.prep")
}
Sleep(200);
- // This pass they should also be retained since our age retention has kept them alive and they will now be picked up and the retention
- // cleared
+ // This pass they should also be retained since our age retention has kept them alive and they will now be picked up and the
+ // retention cleared
{
GcSettings Settings = {.CacheExpireTime = GcClock::Now(),
.ProjectStoreExpireTime = GcClock::Now(),
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index 857e868da..2a81ee3e3 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -2651,19 +2651,22 @@ ParseOplogContainer(const CbObject& ContainerObject,
IoBuffer OpsBuffer(IoBuffer::Wrap, OpsSection.GetData(), OpsSection.GetSize());
IoBuffer SectionPayload = CompressedBuffer::FromCompressedNoValidate(std::move(OpsBuffer)).Decompress().AsIoBuffer();
+ CbValidateError ValidateResult = CbValidateError::None;
+ if (CbObject SectionObject = ValidateAndReadCompactBinaryObject(std::move(SectionPayload), ValidateResult);
+ ValidateResult == CbValidateError::None && ContainerObject)
{
- CbObject SectionObject = LoadCompactBinaryObject(SectionPayload);
- if (!SectionObject)
- {
- remotestore_impl::ReportMessage(OptionalContext,
- fmt::format("Failed to save oplog container: '{}'", "Section has unexpected data type"));
- return RemoteProjectStore::Result{gsl::narrow<int>(HttpResponseCode::BadRequest),
- Timer.GetElapsedTimeMs() / 1000.0,
- "Section has unexpected data type",
- "Failed to save oplog container"};
- }
OutOplogSection = SectionObject;
}
+ else
+ {
+ remotestore_impl::ReportMessage(
+ OptionalContext,
+ fmt::format("Failed to save oplog container: '{}' ('{}')", "Section has unexpected data type", ToString(ValidateResult)));
+ return RemoteProjectStore::Result{gsl::narrow<int>(HttpResponseCode::BadRequest),
+ Timer.GetElapsedTimeMs() / 1000.0,
+ "Section has unexpected data type",
+ "Failed to save oplog container"};
+ }
std::unordered_set<IoHash, IoHash::Hasher> OpsAttachments;
{
CbArrayView OpsArray = OutOplogSection["ops"sv].AsArrayView();
diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp
index 1b2cf036b..d65c2bf06 100644
--- a/src/zenstore/buildstore/buildstore.cpp
+++ b/src/zenstore/buildstore/buildstore.cpp
@@ -3,6 +3,7 @@
#include <zenstore/buildstore/buildstore.h>
#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryutil.h>
#include <zencore/compress.h>
#include <zencore/fmtutils.h>
#include <zencore/logging.h>
@@ -138,24 +139,33 @@ BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc, CidStore&
{
RwLock::ExclusiveLockScope Lock(m_Lock);
- CbObject ManifestReader = LoadCompactBinaryObject(ReadFile(ManifestPath).Flatten());
- Oid ManifestId = ManifestReader["id"].AsObjectId();
- uint32_t Version = ManifestReader["version"].AsUInt32();
- DateTime CreationDate = ManifestReader["createdAt"].AsDateTime();
- ZEN_UNUSED(CreationDate);
- if (ManifestId == Oid::Zero || Version != blobstore::impl::ManifestVersion)
+ CbValidateError ValidateResult = CbValidateError::None;
+ if (CbObject ManifestReader = ValidateAndReadCompactBinaryObject(ReadFile(ManifestPath).Flatten(), ValidateResult);
+ ValidateResult == CbValidateError::None && ManifestReader)
{
- ZEN_WARN("Invalid manifest at {}, wiping state", ManifestPath);
- IsNew = true;
+ Oid ManifestId = ManifestReader["id"].AsObjectId();
+ uint32_t Version = ManifestReader["version"].AsUInt32();
+ DateTime CreationDate = ManifestReader["createdAt"].AsDateTime();
+ ZEN_UNUSED(CreationDate);
+ if (ManifestId == Oid::Zero || Version != blobstore::impl::ManifestVersion)
+ {
+ ZEN_WARN("Invalid manifest at {}, wiping state", ManifestPath);
+ IsNew = true;
+ }
+ else
+ {
+ m_BlobLogFlushPosition = ReadPayloadLog(Lock, BlobLogPath, 0);
+ m_MetaLogFlushPosition = ReadMetadataLog(Lock, MetaLogPath, 0);
+ if (IsFile(AccessTimesPath))
+ {
+ ReadAccessTimes(Lock, AccessTimesPath);
+ }
+ }
}
else
{
- m_BlobLogFlushPosition = ReadPayloadLog(Lock, BlobLogPath, 0);
- m_MetaLogFlushPosition = ReadMetadataLog(Lock, MetaLogPath, 0);
- if (IsFile(AccessTimesPath))
- {
- ReadAccessTimes(Lock, AccessTimesPath);
- }
+ ZEN_WARN("Invalid manifest at {} ('{}'), wiping state", ManifestPath, ToString(ValidateResult));
+ IsNew = true;
}
}
diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp
index 5d9a68919..83301f863 100644
--- a/src/zenstore/cache/cacherpc.cpp
+++ b/src/zenstore/cache/cacherpc.cpp
@@ -190,9 +190,8 @@ CacheRpcHandler::HandleRpcRequest(const CacheRequestContext& Context,
m_CacheStats.RpcRequests.fetch_add(1);
- CbPackage Package;
- CbObjectView Object;
- CbObject ObjectBuffer;
+ CbPackage Package;
+ CbObject Object;
try
{
if (ContentType == ZenContentType::kCbObject)
@@ -203,8 +202,7 @@ CacheRpcHandler::HandleRpcRequest(const CacheRequestContext& Context,
return RpcResponseCode::BadRequest;
}
- ObjectBuffer = LoadCompactBinaryObject(std::move(Body));
- Object = ObjectBuffer;
+ Object = LoadCompactBinaryObject(std::move(Body));
if (!Object)
{
ZEN_WARN("Content format not supported, expected compact binary format");
diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp
index c9278acb4..386a91cb3 100644
--- a/src/zenutil/jupiter/jupiterbuildstorage.cpp
+++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp
@@ -3,6 +3,7 @@
#include <zenutil/jupiter/jupiterbuildstorage.h>
#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryutil.h>
#include <zencore/fmtutils.h>
#include <zencore/scopeguard.h>
#include <zencore/timer.h>
@@ -430,13 +431,41 @@ private:
}
else if (Payload.GetContentType() == ZenContentType::kCbObject)
{
- return LoadCompactBinaryObject(Payload);
+ CbValidateError ValidateResult = CbValidateError::None;
+ if (CbObject Object = ValidateAndReadCompactBinaryObject(IoBuffer(Payload), ValidateResult);
+ ValidateResult == CbValidateError::None)
+ {
+ return Object;
+ }
+ else
+ {
+ throw std::runtime_error(fmt::format("{}: {} ({})",
+ "Invalid compact binary object: '{}'",
+ ErrorContext,
+ ToString(Payload.GetContentType()),
+ ToString(ValidateResult)));
+ }
}
else if (Payload.GetContentType() == ZenContentType::kCompressedBinary)
{
- IoHash RawHash;
- uint64_t RawSize;
- return LoadCompactBinaryObject(CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize));
+ IoHash RawHash;
+ uint64_t RawSize;
+ CbValidateError ValidateResult = CbValidateError::None;
+ if (CbObject Object =
+ ValidateAndReadCompactBinaryObject(CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize),
+ ValidateResult);
+ ValidateResult == CbValidateError::None)
+ {
+ return Object;
+ }
+ else
+ {
+ throw std::runtime_error(fmt::format("{}: {} ({})",
+ "Invalid compresed compact binary object: '{}'",
+ ErrorContext,
+ ToString(Payload.GetContentType()),
+ ToString(ValidateResult)));
+ }
}
else
{
diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp
index 1fd59acdf..c305dc477 100644
--- a/src/zenutil/jupiter/jupitersession.cpp
+++ b/src/zenutil/jupiter/jupitersession.cpp
@@ -4,6 +4,7 @@
#include <zencore/compactbinary.h>
#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinaryutil.h>
#include <zencore/compositebuffer.h>
#include <zencore/compress.h>
#include <zencore/fmtutils.h>
@@ -526,7 +527,15 @@ JupiterSession::PutMultipartBuildBlob(std::string_view Namespace,
ZEN_WARN("{}", StartMultipartResponse.ErrorMessage("startMultipartUpload: "));
return detail::ConvertResponse(StartMultipartResponse, "JupiterSession::PutMultipartBuildBlob"sv);
}
- CbObject ResponseObject = LoadCompactBinaryObject(StartMultipartResponse.ResponsePayload);
+ CbValidateError ValidateResult = CbValidateError::None;
+ CbObject ResponseObject = ValidateAndReadCompactBinaryObject(IoBuffer(StartMultipartResponse.ResponsePayload), ValidateResult);
+ if (ValidateResult != CbValidateError::None)
+ {
+ JupiterResult Result = detail::ConvertResponse(StartMultipartResponse, "JupiterSession::PutMultipartBuildBlob"sv);
+ Result.ErrorCode = (int32)HttpResponseCode::UnsupportedMediaType;
+ Result.Reason = fmt::format("Invalid multipart response object format: '{}'", ToString(ValidateResult));
+ return Result;
+ }
struct WorkloadData
{