diff options
| author | Dan Engelbrecht <[email protected]> | 2025-09-04 13:17:25 +0200 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-09-04 13:17:25 +0200 |
| commit | 9f575bd416e1f7afbd11d4b221074f34bb89605c (patch) | |
| tree | 07c87ccdbc01cdaf13015f46dddfaa71fa791d5b /src | |
| parent | oplog memory usage reduction (#482) (diff) | |
| download | zen-9f575bd416e1f7afbd11d4b221074f34bb89605c.tar.xz zen-9f575bd416e1f7afbd11d4b221074f34bb89605c.zip | |
add validation of compact binary payloads before reading them (#483)
* add validation of compact binary payloads before reading them
Diffstat (limited to 'src')
| -rw-r--r-- | src/zen/cmds/status_cmd.cpp | 22 | ||||
| -rw-r--r-- | src/zen/cmds/up_cmd.cpp | 43 | ||||
| -rw-r--r-- | src/zencore/compactbinaryfile.cpp | 10 | ||||
| -rw-r--r-- | src/zencore/compactbinaryutil.cpp | 33 | ||||
| -rw-r--r-- | src/zencore/include/zencore/compactbinaryutil.h | 9 | ||||
| -rw-r--r-- | src/zenhttp/auth/authmgr.cpp | 13 | ||||
| -rw-r--r-- | src/zenhttp/httpclient.cpp | 25 | ||||
| -rw-r--r-- | src/zenhttp/httpserver.cpp | 64 | ||||
| -rw-r--r-- | src/zenhttp/packageformat.cpp | 47 | ||||
| -rw-r--r-- | src/zenserver/cache/httpstructuredcache.cpp | 67 | ||||
| -rw-r--r-- | src/zenserver/config.cpp | 12 | ||||
| -rw-r--r-- | src/zenserver/projectstore/fileremoteprojectstore.cpp | 10 | ||||
| -rw-r--r-- | src/zenserver/projectstore/httpprojectstore.cpp | 204 | ||||
| -rw-r--r-- | src/zenserver/projectstore/jupiterremoteprojectstore.cpp | 11 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.cpp | 138 | ||||
| -rw-r--r-- | src/zenserver/projectstore/remoteprojectstore.cpp | 23 | ||||
| -rw-r--r-- | src/zenstore/buildstore/buildstore.cpp | 38 | ||||
| -rw-r--r-- | src/zenstore/cache/cacherpc.cpp | 8 | ||||
| -rw-r--r-- | src/zenutil/jupiter/jupiterbuildstorage.cpp | 37 | ||||
| -rw-r--r-- | src/zenutil/jupiter/jupitersession.cpp | 11 |
20 files changed, 536 insertions, 289 deletions
diff --git a/src/zen/cmds/status_cmd.cpp b/src/zen/cmds/status_cmd.cpp index b5764af44..5b172f1a5 100644 --- a/src/zen/cmds/status_cmd.cpp +++ b/src/zen/cmds/status_cmd.cpp @@ -3,6 +3,7 @@ #include "status_cmd.h" #include <zencore/compactbinary.h> +#include <zencore/compactbinaryutil.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> #include <zencore/string.h> @@ -37,14 +38,25 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE_ERROR("Lock file does not exist in directory '{}'", m_DataDir); return 1; } - LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); - std::string Reason; - if (!ValidateLockFileInfo(Info, Reason)) + CbValidateError ValidateResult = CbValidateError::None; + if (CbObject LockFileObject = + ValidateAndReadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"), ValidateResult); + ValidateResult == CbValidateError::None) { - ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); + LockFileInfo Info = ReadLockFilePayload(LockFileObject); + std::string Reason; + if (!ValidateLockFileInfo(Info, Reason)) + { + ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); + return 1; + } + EffectivePort = Info.EffectiveListenPort; + } + else + { + ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, ToString(ValidateResult)); return 1; } - EffectivePort = Info.EffectiveListenPort; } ZenServerState State; diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp index fd330f616..8deb6c70e 100644 --- a/src/zen/cmds/up_cmd.cpp +++ b/src/zen/cmds/up_cmd.cpp @@ -3,6 +3,7 @@ #include "up_cmd.h" #include <zencore/compactbinary.h> +#include <zencore/compactbinaryutil.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> @@ -160,14 +161,25 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("Lock file does not exist in directory '{}'", m_DataDir); return 1; } - LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); - std::string Reason; - if (!ValidateLockFileInfo(Info, Reason)) + CbValidateError ValidateResult = CbValidateError::None; + if (CbObject LockFileObject = + ValidateAndReadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"), ValidateResult); + ValidateResult == CbValidateError::None && LockFileObject) { - ZEN_CONSOLE("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); + LockFileInfo Info = ReadLockFilePayload(LockFileObject); + std::string Reason; + if (!ValidateLockFileInfo(Info, Reason)) + { + ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); + return 1; + } + Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort); + } + else + { + ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, ToString(ValidateResult)); return 1; } - Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort); } if (!Entry) @@ -227,14 +239,25 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE_ERROR("Lock file does not exist in directory '{}'", m_DataDir); return 1; } - LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); - std::string Reason; - if (!ValidateLockFileInfo(Info, Reason)) + CbValidateError ValidateResult = CbValidateError::None; + if (CbObject LockFileObject = + ValidateAndReadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"), ValidateResult); + ValidateResult == CbValidateError::None && LockFileObject) + { + LockFileInfo Info = ReadLockFilePayload(LockFileObject); + std::string Reason; + if (!ValidateLockFileInfo(Info, Reason)) + { + ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); + return 1; + } + Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort); + } + else { - ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); + ZEN_CONSOLE_ERROR("Lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, ToString(ValidateResult)); return 1; } - Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort); } if (Entry) diff --git a/src/zencore/compactbinaryfile.cpp b/src/zencore/compactbinaryfile.cpp index 1526c21d5..ec2fc3cd5 100644 --- a/src/zencore/compactbinaryfile.cpp +++ b/src/zencore/compactbinaryfile.cpp @@ -1,7 +1,7 @@ // Copyright Epic Games, Inc. All Rights Reserved. #include "zencore/compactbinaryfile.h" -#include "zencore/compactbinaryvalidation.h" +#include "zencore/compactbinaryutil.h" #include <zencore/filesystem.h> @@ -19,12 +19,12 @@ LoadCompactBinaryObject(const std::filesystem::path& FilePath) IoBuffer ObjectBuffer = ObjectFile.Flatten(); - if (CbValidateError Result = ValidateCompactBinary(ObjectBuffer, CbValidateMode::Default); Result == CbValidateError::None) + CbValidateError ValidateResult; + CbObject Object = ValidateAndReadCompactBinaryObject(IoBuffer(ObjectBuffer), ValidateResult); + if (ValidateResult == CbValidateError::None) { - CbObject Object = LoadCompactBinaryObject(ObjectBuffer); const IoHash WorkerId = IoHash::HashBuffer(ObjectBuffer); - - return {.Object = Object, .Hash = WorkerId}; + return {.Object = std::move(Object), .Hash = WorkerId}; } return {.Hash = IoHash::Zero}; diff --git a/src/zencore/compactbinaryutil.cpp b/src/zencore/compactbinaryutil.cpp new file mode 100644 index 000000000..c8cde21c3 --- /dev/null +++ b/src/zencore/compactbinaryutil.cpp @@ -0,0 +1,33 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/compactbinaryutil.h> + +#include <zencore/compress.h> +#include <zencore/filesystem.h> + +namespace zen { + +CbObject +ValidateAndReadCompactBinaryObject(const SharedBuffer&& Payload, CbValidateError& OutError) +{ + if (Payload.GetSize() > 0) + { + if (OutError = ValidateCompactBinary(Payload.GetView(), CbValidateMode::Default); OutError == CbValidateError::None) + { + return CbObject(std::move(Payload)); + } + } + return CbObject(); +} + +CbObject +ValidateAndReadCompactBinaryObject(const CompressedBuffer&& Payload, CbValidateError& OutError) +{ + if (CompositeBuffer Decompressed = Payload.DecompressToComposite()) + { + return ValidateAndReadCompactBinaryObject(std::move(Decompressed).Flatten(), OutError); + } + return CbObject(); +} + +} // namespace zen diff --git a/src/zencore/include/zencore/compactbinaryutil.h b/src/zencore/include/zencore/compactbinaryutil.h index 9524d1fc4..d750c6492 100644 --- a/src/zencore/include/zencore/compactbinaryutil.h +++ b/src/zencore/include/zencore/compactbinaryutil.h @@ -6,6 +6,7 @@ #include <zencore/compactbinary.h> #include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryvalidation.h> namespace zen { @@ -43,4 +44,12 @@ RewriteCbObject(CbObjectView InObj, Invocable<CbObjectWriter&, CbFieldView&> aut return Writer.Save(); } +CbObject ValidateAndReadCompactBinaryObject(const SharedBuffer&& Payload, CbValidateError& OutError); +inline CbObject +ValidateAndReadCompactBinaryObject(const IoBuffer&& Payload, CbValidateError& OutError) +{ + return ValidateAndReadCompactBinaryObject(SharedBuffer(std::move(Payload)), OutError); +} +CbObject ValidateAndReadCompactBinaryObject(const CompressedBuffer&& Payload, CbValidateError& OutError); + } // namespace zen diff --git a/src/zenhttp/auth/authmgr.cpp b/src/zenhttp/auth/authmgr.cpp index 6c1a66a99..209276621 100644 --- a/src/zenhttp/auth/authmgr.cpp +++ b/src/zenhttp/auth/authmgr.cpp @@ -5,7 +5,7 @@ #include <zencore/basicfile.h> #include <zencore/compactbinary.h> #include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinaryvalidation.h> +#include <zencore/compactbinaryutil.h> #include <zencore/crypto.h> #include <zencore/filesystem.h> #include <zencore/logging.h> @@ -297,15 +297,14 @@ private: return; } - const CbValidateError ValidationError = ValidateCompactBinary(Buffer, CbValidateMode::All); - - if (ValidationError != CbValidateError::None) + CbValidateError ValidationError; + if (CbObject AuthState = ValidateAndReadCompactBinaryObject(std::move(Buffer), ValidationError); + ValidationError != CbValidateError::None) { - ZEN_WARN("load serialized state FAILED, reason 'Invalid compact binary'"); + ZEN_WARN("load serialized state FAILED, reason '{}'", ToString(ValidationError)); return; } - - if (CbObject AuthState = LoadCompactBinaryObject(Buffer)) + else { for (CbFieldView ProviderView : AuthState["OpenIdProviders"sv]) { diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 30a2bfc65..9ee8cc05a 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -7,6 +7,7 @@ #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinarypackage.h> +#include <zencore/compactbinaryutil.h> #include <zencore/compositebuffer.h> #include <zencore/except.h> #include <zencore/filesystem.h> @@ -984,13 +985,16 @@ HttpClient::TransactPackage(std::string_view Url, CbPackage Package, const KeyVa if (FilterResponse.status_code == 200) { - IoBuffer ResponseBuffer(IoBuffer::Wrap, FilterResponse.text.data(), FilterResponse.text.size()); - CbObject ResponseObject = LoadCompactBinaryObject(ResponseBuffer); - - for (CbFieldView& Entry : ResponseObject["need"]) + IoBuffer ResponseBuffer(IoBuffer::Wrap, FilterResponse.text.data(), FilterResponse.text.size()); + CbValidateError ValidationError = CbValidateError::None; + if (CbObject ResponseObject = ValidateAndReadCompactBinaryObject(std::move(ResponseBuffer), ValidationError); + ValidationError == CbValidateError::None) { - ZEN_ASSERT(Entry.IsHash()); - AttachmentsToSend.push_back(Entry.AsHash()); + for (CbFieldView& Entry : ResponseObject["need"]) + { + ZEN_ASSERT(Entry.IsHash()); + AttachmentsToSend.push_back(Entry.AsHash()); + } } } } @@ -1550,11 +1554,14 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold CbObject HttpClient::Response::AsObject() const { - // TODO: sanity check the payload format etc - if (ResponsePayload) { - return LoadCompactBinaryObject(ResponsePayload); + CbValidateError ValidationError = CbValidateError::None; + if (CbObject ResponseObject = ValidateAndReadCompactBinaryObject(IoBuffer(ResponsePayload), ValidationError); + ValidationError == CbValidateError::None) + { + return ResponseObject; + } } return {}; diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 764f2a2a7..2c063d646 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -18,6 +18,7 @@ #include <zencore/compactbinary.h> #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinarypackage.h> +#include <zencore/compactbinaryutil.h> #include <zencore/iobuffer.h> #include <zencore/logging.h> #include <zencore/stream.h> @@ -665,9 +666,13 @@ HttpServerRequest::ReadPayloadObject() } return CbObject(); } - return LoadCompactBinaryObject(std::move(Payload)); + CbValidateError ValidationError = CbValidateError::None; + if (CbObject ResponseObject = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidationError); + ValidationError == CbValidateError::None) + { + return ResponseObject; + } } - return {}; } @@ -931,42 +936,51 @@ HandlePackageOffers(HttpService& Service, HttpServerRequest& Request, Ref<IHttpP if (PackageHandlerRef) { - CbObject OfferMessage = LoadCompactBinaryObject(Request.ReadPayload()); - - std::vector<IoHash> OfferCids; - - for (auto& CidEntry : OfferMessage["offer"]) + CbValidateError ValidationError = CbValidateError::None; + if (CbObject OfferMessage = ValidateAndReadCompactBinaryObject(IoBuffer(Request.ReadPayload()), ValidationError); + ValidationError == CbValidateError::None) { - if (!CidEntry.IsHash()) + std::vector<IoHash> OfferCids; + + for (auto& CidEntry : OfferMessage["offer"]) { - // Should yield bad request response? + if (!CidEntry.IsHash()) + { + // Should yield bad request response? + + ZEN_WARN("found invalid entry in offer"); - ZEN_WARN("found invalid entry in offer"); + continue; + } - continue; + OfferCids.push_back(CidEntry.AsHash()); } - OfferCids.push_back(CidEntry.AsHash()); - } + ZEN_TRACE("request #{} -> filtering offer of {} entries", Request.RequestId(), OfferCids.size()); + + PackageHandlerRef->FilterOffer(OfferCids); - ZEN_TRACE("request #{} -> filtering offer of {} entries", Request.RequestId(), OfferCids.size()); + ZEN_TRACE("request #{} -> filtered to {} entries", Request.RequestId(), OfferCids.size()); - PackageHandlerRef->FilterOffer(OfferCids); + CbObjectWriter ResponseWriter; + ResponseWriter.BeginArray("need"); - ZEN_TRACE("request #{} -> filtered to {} entries", Request.RequestId(), OfferCids.size()); + for (const IoHash& Cid : OfferCids) + { + ResponseWriter.AddHash(Cid); + } - CbObjectWriter ResponseWriter; - ResponseWriter.BeginArray("need"); + ResponseWriter.EndArray(); - for (const IoHash& Cid : OfferCids) + // Emit filter response + Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); + } + else { - ResponseWriter.AddHash(Cid); + Request.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid request payload: '{}'", ToString(ValidationError))); } - - ResponseWriter.EndArray(); - - // Emit filter response - Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); return true; } } diff --git a/src/zenhttp/packageformat.cpp b/src/zenhttp/packageformat.cpp index 0b7848f79..f622b93ea 100644 --- a/src/zenhttp/packageformat.cpp +++ b/src/zenhttp/packageformat.cpp @@ -4,6 +4,7 @@ #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinarypackage.h> +#include <zencore/compactbinaryutil.h> #include <zencore/compositebuffer.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> @@ -499,6 +500,8 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint { if (Entry.Flags & CbAttachmentEntry::kIsObject) { + CbObject AttachmentObject; + CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer))); if (!CompBuf) { @@ -509,7 +512,18 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint AttachmentBuffer.GetSize(), Entry.AttachmentHash))); } - CbObject AttachmentObject = LoadCompactBinaryObject(std::move(CompBuf)); + else + { + CbValidateError ValidationError = CbValidateError::None; + AttachmentObject = ValidateAndReadCompactBinaryObject(std::move(CompBuf), ValidationError); + if (ValidationError != CbValidateError::None) + { + MalformedAttachments.push_back(std::make_pair( + i, + fmt::format("Invalid format, CbObject for {}. Reason '{}'", Entry.AttachmentHash, ToString(ValidationError)))); + } + } + if (i == 0) { // First payload is always a compact binary object @@ -541,7 +555,15 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint { if (Entry.Flags & CbAttachmentEntry::kIsObject) { - CbObject AttachmentObject = LoadCompactBinaryObject(AttachmentBuffer); + CbValidateError ValidationError = CbValidateError::None; + CbObject AttachmentObject = ValidateAndReadCompactBinaryObject(std::move(AttachmentBuffer), ValidationError); + if (ValidationError != CbValidateError::None) + { + MalformedAttachments.push_back(std::make_pair( + i, + fmt::format("Invalid format, CbObject for {}. Reason '{}'", Entry.AttachmentHash, ToString(ValidationError)))); + } + if (i == 0) { Package.SetObject(AttachmentObject); @@ -709,7 +731,12 @@ CbPackageReader::Finalize() { if (Entry.Flags & CbAttachmentEntry::kIsLocalRef) { - m_RootObject = LoadCompactBinaryObject(MarshalLocalChunkReference(AttachmentBuffer)); + CbValidateError ValidateError = CbValidateError::None; + m_RootObject = ValidateAndReadCompactBinaryObject(MarshalLocalChunkReference(AttachmentBuffer), ValidateError); + if (ValidateError != CbValidateError::None) + { + throw std::runtime_error(fmt::format("Root object format is invalid, reason: '{}'", ToString(ValidateError))); + } } else if (Entry.Flags & CbAttachmentEntry::kIsCompressed) { @@ -718,12 +745,22 @@ CbPackageReader::Finalize() CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer), RawHash, RawSize); if (RawHash == Entry.AttachmentHash) { - m_RootObject = LoadCompactBinaryObject(Compressed); + CbValidateError ValidateError = CbValidateError::None; + m_RootObject = ValidateAndReadCompactBinaryObject(std::move(Compressed), ValidateError); + if (ValidateError != CbValidateError::None) + { + throw std::runtime_error(fmt::format("Root object format is invalid, reason: '{}'", ToString(ValidateError))); + } } } else { - m_RootObject = LoadCompactBinaryObject(std::move(AttachmentBuffer)); + CbValidateError ValidateError = CbValidateError::None; + m_RootObject = ValidateAndReadCompactBinaryObject(std::move(AttachmentBuffer), ValidateError); + if (ValidateError != CbValidateError::None) + { + throw std::runtime_error(fmt::format("Root object format is invalid, reason: '{}'", ToString(ValidateError))); + } } } else diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp index 68f1c602e..08d0b12a7 100644 --- a/src/zenserver/cache/httpstructuredcache.cpp +++ b/src/zenserver/cache/httpstructuredcache.cpp @@ -5,6 +5,7 @@ #include <zencore/compactbinary.h> #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinarypackage.h> +#include <zencore/compactbinaryutil.h> #include <zencore/compactbinaryvalidation.h> #include <zencore/compress.h> #include <zencore/enumflags.h> @@ -842,45 +843,61 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con { if (ContentType == ZenContentType::kCbObject) { - CbPackage Package; - uint32_t MissingCount = 0; - - CbObjectView CacheRecord(ClientResultValue.Value.Data()); - CacheRecord.IterateAttachments([this, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) { - if (SkipData) - { - if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash())) + CbPackage Package; + uint32_t MissingCount = 0; + CbValidateError ValidateError = CbValidateError::None; + if (CbObject PackageObject = ValidateAndReadCompactBinaryObject(std::move(ClientResultValue.Value), ValidateError); + ValidateError == CbValidateError::None) + { + CbObjectView CacheRecord(ClientResultValue.Value.Data()); + CacheRecord.IterateAttachments([this, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) { + if (SkipData) { - MissingCount++; + if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash())) + { + MissingCount++; + } } - } - else - { - if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) + else { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)); - if (Compressed) + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) { - Package.AddAttachment(CbAttachment(Compressed, AttachmentHash.AsHash())); + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)); + if (Compressed) + { + Package.AddAttachment(CbAttachment(Compressed, AttachmentHash.AsHash())); + } + else + { + ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash.AsHash()); + MissingCount++; + } } else { - ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash.AsHash()); MissingCount++; } } - else - { - MissingCount++; - } - } - }); + }); - Success = MissingCount == 0 || PartialRecord; + Success = MissingCount == 0 || PartialRecord; + } + else + { + ZEN_WARN("Invalid compact binary payload returned for {}/{}/{} ({}). Reason: '{}'", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + Ref.ValueContentId, + ToString(ValidateError)); + Success = false; + } if (Success) { - Package.SetObject(LoadCompactBinaryObject(ClientResultValue.Value)); + CbObject PackageObject = LoadCompactBinaryObject(std::move(ClientResultValue.Value)); + + Package.SetObject(std::move(PackageObject)); BinaryWriter MemStream; Package.Save(MemStream); diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index fb2d9b7f4..0397677b9 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -7,6 +7,7 @@ #include <zencore/basicfile.h> #include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryutil.h> #include <zencore/compactbinaryvalidation.h> #include <zencore/crypto.h> #include <zencore/except.h> @@ -72,13 +73,12 @@ ReadAllCentralManifests(const std::filesystem::path& SystemRoot) { try { - FileContents FileData = ReadFile(File); - IoBuffer DataBuffer = FileData.Flatten(); - CbValidateError ValidateError = ValidateCompactBinary(DataBuffer, CbValidateMode::All); - - if (ValidateError == CbValidateError::None) + FileContents FileData = ReadFile(File); + CbValidateError ValidateError; + if (CbObject Manifest = ValidateAndReadCompactBinaryObject(FileData.Flatten(), ValidateError); + ValidateError == CbValidateError::None) { - Manifests.push_back(LoadCompactBinaryObject(DataBuffer)); + Manifests.emplace_back(std::move(Manifest)); } else { diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp index 375e44e59..e550222fd 100644 --- a/src/zenserver/projectstore/fileremoteprojectstore.cpp +++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp @@ -2,6 +2,7 @@ #include "fileremoteprojectstore.h" +#include <zencore/compactbinaryutil.h> #include <zencore/compress.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> @@ -260,11 +261,14 @@ private: ContainerPayload = ContainerFile.ReadAll(); } AddStats(0, ContainerPayload.GetSize(), Timer.GetElapsedTimeUs() * 1000); - Result.ContainerObject = LoadCompactBinaryObject(ContainerPayload); - if (!Result.ContainerObject) + CbValidateError ValidateResult = CbValidateError::None; + if (Result.ContainerObject = ValidateAndReadCompactBinaryObject(std::move(ContainerPayload), ValidateResult); + ValidateResult != CbValidateError::None || !Result.ContainerObject) { Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("The file {} is not formatted as a compact binary object", SourcePath.string()); + Result.Reason = fmt::format("The file {} is not formatted as a compact binary object ('{}')", + SourcePath.string(), + ToString(ValidateResult)); Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; return Result; } diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index feeec3e37..237dc097e 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -1050,36 +1050,44 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req) // chunks are not present on this server. This list is then returned in // the "need" list in the response - IoBuffer Payload = HttpReq.ReadPayload(); - CbObject RequestObject = LoadCompactBinaryObject(Payload); + CbValidateError ValidateResult; + if (CbObject RequestObject = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); + ValidateResult == CbValidateError::None) + { + std::vector<IoHash> NeedList; - std::vector<IoHash> NeedList; + { + eastl::fixed_vector<IoHash, 16> ChunkList; + CbArrayView HaveList = RequestObject["have"sv].AsArrayView(); + ChunkList.reserve(HaveList.Num()); + for (auto& Entry : HaveList) + { + ChunkList.push_back(Entry.AsHash()); + } - { - eastl::fixed_vector<IoHash, 16> ChunkList; - CbArrayView HaveList = RequestObject["have"sv].AsArrayView(); - ChunkList.reserve(HaveList.Num()); - for (auto& Entry : HaveList) + NeedList = FoundLog->CheckPendingChunkReferences(std::span(begin(ChunkList), end(ChunkList)), std::chrono::minutes(2)); + } + + CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1); + Cbo.BeginArray("need"); { - ChunkList.push_back(Entry.AsHash()); + for (const IoHash& Hash : NeedList) + { + ZEN_DEBUG("prep - NEED: {}", Hash); + Cbo << Hash; + } } + Cbo.EndArray(); + CbObject Response = Cbo.Save(); - NeedList = FoundLog->CheckPendingChunkReferences(std::span(begin(ChunkList), end(ChunkList)), std::chrono::minutes(2)); + return HttpReq.WriteResponse(HttpResponseCode::OK, Response); } - - CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1); - Cbo.BeginArray("need"); + else { - for (const IoHash& Hash : NeedList) - { - ZEN_DEBUG("prep - NEED: {}", Hash); - Cbo << Hash; - } + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid compact binary format: '{}'", ToString(ValidateResult))); } - Cbo.EndArray(); - CbObject Response = Cbo.Save(); - - return HttpReq.WriteResponse(HttpResponseCode::OK, Response); } void @@ -1173,7 +1181,9 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver)) { - if (CbObject Core = LoadCompactBinaryObject(Payload)) + CbValidateError ValidateResult; + if (CbObject Core = ValidateAndReadCompactBinaryObject(IoBuffer(Payload), ValidateResult); + ValidateResult == CbValidateError::None && Core) { Package.SetObject(Core); } @@ -1182,7 +1192,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) std::filesystem::path BadPackagePath = Oplog.TempPath() / "bad_packages"sv / fmt::format("session{}_request{}"sv, HttpReq.SessionId(), HttpReq.RequestId()); - ZEN_WARN("Received malformed package! Saving payload to '{}'", BadPackagePath); + ZEN_WARN("Received malformed package ('{}')! Saving payload to '{}'", ToString(ValidateResult), BadPackagePath); WriteFile(BadPackagePath, Payload); @@ -1413,15 +1423,18 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) switch (Payload.GetContentType()) { case ZenContentType::kCbObject: - if (CbObject Object = LoadCompactBinaryObject(Payload)) { - Package.AddAttachment(CbAttachment(Object)); - } - else - { - // Error - malformed object - - ZEN_WARN("malformed object returned for {}", AttachmentHash); + CbValidateError ValidateResult; + if (CbObject Object = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); + ValidateResult == CbValidateError::None && Object) + { + Package.AddAttachment(CbAttachment(Object)); + } + else + { + // Error - malformed object + ZEN_WARN("malformed object returned for {} ('{}')", AttachmentHash, ToString(ValidateResult)); + } } break; @@ -1774,66 +1787,20 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req) return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); } - IoBuffer Payload = HttpReq.ReadPayload(); - CbObject Params = LoadCompactBinaryObject(Payload); - std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`) - std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`) - std::filesystem::path ProjectRoot = - Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) - std::filesystem::path ProjectFilePath = - Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) - - const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; - m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath); - - ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", - ProjectId, - Root, - EngineRoot, - ProjectRoot, - ProjectFilePath, - ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); - - m_ProjectStats.ProjectWriteCount++; - HttpReq.WriteResponse(HttpResponseCode::Created); - } - break; - - case HttpVerb::kPut: - { - if (!m_ProjectStore->AreDiskWritesAllowed()) + CbValidateError ValidateResult; + if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); + ValidateResult == CbValidateError::None) { - return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - } + std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`) + std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`) + std::filesystem::path ProjectRoot = + Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) + std::filesystem::path ProjectFilePath = + Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) - IoBuffer Payload = HttpReq.ReadPayload(); - CbObject Params = LoadCompactBinaryObject(Payload); - std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`) - std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`) - std::filesystem::path ProjectRoot = - Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) - std::filesystem::path ProjectFilePath = - Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) - - if (m_ProjectStore->UpdateProject(ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath)) - { - m_ProjectStats.ProjectWriteCount++; - ZEN_INFO("updated project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", - ProjectId, - Root, - EngineRoot, - ProjectRoot, - ProjectFilePath, - ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); - - HttpReq.WriteResponse(HttpResponseCode::OK); - } - else - { const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath); - m_ProjectStats.ProjectWriteCount++; ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", ProjectId, Root, @@ -1842,8 +1809,71 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req) ProjectFilePath, ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); + m_ProjectStats.ProjectWriteCount++; HttpReq.WriteResponse(HttpResponseCode::Created); } + else + { + HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Malformed compact binary object: '{}'", ToString(ValidateResult))); + } + } + break; + + case HttpVerb::kPut: + { + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + CbValidateError ValidateResult; + if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); + ValidateResult == CbValidateError::None) + { + std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`) + std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`) + std::filesystem::path ProjectRoot = + Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) + std::filesystem::path ProjectFilePath = + Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) + + if (m_ProjectStore->UpdateProject(ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath)) + { + m_ProjectStats.ProjectWriteCount++; + ZEN_INFO("updated project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", + ProjectId, + Root, + EngineRoot, + ProjectRoot, + ProjectFilePath, + ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); + + HttpReq.WriteResponse(HttpResponseCode::OK); + } + else + { + const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; + m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath); + + m_ProjectStats.ProjectWriteCount++; + ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", + ProjectId, + Root, + EngineRoot, + ProjectRoot, + ProjectFilePath, + ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); + + HttpReq.WriteResponse(HttpResponseCode::Created); + } + } + else + { + HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Malformed compact binary object: '{}'", ToString(ValidateResult))); + } } break; diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp index d79ad3cb7..a8d486e2a 100644 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp +++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp @@ -2,6 +2,7 @@ #include "jupiterremoteprojectstore.h" +#include <zencore/compactbinaryutil.h> #include <zencore/compress.h> #include <zencore/fmtutils.h> @@ -249,8 +250,9 @@ private: return Result; } - CbObject ContainerObject = LoadCompactBinaryObject(GetResult.Response); - if (!ContainerObject) + CbValidateError ValidateResult = CbValidateError::None; + if (CbObject ContainerObject = ValidateAndReadCompactBinaryObject(IoBuffer(GetResult.Response), ValidateResult); + ValidateResult != CbValidateError::None || !ContainerObject) { return LoadContainerResult{ RemoteProjectStore::Result{.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError), @@ -262,7 +264,10 @@ private: Key)}, {}}; } - return LoadContainerResult{ConvertResult(GetResult), std::move(ContainerObject)}; + else + { + return LoadContainerResult{ConvertResult(GetResult), std::move(ContainerObject)}; + } } void AddStats(const JupiterResult& Result) diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 56b64cee4..5277d689f 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -5439,67 +5439,70 @@ ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_vie } Project->TouchOplog(OplogId); - CbObject ContainerObject = LoadCompactBinaryObject(Payload); - if (!ContainerObject) - { - return {HttpResponseCode::BadRequest, "Invalid payload format"}; - } - - CidStore& ChunkStore = m_CidStore; - RwLock AttachmentsLock; - tsl::robin_set<IoHash, IoHash::Hasher> Attachments; - - auto HasAttachment = [&ChunkStore](const IoHash& RawHash) { return ChunkStore.ContainsChunk(RawHash); }; - auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector<IoHash>&& ChunkHashes) { - RwLock::ExclusiveLockScope _(AttachmentsLock); - if (BlockHash != IoHash::Zero) - { - Attachments.insert(BlockHash); - } - else - { - Attachments.insert(ChunkHashes.begin(), ChunkHashes.end()); - } - }; - auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) { - RwLock::ExclusiveLockScope _(AttachmentsLock); - Attachments.insert(RawHash); - }; + CbValidateError ValidateResult; + if (CbObject ContainerObject = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); + ValidateResult == CbValidateError::None && ContainerObject) + { + CidStore& ChunkStore = m_CidStore; + RwLock AttachmentsLock; + tsl::robin_set<IoHash, IoHash::Hasher> Attachments; + + auto HasAttachment = [&ChunkStore](const IoHash& RawHash) { return ChunkStore.ContainsChunk(RawHash); }; + auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector<IoHash>&& ChunkHashes) { + RwLock::ExclusiveLockScope _(AttachmentsLock); + if (BlockHash != IoHash::Zero) + { + Attachments.insert(BlockHash); + } + else + { + Attachments.insert(ChunkHashes.begin(), ChunkHashes.end()); + } + }; + auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) { + RwLock::ExclusiveLockScope _(AttachmentsLock); + Attachments.insert(RawHash); + }; - auto OnChunkedAttachment = [](const ChunkedInfo&) {}; + auto OnChunkedAttachment = [](const ChunkedInfo&) {}; - auto OnReferencedAttachments = [&Oplog](std::span<IoHash> RawHashes) { Oplog->CaptureAddedAttachments(RawHashes); }; - // Make sure we retain any attachments we download before writing the oplog - Oplog->EnableUpdateCapture(); - auto _ = MakeGuard([&Oplog]() { Oplog->DisableUpdateCapture(); }); + auto OnReferencedAttachments = [&Oplog](std::span<IoHash> RawHashes) { Oplog->CaptureAddedAttachments(RawHashes); }; + // Make sure we retain any attachments we download before writing the oplog + Oplog->EnableUpdateCapture(); + auto _ = MakeGuard([&Oplog]() { Oplog->DisableUpdateCapture(); }); - RemoteProjectStore::Result RemoteResult = SaveOplogContainer(*Oplog, - ContainerObject, - OnReferencedAttachments, - HasAttachment, - OnNeedBlock, - OnNeedAttachment, - OnChunkedAttachment, - nullptr); + RemoteProjectStore::Result RemoteResult = SaveOplogContainer(*Oplog, + ContainerObject, + OnReferencedAttachments, + HasAttachment, + OnNeedBlock, + OnNeedAttachment, + OnChunkedAttachment, + nullptr); - if (RemoteResult.ErrorCode) - { - return ConvertResult(RemoteResult); - } + if (RemoteResult.ErrorCode) + { + return ConvertResult(RemoteResult); + } - CbObjectWriter Cbo(1 + 1 + 5 + Attachments.size() * (1 + sizeof(IoHash::Hash)) + 1); - Cbo.BeginArray("need"); - { - for (const IoHash& Hash : Attachments) + CbObjectWriter Cbo(1 + 1 + 5 + Attachments.size() * (1 + sizeof(IoHash::Hash)) + 1); + Cbo.BeginArray("need"); { - ZEN_DEBUG("Need attachment {}", Hash); - Cbo << Hash; + for (const IoHash& Hash : Attachments) + { + ZEN_DEBUG("Need attachment {}", Hash); + Cbo << Hash; + } } - } - Cbo.EndArray(); // "need" + Cbo.EndArray(); // "need" - OutResponse = Cbo.Save(); - return {HttpResponseCode::OK, {}}; + OutResponse = Cbo.Save(); + return {HttpResponseCode::OK, {}}; + } + else + { + return {HttpResponseCode::BadRequest, fmt::format("Invalid payload format ('{}')", ToString(ValidateResult))}; + } } std::pair<HttpResponseCode, std::string> @@ -5604,15 +5607,19 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, } break; case HttpContentType::kCbObject: - Cb = LoadCompactBinaryObject(Payload); - if (!Cb) { - HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Content format not supported, expected compact binary format"); - return false; + CbValidateError ValidateResult; + if (Cb = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); + ValidateResult != CbValidateError::None || !Cb) + { + HttpReq.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Content format not supported, expected compact binary format ('{}')", ToString(ValidateResult))); + return false; + } + break; } - break; case HttpContentType::kCbPackage: try { @@ -5917,8 +5924,8 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, ResponseObj.EndArray(); } - // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the new - // chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have + // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the + // new chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have // unreferenced chunks. for (auto It : AddedChunks) { @@ -7768,7 +7775,8 @@ TEST_CASE("project.store.gc.prep") Project1->DeleteOplog("oplog1"sv); } - // Caution - putting breakpoints and stepping through this part of the test likely makes it fails due to expiry time of pending chunks + // Caution - putting breakpoints and stepping through this part of the test likely makes it fails due to expiry time of pending + // chunks { Ref<ProjectStore::Project> Project1 = ProjectStore.OpenProject("proj1"sv); Ref<ProjectStore::Oplog> Oplog = Project1->NewOplog("oplog1"sv, Project1OplogPath); @@ -7790,8 +7798,8 @@ TEST_CASE("project.store.gc.prep") } Sleep(200); - // This pass they should also be retained since our age retention has kept them alive and they will now be picked up and the retention - // cleared + // This pass they should also be retained since our age retention has kept them alive and they will now be picked up and the + // retention cleared { GcSettings Settings = {.CacheExpireTime = GcClock::Now(), .ProjectStoreExpireTime = GcClock::Now(), diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp index 857e868da..2a81ee3e3 100644 --- a/src/zenserver/projectstore/remoteprojectstore.cpp +++ b/src/zenserver/projectstore/remoteprojectstore.cpp @@ -2651,19 +2651,22 @@ ParseOplogContainer(const CbObject& ContainerObject, IoBuffer OpsBuffer(IoBuffer::Wrap, OpsSection.GetData(), OpsSection.GetSize()); IoBuffer SectionPayload = CompressedBuffer::FromCompressedNoValidate(std::move(OpsBuffer)).Decompress().AsIoBuffer(); + CbValidateError ValidateResult = CbValidateError::None; + if (CbObject SectionObject = ValidateAndReadCompactBinaryObject(std::move(SectionPayload), ValidateResult); + ValidateResult == CbValidateError::None && ContainerObject) { - CbObject SectionObject = LoadCompactBinaryObject(SectionPayload); - if (!SectionObject) - { - remotestore_impl::ReportMessage(OptionalContext, - fmt::format("Failed to save oplog container: '{}'", "Section has unexpected data type")); - return RemoteProjectStore::Result{gsl::narrow<int>(HttpResponseCode::BadRequest), - Timer.GetElapsedTimeMs() / 1000.0, - "Section has unexpected data type", - "Failed to save oplog container"}; - } OutOplogSection = SectionObject; } + else + { + remotestore_impl::ReportMessage( + OptionalContext, + fmt::format("Failed to save oplog container: '{}' ('{}')", "Section has unexpected data type", ToString(ValidateResult))); + return RemoteProjectStore::Result{gsl::narrow<int>(HttpResponseCode::BadRequest), + Timer.GetElapsedTimeMs() / 1000.0, + "Section has unexpected data type", + "Failed to save oplog container"}; + } std::unordered_set<IoHash, IoHash::Hasher> OpsAttachments; { CbArrayView OpsArray = OutOplogSection["ops"sv].AsArrayView(); diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index 1b2cf036b..d65c2bf06 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -3,6 +3,7 @@ #include <zenstore/buildstore/buildstore.h> #include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryutil.h> #include <zencore/compress.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> @@ -138,24 +139,33 @@ BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc, CidStore& { RwLock::ExclusiveLockScope Lock(m_Lock); - CbObject ManifestReader = LoadCompactBinaryObject(ReadFile(ManifestPath).Flatten()); - Oid ManifestId = ManifestReader["id"].AsObjectId(); - uint32_t Version = ManifestReader["version"].AsUInt32(); - DateTime CreationDate = ManifestReader["createdAt"].AsDateTime(); - ZEN_UNUSED(CreationDate); - if (ManifestId == Oid::Zero || Version != blobstore::impl::ManifestVersion) + CbValidateError ValidateResult = CbValidateError::None; + if (CbObject ManifestReader = ValidateAndReadCompactBinaryObject(ReadFile(ManifestPath).Flatten(), ValidateResult); + ValidateResult == CbValidateError::None && ManifestReader) { - ZEN_WARN("Invalid manifest at {}, wiping state", ManifestPath); - IsNew = true; + Oid ManifestId = ManifestReader["id"].AsObjectId(); + uint32_t Version = ManifestReader["version"].AsUInt32(); + DateTime CreationDate = ManifestReader["createdAt"].AsDateTime(); + ZEN_UNUSED(CreationDate); + if (ManifestId == Oid::Zero || Version != blobstore::impl::ManifestVersion) + { + ZEN_WARN("Invalid manifest at {}, wiping state", ManifestPath); + IsNew = true; + } + else + { + m_BlobLogFlushPosition = ReadPayloadLog(Lock, BlobLogPath, 0); + m_MetaLogFlushPosition = ReadMetadataLog(Lock, MetaLogPath, 0); + if (IsFile(AccessTimesPath)) + { + ReadAccessTimes(Lock, AccessTimesPath); + } + } } else { - m_BlobLogFlushPosition = ReadPayloadLog(Lock, BlobLogPath, 0); - m_MetaLogFlushPosition = ReadMetadataLog(Lock, MetaLogPath, 0); - if (IsFile(AccessTimesPath)) - { - ReadAccessTimes(Lock, AccessTimesPath); - } + ZEN_WARN("Invalid manifest at {} ('{}'), wiping state", ManifestPath, ToString(ValidateResult)); + IsNew = true; } } diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index 5d9a68919..83301f863 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -190,9 +190,8 @@ CacheRpcHandler::HandleRpcRequest(const CacheRequestContext& Context, m_CacheStats.RpcRequests.fetch_add(1); - CbPackage Package; - CbObjectView Object; - CbObject ObjectBuffer; + CbPackage Package; + CbObject Object; try { if (ContentType == ZenContentType::kCbObject) @@ -203,8 +202,7 @@ CacheRpcHandler::HandleRpcRequest(const CacheRequestContext& Context, return RpcResponseCode::BadRequest; } - ObjectBuffer = LoadCompactBinaryObject(std::move(Body)); - Object = ObjectBuffer; + Object = LoadCompactBinaryObject(std::move(Body)); if (!Object) { ZEN_WARN("Content format not supported, expected compact binary format"); diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index c9278acb4..386a91cb3 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -3,6 +3,7 @@ #include <zenutil/jupiter/jupiterbuildstorage.h> #include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryutil.h> #include <zencore/fmtutils.h> #include <zencore/scopeguard.h> #include <zencore/timer.h> @@ -430,13 +431,41 @@ private: } else if (Payload.GetContentType() == ZenContentType::kCbObject) { - return LoadCompactBinaryObject(Payload); + CbValidateError ValidateResult = CbValidateError::None; + if (CbObject Object = ValidateAndReadCompactBinaryObject(IoBuffer(Payload), ValidateResult); + ValidateResult == CbValidateError::None) + { + return Object; + } + else + { + throw std::runtime_error(fmt::format("{}: {} ({})", + "Invalid compact binary object: '{}'", + ErrorContext, + ToString(Payload.GetContentType()), + ToString(ValidateResult))); + } } else if (Payload.GetContentType() == ZenContentType::kCompressedBinary) { - IoHash RawHash; - uint64_t RawSize; - return LoadCompactBinaryObject(CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize)); + IoHash RawHash; + uint64_t RawSize; + CbValidateError ValidateResult = CbValidateError::None; + if (CbObject Object = + ValidateAndReadCompactBinaryObject(CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize), + ValidateResult); + ValidateResult == CbValidateError::None) + { + return Object; + } + else + { + throw std::runtime_error(fmt::format("{}: {} ({})", + "Invalid compresed compact binary object: '{}'", + ErrorContext, + ToString(Payload.GetContentType()), + ToString(ValidateResult))); + } } else { diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp index 1fd59acdf..c305dc477 100644 --- a/src/zenutil/jupiter/jupitersession.cpp +++ b/src/zenutil/jupiter/jupitersession.cpp @@ -4,6 +4,7 @@ #include <zencore/compactbinary.h> #include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryutil.h> #include <zencore/compositebuffer.h> #include <zencore/compress.h> #include <zencore/fmtutils.h> @@ -526,7 +527,15 @@ JupiterSession::PutMultipartBuildBlob(std::string_view Namespace, ZEN_WARN("{}", StartMultipartResponse.ErrorMessage("startMultipartUpload: ")); return detail::ConvertResponse(StartMultipartResponse, "JupiterSession::PutMultipartBuildBlob"sv); } - CbObject ResponseObject = LoadCompactBinaryObject(StartMultipartResponse.ResponsePayload); + CbValidateError ValidateResult = CbValidateError::None; + CbObject ResponseObject = ValidateAndReadCompactBinaryObject(IoBuffer(StartMultipartResponse.ResponsePayload), ValidateResult); + if (ValidateResult != CbValidateError::None) + { + JupiterResult Result = detail::ConvertResponse(StartMultipartResponse, "JupiterSession::PutMultipartBuildBlob"sv); + Result.ErrorCode = (int32)HttpResponseCode::UnsupportedMediaType; + Result.Reason = fmt::format("Invalid multipart response object format: '{}'", ToString(ValidateResult)); + return Result; + } struct WorkloadData { |