diff options
| author | Martin Ridgers <[email protected]> | 2021-09-15 09:22:32 +0200 |
|---|---|---|
| committer | Martin Ridgers <[email protected]> | 2021-09-15 09:23:33 +0200 |
| commit | 8f5e773529858223beeecf5d1b69c23991df644e (patch) | |
| tree | 2c360c67e028f5ecd7368212b0adf8b23578ff9d /zencore | |
| parent | Use zen::Sleep() in timer.cpp's tests (diff) | |
| parent | Updated function service to new package management API (diff) | |
| download | zen-8f5e773529858223beeecf5d1b69c23991df644e.tar.xz zen-8f5e773529858223beeecf5d1b69c23991df644e.zip | |
Merge main
Diffstat (limited to 'zencore')
| -rw-r--r-- | zencore/compactbinary.cpp | 2 | ||||
| -rw-r--r-- | zencore/compactbinarypackage.cpp | 552 | ||||
| -rw-r--r-- | zencore/compactbinaryvalidation.cpp | 143 | ||||
| -rw-r--r-- | zencore/except.cpp | 8 | ||||
| -rw-r--r-- | zencore/httpclient.cpp | 23 | ||||
| -rw-r--r-- | zencore/httpserver.cpp | 1641 | ||||
| -rw-r--r-- | zencore/include/zencore/compactbinary.h | 22 | ||||
| -rw-r--r-- | zencore/include/zencore/compactbinarypackage.h | 62 | ||||
| -rw-r--r-- | zencore/include/zencore/compactbinaryvalidation.h | 5 | ||||
| -rw-r--r-- | zencore/include/zencore/except.h | 13 | ||||
| -rw-r--r-- | zencore/include/zencore/httpclient.h | 22 | ||||
| -rw-r--r-- | zencore/include/zencore/httpserver.h | 401 | ||||
| -rw-r--r-- | zencore/include/zencore/iobuffer.h | 1 | ||||
| -rw-r--r-- | zencore/include/zencore/logging.h | 1 | ||||
| -rw-r--r-- | zencore/include/zencore/refcount.h | 1 | ||||
| -rw-r--r-- | zencore/iothreadpool.cpp | 36 | ||||
| -rw-r--r-- | zencore/iothreadpool.h | 31 | ||||
| -rw-r--r-- | zencore/logging.cpp | 6 | ||||
| -rw-r--r-- | zencore/zencore.vcxproj | 6 | ||||
| -rw-r--r-- | zencore/zencore.vcxproj.filters | 6 |
20 files changed, 572 insertions, 2410 deletions
diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp index 5fe7f272d..b508d8fe8 100644 --- a/zencore/compactbinary.cpp +++ b/zencore/compactbinary.cpp @@ -12,7 +12,7 @@ namespace zen { -const int DaysToMonth[] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365}; +const int DaysToMonth[] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365}; bool IsLeapYear(int Year) diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp index 7880164f9..9a7e7c098 100644 --- a/zencore/compactbinarypackage.cpp +++ b/zencore/compactbinarypackage.cpp @@ -16,22 +16,45 @@ CbAttachment::CbAttachment(const CompressedBuffer& InValue) : CbAttachment(InVal { } -CbAttachment::CbAttachment(CompressedBuffer&& InValue) +CbAttachment::CbAttachment(const SharedBuffer& InValue) : CbAttachment(CompositeBuffer(InValue)) { - Value.emplace<CompressedBuffer>(std::move(InValue).MakeOwned()); } -CbAttachment::CbAttachment(const SharedBuffer& InValue) -: CbAttachment(InValue.IsNull() ? CompressedBuffer() - : CompressedBuffer::Compress(InValue, OodleCompressor::NotSet, OodleCompressionLevel::None)) +CbAttachment::CbAttachment(const SharedBuffer& InValue, [[maybe_unused]] const IoHash& InHash) +: CbAttachment(CompositeBuffer(InValue), InHash) { } -CbAttachment::CbAttachment(const SharedBuffer& InValue, [[maybe_unused]] const IoHash& InHash) -: CbAttachment(InValue.IsNull() ? CompressedBuffer() - : CompressedBuffer::Compress(InValue, OodleCompressor::NotSet, OodleCompressionLevel::None)) +CbAttachment::CbAttachment(const CompositeBuffer& InValue) : Value{std::in_place_type<BinaryValue>, InValue} +{ + if (std::get<BinaryValue>(Value).Buffer.IsNull()) + { + Value.emplace<nullptr_t>(); + } +} + +CbAttachment::CbAttachment(CompositeBuffer&& InValue) : Value{std::in_place_type<BinaryValue>, InValue} +{ + if (std::get<BinaryValue>(Value).Buffer.IsNull()) + { + Value.emplace<nullptr_t>(); + } +} + +CbAttachment::CbAttachment(CompositeBuffer&& InValue, const IoHash& Hash) : Value{std::in_place_type<BinaryValue>, InValue, Hash} +{ + if (std::get<BinaryValue>(Value).Buffer.IsNull()) + { + Value.emplace<nullptr_t>(); + } +} + +CbAttachment::CbAttachment(CompressedBuffer&& InValue) : Value(std::in_place_type<CompressedBuffer>, InValue) { - // This could be more efficient, and should at the very least try to validate the hash + if (std::get<CompressedBuffer>(Value).IsNull()) + { + Value.emplace<nullptr_t>(); + } } CbAttachment::CbAttachment(const CbObject& InValue, const IoHash* const InHash) @@ -70,114 +93,139 @@ CbAttachment::TryLoad(IoBuffer& InBuffer, BufferAllocator Allocator) bool CbAttachment::TryLoad(CbFieldIterator& Fields) { - const CbObjectView ObjectView = Fields.AsObjectView(); - if (Fields.HasError()) + if (const CbObjectView ObjectView = Fields.AsObjectView(); !Fields.HasError()) + { + // Is a null object or object not prefixed with a precomputed hash value + Value.emplace<CbObjectValue>(CbObject(ObjectView, Fields.GetOuterBuffer()), ObjectView.GetHash()); + ++Fields; + } + else if (const IoHash ObjectAttachmentHash = Fields.AsObjectAttachment(); !Fields.HasError()) + { + // Is an object + ++Fields; + const CbObjectView InnerObjectView = Fields.AsObjectView(); + if (Fields.HasError()) + { + return false; + } + Value.emplace<CbObjectValue>(CbObject(InnerObjectView, Fields.GetOuterBuffer()), ObjectAttachmentHash); + ++Fields; + } + else if (const IoHash BinaryAttachmentHash = Fields.AsBinaryAttachment(); !Fields.HasError()) + { + // Is an uncompressed binary blob + ++Fields; + MemoryView BinaryView = Fields.AsBinaryView(); + if (Fields.HasError()) + { + return false; + } + Value.emplace<BinaryValue>(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer()), BinaryAttachmentHash); + ++Fields; + } + else if (MemoryView BinaryView = Fields.AsBinaryView(); !Fields.HasError()) { - // Is a buffer - const MemoryView BinaryView = Fields.AsBinaryView(); if (BinaryView.GetSize() > 0) { + // Is a compressed binary blob Value.emplace<CompressedBuffer>( CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())).MakeOwned()); - ++Fields; } else { + // Is an uncompressed empty binary blob + Value.emplace<BinaryValue>(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer()), IoHash::HashBuffer(nullptr, 0)); ++Fields; - Value.emplace<CompressedBuffer>(); } } else { - // It's an object - ++Fields; - IoHash Hash; - if (ObjectView) - { - Hash = Fields.AsObjectAttachment(); - ++Fields; - } - else - { - Hash = IoHash::HashBuffer(MemoryView{}); - } - Value.emplace<CbObjectValue>(CbObject(ObjectView, Fields->GetOuterBuffer()), Hash); + return false; } return true; } -bool -CbAttachment::TryLoad(BinaryReader& Reader, BufferAllocator Allocator) +static bool +TryLoad_ArchiveFieldIntoAttachment(CbAttachment& TargetAttachment, CbField&& Field, BinaryReader& Reader, BufferAllocator Allocator) { - CbField Field = LoadCompactBinary(Reader, Allocator); - const CbObjectView ObjectView = Field.AsObjectView(); - - if (Field.HasError()) + if (const CbObjectView ObjectView = Field.AsObjectView(); !Field.HasError()) { - // It's a buffer - const MemoryView BinaryView = Field.AsBinaryView(); - if (BinaryView.GetSize() > 0) + // Is a null object or object not prefixed with a precomputed hash value + TargetAttachment = CbAttachment(CbObject(ObjectView, std::move(Field)), ObjectView.GetHash()); + } + else if (const IoHash ObjectAttachmentHash = Field.AsObjectAttachment(); !Field.HasError()) + { + // Is an object + Field = LoadCompactBinary(Reader, Allocator); + if (!Field.IsObject()) { - Value.emplace<CompressedBuffer>( - CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Field.GetOuterBuffer())).MakeOwned()); + return false; } - else + TargetAttachment = CbAttachment(std::move(Field).AsObject(), ObjectAttachmentHash); + } + else if (const IoHash BinaryAttachmentHash = Field.AsBinaryAttachment(); !Field.HasError()) + { + // Is an uncompressed binary blob + Field = LoadCompactBinary(Reader, Allocator); + SharedBuffer Buffer = Field.AsBinary(); + if (Field.HasError()) { - Value.emplace<CompressedBuffer>(); + return false; } + TargetAttachment = CbAttachment(CompositeBuffer(Buffer), BinaryAttachmentHash); } - else + else if (SharedBuffer Buffer = Field.AsBinary(); !Field.HasError()) { - // It's an object - IoHash Hash; - if (ObjectView) + if (Buffer.GetSize() > 0) { - std::vector<uint8_t> HashBuffer; - CbField HashField = LoadCompactBinary(Reader, [&HashBuffer](uint64_t Size) -> UniqueBuffer { - HashBuffer.resize(Size); - return UniqueBuffer::MakeMutableView(HashBuffer.data(), Size); - }); - Hash = HashField.AsAttachment(); - if (HashField.HasError() || ObjectView.GetHash() != Hash) - { - // Error - return false; - } + // Is a compressed binary blob + TargetAttachment = CbAttachment(CompressedBuffer::FromCompressed(std::move(Buffer))); } else { - Hash = IoHash::HashBuffer(MemoryView()); + // Is an uncompressed empty binary blob + TargetAttachment = CbAttachment(CompositeBuffer(Buffer), IoHash::HashBuffer(nullptr, 0)); } - Value.emplace<CbObjectValue>(CbObject(ObjectView, Field.GetOuterBuffer()), Hash); + } + else + { + return false; } return true; } +bool +CbAttachment::TryLoad(BinaryReader& Reader, BufferAllocator Allocator) +{ + CbField Field = LoadCompactBinary(Reader, Allocator); + return TryLoad_ArchiveFieldIntoAttachment(*this, std::move(Field), Reader, Allocator); +} + void CbAttachment::Save(CbWriter& Writer) const { - if (const CbObjectValue* ObjectValue = std::get_if<CbObjectValue>(&Value)) + if (const CbObjectValue* ObjValue = std::get_if<CbObjectValue>(&Value)) { - Writer.AddObject(ObjectValue->Object); - if (ObjectValue->Object) + if (ObjValue->Object) { - Writer.AddObjectAttachment(ObjectValue->Hash); + Writer.AddObjectAttachment(ObjValue->Hash); } + Writer.AddObject(ObjValue->Object); } - else + else if (const BinaryValue* BinValue = std::get_if<BinaryValue>(&Value)) { - const CompressedBuffer& BufferValue = std::get<CompressedBuffer>(Value); - if (BufferValue.GetRawSize()) - { - Writer.AddBinary(BufferValue.GetCompressed()); - } - else // Null + if (BinValue->Buffer.GetSize() > 0) { - Writer.AddBinary(MemoryView()); + Writer.AddBinaryAttachment(BinValue->Hash); } + Writer.AddBinary(BinValue->Buffer); + } + else if (const CompressedBuffer* BufferValue = std::get_if<CompressedBuffer>(&Value)) + { + Writer.AddBinary(BufferValue->GetCompressed()); } } @@ -192,14 +240,19 @@ CbAttachment::Save(BinaryWriter& Writer) const bool CbAttachment::IsNull() const { - if (const CompressedBuffer* Buffer = std::get_if<CompressedBuffer>(&Value)) - { - return Buffer->IsNull(); - } - else - { - return false; - } + return std::holds_alternative<nullptr_t>(Value); +} + +bool +CbAttachment::IsBinary() const +{ + return std::holds_alternative<BinaryValue>(Value); +} + +bool +CbAttachment::IsCompressedBinary() const +{ + return std::holds_alternative<CompressedBuffer>(Value); } bool @@ -213,25 +266,42 @@ CbAttachment::GetHash() const { if (const CompressedBuffer* Buffer = std::get_if<CompressedBuffer>(&Value)) { - return Buffer->IsNull() ? IoHash::HashBuffer(MemoryView()) : IoHash::FromBLAKE3(Buffer->GetRawHash()); + return IoHash::FromBLAKE3(Buffer->GetRawHash()); } - else + + if (const BinaryValue* BinValue = std::get_if<BinaryValue>(&Value)) { - return std::get<CbObjectValue>(Value).Hash; + return BinValue->Hash; } + + if (const CbObjectValue* ObjectValue = std::get_if<CbObjectValue>(&Value)) + { + return ObjectValue->Hash; + } + + return IoHash::Zero; } -SharedBuffer -CbAttachment::AsBinary() const +CompositeBuffer +CbAttachment::AsCompositeBinary() const { - if (const CompressedBuffer* Buffer = std::get_if<CompressedBuffer>(&Value)) + if (const BinaryValue* BinValue = std::get_if<BinaryValue>(&Value)) { - return Buffer->Decompress(); + return BinValue->Buffer; } - else + + return CompositeBuffer::Null; +} + +SharedBuffer +CbAttachment::AsBinary() const +{ + if (const BinaryValue* BinValue = std::get_if<BinaryValue>(&Value)) { - return std::get<CbObjectValue>(Value).Object.GetBuffer(); + return BinValue->Buffer.Flatten(); } + + return {}; } CompressedBuffer @@ -241,12 +311,8 @@ CbAttachment::AsCompressedBinary() const { return *Buffer; } - else - { - return CompressedBuffer::Compress(std::get<CbObjectValue>(Value).Object.GetBuffer(), - OodleCompressor::NotSet, - OodleCompressionLevel::None); - } + + return CompressedBuffer::Null; } /** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */ @@ -257,10 +323,8 @@ CbAttachment::AsObject() const { return ObjectValue->Object; } - else - { - return {}; - } + + return {}; } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -301,10 +365,7 @@ CbPackage::AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Res if (It != Attachments.end() && *It == Attachment) { CbAttachment& Existing = *It; - if (Attachment.IsObject() && !Existing.IsObject()) - { - Existing = CbAttachment(CbObject(Existing.AsBinary()), Existing.GetHash()); - } + Existing = Attachment; } else { @@ -358,14 +419,14 @@ CbPackage::GatherAttachments(const CbObject& Value, AttachmentResolver Resolver) } else { - AddAttachment(CbAttachment(std::move(Buffer), Hash)); + AddAttachment(CbAttachment(std::move(Buffer))); } } }); } bool -CbPackage::TryLoad(IoBuffer& InBuffer, BufferAllocator Allocator, AttachmentResolver* Mapper) +CbPackage::TryLoad(IoBuffer InBuffer, BufferAllocator Allocator, AttachmentResolver* Mapper) { MemoryInStream InStream(InBuffer.Data(), InBuffer.Size()); BinaryReader Reader(InStream); @@ -377,6 +438,7 @@ bool CbPackage::TryLoad(CbFieldIterator& Fields) { *this = CbPackage(); + while (Fields) { if (Fields.IsNull()) @@ -384,43 +446,76 @@ CbPackage::TryLoad(CbFieldIterator& Fields) ++Fields; break; } - else if (Fields.IsBinary()) + else if (IoHash Hash = Fields.AsHash(); !Fields.HasError() && !Fields.IsAttachment()) { - CbAttachment Attachment; - Attachment.TryLoad(Fields); - AddAttachment(Attachment); - } - else - { - Object = Fields.AsObject(); - if (Fields->HasError()) + ++Fields; + CbObjectView ObjectView = Fields.AsObjectView(); + if (Fields.HasError() || Hash != ObjectView.GetHash()) { return false; } + Object = CbObject(ObjectView, Fields.GetOuterBuffer()); Object.MakeOwned(); + ObjectHash = Hash; ++Fields; - if (Object.CreateIterator()) - { - ObjectHash = Fields.AsObjectAttachment(); - if (Fields.HasError()) - { - return false; - } - ++Fields; - } - else + } + else + { + CbAttachment Attachment; + if (!Attachment.TryLoad(Fields)) { - Object.Reset(); + return false; } + AddAttachment(Attachment); } } - return true; } bool CbPackage::TryLoad(BinaryReader& Reader, BufferAllocator Allocator, AttachmentResolver* Mapper) { + // TODO: this needs to re-grow the ability to accept a reference to an attachment which is + // not embedded + + ZEN_UNUSED(Mapper); + +#if 1 + *this = CbPackage(); + for (;;) + { + CbField Field = LoadCompactBinary(Reader, Allocator); + if (!Field) + { + return false; + } + + if (Field.IsNull()) + { + return true; + } + else if (IoHash Hash = Field.AsHash(); !Field.HasError() && !Field.IsAttachment()) + { + Field = LoadCompactBinary(Reader, Allocator); + CbObjectView ObjectView = Field.AsObjectView(); + if (Field.HasError() || Hash != ObjectView.GetHash()) + { + return false; + } + Object = CbObject(ObjectView, Field.GetOuterBuffer()); + ObjectHash = Hash; + } + else + { + CbAttachment Attachment; + if (!TryLoad_ArchiveFieldIntoAttachment(Attachment, std::move(Field), Reader, Allocator)) + { + return false; + } + AddAttachment(Attachment); + } + } +#else uint8_t StackBuffer[64]; const auto StackAllocator = [&Allocator, &StackBuffer](uint64_t Size) -> UniqueBuffer { if (Size <= sizeof(StackBuffer)) @@ -494,6 +589,7 @@ CbPackage::TryLoad(BinaryReader& Reader, BufferAllocator Allocator, AttachmentRe } } } +#endif } void @@ -501,8 +597,8 @@ CbPackage::Save(CbWriter& Writer) const { if (Object) { + Writer.AddHash(ObjectHash); Writer.AddObject(Object); - Writer.AddObjectAttachment(ObjectHash); } for (const CbAttachment& Attachment : Attachments) { @@ -519,6 +615,136 @@ CbPackage::Save(BinaryWriter& StreamWriter) const Writer.Save(StreamWriter); } +////////////////////////////////////////////////////////////////////////// +// +// Legacy package serialization support +// + +namespace legacy { + + void SaveCbAttachment(const CbAttachment& Attachment, CbWriter& Writer) + { + if (Attachment.IsObject()) + { + CbObject Object = Attachment.AsObject(); + Writer.AddBinary(Object.GetBuffer()); + if (Object) + { + Writer.AddObjectAttachment(Attachment.GetHash()); + } + } + else if (Attachment.IsBinary()) + { + Writer.AddBinary(Attachment.AsBinary()); + Writer.AddBinaryAttachment(Attachment.GetHash()); + } + else if (Attachment.IsNull()) + { + Writer.AddBinary(MemoryView()); + } + else + { + ZEN_NOT_IMPLEMENTED("Compressed binary is not supported in this serialization format"); + } + } + + void SaveCbPackage(const CbPackage& Package, CbWriter& Writer) + { + if (const CbObject& RootObject = Package.GetObject()) + { + Writer.AddObject(RootObject); + Writer.AddObjectAttachment(Package.GetObjectHash()); + } + for (const CbAttachment& Attachment : Package.GetAttachments()) + { + SaveCbAttachment(Attachment, Writer); + } + Writer.AddNull(); + } + + void SaveCbPackage(const CbPackage& Package, BinaryWriter& Ar) + { + CbWriter Writer; + SaveCbPackage(Package, Writer); + Writer.Save(Ar); + } + + bool TryLoadCbPackage(CbPackage& Package, IoBuffer InBuffer, BufferAllocator Allocator, CbPackage::AttachmentResolver* Mapper) + { + MemoryInStream InStream(InBuffer.Data(), InBuffer.Size()); + BinaryReader Reader(InStream); + + return TryLoadCbPackage(Package, Reader, Allocator, Mapper); + } + + bool TryLoadCbPackage(CbPackage& Package, BinaryReader& Reader, BufferAllocator Allocator, CbPackage::AttachmentResolver* Mapper) + { + Package = CbPackage(); + for (;;) + { + CbField ValueField = LoadCompactBinary(Reader, Allocator); + if (!ValueField) + { + return false; + } + if (ValueField.IsNull()) + { + return true; + } + if (ValueField.IsBinary()) + { + const MemoryView View = ValueField.AsBinaryView(); + if (View.GetSize() > 0) + { + SharedBuffer Buffer = SharedBuffer::MakeView(View, ValueField.GetOuterBuffer()).MakeOwned(); + CbField HashField = LoadCompactBinary(Reader, Allocator); + const IoHash& Hash = HashField.AsAttachment(); + if (HashField.HasError() || IoHash::HashBuffer(Buffer) != Hash) + { + return false; + } + if (HashField.IsObjectAttachment()) + { + Package.AddAttachment(CbAttachment(CbObject(std::move(Buffer)), Hash)); + } + else + { + Package.AddAttachment(CbAttachment(CompositeBuffer(std::move(Buffer)), Hash)); + } + } + } + else if (ValueField.IsHash()) + { + const IoHash Hash = ValueField.AsHash(); + + ZEN_ASSERT(Mapper); + + Package.AddAttachment(CbAttachment((*Mapper)(Hash), Hash)); + } + else + { + CbObject Object = ValueField.AsObject(); + if (ValueField.HasError()) + { + return false; + } + + if (Object) + { + CbField HashField = LoadCompactBinary(Reader, Allocator); + IoHash ObjectHash = HashField.AsObjectAttachment(); + if (HashField.HasError() || Object.GetHash() != ObjectHash) + { + return false; + } + Package.SetObject(Object, ObjectHash); + } + } + } + } + +} // namespace legacy + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void @@ -567,8 +793,7 @@ TEST_CASE("usonpackage") CHECK_FALSE(bool(Attachment.AsObject())); CHECK_FALSE(Attachment.IsBinary()); CHECK_FALSE(Attachment.IsObject()); - CHECK(Attachment.GetHash() == IoHash::HashBuffer({})); - TestSaveLoadValidate("Null", Attachment); + CHECK(Attachment.GetHash() == IoHash::Zero); } SUBCASE("Binary Attachment") @@ -596,12 +821,12 @@ TEST_CASE("usonpackage") CHECK_FALSE(Attachment.IsNull()); CHECK(bool(Attachment)); - CHECK(Attachment.AsBinary() == Object.GetBuffer()); + CHECK(Attachment.AsBinary() == SharedBuffer()); CHECK(Attachment.AsObject().Equals(Object)); - CHECK(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsBinary()); CHECK(Attachment.IsObject()); CHECK(Attachment.GetHash() == Object.GetHash()); - TestSaveLoadValidate("CompactBinary", Attachment); + TestSaveLoadValidate("Object", Attachment); } SUBCASE("Binary View") @@ -633,7 +858,7 @@ TEST_CASE("usonpackage") CHECK(Attachment.AsBinary() != ObjectView.GetBuffer()); CHECK(Attachment.AsObject().Equals(Object)); - CHECK(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsBinary()); CHECK(Attachment.IsObject()); CHECK(Attachment.GetHash() == IoHash(Object.GetHash())); } @@ -677,15 +902,14 @@ TEST_CASE("usonpackage") CbFieldIterator FieldsView = CbFieldIterator::MakeRangeView(CbFieldViewIterator(Fields)); Attachment.TryLoad(FieldsView); + MemoryView View; CHECK_FALSE(Attachment.IsNull()); CHECK(bool(Attachment)); - - CHECK(Attachment.AsBinary().GetView().EqualBytes(Value.GetView())); - CHECK_FALSE(FieldsView.GetBuffer().GetView().Contains(Attachment.AsObject().GetBuffer().GetView())); - CHECK(Attachment.IsBinary()); + CHECK(Attachment.AsBinary().GetView().EqualBytes(MemoryView())); + CHECK_FALSE((!Attachment.AsObject().TryGetSerializedView(View) || FieldsView.GetOuterBuffer().GetView().Contains(View))); + CHECK_FALSE(Attachment.IsBinary()); CHECK(Attachment.IsObject()); - CHECK(Attachment.GetHash() == Value.GetHash()); } @@ -696,7 +920,7 @@ TEST_CASE("usonpackage") CHECK(Attachment.IsNull()); CHECK_FALSE(Attachment.IsBinary()); CHECK_FALSE(Attachment.IsObject()); - CHECK(Attachment.GetHash() == IoHash::HashBuffer(SharedBuffer{})); + CHECK(Attachment.GetHash() == IoHash::Zero); } SUBCASE("Binary Empty") @@ -714,7 +938,7 @@ TEST_CASE("usonpackage") const CbAttachment Attachment(CbObject{}); CHECK_FALSE(Attachment.IsNull()); - CHECK(Attachment.IsBinary()); + CHECK_FALSE(Attachment.IsBinary()); CHECK(Attachment.IsObject()); CHECK(Attachment.GetHash() == CbObject().GetHash()); } @@ -833,17 +1057,16 @@ TEST_CASE("usonpackage.serialization") CHECK((Object1Attachment && Object1Attachment->AsObject().Equals(Object1))); CHECK((Object2Attachment && Object2Attachment->AsBinary() == Object2.GetBuffer())); - Package.AddAttachment(CbAttachment(SharedBuffer::Clone(Object1.GetView()))); + SharedBuffer Object1ClonedBuffer = SharedBuffer::Clone(Object1.GetOuterBuffer()); + Package.AddAttachment(CbAttachment(Object1ClonedBuffer)); Package.AddAttachment(CbAttachment(CbObject::Clone(Object2))); CHECK(Package.GetAttachments().size() == 2); CHECK(Package.FindAttachment(Object1.GetHash()) == Object1Attachment); CHECK(Package.FindAttachment(Object2.GetHash()) == Object2Attachment); - CHECK((Object1Attachment && Object1Attachment->AsObject().Equals(Object1))); - CHECK((Object1Attachment && Object1Attachment->AsBinary() == Object1.GetBuffer())); + CHECK((Object1Attachment && Object1Attachment->AsBinary() == Object1ClonedBuffer)); CHECK((Object2Attachment && Object2Attachment->AsObject().Equals(Object2))); - CHECK((Object2Attachment && Object2Attachment->AsBinary() == Object2.GetBuffer())); CHECK(std::is_sorted(begin(Package.GetAttachments()), end(Package.GetAttachments()))); } @@ -884,8 +1107,8 @@ TEST_CASE("usonpackage.serialization") const IoHash Level1Hash = Level1.GetHash(); const auto Resolver = [&Level2, &Level2Hash, &Level3, &Level3Hash, &Level4, &Level4Hash](const IoHash& Hash) -> SharedBuffer { - return Hash == Level2Hash ? Level2.GetBuffer() - : Hash == Level3Hash ? Level3.GetBuffer() + return Hash == Level2Hash ? Level2.GetOuterBuffer() + : Hash == Level3Hash ? Level3.GetOuterBuffer() : Hash == Level4Hash ? Level4 : SharedBuffer(); }; @@ -907,8 +1130,9 @@ TEST_CASE("usonpackage.serialization") const CbAttachment* const Level4Attachment = Package.FindAttachment(Level4Hash); CHECK((Level2Attachment && Level2Attachment->AsObject().Equals(Level2))); CHECK((Level3Attachment && Level3Attachment->AsObject().Equals(Level3))); - CHECK((Level4Attachment && Level4Attachment->AsBinary() != Level4 && - Level4Attachment->AsBinary().GetView().EqualBytes(Level4.GetView()))); + REQUIRE(Level4Attachment); + CHECK(Level4Attachment->AsBinary() != Level4); + CHECK(Level4Attachment->AsBinary().GetView().EqualBytes(Level4.GetView())); CHECK(std::is_sorted(begin(Package.GetAttachments()), end(Package.GetAttachments()))); @@ -932,15 +1156,15 @@ TEST_CASE("usonpackage.serialization") // Out of Order { - CbWriter Writer; - Writer.AddBinary(Level2.GetBuffer()); - Writer.AddObjectAttachment(Level2Hash); - Writer.AddBinary(Level4); - Writer.AddBinaryAttachment(Level4Hash); + CbWriter Writer; + CbAttachment Attachment2(Level2, Level2Hash); + Attachment2.Save(Writer); + CbAttachment Attachment4(Level4); + Attachment4.Save(Writer); + Writer.AddHash(Level1Hash); Writer.AddObject(Level1); - Writer.AddObjectAttachment(Level1Hash); - Writer.AddBinary(Level3.GetBuffer()); - Writer.AddObjectAttachment(Level3Hash); + CbAttachment Attachment3(Level3, Level3Hash); + Attachment3.Save(Writer); Writer.AddNull(); CbFieldIterator Fields = Writer.Save(); @@ -961,11 +1185,9 @@ TEST_CASE("usonpackage.serialization") const MemoryView FieldsOuterBufferView = Fields.GetOuterBuffer().GetView(); CHECK(Level2Attachment->AsObject().Equals(Level2)); - CHECK(FieldsOuterBufferView.Contains(Level2Attachment->AsBinary().GetView())); CHECK(Level2Attachment->GetHash() == Level2Hash); CHECK(Level3Attachment->AsObject().Equals(Level3)); - CHECK(FieldsOuterBufferView.Contains(Level3Attachment->AsBinary().GetView())); CHECK(Level3Attachment->GetHash() == Level3Hash); CHECK(Level4Attachment->AsBinary().GetView().EqualBytes(Level4.GetView())); @@ -983,21 +1205,23 @@ TEST_CASE("usonpackage.serialization") Writer.Reset(); FromArchive.Save(Writer); CbFieldIterator Saved = Writer.Save(); - CHECK(Saved.AsObject().Equals(Level1)); + + CHECK(Saved.AsHash() == Level1Hash); ++Saved; - CHECK(Saved.AsObjectAttachment() == Level1Hash); + CHECK(Saved.AsObject().Equals(Level1)); ++Saved; - CHECK(Saved.AsBinaryView().EqualBytes(Level2.GetView())); + CHECK_EQ(Saved.AsObjectAttachment(), Level2Hash); ++Saved; - CHECK(Saved.AsObjectAttachment() == Level2Hash); + CHECK(Saved.AsObject().Equals(Level2)); ++Saved; - CHECK(Saved.AsBinaryView().EqualBytes(Level3.GetView())); + CHECK_EQ(Saved.AsObjectAttachment(), Level3Hash); ++Saved; - CHECK(Saved.AsObjectAttachment() == Level3Hash); + CHECK(Saved.AsObject().Equals(Level3)); ++Saved; - CHECK(Saved.AsBinaryView().EqualBytes(Level4.GetView())); + CHECK_EQ(Saved.AsBinaryAttachment(), Level4Hash); ++Saved; - CHECK(Saved.AsBinaryAttachment() == Level4Hash); + SharedBuffer SavedLevel4Buffer = SharedBuffer::MakeView(Saved.AsBinaryView()); + CHECK(SavedLevel4Buffer.GetView().EqualBytes(Level4.GetView())); ++Saved; CHECK(Saved.IsNull()); ++Saved; diff --git a/zencore/compactbinaryvalidation.cpp b/zencore/compactbinaryvalidation.cpp index 52f625313..316da76a6 100644 --- a/zencore/compactbinaryvalidation.cpp +++ b/zencore/compactbinaryvalidation.cpp @@ -416,92 +416,125 @@ ValidateCbPackageField(MemoryView& View, CbValidateMode Mode, CbValidateError& E static IoHash ValidateCbPackageAttachment(CbFieldView& Value, MemoryView& View, CbValidateMode Mode, CbValidateError& Error) { - const CbObjectView ObjectView = Value.AsObjectView(); - if (Value.HasError()) + if (const CbObjectView ObjectView = Value.AsObjectView(); !Value.HasError()) { - const MemoryView BinaryView = Value.AsBinaryView(); - if (Value.HasError() && EnumHasAnyFlags(Mode, CbValidateMode::Package)) + return CbObject().GetHash(); + } + + if (const IoHash ObjectAttachmentHash = Value.AsObjectAttachment(); !Value.HasError()) + { + if (CbFieldView ObjectField = ValidateCbPackageField(View, Mode, Error)) { - if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) + const CbObjectView InnerObjectView = ObjectField.AsObjectView(); + if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && ObjectField.HasError()) { AddError(Error, CbValidateError::InvalidPackageFormat); } + else if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) && (ObjectAttachmentHash != InnerObjectView.GetHash())) + { + AddError(Error, CbValidateError::InvalidPackageHash); + } + return ObjectAttachmentHash; } - else if (BinaryView.GetSize()) + } + else if (const IoHash BinaryAttachmentHash = Value.AsBinaryAttachment(); !Value.HasError()) + { + if (CbFieldView BinaryField = ValidateCbPackageField(View, Mode, Error)) { - if (EnumHasAnyFlags(Mode, CbValidateMode::Package | CbValidateMode::PackageHash)) + const MemoryView BinaryView = BinaryField.AsBinaryView(); + if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && BinaryField.HasError()) { - CompressedBuffer Buffer = CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView)); - if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && Buffer.IsNull()) + AddError(Error, CbValidateError::InvalidPackageFormat); + } + else + { + if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && BinaryView.IsEmpty()) { - AddError(Error, CbValidateError::InvalidPackageFormat); + AddError(Error, CbValidateError::NullPackageAttachment); } - if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) && - (IoHash::FromBLAKE3(Buffer.GetRawHash()) != IoHash::HashBuffer(Buffer.DecompressToComposite()))) + if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) && (BinaryAttachmentHash != IoHash::HashBuffer(BinaryView))) { AddError(Error, CbValidateError::InvalidPackageHash); } - return IoHash::FromBLAKE3(Buffer.GetRawHash()); } + return BinaryAttachmentHash; } } - else + else if (const MemoryView BinaryView = Value.AsBinaryView(); !Value.HasError()) { - if (ObjectView) + if (BinaryView.GetSize() > 0) { - if (CbFieldView HashField = ValidateCbPackageField(View, Mode, Error)) + CompressedBuffer Buffer = CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView)); + if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && Buffer.IsNull()) { - const IoHash Hash = HashField.AsAttachment(); - if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && HashField.HasError()) - { - AddError(Error, CbValidateError::InvalidPackageFormat); - } - if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) && (Hash != ObjectView.GetHash())) - { - AddError(Error, CbValidateError::InvalidPackageHash); - } - return Hash; + AddError(Error, CbValidateError::NullPackageAttachment); } + if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) && + (IoHash::FromBLAKE3(Buffer.GetRawHash()) != IoHash::HashBuffer(Buffer.DecompressToComposite()))) + { + AddError(Error, CbValidateError::InvalidPackageHash); + } + return IoHash::FromBLAKE3(Buffer.GetRawHash()); } else { - return CbObject().GetHash(); + if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) + { + AddError(Error, CbValidateError::NullPackageAttachment); + } + return IoHash::HashBuffer(MemoryView()); } } - return {}; -} - -static IoHash -ValidateCbPackageObject(CbFieldView& Value, MemoryView& View, CbValidateMode Mode, CbValidateError& Error) -{ - CbObjectView Object = Value.AsObjectView(); - if (Value.HasError()) + else { if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) { AddError(Error, CbValidateError::InvalidPackageFormat); } } - else if (CbFieldView HashField = ValidateCbPackageField(View, Mode, Error)) + + return IoHash(); +} + +static IoHash +ValidateCbPackageObject(CbFieldView& Value, MemoryView& View, CbValidateMode Mode, CbValidateError& Error) +{ + if (IoHash RootObjectHash = Value.AsHash(); !Value.HasError() && !Value.IsAttachment()) { - const IoHash Hash = HashField.AsAttachment(); + CbFieldView RootObjectField = ValidateCbPackageField(View, Mode, Error); + if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) { - if (!Object) - { - AddError(Error, CbValidateError::NullPackageObject); - } - if (HashField.HasError()) + if (RootObjectField.HasError()) { AddError(Error, CbValidateError::InvalidPackageFormat); } - else if (Hash != Value.GetHash()) + } + + const CbObjectView RootObjectView = RootObjectField.AsObjectView(); + if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) + { + if (!RootObjectView) { - AddError(Error, CbValidateError::InvalidPackageHash); + AddError(Error, CbValidateError::NullPackageObject); } } - return Hash; + + if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) && (RootObjectHash != RootObjectView.GetHash())) + { + AddError(Error, CbValidateError::InvalidPackageHash); + } + + return RootObjectHash; + } + else + { + if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) + { + AddError(Error, CbValidateError::InvalidPackageFormat); + } } + return IoHash(); } @@ -562,24 +595,20 @@ ValidateCompactBinaryPackage(MemoryView View, CbValidateMode Mode) uint32_t ObjectCount = 0; while (CbFieldView Value = ValidateCbPackageField(View, Mode, Error)) { - if (Value.IsBinary()) + if (Value.IsHash() && !Value.IsAttachment()) { - const IoHash Hash = ValidateCbPackageAttachment(Value, View, Mode, Error); - if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) + ValidateCbPackageObject(Value, View, Mode, Error); + if (++ObjectCount > 1 && EnumHasAnyFlags(Mode, CbValidateMode::Package)) { - Attachments.push_back(Hash); - if (Value.AsBinaryView().IsEmpty()) - { - AddError(Error, CbValidateError::NullPackageAttachment); - } + AddError(Error, CbValidateError::MultiplePackageObjects); } } - else if (Value.IsObject()) + else if (Value.IsBinary() || Value.IsAttachment() || Value.IsObject()) { - ValidateCbPackageObject(Value, View, Mode, Error); - if (++ObjectCount > 1 && EnumHasAnyFlags(Mode, CbValidateMode::Package)) + const IoHash Hash = ValidateCbPackageAttachment(Value, View, Mode, Error); + if (EnumHasAnyFlags(Mode, CbValidateMode::Package)) { - AddError(Error, CbValidateError::MultiplePackageObjects); + Attachments.push_back(Hash); } } else if (Value.IsNull()) diff --git a/zencore/except.cpp b/zencore/except.cpp index 00cb826f6..9bd447308 100644 --- a/zencore/except.cpp +++ b/zencore/except.cpp @@ -28,7 +28,13 @@ ThrowLastError(std::string_view Message) std::string GetLastErrorAsString() { - throw std::error_code(::GetLastError(), std::system_category()).message(); + return GetWindowsErrorAsString(::GetLastError()); +} + +std::string +GetWindowsErrorAsString(uint32_t Win32ErrorCode) +{ + return std::error_code(Win32ErrorCode, std::system_category()).message(); } void diff --git a/zencore/httpclient.cpp b/zencore/httpclient.cpp deleted file mode 100644 index 268483403..000000000 --- a/zencore/httpclient.cpp +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/httpclient.h> - -#include <spdlog/spdlog.h> - -#include <doctest/doctest.h> - -namespace zen { - -TEST_CASE("httpclient") -{ - using namespace std::literals; - - SUBCASE("client") {} -} - -void -httpclient_forcelink() -{ -} - -} // namespace zen diff --git a/zencore/httpserver.cpp b/zencore/httpserver.cpp deleted file mode 100644 index e85c5ed2b..000000000 --- a/zencore/httpserver.cpp +++ /dev/null @@ -1,1641 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/httpserver.h> - -#define _WINSOCKAPI_ -#include <zencore/windows.h> -#include "iothreadpool.h" - -#include <atlbase.h> -#include <conio.h> -#include <http.h> -#include <new.h> -#include <zencore/compactbinary.h> -#include <zencore/compactbinarypackage.h> -#include <zencore/iobuffer.h> -#include <zencore/logging.h> -#include <zencore/refcount.h> -#include <zencore/stream.h> -#include <zencore/string.h> -#include <zencore/thread.h> -#include <charconv> -#include <span> -#include <string_view> - -#include <spdlog/spdlog.h> - -#include <doctest/doctest.h> - -#if ZEN_PLATFORM_WINDOWS -# pragma comment(lib, "httpapi.lib") -#endif - -////////////////////////////////////////////////////////////////////////// - -std::wstring -UTF8_to_wstring(const char* in) -{ - std::wstring out; - unsigned int codepoint; - - while (*in != 0) - { - unsigned char ch = static_cast<unsigned char>(*in); - - if (ch <= 0x7f) - codepoint = ch; - else if (ch <= 0xbf) - codepoint = (codepoint << 6) | (ch & 0x3f); - else if (ch <= 0xdf) - codepoint = ch & 0x1f; - else if (ch <= 0xef) - codepoint = ch & 0x0f; - else - codepoint = ch & 0x07; - - ++in; - - if (((*in & 0xc0) != 0x80) && (codepoint <= 0x10ffff)) - { - if (sizeof(wchar_t) > 2) - { - out.append(1, static_cast<wchar_t>(codepoint)); - } - else if (codepoint > 0xffff) - { - out.append(1, static_cast<wchar_t>(0xd800 + (codepoint >> 10))); - out.append(1, static_cast<wchar_t>(0xdc00 + (codepoint & 0x03ff))); - } - else if (codepoint < 0xd800 || codepoint >= 0xe000) - { - out.append(1, static_cast<wchar_t>(codepoint)); - } - } - } - - return out; -} - -////////////////////////////////////////////////////////////////////////// - -const char* -ReasonStringForHttpResultCode(int HttpCode) -{ - switch (HttpCode) - { - // 1xx Informational - - case 100: - return "Continue"; - case 101: - return "Switching Protocols"; - - // 2xx Success - - case 200: - return "OK"; - case 201: - return "Created"; - case 202: - return "Accepted"; - case 204: - return "No Content"; - case 205: - return "Reset Content"; - case 206: - return "Partial Content"; - - // 3xx Redirection - - case 300: - return "Multiple Choices"; - case 301: - return "Moved Permanently"; - case 302: - return "Found"; - case 303: - return "See Other"; - case 304: - return "Not Modified"; - case 305: - return "Use Proxy"; - case 306: - return "Switch Proxy"; - case 307: - return "Temporary Redirect"; - case 308: - return "Permanent Redirect"; - - // 4xx Client errors - - case 400: - return "Bad Request"; - case 401: - return "Unauthorized"; - case 402: - return "Payment Required"; - case 403: - return "Forbidden"; - case 404: - return "Not Found"; - case 405: - return "Method Not Allowed"; - case 406: - return "Not Acceptable"; - case 407: - return "Proxy Authentication Required"; - case 408: - return "Request Timeout"; - case 409: - return "Conflict"; - case 410: - return "Gone"; - case 411: - return "Length Required"; - case 412: - return "Precondition Failed"; - case 413: - return "Payload Too Large"; - case 414: - return "URI Too Long"; - case 415: - return "Unsupported Media Type"; - case 416: - return "Range Not Satisifiable"; - case 417: - return "Expectation Failed"; - case 418: - return "I'm a teapot"; - case 421: - return "Misdirected Request"; - case 422: - return "Unprocessable Entity"; - case 423: - return "Locked"; - case 424: - return "Failed Dependency"; - case 425: - return "Too Early"; - case 426: - return "Upgrade Required"; - case 428: - return "Precondition Required"; - case 429: - return "Too Many Requests"; - case 431: - return "Request Header Fields Too Large"; - - // 5xx Server errors - - case 500: - return "Internal Server Error"; - case 501: - return "Not Implemented"; - case 502: - return "Bad Gateway"; - case 503: - return "Service Unavailable"; - case 504: - return "Gateway Timeout"; - case 505: - return "HTTP Version Not Supported"; - case 506: - return "Variant Also Negotiates"; - case 507: - return "Insufficient Storage"; - case 508: - return "Loop Detected"; - case 510: - return "Not Extended"; - case 511: - return "Network Authentication Required"; - - default: - return "Unknown Result"; - } -} - -namespace zen { - -using namespace std::literals; - -static const uint32_t HashBinary = HashStringDjb2("application/octet-stream"sv); -static const uint32_t HashJson = HashStringDjb2("application/json"sv); -static const uint32_t HashYaml = HashStringDjb2("text/yaml"sv); -static const uint32_t HashText = HashStringDjb2("text/plain"sv); -static const uint32_t HashCompactBinary = HashStringDjb2("application/x-ue-cb"sv); -static const uint32_t HashCompactBinaryPackage = HashStringDjb2("application/x-ue-cbpkg"sv); - -HttpContentType -MapContentType(const std::string_view& ContentTypeString) -{ - if (!ContentTypeString.empty()) - { - const uint32_t CtHash = HashStringDjb2(ContentTypeString); - - if (CtHash == HashBinary) - { - return HttpContentType::kBinary; - } - else if (CtHash == HashCompactBinary) - { - return HttpContentType::kCbObject; - } - else if (CtHash == HashCompactBinaryPackage) - { - return HttpContentType::kCbPackage; - } - else if (CtHash == HashJson) - { - return HttpContentType::kJSON; - } - else if (CtHash == HashYaml) - { - return HttpContentType::kYAML; - } - else if (CtHash == HashText) - { - return HttpContentType::kText; - } - } - - return HttpContentType::kUnknownContentType; -} - -////////////////////////////////////////////////////////////////////////// - -HttpServerRequest::HttpServerRequest() -{ -} - -HttpServerRequest::~HttpServerRequest() -{ -} - -struct CbPackageHeader -{ - uint32_t HeaderMagic; - uint32_t AttachmentCount; - uint32_t Reserved1; - uint32_t Reserved2; -}; - -static constinit uint32_t kCbPkgMagic = 0xaa77aacc; - -struct CbAttachmentEntry -{ - uint64_t AttachmentSize; - uint32_t Reserved1; - IoHash AttachmentHash; -}; - -void -HttpServerRequest::WriteResponse(HttpResponse HttpResponseCode, CbPackage Data) -{ - const std::span<const CbAttachment>& Attachments = Data.GetAttachments(); - - std::vector<IoBuffer> ResponseBuffers; - ResponseBuffers.reserve(3 + Attachments.size()); // TODO: may want to use an additional fudge factor here to avoid growing since each - // attachment is likely to consist of several buffers - - uint64_t TotalAttachmentsSize = 0; - - // Fixed size header - - CbPackageHeader Hdr{.HeaderMagic = kCbPkgMagic, .AttachmentCount = gsl::narrow<uint32_t>(Attachments.size())}; - - ResponseBuffers.push_back(IoBufferBuilder::MakeCloneFromMemory(&Hdr, sizeof Hdr)); - - // Attachment metadata array - - IoBuffer AttachmentMetadataBuffer = IoBuffer{sizeof(CbAttachmentEntry) * (Attachments.size() + /* root */ 1)}; - - CbAttachmentEntry* AttachmentInfo = reinterpret_cast<CbAttachmentEntry*>(AttachmentMetadataBuffer.MutableData()); - - ResponseBuffers.push_back(AttachmentMetadataBuffer); // Attachment metadata - - // Root object - - IoBuffer RootIoBuffer = Data.GetObject().GetBuffer().AsIoBuffer(); - ResponseBuffers.push_back(RootIoBuffer); // Root object - - *AttachmentInfo++ = {.AttachmentSize = RootIoBuffer.Size(), .AttachmentHash = Data.GetObjectHash()}; - - // Attachment payloads - - for (const CbAttachment& Attachment : Attachments) - { - CompressedBuffer AttachmentBuffer = Attachment.AsCompressedBinary(); - CompositeBuffer Compressed = AttachmentBuffer.GetCompressed(); - - *AttachmentInfo++ = {.AttachmentSize = AttachmentBuffer.GetCompressedSize(), - .AttachmentHash = IoHash::FromBLAKE3(AttachmentBuffer.GetRawHash())}; - - for (const SharedBuffer& Segment : Compressed.GetSegments()) - { - ResponseBuffers.push_back(Segment.AsIoBuffer()); - TotalAttachmentsSize += Segment.GetSize(); - } - } - - return WriteResponse(HttpResponseCode, HttpContentType::kCbPackage, ResponseBuffers); -} - -void -HttpServerRequest::WriteResponse(HttpResponse HttpResponseCode, CbObject Data) -{ - SharedBuffer Buf = Data.GetBuffer(); - std::array<IoBuffer, 1> Buffers{IoBufferBuilder::MakeCloneFromMemory(Buf.GetData(), Buf.GetSize())}; - return WriteResponse(HttpResponseCode, HttpContentType::kCbObject, Buffers); -} - -void -HttpServerRequest::WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::string_view ResponseString) -{ - return WriteResponse(HttpResponseCode, ContentType, std::u8string_view{(char8_t*)ResponseString.data(), ResponseString.size()}); -} - -void -HttpServerRequest::WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, IoBuffer Blob) -{ - std::array<IoBuffer, 1> Buffers{Blob}; - return WriteResponse(HttpResponseCode, ContentType, Buffers); -} - -HttpServerRequest::QueryParams -HttpServerRequest::GetQueryParams() -{ - QueryParams Params; - - const std::string_view QStr = QueryString(); - - const char* QueryIt = QStr.data(); - const char* QueryEnd = QueryIt + QStr.size(); - - while (QueryIt != QueryEnd) - { - if (*QueryIt == '&') - { - ++QueryIt; - continue; - } - - const std::string_view Query{QueryIt, QueryEnd}; - - size_t DelimIndex = Query.find('&', 0); - - if (DelimIndex == std::string_view::npos) - { - DelimIndex = Query.size(); - } - - std::string_view ThisQuery{QueryIt, DelimIndex}; - - size_t EqIndex = ThisQuery.find('=', 0); - - if (EqIndex != std::string_view::npos) - { - std::string_view Parm{ThisQuery.data(), EqIndex}; - ThisQuery.remove_prefix(EqIndex + 1); - - Params.KvPairs.emplace_back(Parm, ThisQuery); - } - - QueryIt += DelimIndex; - } - - return Params; -} - -CbObject -HttpServerRequest::ReadPayloadObject() -{ - IoBuffer Payload = ReadPayload(); - - if (Payload) - { - return LoadCompactBinaryObject(std::move(Payload)); - } - else - { - return {}; - } -} - -CbPackage -HttpServerRequest::ReadPayloadPackage() -{ - // TODO: this should not read into a contiguous buffer! - - IoBuffer Payload = ReadPayload(); - MemoryInStream InStream(Payload); - BinaryReader Reader(InStream); - - if (!Payload) - { - return {}; - } - - CbPackage Package; - - CbPackageHeader Hdr; - Reader.Read(&Hdr, sizeof Hdr); - - if (Hdr.HeaderMagic != kCbPkgMagic) - { - // report error - return {}; - } - - uint32_t ChunkCount = Hdr.AttachmentCount + 1; - - std::unique_ptr<CbAttachmentEntry[]> AttachmentEntries{new CbAttachmentEntry[ChunkCount]}; - - Reader.Read(AttachmentEntries.get(), sizeof(CbAttachmentEntry) * ChunkCount); - - for (uint32_t i = 0; i < ChunkCount; ++i) - { - const uint64_t AttachmentSize = AttachmentEntries[i].AttachmentSize; - IoBuffer AttachmentBuffer{AttachmentSize}; - Reader.Read(AttachmentBuffer.MutableData(), AttachmentSize); - CompressedBuffer CompBuf(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer))); - - if (i == 0) - { - Package.SetObject(LoadCompactBinaryObject(CompBuf)); - } - else - { - CbAttachment Attachment(CompBuf); - Package.AddAttachment(Attachment); - } - } - - return Package; -} - -////////////////////////////////////////////////////////////////////////// -// -// http.sys implementation -// - -#if ZEN_PLATFORM_WINDOWS -class HttpSysServer; -class HttpTransaction; - -class HttpSysRequestHandler -{ -public: - HttpSysRequestHandler(HttpTransaction& InRequest) : m_Request(InRequest) {} - virtual ~HttpSysRequestHandler() = default; - - virtual void IssueRequest() = 0; - virtual HttpSysRequestHandler* HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) = 0; - - HttpTransaction& Transaction() { return m_Request; } - -private: - HttpTransaction& m_Request; // Outermost HTTP transaction object -}; - -/** HTTP transaction - - There will be an instance of this per pending and in-flight HTTP transaction - - */ -class HttpTransaction -{ -public: - HttpTransaction(HttpSysServer& Server) : m_HttpServer(Server), m_HttpHandler(&m_InitialHttpHandler) {} - - virtual ~HttpTransaction() {} - - enum class Status - { - kDone, - kRequestPending - }; - - Status HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred); - - static void __stdcall IoCompletionCallback(PTP_CALLBACK_INSTANCE Instance, - PVOID pContext /* HttpSysServer */, - PVOID pOverlapped, - ULONG IoResult, - ULONG_PTR NumberOfBytesTransferred, - PTP_IO Io) - { - UNREFERENCED_PARAMETER(Io); - UNREFERENCED_PARAMETER(Instance); - UNREFERENCED_PARAMETER(pContext); - - // Note that for a given transaction we may be in this completion function on more - // than one thread at any given moment. This means we need to be careful about what - // happens in here - - HttpTransaction* Transaction = CONTAINING_RECORD(pOverlapped, HttpTransaction, m_HttpOverlapped); - - if (Transaction->HandleCompletion(IoResult, NumberOfBytesTransferred) == HttpTransaction::Status::kDone) - { - delete Transaction; - } - } - - void IssueInitialRequest(); - - PTP_IO Iocp(); - HANDLE RequestQueueHandle(); - inline OVERLAPPED* Overlapped() { return &m_HttpOverlapped; } - inline HttpSysServer& Server() { return m_HttpServer; } - - inline PHTTP_REQUEST HttpRequest() { return m_InitialHttpHandler.HttpRequest(); } - -protected: - OVERLAPPED m_HttpOverlapped{}; - HttpSysServer& m_HttpServer; - HttpSysRequestHandler* m_HttpHandler{nullptr}; - RwLock m_Lock; - -private: - struct InitialRequestHandler : public HttpSysRequestHandler - { - inline PHTTP_REQUEST HttpRequest() { return (PHTTP_REQUEST)m_RequestBuffer; } - inline uint32_t RequestBufferSize() const { return sizeof m_RequestBuffer; } - - InitialRequestHandler(HttpTransaction& InRequest) : HttpSysRequestHandler(InRequest) {} - ~InitialRequestHandler() {} - - virtual void IssueRequest() override; - virtual HttpSysRequestHandler* HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) override; - - PHTTP_REQUEST m_HttpRequestPtr = (HTTP_REQUEST*)(m_RequestBuffer); - UCHAR m_RequestBuffer[16384 + sizeof(HTTP_REQUEST)]; - } m_InitialHttpHandler{*this}; -}; - -////////////////////////////////////////////////////////////////////////// - -class HttpMessageResponseRequest : public HttpSysRequestHandler -{ -public: - HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode); - HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, const char* Message); - HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, const void* Payload, size_t PayloadSize); - HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, std::span<IoBuffer> Blobs); - ~HttpMessageResponseRequest(); - - virtual void IssueRequest() override; - virtual HttpSysRequestHandler* HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) override; - - void SuppressResponseBody(); - -private: - std::vector<HTTP_DATA_CHUNK> m_HttpDataChunks; - uint64_t m_TotalDataSize = 0; // Sum of all chunk sizes - - uint16_t m_HttpResponseCode = 0; - uint32_t m_NextDataChunkOffset = 0; // This is used for responses where the number of chunks exceed the maximum number for one API call - uint32_t m_RemainingChunkCount = 0; - bool m_IsInitialResponse = true; - - void Initialize(uint16_t ResponseCode, std::span<IoBuffer> Blobs); - - std::vector<IoBuffer> m_DataBuffers; -}; - -HttpMessageResponseRequest::HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode) : HttpSysRequestHandler(InRequest) -{ - std::array<IoBuffer, 0> buffers; - - Initialize(ResponseCode, buffers); -} - -HttpMessageResponseRequest::HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, const char* Message) -: HttpSysRequestHandler(InRequest) -{ - IoBuffer MessageBuffer(IoBuffer::Wrap, Message, strlen(Message)); - std::array<IoBuffer, 1> buffers({MessageBuffer}); - - Initialize(ResponseCode, buffers); -} - -HttpMessageResponseRequest::HttpMessageResponseRequest(HttpTransaction& InRequest, - uint16_t ResponseCode, - const void* Payload, - size_t PayloadSize) -: HttpSysRequestHandler(InRequest) -{ - IoBuffer MessageBuffer(IoBuffer::Wrap, Payload, PayloadSize); - std::array<IoBuffer, 1> buffers({MessageBuffer}); - - Initialize(ResponseCode, buffers); -} - -HttpMessageResponseRequest::HttpMessageResponseRequest(HttpTransaction& InRequest, uint16_t ResponseCode, std::span<IoBuffer> Blobs) -: HttpSysRequestHandler(InRequest) -{ - Initialize(ResponseCode, Blobs); -} - -HttpMessageResponseRequest::~HttpMessageResponseRequest() -{ -} - -void -HttpMessageResponseRequest::Initialize(uint16_t ResponseCode, std::span<IoBuffer> Blobs) -{ - m_HttpResponseCode = ResponseCode; - - const uint32_t ChunkCount = (uint32_t)Blobs.size(); - - m_HttpDataChunks.resize(ChunkCount); - m_DataBuffers.reserve(ChunkCount); - - for (IoBuffer& Buffer : Blobs) - { - m_DataBuffers.emplace_back(std::move(Buffer)).MakeOwned(); - } - - // Initialize the full array up front - - uint64_t LocalDataSize = 0; - - { - PHTTP_DATA_CHUNK ChunkPtr = m_HttpDataChunks.data(); - - for (IoBuffer& Buffer : m_DataBuffers) - { - const ULONG BufferDataSize = (ULONG)Buffer.Size(); - - ZEN_ASSERT(BufferDataSize); - - IoBufferFileReference FileRef; - if (Buffer.GetFileReference(/* out */ FileRef)) - { - ChunkPtr->DataChunkType = HttpDataChunkFromFileHandle; - ChunkPtr->FromFileHandle.FileHandle = FileRef.FileHandle; - ChunkPtr->FromFileHandle.ByteRange.StartingOffset.QuadPart = FileRef.FileChunkOffset; - ChunkPtr->FromFileHandle.ByteRange.Length.QuadPart = BufferDataSize; - } - else - { - ChunkPtr->DataChunkType = HttpDataChunkFromMemory; - ChunkPtr->FromMemory.pBuffer = (void*)Buffer.Data(); - ChunkPtr->FromMemory.BufferLength = BufferDataSize; - } - ++ChunkPtr; - - LocalDataSize += BufferDataSize; - } - } - - m_RemainingChunkCount = ChunkCount; - m_TotalDataSize = LocalDataSize; -} - -void -HttpMessageResponseRequest::SuppressResponseBody() -{ - m_RemainingChunkCount = 0; - m_HttpDataChunks.clear(); - m_DataBuffers.clear(); -} - -HttpSysRequestHandler* -HttpMessageResponseRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) -{ - ZEN_UNUSED(NumberOfBytesTransferred); - ZEN_UNUSED(IoResult); - - if (m_RemainingChunkCount == 0) - return nullptr; // All done - - return this; -} - -void -HttpMessageResponseRequest::IssueRequest() -{ - HttpTransaction& Tx = Transaction(); - HTTP_REQUEST* const HttpReq = Tx.HttpRequest(); - PTP_IO const Iocp = Tx.Iocp(); - - StartThreadpoolIo(Iocp); - - // Split payload into batches to play well with the underlying API - - const int MaxChunksPerCall = 9999; - - const int ThisRequestChunkCount = std::min<int>(m_RemainingChunkCount, MaxChunksPerCall); - const int ThisRequestChunkOffset = m_NextDataChunkOffset; - - m_RemainingChunkCount -= ThisRequestChunkCount; - m_NextDataChunkOffset += ThisRequestChunkCount; - - ULONG SendFlags = 0; - - if (m_RemainingChunkCount) - { - // We need to make more calls to send the full amount of data - SendFlags |= HTTP_SEND_RESPONSE_FLAG_MORE_DATA; - } - - ULONG SendResult = 0; - - if (m_IsInitialResponse) - { - // Populate response structure - - HTTP_RESPONSE HttpResponse = {}; - - HttpResponse.EntityChunkCount = USHORT(ThisRequestChunkCount); - HttpResponse.pEntityChunks = m_HttpDataChunks.data() + ThisRequestChunkOffset; - - // Content-length header - - char ContentLengthString[32]; - _ui64toa_s(m_TotalDataSize, ContentLengthString, sizeof ContentLengthString, 10); - - PHTTP_KNOWN_HEADER ContentLengthHeader = &HttpResponse.Headers.KnownHeaders[HttpHeaderContentLength]; - ContentLengthHeader->pRawValue = ContentLengthString; - ContentLengthHeader->RawValueLength = (USHORT)strlen(ContentLengthString); - - // Content-type header - - PHTTP_KNOWN_HEADER ContentTypeHeader = &HttpResponse.Headers.KnownHeaders[HttpHeaderContentType]; - - ContentTypeHeader->pRawValue = "application/octet-stream"; /* TODO! We must respect the content type specified */ - ContentTypeHeader->RawValueLength = (USHORT)strlen(ContentTypeHeader->pRawValue); - - HttpResponse.StatusCode = m_HttpResponseCode; - HttpResponse.pReason = ReasonStringForHttpResultCode(m_HttpResponseCode); - HttpResponse.ReasonLength = (USHORT)strlen(HttpResponse.pReason); - - // Cache policy - - HTTP_CACHE_POLICY CachePolicy; - - CachePolicy.Policy = HttpCachePolicyNocache; // HttpCachePolicyUserInvalidates; - CachePolicy.SecondsToLive = 0; - - // Initial response API call - - SendResult = HttpSendHttpResponse(Tx.RequestQueueHandle(), - HttpReq->RequestId, - SendFlags, - &HttpResponse, - &CachePolicy, - NULL, - NULL, - 0, - Tx.Overlapped(), - NULL); - - m_IsInitialResponse = false; - } - else - { - // Subsequent response API calls - - SendResult = HttpSendResponseEntityBody(Tx.RequestQueueHandle(), - HttpReq->RequestId, - SendFlags, - (USHORT)ThisRequestChunkCount, // EntityChunkCount - &m_HttpDataChunks[ThisRequestChunkOffset], // EntityChunks - NULL, // BytesSent - NULL, // Reserved1 - 0, // Reserved2 - Tx.Overlapped(), // Overlapped - NULL // LogData - ); - } - - if ((SendResult != NO_ERROR) // Synchronous completion, but the completion event will still be posted to IOCP - && (SendResult != ERROR_IO_PENDING) // Asynchronous completion - ) - { - // Some error occurred, no completion will be posted - - CancelThreadpoolIo(Iocp); - - spdlog::error("failed to send HTTP response (error: {}) URL: {}", SendResult, HttpReq->pRawUrl); - - throw HttpServerException("Failed to send HTTP response", SendResult); - } -} - -////////////////////////////////////////////////////////////////////////// - -class HttpSysServer -{ - friend class HttpTransaction; - -public: - HttpSysServer(WinIoThreadPool& InThreadPool); - ~HttpSysServer(); - - void Initialize(const wchar_t* UrlPath); - void Run(bool TestMode); - - void RequestExit() { m_ShutdownEvent.Set(); } - - void StartServer(); - void StopServer(); - - void OnHandlingRequest(); - void IssueNewRequestMaybe(); - - inline bool IsOk() const { return m_IsOk; } - - void AddEndpoint(const char* Endpoint, HttpService& Service); - void RemoveEndpoint(const char* Endpoint, HttpService& Service); - -private: - bool m_IsOk = false; - bool m_IsHttpInitialized = false; - WinIoThreadPool& m_ThreadPool; - - std::wstring m_BaseUri; // http://*:nnnn/ - HTTP_SERVER_SESSION_ID m_HttpSessionId = 0; - HTTP_URL_GROUP_ID m_HttpUrlGroupId = 0; - HANDLE m_RequestQueueHandle = 0; - std::atomic_int32_t m_PendingRequests{0}; - int32_t m_MinPendingRequests = 4; - int32_t m_MaxPendingRequests = 32; - Event m_ShutdownEvent; -}; - -HttpSysServer::HttpSysServer(WinIoThreadPool& InThreadPool) : m_ThreadPool(InThreadPool) -{ - ULONG Result = HttpInitialize(HTTPAPI_VERSION_2, HTTP_INITIALIZE_SERVER, nullptr); - - if (Result != NO_ERROR) - { - return; - } - - m_IsHttpInitialized = true; - m_IsOk = true; -} - -HttpSysServer::~HttpSysServer() -{ - if (m_IsHttpInitialized) - { - HttpTerminate(HTTP_INITIALIZE_SERVER, nullptr); - } -} - -void -HttpSysServer::Initialize(const wchar_t* UrlPath) -{ - // check(bIsOk); - - ULONG Result = HttpCreateServerSession(HTTPAPI_VERSION_2, &m_HttpSessionId, 0); - - if (Result != NO_ERROR) - { - // Flag error - - return; - } - - Result = HttpCreateUrlGroup(m_HttpSessionId, &m_HttpUrlGroupId, 0); - - if (Result != NO_ERROR) - { - // Flag error - - return; - } - - m_BaseUri = UrlPath; - - Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, UrlPath, /* #TODO UrlContext */ HTTP_URL_CONTEXT(0), 0); - - if (Result != NO_ERROR) - { - // Flag error - - return; - } - - HTTP_BINDING_INFO HttpBindingInfo = {{0}, 0}; - - Result = HttpCreateRequestQueue(HTTPAPI_VERSION_2, NULL, NULL, 0, &m_RequestQueueHandle); - - if (Result != NO_ERROR) - { - // Flag error! - - return; - } - - HttpBindingInfo.Flags.Present = 1; - HttpBindingInfo.RequestQueueHandle = m_RequestQueueHandle; - - Result = HttpSetUrlGroupProperty(m_HttpUrlGroupId, HttpServerBindingProperty, &HttpBindingInfo, sizeof(HttpBindingInfo)); - - if (Result != NO_ERROR) - { - // Flag error! - - return; - } - - // Create I/O completion port - - m_ThreadPool.CreateIocp(m_RequestQueueHandle, HttpTransaction::IoCompletionCallback, this); - - // Check result! -} - -void -HttpSysServer::StartServer() -{ - int RequestCount = 32; - - for (int i = 0; i < RequestCount; ++i) - { - IssueNewRequestMaybe(); - } -} - -void -HttpSysServer::Run(bool TestMode) -{ - if (TestMode == false) - { - zen::logging::ConsoleLog().info("Zen Server running. Press ESC or Q to quit"); - } - - do - { - int WaitTimeout = -1; - - if (!TestMode) - { - WaitTimeout = 1000; - } - - if (!TestMode && _kbhit() != 0) - { - char c = (char)_getch(); - - if (c == 27 || c == 'Q' || c == 'q') - { - RequestApplicationExit(0); - } - } - - m_ShutdownEvent.Wait(WaitTimeout); - } while (!IsApplicationExitRequested()); -} - -void -HttpSysServer::OnHandlingRequest() -{ - --m_PendingRequests; - - if (m_PendingRequests > m_MinPendingRequests) - { - // We have more than the minimum number of requests pending, just let someone else - // enqueue new requests - return; - } - - IssueNewRequestMaybe(); -} - -void -HttpSysServer::IssueNewRequestMaybe() -{ - if (m_PendingRequests.load(std::memory_order::relaxed) >= m_MaxPendingRequests) - { - return; - } - - std::unique_ptr<HttpTransaction> Request = std::make_unique<HttpTransaction>(*this); - - Request->IssueInitialRequest(); - - // This may end up exceeding the MaxPendingRequests limit, but it's not - // really a problem. I'm doing it this way mostly to avoid dealing with - // exceptions here - ++m_PendingRequests; - - Request.release(); -} - -void -HttpSysServer::StopServer() -{ -} - -void -HttpSysServer::AddEndpoint(const char* UrlPath, HttpService& Service) -{ - if (UrlPath[0] == '/') - { - ++UrlPath; - } - - const std::wstring Path16 = UTF8_to_wstring(UrlPath); - Service.SetUriPrefixLength(Path16.size() + 1 /* leading slash */); - - // Convert to wide string - - std::wstring Url16 = m_BaseUri + Path16; - - ULONG Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, Url16.c_str(), HTTP_URL_CONTEXT(&Service), 0 /* Reserved */); - - if (Result != NO_ERROR) - { - spdlog::error("HttpAddUrlToUrlGroup failed with result {}", Result); - - return; - } -} - -void -HttpSysServer::RemoveEndpoint(const char* UrlPath, HttpService& Service) -{ - ZEN_UNUSED(Service); - - if (UrlPath[0] == '/') - { - ++UrlPath; - } - - const std::wstring Path16 = UTF8_to_wstring(UrlPath); - - // Convert to wide string - - std::wstring Url16 = m_BaseUri + Path16; - - ULONG Result = HttpRemoveUrlFromUrlGroup(m_HttpUrlGroupId, Url16.c_str(), 0); - - if (Result != NO_ERROR) - { - spdlog::error("HttpRemoveUrlFromUrlGroup failed with result {}", Result); - } -} - -////////////////////////////////////////////////////////////////////////// - -class HttpSysServerRequest : public HttpServerRequest -{ -public: - HttpSysServerRequest(HttpTransaction& Tx, HttpService& Service) : m_HttpTx(Tx) - { - PHTTP_REQUEST HttpRequestPtr = Tx.HttpRequest(); - - const int PrefixLength = Service.UriPrefixLength(); - const int AbsPathLength = HttpRequestPtr->CookedUrl.AbsPathLength / sizeof(char16_t); - - if (AbsPathLength >= PrefixLength) - { - // We convert the URI immediately because most of the code involved prefers to deal - // with utf8. This has some performance impact which I'd prefer to avoid but for now - // we just have to live with it - - WideToUtf8({(char16_t*)HttpRequestPtr->CookedUrl.pAbsPath + PrefixLength, gsl::narrow<size_t>(AbsPathLength - PrefixLength)}, - m_Uri); - } - else - { - m_Uri.Reset(); - } - - if (auto QueryStringLength = HttpRequestPtr->CookedUrl.QueryStringLength) - { - --QueryStringLength; - - WideToUtf8({(char16_t*)(HttpRequestPtr->CookedUrl.pQueryString) + 1, QueryStringLength / sizeof(char16_t)}, m_QueryString); - } - else - { - m_QueryString.Reset(); - } - - switch (HttpRequestPtr->Verb) - { - case HttpVerbOPTIONS: - m_Verb = HttpVerb::kOptions; - break; - - case HttpVerbGET: - m_Verb = HttpVerb::kGet; - break; - - case HttpVerbHEAD: - m_Verb = HttpVerb::kHead; - break; - - case HttpVerbPOST: - m_Verb = HttpVerb::kPost; - break; - - case HttpVerbPUT: - m_Verb = HttpVerb::kPut; - break; - - case HttpVerbDELETE: - m_Verb = HttpVerb::kDelete; - break; - - case HttpVerbCOPY: - m_Verb = HttpVerb::kCopy; - break; - - default: - // TODO: invalid request? - m_Verb = (HttpVerb)0; - break; - } - - const HTTP_KNOWN_HEADER& clh = HttpRequestPtr->Headers.KnownHeaders[HttpHeaderContentLength]; - std::string_view cl(clh.pRawValue, clh.RawValueLength); - std::from_chars(cl.data(), cl.data() + cl.size(), m_ContentLength); - - const HTTP_KNOWN_HEADER& CtHdr = HttpRequestPtr->Headers.KnownHeaders[HttpHeaderContentType]; - m_ContentType = MapContentType({CtHdr.pRawValue, CtHdr.RawValueLength}); - - const HTTP_KNOWN_HEADER& AcceptHdr = HttpRequestPtr->Headers.KnownHeaders[HttpHeaderAccept]; - m_AcceptType = MapContentType({AcceptHdr.pRawValue, AcceptHdr.RawValueLength}); - } - - ~HttpSysServerRequest() {} - - virtual IoBuffer ReadPayload() override - { - // This is presently synchronous for simplicity, but we - // need to implement an asynchronous version also - - HTTP_REQUEST* const HttpReq = m_HttpTx.HttpRequest(); - - IoBuffer PayloadBuffer(m_ContentLength); - - HttpContentType ContentType = RequestContentType(); - PayloadBuffer.SetContentType(ContentType); - - uint64_t BytesToRead = m_ContentLength; - - uint8_t* ReadPointer = reinterpret_cast<uint8_t*>(PayloadBuffer.MutableData()); - - // First deal with any payload which has already been copied - // into our request buffer - - const int EntityChunkCount = HttpReq->EntityChunkCount; - - for (int i = 0; i < EntityChunkCount; ++i) - { - HTTP_DATA_CHUNK& EntityChunk = HttpReq->pEntityChunks[i]; - - ZEN_ASSERT(EntityChunk.DataChunkType == HttpDataChunkFromMemory); - - const uint64_t BufferLength = EntityChunk.FromMemory.BufferLength; - - ZEN_ASSERT(BufferLength <= BytesToRead); - - memcpy(ReadPointer, EntityChunk.FromMemory.pBuffer, BufferLength); - - ReadPointer += BufferLength; - BytesToRead -= BufferLength; - } - - // Call http.sys API to receive the remaining data - - static const uint64_t kMaxBytesPerApiCall = 1 * 1024 * 1024; - - while (BytesToRead) - { - ULONG BytesRead = 0; - - const uint64_t BytesToReadThisCall = zen::Min(BytesToRead, kMaxBytesPerApiCall); - - ULONG ApiResult = HttpReceiveRequestEntityBody(m_HttpTx.RequestQueueHandle(), - HttpReq->RequestId, - 0, /* Flags */ - ReadPointer, - gsl::narrow<ULONG>(BytesToReadThisCall), - &BytesRead, - NULL /* Overlapped */ - ); - - if (ApiResult != NO_ERROR && ApiResult != ERROR_HANDLE_EOF) - { - throw HttpServerException("payload read failed", ApiResult); - } - - BytesToRead -= BytesRead; - ReadPointer += BytesRead; - } - - PayloadBuffer.MakeImmutable(); - - return PayloadBuffer; - } - - virtual void WriteResponse(HttpResponse HttpResponseCode) override - { - ZEN_ASSERT(m_IsHandled == false); - - m_Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)HttpResponseCode); - - if (m_SuppressBody) - { - m_Response->SuppressResponseBody(); - } - - m_IsHandled = true; - } - - virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs) override - { - ZEN_ASSERT(m_IsHandled == false); - ZEN_UNUSED(ContentType); - - m_Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)HttpResponseCode, Blobs); - - if (m_SuppressBody) - { - m_Response->SuppressResponseBody(); - } - - m_IsHandled = true; - } - - virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::u8string_view ResponseString) override - { - ZEN_ASSERT(m_IsHandled == false); - ZEN_UNUSED(ContentType); - - m_Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)HttpResponseCode, ResponseString.data(), ResponseString.size()); - - if (m_SuppressBody) - { - m_Response->SuppressResponseBody(); - } - - m_IsHandled = true; - } - - HttpTransaction& m_HttpTx; - HttpMessageResponseRequest* m_Response = nullptr; -}; - -////////////////////////////////////////////////////////////////////////// - -PTP_IO -HttpTransaction::Iocp() -{ - return m_HttpServer.m_ThreadPool.Iocp(); -} - -HANDLE -HttpTransaction::RequestQueueHandle() -{ - return m_HttpServer.m_RequestQueueHandle; -} - -void -HttpTransaction::IssueInitialRequest() -{ - m_InitialHttpHandler.IssueRequest(); -} - -HttpTransaction::Status -HttpTransaction::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) -{ - // We use this to ensure sequential execution of completion handlers - // for any given transaction. - RwLock::ExclusiveLockScope _(m_Lock); - - bool RequestPending = false; - - if (HttpSysRequestHandler* CurrentHandler = m_HttpHandler) - { - const bool IsInitialRequest = (CurrentHandler == &m_InitialHttpHandler); - - if (IsInitialRequest) - { - // Ensure we have a sufficient number of pending requests outstanding - m_HttpServer.OnHandlingRequest(); - } - - m_HttpHandler = CurrentHandler->HandleCompletion(IoResult, NumberOfBytesTransferred); - - if (m_HttpHandler) - { - try - { - m_HttpHandler->IssueRequest(); - - RequestPending = true; - } - catch (std::exception& Ex) - { - spdlog::error("exception caught from IssueRequest(): {}", Ex.what()); - - // something went wrong, no request is pending - } - } - else - { - if (IsInitialRequest == false) - { - delete CurrentHandler; - } - } - } - - m_HttpServer.IssueNewRequestMaybe(); - - if (RequestPending) - { - return Status::kRequestPending; - } - - return Status::kDone; -} - -////////////////////////////////////////////////////////////////////////// - -void -HttpTransaction::InitialRequestHandler::IssueRequest() -{ - PTP_IO Iocp = Transaction().Iocp(); - - StartThreadpoolIo(Iocp); - - HttpTransaction& Tx = Transaction(); - - HTTP_REQUEST* HttpReq = Tx.HttpRequest(); - - ULONG Result = HttpReceiveHttpRequest(Tx.RequestQueueHandle(), - HTTP_NULL_ID, - HTTP_RECEIVE_REQUEST_FLAG_COPY_BODY, - HttpReq, - RequestBufferSize(), - NULL, - Tx.Overlapped()); - - if (Result != ERROR_IO_PENDING && Result != NO_ERROR) - { - CancelThreadpoolIo(Iocp); - - if (Result == ERROR_MORE_DATA) - { - // ProcessReceiveAndPostResponse(pIoRequest, pServerContext->Io, ERROR_MORE_DATA); - } - - // CleanupHttpIoRequest(pIoRequest); - - fprintf(stderr, "HttpReceiveHttpRequest failed, error 0x%lx\n", Result); - - return; - } -} - -HttpSysRequestHandler* -HttpTransaction::InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) -{ - ZEN_UNUSED(IoResult); - ZEN_UNUSED(NumberOfBytesTransferred); - - // Route requests - - try - { - if (HttpService* Service = reinterpret_cast<HttpService*>(m_HttpRequestPtr->UrlContext)) - { - HttpSysServerRequest ThisRequest(Transaction(), *Service); - - Service->HandleRequest(ThisRequest); - - if (!ThisRequest.IsHandled()) - { - return new HttpMessageResponseRequest(Transaction(), 404, "Not found"); - } - - if (ThisRequest.m_Response) - { - return ThisRequest.m_Response; - } - } - - // Unable to route - return new HttpMessageResponseRequest(Transaction(), 404, "Item unknown"); - } - catch (std::exception& ex) - { - // TODO provide more meaningful error output - - return new HttpMessageResponseRequest(Transaction(), 500, ex.what()); - } -} -#endif // ZEN_PLATFORM_WINDOWS - -////////////////////////////////////////////////////////////////////////// - -struct HttpServer::Impl : public RefCounted -{ - WinIoThreadPool m_ThreadPool; - HttpSysServer m_HttpServer; - - Impl(int ThreadCount) : m_ThreadPool(ThreadCount), m_HttpServer(m_ThreadPool) {} - - void Initialize(int BasePort) - { - using namespace std::literals; - - WideStringBuilder<64> BaseUri; - BaseUri << u8"http://*:"sv << int64_t(BasePort) << u8"/"sv; - - m_HttpServer.Initialize(BaseUri.c_str()); - m_HttpServer.StartServer(); - } - - void Run(bool TestMode) { m_HttpServer.Run(TestMode); } - - void RequestExit() { m_HttpServer.RequestExit(); } - - void Cleanup() { m_HttpServer.StopServer(); } - - void AddEndpoint(const char* Endpoint, HttpService& Service) { m_HttpServer.AddEndpoint(Endpoint, Service); } - - void AddEndpoint([[maybe_unused]] const char* endpoint, [[maybe_unused]] std::function<void(HttpServerRequest&)> handler) - { - ZEN_NOT_IMPLEMENTED(); - } -}; - -HttpServer::HttpServer() -{ - m_Impl = new Impl(32); -} - -HttpServer::~HttpServer() -{ - m_Impl->Cleanup(); -} - -void -HttpServer::AddEndpoint(HttpService& Service) -{ - m_Impl->AddEndpoint(Service.BaseUri(), Service); -} - -void -HttpServer::AddEndpoint(const char* endpoint, std::function<void(HttpServerRequest&)> handler) -{ - m_Impl->AddEndpoint(endpoint, handler); -} - -void -HttpServer::Initialize(int BasePort) -{ - m_Impl->Initialize(BasePort); -} - -void -HttpServer::Run(bool TestMode) -{ - m_Impl->Run(TestMode); -} - -void -HttpServer::RequestExit() -{ - m_Impl->RequestExit(); -} - -////////////////////////////////////////////////////////////////////////// - -HttpServerException::HttpServerException(const char* Message, uint32_t Error) : m_ErrorCode(Error) -{ - using namespace fmt::literals; - - m_Message = "{} (HTTP error {})"_format(Message, m_ErrorCode); -} - -const char* -HttpServerException::what() const noexcept -{ - return m_Message.c_str(); -} - -////////////////////////////////////////////////////////////////////////// - -void -HttpRequestRouter::AddPattern(const char* Id, const char* Regex) -{ - ZEN_ASSERT(m_PatternMap.find(Id) == m_PatternMap.end()); - - m_PatternMap.insert({Id, Regex}); -} - -void -HttpRequestRouter::RegisterRoute(const char* Regex, HttpRequestRouter::HandlerFunc_t&& HandlerFunc, HttpVerb SupportedVerbs) -{ - // Expand patterns - - ExtendableStringBuilder<128> ExpandedRegex; - - size_t RegexLen = strlen(Regex); - - for (size_t i = 0; i < RegexLen;) - { - bool matched = false; - - if (Regex[i] == '{' && ((i == 0) || (Regex[i - 1] != '\\'))) - { - // Might have a pattern reference - find closing brace - - for (size_t j = i + 1; j < RegexLen; ++j) - { - if (Regex[j] == '}') - { - std::string Pattern(&Regex[i + 1], j - i - 1); - - if (auto it = m_PatternMap.find(Pattern); it != m_PatternMap.end()) - { - ExpandedRegex.Append(it->second.c_str()); - } - else - { - // Default to anything goes (or should this just be an error?) - - ExpandedRegex.Append("(.+?)"); - } - - // skip ahead - i = j + 1; - - matched = true; - - break; - } - } - } - - if (!matched) - { - ExpandedRegex.Append(Regex[i++]); - } - } - - m_Handlers.emplace_back(ExpandedRegex.c_str(), SupportedVerbs, std::move(HandlerFunc), Regex); -} - -bool -HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request) -{ - const HttpVerb Verb = Request.RequestVerb(); - - std::string_view Uri = Request.RelativeUri(); - HttpRouterRequest RouterRequest(Request); - - for (const auto& Handler : m_Handlers) - { - if ((Handler.Verbs & Verb) == Verb && regex_match(begin(Uri), end(Uri), RouterRequest.m_Match, Handler.RegEx)) - { - Handler.Handler(RouterRequest); - - return true; // Route matched - } - } - - return false; // No route matched -} - -TEST_CASE("http") -{ - using namespace std::literals; - - SUBCASE("router") - { - HttpRequestRouter r; - r.AddPattern("a", "[[:alpha:]]+"); - r.RegisterRoute( - "{a}", - [&](auto) {}, - HttpVerb::kGet); - - // struct TestHttpServerRequest : public HttpServerRequest - //{ - // TestHttpServerRequest(std::string_view Uri) : m_uri{Uri} {} - //}; - - // TestHttpServerRequest req{}; - // r.HandleRequest(req); - } -} - -void -http_forcelink() -{ -} - -} // namespace zen diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h index b214802bf..4fce129ea 100644 --- a/zencore/include/zencore/compactbinary.h +++ b/zencore/include/zencore/compactbinary.h @@ -1098,15 +1098,15 @@ public: /** Access the field as an object. Defaults to an empty object on error. */ inline CbObject AsObject() &; - - /** Access the field as an object. Defaults to an empty object on error. */ inline CbObject AsObject() &&; /** Access the field as an array. Defaults to an empty array on error. */ inline CbArray AsArray() &; - - /** Access the field as an array. Defaults to an empty array on error. */ inline CbArray AsArray() &&; + + /** Access the field as binary. Returns the provided default on error. */ + inline SharedBuffer AsBinary(const SharedBuffer& Default = SharedBuffer()) &; + inline SharedBuffer AsBinary(const SharedBuffer& Default = SharedBuffer()) &&; }; /** @@ -1268,6 +1268,20 @@ CbField::AsArray() && return IsArray() ? CbArray(AsArrayView(), std::move(*this)) : CbArray(); } +inline SharedBuffer +CbField::AsBinary(const SharedBuffer& Default) & +{ + const MemoryView View = AsBinaryView(); + return !HasError() ? SharedBuffer::MakeView(View, GetOuterBuffer()) : Default; +} + +inline SharedBuffer +CbField::AsBinary(const SharedBuffer& Default) && +{ + const MemoryView View = AsBinaryView(); + return !HasError() ? SharedBuffer::MakeView(View, std::move(*this).GetOuterBuffer()) : Default; +} + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /** diff --git a/zencore/include/zencore/compactbinarypackage.h b/zencore/include/zencore/compactbinarypackage.h index d60155d1a..e31bc4bfd 100644 --- a/zencore/include/zencore/compactbinarypackage.h +++ b/zencore/include/zencore/compactbinarypackage.h @@ -38,18 +38,27 @@ public: CbAttachment() = default; /** Construct a compact binary attachment. Value is cloned if not owned. */ - inline explicit CbAttachment(const CbObject& Value) : CbAttachment(Value, nullptr) {} + inline explicit CbAttachment(const CbObject& InValue) : CbAttachment(InValue, nullptr) {} /** Construct a compact binary attachment. Value is cloned if not owned. Hash must match Value. */ - inline explicit CbAttachment(const CbObject& Value, const IoHash& Hash) : CbAttachment(Value, &Hash) {} + inline explicit CbAttachment(const CbObject& InValue, const IoHash& Hash) : CbAttachment(InValue, &Hash) {} - /** Construct a binary attachment. Value is cloned if not owned. */ - ZENCORE_API explicit CbAttachment(const SharedBuffer& Value); + /** Construct a raw binary attachment. Value is cloned if not owned. */ + ZENCORE_API explicit CbAttachment(const SharedBuffer& InValue); - /** Construct a binary attachment. Value is cloned if not owned. Hash must match Value. */ - ZENCORE_API explicit CbAttachment(const SharedBuffer& Value, const IoHash& Hash); + /** Construct a raw binary attachment. Value is cloned if not owned. Hash must match Value. */ + ZENCORE_API explicit CbAttachment(const SharedBuffer& InValue, const IoHash& Hash); - /** Construct a binary attachment. Value is cloned if not owned. */ + /** Construct a raw binary attachment. Value is cloned if not owned. */ + ZENCORE_API explicit CbAttachment(const CompositeBuffer& InValue); + + /** Construct a raw binary attachment. Value is cloned if not owned. */ + ZENCORE_API explicit CbAttachment(CompositeBuffer&& InValue); + + /** Construct a raw binary attachment. Value is cloned if not owned. */ + ZENCORE_API explicit CbAttachment(CompositeBuffer&& InValue, const IoHash& Hash); + + /** Construct a compressed binary attachment. Value is cloned if not owned. */ ZENCORE_API explicit CbAttachment(const CompressedBuffer& InValue); ZENCORE_API explicit CbAttachment(CompressedBuffer&& InValue); @@ -66,13 +75,19 @@ public: ZENCORE_API [[nodiscard]] SharedBuffer AsBinary() const; /** Access the attachment as compressed binary. Defaults to a null buffer if the attachment is null. */ + ZENCORE_API [[nodiscard]] CompositeBuffer AsCompositeBinary() const; + + /** Access the attachment as compressed binary. Defaults to a null buffer if the attachment is null. */ ZENCORE_API [[nodiscard]] CompressedBuffer AsCompressedBinary() const; /** Access the attachment as compact binary. Defaults to a field iterator with no value on error. */ ZENCORE_API [[nodiscard]] CbObject AsObject() const; - /** Returns true if the attachment is either binary or an object */ - [[nodiscard]] inline bool IsBinary() const { return !IsNull(); } + /** Returns true if the attachment is binary */ + ZENCORE_API [[nodiscard]] bool IsBinary() const; + + /** Returns true if the attachment is compressed binary */ + ZENCORE_API [[nodiscard]] bool IsCompressedBinary() const; /** Returns whether the attachment is an object. */ ZENCORE_API [[nodiscard]] bool IsObject() const; @@ -122,7 +137,19 @@ private: CbObjectValue(CbObject&& InObject, const IoHash& InHash) : Object(std::move(InObject)), Hash(InHash) {} }; - std::variant<CompressedBuffer, CbObjectValue> Value; + struct BinaryValue + { + CompositeBuffer Buffer; + IoHash Hash; + + BinaryValue(const CompositeBuffer& InBuffer) : Buffer(InBuffer.MakeOwned()), Hash(IoHash::HashBuffer(InBuffer)) {} + BinaryValue(const CompositeBuffer& InBuffer, const IoHash& InHash) : Buffer(InBuffer.MakeOwned()), Hash(InHash) {} + BinaryValue(CompositeBuffer&& InBuffer) : Buffer(std::move(InBuffer)), Hash(IoHash::HashBuffer(Buffer)) {} + BinaryValue(CompositeBuffer&& InBuffer, const IoHash& InHash) : Buffer(std::move(InBuffer)), Hash(InHash) {} + BinaryValue(SharedBuffer&& InBuffer, const IoHash& InHash) : Buffer(std::move(InBuffer)), Hash(InHash) {} + }; + + std::variant<nullptr_t, CbObjectValue, BinaryValue, CompressedBuffer> Value; }; /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -290,9 +317,7 @@ public: * The iterator is advanced as object and attachment fields are consumed from it. */ ZENCORE_API bool TryLoad(CbFieldIterator& Fields); - - ZENCORE_API bool TryLoad(IoBuffer& Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc, AttachmentResolver* Mapper = nullptr); - + ZENCORE_API bool TryLoad(IoBuffer Buffer, BufferAllocator Allocator = UniqueBuffer::Alloc, AttachmentResolver* Mapper = nullptr); ZENCORE_API bool TryLoad(BinaryReader& Reader, BufferAllocator Allocator = UniqueBuffer::Alloc, AttachmentResolver* Mapper = nullptr); /** Save the object and attachments into the writer as a stream of compact binary fields. */ @@ -313,6 +338,17 @@ private: IoHash ObjectHash; }; +namespace legacy { + void SaveCbAttachment(const CbAttachment& Attachment, CbWriter& Writer); + void SaveCbPackage(const CbPackage& Package, CbWriter& Writer); + void SaveCbPackage(const CbPackage& Package, BinaryWriter& Ar); + bool TryLoadCbPackage(CbPackage& Package, IoBuffer Buffer, BufferAllocator Allocator, CbPackage::AttachmentResolver* Mapper = nullptr); + bool TryLoadCbPackage(CbPackage& Package, + BinaryReader& Reader, + BufferAllocator Allocator, + CbPackage::AttachmentResolver* Mapper = nullptr); +} // namespace legacy + void usonpackage_forcelink(); // internal } // namespace zen diff --git a/zencore/include/zencore/compactbinaryvalidation.h b/zencore/include/zencore/compactbinaryvalidation.h index 9799c594a..b1fab9572 100644 --- a/zencore/include/zencore/compactbinaryvalidation.h +++ b/zencore/include/zencore/compactbinaryvalidation.h @@ -58,10 +58,13 @@ enum class CbValidateMode : uint32_t Padding = 1 << 3, /** - * Validate that a package or attachment has the expected fields and matches its saved hashes. + * Validate that a package or attachment has the expected fields. */ Package = 1 << 4, + /** + * Validate that a package or attachment matches its saved hashes. + */ PackageHash = 1 << 5, /** Perform all validation described above. */ diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h index 0ae31dc71..8625f01d0 100644 --- a/zencore/include/zencore/except.h +++ b/zencore/include/zencore/except.h @@ -55,5 +55,18 @@ ThrowSystemException(const char* Message) ZENCORE_API void ThrowLastError(std::string_view Message); ZENCORE_API void ThrowLastError(std::string_view Message, const std::source_location& Location); ZENCORE_API std::string GetLastErrorAsString(); +ZENCORE_API std::string GetWindowsErrorAsString(uint32_t Win32ErrorCode); + +inline std::error_code +MakeWin32ErrorCode(uint32_t Win32ErrorCode) noexcept +{ + return std::error_code(Win32ErrorCode, std::system_category()); +} + +inline std::error_code +MakeErrorCodeFromLastError() noexcept +{ + return std::error_code(::GetLastError(), std::system_category()); +} } // namespace zen diff --git a/zencore/include/zencore/httpclient.h b/zencore/include/zencore/httpclient.h deleted file mode 100644 index 4b30eb09b..000000000 --- a/zencore/include/zencore/httpclient.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -#include <zencore/string.h> -#include <gsl/gsl-lite.hpp> - -namespace zen { - -/** Asynchronous HTTP client implementation for Zen use cases - */ -class HttpClient -{ -public: -private: -}; - -} // namespace zen - -void httpclient_forcelink(); // internal diff --git a/zencore/include/zencore/httpserver.h b/zencore/include/zencore/httpserver.h deleted file mode 100644 index 19ac8732e..000000000 --- a/zencore/include/zencore/httpserver.h +++ /dev/null @@ -1,401 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencore.h" - -#include <zencore/enumflags.h> -#include <zencore/iobuffer.h> -#include <zencore/refcount.h> -#include <zencore/string.h> - -#include <functional> -#include <gsl/gsl-lite.hpp> -#include <list> -#include <regex> -#include <span> -#include <unordered_map> - -namespace zen { - -using HttpContentType = ZenContentType; - -class IoBuffer; -class CbObject; -class CbPackage; -class StringBuilderBase; - -enum class HttpVerb -{ - kGet = 1 << 0, - kPut = 1 << 1, - kPost = 1 << 2, - kDelete = 1 << 3, - kHead = 1 << 4, - kCopy = 1 << 5, - kOptions = 1 << 6 -}; - -gsl_DEFINE_ENUM_BITMASK_OPERATORS(HttpVerb); - -enum class HttpResponse -{ - // 1xx - Informational - - Continue = 100, //!< Indicates that the initial part of a request has been received and has not yet been rejected by the server. - SwitchingProtocols = 101, //!< Indicates that the server understands and is willing to comply with the client's request, via the - //!< Upgrade header field, for a change in the application protocol being used on this connection. - Processing = 102, //!< Is an interim response used to inform the client that the server has accepted the complete request, but has not - //!< yet completed it. - EarlyHints = 103, //!< Indicates to the client that the server is likely to send a final response with the header fields included in - //!< the informational response. - - // 2xx - Successful - - OK = 200, //!< Indicates that the request has succeeded. - Created = 201, //!< Indicates that the request has been fulfilled and has resulted in one or more new resources being created. - Accepted = 202, //!< Indicates that the request has been accepted for processing, but the processing has not been completed. - NonAuthoritativeInformation = 203, //!< Indicates that the request was successful but the enclosed payload has been modified from that - //!< of the origin server's 200 (OK) response by a transforming proxy. - NoContent = 204, //!< Indicates that the server has successfully fulfilled the request and that there is no additional content to send - //!< in the response payload body. - ResetContent = 205, //!< Indicates that the server has fulfilled the request and desires that the user agent reset the \"document - //!< view\", which caused the request to be sent, to its original state as received from the origin server. - PartialContent = 206, //!< Indicates that the server is successfully fulfilling a range request for the target resource by transferring - //!< one or more parts of the selected representation that correspond to the satisfiable ranges found in the - //!< requests's Range header field. - MultiStatus = 207, //!< Provides status for multiple independent operations. - AlreadyReported = 208, //!< Used inside a DAV:propstat response element to avoid enumerating the internal members of multiple bindings - //!< to the same collection repeatedly. [RFC 5842] - IMUsed = 226, //!< The server has fulfilled a GET request for the resource, and the response is a representation of the result of one - //!< or more instance-manipulations applied to the current instance. - - // 3xx - Redirection - - MultipleChoices = 300, //!< Indicates that the target resource has more than one representation, each with its own more specific - //!< identifier, and information about the alternatives is being provided so that the user (or user agent) can - //!< select a preferred representation by redirecting its request to one or more of those identifiers. - MovedPermanently = 301, //!< Indicates that the target resource has been assigned a new permanent URI and any future references to this - //!< resource ought to use one of the enclosed URIs. - Found = 302, //!< Indicates that the target resource resides temporarily under a different URI. - SeeOther = 303, //!< Indicates that the server is redirecting the user agent to a different resource, as indicated by a URI in the - //!< Location header field, that is intended to provide an indirect response to the original request. - NotModified = 304, //!< Indicates that a conditional GET request has been received and would have resulted in a 200 (OK) response if it - //!< were not for the fact that the condition has evaluated to false. - UseProxy = 305, //!< \deprecated \parblock Due to security concerns regarding in-band configuration of a proxy. \endparblock - //!< The requested resource MUST be accessed through the proxy given by the Location field. - TemporaryRedirect = 307, //!< Indicates that the target resource resides temporarily under a different URI and the user agent MUST NOT - //!< change the request method if it performs an automatic redirection to that URI. - PermanentRedirect = 308, //!< The target resource has been assigned a new permanent URI and any future references to this resource - //!< ought to use one of the enclosed URIs. [...] This status code is similar to 301 Moved Permanently - //!< (Section 7.3.2 of rfc7231), except that it does not allow rewriting the request method from POST to GET. - - // 4xx - Client Error - BadRequest = 400, //!< Indicates that the server cannot or will not process the request because the received syntax is invalid, - //!< nonsensical, or exceeds some limitation on what the server is willing to process. - Unauthorized = 401, //!< Indicates that the request has not been applied because it lacks valid authentication credentials for the - //!< target resource. - PaymentRequired = 402, //!< *Reserved* - Forbidden = 403, //!< Indicates that the server understood the request but refuses to authorize it. - NotFound = 404, //!< Indicates that the origin server did not find a current representation for the target resource or is not willing - //!< to disclose that one exists. - MethodNotAllowed = 405, //!< Indicates that the method specified in the request-line is known by the origin server but not supported by - //!< the target resource. - NotAcceptable = 406, //!< Indicates that the target resource does not have a current representation that would be acceptable to the - //!< user agent, according to the proactive negotiation header fields received in the request, and the server is - //!< unwilling to supply a default representation. - ProxyAuthenticationRequired = - 407, //!< Is similar to 401 (Unauthorized), but indicates that the client needs to authenticate itself in order to use a proxy. - RequestTimeout = - 408, //!< Indicates that the server did not receive a complete request message within the time that it was prepared to wait. - Conflict = 409, //!< Indicates that the request could not be completed due to a conflict with the current state of the resource. - Gone = 410, //!< Indicates that access to the target resource is no longer available at the origin server and that this condition is - //!< likely to be permanent. - LengthRequired = 411, //!< Indicates that the server refuses to accept the request without a defined Content-Length. - PreconditionFailed = - 412, //!< Indicates that one or more preconditions given in the request header fields evaluated to false when tested on the server. - PayloadTooLarge = 413, //!< Indicates that the server is refusing to process a request because the request payload is larger than the - //!< server is willing or able to process. - URITooLong = 414, //!< Indicates that the server is refusing to service the request because the request-target is longer than the - //!< server is willing to interpret. - UnsupportedMediaType = 415, //!< Indicates that the origin server is refusing to service the request because the payload is in a format - //!< not supported by the target resource for this method. - RangeNotSatisfiable = 416, //!< Indicates that none of the ranges in the request's Range header field overlap the current extent of the - //!< selected resource or that the set of ranges requested has been rejected due to invalid ranges or an - //!< excessive request of small or overlapping ranges. - ExpectationFailed = 417, //!< Indicates that the expectation given in the request's Expect header field could not be met by at least - //!< one of the inbound servers. - ImATeapot = 418, //!< Any attempt to brew coffee with a teapot should result in the error code 418 I'm a teapot. - UnprocessableEntity = 422, //!< Means the server understands the content type of the request entity (hence a 415(Unsupported Media - //!< Type) status code is inappropriate), and the syntax of the request entity is correct (thus a 400 (Bad - //!< Request) status code is inappropriate) but was unable to process the contained instructions. - Locked = 423, //!< Means the source or destination resource of a method is locked. - FailedDependency = 424, //!< Means that the method could not be performed on the resource because the requested action depended on - //!< another action and that action failed. - UpgradeRequired = 426, //!< Indicates that the server refuses to perform the request using the current protocol but might be willing to - //!< do so after the client upgrades to a different protocol. - PreconditionRequired = 428, //!< Indicates that the origin server requires the request to be conditional. - TooManyRequests = 429, //!< Indicates that the user has sent too many requests in a given amount of time (\"rate limiting\"). - RequestHeaderFieldsTooLarge = - 431, //!< Indicates that the server is unwilling to process the request because its header fields are too large. - UnavailableForLegalReasons = - 451, //!< This status code indicates that the server is denying access to the resource in response to a legal demand. - - // 5xx - Server Error - - InternalServerError = - 500, //!< Indicates that the server encountered an unexpected condition that prevented it from fulfilling the request. - NotImplemented = 501, //!< Indicates that the server does not support the functionality required to fulfill the request. - BadGateway = 502, //!< Indicates that the server, while acting as a gateway or proxy, received an invalid response from an inbound - //!< server it accessed while attempting to fulfill the request. - ServiceUnavailable = 503, //!< Indicates that the server is currently unable to handle the request due to a temporary overload or - //!< scheduled maintenance, which will likely be alleviated after some delay. - GatewayTimeout = 504, //!< Indicates that the server, while acting as a gateway or proxy, did not receive a timely response from an - //!< upstream server it needed to access in order to complete the request. - HTTPVersionNotSupported = 505, //!< Indicates that the server does not support, or refuses to support, the protocol version that was - //!< used in the request message. - VariantAlsoNegotiates = - 506, //!< Indicates that the server has an internal configuration error: the chosen variant resource is configured to engage in - //!< transparent content negotiation itself, and is therefore not a proper end point in the negotiation process. - InsufficientStorage = 507, //!< Means the method could not be performed on the resource because the server is unable to store the - //!< representation needed to successfully complete the request. - LoopDetected = 508, //!< Indicates that the server terminated an operation because it encountered an infinite loop while processing a - //!< request with "Depth: infinity". [RFC 5842] - NotExtended = 510, //!< The policy for accessing the resource has not been met in the request. [RFC 2774] - NetworkAuthenticationRequired = 511, //!< Indicates that the client needs to authenticate to gain network access. -}; - -/** HTTP Server Request - */ -class HttpServerRequest -{ -public: - HttpServerRequest(); - ~HttpServerRequest(); - - // Synchronous operations - - [[nodiscard]] inline std::string_view RelativeUri() const { return m_Uri; } // Returns URI without service prefix - [[nodiscard]] inline std::string_view QueryString() const { return m_QueryString; } - inline bool IsHandled() const { return m_IsHandled; } - - struct QueryParams - { - std::vector<std::pair<std::string_view, std::string_view>> KvPairs; - - std::string_view GetValue(std::string_view ParamName) - { - for (const auto& Kv : KvPairs) - { - const std::string_view& Key = Kv.first; - - if (Key.size() == ParamName.size()) - { - if (0 == _strnicmp(Key.data(), ParamName.data(), Key.size())) - { - return Kv.second; - } - } - } - - return std::string_view(); - } - }; - - QueryParams GetQueryParams(); - - inline HttpVerb RequestVerb() const { return m_Verb; } - inline HttpContentType RequestContentType() { return m_ContentType; } - inline HttpContentType AcceptContentType() { return m_AcceptType; } - - const char* HeaderAccept() const; - const char* HeaderAcceptEncoding() const; - const char* HeaderContentType() const; - const char* HeaderContentEncoding() const; - inline uint64_t HeaderContentLength() const { return m_ContentLength; } - - void SetSuppressResponseBody() { m_SuppressBody = true; } - - // Asynchronous operations - - /** Read POST/PUT payload - - This will return a null buffer if the contents are not fully available yet, and the handler should - at that point return - another completion request will be issued once the contents have been received - fully. - - NOTE: in practice, via the http.sys implementation this always operates synchronously. This should - be updated to provide fully asynchronous operation for better scalability on shared instances - */ - virtual IoBuffer ReadPayload() = 0; - - ZENCORE_API CbObject ReadPayloadObject(); - ZENCORE_API CbPackage ReadPayloadPackage(); - - /** Respond with payload - - Note that this is destructive in the sense that the IoBuffer instances referred to by Blobs will be - moved into our response handler array where they are kept alive, in order to reduce ref-counting storms - */ - virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs) = 0; - virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, IoBuffer Blob); - virtual void WriteResponse(HttpResponse HttpResponseCode) = 0; - - virtual void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::u8string_view ResponseString) = 0; - - void WriteResponse(HttpResponse HttpResponseCode, CbObject Data); - void WriteResponse(HttpResponse HttpResponseCode, CbPackage Package); - void WriteResponse(HttpResponse HttpResponseCode, HttpContentType ContentType, std::string_view ResponseString); - -protected: - bool m_IsHandled = false; - bool m_SuppressBody = false; - HttpVerb m_Verb = HttpVerb::kGet; - uint64_t m_ContentLength = ~0ull; - HttpContentType m_ContentType = HttpContentType::kBinary; - HttpContentType m_AcceptType = HttpContentType::kUnknownContentType; - ExtendableStringBuilder<256> m_Uri; - ExtendableStringBuilder<256> m_QueryString; -}; - -class HttpServerException : public std::exception -{ -public: - HttpServerException(const char* Message, uint32_t Error); - - virtual const char* what() const noexcept override; - -private: - uint32_t m_ErrorCode; - std::string m_Message; -}; - -/** - * Base class for implementing an HTTP "service" - * - * A service exposes one or more endpoints with a certain URI prefix - * - */ - -class HttpService -{ -public: - HttpService() = default; - virtual ~HttpService() = default; - - virtual const char* BaseUri() const = 0; - virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) = 0; - - // Internals - - inline void SetUriPrefixLength(size_t PrefixLength) { m_UriPrefixLength = (int)PrefixLength; } - inline int UriPrefixLength() const { return m_UriPrefixLength; } - -private: - int m_UriPrefixLength = 0; -}; - -/** HTTP server - * - * Implements the main event loop to service HTTP requests, and handles routing - * requests to the appropriate endpoint handler as registered via AddEndpoint - */ -class HttpServer -{ -public: - HttpServer(); - ~HttpServer(); - - void AddEndpoint(const char* endpoint, std::function<void(HttpServerRequest&)> handler); - void AddEndpoint(HttpService& Service); - - void Initialize(int BasePort); - void Run(bool TestMode); - void RequestExit(); - -private: - struct Impl; - - RefPtr<Impl> m_Impl; -}; - -////////////////////////////////////////////////////////////////////////// - -class HttpRouterRequest -{ -public: - HttpRouterRequest(HttpServerRequest& Request) : m_HttpRequest(Request) {} - - ZENCORE_API std::string GetCapture(uint32_t Index) const; - inline HttpServerRequest& ServerRequest() { return m_HttpRequest; } - -private: - using MatchResults_t = std::match_results<std::string_view::const_iterator>; - - HttpServerRequest& m_HttpRequest; - MatchResults_t m_Match; - - friend class HttpRequestRouter; -}; - -inline std::string -HttpRouterRequest::GetCapture(uint32_t Index) const -{ - ZEN_ASSERT(Index < m_Match.size()); - - return m_Match[Index]; -} - -////////////////////////////////////////////////////////////////////////// - -/** HTTP request router helper - * - * This helper class allows a service implementer to register one or more - * endpoints using pattern matching (currently using regex matching) - * - */ - -class HttpRequestRouter -{ -public: - typedef std::function<void(HttpRouterRequest&)> HandlerFunc_t; - - void AddPattern(const char* Id, const char* Regex); - void RegisterRoute(const char* Regex, HandlerFunc_t&& HandlerFunc, HttpVerb SupportedVerbs); - bool HandleRequest(zen::HttpServerRequest& Request); - -private: - struct HandlerEntry - { - HandlerEntry(const char* Regex, HttpVerb SupportedVerbs, HandlerFunc_t&& Handler, const char* Pattern) - : RegEx(Regex, std::regex::icase | std::regex::ECMAScript) - , Verbs(SupportedVerbs) - , Handler(std::move(Handler)) - , Pattern(Pattern) - { - } - - ~HandlerEntry() = default; - - std::regex RegEx; - HttpVerb Verbs; - HandlerFunc_t Handler; - const char* Pattern; - }; - - std::list<HandlerEntry> m_Handlers; - std::unordered_map<std::string, std::string> m_PatternMap; -}; - -////////////////////////////////////////////////////////////////////////// -// -// HTTP Client -// - -class HttpClient -{ -}; - -} // namespace zen - -void http_forcelink(); // internal diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h index 121b73adc..034c3566f 100644 --- a/zencore/include/zencore/iobuffer.h +++ b/zencore/include/zencore/iobuffer.h @@ -19,6 +19,7 @@ enum class ZenContentType : uint8_t kCbObject, kCbPackage, kYAML, + kCbPackageOffer, kUnknownContentType }; diff --git a/zencore/include/zencore/logging.h b/zencore/include/zencore/logging.h index 7a08cc48b..a2404a5e9 100644 --- a/zencore/include/zencore/logging.h +++ b/zencore/include/zencore/logging.h @@ -19,5 +19,4 @@ spdlog::logger& Get(std::string_view Name); void InitializeLogging(); void ShutdownLogging(); - } // namespace zen::logging diff --git a/zencore/include/zencore/refcount.h b/zencore/include/zencore/refcount.h index 288b649c6..50bd82f59 100644 --- a/zencore/include/zencore/refcount.h +++ b/zencore/include/zencore/refcount.h @@ -117,6 +117,7 @@ public: [[nodiscard]] inline bool IsNull() const { return m_Ref == nullptr; } inline explicit operator bool() const { return m_Ref != nullptr; } inline T* operator->() const { return m_Ref; } + inline T* Get() const { return m_Ref; } inline std::strong_ordering operator<=>(const Ref& Rhs) const = default; diff --git a/zencore/iothreadpool.cpp b/zencore/iothreadpool.cpp deleted file mode 100644 index 4ed81d7a2..000000000 --- a/zencore/iothreadpool.cpp +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "iothreadpool.h" - -namespace zen { - -WinIoThreadPool::WinIoThreadPool(int InThreadCount) -{ - // Thread pool setup - - m_ThreadPool = CreateThreadpool(NULL); - - SetThreadpoolThreadMinimum(m_ThreadPool, InThreadCount); - SetThreadpoolThreadMaximum(m_ThreadPool, InThreadCount * 2); - - InitializeThreadpoolEnvironment(&m_CallbackEnvironment); - - m_CleanupGroup = CreateThreadpoolCleanupGroup(); - - SetThreadpoolCallbackPool(&m_CallbackEnvironment, m_ThreadPool); - - SetThreadpoolCallbackCleanupGroup(&m_CallbackEnvironment, m_CleanupGroup, NULL); -} - -WinIoThreadPool::~WinIoThreadPool() -{ - CloseThreadpool(m_ThreadPool); -} - -void -WinIoThreadPool::CreateIocp(HANDLE IoHandle, PTP_WIN32_IO_CALLBACK Callback, void* Context) -{ - m_ThreadPoolIo = CreateThreadpoolIo(IoHandle, Callback, Context, &m_CallbackEnvironment); -} - -} // namespace zen diff --git a/zencore/iothreadpool.h b/zencore/iothreadpool.h deleted file mode 100644 index f64868540..000000000 --- a/zencore/iothreadpool.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/windows.h> - -namespace zen { - -////////////////////////////////////////////////////////////////////////// -// -// Thread pool. Implemented in terms of Windows thread pool right now, will -// need a cross-platform implementation eventually -// - -class WinIoThreadPool -{ -public: - WinIoThreadPool(int InThreadCount); - ~WinIoThreadPool(); - - void CreateIocp(HANDLE IoHandle, PTP_WIN32_IO_CALLBACK Callback, void* Context); - inline PTP_IO Iocp() const { return m_ThreadPoolIo; } - -private: - PTP_POOL m_ThreadPool = nullptr; - PTP_CLEANUP_GROUP m_CleanupGroup = nullptr; - PTP_IO m_ThreadPoolIo = nullptr; - TP_CALLBACK_ENVIRON m_CallbackEnvironment; -}; - -} // namespace zen diff --git a/zencore/logging.cpp b/zencore/logging.cpp index 89d588650..6441fc3bc 100644 --- a/zencore/logging.cpp +++ b/zencore/logging.cpp @@ -1,3 +1,5 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + #include "zencore/logging.h" #include <spdlog/sinks/stdout_color_sinks.h> @@ -42,12 +44,12 @@ ConsoleLog() return *ConLogger; } -void +void InitializeLogging() { } -void +void ShutdownLogging() { spdlog::drop_all(); diff --git a/zencore/zencore.vcxproj b/zencore/zencore.vcxproj index 4040d5ae1..4f1e63670 100644 --- a/zencore/zencore.vcxproj +++ b/zencore/zencore.vcxproj @@ -122,8 +122,6 @@ <ClInclude Include="include\zencore\compress.h" /> <ClInclude Include="include\zencore\filesystem.h" /> <ClInclude Include="include\zencore\fmtutils.h" /> - <ClInclude Include="include\zencore\httpclient.h" /> - <ClInclude Include="include\zencore\httpserver.h" /> <ClInclude Include="include\zencore\intmath.h" /> <ClInclude Include="include\zencore\iohash.h" /> <ClInclude Include="include\zencore\logging.h" /> @@ -154,7 +152,6 @@ <ClInclude Include="include\zencore\windows.h" /> <ClInclude Include="include\zencore\xxhash.h" /> <ClInclude Include="include\zencore\zencore.h" /> - <ClInclude Include="iothreadpool.h" /> </ItemGroup> <ItemGroup> <ClCompile Include="blake3.cpp" /> @@ -163,11 +160,8 @@ <ClCompile Include="crc32.cpp" /> <ClCompile Include="except.cpp" /> <ClCompile Include="filesystem.cpp" /> - <ClCompile Include="httpclient.cpp" /> - <ClCompile Include="httpserver.cpp" /> <ClCompile Include="intmath.cpp" /> <ClCompile Include="iohash.cpp" /> - <ClCompile Include="iothreadpool.cpp" /> <ClCompile Include="logging.cpp" /> <ClCompile Include="md5.cpp" /> <ClCompile Include="memory.cpp" /> diff --git a/zencore/zencore.vcxproj.filters b/zencore/zencore.vcxproj.filters index 3a291e967..de3d915b8 100644 --- a/zencore/zencore.vcxproj.filters +++ b/zencore/zencore.vcxproj.filters @@ -21,7 +21,6 @@ <ClInclude Include="include\zencore\enumflags.h" /> <ClInclude Include="include\zencore\except.h" /> <ClInclude Include="include\zencore\filesystem.h" /> - <ClInclude Include="include\zencore\httpserver.h" /> <ClInclude Include="include\zencore\refcount.h" /> <ClInclude Include="include\zencore\memory.h" /> <ClInclude Include="include\zencore\windows.h" /> @@ -31,11 +30,9 @@ <ClInclude Include="include\zencore\compactbinarybuilder.h" /> <ClInclude Include="include\zencore\compactbinarypackage.h" /> <ClInclude Include="include\zencore\compactbinaryvalidation.h" /> - <ClInclude Include="include\zencore\httpclient.h" /> <ClInclude Include="include\zencore\md5.h" /> <ClInclude Include="include\zencore\fmtutils.h" /> <ClInclude Include="include\zencore\xxhash.h" /> - <ClInclude Include="iothreadpool.h" /> <ClInclude Include="include\zencore\varint.h" /> <ClInclude Include="include\zencore\endian.h" /> <ClInclude Include="include\zencore\compositebuffer.h" /> @@ -53,7 +50,6 @@ <ClCompile Include="uid.cpp" /> <ClCompile Include="blake3.cpp" /> <ClCompile Include="filesystem.cpp" /> - <ClCompile Include="httpserver.cpp" /> <ClCompile Include="memory.cpp" /> <ClCompile Include="refcount.cpp" /> <ClCompile Include="stats.cpp" /> @@ -68,11 +64,9 @@ <ClCompile Include="compactbinarybuilder.cpp" /> <ClCompile Include="compactbinarypackage.cpp" /> <ClCompile Include="compactbinaryvalidation.cpp" /> - <ClCompile Include="httpclient.cpp" /> <ClCompile Include="md5.cpp" /> <ClCompile Include="except.cpp" /> <ClCompile Include="xxhash.cpp" /> - <ClCompile Include="iothreadpool.cpp" /> <ClCompile Include="compress.cpp" /> <ClCompile Include="compositebuffer.cpp" /> <ClCompile Include="crc32.cpp" /> |