diff options
28 files changed, 408 insertions, 1075 deletions
diff --git a/zen/cmds/run.cpp b/zen/cmds/run.cpp index 4ffbf820c..94eb7ef6d 100644 --- a/zen/cmds/run.cpp +++ b/zen/cmds/run.cpp @@ -139,9 +139,8 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } PrepReq.EndArray(); - zen::MemoryOutStream MemOut; - zen::BinaryWriter MemWriter(MemOut); - PrepReq.Save(MemWriter); + zen::BinaryWriter MemOut; + PrepReq.Save(MemOut); Zen1.WaitUntilReady(); diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp index fbdcd24e9..88757d47f 100644 --- a/zencore/compactbinarypackage.cpp +++ b/zencore/compactbinarypackage.cpp @@ -83,8 +83,7 @@ CbAttachment::CbAttachment(const CbObject& InValue, const IoHash* const InHash) bool CbAttachment::TryLoad(IoBuffer& InBuffer, BufferAllocator Allocator) { - MemoryInStream InStream(InBuffer.Data(), InBuffer.Size()); - BinaryReader Reader(InStream); + BinaryReader Reader(InBuffer.Data(), InBuffer.Size()); return TryLoad(Reader, Allocator); } @@ -427,8 +426,7 @@ CbPackage::GatherAttachments(const CbObject& Value, AttachmentResolver Resolver) bool CbPackage::TryLoad(IoBuffer InBuffer, BufferAllocator Allocator, AttachmentResolver* Mapper) { - MemoryInStream InStream(InBuffer.Data(), InBuffer.Size()); - BinaryReader Reader(InStream); + BinaryReader Reader(InBuffer.Data(), InBuffer.Size()); return TryLoad(Reader, Allocator, Mapper); } @@ -670,8 +668,7 @@ namespace legacy { bool TryLoadCbPackage(CbPackage& Package, IoBuffer InBuffer, BufferAllocator Allocator, CbPackage::AttachmentResolver* Mapper) { - MemoryInStream InStream(InBuffer.Data(), InBuffer.Size()); - BinaryReader Reader(InStream); + BinaryReader Reader(InBuffer.Data(), InBuffer.Size()); return TryLoadCbPackage(Package, Reader, Allocator, Mapper); } @@ -764,24 +761,22 @@ TEST_CASE("usonpackage") Attachment.Save(Writer); CbFieldIterator Fields = Writer.Save(); - MemoryOutStream WriteStream; - BinaryWriter StreamWriter{WriteStream}; + BinaryWriter StreamWriter; Attachment.Save(StreamWriter); - CHECK(MakeMemoryView(WriteStream).EqualBytes(Fields.GetRangeBuffer().GetView())); - CHECK(ValidateCompactBinaryRange(MakeMemoryView(WriteStream), CbValidateMode::All) == CbValidateError::None); - CHECK(ValidateObjectAttachment(MakeMemoryView(WriteStream), CbValidateMode::All) == CbValidateError::None); + CHECK(MakeMemoryView(StreamWriter).EqualBytes(Fields.GetRangeBuffer().GetView())); + CHECK(ValidateCompactBinaryRange(MakeMemoryView(StreamWriter), CbValidateMode::All) == CbValidateError::None); + CHECK(ValidateObjectAttachment(MakeMemoryView(StreamWriter), CbValidateMode::All) == CbValidateError::None); CbAttachment FromFields; FromFields.TryLoad(Fields); CHECK(!bool(Fields)); CHECK(FromFields == Attachment); - CbAttachment FromArchive; - MemoryInStream InStream(MakeMemoryView(WriteStream)); - BinaryReader Reader(InStream); + CbAttachment FromArchive; + BinaryReader Reader(MakeMemoryView(StreamWriter)); FromArchive.TryLoad(Reader); - CHECK(Reader.CurrentOffset() == InStream.Size()); + CHECK(Reader.CurrentOffset() == Reader.Size()); CHECK(FromArchive == Attachment); }; @@ -956,9 +951,8 @@ TEST_CASE("usonpackage.serialization") Package.Save(Writer); CbFieldIterator Fields = Writer.Save(); - MemoryOutStream MemStream; - BinaryWriter WriteAr(MemStream); - Package.Save(WriteAr); + BinaryWriter MemStream; + Package.Save(MemStream); CHECK(MakeMemoryView(MemStream).EqualBytes(Fields.GetRangeBuffer().GetView())); CHECK(ValidateCompactBinaryRange(MakeMemoryView(MemStream), CbValidateMode::All) == CbValidateError::None); @@ -969,11 +963,10 @@ TEST_CASE("usonpackage.serialization") CHECK_FALSE(bool(Fields)); CHECK(FromFields == Package); - CbPackage FromArchive; - MemoryInStream ReadMemStream(MakeMemoryView(MemStream)); - BinaryReader ReadAr(ReadMemStream); + CbPackage FromArchive; + BinaryReader ReadAr(MakeMemoryView(MemStream)); FromArchive.TryLoad(ReadAr); - CHECK(ReadAr.CurrentOffset() == ReadMemStream.Size()); + CHECK(ReadAr.CurrentOffset() == ReadAr.Size()); CHECK(FromArchive == Package); }; @@ -1195,12 +1188,10 @@ TEST_CASE("usonpackage.serialization") CHECK(FieldsOuterBufferView.Contains(Level4Attachment->AsBinary().GetView())); CHECK(Level4Attachment->GetHash() == Level4Hash); - MemoryOutStream WriteStream; - BinaryWriter WriteAr(WriteStream); - Writer.Save(WriteAr); - CbPackage FromArchive; - MemoryInStream ReadStream(MakeMemoryView(WriteStream)); - BinaryReader ReadAr(ReadStream); + BinaryWriter WriteStream; + Writer.Save(WriteStream); + CbPackage FromArchive; + BinaryReader ReadAr(MakeMemoryView(WriteStream)); FromArchive.TryLoad(ReadAr); Writer.Reset(); diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h index 6f3609d19..54801f9ac 100644 --- a/zencore/include/zencore/iobuffer.h +++ b/zencore/include/zencore/iobuffer.h @@ -28,6 +28,35 @@ enum class ZenContentType : uint8_t kCOUNT }; +inline std::string_view +ToString(ZenContentType ContentType) +{ + using namespace std::literals; + + switch (ContentType) + { + default: + case ZenContentType::kUnknownContentType: + return "unknown"sv; + case ZenContentType::kBinary: + return "binary"sv; + case ZenContentType::kText: + return "text"sv; + case ZenContentType::kJSON: + return "json"sv; + case ZenContentType::kCbObject: + return "cb-object"sv; + case ZenContentType::kCbPackage: + return "cb-package"sv; + case ZenContentType::kCbPackageOffer: + return "cb-package-offer"sv; + case ZenContentType::kCompressedBinary: + return "compressed-binary"sv; + case ZenContentType::kYAML: + return "yaml"sv; + } +} + struct IoBufferFileReference { void* FileHandle; diff --git a/zencore/include/zencore/refcount.h b/zencore/include/zencore/refcount.h index 50bd82f59..0a1e15614 100644 --- a/zencore/include/zencore/refcount.h +++ b/zencore/include/zencore/refcount.h @@ -17,7 +17,7 @@ namespace zen { class RefCounted { public: - RefCounted() : m_RefCount(IsPointerToStack(this)){}; + RefCounted() = default; virtual ~RefCounted() = default; inline uint32_t AddRef() const { return AtomicIncrement(const_cast<RefCounted*>(this)->m_RefCount); } diff --git a/zencore/include/zencore/stream.h b/zencore/include/zencore/stream.h index a0e165bdc..9d1a7628c 100644 --- a/zencore/include/zencore/stream.h +++ b/zencore/include/zencore/stream.h @@ -5,51 +5,30 @@ #include "zencore.h" #include <zencore/memory.h> -#include <zencore/refcount.h> #include <zencore/thread.h> -#include <string_view> #include <vector> namespace zen { /** - * Basic byte stream interface - * - * This is intended as a minimal base class offering only the absolute minimum of functionality. - * - * IMPORTANT: To better support concurrency, this abstraction offers no "file pointer". Thus - * every read or write operation needs to specify the offset from which they wish to read. - * - * Most client code will likely want to use reader/writer classes like BinaryWriter/BinaryReader - * + * Binary stream writer */ -class OutStream : public RefCounted -{ -public: - virtual void Write(const void* Data, size_t ByteCount, uint64_t Offset) = 0; - virtual void Flush() = 0; -}; -class InStream : public RefCounted +class BinaryWriter { public: - virtual void Read(void* DataPtr, size_t ByteCount, uint64_t Offset) = 0; - virtual uint64_t Size() const = 0; - uint64_t GetSize() const { return Size(); } -}; + inline BinaryWriter() = default; + ~BinaryWriter() = default; -/** - * Stream which writes into a growing memory buffer - */ -class MemoryOutStream : public OutStream -{ -public: - MemoryOutStream() = default; - ~MemoryOutStream() = default; + inline void Write(const void* DataPtr, size_t ByteCount) + { + Write(DataPtr, ByteCount, m_Offset); + m_Offset += ByteCount; + } + + inline uint64_t CurrentOffset() const { return m_Offset; } - virtual void Write(const void* DataPtr, size_t ByteCount, uint64_t Offset) override; - virtual void Flush() override; inline const uint8_t* Data() const { return m_Buffer.data(); } inline const uint8_t* GetData() const { return m_Buffer.data(); } inline uint64_t Size() const { return m_Buffer.size(); } @@ -58,263 +37,45 @@ public: private: RwLock m_Lock; std::vector<uint8_t> m_Buffer; + uint64_t m_Offset = 0; + + void Write(const void* DataPtr, size_t ByteCount, uint64_t Offset); }; inline MemoryView -MakeMemoryView(const MemoryOutStream& Stream) +MakeMemoryView(const BinaryWriter& Stream) { return MemoryView(Stream.Data(), Stream.Size()); } /** - * Stream which reads from a memory buffer - */ -class MemoryInStream : public InStream -{ -public: - MemoryInStream(const void* Buffer, size_t Size); - MemoryInStream(MemoryView View) : MemoryInStream(View.GetData(), View.GetSize()) {} - ~MemoryInStream() = default; - - virtual void Read(void* DataPtr, size_t ByteCount, uint64_t ReadOffset) override; - virtual uint64_t Size() const override { return m_Buffer.size(); } - inline const uint8_t* Data() const { return m_Buffer.data(); } - inline const uint8_t* GetData() const { return m_Buffer.data(); } - -private: - RwLock m_Lock; - std::vector<uint8_t> m_Buffer; -}; - -/** - * Binary stream writer - */ - -class BinaryWriter -{ -public: - inline BinaryWriter(OutStream& Stream) : m_Stream(&Stream) {} - ~BinaryWriter() = default; - - inline void Write(const void* DataPtr, size_t ByteCount) - { - m_Stream->Write(DataPtr, ByteCount, m_Offset); - m_Offset += ByteCount; - } - - uint64_t CurrentOffset() const { return m_Offset; } - -private: - RefPtr<OutStream> m_Stream; - uint64_t m_Offset = 0; -}; - -inline BinaryWriter& -operator<<(BinaryWriter& Writer, bool Value) -{ - Writer.Write(&Value, sizeof Value); - return Writer; -} -inline BinaryWriter& -operator<<(BinaryWriter& Writer, int8_t Value) -{ - Writer.Write(&Value, sizeof Value); - return Writer; -} -inline BinaryWriter& -operator<<(BinaryWriter& Writer, int16_t Value) -{ - Writer.Write(&Value, sizeof Value); - return Writer; -} -inline BinaryWriter& -operator<<(BinaryWriter& Writer, int32_t Value) -{ - Writer.Write(&Value, sizeof Value); - return Writer; -} -inline BinaryWriter& -operator<<(BinaryWriter& Writer, int64_t Value) -{ - Writer.Write(&Value, sizeof Value); - return Writer; -} -inline BinaryWriter& -operator<<(BinaryWriter& Writer, uint8_t Value) -{ - Writer.Write(&Value, sizeof Value); - return Writer; -} -inline BinaryWriter& -operator<<(BinaryWriter& Writer, uint16_t Value) -{ - Writer.Write(&Value, sizeof Value); - return Writer; -} -inline BinaryWriter& -operator<<(BinaryWriter& Writer, uint32_t Value) -{ - Writer.Write(&Value, sizeof Value); - return Writer; -} -inline BinaryWriter& -operator<<(BinaryWriter& Writer, uint64_t Value) -{ - Writer.Write(&Value, sizeof Value); - return Writer; -} - -/** * Binary stream reader */ class BinaryReader { public: - inline BinaryReader(InStream& Stream) : m_Stream(&Stream) {} - ~BinaryReader() = default; - - inline void Read(void* DataPtr, size_t ByteCount) + inline BinaryReader(const void* Buffer, uint64_t Size) : m_BufferBase(reinterpret_cast<const uint8_t*>(Buffer)), m_BufferSize(Size) {} + inline BinaryReader(MemoryView Buffer) + : m_BufferBase(reinterpret_cast<const uint8_t*>(Buffer.GetData())) + , m_BufferSize(Buffer.GetSize()) { - m_Stream->Read(DataPtr, ByteCount, m_Offset); - m_Offset += ByteCount; } - void Seek(uint64_t Offset) - { - ZEN_ASSERT(Offset <= m_Stream->Size()); - m_Offset = Offset; - } - - void Skip(uint64_t SkipOffset) + inline void Read(void* DataPtr, size_t ByteCount) { - ZEN_ASSERT((m_Offset + SkipOffset) <= m_Stream->Size()); - m_Offset += SkipOffset; + memcpy(DataPtr, m_BufferBase + m_Offset, ByteCount); + m_Offset += ByteCount; } + inline uint64_t Size() const { return m_BufferSize; } + inline uint64_t GetSize() const { return Size(); } inline uint64_t CurrentOffset() const { return m_Offset; } - inline uint64_t AvailableBytes() const { return m_Stream->Size() - m_Offset; } - -private: - RefPtr<InStream> m_Stream; - uint64_t m_Offset = 0; -}; - -inline BinaryReader& -operator>>(BinaryReader& Reader, bool& Value) -{ - Reader.Read(&Value, sizeof Value); - return Reader; -} -inline BinaryReader& -operator>>(BinaryReader& Reader, int8_t& Value) -{ - Reader.Read(&Value, sizeof Value); - return Reader; -} -inline BinaryReader& -operator>>(BinaryReader& Reader, int16_t& Value) -{ - Reader.Read(&Value, sizeof Value); - return Reader; -} -inline BinaryReader& -operator>>(BinaryReader& Reader, int32_t& Value) -{ - Reader.Read(&Value, sizeof Value); - return Reader; -} -inline BinaryReader& -operator>>(BinaryReader& Reader, int64_t& Value) -{ - Reader.Read(&Value, sizeof Value); - return Reader; -} -inline BinaryReader& -operator>>(BinaryReader& Reader, uint8_t& Value) -{ - Reader.Read(&Value, sizeof Value); - return Reader; -} -inline BinaryReader& -operator>>(BinaryReader& Reader, uint16_t& Value) -{ - Reader.Read(&Value, sizeof Value); - return Reader; -} -inline BinaryReader& -operator>>(BinaryReader& Reader, uint32_t& Value) -{ - Reader.Read(&Value, sizeof Value); - return Reader; -} -inline BinaryReader& -operator>>(BinaryReader& Reader, uint64_t& Value) -{ - Reader.Read(&Value, sizeof Value); - return Reader; -} - -/** - * Text stream writer - */ - -class TextWriter -{ -public: - ZENCORE_API TextWriter(OutStream& Stream); - ZENCORE_API ~TextWriter(); - - ZENCORE_API virtual void Write(const void* DataPtr, size_t ByteCount); - ZENCORE_API void Writef(const char* FormatString, ...); - - inline uint64_t CurrentOffset() const { return m_CurrentOffset; } - -private: - RefPtr<OutStream> m_Stream; - uint64_t m_CurrentOffset = 0; -}; - -ZENCORE_API TextWriter& operator<<(TextWriter& Writer, const char* Value); -ZENCORE_API TextWriter& operator<<(TextWriter& Writer, const std::string_view& Value); -ZENCORE_API TextWriter& operator<<(TextWriter& Writer, bool Value); -ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int8_t Value); -ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int16_t Value); -ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int32_t Value); -ZENCORE_API TextWriter& operator<<(TextWriter& Writer, int64_t Value); -ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint8_t Value); -ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint16_t Value); -ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint32_t Value); -ZENCORE_API TextWriter& operator<<(TextWriter& Writer, uint64_t Value); - -class IndentTextWriter : public TextWriter -{ -public: - ZENCORE_API IndentTextWriter(OutStream& stream); - ZENCORE_API ~IndentTextWriter(); - - ZENCORE_API virtual void Write(const void* DataPtr, size_t ByteCount) override; - - inline void Indent(int Amount) { m_IndentAmount += Amount; } - - struct Scope - { - Scope(IndentTextWriter& Outer, int IndentAmount = 2) : m_Outer(Outer), m_IndentAmount(IndentAmount) - { - m_Outer.Indent(IndentAmount); - } - - ~Scope() { m_Outer.Indent(-m_IndentAmount); } - - private: - IndentTextWriter& m_Outer; - int m_IndentAmount; - }; private: - int m_IndentAmount = 0; - int m_LineCursor = 0; - char m_LineBuffer[2048]; + const uint8_t* m_BufferBase; + uint64_t m_BufferSize; + uint64_t m_Offset = 0; }; void stream_forcelink(); // internal diff --git a/zencore/include/zencore/streamutil.h b/zencore/include/zencore/streamutil.h deleted file mode 100644 index 190cd18eb..000000000 --- a/zencore/include/zencore/streamutil.h +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <fmt/format.h> -#include <zencore/string.h> -#include <string> -#include <string_view> - -#include "blake3.h" -#include "iohash.h" -#include "sha1.h" -#include "stream.h" - -namespace zen { - -ZENCORE_API BinaryWriter& operator<<(BinaryWriter& writer, const std::string_view& value); -ZENCORE_API BinaryReader& operator>>(BinaryReader& reader, std::string& value); - -ZENCORE_API BinaryWriter& operator<<(BinaryWriter& writer, const std::wstring_view& value); -ZENCORE_API BinaryReader& operator>>(BinaryReader& reader, std::wstring& value); -ZENCORE_API TextWriter& operator<<(TextWriter& writer, const std::wstring_view& value); - -inline BinaryWriter& -operator<<(BinaryWriter& writer, const SHA1& value) -{ - writer.Write(value.Hash, sizeof value.Hash); - return writer; -} -inline BinaryReader& -operator>>(BinaryReader& reader, SHA1& value) -{ - reader.Read(value.Hash, sizeof value.Hash); - return reader; -} -ZENCORE_API TextWriter& operator<<(TextWriter& writer, const zen::SHA1& value); - -inline BinaryWriter& -operator<<(BinaryWriter& writer, const BLAKE3& value) -{ - writer.Write(value.Hash, sizeof value.Hash); - return writer; -} -inline BinaryReader& -operator>>(BinaryReader& reader, BLAKE3& value) -{ - reader.Read(value.Hash, sizeof value.Hash); - return reader; -} -ZENCORE_API TextWriter& operator<<(TextWriter& writer, const BLAKE3& value); - -inline BinaryWriter& -operator<<(BinaryWriter& writer, const IoHash& value) -{ - writer.Write(value.Hash, sizeof value.Hash); - return writer; -} -inline BinaryReader& -operator>>(BinaryReader& reader, IoHash& value) -{ - reader.Read(value.Hash, sizeof value.Hash); - return reader; -} -ZENCORE_API TextWriter& operator<<(TextWriter& writer, const IoHash& value); - -} // namespace zen - -////////////////////////////////////////////////////////////////////////// - -template<> -struct fmt::formatter<zen::IoHash> -{ - constexpr auto parse(format_parse_context& ctx) - { - // Parse the presentation format and store it in the formatter: - auto it = ctx.begin(), end = ctx.end(); - - // Check if reached the end of the range: - if (it != end && *it != '}') - throw format_error("invalid format"); - - // Return an iterator past the end of the parsed range: - return it; - } - - template<typename FormatContext> - auto format(const zen::IoHash& h, FormatContext& ctx) - { - zen::ExtendableStringBuilder<48> String; - h.ToHexString(String); - return format_to(ctx.out(), std::string_view(String)); - } -}; - -template<> -struct fmt::formatter<zen::BLAKE3> -{ - constexpr auto parse(format_parse_context& ctx) - { - // Parse the presentation format and store it in the formatter: - auto it = ctx.begin(), end = ctx.end(); - - // Check if reached the end of the range: - if (it != end && *it != '}') - throw format_error("invalid format"); - - // Return an iterator past the end of the parsed range: - return it; - } - - template<typename FormatContext> - auto format(const zen::BLAKE3& h, FormatContext& ctx) - { - zen::ExtendableStringBuilder<80> String; - h.ToHexString(String); - return format_to(ctx.out(), std::string_view(String)); - } -}; diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index 4b9c1af1b..136ed2944 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -112,7 +112,7 @@ #if ZEN_ARCH_ARM64 // On ARM we can't do this because the executable will require jumps larger // than the branch instruction can handle. Clang will only generate -// the trampolines in the .text segment of the binary. If the uedbg segment +// the trampolines in the .text segment of the binary. If the zcold segment // is present it will generate code that it cannot link. # define ZEN_DEBUG_SECTION #else @@ -121,7 +121,7 @@ // it well off the hot path and hopefully out of the instruction cache. It also // facilitates reasoning about the makeup of a compiled/linked binary. # define ZEN_DEBUG_SECTION ZEN_CODE_SECTION(".zcold") -#endif // DO_CHECK || DO_GUARD_SLOW +#endif namespace zen { @@ -194,16 +194,18 @@ char (&ZenArrayCountHelper(const T (&)[N]))[N + 1]; namespace zen { -ZENCORE_API bool IsPointerToStack(const void* ptr); // Query if pointer is within the stack of the currently executing thread ZENCORE_API bool IsApplicationExitRequested(); ZENCORE_API void RequestApplicationExit(int ExitCode); ZENCORE_API bool IsDebuggerPresent(); +ZENCORE_API void SetIsInteractiveSession(bool Value); ZENCORE_API bool IsInteractiveSession(); ZENCORE_API void zencore_forcelinktests(); } // namespace zen +////////////////////////////////////////////////////////////////////////// + #ifndef ZEN_USE_MIMALLOC # define ZEN_USE_MIMALLOC 1 #endif diff --git a/zencore/stream.cpp b/zencore/stream.cpp index ead0b014b..aa9705764 100644 --- a/zencore/stream.cpp +++ b/zencore/stream.cpp @@ -10,269 +10,19 @@ namespace zen { -MemoryInStream::MemoryInStream(const void* buffer, size_t size) -: m_Buffer(reinterpret_cast<const uint8_t*>(buffer), reinterpret_cast<const uint8_t*>(buffer) + size) -{ -} - void -MemoryInStream::Read(void* buffer, size_t byteCount, uint64_t offset) +BinaryWriter::Write(const void* data, size_t ByteCount, uint64_t Offset) { RwLock::ExclusiveLockScope _(m_Lock); - const size_t needEnd = offset + byteCount; - - if (needEnd > m_Buffer.size()) - throw std::runtime_error("read past end of file!"); // TODO: better exception - - memcpy(buffer, m_Buffer.data() + offset, byteCount); -} - -void -MemoryOutStream::Write(const void* data, size_t byteCount, uint64_t offset) -{ - RwLock::ExclusiveLockScope _(m_Lock); - - const size_t needEnd = offset + byteCount; - - if (needEnd > m_Buffer.size()) - m_Buffer.resize(needEnd); - - memcpy(m_Buffer.data() + offset, data, byteCount); -} - -void -MemoryOutStream::Flush() -{ - // No-op -} - -////////////////////////////////////////////////////////////////////////// - -TextWriter::TextWriter(OutStream& stream) : m_Stream(&stream) -{ -} - -TextWriter::~TextWriter() = default; - -void -TextWriter::Write(const void* data, size_t byteCount) -{ - m_Stream->Write(data, byteCount, m_CurrentOffset); - m_CurrentOffset += byteCount; -} - -TextWriter& -operator<<(TextWriter& Writer, const char* value) -{ - if (value) - Writer.Write(value, strlen(value)); - else - Writer.Write("(null)", 6); - - return Writer; -} - -TextWriter& -operator<<(TextWriter& writer, const std::string_view& value) -{ - writer.Write(value.data(), value.size()); - - return writer; -} - -TextWriter& -operator<<(TextWriter& writer, bool value) -{ - if (value) - writer.Write("true", 4); - else - writer.Write("false", 5); - - return writer; -} - -TextWriter& -operator<<(TextWriter& writer, int8_t value) -{ - char buffer[16]; -#if ZEN_PLATFORM_WINDOWS - _itoa_s(value, buffer, 10); -#else - sprintf(buffer, "%d", value); -#endif - writer << buffer; - return writer; -} - -TextWriter& -operator<<(TextWriter& writer, int16_t value) -{ - char buffer[16]; -#if ZEN_PLATFORM_WINDOWS - _itoa_s(value, buffer, 10); -#else - sprintf(buffer, "%d", value); -#endif - writer << buffer; - return writer; -} - -TextWriter& -operator<<(TextWriter& writer, int32_t value) -{ - char buffer[16]; -#if ZEN_PLATFORM_WINDOWS - _itoa_s(value, buffer, 10); -#else - sprintf(buffer, "%d", value); -#endif - writer << buffer; - return writer; -} - -TextWriter& -operator<<(TextWriter& writer, int64_t value) -{ - char buffer[32]; -#if ZEN_PLATFORM_WINDOWS - _i64toa_s(value, buffer, sizeof buffer, 10); -#else - sprintf(buffer, "%" PRId64, value); -#endif - writer << buffer; - return writer; -} - -TextWriter& -operator<<(TextWriter& writer, uint8_t value) -{ - char buffer[16]; -#if ZEN_PLATFORM_WINDOWS - _ultoa_s(value, buffer, 10); -#else - sprintf(buffer, "%u", value); -#endif - writer << buffer; - return writer; -} - -TextWriter& -operator<<(TextWriter& writer, uint16_t value) -{ - char buffer[16]; -#if ZEN_PLATFORM_WINDOWS - _ultoa_s(value, buffer, 10); -#else - sprintf(buffer, "%u", value); -#endif - writer << buffer; - return writer; -} - -TextWriter& -operator<<(TextWriter& writer, uint32_t value) -{ - char buffer[16]; -#if ZEN_PLATFORM_WINDOWS - _ultoa_s(value, buffer, 10); -#else - sprintf(buffer, "%u", value); -#endif - writer << buffer; - return writer; -} - -TextWriter& -operator<<(TextWriter& writer, uint64_t value) -{ - char buffer[32]; -#if ZEN_PLATFORM_WINDOWS - _ui64toa_s(value, buffer, sizeof buffer, 10); -#else - sprintf(buffer, "%" PRIu64, value); -#endif - writer << buffer; - return writer; -} - -void -TextWriter::Writef(const char* formatString, ...) -{ - va_list args; - va_start(args, formatString); - - char* tempBuffer = nullptr; - char buffer[4096]; - int rv = vsnprintf(buffer, sizeof buffer, formatString, args); - - ZEN_ASSERT(rv >= 0); - - if (uint32_t(rv) > sizeof buffer) - { - // Need more room -- allocate temporary buffer - - tempBuffer = (char*)Memory::Alloc(rv + 1, 8); - - int rv2 = vsnprintf(tempBuffer, rv + 1, formatString, args); - - ZEN_ASSERT(rv >= 0); - ZEN_ASSERT(rv2 <= rv); - - rv = rv2; - } - - m_Stream->Write(tempBuffer ? tempBuffer : buffer, rv, m_CurrentOffset); - m_CurrentOffset += rv; - - if (tempBuffer) - Memory::Free(tempBuffer); - - va_end(args); -} - -////////////////////////////////////////////////////////////////////////// - -IndentTextWriter::IndentTextWriter(OutStream& stream) : TextWriter(stream) -{ -} - -IndentTextWriter::~IndentTextWriter() -{ -} + const size_t NeedEnd = Offset + ByteCount; -void -IndentTextWriter::Write(const void* data, size_t byteCount) -{ - const uint8_t* src = reinterpret_cast<const uint8_t*>(data); - int cur = m_LineCursor; - - while (byteCount) + if (NeedEnd > m_Buffer.size()) { - char c = *src++; - - if (cur == 0) - { - const char indentSpaces[] = - " " - " "; - - cur = std::min<int>(m_IndentAmount, sizeof indentSpaces - 1); - memcpy(m_LineBuffer, indentSpaces, cur); - } - - m_LineBuffer[cur++] = c; - --byteCount; - - if (c == '\n' || cur == sizeof m_LineBuffer) - { - TextWriter::Write(m_LineBuffer, cur); - - cur = 0; - } + m_Buffer.resize(NeedEnd); } - m_LineCursor = cur; + memcpy(m_Buffer.data() + Offset, data, ByteCount); } ////////////////////////////////////////////////////////////////////////// @@ -287,58 +37,6 @@ stream_forcelink() { } -TEST_CASE("BinaryWriter and BinaryWriter") -{ - MemoryOutStream stream; - BinaryWriter writer(stream); - - CHECK(writer.CurrentOffset() == 0); - - writer.Write("foo!", 4); - CHECK(writer.CurrentOffset() == 4); - - writer << uint8_t(42) << uint16_t(42) << uint32_t(42) << uint64_t(42); - writer << int8_t(42) << int16_t(42) << int32_t(42) << int64_t(42); - - CHECK(writer.CurrentOffset() == (4 + 15 * 2)); - - // Read the data back - - MemoryInStream instream(stream.Data(), stream.Size()); - BinaryReader reader(instream); - CHECK(reader.CurrentOffset() == 0); - - char buffer[4]; - reader.Read(buffer, 4); - CHECK(reader.CurrentOffset() == 4); - - CHECK(memcmp(buffer, "foo!", 4) == 0); - - uint8_t ui8 = 0; - uint16_t ui16 = 0; - uint32_t ui32 = 0; - uint64_t ui64 = 0; - int8_t i8 = 0; - int16_t i16 = 0; - int32_t i32 = 0; - int64_t i64 = 0; - - reader >> ui8 >> ui16 >> ui32 >> ui64; - reader >> i8 >> i16 >> i32 >> i64; - - CHECK(reader.CurrentOffset() == (4 + 15 * 2)); - - CHECK(ui8 == 42); - CHECK(ui16 == 42); - CHECK(ui32 == 42); - CHECK(ui64 == 42); - - CHECK(i8 == 42); - CHECK(i16 == 42); - CHECK(i32 == 42); - CHECK(i64 == 42); -} - #endif } // namespace zen diff --git a/zencore/streamutil.cpp b/zencore/streamutil.cpp deleted file mode 100644 index d3ed5ceaa..000000000 --- a/zencore/streamutil.cpp +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <zencore/streamutil.h> -#include <zencore/string.h> - -namespace zen { - -BinaryWriter& -operator<<(BinaryWriter& writer, const std::string_view& value) -{ - writer.Write(value.data(), value.size()); - writer << uint8_t(0); - - return writer; -} - -BinaryReader& -operator>>(BinaryReader& reader, std::string& value) -{ - for (;;) - { - uint8_t x; - reader.Read(&x, 1); - - if (x == 0) - return reader; - - value.push_back(char(x)); - } -} - -BinaryWriter& -operator<<(BinaryWriter& writer, const std::wstring_view& value) -{ - // write as utf8 - - ExtendableStringBuilder<128> utf8; - WideToUtf8(value, utf8); - - writer.Write(utf8.c_str(), utf8.Size() + 1); - - return writer; -} - -BinaryReader& -operator>>(BinaryReader& reader, std::wstring& value) -{ - // read as utf8 - - std::string v8; - reader >> v8; - - ExtendableWideStringBuilder<128> wstr; - Utf8ToWide(v8, wstr); - - value = wstr.c_str(); - - return reader; -} - -TextWriter& -operator<<(TextWriter& writer, const zen::SHA1& value) -{ - zen::SHA1::String_t buffer; - value.ToHexString(buffer); - - writer.Write(buffer, zen::SHA1::StringLength); - - return writer; -} - -TextWriter& -operator<<(TextWriter& writer, const zen::BLAKE3& value) -{ - zen::BLAKE3::String_t buffer; - value.ToHexString(buffer); - - writer.Write(buffer, zen::BLAKE3::StringLength); - - return writer; -} - -TextWriter& -operator<<(TextWriter& writer, const zen::IoHash& value) -{ - zen::IoHash::String_t buffer; - value.ToHexString(buffer); - - writer.Write(buffer, zen::IoHash::StringLength); - - return writer; -} - -TextWriter& -operator<<(TextWriter& writer, const std::wstring_view& value) -{ - ExtendableStringBuilder<128> v8; - WideToUtf8(value, v8); - - writer.Write(v8.c_str(), v8.Size()); - return writer; -} - -} // namespace zen diff --git a/zencore/uid.cpp b/zencore/uid.cpp index d4b708288..86cdfae3a 100644 --- a/zencore/uid.cpp +++ b/zencore/uid.cpp @@ -99,11 +99,10 @@ Oid::ToString(char OutString[StringLength]) StringBuilderBase& Oid::ToString(StringBuilderBase& OutString) const { - char str[StringLength + 1]; - ToHexBytes(reinterpret_cast<const uint8_t*>(OidBits), sizeof(Oid::OidBits), str); - str[2 * sizeof(Oid)] = '\0'; + String_t Str; + ToHexBytes(reinterpret_cast<const uint8_t*>(OidBits), sizeof(Oid::OidBits), Str); - OutString.AppendRange(str, &str[25]); + OutString.AppendRange(Str, &Str[StringLength]); return OutString; } diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp index 3eb43c558..19acdd1f5 100644 --- a/zencore/zencore.cpp +++ b/zencore/zencore.cpp @@ -35,31 +35,6 @@ namespace zen { ////////////////////////////////////////////////////////////////////////// bool -IsPointerToStack(const void* ptr) -{ -#if ZEN_PLATFORM_WINDOWS - ULONG_PTR low, high; - GetCurrentThreadStackLimits(&low, &high); - - const uintptr_t intPtr = reinterpret_cast<uintptr_t>(ptr); - - return (intPtr - low) < (high - low); -#elif ZEN_PLATFORM_LINUX - pthread_t self = pthread_self(); - - pthread_attr_t attr; - pthread_getattr_np(self, &attr); - - void* low; - size_t size; - pthread_attr_getstack(&attr, &low, &size); - - return (uintptr_t(ptr) - uintptr_t(low)) < uintptr_t(size); -#elif 0 -#endif -} - -bool IsDebuggerPresent() { #if ZEN_PLATFORM_WINDOWS @@ -69,21 +44,36 @@ IsDebuggerPresent() #endif } +std::optional<bool> InteractiveSessionFlag; + +void +SetIsInteractiveSession(bool Value) +{ + InteractiveSessionFlag = Value; +} + bool IsInteractiveSession() { -#if ZEN_PLATFORM_WINDOWS - DWORD dwSessionId = 0; - if (ProcessIdToSessionId(GetCurrentProcessId(), &dwSessionId)) + if (!InteractiveSessionFlag.has_value()) { - return (dwSessionId != 0); - } - - return false; +#if ZEN_PLATFORM_WINDOWS + DWORD dwSessionId = 0; + if (ProcessIdToSessionId(GetCurrentProcessId(), &dwSessionId)) + { + InteractiveSessionFlag = (dwSessionId != 0); + } + else + { + InteractiveSessionFlag = false; + } #else - // TODO: figure out what makes sense here - return true; + // TODO: figure out what actually makes sense here + InteractiveSessionFlag = true; #endif + } + + return InteractiveSessionFlag.value(); } ////////////////////////////////////////////////////////////////////////// diff --git a/zencore/zencore.vcxproj b/zencore/zencore.vcxproj index 3adf779ed..421802d3f 100644 --- a/zencore/zencore.vcxproj +++ b/zencore/zencore.vcxproj @@ -115,6 +115,7 @@ <ClInclude Include="include\zencore\atomic.h" /> <ClInclude Include="include\zencore\base64.h" /> <ClInclude Include="include\zencore\blake3.h" /> + <ClInclude Include="include\zencore\compactbinaryvalue.h" /> <ClInclude Include="include\zencore\compositebuffer.h" /> <ClInclude Include="include\zencore\crc32.h" /> <ClInclude Include="include\zencore\endian.h" /> @@ -140,7 +141,6 @@ <ClInclude Include="include\zencore\sharedbuffer.h" /> <ClInclude Include="include\zencore\stats.h" /> <ClInclude Include="include\zencore\stream.h" /> - <ClInclude Include="include\zencore\streamutil.h" /> <ClInclude Include="include\zencore\string.h" /> <ClInclude Include="include\zencore\targetver.h" /> <ClInclude Include="include\zencore\testing.h" /> @@ -183,7 +183,6 @@ <ClCompile Include="sharedbuffer.cpp" /> <ClCompile Include="stats.cpp" /> <ClCompile Include="stream.cpp" /> - <ClCompile Include="streamutil.cpp" /> <ClCompile Include="string.cpp" /> <ClCompile Include="testutils.cpp" /> <ClCompile Include="thread.cpp" /> diff --git a/zencore/zencore.vcxproj.filters b/zencore/zencore.vcxproj.filters index 92aa0db1d..e701e9354 100644 --- a/zencore/zencore.vcxproj.filters +++ b/zencore/zencore.vcxproj.filters @@ -12,7 +12,6 @@ <ClInclude Include="include\zencore\timer.h" /> <ClInclude Include="include\zencore\thread.h" /> <ClInclude Include="include\zencore\string.h" /> - <ClInclude Include="include\zencore\streamutil.h" /> <ClInclude Include="include\zencore\stream.h" /> <ClInclude Include="include\zencore\stats.h" /> <ClInclude Include="include\zencore\blake3.h" /> @@ -45,6 +44,7 @@ <ClInclude Include="include\zencore\testing.h" /> <ClInclude Include="include\zencore\mpscqueue.h" /> <ClInclude Include="include\zencore\base64.h" /> + <ClInclude Include="include\zencore\compactbinaryvalue.h" /> </ItemGroup> <ItemGroup> <ClCompile Include="sha1.cpp" /> @@ -57,7 +57,6 @@ <ClCompile Include="refcount.cpp" /> <ClCompile Include="stats.cpp" /> <ClCompile Include="stream.cpp" /> - <ClCompile Include="streamutil.cpp" /> <ClCompile Include="string.cpp" /> <ClCompile Include="thread.cpp" /> <ClCompile Include="timer.cpp" /> diff --git a/zenhttp/httpclient.cpp b/zenhttp/httpclient.cpp index 20550b0c9..6e915e613 100644 --- a/zenhttp/httpclient.cpp +++ b/zenhttp/httpclient.cpp @@ -66,12 +66,11 @@ HttpClient::TransactPackage(std::string_view Url, CbPackage Package) Writer.EndArray(); - MemoryOutStream MemOut; - BinaryWriter MemWriter(MemOut); + BinaryWriter MemWriter; Writer.Save(MemWriter); Sess.SetHeader({{"Content-Type", "application/x-ue-offer"}, {"UE-Session", m_SessionId}, {"UE-Request", RequestIdString}}); - Sess.SetBody(cpr::Body{(const char*)MemOut.Data(), MemOut.Size()}); + Sess.SetBody(cpr::Body{(const char*)MemWriter.Data(), MemWriter.Size()}); cpr::Response FilterResponse = Sess.Post(); diff --git a/zenhttp/httpshared.cpp b/zenhttp/httpshared.cpp index b0c5493db..c703409af 100644 --- a/zenhttp/httpshared.cpp +++ b/zenhttp/httpshared.cpp @@ -2,11 +2,13 @@ #include <zenhttp/httpshared.h> +#include <zencore/compactbinarybuilder.h> #include <zencore/compactbinarypackage.h> #include <zencore/compositebuffer.h> #include <zencore/iobuffer.h> #include <zencore/iohash.h> #include <zencore/stream.h> +#include <zencore/testing.h> #include <span> #include <vector> @@ -58,9 +60,7 @@ FormatPackageMessage(const CbPackage& Data) IoBuffer RootIoBuffer = Data.GetObject().GetBuffer().AsIoBuffer(); ResponseBuffers.push_back(RootIoBuffer); // Root object - *AttachmentInfo++ = {.AttachmentSize = RootIoBuffer.Size(), - .Flags = CbAttachmentEntry::kIsObject, - .AttachmentHash = Data.GetObjectHash()}; + *AttachmentInfo++ = {.PayloadSize = RootIoBuffer.Size(), .Flags = CbAttachmentEntry::kIsObject, .AttachmentHash = Data.GetObjectHash()}; // Attachment payloads @@ -74,7 +74,7 @@ FormatPackageMessage(const CbPackage& Data) { CompositeBuffer Compressed = AttachmentBuffer.GetCompressed(); - *AttachmentInfo++ = {.AttachmentSize = AttachmentBuffer.GetCompressedSize(), + *AttachmentInfo++ = {.PayloadSize = AttachmentBuffer.GetCompressedSize(), .Flags = CbAttachmentEntry::kIsCompressed, .AttachmentHash = IoHash::FromBLAKE3(AttachmentBuffer.GetRawHash())}; @@ -89,13 +89,13 @@ FormatPackageMessage(const CbPackage& Data) IoBuffer ObjIoBuffer = AttachmentObject.GetBuffer().AsIoBuffer(); ResponseBuffers.push_back(ObjIoBuffer); - *AttachmentInfo++ = {.AttachmentSize = ObjIoBuffer.Size(), + *AttachmentInfo++ = {.PayloadSize = ObjIoBuffer.Size(), .Flags = CbAttachmentEntry::kIsObject, .AttachmentHash = Attachment.GetHash()}; } else if (CompositeBuffer AttachmentBinary = Attachment.AsCompositeBinary()) { - *AttachmentInfo++ = {.AttachmentSize = AttachmentBinary.GetSize(), .Flags = 0, .AttachmentHash = Attachment.GetHash()}; + *AttachmentInfo++ = {.PayloadSize = AttachmentBinary.GetSize(), .Flags = 0, .AttachmentHash = Attachment.GetHash()}; for (const SharedBuffer& Segment : AttachmentBinary.GetSegments()) { @@ -120,8 +120,7 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint return {}; } - MemoryInStream InStream(Payload); - BinaryReader Reader(InStream); + BinaryReader Reader(Payload); CbPackageHeader Hdr; Reader.Read(&Hdr, sizeof Hdr); @@ -142,7 +141,7 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint for (uint32_t i = 0; i < ChunkCount; ++i) { const CbAttachmentEntry& Entry = AttachmentEntries[i]; - const uint64_t AttachmentSize = Entry.AttachmentSize; + const uint64_t AttachmentSize = Entry.PayloadSize; IoBuffer AttachmentBuffer = CreateBuffer(Entry.AttachmentHash, AttachmentSize); ZEN_ASSERT(AttachmentBuffer); @@ -195,4 +194,123 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint return Package; } -} // namespace zen
\ No newline at end of file +CbPackageReader::CbPackageReader() : m_CreateBuffer([](const IoHash&, uint64_t Size) -> IoBuffer { return IoBuffer{Size}; }) +{ +} + +CbPackageReader::~CbPackageReader() +{ +} + +void +CbPackageReader::SetPayloadBufferCreator(std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> CreateBuffer) +{ + m_CreateBuffer = CreateBuffer; +} + +/** Process data + */ +uint64_t +CbPackageReader::ProcessHeaderData(const void* Data, uint64_t DataBytes) +{ + ZEN_ASSERT(m_CurrentState != State::kReadingBuffers); + + switch (m_CurrentState) + { + case State::kInitialState: + ZEN_ASSERT(Data == nullptr); + m_CurrentState = State::kReadingHeader; + return sizeof m_PackageHeader; + + case State::kReadingHeader: + ZEN_ASSERT(DataBytes == sizeof m_PackageHeader); + memcpy(&m_PackageHeader, Data, sizeof m_PackageHeader); + ZEN_ASSERT(m_PackageHeader.HeaderMagic == kCbPkgMagic); + m_CurrentState = State::kReadingAttachmentEntries; + m_AttachmentEntries.resize(m_PackageHeader.AttachmentCount + 1); + return (m_PackageHeader.AttachmentCount + 1) * sizeof(CbAttachmentEntry); + + case State::kReadingAttachmentEntries: + ZEN_ASSERT(DataBytes == ((m_PackageHeader.AttachmentCount + 1) * sizeof(CbAttachmentEntry))); + memcpy(m_AttachmentEntries.data(), Data, DataBytes); + + for (CbAttachmentEntry& Entry : m_AttachmentEntries) + { + m_PayloadBuffers.push_back(IoBuffer{Entry.PayloadSize}); + } + + m_CurrentState = State::kReadingBuffers; + return 0; + + default: + ZEN_ASSERT(false); + return 0; + } +} + +/** + ______________________ _____________________________ + \__ ___/\_ _____// _____/\__ ___/ _____/ + | | | __)_ \_____ \ | | \_____ \ + | | | \/ \ | | / \ + |____| /_______ /_______ / |____| /_______ / + \/ \/ \/ + */ + +#if ZEN_WITH_TESTS + +TEST_CASE("CbPackage.Serialization") +{ + // Make a test package + + CbAttachment Attach1{SharedBuffer::MakeView(MakeMemoryView("abcd"))}; + CbAttachment Attach2{SharedBuffer::MakeView(MakeMemoryView("efgh"))}; + + CbObjectWriter Cbo; + Cbo.AddAttachment("abcd", Attach1); + Cbo.AddAttachment("efgh", Attach2); + + CbPackage Pkg; + Pkg.AddAttachment(Attach1); + Pkg.AddAttachment(Attach2); + Pkg.SetObject(Cbo.Save()); + + SharedBuffer Buffer = FormatPackageMessageBuffer(Pkg).Flatten(); + const uint8_t* CursorPtr = reinterpret_cast<const uint8_t*>(Buffer.GetData()); + uint64_t RemainingBytes = Buffer.GetSize(); + + auto ConsumeBytes = [&](uint64_t ByteCount) { + ZEN_ASSERT(ByteCount <= RemainingBytes); + void* ReturnPtr = (void*)CursorPtr; + CursorPtr += ByteCount; + RemainingBytes -= ByteCount; + return ReturnPtr; + }; + + auto CopyBytes = [&](void* TargetBuffer, uint64_t ByteCount) { + ZEN_ASSERT(ByteCount <= RemainingBytes); + memcpy(TargetBuffer, CursorPtr, ByteCount); + CursorPtr += ByteCount; + RemainingBytes -= ByteCount; + }; + + CbPackageReader Reader; + uint64_t InitialRead = Reader.ProcessHeaderData(nullptr, 0); + uint64_t NextBytes = Reader.ProcessHeaderData(ConsumeBytes(InitialRead), InitialRead); + NextBytes = Reader.ProcessHeaderData(ConsumeBytes(NextBytes), NextBytes); + auto Buffers = Reader.GetPayloadBuffers(); + + for (auto& PayloadBuffer : Buffers) + { + CopyBytes(PayloadBuffer.MutableData(), PayloadBuffer.GetSize()); + } +} + +void +forcelink_httpshared() +{ +} + +#endif + +} // namespace zen diff --git a/zenhttp/httpshared.h b/zenhttp/httpshared.h deleted file mode 100644 index 92c1ef9c6..000000000 --- a/zenhttp/httpshared.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/iobuffer.h> -#include <zencore/iohash.h> - -#include <functional> - -namespace zen { - -class IoBuffer; -class CbPackage; -class CompositeBuffer; - -struct CbPackageHeader -{ - uint32_t HeaderMagic; - uint32_t AttachmentCount; - uint32_t Reserved1; - uint32_t Reserved2; -}; - -static_assert(sizeof(CbPackageHeader) == 16); - -static constinit uint32_t kCbPkgMagic = 0xaa77aacc; - -struct CbAttachmentEntry -{ - uint64_t AttachmentSize; - uint32_t Flags; - IoHash AttachmentHash; - - enum - { - kIsCompressed = (1u << 0), // Is marshaled using compressed buffer storage format - kIsObject = (1u << 1), // Is compact binary object - }; -}; - -static_assert(sizeof(CbAttachmentEntry) == 32); - -std::vector<IoBuffer> FormatPackageMessage(const CbPackage& Data); -CompositeBuffer FormatPackageMessageBuffer(const CbPackage& Data); -CbPackage ParsePackageMessage( - IoBuffer Payload, - std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> CreateBuffer = [](const IoHash&, uint64_t Size) -> IoBuffer { - return IoBuffer{Size}; - }); - -} // namespace zen diff --git a/zenhttp/include/zenhttp/httpshared.h b/zenhttp/include/zenhttp/httpshared.h index 92c1ef9c6..2e728577d 100644 --- a/zenhttp/include/zenhttp/httpshared.h +++ b/zenhttp/include/zenhttp/httpshared.h @@ -13,10 +13,23 @@ class IoBuffer; class CbPackage; class CompositeBuffer; +/** _____ _ _____ _ + / ____| | | __ \ | | + | | | |__ | |__) |_ _ ___| | ____ _ __ _ ___ + | | | '_ \| ___/ _` |/ __| |/ / _` |/ _` |/ _ \ + | |____| |_) | | | (_| | (__| < (_| | (_| | __/ + \_____|_.__/|_| \__,_|\___|_|\_\__,_|\__, |\___| + __/ | + |___/ + + Structures and code related to handling CbPackage transactions + + */ + struct CbPackageHeader { uint32_t HeaderMagic; - uint32_t AttachmentCount; + uint32_t AttachmentCount; // TODO: should add ability to opt out of implicit root document? uint32_t Reserved1; uint32_t Reserved2; }; @@ -27,7 +40,7 @@ static constinit uint32_t kCbPkgMagic = 0xaa77aacc; struct CbAttachmentEntry { - uint64_t AttachmentSize; + uint64_t PayloadSize; uint32_t Flags; IoHash AttachmentHash; @@ -35,6 +48,7 @@ struct CbAttachmentEntry { kIsCompressed = (1u << 0), // Is marshaled using compressed buffer storage format kIsObject = (1u << 1), // Is compact binary object + kIsError = (1u << 2), // Is error (compact binary formatted) object }; }; @@ -48,4 +62,41 @@ CbPackage ParsePackageMessage( return IoBuffer{Size}; }); +/** Streaming reader for compact binary packages + + The goal is to ultimately support zero-copy I/O, but for now there'll be some + copying involved on some platforms at least. + + */ +class CbPackageReader +{ +public: + CbPackageReader(); + ~CbPackageReader(); + + void SetPayloadBufferCreator(std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> CreateBuffer); + + /** Process header data + */ + uint64_t ProcessHeaderData(const void* Data, uint64_t DataBytes); + + std::span<IoBuffer> GetPayloadBuffers() { return m_PayloadBuffers; } + +private: + enum class State + { + kInitialState, + kReadingHeader, + kReadingAttachmentEntries, + kReadingBuffers + } m_CurrentState = State::kInitialState; + + std::function<IoBuffer(const IoHash& Cid, uint64_t Size)> m_CreateBuffer; + std::vector<IoBuffer> m_PayloadBuffers; + std::vector<CbAttachmentEntry> m_AttachmentEntries; + CbPackageHeader m_PackageHeader; +}; + +void forcelink_httpshared(); + } // namespace zen diff --git a/zenhttp/zenhttp.cpp b/zenhttp/zenhttp.cpp index 637486f55..0194abdcb 100644 --- a/zenhttp/zenhttp.cpp +++ b/zenhttp/zenhttp.cpp @@ -3,6 +3,7 @@ #include <zenhttp/zenhttp.h> #include <zenhttp/httpserver.h> +#include <zenhttp/httpshared.h> namespace zen { @@ -10,6 +11,7 @@ void zenhttp_forcelinktests() { http_forcelink(); + forcelink_httpshared(); } -} // namespace zen
\ No newline at end of file +} // namespace zen diff --git a/zenhttp/zenhttp.vcxproj b/zenhttp/zenhttp.vcxproj index eca9898d3..899cf4bd1 100644 --- a/zenhttp/zenhttp.vcxproj +++ b/zenhttp/zenhttp.vcxproj @@ -104,12 +104,12 @@ </ItemGroup> <ItemGroup> <ClInclude Include="httpnull.h" /> - <ClInclude Include="httpshared.h" /> <ClInclude Include="httpsys.h" /> <ClInclude Include="httpuws.h" /> <ClInclude Include="include\zenhttp\httpclient.h" /> <ClInclude Include="include\zenhttp\httpcommon.h" /> <ClInclude Include="include\zenhttp\httpserver.h" /> + <ClInclude Include="include\zenhttp\httpshared.h" /> <ClInclude Include="include\zenhttp\zenhttp.h" /> <ClInclude Include="iothreadpool.h" /> </ItemGroup> diff --git a/zenhttp/zenhttp.vcxproj.filters b/zenhttp/zenhttp.vcxproj.filters index 17f71bed1..2e968055c 100644 --- a/zenhttp/zenhttp.vcxproj.filters +++ b/zenhttp/zenhttp.vcxproj.filters @@ -18,8 +18,8 @@ <ClInclude Include="include\zenhttp\zenhttp.h" /> <ClInclude Include="httpnull.h" /> <ClInclude Include="httpuws.h" /> - <ClInclude Include="httpshared.h" /> <ClInclude Include="include\zenhttp\httpcommon.h" /> + <ClInclude Include="include\zenhttp\httpshared.h" /> </ItemGroup> <ItemGroup> <None Include="xmake.lua" /> diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index 794a5fe94..9b5a48128 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -968,9 +968,8 @@ TEST_CASE("project.basic") Body << "engine" << "/zooom"; - zen::MemoryOutStream MemOut; - zen::BinaryWriter Writer{MemOut}; - Body.Save(Writer); + zen::BinaryWriter MemOut; + Body.Save(MemOut); auto Response = cpr::Post(cpr::Url{BaseUri.c_str()}, cpr::Body{(const char*)MemOut.Data(), MemOut.Size()}); CHECK(Response.status_code == 201); @@ -1035,11 +1034,10 @@ TEST_CASE("project.basic") zen::CbObject Op = OpWriter.Save(); - zen::MemoryOutStream MemOut; - zen::BinaryWriter Writer(MemOut); - zen::CbPackage OpPackage(Op); + zen::BinaryWriter MemOut; + zen::CbPackage OpPackage(Op); OpPackage.AddAttachment(Attach); - OpPackage.Save(Writer); + OpPackage.Save(MemOut); { zen::StringBuilder<64> PostUri; @@ -1130,9 +1128,8 @@ TEST_CASE("zcache.basic") zen::CbObjectWriter Cbo; Cbo << "index" << i; - zen::MemoryOutStream MemOut; - zen::BinaryWriter Writer{MemOut}; - Cbo.Save(Writer); + zen::BinaryWriter MemOut; + Cbo.Save(MemOut); zen::IoHash Key = HashKey(i); @@ -1160,9 +1157,8 @@ TEST_CASE("zcache.basic") zen::CbObjectWriter Cbo; Cbo << "index" << 42; - zen::MemoryOutStream MemOut; - zen::BinaryWriter Writer{MemOut}; - Cbo.Save(Writer); + zen::BinaryWriter MemOut; + Cbo.Save(MemOut); zen::IoHash Key = HashKey(442); @@ -1218,10 +1214,9 @@ TEST_CASE("zcache.cbpackage") }; auto SerializeToBuffer = [](zen::CbPackage Package) -> zen::IoBuffer { - zen::MemoryOutStream MemStream; - zen::BinaryWriter Writer(MemStream); + zen::BinaryWriter MemStream; - Package.Save(Writer); + Package.Save(MemStream); return zen::IoBuffer(zen::IoBuffer::Clone, MemStream.Data(), MemStream.Size()); }; @@ -1469,9 +1464,8 @@ TEST_CASE("zcache.policy") }; auto ToBuffer = [](zen::CbPackage Package) -> zen::IoBuffer { - zen::MemoryOutStream MemStream; - zen::BinaryWriter Writer(MemStream); - Package.Save(Writer); + zen::BinaryWriter MemStream; + Package.Save(MemStream); return zen::IoBuffer(zen::IoBuffer::Clone, MemStream.Data(), MemStream.Size()); }; @@ -1828,8 +1822,7 @@ struct RemoteExecutionRequest } PrepReq.EndArray(); - zen::BinaryWriter MemWriter(m_MemOut); - PrepReq.Save(MemWriter); + PrepReq.Save(m_MemOut); } void Prep() @@ -1931,7 +1924,7 @@ private: const std::string m_BaseUri = "http://{}:{}/exec/jobs"_format(m_HostName, m_PortNumber); const std::string m_CasUri = "http://{}:{}/cas"_format(m_HostName, m_PortNumber); Visitor m_Visit{m_TreePath}; - zen::MemoryOutStream m_MemOut; + zen::BinaryWriter m_MemOut; }; TEST_CASE("exec.basic") diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 3ac1ec37f..74cee6614 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -290,9 +290,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (QueryUpstream) { - const ZenContentType CacheRecordType = Ref.BucketSegment == "legacy"sv ? ZenContentType::kBinary - : AcceptType == ZenContentType::kCbPackage ? ZenContentType::kCbPackage - : ZenContentType::kCbObject; + const ZenContentType CacheRecordType = AcceptType; if (auto UpstreamResult = m_UpstreamCache->GetCacheRecord({Ref.BucketSegment, Ref.HashKey}, CacheRecordType); UpstreamResult.Success) @@ -321,9 +319,10 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request else { Success = false; - ZEN_WARN("Get - cache record '{}/{}' FAILED, invalid compact binary object from upstream", + ZEN_WARN("Get - '{}/{}' '{}' FAILED, invalid compact binary object from upstream", Ref.BucketSegment, - Ref.HashKey); + Ref.HashKey, + ToString(AcceptType)); } } @@ -353,9 +352,10 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request } else { - ZEN_WARN("GET - cache record '{}/{}' FAILED, upstream attachment not compressed", + ZEN_WARN("Get - '{}/{}' '{}' FAILED, upstream attachment not compressed", Ref.BucketSegment, - Ref.HashKey); + Ref.HashKey, + ToString(ZenContentType::kCbPackage)); } } AttachmentCount++; @@ -370,9 +370,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request CbPackage PackageWithoutAttachments; PackageWithoutAttachments.SetObject(CacheRecord); - MemoryOutStream MemStream; - BinaryWriter Writer(MemStream); - PackageWithoutAttachments.Save(Writer); + BinaryWriter MemStream; + PackageWithoutAttachments.Save(MemStream); Value.Value = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); } @@ -380,15 +379,16 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request else { Success = false; - ZEN_WARN("GET - cache record '{}/{}' FAILED, attachments missing in upstream package", + ZEN_WARN("Get - '{}/{}' '{}' FAILED, attachments missing in upstream package", Ref.BucketSegment, - Ref.HashKey); + Ref.HashKey, + ToString(AcceptType)); } } else { Success = false; - ZEN_WARN("GET - cache record '{}/{}' FAILED, invalid upstream package", Ref.BucketSegment, Ref.HashKey); + ZEN_WARN("Get - '{}/{}' '{}' FAILED, invalid upstream package", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); } } } @@ -396,8 +396,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (!Success) { - ZEN_DEBUG("MISS - '{}/{}'", Ref.BucketSegment, Ref.HashKey); - + ZEN_DEBUG("MISS - '{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); return Request.WriteResponse(HttpResponseCode::NotFound); } @@ -409,7 +408,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (ValidationResult != CbValidateError::None) { - ZEN_WARN("GET - cache record '{}/{}' FAILED, invalid compact binary object", Ref.BucketSegment, Ref.HashKey); + ZEN_WARN("GET - '{}/{}' '{}' FAILED, invalid compact binary object", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType)); return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Invalid cache record"sv); } @@ -435,9 +434,10 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request if (ValidCount != AttachmentCount) { - ZEN_WARN("GET - cache record '{}/{}' FAILED, found '{}' of '{}' attachments", + ZEN_WARN("GET - '{}/{}' '{}' FAILED, found '{}' of '{}' attachments", Ref.BucketSegment, Ref.HashKey, + ToString(AcceptType), ValidCount, AttachmentCount); @@ -447,15 +447,15 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request Package.SetObject(LoadCompactBinaryObject(Value.Value)); - ZEN_DEBUG("HIT - '{}/{}' {}, {} attachments (LOCAL)", + ZEN_DEBUG("HIT - '{}/{}' {} '{}', {} attachments (LOCAL)", Ref.BucketSegment, Ref.HashKey, NiceBytes(AttachmentBytes + Value.Value.Size()), + ToString(HttpContentType::kCbPackage), AttachmentCount); - MemoryOutStream MemStream; - BinaryWriter Writer(MemStream); - Package.Save(Writer); + BinaryWriter MemStream; + Package.Save(MemStream); IoBuffer Response(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); @@ -463,10 +463,11 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request } else { - ZEN_DEBUG("HIT - '{}/{}' {} ({})", + ZEN_DEBUG("HIT - '{}/{}' {} '{}' ({})", Ref.BucketSegment, Ref.HashKey, NiceBytes(Value.Value.Size()), + ToString(Value.Value.GetContentType()), InUpstreamCache ? "UPSTREAM" : "LOCAL"); Request.WriteResponse(HttpResponseCode::OK, Value.Value.GetContentType(), Value.Value); @@ -486,10 +487,12 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request const HttpContentType ContentType = Request.RequestContentType(); const bool StoreUpstream = m_UpstreamCache && (CachePolicy::StoreRemote == (Policy & CachePolicy::StoreRemote)); - if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kUnknownContentType) + Body.SetContentType(ContentType); + + if (ContentType == HttpContentType::kBinary) { + ZEN_DEBUG("PUT - '{}/{}' {} '{}'", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ToString(ContentType)); m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body}); - ZEN_DEBUG("PUT - binary '{}/{}' {}", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size())); if (StoreUpstream) { @@ -505,7 +508,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (ValidationResult != CbValidateError::None) { - ZEN_WARN("PUT - cache record '{}/{}' ({} bytes) FAILED, invalid compact binary", Ref.BucketSegment, Ref.HashKey, Body.Size()); + ZEN_WARN("PUT - '{}/{}' '{}' FAILED, invalid compact binary", Ref.BucketSegment, Ref.HashKey, ToString(ContentType)); return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Compact binary validation failed"sv); } @@ -527,7 +530,13 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (ValidCacheRecord) { - ZEN_DEBUG("PUT - cache record '{}/{}' {}, {} attachments", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.Size()), ValidCount); + ZEN_DEBUG("PUT - '{}/{}' {} '{}', {} attachments", + Ref.BucketSegment, + Ref.HashKey, + NiceBytes(Body.Size()), + ToString(ContentType), + ValidCount); + m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body}); if (StoreUpstream) @@ -542,9 +551,10 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request } else { - ZEN_WARN("PUT - cache record '{}/{}' FAILED, found {}/{} attachments", + ZEN_WARN("PUT - '{}/{}' '{}' FAILED, found {}/{} attachments", Ref.BucketSegment, Ref.HashKey, + ToString(ContentType), ValidCount, AttachmentCount); @@ -557,7 +567,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (!Package.TryLoad(Body)) { - ZEN_WARN("PUT - cache record '{}/{}' FAILED, invalid package", Ref.BucketSegment, Ref.HashKey); + ZEN_WARN("PUT - '{}/{}' '{}' FAILED, invalid package", Ref.BucketSegment, Ref.HashKey, ToString(ContentType)); return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"sv); } @@ -586,17 +596,19 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request } else { - ZEN_WARN("PUT - cache record '{}/{}' FAILED, attachment '{}' is not compressed", + ZEN_WARN("PUT - '{}/{}' '{}' FAILED, attachment '{}' is not compressed", Ref.BucketSegment, Ref.HashKey, + ToString(HttpContentType::kCbPackage), AttachmentHash.AsHash()); } } else { - ZEN_WARN("PUT - cache record '{}/{}' FAILED, missing attachment '{}'", + ZEN_WARN("PUT - '{}/{}' '{}' FAILED, missing attachment '{}'", Ref.BucketSegment, Ref.HashKey, + ToString(HttpContentType::kCbPackage), AttachmentHash.AsHash()); } }); @@ -608,13 +620,17 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachments"sv); } - ZEN_DEBUG("PUT - cache record '{}/{}' {}, {}/{} new attachments", + ZEN_DEBUG("PUT - '{}/{}' {} '{}', {}/{} new attachments", Ref.BucketSegment, Ref.HashKey, NiceBytes(Body.GetSize()), + ToString(ContentType), NewAttachmentCount, Attachments.size()); + IoBuffer CacheRecordValue = CacheRecord.GetBuffer().AsIoBuffer(); + CacheRecordValue.SetContentType(ZenContentType::kCbObject); + m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = CacheRecord.GetBuffer().AsIoBuffer()}); if (StoreUpstream) @@ -629,7 +645,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request } else { - Request.WriteResponse(HttpResponseCode::BadRequest); + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Content-Type invalid"sv); } } @@ -692,12 +708,12 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques return Request.WriteResponse(HttpResponseCode::NotFound); } - ZEN_DEBUG("HIT - '{}/{}/{}' {} (type: {}) ({})", + ZEN_DEBUG("HIT - '{}/{}/{}' {} '{}' ({})", Ref.BucketSegment, Ref.HashKey, Ref.PayloadId, NiceBytes(Payload.Size()), - Payload.GetContentType(), + ToString(Payload.GetContentType()), InUpstreamCache ? "UPSTREAM" : "LOCAL"); Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Payload); @@ -716,6 +732,8 @@ HttpStructuredCacheService::HandlePutCachePayload(zen::HttpServerRequest& Reques return Request.WriteResponse(HttpResponseCode::BadRequest); } + Body.SetContentType(Request.RequestContentType()); + IoHash ChunkHash = IoHash::HashBuffer(Body); CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Body)); @@ -733,12 +751,12 @@ HttpStructuredCacheService::HandlePutCachePayload(zen::HttpServerRequest& Reques m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash); - ZEN_DEBUG("PUT - payload '{}/{}/{}' {} (type: {}) {}", + ZEN_DEBUG("PUT - '{}/{}/{}' {} '{}' ({})", Ref.BucketSegment, Ref.HashKey, Ref.PayloadId, NiceBytes(Body.Size()), - Body.GetContentType(), + ToString(Body.GetContentType()), Result.New ? "NEW" : "OLD"); const HttpResponseCode ResponseCode = Result.New ? HttpResponseCode::Created : HttpResponseCode::OK; diff --git a/zenserver/casstore.cpp b/zenserver/casstore.cpp index b36dcc09f..88525bd36 100644 --- a/zenserver/casstore.cpp +++ b/zenserver/casstore.cpp @@ -2,8 +2,8 @@ #include "casstore.h" +#include <zencore/fmtutils.h> #include <zencore/logging.h> -#include <zencore/streamutil.h> #include <gsl/gsl-lite.hpp> @@ -14,52 +14,6 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store) m_Router.AddPattern("cas", "([0-9A-Fa-f]{40})"); m_Router.RegisterRoute( - "batch", - [this](HttpRouterRequest& Req) { - HttpServerRequest& ServerRequest = Req.ServerRequest(); - - IoBuffer Payload = ServerRequest.ReadPayload(); - uint64_t EntryCount = Payload.Size() / sizeof(IoHash); - - if ((EntryCount * sizeof(IoHash)) != Payload.Size()) - { - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest); - } - - const IoHash* Hashes = reinterpret_cast<const IoHash*>(Payload.Data()); - std::vector<IoBuffer> Values; - - MemoryOutStream HeaderStream; - BinaryWriter HeaderWriter(HeaderStream); - - Values.emplace_back(); // Placeholder for header - - // Build response header - HeaderWriter << uint32_t(0x12340000) << uint32_t(0); - - for (uint64_t i = 0; i < EntryCount; ++i) - { - IoHash ChunkHash = Hashes[i]; - IoBuffer Value = m_CasStore.FindChunk(ChunkHash); - - if (Value) - { - Values.emplace_back(std::move(Value)); - HeaderWriter << ChunkHash << uint64_t(Value.Size()); - } - } - - // Make real header - - const_cast<uint32_t*>(reinterpret_cast<const uint32_t*>(HeaderStream.Data()))[1] = uint32_t(Values.size() - 1); - - Values[0] = IoBufferBuilder::MakeCloneFromMemory(HeaderStream.Data(), HeaderStream.Size()); - - ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Values); - }, - HttpVerb::kPost); - - m_Router.RegisterRoute( "{cas}", [this](HttpRouterRequest& Req) { IoHash Hash = IoHash::FromHexString(Req.GetCapture(1)); diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index 15d9e0141..a522aa35b 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -767,8 +767,8 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action) lpThreadAttributes, bInheritHandles, dwCreationFlags, - nullptr, // (LPVOID)EnvironmentBlock.c_str(), // Environment block - SandboxPath.c_str(), // Current directory + (LPVOID)EnvironmentBlock.Data(), // Environment block + SandboxPath.c_str(), // Current directory &StartupInfo, /* out */ &ProcessInformation); diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index 1f4239b23..1a9eb2c67 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -635,8 +635,7 @@ ProjectStore::Project::Read() void ProjectStore::Project::Write() { - MemoryOutStream Mem; - BinaryWriter Writer(Mem); + BinaryWriter Mem; CbObjectWriter Cfg; Cfg << "id" << Identifier; @@ -644,7 +643,7 @@ ProjectStore::Project::Write() Cfg << "project" << ProjectRootDir; Cfg << "engine" << EngineRootDir; - Cfg.Save(Writer); + Cfg.Save(Mem); CreateDirectories(m_OplogStoragePath); @@ -951,9 +950,8 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) // Parse Request - IoBuffer Payload = HttpReq.ReadPayload(); - MemoryInStream MemIn(Payload.Data(), Payload.Size()); - BinaryReader Reader(MemIn); + IoBuffer Payload = HttpReq.ReadPayload(); + BinaryReader Reader(Payload); struct RequestHeader { diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index f056c1c76..0dd16cd06 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -172,9 +172,8 @@ namespace detail { if (Result.Success) { - MemoryOutStream MemStream; - BinaryWriter Writer(MemStream); - Package.Save(Writer); + BinaryWriter MemStream; + Package.Save(MemStream); Result.Response = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); } @@ -420,9 +419,8 @@ namespace detail { } } - MemoryOutStream MemStream; - BinaryWriter Writer(MemStream); - Package.Save(Writer); + BinaryWriter MemStream; + Package.Save(MemStream); IoBuffer PackagePayload(IoBuffer::Wrap, MemStream.Data(), MemStream.Size()); for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp index 530bed32a..c988a6b0b 100644 --- a/zenserver/upstream/zen.cpp +++ b/zenserver/upstream/zen.cpp @@ -181,10 +181,9 @@ Mesh::BroadcastPacket(CbObjectWriter& Obj) detail::MessageHeader* Message = reinterpret_cast<detail::MessageHeader*>(MessageBuffer); *Message = {}; - MemoryOutStream MemOut; - BinaryWriter Writer(MemOut); + BinaryWriter MemOut; - Obj.Save(Writer); + Obj.Save(MemOut); // TODO: check that it fits in a packet! diff --git a/zenserver/windows/service.cpp b/zenserver/windows/service.cpp index 017b5f9a7..23cefb7b5 100644 --- a/zenserver/windows/service.cpp +++ b/zenserver/windows/service.cpp @@ -2,6 +2,7 @@ #include "service.h" +#include <zencore/except.h> #include <zencore/zencore.h> #include <stdio.h> @@ -146,26 +147,34 @@ CallMain(DWORD, LPSTR*) int WindowsService::ServiceMain() { - if (zen::IsInteractiveSession()) - { - // Not actually running as a service - return Run(); - } - else + gSvc = this; + + SERVICE_TABLE_ENTRY DispatchTable[] = {{(LPWSTR)SVCNAME, (LPSERVICE_MAIN_FUNCTION)&CallMain}, {NULL, NULL}}; + + // This call returns when the service has stopped. + // The process should simply terminate when the call returns. + + if (!StartServiceCtrlDispatcher(DispatchTable)) { - gSvc = this; + const DWORD dwError = zen::GetLastError(); - SERVICE_TABLE_ENTRY DispatchTable[] = {{(LPWSTR)SVCNAME, (LPSERVICE_MAIN_FUNCTION)&CallMain}, {NULL, NULL}}; + if (dwError == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) + { + // Not actually running as a service + gSvc = nullptr; - // This call returns when the service has stopped. - // The process should simply terminate when the call returns. + zen::SetIsInteractiveSession(true); - if (!StartServiceCtrlDispatcher(DispatchTable)) + return Run(); + } + else { - SvcReportEvent((LPTSTR)L"StartServiceCtrlDispatcher"); + zen::ThrowSystemError(dwError, "StartServiceCtrlDispatcher failed"); } } + zen::SetIsInteractiveSession(false); + return 0; } |