diff options
| author | Martin Ridgers <[email protected]> | 2021-10-20 13:19:39 +0200 |
|---|---|---|
| committer | Martin Ridgers <[email protected]> | 2021-10-20 13:19:39 +0200 |
| commit | 13feae57532a1ad4d14feb5ee00f397fa6c370cc (patch) | |
| tree | 4bfdfe0aa239435baddef9ee64965c39ec7f6551 | |
| parent | Compile fixes due to std::fs::path's differing character types (diff) | |
| parent | zen server: Added root manifest, with support for explicit schema versioning (diff) | |
| download | zen-13feae57532a1ad4d14feb5ee00f397fa6c370cc.tar.xz zen-13feae57532a1ad4d14feb5ee00f397fa6c370cc.zip | |
Merged main
| -rw-r--r-- | zencore/compactbinary.cpp | 91 | ||||
| -rw-r--r-- | zencore/include/zencore/compactbinary.h | 2 | ||||
| -rw-r--r-- | zencore/include/zencore/iobuffer.h | 3 | ||||
| -rw-r--r-- | zencore/include/zencore/string.h | 4 | ||||
| -rw-r--r-- | zencore/string.cpp | 6 | ||||
| -rw-r--r-- | zenhttp/httpasio.cpp | 50 | ||||
| -rw-r--r-- | zenhttp/httpsys.cpp | 12 | ||||
| -rw-r--r-- | zenserver-test/zenserver-test.cpp | 13 | ||||
| -rw-r--r-- | zenserver/projectstore.cpp | 2 | ||||
| -rw-r--r-- | zenserver/zenserver.cpp | 117 |
10 files changed, 233 insertions, 67 deletions
diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp index 6a95ce555..ab0904b2b 100644 --- a/zencore/compactbinary.cpp +++ b/zencore/compactbinary.cpp @@ -16,6 +16,10 @@ #include <fmt/format.h> #include <string_view> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #if ZEN_WITH_TESTS # include <json11.hpp> # include <zencore/compactbinarybuilder.h> @@ -42,6 +46,18 @@ IsLeapYear(int Year) return false; } +DateTime +DateTime::Now() +{ +#if ZEN_PLATFORM_WINDOWS + FILETIME SysTime; + GetSystemTimeAsFileTime(&SysTime); + return DateTime{504911232000000000ull + (uint64_t(SysTime.dwHighDateTime) << 32) | SysTime.dwLowDateTime}; +#else +# error Needs implementation +#endif +} + void DateTime::Set(int Year, int Month, int Day, int Hour, int Minute, int Second, int MilliSecond) { @@ -96,31 +112,31 @@ TimeSpan::ToString(const char* Format) const switch (*Format) { case 'd': - Result.Append("%i"_format(GetDays())); + Result.Append("{}"_format(GetDays())); break; case 'D': - Result.Append("%08i"_format(GetDays())); + Result.Append("{:08}"_format(GetDays())); break; case 'h': - Result.Append("%02i"_format(GetHours())); + Result.Append("{:02}"_format(GetHours())); break; case 'm': - Result.Append("%02i"_format(GetMinutes())); + Result.Append("{:02}"_format(GetMinutes())); break; case 's': - Result.Append("%02i"_format(GetSeconds())); + Result.Append("{:02}"_format(GetSeconds())); break; case 'f': - Result.Append("%03i"_format(GetFractionMilli())); + Result.Append("{:03}"_format(GetFractionMilli())); break; case 'u': - Result.Append("%06i"_format(GetFractionMicro())); + Result.Append("{:06}"_format(GetFractionMicro())); break; case 't': - Result.Append("%07i"_format(GetFractionTicks())); + Result.Append("{:07}"_format(GetFractionTicks())); break; case 'n': - Result.Append("%09i"_format(GetFractionNano())); + Result.Append("{:09}"_format(GetFractionNano())); break; default: Result.Append(*Format); @@ -247,8 +263,8 @@ DateTime::ToString(const char* Format) const { using namespace fmt::literals; - StringBuilder<32> Result; - int Year, Month, Day; + ExtendableStringBuilder<32> Result; + int Year, Month, Day; GetDate(Year, Month, Day); @@ -263,32 +279,32 @@ DateTime::ToString(const char* Format) const // case 'a': Result.Append(IsMorning() ? TEXT("am") : TEXT("pm")); break; // case 'A': Result.Append(IsMorning() ? TEXT("AM") : TEXT("PM")); break; case 'd': - Result.Append("%02i"_format(Day)); + Result.Append("{:02}"_format(Day)); break; // case 'D': Result.Appendf(TEXT("%03i"), GetDayOfYear()); break; case 'm': - Result.Append("%02i"_format(Month)); + Result.Append("{:02}"_format(Month)); break; case 'y': - Result.Append("%02i"_format(Year % 100)); + Result.Append("{:02}"_format(Year % 100)); break; case 'Y': - Result.Append("%04i"_format(Year % 100)); + Result.Append("{:04}"_format(Year)); break; case 'h': - Result.Append("%02i"_format(GetHour12())); + Result.Append("{:02}"_format(GetHour12())); break; case 'H': - Result.Append("%02i"_format(GetHour())); + Result.Append("{:02}"_format(GetHour())); break; case 'M': - Result.Append("%02i"_format(GetMinute())); + Result.Append("{:02}"_format(GetMinute())); break; case 'S': - Result.Append("%02i"_format(GetSecond())); + Result.Append("{:02}"_format(GetSecond())); break; case 's': - Result.Append("%03i"_format(GetMillisecond())); + Result.Append("{:03}"_format(GetMillisecond())); break; default: Result.Append(*Format); @@ -1659,10 +1675,10 @@ private: } private: - StringBuilderBase& Builder; - StringBuilder<32> NewLineAndIndent; - bool NeedsComma{false}; - bool NeedsNewLine{false}; + StringBuilderBase& Builder; + ExtendableStringBuilder<32> NewLineAndIndent; + bool NeedsComma{false}; + bool NeedsNewLine{false}; }; void @@ -1902,6 +1918,33 @@ TEST_CASE("uson.json") CHECK(DoubleValue == 0); } } + +TEST_CASE("uson.datetime") +{ + using namespace std::literals; + + { + DateTime D1600(1601, 1, 1); + CHECK_EQ(D1600.GetYear(), 1601); + CHECK_EQ(D1600.GetMonth(), 1); + CHECK_EQ(D1600.GetDay(), 1); + CHECK_EQ(D1600.GetHour(), 0); + CHECK_EQ(D1600.GetMinute(), 0); + CHECK_EQ(D1600.GetSecond(), 0); + + CHECK_EQ(D1600.ToIso8601(), "1601-01-01T00:00:00.000Z"sv); + } + + { + DateTime D72(1972, 2, 23, 17, 30, 10); + CHECK_EQ(D72.GetYear(), 1972); + CHECK_EQ(D72.GetMonth(), 2); + CHECK_EQ(D72.GetDay(), 23); + CHECK_EQ(D72.GetHour(), 17); + CHECK_EQ(D72.GetMinute(), 30); + CHECK_EQ(D72.GetSecond(), 10); + } +} #endif } // namespace zen diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h index ab01402f8..06331c510 100644 --- a/zencore/include/zencore/compactbinary.h +++ b/zencore/include/zencore/compactbinary.h @@ -43,6 +43,8 @@ public: inline uint64_t GetTicks() const { return Ticks; } + static DateTime Now(); + int GetYear() const; int GetMonth() const; int GetDay() const; diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h index 60fee1dc5..847bed606 100644 --- a/zencore/include/zencore/iobuffer.h +++ b/zencore/include/zencore/iobuffer.h @@ -357,6 +357,9 @@ public: [[nodiscard]] inline ZenContentType GetContentType() const { return m_Core->GetContentType(); } [[nodiscard]] ZENCORE_API bool GetFileReference(IoBufferFileReference& OutRef) const; + template<typename T> + [[nodiscard]] const T* Data() const { return reinterpret_cast<const T*>(m_Core->DataPointer()); } + private: RefPtr<IoBufferCore> m_Core = new IoBufferCore; diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h index e2a957786..47fd5643d 100644 --- a/zencore/include/zencore/string.h +++ b/zencore/include/zencore/string.h @@ -631,7 +631,7 @@ HashStringAsLowerDjb2(const std::string_view& InString) { uint32_t HashValue = 5381; - for (int CurChar : InString) + for (uint8_t CurChar : InString) { CurChar -= ((CurChar - 'A') <= ('Z' - 'A')) * ('A' - 'a'); // this should be compiled into branchless logic HashValue = HashValue * 33 + CurChar; @@ -649,7 +649,7 @@ ToLower(const std::string_view& InString) for (char& CurChar : Out) { - CurChar -= ((CurChar - 'A') <= ('Z' - 'A')) * ('A' - 'a'); // this should be compiled into branchless logic + CurChar -= (uint8_t(CurChar - 'A') <= ('Z' - 'A')) * ('A' - 'a'); // this should be compiled into branchless logic } return Out; diff --git a/zencore/string.cpp b/zencore/string.cpp index 76cd69871..3fa32692e 100644 --- a/zencore/string.cpp +++ b/zencore/string.cpp @@ -945,6 +945,12 @@ TEST_CASE("string") CHECK(HashStringAsLowerDjb2("aBCd"sv) == HashStringDjb2(ToLower("aBCd"sv))); } + SUBCASE("tolower") + { + CHECK_EQ(ToLower("te!st"sv), "te!st"sv); + CHECK_EQ(ToLower("TE%St"sv), "te%st"sv); + } + SUBCASE("ForEachStrTok") { const auto Tokens = "here,is,my,different,tokens"sv; diff --git a/zenhttp/httpasio.cpp b/zenhttp/httpasio.cpp index 12fbdb61e..ce6503a96 100644 --- a/zenhttp/httpasio.cpp +++ b/zenhttp/httpasio.cpp @@ -2,14 +2,17 @@ #include "httpasio.h" +#include <zencore/logging.h> #include <zenhttp/httpserver.h> -#include <conio.h> -#include <zencore/logging.h> +#include <deque> +#include <memory_resource> +ZEN_THIRD_PARTY_INCLUDES_START +#include <conio.h> #include <http_parser.h> #include <asio.hpp> -#include <deque> +ZEN_THIRD_PARTY_INCLUDES_END namespace zen::asio_http { @@ -24,6 +27,7 @@ struct HttpServerConnection; static constinit uint32_t HashContentLength = HashStringAsLowerDjb2("Content-Length"sv); static constinit uint32_t HashContentType = HashStringAsLowerDjb2("Content-Type"sv); static constinit uint32_t HashAccept = HashStringAsLowerDjb2("Accept"sv); +static constinit uint32_t HashExpect = HashStringAsLowerDjb2("Expect"sv); static constinit uint32_t HashSession = HashStringAsLowerDjb2("UE-Session"sv); static constinit uint32_t HashRequest = HashStringAsLowerDjb2("UE-Request"sv); @@ -31,7 +35,7 @@ inline spdlog::logger& InitLogger() { spdlog::logger& Logger = logging::Get("asio"); - // Logger.set_level(spdlog::level::trace); + //Logger.set_level(spdlog::level::trace); return Logger; } @@ -144,7 +148,6 @@ private: }; HttpServerConnection& m_Connection; - char m_HeaderBuffer[512]; char* m_HeaderCursor = m_HeaderBuffer; char* m_Url = nullptr; size_t m_UrlLength = 0; @@ -158,13 +161,15 @@ private: int8_t m_ContentLengthHeaderIndex; int8_t m_AcceptHeaderIndex; int8_t m_ContentTypeHeaderIndex; - int m_RequestId = -1; + HttpVerb m_RequestVerb; + bool m_KeepAlive = false; + bool m_Expect100Continue = false; + int m_RequestId = -1; Oid m_SessionId{}; IoBuffer m_BodyBuffer; - uint64_t m_BodyPosition; + uint64_t m_BodyPosition = 0; http_parser m_Parser; - HttpVerb m_RequestVerb; - bool m_KeepAlive = false; + char m_HeaderBuffer[512]; void AppendInputBytes(const char* Data, size_t Bytes); void AppendCurrentHeader(); @@ -327,6 +332,7 @@ HttpServerConnection::HttpServerConnection(HttpAsioServerImpl& Server, std::uniq HttpServerConnection::~HttpServerConnection() { + ZEN_TRACE("destroying connection #{}", m_ConnectionId); } void @@ -701,6 +707,18 @@ HttpRequest::AppendCurrentHeader() { std::from_chars(HeaderValue.data(), HeaderValue.data() + HeaderValue.size(), m_RequestId); } + else if (HeaderHash == HashExpect) + { + if (HeaderValue == "100-continue"sv) + { + // We don't currently do anything with this + m_Expect100Continue = true; + } + else + { + ZEN_INFO("Unexpected expect - Expect: {}", HeaderValue); + } + } m_Headers.emplace_back(HeaderName, HeaderValue); } @@ -744,10 +762,6 @@ HttpRequest::OnHeadersComplete() { m_BodyBuffer = IoBuffer(ContentLength); } - else - { - m_BodyBuffer = {}; - } m_BodyBuffer.SetContentType(ContentType()); @@ -827,13 +841,14 @@ HttpRequest::ResetState() m_CurrentHeaderName = nullptr; m_Url = nullptr; m_UrlLength = 0; + m_QueryString = nullptr; + m_QueryLength = 0; m_ContentLengthHeaderIndex = -1; m_AcceptHeaderIndex = -1; m_ContentTypeHeaderIndex = -1; - - m_BodyBuffer = {}; - m_BodyPosition = 0; - + m_Expect100Continue = false; + m_BodyBuffer = {}; + m_BodyPosition = 0; m_Headers.clear(); } @@ -1115,6 +1130,7 @@ HttpAsioServerImpl::RouteRequest(std::string_view Url) namespace zen { HttpAsioServer::HttpAsioServer() : m_Impl(std::make_unique<asio_http::HttpAsioServerImpl>()) { + ZEN_DEBUG("Request object size: {} ({:#x})", sizeof(asio_http::HttpRequest), sizeof(asio_http::HttpRequest)); } HttpAsioServer::~HttpAsioServer() diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index 43dfe30d8..242954912 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -341,8 +341,6 @@ HttpMessageResponseRequest::~HttpMessageResponseRequest() void HttpMessageResponseRequest::InitializeForPayload(uint16_t ResponseCode, std::span<IoBuffer> BlobList) { - m_ResponseCode = ResponseCode; - const uint32_t ChunkCount = gsl::narrow<uint32_t>(BlobList.size()); m_HttpDataChunks.reserve(ChunkCount); @@ -407,6 +405,16 @@ HttpMessageResponseRequest::InitializeForPayload(uint16_t ResponseCode, std::spa m_RemainingChunkCount = gsl::narrow<uint32_t>(m_HttpDataChunks.size()); m_TotalDataSize = LocalDataSize; + + if (m_TotalDataSize == 0 && ResponseCode == 200) + { + // Some HTTP clients really don't like empty responses unless a 204 response is sent + m_ResponseCode = uint16_t(HttpResponseCode::NoContent); + } + else + { + m_ResponseCode = ResponseCode; + } } void diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index 0f9026e76..ee9ab7cb3 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -1036,10 +1036,11 @@ TEST_CASE("project.basic") zen::CbObject Op = OpWriter.Save(); - zen::BinaryWriter MemOut; zen::CbPackage OpPackage(Op); OpPackage.AddAttachment(Attach); - OpPackage.Save(MemOut); + + zen::BinaryWriter MemOut; + legacy::SaveCbPackage(OpPackage, MemOut); { zen::StringBuilder<64> PostUri; @@ -1831,7 +1832,7 @@ TEST_CASE("zcache.policy") { cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key)}, cpr::Header{{"Accept", "application/x-ue-cbpkg"}}); - CHECK(Result.status_code == 200); + CHECK(IsHttpSuccessCode(Result.status_code)); CHECK(Result.text.size() == 0); } @@ -1839,7 +1840,7 @@ TEST_CASE("zcache.policy") { cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key)}, cpr::Header{{"Accept", "application/x-ue-cbobject"}}); - CHECK(Result.status_code == 200); + CHECK(IsHttpSuccessCode(Result.status_code)); CHECK(Result.text.size() == 0); } @@ -1847,7 +1848,7 @@ TEST_CASE("zcache.policy") { cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key, PayloadId)}, cpr::Header{{"Accept", "application/x-ue-comp"}}); - CHECK(Result.status_code == 200); + CHECK(IsHttpSuccessCode(Result.status_code)); CHECK(Result.text.size() == 0); } } @@ -1875,7 +1876,7 @@ TEST_CASE("zcache.policy") { cpr::Response Result = cpr::Get(cpr::Url{"{}/{}/{}?skip=data"_format(Cfg.BaseUri, Bucket, Key)}, cpr::Header{{"Accept", "application/octet-stream"}}); - CHECK(Result.status_code == 200); + CHECK(IsHttpSuccessCode(Result.status_code)); CHECK(Result.text.size() == 0); } } diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index 0e1ba01cb..331590720 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -1613,6 +1613,8 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) } m_ProjectStore->DeleteProject(ProjectId); + + return Req.ServerRequest().WriteResponse(HttpResponseCode::NoContent); } break; } diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index c9b52604d..269db7394 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -1,6 +1,7 @@ // Copyright Epic Games, Inc. All Rights Reserved. #include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryvalidation.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/iobuffer.h> @@ -40,6 +41,8 @@ ZEN_THIRD_PARTY_INCLUDES_END # define BUILD_VERSION ("dev-build") #endif +#define ZEN_SCHEMA_VERSION 1 + ////////////////////////////////////////////////////////////////////////// // We don't have any doctest code in this file but this is needed to bring // in some shared code into the executable @@ -154,6 +157,78 @@ public: throw std::runtime_error("Failed to create mutex '{}' - is another instance already running?"_format(MutexName).c_str()); } + // Check root manifest to deal with schema versioning + + bool WipeState = false; + std::string WipeReason = "Unspecified"; + + std::filesystem::path ManifestPath = m_DataRoot / "root_manifest"; + FileContents ManifestData = zen::ReadFile(ManifestPath); + + if (ManifestData.ErrorCode) + { + WipeState = true; + WipeReason = "No manifest present at '{}'"_format(ManifestPath); + } + else + { + IoBuffer Manifest = ManifestData.Flatten(); + + if (CbValidateError ValidationResult = ValidateCompactBinary(Manifest, CbValidateMode::All); + ValidationResult != CbValidateError::None) + { + ZEN_ERROR("Manifest validation failed: {}, state will be wiped", ValidationResult); + + WipeState = true; + WipeReason = "Validation of manifest at '{}' failed: {}"_format(ManifestPath, ValidationResult); + } + else + { + m_RootManifest = LoadCompactBinaryObject(Manifest); + + const int32_t ManifestVersion = m_RootManifest["schema_version"].AsInt32(0); + + if (ManifestVersion != ZEN_SCHEMA_VERSION) + { + WipeState = true; + WipeReason = "Manifest schema version: {}, differs from required: {}"_format(ManifestVersion, ZEN_SCHEMA_VERSION); + } + } + } + + // Handle any state wipe + + if (WipeState) + { + ZEN_WARN("Wiping state at '{}' - reason: '{}'", m_DataRoot, WipeReason); + + std::error_code Ec; + for (const std::filesystem::directory_entry& DirEntry : std::filesystem::directory_iterator{m_DataRoot, Ec}) + { + if (DirEntry.is_directory()) + { + ZEN_INFO("Deleting '{}'", DirEntry.path()); + + std::filesystem::remove_all(DirEntry.path(), Ec); + + if (Ec) + { + ZEN_WARN("Delete of '{}' returned error: '{}'", DirEntry.path(), Ec.message()); + } + } + } + + ZEN_INFO("Wiped all directories in data root"); + + // Write new manifest + + CbObjectWriter Cbo; + Cbo << "schema_version" << ZEN_SCHEMA_VERSION; + m_RootManifest = Cbo.Save(); + + WriteFile(ManifestPath, m_RootManifest.GetBuffer().AsIoBuffer()); + } + // Ok so now we're configured, let's kick things off m_Http = zen::CreateHttpServer(HttpServerClass); @@ -409,6 +484,7 @@ private: ZenServerState::ZenServerEntry* m_ServerEntry = nullptr; bool m_IsDedicatedMode = false; bool m_TestMode = false; + CbObject m_RootManifest; std::filesystem::path m_DataRoot; std::filesystem::path m_ContentRoot; std::jthread m_IoRunner; @@ -693,27 +769,36 @@ main(int argc, char* argv[]) mi_version(); #endif - ZenServerOptions GlobalOptions; - ZenServiceConfig ServiceConfig; - ParseGlobalCliOptions(argc, argv, GlobalOptions, ServiceConfig); - InitializeLogging(GlobalOptions); + try + { + ZenServerOptions GlobalOptions; + ZenServiceConfig ServiceConfig; + ParseGlobalCliOptions(argc, argv, GlobalOptions, ServiceConfig); + InitializeLogging(GlobalOptions); #if ZEN_PLATFORM_WINDOWS - if (GlobalOptions.InstallService) - { - WindowsService::Install(); + if (GlobalOptions.InstallService) + { + WindowsService::Install(); - std::exit(0); - } + std::exit(0); + } - if (GlobalOptions.UninstallService) - { - WindowsService::Delete(); + if (GlobalOptions.UninstallService) + { + WindowsService::Delete(); - std::exit(0); - } + std::exit(0); + } #endif - ZenWindowsService App(GlobalOptions, ServiceConfig); - return App.ServiceMain(); + ZenWindowsService App(GlobalOptions, ServiceConfig); + return App.ServiceMain(); + } + catch (std::exception& Ex) + { + fprintf(stderr, "ERROR: Caught exception in main: '%s'", Ex.what()); + + return 1; + } } |