diff options
37 files changed, 302 insertions, 96 deletions
diff --git a/.gitignore b/.gitignore index bf237a7b3..4ceeab867 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,8 @@ bld/ [Bb]in/ [Oo]bj/ [Ll]og/ +vsxmake20*/ +vs20*/ # Visual Studio Code options directory .vscode/ @@ -55,11 +55,65 @@ You can then build from the command line: It's also possible to generate project files for Visual Studio via -`d:\zen> xmake project -k vsxmake` +`d:\zen> xmake project -k vsxmake -a x64` ## Building on Linux -... coming soon +The following instructions have been collated using Ubuntu 20.04. + +At the time of writing only GCC v11 supports the C++20 features used by Zen. As +this version is not available in the default package repositories we need to +get GCC from one of Ubuntu's toolchain repositories; + +``` +sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test +sudo apt install -y gcc-11 +gcc-11 --version +``` + +Next we need the `xmake` build system. For this we will download and install +`xmake` as a `.deb` package. Check the project's release page for more up to +date `.deb` files; + +``` +wget https://github.com/xmake-io/xmake/releases/download/v2.5.7/xmake-v2.5.7.amd64.deb +sudo dpkg -i xmake-v2.5.7.amd64.deb +xmake --version +``` + +For some of Zen's third party dependencies are provided by Microsoft's `vcpkg` +C++ library manager. After cloning the project there is a initialisation step; + +``` +git clone https://github.com/microsoft/vcpkg.git ~/zen/vcpkg +~/zen/vcpkg/bootstrap-vcpkg.sh +``` + +`xmake` uses an environment variable to find `vcpkg`. Alternatively this can be +done by including `VCPKG_ROOT=...` on the command line when invoking `xmake`; + +``` +export VCPKG_ROOT=~/zen/vcpkg +``` + +Clone the Zen project and tell `xmake` to use the correct GCC version; + +``` +git clone https://github.com/EpicGames/zen.git ~/zen/main +cd ~/zen/main +xmake config --plat=linux --cxx=g++-11 --cc=gcc-11 +``` + +Now we are ready to build Zen. The `--mode=debug` is optional. It has been +included below to show how to change build variants. The `-y` skips `xmake` from +prompting about updating `vcpkg` packages; + +``` +xmake config --mode=debug +xmake build -y +``` + +The `xmake` flags `-vD` can be useful to diagnose `xmake` issues. ## Building on Mac @@ -39,10 +39,24 @@ else end if is_os("windows") then - add_defines("_CRT_SECURE_NO_WARNINGS", "_UNICODE", "UNICODE", "_WIN32_WINNT=0x0A00") + add_defines( + "_CRT_SECURE_NO_WARNINGS", + "_UNICODE", + "UNICODE", + "_CONSOLE", + "_WIN32_WINNT=0x0A00" + ) -- add_ldflags("/MAP") end +if is_os("linux") then + add_cxxflags("-Wno-unused-variable") + add_cxxflags("-Wno-unused-parameter") + add_cxxflags("-Wno-strict-aliasing") + add_cxxflags("-Wno-implicit-fallthrough") + add_cxxflags("-Wno-missing-field-initializers") +end + add_defines("USE_SENTRY=1") add_defines("ZEN_USE_MIMALLOC=1") @@ -59,8 +73,6 @@ option("httpsys") add_defines("ZEN_WITH_HTTPSYS") option_end() -add_defines("UNICODE", "_CONSOLE") - set_warnings("allextra", "error") set_languages("cxx20") diff --git a/zen/cmds/run.cpp b/zen/cmds/run.cpp index 8cbf13566..085e74061 100644 --- a/zen/cmds/run.cpp +++ b/zen/cmds/run.cpp @@ -152,7 +152,7 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (auto It = Visit.m_HashToFile.find(NeedHash); It != Visit.m_HashToFile.end()) { - zen::IoBuffer FileData = zen::IoBufferBuilder::MakeFromFile(It->second.c_str()); + zen::IoBuffer FileData = zen::IoBufferBuilder::MakeFromFile(It->second); cpr::Response CasResponse = cpr::Post(cpr::Url("http://localhost:13337/cas"), cpr::Body((const char*)FileData.Data(), FileData.Size())); diff --git a/zencore/blake3.cpp b/zencore/blake3.cpp index 663f21b6d..5869f19a3 100644 --- a/zencore/blake3.cpp +++ b/zencore/blake3.cpp @@ -8,7 +8,9 @@ #include <zencore/zencore.h> #include "../3rdparty/BLAKE3/c/blake3.h" -#pragma comment(lib, "blake3.lib") +#if ZEN_PLATFORM_WINDOWS +# pragma comment(lib, "blake3.lib") +#endif #include <string.h> diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp index c4694733c..6a95ce555 100644 --- a/zencore/compactbinary.cpp +++ b/zencore/compactbinary.cpp @@ -87,7 +87,7 @@ TimeSpan::ToString(const char* Format) const StringBuilder<128> Result; - Result.Append((Ticks < 0) ? '-' : '+'); + Result.Append((int64_t(Ticks) < 0) ? '-' : '+'); while (*Format != '\0') { @@ -1838,11 +1838,11 @@ TEST_CASE("uson.json") << "ValueTwo"; CbObject Obj = Writer.Save(); - StringBuilder<128> Sb; - const std::string_view JsonText = Obj.ToJson(Sb).ToView(); + StringBuilder<128> Sb; + const char* JsonText = Obj.ToJson(Sb).Data(); std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText.data(), JsonError); + json11::Json Json = json11::Json::parse(JsonText, JsonError); const std::string ValueOne = Json["KeyOne"].string_value(); const std::string ValueTwo = Json["KeyTwo"].string_value(); @@ -1863,11 +1863,11 @@ TEST_CASE("uson.json") CbObject Obj = Writer.Save(); - StringBuilder<128> Sb; - const std::string_view JsonText = Obj.ToJson(Sb).ToView(); + StringBuilder<128> Sb; + const char* JsonText = Obj.ToJson(Sb).Data(); std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText.data(), JsonError); + json11::Json Json = json11::Json::parse(JsonText, JsonError); const float FloatValue = float(Json["Float"].number_value()); const double DoubleValue = Json["Double"].number_value(); @@ -1888,11 +1888,11 @@ TEST_CASE("uson.json") CbObject Obj = Writer.Save(); - StringBuilder<128> Sb; - const std::string_view JsonText = Obj.ToJson(Sb).ToView(); + StringBuilder<128> Sb; + const char* JsonText = Obj.ToJson(Sb).Data(); std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText.data(), JsonError); + json11::Json Json = json11::Json::parse(JsonText, JsonError); const double FloatValue = Json["FloatNan"].number_value(); const double DoubleValue = Json["DoubleNan"].number_value(); diff --git a/zencore/compactbinarybuilder.cpp b/zencore/compactbinarybuilder.cpp index fa5b6a69b..3be24c149 100644 --- a/zencore/compactbinarybuilder.cpp +++ b/zencore/compactbinarybuilder.cpp @@ -985,7 +985,12 @@ TEST_CASE("usonbuilder.string") SUBCASE("Non-ASCII String") { +#if ZEN_SIZEOF_WCHAR_T == 2 wchar_t Value[2] = {0xd83d, 0xde00}; +#else + wchar_t Value[1] = {0x1f600}; +#endif + Writer.AddString("\xf0\x9f\x98\x80"sv); Writer.AddString(std::wstring_view(Value, ZEN_ARRAY_COUNT(Value))); CbFieldIterator Fields = Writer.Save(); diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index b8e8eff55..c3edf656e 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -505,7 +505,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer ThrowSystemException(hRes, "File write failed for '{}'"_format(Path).c_str()); } #else - if (write(Fd, DataPtr, WriteSize) != WriteSize) + if (write(Fd, DataPtr, WriteSize) != int64_t(WriteSize)) { ThrowLastError("File write failed for '{}'"_format(Path)); } @@ -572,7 +572,9 @@ ReadFile(std::filesystem::path Path) int Fd = open(Path.c_str(), O_RDONLY); if (Fd < 0) { - return FileContents{.ErrorCode = std::error_code(zen::GetLastError(), std::system_category())}; + FileContents Ret; + Ret.ErrorCode = std::error_code(zen::GetLastError(), std::system_category()); + return Ret; } static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); @@ -731,7 +733,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr ThrowLastError("Failed to open directory for traversal: {}"_format(RootDir.c_str())); } - for (struct dirent* Entry; Entry = readdir(Dir);) + for (struct dirent* Entry; (Entry = readdir(Dir));) { const char* FileName = Entry->d_name; @@ -787,14 +789,15 @@ PathFromHandle(void* NativeHandle) return FullPath; #elif ZEN_PLATFORM_LINUX - char Buffer[256]; - sprintf(Buffer, "/proc/%d/fd/%d", getpid(), int(uintptr_t(NativeHandle))); - ssize_t BytesRead = readlink(Buffer, Buffer, sizeof(Buffer) - 1); + char Link[256]; + char Path[64]; + sprintf(Path, "/proc/%d/fd/%d", getpid(), int(uintptr_t(NativeHandle))); + ssize_t BytesRead = readlink(Path, Link, sizeof(Link) - 1); if (BytesRead <= 0) return std::filesystem::path(); - Buffer[BytesRead] = '\0'; - return Buffer; + Link[BytesRead] = '\0'; + return Link; #else # error Unimplemented platform #endif // ZEN_PLATFORM_WINDOWS @@ -809,14 +812,15 @@ GetRunningExecutablePath() return {std::wstring_view(ExePath, PathLength)}; #elif ZEN_PLATFORM_LINUX - char Buffer[256]; - sprintf(Buffer, "/proc/%d/exe", getpid()); - ssize_t BytesRead = readlink(Buffer, Buffer, sizeof(Buffer) - 1); + char Link[256]; + char Path[64]; + sprintf(Path, "/proc/%d/exe", getpid()); + ssize_t BytesRead = readlink(Path, Link, sizeof(Link) - 1); if (BytesRead < 0) return {}; - Buffer[BytesRead] = '\0'; - return Buffer; + Link[BytesRead] = '\0'; + return Link; #else # error Unimplemented platform #endif // ZEN_PLATFORM_WINDOWS diff --git a/zencore/include/zencore/intmath.h b/zencore/include/zencore/intmath.h index 7619e1950..0d0ceff16 100644 --- a/zencore/include/zencore/intmath.h +++ b/zencore/include/zencore/intmath.h @@ -107,7 +107,7 @@ FloorLog2(uint32_t Value) static inline uint32_t CountLeadingZeros(uint32_t Value) { - unsigned long Log2; + unsigned long Log2 = 0; _BitScanReverse64(&Log2, (uint64_t(Value) << 1) | 1); return 32 - Log2; } @@ -115,7 +115,7 @@ CountLeadingZeros(uint32_t Value) static inline uint64_t FloorLog2_64(uint64_t Value) { - unsigned long Log2; + unsigned long Log2 = 0; long Mask = -long(_BitScanReverse64(&Log2, Value) != 0); return Log2 & Mask; } @@ -123,7 +123,7 @@ FloorLog2_64(uint64_t Value) static inline uint64_t CountLeadingZeros64(uint64_t Value) { - unsigned long Log2; + unsigned long Log2 = 0; long Mask = -long(_BitScanReverse64(&Log2, Value) != 0); return ((63 - Log2) & Mask) | (64 & ~Mask); } diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h index db462e238..60fee1dc5 100644 --- a/zencore/include/zencore/iobuffer.h +++ b/zencore/include/zencore/iobuffer.h @@ -368,10 +368,8 @@ private: class IoBufferBuilder { - using path_char_t = std::filesystem::path::value_type; - public: - ZENCORE_API static IoBuffer MakeFromFile(const path_char_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); + ZENCORE_API static IoBuffer MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); ZENCORE_API static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName); ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); ZENCORE_API static IoBuffer ReadFromFileMaybe(IoBuffer& InBuffer); diff --git a/zencore/include/zencore/refcount.h b/zencore/include/zencore/refcount.h index 1873ce48e..325863051 100644 --- a/zencore/include/zencore/refcount.h +++ b/zencore/include/zencore/refcount.h @@ -160,7 +160,7 @@ public: private: T* m_Ref = nullptr; - template<class T> + template<class U> friend class Ref; }; diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h index a94e063a4..e2a957786 100644 --- a/zencore/include/zencore/string.h +++ b/zencore/include/zencore/string.h @@ -424,7 +424,7 @@ public: inline std::wstring_view ToView() const { return std::wstring_view{Data(), Size()}; } inline std::wstring toString() const { return std::wstring{Data(), Size()}; } - inline StringBuilderImpl& operator<<(const std::u16string_view str) { return Append((const wchar_t*)str.data(), str.size()); } + inline StringBuilderImpl& operator<<(const std::wstring_view str) { return Append((const wchar_t*)str.data(), str.size()); } inline StringBuilderImpl& operator<<(const wchar_t* str) { return Append(str); } using StringBuilderImpl:: operator<<; }; @@ -460,7 +460,6 @@ std::wstring Utf8ToWide(const std::string_view& wstr); void WideToUtf8(const wchar_t* wstr, StringBuilderBase& out); std::string WideToUtf8(const wchar_t* wstr); -void WideToUtf8(const std::u16string_view& wstr, StringBuilderBase& out); void WideToUtf8(const std::wstring_view& wstr, StringBuilderBase& out); std::string WideToUtf8(const std::wstring_view Wstr); diff --git a/zencore/include/zencore/thread.h b/zencore/include/zencore/thread.h index 9fc4c87a2..48a58a261 100644 --- a/zencore/include/zencore/thread.h +++ b/zencore/include/zencore/thread.h @@ -143,7 +143,7 @@ public: ZENCORE_API bool Wait(int TimeoutMs = -1); ZENCORE_API void Terminate(int ExitCode); ZENCORE_API void Reset(); - inline [[nodiscard]] int Pid() const { return m_Pid; } + [[nodiscard]] inline int Pid() const { return m_Pid; } private: void* m_ProcessHandle = nullptr; diff --git a/zencore/include/zencore/uid.h b/zencore/include/zencore/uid.h index f4e9ab65a..d25aa8059 100644 --- a/zencore/include/zencore/uid.h +++ b/zencore/include/zencore/uid.h @@ -74,7 +74,7 @@ struct Oid size_t operator()(const Oid& id) const { const size_t seed = id.OidBits[0]; - return (seed << 6) + (seed >> 2) + 0x9e3779b9 + uint64_t(id.OidBits[1]) | (uint64_t(id.OidBits[2]) << 32); + return ((seed << 6) + (seed >> 2) + 0x9e3779b9 + uint64_t(id.OidBits[1])) | (uint64_t(id.OidBits[2]) << 32); } }; diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index d654770d0..de848146a 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -110,6 +110,13 @@ #define ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS 1 +#if defined(__SIZEOF_WCHAR_T__) && __SIZEOF_WCHAR_T__ == 4 +# define ZEN_SIZEOF_WCHAR_T 4 +#else + static_assert(sizeof(wchar_t) == 2, "wchar_t is expected to be two bytes in size"); +# define ZEN_SIZEOF_WCHAR_T 2 +#endif + ////////////////////////////////////////////////////////////////////////// // Assert // @@ -229,9 +236,15 @@ ZENCORE_API void zencore_forcelinktests(); ////////////////////////////////////////////////////////////////////////// #if ZEN_COMPILER_MSC -# define ZEN_DISABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", off)) +# define ZEN_DISABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", off)) # define ZEN_ENABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", on)) -#else +#elif ZEN_COMPILER_GCC +# define ZEN_DISABLE_OPTIMIZATION_ACTUAL _Pragma( "GCC push_options" ) \ + _Pragma( "GCC optimize (\"O0\")" ) +# define ZEN_ENABLE_OPTIMIZATION_ACTUAL _Pragma( "GCC pop_options" ) +#elif ZEN_COMPILER_CLANG +# define ZEN_DISABLE_OPTIMIZATION_ACTUAL _Pragma("clang optimize off") +# define ZEN_ENABLE_OPTIMIZATION_ACTUAL _Pragma("clang optimize on") #endif // Set up optimization control macros, now that we have both the build settings and the platform macros diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index 6cee3f60d..63d5ac98d 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -417,14 +417,14 @@ IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t } IoBuffer -IoBufferBuilder::MakeFromFile(const path_char_t* FileName, uint64_t Offset, uint64_t Size) +IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size) { uint64_t FileSize; #if ZEN_PLATFORM_WINDOWS CAtlFile DataFile; - HRESULT hRes = DataFile.Create(FileName, GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); + HRESULT hRes = DataFile.Create(FileName.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); if (FAILED(hRes)) { @@ -433,7 +433,7 @@ IoBufferBuilder::MakeFromFile(const path_char_t* FileName, uint64_t Offset, uint DataFile.GetSize((ULONGLONG&)FileSize); #else - int Fd = open(FileName, O_RDONLY); + int Fd = open(FileName.c_str(), O_RDONLY); if (Fd < 0) { return {}; diff --git a/zencore/stats.cpp b/zencore/stats.cpp index 0c0647999..1bb6f6de0 100644 --- a/zencore/stats.cpp +++ b/zencore/stats.cpp @@ -227,7 +227,7 @@ UniformSample::Snapshot() const uint64_t ValuesSize = Size(); std::vector<double> Values(ValuesSize); - for (int i = 0; i < ValuesSize; ++i) + for (int i = 0, n = int(ValuesSize); i < n; ++i) { Values[i] = double(m_Values[i]); } diff --git a/zencore/string.cpp b/zencore/string.cpp index 8e7921bb6..76cd69871 100644 --- a/zencore/string.cpp +++ b/zencore/string.cpp @@ -30,6 +30,17 @@ utf16to8_impl(u16bit_iterator StartIt, u16bit_iterator EndIt, ::zen::StringBuild } } +template<typename u32bit_iterator> +void +utf32to8_impl(u32bit_iterator StartIt, u32bit_iterator EndIt, ::zen::StringBuilderBase& OutString) +{ + for (; StartIt != EndIt; ++StartIt) + { + wchar_t cp = *StartIt; + OutString.AppendCodepoint(cp); + } +} + ////////////////////////////////////////////////////////////////////////// namespace zen { @@ -76,7 +87,7 @@ FilepathFindExtension(const std::string_view& Path, const char* ExtensionToMatch // Look for extension introducer ('.') - for (size_t i = PathLen - 1; i >= 0; --i) + for (int64_t i = PathLen - 1; i >= 0; --i) { if (Path[i] == '.') return Path.data() + i; @@ -164,26 +175,24 @@ Utf8ToWide(const std::u8string_view& Str8, WideStringBuilderBase& OutString) void WideToUtf8(const wchar_t* Wstr, StringBuilderBase& OutString) { - WideToUtf8(std::u16string_view{(char16_t*)Wstr}, OutString); + WideToUtf8(std::wstring_view{Wstr}, OutString); } void WideToUtf8(const std::wstring_view& Wstr, StringBuilderBase& OutString) { - WideToUtf8(std::u16string_view{(char16_t*)Wstr.data(), Wstr.size()}, OutString); -} - -void -WideToUtf8(const std::u16string_view& Wstr, StringBuilderBase& OutString) -{ +#if ZEN_SIZEOF_WCHAR_T == 2 utf16to8_impl(begin(Wstr), end(Wstr), OutString); +#else + utf32to8_impl(begin(Wstr), end(Wstr), OutString); +#endif } std::string WideToUtf8(const wchar_t* Wstr) { ExtendableStringBuilder<128> String; - WideToUtf8(std::u16string_view{(char16_t*)Wstr}, String); + WideToUtf8(std::wstring_view{Wstr}, String); return String.c_str(); } @@ -192,7 +201,7 @@ std::string WideToUtf8(const std::wstring_view Wstr) { ExtendableStringBuilder<128> String; - WideToUtf8(std::u16string_view{(char16_t*)Wstr.data(), Wstr.size()}, String); + WideToUtf8(std::wstring_view{Wstr.data(), Wstr.size()}, String); return String.c_str(); } diff --git a/zencore/thread.cpp b/zencore/thread.cpp index da711fe89..2a38283e1 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -4,6 +4,7 @@ #include <zencore/except.h> #include <zencore/string.h> +#include <zencore/testing.h> #if ZEN_PLATFORM_WINDOWS # include <zencore/windows.h> @@ -418,7 +419,9 @@ IsProcessRunning(int pid) return true; #else - ZEN_NOT_IMPLEMENTED(); + char Buffer[64]; + sprintf(Buffer, "/proc/%d", pid); + return access(Buffer, F_OK) == 0; #endif } @@ -447,9 +450,20 @@ Sleep(int ms) // Testing related code follows... // +#if ZEN_WITH_TESTS + void thread_forcelink() { } +TEST_CASE("Thread") +{ + int Pid = GetCurrentProcessId(); + CHECK(Pid > 0); + CHECK(IsProcessRunning(Pid)); +} + +#endif // ZEN_WITH_TESTS + } // namespace zen diff --git a/zencore/xmake.lua b/zencore/xmake.lua index 5de7f476d..1abd86b7b 100644 --- a/zencore/xmake.lua +++ b/zencore/xmake.lua @@ -2,13 +2,24 @@ target('zencore') set_kind("static") add_files("**.cpp") add_includedirs("include", {public=true}) - add_includedirs("..\\3rdparty\\utfcpp\\source") - add_linkdirs("$(projectdir)/3rdparty/BLAKE3/lib/Win64", "$(projectdir)/3rdparty/Oodle/lib/Win64") + add_includedirs("$(projectdir)/3rdparty/utfcpp/source") + if is_os("windows") then + add_linkdirs("$(projectdir)/3rdparty/BLAKE3/lib/Win64") + add_linkdirs("$(projectdir)/3rdparty/Oodle/lib/Win64") + elseif is_os("linux") then + add_linkdirs("$(projectdir)/3rdparty/BLAKE3/lib/Linux_x64") + add_linkdirs("$(projectdir)/3rdparty/Oodle/lib/Linux_x64") + add_links("blake3") + add_links("oo2corelinux64") + add_links("pthread") + end add_packages( "vcpkg::spdlog", "vcpkg::fmt", "vcpkg::doctest", "vcpkg::lz4", + "vcpkg::mimalloc", + "vcpkg::json11", "vcpkg::cpr", "vcpkg::curl", -- required by cpr "vcpkg::zlib", -- required by curl diff --git a/zenhttp/httpnull.cpp b/zenhttp/httpnull.cpp index e49051ac5..4208421e1 100644 --- a/zenhttp/httpnull.cpp +++ b/zenhttp/httpnull.cpp @@ -2,9 +2,12 @@ #include "httpnull.h" -#include <conio.h> #include <zencore/logging.h> +#if ZEN_PLATFORM_WINDOWS +# include <conio.h> +#endif + namespace zen { HttpNullServer::HttpNullServer() @@ -32,6 +35,7 @@ HttpNullServer::Run(bool IsInteractiveSession) { const bool TestMode = !IsInteractiveSession; +#if ZEN_PLATFORM_WINDOWS if (TestMode == false) { zen::logging::ConsoleLog().info("Zen Server running (null HTTP). Press ESC or Q to quit"); @@ -58,6 +62,18 @@ HttpNullServer::Run(bool IsInteractiveSession) m_ShutdownEvent.Wait(WaitTimeout); } while (!IsApplicationExitRequested()); +#else + if (TestMode == false) + { + zen::logging::ConsoleLog().info("Zen Server running (null HTTP). Ctrl-C to quit"); + } + + do + { + zen::Sleep(1493); + } + while (!IsApplicationExitRequested()); +#endif } void diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index f3568bbd1..43dfe30d8 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -1213,7 +1213,7 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService& const HTTP_REQUEST* HttpRequestPtr = Tx.HttpRequest(); const int PrefixLength = Service.UriPrefixLength(); - const int AbsPathLength = HttpRequestPtr->CookedUrl.AbsPathLength / sizeof(char16_t); + const int AbsPathLength = HttpRequestPtr->CookedUrl.AbsPathLength / sizeof(wchar_t); HttpContentType AcceptContentType = HttpContentType::kUnknownContentType; @@ -1223,7 +1223,7 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService& // with utf8. This is overhead which I'd prefer to avoid but for now we just have // to live with it - WideToUtf8({(char16_t*)HttpRequestPtr->CookedUrl.pAbsPath + PrefixLength, gsl::narrow<size_t>(AbsPathLength - PrefixLength)}, + WideToUtf8({(wchar_t*)HttpRequestPtr->CookedUrl.pAbsPath + PrefixLength, gsl::narrow<size_t>(AbsPathLength - PrefixLength)}, m_UriUtf8); std::string_view UriSuffix8{m_UriUtf8}; @@ -1260,7 +1260,7 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService& { --QueryStringLength; // We skip the leading question mark - WideToUtf8({(char16_t*)(HttpRequestPtr->CookedUrl.pQueryString) + 1, QueryStringLength / sizeof(char16_t)}, m_QueryStringUtf8); + WideToUtf8({(wchar_t*)(HttpRequestPtr->CookedUrl.pQueryString) + 1, QueryStringLength / sizeof(wchar_t)}, m_QueryStringUtf8); } else { diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index 9c9b73042..0f9026e76 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -1703,11 +1703,12 @@ TEST_CASE("zcache.policy") zen::IoHash Key; zen::IoHash PayloadId; - zen::CbPackage Package = GeneratePackage(Key, PayloadId); - auto Buf = ToBuffer(Package); // Store package locally { + zen::CbPackage Package = GeneratePackage(Key, PayloadId); + auto Buf = ToBuffer(Package); + CHECK(Package.GetAttachments().size() != 0); cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}"_format(LocalCfg.BaseUri, Bucket, Key)}, cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()}, @@ -1760,11 +1761,12 @@ TEST_CASE("zcache.policy") zen::IoHash Key; zen::IoHash PayloadId; - zen::CbPackage Package = GeneratePackage(Key, PayloadId); - auto Buf = ToBuffer(Package); // Store package upstream { + zen::CbPackage Package = GeneratePackage(Key, PayloadId); + auto Buf = ToBuffer(Package); + CHECK(Package.GetAttachments().size() != 0); cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}"_format(UpstreamCfg.BaseUri, Bucket, Key)}, cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()}, @@ -1926,7 +1928,7 @@ struct RemoteExecutionRequest if (auto It = m_Visit.m_HashToFile.find(NeedHash); It != m_Visit.m_HashToFile.end()) { - zen::IoBuffer FileData = zen::IoBufferBuilder::MakeFromFile(It->second.c_str()); + zen::IoBuffer FileData = zen::IoBufferBuilder::MakeFromFile(It->second); cpr::Response CasResponse = cpr::Post(cpr::Url(m_CasUri), cpr::Body((const char*)FileData.Data(), FileData.Size())); diff --git a/zenserver/config.cpp b/zenserver/config.cpp index cbdbebb03..799891289 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -279,7 +279,7 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv using namespace fmt::literals; std::filesystem::path ConfigScript = DataRoot / "zen_cfg.lua"; - zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(ConfigScript.native().c_str()); + zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(ConfigScript); if (LuaScript) { diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index 39348b436..0e1ba01cb 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -328,7 +328,7 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId) std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second.ServerPath; - return IoBufferBuilder::MakeFromFile(FilePath.native().c_str()); + return IoBufferBuilder::MakeFromFile(FilePath); } if (auto MetaIt = m_MetaMap.find(ChunkId); MetaIt != m_MetaMap.end()) diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp index a4bbfa340..bc4825419 100644 --- a/zenstore/CAS.cpp +++ b/zenstore/CAS.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include <zenstore/cas.h> +#include <zenstore/CAS.h> #include "compactcas.h" #include "filecas.h" @@ -180,7 +180,7 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) id.ToString(manifest); Marker.Open(ManifestPath.c_str(), /* IsCreate */ true); - Marker.Write(manifest.c_str(), (DWORD)manifest.Size(), 0); + Marker.Write(manifest.c_str(), uint32_t(manifest.Size()), 0); } } diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp index bbb9e1036..dd3598319 100644 --- a/zenstore/basicfile.cpp +++ b/zenstore/basicfile.cpp @@ -8,6 +8,13 @@ #include <zencore/testing.h> #include <zencore/testutils.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#else +# include <sys/stat.h> +# include <fcntl.h> +#endif + #include <fmt/format.h> #include <gsl/gsl-lite.hpp> @@ -37,6 +44,7 @@ BasicFile::Open(std::filesystem::path FileName, bool IsCreate, std::error_code& { Ec.clear(); +#if ZEN_PLATFORM_WINDOWS const DWORD dwCreationDisposition = IsCreate ? CREATE_ALWAYS : OPEN_EXISTING; DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; const DWORD dwShareMode = FILE_SHARE_READ; @@ -62,6 +70,19 @@ BasicFile::Open(std::filesystem::path FileName, bool IsCreate, std::error_code& return; } +#else + int OpenFlags = O_RDWR; + OpenFlags |= IsCreate ? O_CREAT|O_TRUNC : 0; + + int Fd = open(FileName.c_str(), O_RDWR, 0666); + if (Fd < 0) + { + Ec = zen::MakeErrorCodeFromLastError(); + return; + } + + void* FileHandle = (void*)(uintptr_t(Fd)); +#endif m_FileHandle = FileHandle; } @@ -71,7 +92,12 @@ BasicFile::Close() { if (m_FileHandle) { +#if ZEN_PLATFORM_WINDOWS ::CloseHandle(m_FileHandle); +#else + int Fd = int(uintptr_t(m_FileHandle)); + close(Fd); +#endif m_FileHandle = nullptr; } } @@ -85,6 +111,7 @@ BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) { const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize); +#if ZEN_PLATFORM_WINDOWS OVERLAPPED Ovl{}; Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); @@ -94,6 +121,12 @@ BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) BOOL Success = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl); ZEN_ASSERT(dwNumberOfBytesRead == NumberOfBytesToRead); +#else + static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); + int Fd = int(uintptr_t(m_FileHandle)); + int BytesRead = pread(Fd, Data, NumberOfBytesToRead, FileOffset); + bool Success = (BytesRead > 0); +#endif if (!Success) { @@ -160,6 +193,7 @@ BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::erro { const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize); +#if ZEN_PLATFORM_WINDOWS OVERLAPPED Ovl{}; Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); @@ -168,6 +202,12 @@ BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::erro DWORD dwNumberOfBytesWritten = 0; BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl); +#else + static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); + int Fd = int(uintptr_t(m_FileHandle)); + int BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, FileOffset); + bool Success = (BytesWritten > 0); +#endif if (!Success) { @@ -209,16 +249,29 @@ BasicFile::WriteAll(IoBuffer Data, std::error_code& Ec) void BasicFile::Flush() { +#if ZEN_PLATFORM_WINDOWS FlushFileBuffers(m_FileHandle); +#else + int Fd = int(uintptr_t(m_FileHandle)); + fsync(Fd); +#endif } uint64_t BasicFile::FileSize() { +#if ZEN_PLATFORM_WINDOWS ULARGE_INTEGER liFileSize; liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart); return uint64_t(liFileSize.QuadPart); +#else + int Fd = int(uintptr_t(m_FileHandle)); + static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); + struct stat Stat; + fstat(Fd, &Stat); + return uint64_t(Stat.st_size); +#endif } ////////////////////////////////////////////////////////////////////////// @@ -233,11 +286,16 @@ TemporaryFile::Close() { if (m_FileHandle) { +#if ZEN_PLATFORM_WINDOWS // Mark file for deletion when final handle is closed FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE}; SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi); +#else + std::filesystem::path FilePath = zen::PathFromHandle(m_FileHandle); + unlink(FilePath.c_str()); +#endif BasicFile::Close(); } diff --git a/zenstore/caslog.cpp b/zenstore/caslog.cpp index 2bac6affd..449e2fc58 100644 --- a/zenstore/caslog.cpp +++ b/zenstore/caslog.cpp @@ -1,8 +1,8 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include <zenstore/cas.h> +#include <zenstore/CAS.h> -#include "CompactCas.h" +#include "compactcas.h" #include <zencore/except.h> #include <zencore/filesystem.h> @@ -121,7 +121,7 @@ CasLogFile::Replay(std::function<void(const void*)>&& Handler) m_File.Read(ReadBuffer.data(), LogDataSize, LogBaseOffset); - for (int i = 0; i < LogEntryCount; ++i) + for (int i = 0; i < int(LogEntryCount); ++i) { Handler(ReadBuffer.data() + (i * m_RecordSize)); } @@ -135,7 +135,7 @@ CasLogFile::Append(const void* DataPointer, uint64_t DataSize) uint64_t AppendOffset = m_AppendOffset.fetch_add(DataSize); std::error_code Ec; - m_File.Write(DataPointer, gsl::narrow<DWORD>(DataSize), AppendOffset, Ec); + m_File.Write(DataPointer, gsl::narrow<uint32_t>(DataSize), AppendOffset, Ec); if (Ec) { diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp index 612f87c7c..3631722d4 100644 --- a/zenstore/compactcas.cpp +++ b/zenstore/compactcas.cpp @@ -1,8 +1,8 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include <zenstore/cas.h> +#include <zenstore/CAS.h> -#include "CompactCas.h" +#include "compactcas.h" #include <zencore/except.h> #include <zencore/logging.h> diff --git a/zenstore/compactcas.h b/zenstore/compactcas.h index a512c3d93..47e989fe9 100644 --- a/zenstore/compactcas.h +++ b/zenstore/compactcas.h @@ -9,11 +9,14 @@ #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/uid.h> -#include <zencore/windows.h> #include <zenstore/basicfile.h> -#include <zenstore/cas.h> +#include <zenstore/CAS.h> #include <zenstore/caslog.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + namespace zen { ////////////////////////////////////////////////////////////////////////// diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp index a37450cd8..3941499f1 100644 --- a/zenstore/filecas.cpp +++ b/zenstore/filecas.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "FileCas.h" +#include "filecas.h" #include <zencore/except.h> #include <zencore/filesystem.h> @@ -23,7 +23,9 @@ #include <unordered_map> ZEN_THIRD_PARTY_INCLUDES_START -#include <atlfile.h> +#if ZEN_PLATFORM_WINDOWS +# include <atlfile.h> +#endif ZEN_THIRD_PARTY_INCLUDES_END namespace zen { diff --git a/zenstore/filecas.h b/zenstore/filecas.h index 14314ce52..8f36c0170 100644 --- a/zenstore/filecas.h +++ b/zenstore/filecas.h @@ -8,7 +8,7 @@ #include <zencore/iohash.h> #include <zencore/string.h> #include <zencore/thread.h> -#include <zenstore/cas.h> +#include <zenstore/CAS.h> #include <functional> diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/CAS.h index 86e7e78d9..7f8b5bfa2 100644 --- a/zenstore/include/zenstore/CAS.h +++ b/zenstore/include/zenstore/CAS.h @@ -41,9 +41,9 @@ public: void AddChunksToSet(std::span<const IoHash> HashesToAdd); void RemoveChunksIf(std::function<bool(const IoHash& CandidateHash)>&& Predicate); void IterateChunks(std::function<void(const IoHash& ChunkHash)>&& Callback); - inline [[nodiscard]] bool ContainsChunk(const IoHash& Hash) const { return m_ChunkSet.find(Hash) != m_ChunkSet.end(); } - inline [[nodiscard]] bool IsEmpty() const { return m_ChunkSet.empty(); } - inline [[nodiscard]] size_t GetSize() const { return m_ChunkSet.size(); } + [[nodiscard]] inline bool ContainsChunk(const IoHash& Hash) const { return m_ChunkSet.find(Hash) != m_ChunkSet.end(); } + [[nodiscard]] inline bool IsEmpty() const { return m_ChunkSet.empty(); } + [[nodiscard]] inline size_t GetSize() const { return m_ChunkSet.size(); } private: // Q: should we protect this with a lock, or is that a higher level concern? diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h index 7ae35dea6..d30479b91 100644 --- a/zenstore/include/zenstore/basicfile.h +++ b/zenstore/include/zenstore/basicfile.h @@ -5,7 +5,6 @@ #include "zenstore.h" #include <zencore/iobuffer.h> -#include <zencore/windows.h> #include <filesystem> #include <functional> diff --git a/zenstore/include/zenstore/caslog.h b/zenstore/include/zenstore/caslog.h index 00b987383..8189da87e 100644 --- a/zenstore/include/zenstore/caslog.h +++ b/zenstore/include/zenstore/caslog.h @@ -8,9 +8,12 @@ #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/uid.h> -#include <zencore/windows.h> #include <zenstore/basicfile.h> -#include <zenstore/cas.h> +#include <zenstore/CAS.h> + +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif #include <functional> diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h index 5f567e7fc..26bf2bbde 100644 --- a/zenstore/include/zenstore/cidstore.h +++ b/zenstore/include/zenstore/cidstore.h @@ -8,9 +8,7 @@ #include <zencore/iohash.h> #include <zenstore/CAS.h> -namespace std::filesystem { -class path; -} +#include <filesystem> namespace zen { diff --git a/zenutil/zenserverprocess.cpp b/zenutil/zenserverprocess.cpp index 55b592ab1..ab3454a91 100644 --- a/zenutil/zenserverprocess.cpp +++ b/zenutil/zenserverprocess.cpp @@ -9,11 +9,13 @@ #include <zencore/session.h> #include <zencore/string.h> -#include <atlbase.h> -#include <shellapi.h> #include <source_location> -#include <zencore/windows.h> +#if ZEN_PLATFORM_WINDOWS +# include <atlbase.h> +# include <shellapi.h> +# include <zencore/windows.h> +#endif ////////////////////////////////////////////////////////////////////////// |