From 68897f04bd60624fb57d1bd5add4ca7aed1d5e50 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 17 Sep 2021 15:39:49 +0200 Subject: Added IsDebuggerPresent() query function to query whether a debugger is currently attached to the running process --- zencore/include/zencore/zencore.h | 2 +- zencore/zencore.cpp | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index da17e61e3..14f915e76 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -155,7 +155,7 @@ namespace zen { ZENCORE_API bool IsPointerToStack(const void* ptr); // Query if pointer is within the stack of the currently executing thread ZENCORE_API bool IsApplicationExitRequested(); ZENCORE_API void RequestApplicationExit(int ExitCode); - +ZENCORE_API bool IsDebuggerPresent(); ZENCORE_API void zencore_forcelinktests(); } diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp index 5899f014d..bb457ed0a 100644 --- a/zencore/zencore.cpp +++ b/zencore/zencore.cpp @@ -30,6 +30,8 @@ namespace zen { +////////////////////////////////////////////////////////////////////////// + bool IsPointerToStack(const void* ptr) { @@ -55,6 +57,18 @@ IsPointerToStack(const void* ptr) #endif } +bool +IsDebuggerPresent() +{ +#if ZEN_PLATFORM_WINDOWS + return ::IsDebuggerPresent(); +#else + return false; +#endif +} + +////////////////////////////////////////////////////////////////////////// + AssertException::AssertException(const char* Msg) : m_Msg(Msg) { } -- cgit v1.2.3 From 0080e35ddb363dcac604cd77c1a07c98e4e75f4b Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 17 Sep 2021 15:42:30 +0200 Subject: clang-format --- zen/cmds/copy.cpp | 2 +- zen/cmds/copy.h | 2 +- zen/cmds/dedup.cpp | 2 +- zen/cmds/dedup.h | 2 +- zen/cmds/deploy.cpp | 2 +- zen/cmds/deploy.h | 2 +- zen/cmds/hash.cpp | 2 +- zen/cmds/hash.h | 2 +- zen/cmds/run.cpp | 2 +- zen/cmds/run.h | 2 +- zen/cmds/scrub.cpp | 4 ++-- zen/cmds/scrub.h | 2 +- zen/cmds/status.cpp | 2 +- zen/cmds/status.h | 2 +- zen/cmds/top.cpp | 11 ++++++----- zen/cmds/top.h | 2 +- zen/cmds/up.cpp | 2 +- zen/cmds/up.h | 2 +- zen/zen.cpp | 5 ++--- zen/zen.vcxproj | 3 +++ zen/zen.vcxproj.filters | 3 +++ zencore/include/zencore/zencore.h | 2 +- zenserver-test/zenserver-test.cpp | 4 ++-- zenserver/diag/logging.cpp | 12 +++++++++++- zenutil/include/zenutil/zenserverprocess.h | 2 +- 25 files changed, 47 insertions(+), 31 deletions(-) diff --git a/zen/cmds/copy.cpp b/zen/cmds/copy.cpp index 6b3965f99..947d54e07 100644 --- a/zen/cmds/copy.cpp +++ b/zen/cmds/copy.cpp @@ -97,4 +97,4 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/copy.h b/zen/cmds/copy.h index c2a7fd23b..322cf3f2f 100644 --- a/zen/cmds/copy.h +++ b/zen/cmds/copy.h @@ -25,4 +25,4 @@ private: bool m_NoClone = false; }; -} +} // namespace zen diff --git a/zen/cmds/dedup.cpp b/zen/cmds/dedup.cpp index f95a87518..e71314622 100644 --- a/zen/cmds/dedup.cpp +++ b/zen/cmds/dedup.cpp @@ -293,4 +293,4 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/dedup.h b/zen/cmds/dedup.h index c955d4bbd..7932d10e6 100644 --- a/zen/cmds/dedup.h +++ b/zen/cmds/dedup.h @@ -27,4 +27,4 @@ private: size_t m_SizeThreshold = 1024 * 1024; }; -} +} // namespace zen diff --git a/zen/cmds/deploy.cpp b/zen/cmds/deploy.cpp index faaf0030c..d60392dd5 100644 --- a/zen/cmds/deploy.cpp +++ b/zen/cmds/deploy.cpp @@ -83,4 +83,4 @@ DeployCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/deploy.h b/zen/cmds/deploy.h index 565e11bf3..975caf9e9 100644 --- a/zen/cmds/deploy.h +++ b/zen/cmds/deploy.h @@ -26,4 +26,4 @@ private: bool m_IsClean = false; }; -} +} // namespace zen diff --git a/zen/cmds/hash.cpp b/zen/cmds/hash.cpp index e9484c453..0a7989ffc 100644 --- a/zen/cmds/hash.cpp +++ b/zen/cmds/hash.cpp @@ -126,4 +126,4 @@ HashCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/hash.h b/zen/cmds/hash.h index 8ed8e9fa3..3df9063ea 100644 --- a/zen/cmds/hash.h +++ b/zen/cmds/hash.h @@ -26,4 +26,4 @@ private: std::string m_OutputFile; }; -} +} // namespace zen diff --git a/zen/cmds/run.cpp b/zen/cmds/run.cpp index 711a3a341..ceeb4ddae 100644 --- a/zen/cmds/run.cpp +++ b/zen/cmds/run.cpp @@ -184,4 +184,4 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/run.h b/zen/cmds/run.h index f4eb4ef76..3e1e3f2b2 100644 --- a/zen/cmds/run.h +++ b/zen/cmds/run.h @@ -23,4 +23,4 @@ private: std::string m_ExeTree; }; -} +} // namespace zen diff --git a/zen/cmds/scrub.cpp b/zen/cmds/scrub.cpp index 73d2b45ee..c0fe8ca61 100644 --- a/zen/cmds/scrub.cpp +++ b/zen/cmds/scrub.cpp @@ -31,7 +31,7 @@ GcCommand::~GcCommand() { } -int +int GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_UNUSED(GlobalOptions, argc, argv); @@ -39,4 +39,4 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/scrub.h b/zen/cmds/scrub.h index a3f25c259..561ae578d 100644 --- a/zen/cmds/scrub.h +++ b/zen/cmds/scrub.h @@ -36,4 +36,4 @@ private: cxxopts::Options m_Options{"gc", "Garbage collect zen storage"}; }; -} +} // namespace zen diff --git a/zen/cmds/status.cpp b/zen/cmds/status.cpp index 1050b5b96..10970e3c2 100644 --- a/zen/cmds/status.cpp +++ b/zen/cmds/status.cpp @@ -20,4 +20,4 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/status.h b/zen/cmds/status.h index f2c68b96a..acde280c5 100644 --- a/zen/cmds/status.h +++ b/zen/cmds/status.h @@ -19,4 +19,4 @@ private: cxxopts::Options m_Options{"status", "Show zen status"}; }; -} +} // namespace zen diff --git a/zen/cmds/top.cpp b/zen/cmds/top.cpp index 3ff7edcda..3e3942d31 100644 --- a/zen/cmds/top.cpp +++ b/zen/cmds/top.cpp @@ -2,9 +2,9 @@ #include "top.h" +#include #include #include -#include #include #include @@ -32,7 +32,7 @@ TopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } - int n = 0; + int n = 0; const int HeaderPeriod = 20; for (;;) @@ -42,14 +42,15 @@ TopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("{:>5} {:>6} {:>24}", "port", "pid", "session"); } - State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { ZEN_CONSOLE("{:5} {:6} {:24}", Entry.ListenPort, Entry.Pid, Entry.GetSessionId()); }); + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + ZEN_CONSOLE("{:5} {:6} {:24}", Entry.ListenPort, Entry.Pid, Entry.GetSessionId()); + }); zen::Sleep(1000); State.Sweep(); } - return 0; } @@ -79,4 +80,4 @@ PsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/top.h b/zen/cmds/top.h index a842089ff..d8bf91a1c 100644 --- a/zen/cmds/top.h +++ b/zen/cmds/top.h @@ -32,4 +32,4 @@ private: cxxopts::Options m_Options{"ps", "Enumerate running Zen server instances"}; }; -} +} // namespace zen diff --git a/zen/cmds/up.cpp b/zen/cmds/up.cpp index 048133fc9..17cba3794 100644 --- a/zen/cmds/up.cpp +++ b/zen/cmds/up.cpp @@ -101,4 +101,4 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/up.h b/zen/cmds/up.h index fda62693a..fe1ed7a0c 100644 --- a/zen/cmds/up.h +++ b/zen/cmds/up.h @@ -32,4 +32,4 @@ private: cxxopts::Options m_Options{"down", "Bring down zen service"}; }; -} +} // namespace zen diff --git a/zen/zen.cpp b/zen/zen.cpp index 6390c832f..9c373face 100644 --- a/zen/zen.cpp +++ b/zen/zen.cpp @@ -42,7 +42,6 @@ ////////////////////////////////////////////////////////////////////////// - class TemplateCommand : public ZenCmdBase { public: @@ -125,7 +124,7 @@ main(int argc, char** argv) ZenCmdBase* Cmd; const char* CmdSummary; } Commands[] = { - // clang-format off + // clang-format off {"chunk", &ChunkCmd, "Perform chunking"}, {"copy", &CopyCmd, "Copy file(s)"}, {"deploy", &DeployCmd, "Deploy data"}, @@ -139,7 +138,7 @@ main(int argc, char** argv) {"top", &TopCmd, "Monitor zen server activity"}, {"up", &UpCmd, "Bring zen server up"}, {"down", &DownCmd, "Bring zen server down"}, - // clang-format on + // clang-format on }; // Build set containing available commands diff --git a/zen/zen.vcxproj b/zen/zen.vcxproj index ff10f4c59..fb0674e87 100644 --- a/zen/zen.vcxproj +++ b/zen/zen.vcxproj @@ -136,6 +136,9 @@ {77f8315d-b21d-4db0-9a6f-2d3359f88a70} + + + diff --git a/zen/zen.vcxproj.filters b/zen/zen.vcxproj.filters index a38771944..9002f01c2 100644 --- a/zen/zen.vcxproj.filters +++ b/zen/zen.vcxproj.filters @@ -63,4 +63,7 @@ {2e06a54c-52be-4260-9275-a4232d01a53c} + + + \ No newline at end of file diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index 14f915e76..8011ab241 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -158,7 +158,7 @@ ZENCORE_API void RequestApplicationExit(int ExitCode); ZENCORE_API bool IsDebuggerPresent(); ZENCORE_API void zencore_forcelinktests(); -} +} // namespace zen ////////////////////////////////////////////////////////////////////////// diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index 48bbacd80..17b8b76eb 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -670,7 +670,7 @@ main(int argc, char** argv) zen::logging::InitializeLogging(); spdlog::set_level(spdlog::level::debug); - spdlog::set_formatter(std::make_unique<::logging::full_formatter>("test", std::chrono::system_clock::now())); + spdlog::set_formatter(std::make_unique< ::logging::full_formatter>("test", std::chrono::system_clock::now())); std::filesystem::path ProgramBaseDir = std::filesystem::path(argv[0]).parent_path(); std::filesystem::path TestBaseDir = ProgramBaseDir.parent_path().parent_path() / ".test"; @@ -1773,5 +1773,5 @@ TEST_CASE("lifetime.owner.2") } # endif -} +} // namespace zen::tests #endif diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp index 4ba4835af..48c87dd68 100644 --- a/zenserver/diag/logging.cpp +++ b/zenserver/diag/logging.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -207,7 +208,7 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) if (GlobalOptions.IsTest) { LogLevel = spdlog::level::trace; - IsAsync = false; + IsAsync = false; } if (IsAsync) @@ -234,6 +235,15 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) Sinks.push_back(ConsoleSink); Sinks.push_back(FileSink); +#if ZEN_PLATFORM_WINDOWS + if (zen::IsDebuggerPresent()) + { + auto DebugSink = std::make_shared(); + DebugSink->set_level(spdlog::level::debug); + Sinks.push_back(DebugSink); + } +#endif + // Jupiter - only log HTTP traffic to file auto JupiterLogger = std::make_shared("jupiter", FileSink); diff --git a/zenutil/include/zenutil/zenserverprocess.h b/zenutil/include/zenutil/zenserverprocess.h index f0924f048..005f7adff 100644 --- a/zenutil/include/zenutil/zenserverprocess.h +++ b/zenutil/include/zenutil/zenserverprocess.h @@ -131,4 +131,4 @@ private: ZenServerEntry* m_OurEntry = nullptr; }; -} +} // namespace zen -- cgit v1.2.3 From 025999f216f0fe99d0eacd1f7a9c6276b834bcb8 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 17 Sep 2021 19:04:26 +0200 Subject: Removed WindowsException from public headers --- zencore/except.cpp | 35 ++++++++++++++++++++++++ zencore/include/zencore/except.h | 51 +++-------------------------------- zencore/thread.cpp | 3 ++- zenserver/experimental/usnjournal.cpp | 10 +++---- 4 files changed, 45 insertions(+), 54 deletions(-) diff --git a/zencore/except.cpp b/zencore/except.cpp index 9bf043f4a..c06a26edd 100644 --- a/zencore/except.cpp +++ b/zencore/except.cpp @@ -7,6 +7,41 @@ namespace zen { #if ZEN_PLATFORM_WINDOWS +class WindowsException : public std::exception +{ +public: + WindowsException(std::string_view Message) + { + m_hResult = HRESULT_FROM_WIN32(GetLastError()); + m_Message = Message; + } + + WindowsException(HRESULT hRes, std::string_view Message) + { + m_hResult = hRes; + m_Message = Message; + } + + WindowsException(HRESULT hRes, const char* Message, const char* Detail) + { + m_hResult = hRes; + + ExtendableStringBuilder<128> msg; + msg.Append(Message); + msg.Append(" (detail: '"); + msg.Append(Detail); + msg.Append("')"); + + m_Message = msg.c_str(); + } + + virtual const char* what() const override { return m_Message.c_str(); } + +private: + std::string m_Message; + HRESULT m_hResult; +}; + void ThrowSystemException([[maybe_unused]] HRESULT hRes, [[maybe_unused]] std::string_view Message) { diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h index 08330e4bc..3bffbd3bb 100644 --- a/zencore/include/zencore/except.h +++ b/zencore/include/zencore/except.h @@ -15,63 +15,18 @@ namespace zen { #if ZEN_PLATFORM_WINDOWS -class WindowsException : public std::exception -{ -public: - WindowsException(std::string_view Message) - { - m_hResult = HRESULT_FROM_WIN32(GetLastError()); - m_Message = Message; - } - - WindowsException(HRESULT hRes, std::string_view Message) - { - m_hResult = hRes; - m_Message = Message; - } - - WindowsException(HRESULT hRes, const char* Message, const char* Detail) - { - m_hResult = hRes; - - ExtendableStringBuilder<128> msg; - msg.Append(Message); - msg.Append(" (detail: '"); - msg.Append(Detail); - msg.Append("')"); - - m_Message = msg.c_str(); - } - - virtual const char* what() const override { return m_Message.c_str(); } - -private: - std::string m_Message; - HRESULT m_hResult; -}; - -ZENCORE_API void ThrowSystemException(HRESULT hRes, std::string_view Message); +ZENCORE_API void ThrowSystemException [[noreturn]] (HRESULT hRes, std::string_view Message); #endif // ZEN_PLATFORM_WINDOWS -ZENCORE_API void ThrowLastError(std::string_view Message); +ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message); #if __cpp_lib_source_location -ZENCORE_API void ThrowLastError(std::string_view Message, const std::source_location& Location); +ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message, const std::source_location& Location); #endif ZENCORE_API std::string GetLastErrorAsString(); ZENCORE_API std::string GetWindowsErrorAsString(uint32_t Win32ErrorCode); -inline void -ThrowSystemException(const char* Message) -{ -#if ZEN_PLATFORM_WINDOWS - throw WindowsException(Message); -#else - ThrowLastError(Message); -#endif -} - inline int32_t GetLastError() { diff --git a/zencore/thread.cpp b/zencore/thread.cpp index 1c7e4b3ab..c02bf508a 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -257,7 +257,8 @@ ProcessHandle::Wait(int TimeoutMs) case WAIT_FAILED: // What might go wrong here, and what is meaningful to act on? - throw WindowsException("Process::Wait failed"); + using namespace std::literals; + ThrowLastError("Process::Wait failed"sv); } return false; diff --git a/zenserver/experimental/usnjournal.cpp b/zenserver/experimental/usnjournal.cpp index ab83b8a1c..1e765fbe5 100644 --- a/zenserver/experimental/usnjournal.cpp +++ b/zenserver/experimental/usnjournal.cpp @@ -34,14 +34,14 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) if (!Success) { - zen::ThrowSystemException("GetVolumePathName failed"); + zen::ThrowLastError("GetVolumePathName failed"); } Success = GetVolumeNameForVolumeMountPoint(VolumePathName, VolumeName, ZEN_ARRAY_COUNT(VolumeName)); if (!Success) { - zen::ThrowSystemException("GetVolumeNameForVolumeMountPoint failed"); + zen::ThrowLastError("GetVolumeNameForVolumeMountPoint failed"); } // Chop off trailing slash since we want to open a volume handle, not a handle to the volume root directory @@ -64,7 +64,7 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) if (m_VolumeHandle == INVALID_HANDLE_VALUE) { - ThrowSystemException("Volume handle open failed"); + ThrowLastError("Volume handle open failed"); } // Figure out which file system is in use for volume @@ -86,7 +86,7 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) if (!Success) { - ThrowSystemException("Failed to get volume information"); + ThrowLastError("Failed to get volume information"); } ZEN_DEBUG("File system type is {}", WideToUtf8(FileSystemName)); @@ -173,7 +173,7 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) if (!Success) { - ThrowSystemException("GetFileInformationByHandleEx failed"); + ThrowLastError("GetFileInformationByHandleEx failed"); } const Frn VolumeRootFrn = FileInformation.FileId; -- cgit v1.2.3 From feb3e97f486e474f0c5775a4c407ef6db540df49 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 17 Sep 2021 19:09:10 +0200 Subject: Assert improvements --- zencore/include/zencore/zencore.h | 76 +++++++++++++++++++++++++++------------ zencore/zencore.cpp | 10 ------ 2 files changed, 54 insertions(+), 32 deletions(-) diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index 8011ab241..4f9dc6322 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -3,7 +3,7 @@ #pragma once #include -#include +#include #include ////////////////////////////////////////////////////////////////////////// @@ -90,37 +90,69 @@ // Assert // +#if ZEN_PLATFORM_WINDOWS +// Tells the compiler to put the decorated function in a certain section (aka. segment) of the executable. +# define ZEN_CODE_SECTION(Name) __declspec(code_seg(Name)) +# define ZEN_FORCENOINLINE __declspec(noinline) /* Force code to NOT be inline */ +#else +# define ZEN_CODE_SECTION(Name) +# define ZEN_FORCENOINLINE +#endif + +#if ZEN_ARCH_ARM64 +// On ARM we can't do this because the executable will require jumps larger +// than the branch instruction can handle. Clang will only generate +// the trampolines in the .text segment of the binary. If the uedbg segment +// is present it will generate code that it cannot link. +# define ZEN_DEBUG_SECTION +#else +// We'll put all assert implementation code into a separate section in the linked +// executable. This code should never execute so using a separate section keeps +// it well off the hot path and hopefully out of the instruction cache. It also +// facilitates reasoning about the makeup of a compiled/linked binary. +# define ZEN_DEBUG_SECTION ZEN_CODE_SECTION(".zcold") +#endif // DO_CHECK || DO_GUARD_SLOW + namespace zen { -class AssertException : public std::exception +class AssertException : public std::logic_error { public: - AssertException(const char* Msg); - ~AssertException(); - - [[nodiscard]] virtual char const* what() const noexcept override { return m_Msg.c_str(); } - -private: - std::string m_Msg; + AssertException(const char* Msg) : std::logic_error(Msg) {} }; } // namespace zen -#define ZEN_ASSERT(x, ...) \ - do \ - { \ - if (x) \ - break; \ - throw ::zen::AssertException{#x}; \ +template +RetType ZEN_FORCENOINLINE ZEN_DEBUG_SECTION +DispatchAssert(InnerType&& Inner, ArgTypes const&... Args) +{ + return Inner(Args...); +} + +#define ZEN_ASSERT(x, ...) \ + do \ + { \ + if (x) [[unlikely]] \ + break; \ + struct Impl \ + { \ + static void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION ExecThrow [[noreturn]] () { throw ::zen::AssertException{#x}; } \ + }; \ + Impl::ExecThrow(); \ } while (false) #ifndef NDEBUG -# define ZEN_ASSERT_SLOW(x, ...) \ - do \ - { \ - if (x) \ - break; \ - throw ::zen::AssertException{#x}; \ +# define ZEN_ASSERT_SLOW(x, ...) \ + do \ + { \ + if (x) [[unlikely]] \ + break; \ + struct Impl \ + { \ + static void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION ExecThrow [[noreturn]] () { throw ::zen::AssertException{#x}; } \ + }; \ + Impl::ExecThrow(); \ } while (false) #else # define ZEN_ASSERT_SLOW(x, ...) @@ -148,7 +180,7 @@ char (&ZenArrayCountHelper(const T (&)[N]))[N + 1]; #define ZEN_UNUSED(...) ((void)__VA_ARGS__) #define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false, __VA_ARGS__) -#define ZENCORE_API // Placeholder to allow DLL configs in the future +#define ZENCORE_API // Placeholder to allow DLL configs in the future (maybe) namespace zen { diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp index bb457ed0a..d0b1135dc 100644 --- a/zencore/zencore.cpp +++ b/zencore/zencore.cpp @@ -69,16 +69,6 @@ IsDebuggerPresent() ////////////////////////////////////////////////////////////////////////// -AssertException::AssertException(const char* Msg) : m_Msg(Msg) -{ -} - -AssertException::~AssertException() -{ -} - -////////////////////////////////////////////////////////////////////////// - static int s_ApplicationExitCode = 0; static bool s_ApplicationExitRequested; -- cgit v1.2.3 From 91ac797335b1363952595520e561fdf3b7e33a1a Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 17 Sep 2021 19:10:53 +0200 Subject: Added UNICODE and defined _WIN32_WINNT to match the sln --- xmake.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake.lua b/xmake.lua index f5a37e12d..d18962126 100644 --- a/xmake.lua +++ b/xmake.lua @@ -36,7 +36,7 @@ if is_mode("debug") then end if is_os("windows") then - add_defines("_CRT_SECURE_NO_WARNINGS") + add_defines("_CRT_SECURE_NO_WARNINGS", "_UNICODE", "UNICODE", "_WIN32_WINNT=0x0A00") end add_defines("USE_SENTRY=1") -- cgit v1.2.3 From 03dddb20a2a378eaf8c529a986ccfaeb8ee019a7 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 17 Sep 2021 19:11:11 +0200 Subject: Implemented basics for Windows server support (not yet 100% - needs to properly report service state etc to the OS) --- zen.sln | 1 + zenserver/config.cpp | 15 + zenserver/config.h | 14 +- zenserver/windows/service.cpp | 655 ++++++++++++++++++++++++++++++++++++ zenserver/windows/service.h | 15 + zenserver/xmake.lua | 2 + zenserver/zenserver.cpp | 26 +- zenserver/zenserver.vcxproj | 2 + zenserver/zenserver.vcxproj.filters | 2 + 9 files changed, 723 insertions(+), 9 deletions(-) create mode 100644 zenserver/windows/service.cpp create mode 100644 zenserver/windows/service.h diff --git a/zen.sln b/zen.sln index 2ceb9e58c..e917d5b74 100644 --- a/zen.sln +++ b/zen.sln @@ -11,6 +11,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{4EA55E5B-1 README.md = README.md RESTAPI.md = RESTAPI.md vcpkg.json = vcpkg.json + xmake.lua = xmake.lua EndProjectSection EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zencore", "zencore\zencore.vcxproj", "{D75BF9AB-C61E-4FFF-AD59-1563430F05E2}" diff --git a/zenserver/config.cpp b/zenserver/config.cpp index 578a3a202..092fc6998 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -77,6 +77,21 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z cxxopts::value(GlobalOptions.ChildId), ""); +#if ZEN_PLATFORM_WINDOWS + options.add_option("lifetime", + "", + "install", + "Install zenserver as a Windows service", + cxxopts::value(GlobalOptions.InstallService), + ""); + options.add_option("lifetime", + "", + "uninstall", + "Uninstall zenserver as a Windows service", + cxxopts::value(GlobalOptions.UninstallService), + ""); +#endif + options.add_option("network", "p", "port", diff --git a/zenserver/config.h b/zenserver/config.h index 80ec86905..c06102384 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -9,12 +9,14 @@ struct ZenServerOptions { bool IsDebug = false; bool IsTest = false; - bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements - int BasePort = 1337; // Service listen port (used for both UDP and TCP) - int OwnerPid = 0; // Parent process id (zero for standalone) - std::string ChildId; // Id assigned by parent process (used for lifetime management) - std::string LogId; // Id for tagging log output - std::filesystem::path DataDir; // Root directory for state (used for testing) + bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements + int BasePort = 1337; // Service listen port (used for both UDP and TCP) + int OwnerPid = 0; // Parent process id (zero for standalone) + std::string ChildId; // Id assigned by parent process (used for lifetime management) + bool InstallService = false; // Flag used to initiate service install (temporary) + bool UninstallService = false; // Flag used to initiate service uninstall (temporary) + std::string LogId; // Id for tagging log output + std::filesystem::path DataDir; // Root directory for state (used for testing) }; struct ZenUpstreamJupiterConfig diff --git a/zenserver/windows/service.cpp b/zenserver/windows/service.cpp new file mode 100644 index 000000000..7a7864b39 --- /dev/null +++ b/zenserver/windows/service.cpp @@ -0,0 +1,655 @@ +#include "service.h" + +#include + +#include +#include +#include + +#define SVCNAME L"Zen Store" + +SERVICE_STATUS gSvcStatus; +SERVICE_STATUS_HANDLE gSvcStatusHandle; +HANDLE ghSvcStopEvent = NULL; + +void SvcInstall(void); +void WINAPI SvcCtrlHandler(DWORD); +void WINAPI SvcMain(DWORD, LPTSTR*); + +void ReportSvcStatus(DWORD, DWORD, DWORD); +void SvcInit(DWORD, LPTSTR*); +void SvcReportEvent(LPTSTR); + +// +// Purpose: +// Installs a service in the SCM database +// +// Parameters: +// None +// +// Return value: +// None +// +VOID +SvcInstall() +{ + SC_HANDLE schSCManager; + SC_HANDLE schService; + TCHAR szPath[MAX_PATH]; + + if (!GetModuleFileName(NULL, szPath, MAX_PATH)) + { + printf("Cannot install service (%d)\n", GetLastError()); + return; + } + + // Get a handle to the SCM database. + + schSCManager = OpenSCManager(NULL, // local computer + NULL, // ServicesActive database + SC_MANAGER_ALL_ACCESS); // full access rights + + if (NULL == schSCManager) + { + printf("OpenSCManager failed (%d)\n", GetLastError()); + return; + } + + // Create the service + + schService = CreateService(schSCManager, // SCM database + SVCNAME, // name of service + SVCNAME, // service name to display + SERVICE_ALL_ACCESS, // desired access + SERVICE_WIN32_OWN_PROCESS, // service type + SERVICE_DEMAND_START, // start type + SERVICE_ERROR_NORMAL, // error control type + szPath, // path to service's binary + NULL, // no load ordering group + NULL, // no tag identifier + NULL, // no dependencies + NULL, // LocalSystem account + NULL); // no password + + if (schService == NULL) + { + printf("CreateService failed (%d)\n", GetLastError()); + CloseServiceHandle(schSCManager); + return; + } + else + printf("Service installed successfully\n"); + + CloseServiceHandle(schService); + CloseServiceHandle(schSCManager); +} + +// +// Purpose: +// Deletes a service from the SCM database +// +// Parameters: +// None +// +// Return value: +// None +// +void SvcDelete() +{ + SC_HANDLE schSCManager; + SC_HANDLE schService; + + // Get a handle to the SCM database. + + schSCManager = OpenSCManager(NULL, // local computer + NULL, // ServicesActive database + SC_MANAGER_ALL_ACCESS); // full access rights + + if (NULL == schSCManager) + { + printf("OpenSCManager failed (%d)\n", GetLastError()); + return; + } + + // Get a handle to the service. + + schService = OpenService(schSCManager, // SCM database + SVCNAME, // name of service + DELETE); // need delete access + + if (schService == NULL) + { + printf("OpenService failed (%d)\n", GetLastError()); + CloseServiceHandle(schSCManager); + return; + } + + // Delete the service. + + if (!DeleteService(schService)) + { + printf("DeleteService failed (%d)\n", GetLastError()); + } + else + printf("Service deleted successfully\n"); + + CloseServiceHandle(schService); + CloseServiceHandle(schSCManager); +} + +// +// Purpose: +// Retrieves and displays the current service configuration. +// +// Parameters: +// None +// +// Return value: +// None +// +void __stdcall DoQuerySvc() +{ + SC_HANDLE schSCManager{}; + SC_HANDLE schService{}; + LPQUERY_SERVICE_CONFIG lpsc{}; + LPSERVICE_DESCRIPTION lpsd{}; + DWORD dwBytesNeeded{}, cbBufSize{}, dwError{}; + + // Get a handle to the SCM database. + + schSCManager = OpenSCManager(NULL, // local computer + NULL, // ServicesActive database + SC_MANAGER_ALL_ACCESS); // full access rights + + if (NULL == schSCManager) + { + printf("OpenSCManager failed (%d)\n", GetLastError()); + return; + } + + // Get a handle to the service. + + schService = OpenService(schSCManager, // SCM database + SVCNAME, // name of service + SERVICE_QUERY_CONFIG); // need query config access + + if (schService == NULL) + { + printf("OpenService failed (%d)\n", GetLastError()); + CloseServiceHandle(schSCManager); + return; + } + + // Get the configuration information. + + if (!QueryServiceConfig(schService, NULL, 0, &dwBytesNeeded)) + { + dwError = GetLastError(); + if (ERROR_INSUFFICIENT_BUFFER == dwError) + { + cbBufSize = dwBytesNeeded; + lpsc = (LPQUERY_SERVICE_CONFIG)LocalAlloc(LMEM_FIXED, cbBufSize); + } + else + { + printf("QueryServiceConfig failed (%d)", dwError); + goto cleanup; + } + } + + if (!QueryServiceConfig(schService, lpsc, cbBufSize, &dwBytesNeeded)) + { + printf("QueryServiceConfig failed (%d)", GetLastError()); + goto cleanup; + } + + if (!QueryServiceConfig2(schService, SERVICE_CONFIG_DESCRIPTION, NULL, 0, &dwBytesNeeded)) + { + dwError = GetLastError(); + if (ERROR_INSUFFICIENT_BUFFER == dwError) + { + cbBufSize = dwBytesNeeded; + lpsd = (LPSERVICE_DESCRIPTION)LocalAlloc(LMEM_FIXED, cbBufSize); + } + else + { + printf("QueryServiceConfig2 failed (%d)", dwError); + goto cleanup; + } + } + + if (!QueryServiceConfig2(schService, SERVICE_CONFIG_DESCRIPTION, (LPBYTE)lpsd, cbBufSize, &dwBytesNeeded)) + { + printf("QueryServiceConfig2 failed (%d)", GetLastError()); + goto cleanup; + } + + // Print the configuration information. + + _tprintf(TEXT("%s configuration: \n"), SVCNAME); + _tprintf(TEXT(" Type: 0x%x\n"), lpsc->dwServiceType); + _tprintf(TEXT(" Start Type: 0x%x\n"), lpsc->dwStartType); + _tprintf(TEXT(" Error Control: 0x%x\n"), lpsc->dwErrorControl); + _tprintf(TEXT(" Binary path: %s\n"), lpsc->lpBinaryPathName); + _tprintf(TEXT(" Account: %s\n"), lpsc->lpServiceStartName); + + if (lpsd->lpDescription != NULL && lstrcmp(lpsd->lpDescription, TEXT("")) != 0) + _tprintf(TEXT(" Description: %s\n"), lpsd->lpDescription); + if (lpsc->lpLoadOrderGroup != NULL && lstrcmp(lpsc->lpLoadOrderGroup, TEXT("")) != 0) + _tprintf(TEXT(" Load order group: %s\n"), lpsc->lpLoadOrderGroup); + if (lpsc->dwTagId != 0) + _tprintf(TEXT(" Tag ID: %d\n"), lpsc->dwTagId); + if (lpsc->lpDependencies != NULL && lstrcmp(lpsc->lpDependencies, TEXT("")) != 0) + _tprintf(TEXT(" Dependencies: %s\n"), lpsc->lpDependencies); + + LocalFree(lpsc); + LocalFree(lpsd); + +cleanup: + CloseServiceHandle(schService); + CloseServiceHandle(schSCManager); +} +// +// Purpose: +// Disables the service. +// +// Parameters: +// None +// +// Return value: +// None +// +VOID __stdcall DoDisableSvc() +{ + SC_HANDLE schSCManager; + SC_HANDLE schService; + + // Get a handle to the SCM database. + + schSCManager = OpenSCManager(NULL, // local computer + NULL, // ServicesActive database + SC_MANAGER_ALL_ACCESS); // full access rights + + if (NULL == schSCManager) + { + printf("OpenSCManager failed (%d)\n", GetLastError()); + return; + } + + // Get a handle to the service. + + schService = OpenService(schSCManager, // SCM database + SVCNAME, // name of service + SERVICE_CHANGE_CONFIG); // need change config access + + if (schService == NULL) + { + printf("OpenService failed (%d)\n", GetLastError()); + CloseServiceHandle(schSCManager); + return; + } + + // Change the service start type. + + if (!ChangeServiceConfig(schService, // handle of service + SERVICE_NO_CHANGE, // service type: no change + SERVICE_DISABLED, // service start type + SERVICE_NO_CHANGE, // error control: no change + NULL, // binary path: no change + NULL, // load order group: no change + NULL, // tag ID: no change + NULL, // dependencies: no change + NULL, // account name: no change + NULL, // password: no change + NULL)) // display name: no change + { + printf("ChangeServiceConfig failed (%d)\n", GetLastError()); + } + else + printf("Service disabled successfully.\n"); + + CloseServiceHandle(schService); + CloseServiceHandle(schSCManager); +} + +// +// Purpose: +// Enables the service. +// +// Parameters: +// None +// +// Return value: +// None +// +VOID __stdcall DoEnableSvc() +{ + SC_HANDLE schSCManager; + SC_HANDLE schService; + + // Get a handle to the SCM database. + + schSCManager = OpenSCManager(NULL, // local computer + NULL, // ServicesActive database + SC_MANAGER_ALL_ACCESS); // full access rights + + if (NULL == schSCManager) + { + printf("OpenSCManager failed (%d)\n", GetLastError()); + return; + } + + // Get a handle to the service. + + schService = OpenService(schSCManager, // SCM database + SVCNAME, // name of service + SERVICE_CHANGE_CONFIG); // need change config access + + if (schService == NULL) + { + printf("OpenService failed (%d)\n", GetLastError()); + CloseServiceHandle(schSCManager); + return; + } + + // Change the service start type. + + if (!ChangeServiceConfig(schService, // handle of service + SERVICE_NO_CHANGE, // service type: no change + SERVICE_DEMAND_START, // service start type + SERVICE_NO_CHANGE, // error control: no change + NULL, // binary path: no change + NULL, // load order group: no change + NULL, // tag ID: no change + NULL, // dependencies: no change + NULL, // account name: no change + NULL, // password: no change + NULL)) // display name: no change + { + printf("ChangeServiceConfig failed (%d)\n", GetLastError()); + } + else + printf("Service enabled successfully.\n"); + + CloseServiceHandle(schService); + CloseServiceHandle(schSCManager); +} +// +// Purpose: +// Updates the service description to "This is a test description". +// +// Parameters: +// None +// +// Return value: +// None +// +VOID __stdcall DoUpdateSvcDesc() +{ + SC_HANDLE schSCManager; + SC_HANDLE schService; + SERVICE_DESCRIPTION sd; + TCHAR szDesc[] = TEXT("This is a test description"); + + // Get a handle to the SCM database. + + schSCManager = OpenSCManager(NULL, // local computer + NULL, // ServicesActive database + SC_MANAGER_ALL_ACCESS); // full access rights + + if (NULL == schSCManager) + { + printf("OpenSCManager failed (%d)\n", GetLastError()); + return; + } + + // Get a handle to the service. + + schService = OpenService(schSCManager, // SCM database + SVCNAME, // name of service + SERVICE_CHANGE_CONFIG); // need change config access + + if (schService == NULL) + { + printf("OpenService failed (%d)\n", GetLastError()); + CloseServiceHandle(schSCManager); + return; + } + + // Change the service description. + + sd.lpDescription = szDesc; + + if (!ChangeServiceConfig2(schService, // handle to service + SERVICE_CONFIG_DESCRIPTION, // change: description + &sd)) // new description + { + printf("ChangeServiceConfig2 failed\n"); + } + else + printf("Service description updated successfully.\n"); + + CloseServiceHandle(schService); + CloseServiceHandle(schSCManager); +} + +// +// Purpose: +// Entry point for the service +// +// Parameters: +// dwArgc - Number of arguments in the lpszArgv array +// lpszArgv - Array of strings. The first string is the name of +// the service and subsequent strings are passed by the process +// that called the StartService function to start the service. +// +// Return value: +// None. +// +VOID WINAPI +SvcMain(DWORD dwArgc, LPTSTR* lpszArgv) +{ + // Register the handler function for the service + + gSvcStatusHandle = RegisterServiceCtrlHandler(SVCNAME, SvcCtrlHandler); + + if (!gSvcStatusHandle) + { + SvcReportEvent((LPTSTR)TEXT("RegisterServiceCtrlHandler")); + return; + } + + // These SERVICE_STATUS members remain as set here + + gSvcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS; + gSvcStatus.dwServiceSpecificExitCode = 0; + + // Report initial status to the SCM + + ReportSvcStatus(SERVICE_START_PENDING, NO_ERROR, 3000); + + // Perform service-specific initialization and work. + + SvcInit(dwArgc, lpszArgv); +} + +// +// Purpose: +// The service code +// +// Parameters: +// dwArgc - Number of arguments in the lpszArgv array +// lpszArgv - Array of strings. The first string is the name of +// the service and subsequent strings are passed by the process +// that called the StartService function to start the service. +// +// Return value: +// None +// +VOID +SvcInit(DWORD dwArgc, LPTSTR* lpszArgv) +{ + // TO_DO: Declare and set any required variables. + // Be sure to periodically call ReportSvcStatus() with + // SERVICE_START_PENDING. If initialization fails, call + // ReportSvcStatus with SERVICE_STOPPED. + + ZEN_UNUSED(lpszArgv, dwArgc); + + // Create an event. The control handler function, SvcCtrlHandler, + // signals this event when it receives the stop control code. + + ghSvcStopEvent = CreateEvent(NULL, // default security attributes + TRUE, // manual reset event + FALSE, // not signaled + NULL); // no name + + if (ghSvcStopEvent == NULL) + { + ReportSvcStatus(SERVICE_STOPPED, GetLastError(), 0); + return; + } + + // Report running status when initialization is complete. + + ReportSvcStatus(SERVICE_RUNNING, NO_ERROR, 0); + + // TO_DO: Perform work until service stops. + + while (1) + { + // Check whether to stop the service. + + WaitForSingleObject(ghSvcStopEvent, INFINITE); + + ReportSvcStatus(SERVICE_STOPPED, NO_ERROR, 0); + return; + } +} + +// +// Purpose: +// Sets the current service status and reports it to the SCM. +// +// Parameters: +// dwCurrentState - The current state (see SERVICE_STATUS) +// dwWin32ExitCode - The system error code +// dwWaitHint - Estimated time for pending operation, +// in milliseconds +// +// Return value: +// None +// +VOID +ReportSvcStatus(DWORD dwCurrentState, DWORD dwWin32ExitCode, DWORD dwWaitHint) +{ + static DWORD dwCheckPoint = 1; + + // Fill in the SERVICE_STATUS structure. + + gSvcStatus.dwCurrentState = dwCurrentState; + gSvcStatus.dwWin32ExitCode = dwWin32ExitCode; + gSvcStatus.dwWaitHint = dwWaitHint; + + if (dwCurrentState == SERVICE_START_PENDING) + gSvcStatus.dwControlsAccepted = 0; + else + gSvcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP; + + if ((dwCurrentState == SERVICE_RUNNING) || (dwCurrentState == SERVICE_STOPPED)) + gSvcStatus.dwCheckPoint = 0; + else + gSvcStatus.dwCheckPoint = dwCheckPoint++; + + // Report the status of the service to the SCM. + SetServiceStatus(gSvcStatusHandle, &gSvcStatus); +} + +// +// Purpose: +// Called by SCM whenever a control code is sent to the service +// using the ControlService function. +// +// Parameters: +// dwCtrl - control code +// +// Return value: +// None +// +VOID WINAPI +SvcCtrlHandler(DWORD dwCtrl) +{ + // Handle the requested control code. + + switch (dwCtrl) + { + case SERVICE_CONTROL_STOP: + ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, 0); + + // Signal the service to stop. + + SetEvent(ghSvcStopEvent); + ReportSvcStatus(gSvcStatus.dwCurrentState, NO_ERROR, 0); + + return; + + case SERVICE_CONTROL_INTERROGATE: + break; + + default: + break; + } +} + +// +// Purpose: +// Logs messages to the event log +// +// Parameters: +// szFunction - name of function that failed +// +// Return value: +// None +// +// Remarks: +// The service must have an entry in the Application event log. +// +VOID +SvcReportEvent(LPTSTR szFunction) +{ + ZEN_UNUSED(szFunction); + + // HANDLE hEventSource; + // LPCTSTR lpszStrings[2]; + // TCHAR Buffer[80]; + + // hEventSource = RegisterEventSource(NULL, SVCNAME); + + // if (NULL != hEventSource) + //{ + // StringCchPrintf(Buffer, 80, TEXT("%s failed with %d"), szFunction, GetLastError()); + + // lpszStrings[0] = SVCNAME; + // lpszStrings[1] = Buffer; + + // ReportEvent(hEventSource, // event log handle + // EVENTLOG_ERROR_TYPE, // event type + // 0, // event category + // SVC_ERROR, // event identifier + // NULL, // no security identifier + // 2, // size of lpszStrings array + // 0, // no binary data + // lpszStrings, // array of strings + // NULL); // no binary data + + // DeregisterEventSource(hEventSource); + //} +} + +WindowsServiceBase::WindowsServiceBase() +{ +} + +WindowsServiceBase::~WindowsServiceBase() +{ +} diff --git a/zenserver/windows/service.h b/zenserver/windows/service.h new file mode 100644 index 000000000..0f76d7447 --- /dev/null +++ b/zenserver/windows/service.h @@ -0,0 +1,15 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +void SvcInstall(void); +void SvcDelete(); + +class WindowsServiceBase +{ +public: + WindowsServiceBase(); + ~WindowsServiceBase(); + +private: +}; \ No newline at end of file diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua index bb70846fa..7a6981fcd 100644 --- a/zenserver/xmake.lua +++ b/zenserver/xmake.lua @@ -14,6 +14,8 @@ target("zenserver") add_ldflags("/MANIFEST:EMBED") add_ldflags("/MANIFESTUAC:level='requireAdministrator'") add_ldflags("/LTCG") + else + del_files("windows/**") end add_options("vfs") diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index eb0324161..f36cfba48 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -11,9 +11,9 @@ #include #include #include -#include #include #include +#include #include #include @@ -40,6 +40,10 @@ #include "config.h" #include "diag/logging.h" +#if ZEN_PLATFORM_WINDOWS +# include "windows/service.h" +#endif + ////////////////////////////////////////////////////////////////////////// // Sentry // @@ -422,7 +426,7 @@ private: bool m_DebugOptionForcedCrash = false; }; -} +} // namespace zen int main(int argc, char* argv[]) @@ -436,6 +440,22 @@ main(int argc, char* argv[]) ParseGlobalCliOptions(argc, argv, GlobalOptions, ServiceConfig); InitializeLogging(GlobalOptions); +#if ZEN_PLATFORM_WINDOWS + if (GlobalOptions.InstallService) + { + SvcInstall(); + + std::exit(0); + } + + if (GlobalOptions.UninstallService) + { + SvcDelete(); + + std::exit(0); + } +#endif + #if USE_SENTRY // Initialize sentry.io client @@ -443,7 +463,7 @@ main(int argc, char* argv[]) sentry_options_set_dsn(SentryOptions, "https://8ba3441bebc941c1ae24b8cd2fd25d55@o10593.ingest.sentry.io/5919284"); sentry_init(SentryOptions); - auto _ = zen::MakeGuard([&] { sentry_close(); }); + auto _ = zen::MakeGuard([] { sentry_close(); }); #endif try diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj index aa9d538a5..db657d192 100644 --- a/zenserver/zenserver.vcxproj +++ b/zenserver/zenserver.vcxproj @@ -123,6 +123,7 @@ + @@ -142,6 +143,7 @@ + diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters index a86a6d96d..250c55812 100644 --- a/zenserver/zenserver.vcxproj.filters +++ b/zenserver/zenserver.vcxproj.filters @@ -39,6 +39,7 @@ upstream + @@ -73,6 +74,7 @@ upstream + -- cgit v1.2.3 From de65c608c2cefc4be771c11b5bb7132a84fa46eb Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 17 Sep 2021 23:04:11 +0200 Subject: Added better handling for read-only mode --- zenutil/include/zenutil/zenserverprocess.h | 2 ++ zenutil/zenserverprocess.cpp | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/zenutil/include/zenutil/zenserverprocess.h b/zenutil/include/zenutil/zenserverprocess.h index 005f7adff..979db349a 100644 --- a/zenutil/include/zenutil/zenserverprocess.h +++ b/zenutil/include/zenutil/zenserverprocess.h @@ -123,12 +123,14 @@ public: ZenServerEntry* Register(int ListenPort); void Sweep(); void Snapshot(std::function&& Callback); + inline bool IsReadOnly() const { return m_IsReadOnly; } private: void* m_hMapFile = nullptr; ZenServerEntry* m_Data = nullptr; int m_MaxEntryCount = 131072 / sizeof(ZenServerEntry); ZenServerEntry* m_OurEntry = nullptr; + bool m_IsReadOnly = true; }; } // namespace zen diff --git a/zenutil/zenserverprocess.cpp b/zenutil/zenserverprocess.cpp index c504ce7da..c6bee1ed7 100644 --- a/zenutil/zenserverprocess.cpp +++ b/zenutil/zenserverprocess.cpp @@ -129,7 +129,8 @@ ZenServerState::Initialize() zen::ThrowLastError("Could not map view of Zen server state"); } - m_Data = reinterpret_cast(pBuf); + m_Data = reinterpret_cast(pBuf); + m_IsReadOnly = false; } bool @@ -221,6 +222,8 @@ ZenServerState::Sweep() return; } + ZEN_ASSERT(m_IsReadOnly == false); + for (int i = 0; i < m_MaxEntryCount; ++i) { ZenServerEntry& Entry = m_Data[i]; -- cgit v1.2.3 From f1992cb73b7e7ff5a20740c0085ce8cb5b72d01f Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 17 Sep 2021 23:07:26 +0200 Subject: Don't sweep instance table in read-only mode --- zen/cmds/top.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zen/cmds/top.cpp b/zen/cmds/top.cpp index 3e3942d31..315d8cb38 100644 --- a/zen/cmds/top.cpp +++ b/zen/cmds/top.cpp @@ -48,7 +48,10 @@ TopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) zen::Sleep(1000); - State.Sweep(); + if (!State.IsReadOnly()) + { + State.Sweep(); + } } return 0; -- cgit v1.2.3 From a6244f914633a4245fc71a987cb42fce0b7d8671 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 17 Sep 2021 23:07:45 +0200 Subject: Added ThrowSystemError() helper --- zencore/except.cpp | 6 ++++++ zencore/include/zencore/except.h | 2 ++ 2 files changed, 8 insertions(+) diff --git a/zencore/except.cpp b/zencore/except.cpp index c06a26edd..44b8edffb 100644 --- a/zencore/except.cpp +++ b/zencore/except.cpp @@ -63,6 +63,12 @@ ThrowLastError(std::string_view Message) throw std::system_error(std::error_code(zen::GetLastError(), std::system_category()), std::string(Message)); } +void +ThrowSystemError(uint32_t ErrorCode, std::string_view Message) +{ + throw std::system_error(std::error_code(ErrorCode, std::system_category()), std::string(Message)); +} + std::string GetLastErrorAsString() { diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h index 3bffbd3bb..5cfefb1e2 100644 --- a/zencore/include/zencore/except.h +++ b/zencore/include/zencore/except.h @@ -24,6 +24,8 @@ ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message); ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message, const std::source_location& Location); #endif +ZENCORE_API void ThrowSystemError [[noreturn]] (uint32_t ErrorCode, std::string_view Message); + ZENCORE_API std::string GetLastErrorAsString(); ZENCORE_API std::string GetWindowsErrorAsString(uint32_t Win32ErrorCode); -- cgit v1.2.3 From 28de689b30b64c0c2691b05866c27c84e0a21ff3 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 17 Sep 2021 23:15:56 +0200 Subject: IsProcessRunning now throws if it fails the function fails to get a handle to the process due to an error (unless it is because the process does not exist) --- zencore/thread.cpp | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/zencore/thread.cpp b/zencore/thread.cpp index c02bf508a..c92cca6de 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -334,16 +334,32 @@ ProcessMonitor::IsActive() const bool IsProcessRunning(int pid) { + // This function is arguably not super useful, a pid can be re-used + // by the OS so holding on to a pid and polling it over some time + // period will not necessarily tell you what you probably want to know. + +#if ZEN_PLATFORM_WINDOWS HANDLE hProc = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid); - if (hProc == NULL) + if (!hProc) { - return false; + DWORD Error = zen::GetLastError(); + + if (Error == ERROR_INVALID_PARAMETER) + { + return false; + } + + using namespace fmt::literals; + ThrowSystemError(Error, "failed to open process with pid {}"_format(pid)); } CloseHandle(hProc); return true; +#else + ZEN_NOT_IMPLEMENTED(); +#endif } int -- cgit v1.2.3 From 91c305d98ab2a0482114c2d52d23ef787f08190d Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 17 Sep 2021 23:16:34 +0200 Subject: Added IsInteractiveSession() query to help identify if the process is running as a daemon or as an interactive process --- zencore/include/zencore/zencore.h | 2 ++ zencore/zencore.cpp | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index 4f9dc6322..310f6c4ed 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -188,6 +188,8 @@ ZENCORE_API bool IsPointerToStack(const void* ptr); // Query if pointer is with ZENCORE_API bool IsApplicationExitRequested(); ZENCORE_API void RequestApplicationExit(int ExitCode); ZENCORE_API bool IsDebuggerPresent(); +ZENCORE_API bool IsInteractiveSession(); + ZENCORE_API void zencore_forcelinktests(); } // namespace zen diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp index d0b1135dc..122719d90 100644 --- a/zencore/zencore.cpp +++ b/zencore/zencore.cpp @@ -67,6 +67,23 @@ IsDebuggerPresent() #endif } +bool +IsInteractiveSession() +{ +#if ZEN_PLATFORM_WINDOWS + DWORD dwSessionId = 0; + if (ProcessIdToSessionId(GetCurrentProcessId(), &dwSessionId)) + { + return (dwSessionId != 0); + } + + return false; +#else + // TODO: figure out what makes sense here + return true; +#endif +} + ////////////////////////////////////////////////////////////////////////// static int s_ApplicationExitCode = 0; -- cgit v1.2.3 From 0ee89539ead8631b02953ddf601770aefa557edb Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 17 Sep 2021 23:18:20 +0200 Subject: zenserver can now run as a Windows service. We'll still need to improve how data files are found as the current defaults are relative to the user directory which ends up being in the Windows folder when running as the local system user --- zenhttp/httpnull.cpp | 4 +- zenhttp/httpnull.h | 2 +- zenhttp/httpsys.cpp | 23 ++-- zenhttp/httpuws.cpp | 4 +- zenhttp/httpuws.h | 2 +- zenhttp/include/zenhttp/httpserver.h | 2 +- zenserver/windows/service.cpp | 246 ++++++++++++++++------------------- zenserver/windows/service.h | 21 +-- zenserver/zenserver.cpp | 79 +++++++---- 9 files changed, 201 insertions(+), 182 deletions(-) diff --git a/zenhttp/httpnull.cpp b/zenhttp/httpnull.cpp index 57cba13d3..e49051ac5 100644 --- a/zenhttp/httpnull.cpp +++ b/zenhttp/httpnull.cpp @@ -28,8 +28,10 @@ HttpNullServer::Initialize(int BasePort) } void -HttpNullServer::Run(bool TestMode) +HttpNullServer::Run(bool IsInteractiveSession) { + const bool TestMode = !IsInteractiveSession; + if (TestMode == false) { zen::logging::ConsoleLog().info("Zen Server running (null HTTP). Press ESC or Q to quit"); diff --git a/zenhttp/httpnull.h b/zenhttp/httpnull.h index b15b1b123..867bbe4d2 100644 --- a/zenhttp/httpnull.h +++ b/zenhttp/httpnull.h @@ -19,7 +19,7 @@ public: virtual void RegisterService(HttpService& Service) override; virtual void Initialize(int BasePort) override; - virtual void Run(bool TestMode) override; + virtual void Run(bool IsInteractiveSession) override; virtual void RequestExit() override; private: diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index c2d4ef14c..b5313021c 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -707,29 +707,30 @@ HttpSysServer::StartServer() } void -HttpSysServer::Run(bool TestMode) +HttpSysServer::Run(bool IsInteractive) { - if (TestMode == false) + if (IsInteractive) { zen::logging::ConsoleLog().info("Zen Server running. Press ESC or Q to quit"); } do { - int WaitTimeout = -1; + //int WaitTimeout = -1; + int WaitTimeout = 100; - if (!TestMode) + if (IsInteractive) { WaitTimeout = 1000; - } - - if (!TestMode && _kbhit() != 0) - { - char c = (char)_getch(); - if (c == 27 || c == 'Q' || c == 'q') + if (_kbhit() != 0) { - RequestApplicationExit(0); + char c = (char)_getch(); + + if (c == 27 || c == 'Q' || c == 'q') + { + RequestApplicationExit(0); + } } } diff --git a/zenhttp/httpuws.cpp b/zenhttp/httpuws.cpp index 992809b17..e062e7747 100644 --- a/zenhttp/httpuws.cpp +++ b/zenhttp/httpuws.cpp @@ -37,8 +37,10 @@ HttpUwsServer::Initialize(int BasePort) } void -HttpUwsServer::Run(bool TestMode) +HttpUwsServer::Run(bool IsInteractive) { + const bool TestMode = !IsInteractive; + if (TestMode == false) { zen::logging::ConsoleLog().info("Zen Server running (null HTTP). Press ESC or Q to quit"); diff --git a/zenhttp/httpuws.h b/zenhttp/httpuws.h index ec55ae2fd..5e300202f 100644 --- a/zenhttp/httpuws.h +++ b/zenhttp/httpuws.h @@ -16,7 +16,7 @@ public: virtual void RegisterService(HttpService& Service) override; virtual void Initialize(int BasePort) override; - virtual void Run(bool TestMode) override; + virtual void Run(bool IsInteractiveSession) override; virtual void RequestExit() override; private: diff --git a/zenhttp/include/zenhttp/httpserver.h b/zenhttp/include/zenhttp/httpserver.h index ed6075c92..6a7dc8a70 100644 --- a/zenhttp/include/zenhttp/httpserver.h +++ b/zenhttp/include/zenhttp/httpserver.h @@ -167,7 +167,7 @@ class HttpServer : public RefCounted public: virtual void RegisterService(HttpService& Service) = 0; virtual void Initialize(int BasePort) = 0; - virtual void Run(bool TestMode) = 0; + virtual void Run(bool IsInteractiveSession) = 0; virtual void RequestExit() = 0; }; diff --git a/zenserver/windows/service.cpp b/zenserver/windows/service.cpp index 7a7864b39..bd80e0c2c 100644 --- a/zenserver/windows/service.cpp +++ b/zenserver/windows/service.cpp @@ -1,3 +1,5 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + #include "service.h" #include @@ -12,14 +14,19 @@ SERVICE_STATUS gSvcStatus; SERVICE_STATUS_HANDLE gSvcStatusHandle; HANDLE ghSvcStopEvent = NULL; -void SvcInstall(void); -void WINAPI SvcCtrlHandler(DWORD); -void WINAPI SvcMain(DWORD, LPTSTR*); +void SvcInstall(void); void ReportSvcStatus(DWORD, DWORD, DWORD); -void SvcInit(DWORD, LPTSTR*); void SvcReportEvent(LPTSTR); +WindowsService::WindowsService() +{ +} + +WindowsService::~WindowsService() +{ +} + // // Purpose: // Installs a service in the SCM database @@ -31,7 +38,7 @@ void SvcReportEvent(LPTSTR); // None // VOID -SvcInstall() +WindowsService::Install() { SC_HANDLE schSCManager; SC_HANDLE schService; @@ -84,17 +91,8 @@ SvcInstall() CloseServiceHandle(schSCManager); } -// -// Purpose: -// Deletes a service from the SCM database -// -// Parameters: -// None -// -// Return value: -// None -// -void SvcDelete() +void +WindowsService::Delete() { SC_HANDLE schSCManager; SC_HANDLE schService; @@ -137,6 +135,89 @@ void SvcDelete() CloseServiceHandle(schSCManager); } +WindowsService* gSvc; + +void WINAPI +CallMain(DWORD, LPSTR*) +{ + gSvc->SvcMain(); +} + +int +WindowsService::ServiceMain() +{ + if (zen::IsInteractiveSession()) + { + // Not actually running as a service + return Run(); + } + else + { + gSvc = this; + + SERVICE_TABLE_ENTRY DispatchTable[] = {{(LPWSTR)SVCNAME, (LPSERVICE_MAIN_FUNCTION)&CallMain}, {NULL, NULL}}; + + // This call returns when the service has stopped. + // The process should simply terminate when the call returns. + + if (!StartServiceCtrlDispatcher(DispatchTable)) + { + SvcReportEvent((LPTSTR)L"StartServiceCtrlDispatcher"); + } + } + + return 0; +} + +int +WindowsService::SvcMain() +{ + // Register the handler function for the service + + gSvcStatusHandle = RegisterServiceCtrlHandler(SVCNAME, SvcCtrlHandler); + + if (!gSvcStatusHandle) + { + SvcReportEvent((LPTSTR)TEXT("RegisterServiceCtrlHandler")); + + return 1; + } + + // These SERVICE_STATUS members remain as set here + + gSvcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS; + gSvcStatus.dwServiceSpecificExitCode = 0; + + // Report initial status to the SCM + + ReportSvcStatus(SERVICE_START_PENDING, NO_ERROR, 3000); + + // Create an event. The control handler function, SvcCtrlHandler, + // signals this event when it receives the stop control code. + + ghSvcStopEvent = CreateEvent(NULL, // default security attributes + TRUE, // manual reset event + FALSE, // not signaled + NULL); // no name + + if (ghSvcStopEvent == NULL) + { + ReportSvcStatus(SERVICE_STOPPED, GetLastError(), 0); + + return 1; + } + + // Report running status when initialization is complete. + + ReportSvcStatus(SERVICE_RUNNING, NO_ERROR, 0); + + int ReturnCode = Run(); + + ReportSvcStatus(SERVICE_STOPPED, NO_ERROR, 0); + + return ReturnCode; +} + // // Purpose: // Retrieves and displays the current service configuration. @@ -147,7 +228,8 @@ void SvcDelete() // Return value: // None // -void __stdcall DoQuerySvc() +void +DoQuerySvc() { SC_HANDLE schSCManager{}; SC_HANDLE schService{}; @@ -249,6 +331,7 @@ cleanup: CloseServiceHandle(schService); CloseServiceHandle(schSCManager); } + // // Purpose: // Disables the service. @@ -259,7 +342,8 @@ cleanup: // Return value: // None // -VOID __stdcall DoDisableSvc() +void +DoDisableSvc() { SC_HANDLE schSCManager; SC_HANDLE schService; @@ -384,7 +468,8 @@ VOID __stdcall DoEnableSvc() // Return value: // None // -VOID __stdcall DoUpdateSvcDesc() +void +DoUpdateSvcDesc() { SC_HANDLE schSCManager; SC_HANDLE schService; @@ -433,100 +518,6 @@ VOID __stdcall DoUpdateSvcDesc() CloseServiceHandle(schSCManager); } -// -// Purpose: -// Entry point for the service -// -// Parameters: -// dwArgc - Number of arguments in the lpszArgv array -// lpszArgv - Array of strings. The first string is the name of -// the service and subsequent strings are passed by the process -// that called the StartService function to start the service. -// -// Return value: -// None. -// -VOID WINAPI -SvcMain(DWORD dwArgc, LPTSTR* lpszArgv) -{ - // Register the handler function for the service - - gSvcStatusHandle = RegisterServiceCtrlHandler(SVCNAME, SvcCtrlHandler); - - if (!gSvcStatusHandle) - { - SvcReportEvent((LPTSTR)TEXT("RegisterServiceCtrlHandler")); - return; - } - - // These SERVICE_STATUS members remain as set here - - gSvcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS; - gSvcStatus.dwServiceSpecificExitCode = 0; - - // Report initial status to the SCM - - ReportSvcStatus(SERVICE_START_PENDING, NO_ERROR, 3000); - - // Perform service-specific initialization and work. - - SvcInit(dwArgc, lpszArgv); -} - -// -// Purpose: -// The service code -// -// Parameters: -// dwArgc - Number of arguments in the lpszArgv array -// lpszArgv - Array of strings. The first string is the name of -// the service and subsequent strings are passed by the process -// that called the StartService function to start the service. -// -// Return value: -// None -// -VOID -SvcInit(DWORD dwArgc, LPTSTR* lpszArgv) -{ - // TO_DO: Declare and set any required variables. - // Be sure to periodically call ReportSvcStatus() with - // SERVICE_START_PENDING. If initialization fails, call - // ReportSvcStatus with SERVICE_STOPPED. - - ZEN_UNUSED(lpszArgv, dwArgc); - - // Create an event. The control handler function, SvcCtrlHandler, - // signals this event when it receives the stop control code. - - ghSvcStopEvent = CreateEvent(NULL, // default security attributes - TRUE, // manual reset event - FALSE, // not signaled - NULL); // no name - - if (ghSvcStopEvent == NULL) - { - ReportSvcStatus(SERVICE_STOPPED, GetLastError(), 0); - return; - } - - // Report running status when initialization is complete. - - ReportSvcStatus(SERVICE_RUNNING, NO_ERROR, 0); - - // TO_DO: Perform work until service stops. - - while (1) - { - // Check whether to stop the service. - - WaitForSingleObject(ghSvcStopEvent, INFINITE); - - ReportSvcStatus(SERVICE_STOPPED, NO_ERROR, 0); - return; - } -} - // // Purpose: // Sets the current service status and reports it to the SCM. @@ -565,21 +556,13 @@ ReportSvcStatus(DWORD dwCurrentState, DWORD dwWin32ExitCode, DWORD dwWaitHint) SetServiceStatus(gSvcStatusHandle, &gSvcStatus); } -// -// Purpose: -// Called by SCM whenever a control code is sent to the service -// using the ControlService function. -// -// Parameters: -// dwCtrl - control code -// -// Return value: -// None -// -VOID WINAPI -SvcCtrlHandler(DWORD dwCtrl) +void +WindowsService::SvcCtrlHandler(DWORD dwCtrl) { // Handle the requested control code. + // + // Called by SCM whenever a control code is sent to the service + // using the ControlService function. switch (dwCtrl) { @@ -589,8 +572,9 @@ SvcCtrlHandler(DWORD dwCtrl) // Signal the service to stop. SetEvent(ghSvcStopEvent); - ReportSvcStatus(gSvcStatus.dwCurrentState, NO_ERROR, 0); + zen::RequestApplicationExit(0); + ReportSvcStatus(gSvcStatus.dwCurrentState, NO_ERROR, 0); return; case SERVICE_CONTROL_INTERROGATE: @@ -645,11 +629,3 @@ SvcReportEvent(LPTSTR szFunction) // DeregisterEventSource(hEventSource); //} } - -WindowsServiceBase::WindowsServiceBase() -{ -} - -WindowsServiceBase::~WindowsServiceBase() -{ -} diff --git a/zenserver/windows/service.h b/zenserver/windows/service.h index 0f76d7447..7c9610983 100644 --- a/zenserver/windows/service.h +++ b/zenserver/windows/service.h @@ -2,14 +2,19 @@ #pragma once -void SvcInstall(void); -void SvcDelete(); - -class WindowsServiceBase +class WindowsService { public: - WindowsServiceBase(); - ~WindowsServiceBase(); + WindowsService(); + ~WindowsService(); + + virtual int Run() = 0; + + int ServiceMain(); + + static void Install(); + static void Delete(); -private: -}; \ No newline at end of file + int SvcMain(); + static void __stdcall SvcCtrlHandler(unsigned long); +}; diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 83580b288..ea4a2915e 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -316,7 +316,9 @@ public: __debugbreak(); } - m_Http->Run(m_TestMode); + const bool IsInteractiveMode = zen::IsInteractiveSession() && !m_TestMode; + + m_Http->Run(IsInteractiveMode); ZEN_INFO(ZEN_APP_NAME " exiting"); @@ -436,33 +438,29 @@ private: } // namespace zen -int -main(int argc, char* argv[]) +class ZenWindowsService : public WindowsService { - using namespace zen; - - mi_version(); - - ZenServerOptions GlobalOptions; - ZenServiceConfig ServiceConfig; - ParseGlobalCliOptions(argc, argv, GlobalOptions, ServiceConfig); - InitializeLogging(GlobalOptions); - -#if ZEN_PLATFORM_WINDOWS - if (GlobalOptions.InstallService) +public: + ZenWindowsService(ZenServerOptions& GlobalOptions, ZenServiceConfig& ServiceConfig) + : m_GlobalOptions(GlobalOptions) + , m_ServiceConfig(ServiceConfig) { - SvcInstall(); - - std::exit(0); } - if (GlobalOptions.UninstallService) - { - SvcDelete(); + ZenWindowsService(const ZenWindowsService&) = delete; + ZenWindowsService& operator=(const ZenWindowsService&) = delete; - std::exit(0); - } -#endif + virtual int Run() override; + +private: + ZenServerOptions& m_GlobalOptions; + ZenServiceConfig& m_ServiceConfig; +}; + +int +ZenWindowsService::Run() +{ + using namespace zen; #if USE_SENTRY // Initialize sentry.io client @@ -474,6 +472,9 @@ main(int argc, char* argv[]) auto _ = zen::MakeGuard([] { sentry_close(); }); #endif + auto& GlobalOptions = m_GlobalOptions; + auto& ServiceConfig = m_ServiceConfig; + try { // Prototype config system, we'll see how this pans out @@ -559,3 +560,35 @@ main(int argc, char* argv[]) return 0; } + +int +main(int argc, char* argv[]) +{ + using namespace zen; + + mi_version(); + + ZenServerOptions GlobalOptions; + ZenServiceConfig ServiceConfig; + ParseGlobalCliOptions(argc, argv, GlobalOptions, ServiceConfig); + InitializeLogging(GlobalOptions); + +#if ZEN_PLATFORM_WINDOWS + if (GlobalOptions.InstallService) + { + WindowsService::Install(); + + std::exit(0); + } + + if (GlobalOptions.UninstallService) + { + WindowsService::Delete(); + + std::exit(0); + } +#endif + + ZenWindowsService App(GlobalOptions, ServiceConfig); + return App.ServiceMain(); +} -- cgit v1.2.3 From a79a7830472e9b994eacbb22c1a9fd223c4cc2c7 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 17 Sep 2021 23:32:05 +0200 Subject: Changed file logging to use a rotating log strategy --- zenserver/diag/logging.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp index 48c87dd68..1df35fc14 100644 --- a/zenserver/diag/logging.cpp +++ b/zenserver/diag/logging.cpp @@ -9,7 +9,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -224,7 +226,19 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) // Sinks auto ConsoleSink = std::make_shared(); - auto FileSink = std::make_shared(zen::WideToUtf8(LogPath.c_str()), /* truncate */ true); + +#if 0 + auto FileSink = std::make_shared(zen::WideToUtf8(LogPath.c_str()), + 0, + 0, + /* truncate */ false, + uint16_t(/* max files */ 14)); +#else + auto FileSink = std::make_shared(zen::WideToUtf8(LogPath.c_str()), + /* max size */ 128 * 1024 * 1024, + /* max files */ 16, + /* rotate on open */ true); +#endif // Default @@ -250,6 +264,8 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) spdlog::register_logger(JupiterLogger); JupiterLogger->set_level(LogLevel); + // Zen - only log HTTP traffic to file + auto ZenClientLogger = std::make_shared("zenclient", FileSink); spdlog::register_logger(ZenClientLogger); ZenClientLogger->set_level(LogLevel); @@ -258,6 +274,7 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) spdlog::set_level(LogLevel); spdlog::flush_on(spdlog::level::err); + spdlog::flush_every(std::chrono::seconds{2}); spdlog::set_formatter(std::make_unique(GlobalOptions.LogId, std::chrono::system_clock::now())); } -- cgit v1.2.3 From ab504946e92a261d6782748e7cd4f4cc70f98a25 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 17 Sep 2021 23:48:40 +0200 Subject: Eliminated some redundant logger configuration --- zenserver/diag/logging.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp index 1df35fc14..41b140f90 100644 --- a/zenserver/diag/logging.cpp +++ b/zenserver/diag/logging.cpp @@ -262,13 +262,11 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) auto JupiterLogger = std::make_shared("jupiter", FileSink); spdlog::register_logger(JupiterLogger); - JupiterLogger->set_level(LogLevel); // Zen - only log HTTP traffic to file auto ZenClientLogger = std::make_shared("zenclient", FileSink); spdlog::register_logger(ZenClientLogger); - ZenClientLogger->set_level(LogLevel); // Configure all registered loggers according to settings -- cgit v1.2.3 From 60898b9fcc57bdc0408f6a48de161a7989b91957 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sat, 18 Sep 2021 11:26:46 +0200 Subject: Simplified AnyUserSecurityAttributes helper --- zenutil/zenserverprocess.cpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/zenutil/zenserverprocess.cpp b/zenutil/zenserverprocess.cpp index c6bee1ed7..9e370ae10 100644 --- a/zenutil/zenserverprocess.cpp +++ b/zenutil/zenserverprocess.cpp @@ -40,20 +40,16 @@ namespace zenutil { m_Attributes.nLength = sizeof m_Attributes; m_Attributes.bInheritHandle = false; // Disable inheritance - const BOOL success = InitializeSecurityDescriptor(&m_Sd, SECURITY_DESCRIPTOR_REVISION); + const BOOL Success = InitializeSecurityDescriptor(&m_Sd, SECURITY_DESCRIPTOR_REVISION); - if (success) + if (Success) { - const BOOL bSetOk = SetSecurityDescriptorDacl(&m_Sd, TRUE, (PACL)NULL, FALSE); - - if (bSetOk) - { - m_Attributes.lpSecurityDescriptor = &m_Sd; - } - else + if (!SetSecurityDescriptorDacl(&m_Sd, TRUE, (PACL)NULL, FALSE)) { zen::ThrowLastError("SetSecurityDescriptorDacl failed", std::source_location::current()); } + + m_Attributes.lpSecurityDescriptor = &m_Sd; } } }; -- cgit v1.2.3 From 854cb25ac8dc66f553c2033b7e13dfe5b76cd4bc Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 17:56:31 +0200 Subject: Exclude build outputs from vs-chromium --- vs-chromium-project.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vs-chromium-project.txt b/vs-chromium-project.txt index 2b2e15bc7..2bb89a55c 100644 --- a/vs-chromium-project.txt +++ b/vs-chromium-project.txt @@ -3,6 +3,6 @@ [SourceExplorer.ignore] .git/ -.x64/ +x64/ *.suo -**/.x64/ +**/x64/ -- cgit v1.2.3 From 8b5213bd050634e17b6215cbe25228b4cc6128ef Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 18:35:56 +0200 Subject: Changed BasicFile implementation * No longer uses ATL on Windows (we just use raw Win32 API) * Added non-throwing Open() implementation * Added beginnings of a test suite, for verifying cross-platform implementation --- zenstore/basicfile.cpp | 90 ++++++++++++++++++++++++++++------- zenstore/include/zenstore/basicfile.h | 21 +++++--- 2 files changed, 88 insertions(+), 23 deletions(-) diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp index 35ccdd042..75de638cf 100644 --- a/zenstore/basicfile.cpp +++ b/zenstore/basicfile.cpp @@ -5,7 +5,9 @@ #include #include #include +#include +#include #include #include @@ -13,16 +15,54 @@ namespace zen { using namespace fmt::literals; +BasicFile::~BasicFile() +{ + Close(); +} + void -BasicFile::Open(std::filesystem::path FileName, bool isCreate) +BasicFile::Open(std::filesystem::path FileName, bool IsCreate) { - const DWORD dwCreationDisposition = isCreate ? CREATE_ALWAYS : OPEN_EXISTING; + std::error_code Ec; + Open(FileName, IsCreate, Ec); - HRESULT hRes = m_File.Create(FileName.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, dwCreationDisposition); + if (Ec) + { + throw std::system_error(Ec, "failed to open file '{}'"_format(FileName)); + } +} - if (FAILED(hRes)) +void +BasicFile::Open(std::filesystem::path FileName, bool IsCreate, std::error_code& Ec) +{ + const DWORD dwCreationDisposition = IsCreate ? CREATE_ALWAYS : OPEN_EXISTING; + const DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; + const DWORD dwShareMode = FILE_SHARE_READ; + const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; + HANDLE hTemplateFile = nullptr; + + HANDLE FileHandle = CreateFile(FileName.c_str(), + dwDesiredAccess, + dwShareMode, + /* lpSecurityAttributes */ nullptr, + dwCreationDisposition, + dwFlagsAndAttributes, + hTemplateFile); + + if (FileHandle == INVALID_HANDLE_VALUE) { - ThrowSystemException(hRes, "Failed to open bucket sobs file '{}'"_format(FileName)); + Ec = zen::MakeErrorCodeFromLastError(); + } + + m_FileHandle = FileHandle; +} + +void +BasicFile::Close() +{ + if (m_FileHandle) + { + ::CloseHandle(m_FileHandle); } } @@ -34,11 +74,14 @@ BasicFile::Read(void* Data, uint64_t Size, uint64_t Offset) Ovl.Offset = DWORD(Offset & 0xffff'ffffu); Ovl.OffsetHigh = DWORD(Offset >> 32); - HRESULT hRes = m_File.Read(Data, gsl::narrow(Size), &Ovl); + DWORD dwNumberOfBytesToRead = gsl::narrow(Size); + DWORD dwNumberOfBytesRead = 0; + + BOOL Success = ::ReadFile(m_FileHandle, Data, dwNumberOfBytesToRead, &dwNumberOfBytesRead, &Ovl); - if (FAILED(hRes)) + if (!Success) { - ThrowSystemException(hRes, "Failed to read from file '{}'"_format(zen::PathFromHandle(m_File))); + ThrowLastError("Failed to read from file '{}'"_format(zen::PathFromHandle(m_FileHandle))); } } @@ -60,33 +103,46 @@ BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset) Ovl.Offset = DWORD(Offset & 0xffff'ffffu); Ovl.OffsetHigh = DWORD(Offset >> 32); - HRESULT hRes = m_File.Write(Data, gsl::narrow(Size), &Ovl); + DWORD dwNumberOfBytesToWrite = gsl::narrow(Size); + DWORD dwNumberOfBytesWritten = 0; + + BOOL Success = ::WriteFile(m_FileHandle, Data, dwNumberOfBytesToWrite, &dwNumberOfBytesWritten, &Ovl); - if (FAILED(hRes)) + if (!Success) { - ThrowSystemException(hRes, "Failed to write to file '{}'"_format(zen::PathFromHandle(m_File))); + ThrowLastError("Failed to write to file '{}'"_format(zen::PathFromHandle(m_FileHandle))); } } void BasicFile::Flush() { - m_File.Flush(); + FlushFileBuffers(m_FileHandle); } uint64_t BasicFile::FileSize() { - ULONGLONG Sz; - m_File.GetSize(Sz); + ULARGE_INTEGER liFileSize; + liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart); - return uint64_t(Sz); + return uint64_t(liFileSize.QuadPart); +} + +TEST_CASE("BasicFile") +{ + ScopedCurrentDirectoryChange _; + + BasicFile File1; + CHECK_THROWS(File1.Open("zonk", false)); + CHECK_NOTHROW(File1.Open("zonk", true)); + CHECK_NOTHROW(File1.Write("abcd", 4, 0)); + CHECK(File1.FileSize() == 4); } void -BasicFile::Close() +basicfile_forcelink() { - m_File.Close(); } } // namespace zen diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h index c6f61d466..0daa5d39f 100644 --- a/zenstore/include/zenstore/basicfile.h +++ b/zenstore/include/zenstore/basicfile.h @@ -2,34 +2,43 @@ #pragma once -#include -#include +#include "zenstore.h" +#include #include -#include #include namespace zen { /** * Probably the most basic file abstraction in the universe + * + * One thing of note is that there is no notion of a "current file position" + * in this API -- all reads and writes are done from explicit offsets in + * the file. This avoids concurrency issues which can occur otherwise. + * */ class BasicFile { public: + BasicFile() = default; + ~BasicFile(); void Open(std::filesystem::path FileName, bool IsCreate); + void Open(std::filesystem::path FileName, bool IsCreate, std::error_code& Ec); + void Close(); void Read(void* Data, uint64_t Size, uint64_t Offset); void Write(const void* Data, uint64_t Size, uint64_t Offset); void Flush(); uint64_t FileSize(); - void* Handle() { return m_File; } - void Close(); + void* Handle() { return m_FileHandle; } IoBuffer ReadAll(); private: - CAtlFile m_File; + void* m_FileHandle = nullptr; // This is either null or valid }; +ZENCORE_API void basicfile_forcelink(); + } // namespace zen -- cgit v1.2.3 From f920793c7585a11c60346d306108f90368878d32 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 18:36:50 +0200 Subject: Added testutils for use in writing tests Currently contains helpers for managing temporary directories used in tests --- zencore/include/zencore/testutils.h | 31 +++++++++++++++++++++++++++++++ zencore/testutils.cpp | 33 +++++++++++++++++++++++++++++++++ zencore/zencore.vcxproj | 2 ++ zencore/zencore.vcxproj.filters | 2 ++ 4 files changed, 68 insertions(+) create mode 100644 zencore/include/zencore/testutils.h create mode 100644 zencore/testutils.cpp diff --git a/zencore/include/zencore/testutils.h b/zencore/include/zencore/testutils.h new file mode 100644 index 000000000..72d985d5c --- /dev/null +++ b/zencore/include/zencore/testutils.h @@ -0,0 +1,31 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +namespace zen { + +std::filesystem::path CreateTemporaryDirectory(); + +class ScopedTemporaryDirectory +{ +public: + ScopedTemporaryDirectory(); + ~ScopedTemporaryDirectory(); + + std::filesystem::path& Path() { return m_RootPath; } + +private: + std::filesystem::path m_RootPath; +}; + +struct ScopedCurrentDirectoryChange +{ + std::filesystem::path OldPath{std::filesystem::current_path()}; + + ScopedCurrentDirectoryChange() { std::filesystem::current_path(CreateTemporaryDirectory()); } + ~ScopedCurrentDirectoryChange() { std::filesystem::current_path(OldPath); } +}; + +} // namespace zen diff --git a/zencore/testutils.cpp b/zencore/testutils.cpp new file mode 100644 index 000000000..116491950 --- /dev/null +++ b/zencore/testutils.cpp @@ -0,0 +1,33 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencore/testutils.h" +#include +#include "zencore/string.h" + +namespace zen { + +static std::atomic Sequence{0}; + +std::filesystem::path +CreateTemporaryDirectory() +{ + std::error_code Ec; + + std::filesystem::path DirPath = std::filesystem::temp_directory_path() / GetSessionIdString() / IntNum(++Sequence).c_str(); + std::filesystem::remove_all(DirPath, Ec); + std::filesystem::create_directories(DirPath); + + return DirPath; +} + +ScopedTemporaryDirectory::ScopedTemporaryDirectory() : m_RootPath(CreateTemporaryDirectory()) +{ +} + +ScopedTemporaryDirectory::~ScopedTemporaryDirectory() +{ + std::error_code Ec; + std::filesystem::remove_all(m_RootPath, Ec); +} + +} // namespace zen \ No newline at end of file diff --git a/zencore/zencore.vcxproj b/zencore/zencore.vcxproj index 150c42cd6..59b0bafcc 100644 --- a/zencore/zencore.vcxproj +++ b/zencore/zencore.vcxproj @@ -142,6 +142,7 @@ + @@ -181,6 +182,7 @@ + diff --git a/zencore/zencore.vcxproj.filters b/zencore/zencore.vcxproj.filters index ea0f8a912..6c6b308f8 100644 --- a/zencore/zencore.vcxproj.filters +++ b/zencore/zencore.vcxproj.filters @@ -42,6 +42,7 @@ + @@ -74,6 +75,7 @@ + -- cgit v1.2.3 From 2eab660b90a760dd9d249d7b01324a2bd08229d1 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 18:37:39 +0200 Subject: Removed unnecessary ATL includes --- zen/zen.h | 1 - 1 file changed, 1 deletion(-) diff --git a/zen/zen.h b/zen/zen.h index c90f3169a..1c8d102d3 100644 --- a/zen/zen.h +++ b/zen/zen.h @@ -10,7 +10,6 @@ #include #include -#include #include struct ZenCliOptions -- cgit v1.2.3 From ed9d27f5e7b2e8eda835e67af302f097e5932366 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 18:38:31 +0200 Subject: Added GetSessionIdString() which returns a text representation of the session id --- zencore/include/zencore/session.h | 3 ++- zencore/include/zencore/uid.h | 1 + zencore/session.cpp | 15 ++++++++++++++- zencore/uid.cpp | 8 +++++++- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/zencore/include/zencore/session.h b/zencore/include/zencore/session.h index 2da41b2c8..dd90197bf 100644 --- a/zencore/include/zencore/session.h +++ b/zencore/include/zencore/session.h @@ -8,6 +8,7 @@ namespace zen { struct Oid; -ZENCORE_API Oid GetSessionId(); +ZENCORE_API [[nodiscard]] Oid GetSessionId(); +ZENCORE_API [[nodiscard]] std::string_view GetSessionIdString(); } // namespace zen diff --git a/zencore/include/zencore/uid.h b/zencore/include/zencore/uid.h index f095c49ef..f4e9ab65a 100644 --- a/zencore/include/zencore/uid.h +++ b/zencore/include/zencore/uid.h @@ -60,6 +60,7 @@ struct Oid const Oid& Generate(); [[nodiscard]] static Oid FromHexString(const std::string_view String); StringBuilderBase& ToString(StringBuilderBase& OutString) const; + void ToString(char OutString[StringLength]); [[nodiscard]] static Oid FromMemory(const void* Ptr); auto operator<=>(const Oid& rhs) const = default; diff --git a/zencore/session.cpp b/zencore/session.cpp index d57d3685b..ce4bfae1b 100644 --- a/zencore/session.cpp +++ b/zencore/session.cpp @@ -9,14 +9,27 @@ namespace zen { static Oid GlobalSessionId; +static char GlobalSessionString[Oid::StringLength]; static std::once_flag SessionInitFlag; Oid GetSessionId() { - std::call_once(SessionInitFlag, [&] { GlobalSessionId.Generate(); }); + std::call_once(SessionInitFlag, [&] { + GlobalSessionId.Generate(); + GlobalSessionId.ToString(GlobalSessionString); + }); return GlobalSessionId; } +std::string_view +GetSessionIdString() +{ + // Ensure we actually have a generated session identifier + std::ignore = GetSessionId(); + + return std::string_view(GlobalSessionString, Oid::StringLength); +} + } // namespace zen \ No newline at end of file diff --git a/zencore/uid.cpp b/zencore/uid.cpp index d946638ec..ed00b1814 100644 --- a/zencore/uid.cpp +++ b/zencore/uid.cpp @@ -91,10 +91,16 @@ Oid::FromMemory(const void* Ptr) return Id; } +void +Oid::ToString(char OutString[StringLength]) +{ + ToHexBytes(reinterpret_cast(OidBits), sizeof(Oid::OidBits), OutString); +} + StringBuilderBase& Oid::ToString(StringBuilderBase& OutString) const { - char str[25]; + char str[StringLength + 1]; ToHexBytes(reinterpret_cast(OidBits), sizeof(Oid::OidBits), str); str[2 * sizeof(Oid)] = '\0'; -- cgit v1.2.3 From e7e26b8d581229de469cd309a5fea7ba39d84f55 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 19:25:23 +0200 Subject: Added module cpp/h for zenstore --- zenstore/zenstore.cpp | 17 +++++++++++++++++ zenstore/zenstore.vcxproj | 5 +++++ zenstore/zenstore.vcxproj.filters | 5 +++++ 3 files changed, 27 insertions(+) create mode 100644 zenstore/zenstore.cpp diff --git a/zenstore/zenstore.cpp b/zenstore/zenstore.cpp new file mode 100644 index 000000000..cd16e5634 --- /dev/null +++ b/zenstore/zenstore.cpp @@ -0,0 +1,17 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zenstore/zenstore.h" + +#include +#include + +namespace zen { + +void +zenstore_forcelinktests() +{ + basicfile_forcelink(); + CAS_forcelink(); +} + +} // namespace zen diff --git a/zenstore/zenstore.vcxproj b/zenstore/zenstore.vcxproj index 8d665f2c3..eb2ecd02b 100644 --- a/zenstore/zenstore.vcxproj +++ b/zenstore/zenstore.vcxproj @@ -19,6 +19,7 @@ + @@ -29,12 +30,16 @@ + {d75bf9ab-c61e-4fff-ad59-1563430f05e2} + + + 16.0 Win32Proj diff --git a/zenstore/zenstore.vcxproj.filters b/zenstore/zenstore.vcxproj.filters index 3dfb89dbf..8a52c69f6 100644 --- a/zenstore/zenstore.vcxproj.filters +++ b/zenstore/zenstore.vcxproj.filters @@ -9,6 +9,7 @@ + @@ -19,5 +20,9 @@ + + + + \ No newline at end of file -- cgit v1.2.3 From 347352b5760f42a026fe8edaf7cef1deee64ae7c Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 19:25:53 +0200 Subject: Added zenstore-test test driver for zenstore tests --- xmake.lua | 3 +- zen.sln | 8 ++ zenstore-test/xmake.lua | 5 ++ zenstore-test/zenstore-test.cpp | 23 ++++++ zenstore-test/zenstore-test.vcxproj | 121 ++++++++++++++++++++++++++++ zenstore-test/zenstore-test.vcxproj.filters | 9 +++ 6 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 zenstore-test/xmake.lua create mode 100644 zenstore-test/zenstore-test.cpp create mode 100644 zenstore-test/zenstore-test.vcxproj create mode 100644 zenstore-test/zenstore-test.vcxproj.filters diff --git a/xmake.lua b/xmake.lua index d18962126..8288963e0 100644 --- a/xmake.lua +++ b/xmake.lua @@ -64,6 +64,7 @@ set_symbols("debug") includes("zencore", "zencore-test") includes("zenhttp") -includes("zenstore", "zenutil") +includes("zenstore", "zenstore-test") +includes("zenutil") includes("zenserver", "zenserver-test") includes("zen") diff --git a/zen.sln b/zen.sln index e917d5b74..052c030ae 100644 --- a/zen.sln +++ b/zen.sln @@ -48,6 +48,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zentest-appstub", "zentest- EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zenhttp", "zenhttp\zenhttp.vcxproj", "{8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zenstore-test", "zenstore-test\zenstore-test.vcxproj", "{C001A3DF-B76E-4989-B576-FE2B78AB2580}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -112,6 +114,12 @@ Global {8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Release|x64.ActiveCfg = Release|x64 {8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Release|x64.Build.0 = Release|x64 {8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Release|x86.ActiveCfg = Release|x64 + {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Debug|x64.ActiveCfg = Debug|x64 + {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Debug|x64.Build.0 = Debug|x64 + {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Debug|x86.ActiveCfg = Debug|x64 + {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Release|x64.ActiveCfg = Release|x64 + {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Release|x64.Build.0 = Release|x64 + {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Release|x86.ActiveCfg = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/zenstore-test/xmake.lua b/zenstore-test/xmake.lua new file mode 100644 index 000000000..c8995dab2 --- /dev/null +++ b/zenstore-test/xmake.lua @@ -0,0 +1,5 @@ +target("zenstore-test") + set_kind("binary") + add_files("*.cpp") + add_deps("zenstore", "zencore") + add_packages("vcpkg::doctest") diff --git a/zenstore-test/zenstore-test.cpp b/zenstore-test/zenstore-test.cpp new file mode 100644 index 000000000..ed9a12566 --- /dev/null +++ b/zenstore-test/zenstore-test.cpp @@ -0,0 +1,23 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include +#include +#include + +#define DOCTEST_CONFIG_IMPLEMENT +#include +#undef DOCTEST_CONFIG_IMPLEMENT + +void +forceLinkTests() +{ + zen::zenstore_forcelinktests(); +} + +int +main(int argc, char* argv[]) +{ + zen::logging::InitializeLogging(); + + return doctest::Context(argc, argv).run(); +} diff --git a/zenstore-test/zenstore-test.vcxproj b/zenstore-test/zenstore-test.vcxproj new file mode 100644 index 000000000..201594a25 --- /dev/null +++ b/zenstore-test/zenstore-test.vcxproj @@ -0,0 +1,121 @@ + + + + + Debug + x64 + + + Release + x64 + + + + 15.0 + {C001A3DF-B76E-4989-B576-FE2B78AB2580} + Win32Proj + zenstoretest + 10.0 + + + + Application + true + v142 + Unicode + + + Application + false + v142 + false + Unicode + + + + + + + + + + + + + + + + + true + + + false + + + true + true + --overlay-ports=$(SolutionDir)vcpkg_overlay-ports + + + true + true + --overlay-ports=$(SolutionDir)vcpkg_overlay-ports + + + + NotUsing + Disabled + true + _DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + stdcpplatest + MultiThreadedDebug + + + Console + true + kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + %(AdditionalLibraryDirectories) + + + + + + NotUsing + MaxSpeed + true + true + true + NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + false + stdcpplatest + MultiThreaded + + + Console + true + true + true + NotSet + + + + + + + + + + + + {d75bf9ab-c61e-4fff-ad59-1563430f05e2} + + + {26cbbaeb-14c1-4efc-877d-80f48215651c} + + + + + + \ No newline at end of file diff --git a/zenstore-test/zenstore-test.vcxproj.filters b/zenstore-test/zenstore-test.vcxproj.filters new file mode 100644 index 000000000..000599c79 --- /dev/null +++ b/zenstore-test/zenstore-test.vcxproj.filters @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file -- cgit v1.2.3 From 32d040abb65d323ed590716254a35d5d0afaad1a Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 19:26:12 +0200 Subject: Added generated makefiles to .gitignore --- .gitignore | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index df6c1a9f8..bf237a7b3 100644 --- a/.gitignore +++ b/.gitignore @@ -204,9 +204,8 @@ ServiceFabricBackup/ *.rptproj.bak - - - +# generated build files +makefile # Python Tools for Visual Studio (PTVS) __pycache__/ -- cgit v1.2.3 From ff3f83e399ba6435636c0f5780baf6b308787593 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 19:27:57 +0200 Subject: Added zenstore.h and made headers use it --- zenstore/include/zenstore/caslog.h | 2 +- zenstore/include/zenstore/cidstore.h | 2 ++ zenstore/include/zenstore/scrub.h | 2 ++ zenstore/include/zenstore/zenstore.h | 13 +++++++++++++ 4 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 zenstore/include/zenstore/zenstore.h diff --git a/zenstore/include/zenstore/caslog.h b/zenstore/include/zenstore/caslog.h index aea855e4c..3d558bee0 100644 --- a/zenstore/include/zenstore/caslog.h +++ b/zenstore/include/zenstore/caslog.h @@ -2,7 +2,7 @@ #pragma once -#include +#include "zenstore.h" #include #include diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h index 76a33c915..f023ada40 100644 --- a/zenstore/include/zenstore/cidstore.h +++ b/zenstore/include/zenstore/cidstore.h @@ -2,6 +2,8 @@ #pragma once +#include "zenstore.h" + #include #include #include diff --git a/zenstore/include/zenstore/scrub.h b/zenstore/include/zenstore/scrub.h index 5a34d4860..4948afcd5 100644 --- a/zenstore/include/zenstore/scrub.h +++ b/zenstore/include/zenstore/scrub.h @@ -2,6 +2,8 @@ #pragma once +#include "zenstore.h" + #include #include diff --git a/zenstore/include/zenstore/zenstore.h b/zenstore/include/zenstore/zenstore.h new file mode 100644 index 000000000..46d62029d --- /dev/null +++ b/zenstore/include/zenstore/zenstore.h @@ -0,0 +1,13 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#define ZENSTORE_API + +namespace zen { + +ZENSTORE_API void zenstore_forcelinktests(); + +} -- cgit v1.2.3 From 8f82467ea5e8e90e459d78d603c67a7938ae8ead Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 19:30:16 +0200 Subject: Changed some code over from ATL to BasicFile and added Scrub() stubs. --- zenstore/CAS.cpp | 85 ++++++++++++++++++++++++++++++----------- zenstore/compactcas.cpp | 43 +++++++++++++++++---- zenstore/compactcas.h | 11 ++++-- zenstore/filecas.cpp | 5 +++ zenstore/filecas.h | 1 + zenstore/include/zenstore/CAS.h | 3 +- 6 files changed, 113 insertions(+), 35 deletions(-) diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp index e77c0ed64..af0fcc609 100644 --- a/zenstore/CAS.cpp +++ b/zenstore/CAS.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -20,15 +21,17 @@ #include #include -struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive- -#include - ////////////////////////////////////////////////////////////////////////// namespace zen { /** - * Slightly less naive CAS store + * CAS store implementation + * + * Uses a basic strategy of splitting payloads by size, to improve ability to reclaim space + * quickly for unused large chunks and to maintain locality for small chunks which are + * frequently accessed together. + * */ class CasImpl : public CasStore { @@ -41,10 +44,9 @@ public: virtual IoBuffer FindChunk(const IoHash& ChunkHash) override; virtual void FilterChunks(CasChunkSet& InOutChunks) override; virtual void Flush() override; + virtual void Scrub() override; private: - void PickDefaultDirectory(); - CasContainerStrategy m_TinyStrategy; CasContainerStrategy m_SmallStrategy; FileCasStrategy m_LargeStrategy; @@ -63,13 +65,16 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) { m_Config = InConfig; - ZEN_INFO("initializing CAS pool at {}", m_Config.RootDirectory); + ZEN_INFO("initializing CAS pool at '{}'", m_Config.RootDirectory); // Ensure root directory exists - create if it doesn't exist already std::filesystem::create_directories(m_Config.RootDirectory); // Open or create manifest + // + // The manifest is not currently fully implemented. The goal is to + // use it for recovery and configuration bool IsNewStore = false; @@ -77,23 +82,22 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) std::filesystem::path ManifestPath = m_Config.RootDirectory; ManifestPath /= ".ucas_root"; - CAtlFile marker; - HRESULT hRes = marker.Create(ManifestPath.c_str(), GENERIC_READ, 0, OPEN_EXISTING); + std::error_code Ec; + BasicFile Marker; + Marker.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec); - if (FAILED(hRes)) + if (Ec) { IsNewStore = true; ExtendableStringBuilder<128> manifest; - manifest.Append("#CAS_ROOT\n"); // TODO: should write something meaningful here + manifest.Append("#CAS_ROOT\n"); manifest.Append("ID="); zen::Oid id = zen::Oid::NewOid(); id.ToString(manifest); - hRes = marker.Create(ManifestPath.c_str(), GENERIC_WRITE, 0, CREATE_ALWAYS); - - if (SUCCEEDED(hRes)) - marker.Write(manifest.c_str(), (DWORD)manifest.Size()); + Marker.Open(ManifestPath.c_str(), /* IsCreate */ true); + Marker.Write(manifest.c_str(), (DWORD)manifest.Size(), 0); } } @@ -160,6 +164,14 @@ CasImpl::Flush() m_LargeStrategy.Flush(); } +void +CasImpl::Scrub() +{ + m_SmallStrategy.Scrub(); + m_TinyStrategy.Scrub(); + m_LargeStrategy.Scrub(); +} + ////////////////////////////////////////////////////////////////////////// CasStore* @@ -173,18 +185,45 @@ CreateCasStore() // Testing related code follows... // -void -CAS_forcelink() -{ -} - TEST_CASE("CasStore") { + ScopedTemporaryDirectory TempDir; + zen::CasStoreConfiguration config; - config.RootDirectory = "c:\\temp\\test"; + config.RootDirectory = TempDir.Path(); + + std::unique_ptr Store{CreateCasStore()}; + Store->Initialize(config); + Store->Scrub(); + + IoBuffer Value1{16}; + memcpy(Value1.MutableData(), "1234567890123456", 16); + IoHash Hash1 = IoHash::HashBuffer(Value1.Data(), Value1.Size()); + CasStore::InsertResult Result1 = Store->InsertChunk(Value1, Hash1); + CHECK(Result1.New); + + IoBuffer Value2{16}; + memcpy(Value2.MutableData(), "ABCDEFGHIJKLMNOP", 16); + IoHash Hash2 = IoHash::HashBuffer(Value2.Data(), Value2.Size()); + CasStore::InsertResult Result2 = Store->InsertChunk(Value2, Hash2); + CHECK(Result2.New); + + CasChunkSet ChunkSet; + ChunkSet.AddChunk(Hash1); + ChunkSet.AddChunk(Hash2); + + Store->FilterChunks(ChunkSet); + CHECK(ChunkSet.GetChunkSet().size() == 0); + + IoBuffer Lookup1 = Store->FindChunk(Hash1); + CHECK(Lookup1); + IoBuffer Lookup2 = Store->FindChunk(Hash2); + CHECK(Lookup2); +} - std::unique_ptr store{CreateCasStore()}; - store->Initialize(config); +void +CAS_forcelink() +{ } } // namespace zen diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp index 4407d8b08..71d52e56a 100644 --- a/zenstore/compactcas.cpp +++ b/zenstore/compactcas.cpp @@ -10,13 +10,9 @@ #include #include -#include - -#include - -struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive- -#include #include +#include +#include ////////////////////////////////////////////////////////////////////////// @@ -43,7 +39,9 @@ CasContainerStrategy::Initialize(const std::string_view ContainerBaseName, uint6 uint64_t MaxFileOffset = 0; { - // This is not technically necessary but may help future static analysis + // This is not technically necessary (nobody should be accessing us from + // another thread at this stage) but may help static analysis + zen::RwLock::ExclusiveLockScope _(m_LocationMapLock); m_CasLog.Replay([&](const CasDiskIndexEntry& Record) { @@ -133,6 +131,13 @@ CasContainerStrategy::HaveChunk(const IoHash& ChunkHash) void CasContainerStrategy::FilterChunks(CasChunkSet& InOutChunks) { + // This implementation is good enough for relatively small + // chunk sets (in terms of chunk identifiers), but would + // benefit from a better implementation which removes + // items incrementally for large sets, especially when + // we're likely to already have a large proportion of the + // chunks in the set + std::unordered_set HaveSet; for (const IoHash& Hash : InOutChunks.GetChunkSet()) @@ -157,4 +162,28 @@ CasContainerStrategy::Flush() m_SmallObjectFile.Flush(); } +void +CasContainerStrategy::Scrub() +{ + RwLock::SharedLockScope _(m_LocationMapLock); +} + +void +CasContainerStrategy::MakeSnapshot() +{ + RwLock::SharedLockScope _(m_LocationMapLock); + + std::vector Entries{m_LocationMap.size()}; + + uint64_t EntryIndex = 0; + for (auto& Entry : m_LocationMap) + { + CasDiskIndexEntry& IndexEntry = Entries[EntryIndex++]; + IndexEntry.Key = Entry.first; + IndexEntry.Location = Entry.second; + } + + m_SmallObjectIndex.Write(Entries.data(), Entries.size() * sizeof(CasDiskIndexEntry), 0); +} + } // namespace zen diff --git a/zenstore/compactcas.h b/zenstore/compactcas.h index 05bbf81f6..63d8f8511 100644 --- a/zenstore/compactcas.h +++ b/zenstore/compactcas.h @@ -14,9 +14,6 @@ #include #include -#include -#include - namespace zen { ////////////////////////////////////////////////////////////////////////// @@ -27,7 +24,10 @@ namespace zen { struct CasDiskLocation { uint64_t Offset; - uint32_t Size; // TODO: Make this more like the IoStore index so we can store larger chunks (should be five bytes) + // If we wanted to be able to store larger chunks using this storage mechanism then + // we could make this more like the IoStore index so we can store larger chunks. + // I.e use five bytes for size and seven for offset + uint32_t Size; }; struct CasDiskIndexEntry @@ -58,6 +58,7 @@ struct CasContainerStrategy void FilterChunks(CasChunkSet& InOutChunks); void Initialize(const std::string_view ContainerBaseName, uint64_t Alignment, bool IsNewStore); void Flush(); + void Scrub(); private: const CasStoreConfiguration& m_Config; @@ -73,6 +74,8 @@ private: RwLock m_InsertLock; // used to serialize inserts std::atomic m_CurrentInsertOffset = 0; std::atomic m_CurrentIndexOffset = 0; + + void MakeSnapshot(); }; } // namespace zen diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp index 170f13875..5fdf505d4 100644 --- a/zenstore/filecas.cpp +++ b/zenstore/filecas.cpp @@ -352,6 +352,11 @@ FileCasStrategy::Flush() // maintain a log of when chunks were created } +void +FileCasStrategy::Scrub() +{ +} + void FileCasStrategy::GarbageCollect(GcContext& GcCtx) { diff --git a/zenstore/filecas.h b/zenstore/filecas.h index 448d1a05f..c7cd9d7ca 100644 --- a/zenstore/filecas.h +++ b/zenstore/filecas.h @@ -22,6 +22,7 @@ struct FileCasStrategy void FilterChunks(CasChunkSet& InOutChunks); void Flush(); void GarbageCollect(GcContext& GcCtx); + void Scrub(); private: const CasStoreConfiguration& m_Config; diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/CAS.h index b4de533dd..c6c919593 100644 --- a/zenstore/include/zenstore/CAS.h +++ b/zenstore/include/zenstore/CAS.h @@ -2,7 +2,7 @@ #pragma once -#include +#include "zenstore.h" #include #include @@ -76,6 +76,7 @@ public: virtual IoBuffer FindChunk(const IoHash& ChunkHash) = 0; virtual void FilterChunks(CasChunkSet& InOutChunks) = 0; virtual void Flush() = 0; + virtual void Scrub() = 0; protected: CasStoreConfiguration m_Config; -- cgit v1.2.3 From 1e5b41352c7d45f34cb1a51e3d3811a4a994592a Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 19:31:13 +0200 Subject: clang-format again --- zenhttp/httpsys.cpp | 2 +- zenserver/windows/service.cpp | 2 +- zenstore/compactcas.cpp | 4 ++-- zenstore/compactcas.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index b5313021c..3c5e5d8d3 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -716,7 +716,7 @@ HttpSysServer::Run(bool IsInteractive) do { - //int WaitTimeout = -1; + // int WaitTimeout = -1; int WaitTimeout = 100; if (IsInteractive) diff --git a/zenserver/windows/service.cpp b/zenserver/windows/service.cpp index bd80e0c2c..017b5f9a7 100644 --- a/zenserver/windows/service.cpp +++ b/zenserver/windows/service.cpp @@ -560,7 +560,7 @@ void WindowsService::SvcCtrlHandler(DWORD dwCtrl) { // Handle the requested control code. - // + // // Called by SCM whenever a control code is sent to the service // using the ControlService function. diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp index 71d52e56a..fd223e284 100644 --- a/zenstore/compactcas.cpp +++ b/zenstore/compactcas.cpp @@ -179,8 +179,8 @@ CasContainerStrategy::MakeSnapshot() for (auto& Entry : m_LocationMap) { CasDiskIndexEntry& IndexEntry = Entries[EntryIndex++]; - IndexEntry.Key = Entry.first; - IndexEntry.Location = Entry.second; + IndexEntry.Key = Entry.first; + IndexEntry.Location = Entry.second; } m_SmallObjectIndex.Write(Entries.data(), Entries.size() * sizeof(CasDiskIndexEntry), 0); diff --git a/zenstore/compactcas.h b/zenstore/compactcas.h index 63d8f8511..9921c9e6c 100644 --- a/zenstore/compactcas.h +++ b/zenstore/compactcas.h @@ -27,7 +27,7 @@ struct CasDiskLocation // If we wanted to be able to store larger chunks using this storage mechanism then // we could make this more like the IoStore index so we can store larger chunks. // I.e use five bytes for size and seven for offset - uint32_t Size; + uint32_t Size; }; struct CasDiskIndexEntry -- cgit v1.2.3 From c86315ea955408659eb7ea32798693975e27b9b7 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 22:35:12 +0200 Subject: Changed so Windows also uses the portable std::mutex implementation and reworked some code which would not compile after the change --- zencore/include/zencore/thread.h | 8 +----- zencore/thread.cpp | 16 ------------ zenserver-test/zenserver-test.cpp | 52 ++++++++++++++++++++------------------- 3 files changed, 28 insertions(+), 48 deletions(-) diff --git a/zencore/include/zencore/thread.h b/zencore/include/zencore/thread.h index e65867ec4..7889682cd 100644 --- a/zencore/include/zencore/thread.h +++ b/zencore/include/zencore/thread.h @@ -4,9 +4,7 @@ #include "zencore.h" -#if !ZEN_PLATFORM_WINDOWS -# include -#endif +#include #include @@ -66,11 +64,7 @@ public: }; private: -#if ZEN_PLATFORM_WINDOWS - void* m_Srw = nullptr; -#else std::shared_mutex m_Mutex; -#endif }; /** Basic abstraction of a simple event synchronization mechanism (aka 'binary semaphore') diff --git a/zencore/thread.cpp b/zencore/thread.cpp index c92cca6de..d4f101454 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -17,41 +17,25 @@ namespace zen { void RwLock::AcquireShared() { -#if ZEN_PLATFORM_WINDOWS - AcquireSRWLockShared((PSRWLOCK)&m_Srw); -#else m_Mutex.lock_shared(); -#endif } void RwLock::ReleaseShared() { -#if ZEN_PLATFORM_WINDOWS - ReleaseSRWLockShared((PSRWLOCK)&m_Srw); -#else m_Mutex.unlock_shared(); -#endif } void RwLock::AcquireExclusive() { -#if ZEN_PLATFORM_WINDOWS - AcquireSRWLockExclusive((PSRWLOCK)&m_Srw); -#else m_Mutex.lock(); -#endif } void RwLock::ReleaseExclusive() { -#if ZEN_PLATFORM_WINDOWS - ReleaseSRWLockExclusive((PSRWLOCK)&m_Srw); -#else m_Mutex.unlock(); -#endif } Event::Event() diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index 7629e2ea1..a556d896e 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -37,6 +38,7 @@ #include #include #include +#include #include #include @@ -80,7 +82,7 @@ class HttpClientConnection static HttpClientConnection* This(http_parser* Parser) { return (HttpClientConnection*)Parser->data; }; public: - HttpClientConnection(asio::io_context& IoContext, HttpConnectionPool& Pool, asio::ip::tcp::socket&& InSocket) + HttpClientConnection(asio::io_context& IoContext, zen::Ref Pool, asio::ip::tcp::socket&& InSocket) : m_IoContext(IoContext) , m_Pool(Pool) , m_Resolver(IoContext) @@ -89,8 +91,8 @@ public: } ~HttpClientConnection() {} - HttpConnectionPool& ConnectionPool() { return m_Pool; } - void SetKeepAlive(bool NewState) { m_KeepAlive = NewState; } + zen::Ref ConnectionPool() { return m_Pool; } + void SetKeepAlive(bool NewState) { m_KeepAlive = NewState; } void Get(const std::string_view Server, int Port, const std::string_view Path) { @@ -227,16 +229,16 @@ private: } private: - asio::io_context& m_IoContext; - HttpConnectionPool& m_Pool; - asio::ip::tcp::resolver m_Resolver; - asio::ip::tcp::socket m_Socket; - std::string m_Uri; - std::string m_RequestBody; // Initial request data - http_parser m_HttpParser{}; - http_parser_settings m_HttpParserSettings{}; - uint8_t m_ResponseIoBuffer[4096]; - asio::mutable_buffer m_ResponseBuffer{m_ResponseIoBuffer, sizeof m_ResponseIoBuffer}; + asio::io_context& m_IoContext; + zen::Ref m_Pool; + asio::ip::tcp::resolver m_Resolver; + asio::ip::tcp::socket m_Socket; + std::string m_Uri; + std::string m_RequestBody; // Initial request data + http_parser m_HttpParser{}; + http_parser_settings m_HttpParserSettings{}; + uint8_t m_ResponseIoBuffer[4096]; + asio::mutable_buffer m_ResponseBuffer{m_ResponseIoBuffer, sizeof m_ResponseIoBuffer}; enum class RequestState { @@ -259,7 +261,7 @@ private: ////////////////////////////////////////////////////////////////////////// -class HttpConnectionPool +class HttpConnectionPool : public zen::RefCounted { public: HttpConnectionPool(asio::io_context& Context, std::string_view HostName, uint16_t Port); @@ -322,7 +324,7 @@ HttpConnectionPool::GetConnection() return nullptr; } - return std::make_unique(m_Context, *this, std::move(Socket)); + return std::make_unique(m_Context, this, std::move(Socket)); } std::unique_ptr Connection{m_AvailableConnections.back()}; @@ -347,15 +349,15 @@ public: std::unique_ptr GetConnection(std::string_view HostName, uint16_t Port) { - return ConnectionPool(HostName, Port).GetConnection(); + return ConnectionPool(HostName, Port)->GetConnection(); } void ReturnConnection(std::unique_ptr Connection) { - Connection->ConnectionPool().ReturnConnection(std::move(Connection)); + Connection->ConnectionPool()->ReturnConnection(std::move(Connection)); } - HttpConnectionPool& ConnectionPool(std::string_view HostName, uint16_t Port) + zen::Ref ConnectionPool(std::string_view HostName, uint16_t Port) { zen::RwLock::ExclusiveLockScope _(m_Lock); ConnectionId ConnId{std::string(HostName), Port}; @@ -364,7 +366,7 @@ public: { // Not found - create new entry - auto In = m_ConnectionPools.insert({ConnId, std::move(HttpConnectionPool(m_Context, HostName, Port))}); + auto In = m_ConnectionPools.emplace(ConnId, new HttpConnectionPool(m_Context, HostName, Port)); return In.first->second; } @@ -393,8 +395,8 @@ private: uint16_t Port; }; - zen::RwLock m_Lock; - std::map m_ConnectionPools; + zen::RwLock m_Lock; + std::map> m_ConnectionPools; }; ////////////////////////////////////////////////////////////////////////// @@ -670,7 +672,7 @@ main(int argc, char** argv) zen::logging::InitializeLogging(); spdlog::set_level(spdlog::level::debug); - spdlog::set_formatter(std::make_unique< ::logging::full_formatter>("test", std::chrono::system_clock::now())); + spdlog::set_formatter(std::make_unique<::logging::full_formatter>("test", std::chrono::system_clock::now())); std::filesystem::path ProgramBaseDir = std::filesystem::path(argv[0]).parent_path(); std::filesystem::path TestBaseDir = ProgramBaseDir.parent_path().parent_path() / ".test"; @@ -1919,9 +1921,9 @@ public: ZenServerInstance& GetInstance(int Index) { return *m_Instances[Index]; } private: - std::string m_HelperId; - int m_ServerCount = 0; - std::vector > m_Instances; + std::string m_HelperId; + int m_ServerCount = 0; + std::vector> m_Instances; }; TEST_CASE("http.basics") -- cgit v1.2.3 From fe068c76e0f38dabf80bff2730ff9c713763d707 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 23:17:44 +0200 Subject: Added BasicFile::StreamFile helper function to support reading large files in a chunked fashion (will be using memory-mapped strategy in the future where it makes sense) --- zenstore/basicfile.cpp | 23 +++++++++++++++++++++++ zenstore/include/zenstore/basicfile.h | 6 ++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp index 75de638cf..126f2a8b3 100644 --- a/zenstore/basicfile.cpp +++ b/zenstore/basicfile.cpp @@ -95,6 +95,29 @@ BasicFile::ReadAll() return Buffer; } +void +BasicFile::StreamFile(std::function&& ChunkFun) +{ + const uint64_t ChunkSize = 128 * 1024; + IoBuffer ReadBuffer{ChunkSize}; + void* BufferPtr = ReadBuffer.MutableData(); + + uint64_t RemainBytes = FileSize(); + uint64_t CurrentOffset = 0; + + while (RemainBytes) + { + const uint64_t ThisChunkBytes = zen::Min(ChunkSize, RemainBytes); + + Read(BufferPtr, ThisChunkBytes, CurrentOffset); + + ChunkFun(BufferPtr, ThisChunkBytes); + + CurrentOffset += ThisChunkBytes; + RemainBytes -= ThisChunkBytes; + } +} + void BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset) { diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h index 0daa5d39f..a5874cfe4 100644 --- a/zenstore/include/zenstore/basicfile.h +++ b/zenstore/include/zenstore/basicfile.h @@ -8,6 +8,7 @@ #include #include +#include namespace zen { @@ -28,8 +29,9 @@ public: void Open(std::filesystem::path FileName, bool IsCreate); void Open(std::filesystem::path FileName, bool IsCreate, std::error_code& Ec); void Close(); - void Read(void* Data, uint64_t Size, uint64_t Offset); - void Write(const void* Data, uint64_t Size, uint64_t Offset); + void Read(void* Data, uint64_t Size, uint64_t FileOffset); + void StreamFile(std::function&& ChunkFun); + void Write(const void* Data, uint64_t Size, uint64_t FileOffset); void Flush(); uint64_t FileSize(); void* Handle() { return m_FileHandle; } -- cgit v1.2.3 From 3cf9dedfd08fe4d7a049e51b14a937f7a34afce3 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 23:19:06 +0200 Subject: Implemended basic scrubbing / detection of disk corruption. Still needs more code to propagate errors and make adjustments to account for them in higher level data structures --- zenstore/CAS.cpp | 24 ++++++++--- zenstore/compactcas.cpp | 93 +++++++++++++++++++++++++++++++++++++--- zenstore/compactcas.h | 7 +-- zenstore/filecas.cpp | 95 +++++++++++++++++++++++++++++++++-------- zenstore/filecas.h | 15 +++++-- zenstore/include/zenstore/CAS.h | 21 ++++----- 6 files changed, 207 insertions(+), 48 deletions(-) diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp index af0fcc609..b3ef7a320 100644 --- a/zenstore/CAS.cpp +++ b/zenstore/CAS.cpp @@ -25,6 +25,11 @@ namespace zen { +void +ScrubContext::ReportBadChunks(std::span BadChunks) +{ +} + /** * CAS store implementation * @@ -44,7 +49,7 @@ public: virtual IoBuffer FindChunk(const IoHash& ChunkHash) override; virtual void FilterChunks(CasChunkSet& InOutChunks) override; virtual void Flush() override; - virtual void Scrub() override; + virtual void Scrub(ScrubContext& Ctx) override; private: CasContainerStrategy m_TinyStrategy; @@ -52,7 +57,7 @@ private: FileCasStrategy m_LargeStrategy; }; -CasImpl::CasImpl() : m_TinyStrategy(m_Config, m_Stats), m_SmallStrategy(m_Config, m_Stats), m_LargeStrategy(m_Config, m_Stats) +CasImpl::CasImpl() : m_TinyStrategy(m_Config), m_SmallStrategy(m_Config), m_LargeStrategy(m_Config) { } @@ -105,6 +110,9 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) m_TinyStrategy.Initialize("tobs", 16, IsNewStore); m_SmallStrategy.Initialize("sobs", 4096, IsNewStore); + + ScrubContext Ctx; + Scrub(Ctx); } CasStore::InsertResult @@ -165,11 +173,11 @@ CasImpl::Flush() } void -CasImpl::Scrub() +CasImpl::Scrub(ScrubContext& Ctx) { - m_SmallStrategy.Scrub(); - m_TinyStrategy.Scrub(); - m_LargeStrategy.Scrub(); + m_SmallStrategy.Scrub(Ctx); + m_TinyStrategy.Scrub(Ctx); + m_LargeStrategy.Scrub(Ctx); } ////////////////////////////////////////////////////////////////////////// @@ -194,7 +202,9 @@ TEST_CASE("CasStore") std::unique_ptr Store{CreateCasStore()}; Store->Initialize(config); - Store->Scrub(); + + ScrubContext Ctx; + Store->Scrub(Ctx); IoBuffer Value1{16}; memcpy(Value1.MutableData(), "1234567890123456", 16); diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp index fd223e284..0f9349ab0 100644 --- a/zenstore/compactcas.cpp +++ b/zenstore/compactcas.cpp @@ -18,6 +18,14 @@ namespace zen { +CasContainerStrategy::CasContainerStrategy(const CasStoreConfiguration& Config) : m_Config(Config) +{ +} + +CasContainerStrategy::~CasContainerStrategy() +{ +} + void CasContainerStrategy::Initialize(const std::string_view ContainerBaseName, uint64_t Alignment, bool IsNewStore) { @@ -101,9 +109,8 @@ IoBuffer CasContainerStrategy::FindChunk(const IoHash& ChunkHash) { RwLock::SharedLockScope _(m_LocationMapLock); - auto KeyIt = m_LocationMap.find(ChunkHash); - if (KeyIt != m_LocationMap.end()) + if (auto KeyIt = m_LocationMap.find(ChunkHash); KeyIt != m_LocationMap.end()) { const CasDiskLocation& Location = KeyIt->second; return zen::IoBufferBuilder::MakeFromFileHandle(m_SmallObjectFile.Handle(), Location.Offset, Location.Size); @@ -118,9 +125,8 @@ bool CasContainerStrategy::HaveChunk(const IoHash& ChunkHash) { RwLock::SharedLockScope _(m_LocationMapLock); - auto KeyIt = m_LocationMap.find(ChunkHash); - if (KeyIt != m_LocationMap.end()) + if (auto KeyIt = m_LocationMap.find(ChunkHash); KeyIt != m_LocationMap.end()) { return true; } @@ -163,9 +169,84 @@ CasContainerStrategy::Flush() } void -CasContainerStrategy::Scrub() +CasContainerStrategy::Scrub(ScrubContext& Ctx) { - RwLock::SharedLockScope _(m_LocationMapLock); + const uint64_t WindowSize = 4 * 1024 * 1024; + uint64_t WindowStart = 0; + uint64_t WindowEnd = WindowSize; + const uint64_t FileSize = m_SmallObjectFile.FileSize(); + + std::vector BigChunks; + std::vector BadChunks; + + // We do a read sweep through the payloads file and validate + // any entries that are contained within each segment, with + // the assumption that most entries will be checked in this + // pass. An alternative strategy would be to use memory mapping. + + { + IoBuffer ReadBuffer{WindowSize}; + void* BufferBase = ReadBuffer.MutableData(); + + RwLock::SharedLockScope _(m_LocationMapLock); + + do + { + const uint64_t ChunkSize = zen::Min(WindowSize, FileSize - WindowStart); + m_SmallObjectFile.Read(BufferBase, ChunkSize, WindowStart); + + for (auto& Entry : m_LocationMap) + { + const uint64_t EntryOffset = Entry.second.Offset; + + if ((EntryOffset >= WindowStart) && (EntryOffset < WindowEnd)) + { + const uint64_t EntryEnd = EntryOffset + Entry.second.Size; + + if (EntryEnd >= WindowEnd) + { + BigChunks.push_back({.Key = Entry.first, .Location = Entry.second}); + + continue; + } + + const IoHash ComputedHash = IoHash::HashBuffer(BufferBase, Entry.second.Size); + + if (Entry.first != ComputedHash) + { + // Hash mismatch + + BadChunks.push_back({.Key = Entry.first, .Location = Entry.second}); + } + } + } + + WindowStart += WindowSize; + WindowEnd += WindowSize; + } while (WindowStart < FileSize); + } + + // Deal with large chunks + + for (const CasDiskIndexEntry& Entry : BigChunks) + { + } + + // Deal with bad chunks by removing them from our lookup map + + std::vector BadChunkHashes; + + for (const CasDiskIndexEntry& Entry : BadChunks) + { + BadChunkHashes.push_back(Entry.Key); + m_LocationMap.erase(Entry.Key); + } + + // Let whomever it concerns know about the bad chunks. This could + // be used to invalidate higher level data structures more efficiently + // than a full validation pass might be able to do + + Ctx.ReportBadChunks(BadChunkHashes); } void diff --git a/zenstore/compactcas.h b/zenstore/compactcas.h index 9921c9e6c..101e6b1b7 100644 --- a/zenstore/compactcas.h +++ b/zenstore/compactcas.h @@ -50,7 +50,9 @@ static_assert(sizeof(CasDiskIndexEntry) == 32); struct CasContainerStrategy { - CasContainerStrategy(const CasStoreConfiguration& Config, CasStore::Stats& Stats) : m_Config(Config), m_Stats(Stats) {} + CasContainerStrategy(const CasStoreConfiguration& Config); + ~CasContainerStrategy(); + CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash); CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& chunkHash); IoBuffer FindChunk(const IoHash& ChunkHash); @@ -58,11 +60,10 @@ struct CasContainerStrategy void FilterChunks(CasChunkSet& InOutChunks); void Initialize(const std::string_view ContainerBaseName, uint64_t Alignment, bool IsNewStore); void Flush(); - void Scrub(); + void Scrub(ScrubContext& Ctx); private: const CasStoreConfiguration& m_Config; - CasStore::Stats& m_Stats; uint64_t m_PayloadAlignment = 1 << 4; bool m_IsInitialized = false; BasicFile m_SmallObjectFile; diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp index 5fdf505d4..3314beb7e 100644 --- a/zenstore/filecas.cpp +++ b/zenstore/filecas.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -17,6 +18,7 @@ #include #include +// clang-format off #include struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive- @@ -24,13 +26,19 @@ struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax erro #include // clang-format on -// -////////////////////////////////////////////////////////////////////////// namespace zen { using namespace fmt::literals; +FileCasStrategy::FileCasStrategy(const CasStoreConfiguration& Config) : m_Config(Config) +{ +} + +FileCasStrategy::~FileCasStrategy() +{ +} + WideStringBuilderBase& FileCasStrategy::MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHash& ChunkHash, size_t& OutShard2len) { @@ -56,7 +64,7 @@ FileCasStrategy::MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHas OutShard2len = ShardedPath.Size(); ShardedPath.Append('\\'); - ShardedPath.AppendAsciiRange(str + 6, str + 64); + ShardedPath.AppendAsciiRange(str + 5, str + 64); return ShardedPath; } @@ -259,12 +267,9 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize } // We cannot rely on RAII to close the file handle since it would be closed - // *after* the lock is released due to the initialization order. + // *after* the lock is released due to the initialization order PayloadFile.Close(); - AtomicIncrement(m_Stats.PutCount); - AtomicAdd(m_Stats.PutBytes, ChunkSize); - return {.New = true}; } @@ -279,15 +284,7 @@ FileCasStrategy::FindChunk(const IoHash& ChunkHash) RwLock::SharedLockScope _(LockForHash(ChunkHash)); - auto Chunk = IoBufferBuilder::MakeFromFile(ShardedPath.c_str()); - - if (Chunk) - { - AtomicIncrement(m_Stats.GetCount); - AtomicAdd(m_Stats.GetBytes, Chunk.Size()); - } - - return Chunk; + return IoBufferBuilder::MakeFromFile(ShardedPath.c_str()); } bool @@ -337,6 +334,56 @@ FileCasStrategy::FilterChunks(CasChunkSet& InOutChunks) } } +void +FileCasStrategy::IterateChunks(std::function&& Callback) +{ + struct Visitor : public FileSystemTraversal::TreeVisitor + { + Visitor(const std::filesystem::path& RootDir) : RootDirectory(RootDir) {} + virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& File, uint64_t FileSize) override + { + std::filesystem::path RelPath = std::filesystem::relative(Parent, RootDirectory); + + std::wstring PathString = RelPath.native(); + + if ((PathString.size() == (3 + 2 + 1)) && (File.size() == (40 - 3 - 2))) + { + if (PathString.at(3) == std::filesystem::path::preferred_separator) + { + PathString.erase(3, 1); + } + PathString.append(File); + + StringBuilder<64> Utf8; + WideToUtf8(PathString, Utf8); + + // TODO: should validate that we're actually dealing with a valid hex string here + + IoHash NameHash = IoHash::FromHexString({Utf8.Data(), Utf8.Size()}); + + BasicFile PayloadFile; + std::error_code Ec; + PayloadFile.Open(Parent / File, false, Ec); + + if (!Ec) + { + Callback(NameHash, PayloadFile); + } + } + } + + virtual bool VisitDirectory(const std::filesystem::path& Parent, const std::wstring_view& DirectoryName) { return true; } + + const std::filesystem::path& RootDirectory; + std::function Callback; + } CasVisitor{m_Config.RootDirectory}; + + CasVisitor.Callback = std::move(Callback); + + FileSystemTraversal Traversal; + Traversal.TraverseFileSystem(m_Config.RootDirectory, CasVisitor); +} + void FileCasStrategy::Flush() { @@ -353,8 +400,22 @@ FileCasStrategy::Flush() } void -FileCasStrategy::Scrub() +FileCasStrategy::Scrub(ScrubContext& Ctx) { + std::vector BadHashes; + + IterateChunks([&](const IoHash& Hash, BasicFile& Payload) { + IoHashStream Hasher; + Payload.StreamFile([&](const void* Data, size_t Size) { Hasher.Append(Data, Size); }); + IoHash ComputedHash = Hasher.GetHash(); + + if (ComputedHash != Hash) + { + BadHashes.push_back(Hash); + } + }); + + Ctx.ReportBadChunks(BadHashes); } void diff --git a/zenstore/filecas.h b/zenstore/filecas.h index c7cd9d7ca..885973810 100644 --- a/zenstore/filecas.h +++ b/zenstore/filecas.h @@ -10,11 +10,20 @@ #include #include +#include + namespace zen { +class BasicFile; + +/** CAS storage strategy using a file-per-chunk storage strategy +*/ + struct FileCasStrategy { - FileCasStrategy(const CasStoreConfiguration& Config, CasStore::Stats& Stats) : m_Config(Config), m_Stats(Stats) {} + FileCasStrategy(const CasStoreConfiguration& Config); + ~FileCasStrategy(); + CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash); CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash); IoBuffer FindChunk(const IoHash& ChunkHash); @@ -22,16 +31,16 @@ struct FileCasStrategy void FilterChunks(CasChunkSet& InOutChunks); void Flush(); void GarbageCollect(GcContext& GcCtx); - void Scrub(); + void Scrub(ScrubContext& Ctx); private: const CasStoreConfiguration& m_Config; - CasStore::Stats& m_Stats; RwLock m_Lock; RwLock m_ShardLocks[256]; // TODO: these should be spaced out so they don't share cache lines inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardLocks[Hash.Hash[19]]; } static WideStringBuilderBase& MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHash& ChunkHash, size_t& OutShard2len); + void IterateChunks(std::function&& Callback); }; } // namespace zen diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/CAS.h index c6c919593..bb310b179 100644 --- a/zenstore/include/zenstore/CAS.h +++ b/zenstore/include/zenstore/CAS.h @@ -37,6 +37,14 @@ public: private: }; +class ScrubContext +{ +public: + virtual void ReportBadChunks(std::span BadChunks); + +private: +}; + class CasChunkSet { public: @@ -54,17 +62,7 @@ class CasStore public: virtual ~CasStore() = default; - struct Stats - { - uint64_t PutBytes = 0; - uint64_t PutCount = 0; - - uint64_t GetBytes = 0; - uint64_t GetCount = 0; - }; - const CasStoreConfiguration& Config() { return m_Config; } - const Stats& GetStats() const { return m_Stats; } struct InsertResult { @@ -76,11 +74,10 @@ public: virtual IoBuffer FindChunk(const IoHash& ChunkHash) = 0; virtual void FilterChunks(CasChunkSet& InOutChunks) = 0; virtual void Flush() = 0; - virtual void Scrub() = 0; + virtual void Scrub(ScrubContext& Ctx) = 0; protected: CasStoreConfiguration m_Config; - Stats m_Stats; }; ZENCORE_API CasStore* CreateCasStore(); -- cgit v1.2.3 From 6240dbc8aac714998e38b92850c3bff3d579f2ab Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 23:30:03 +0200 Subject: Implemented BasicFile::StreamByteRange --- zenstore/basicfile.cpp | 10 ++++++++-- zenstore/include/zenstore/basicfile.h | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp index 126f2a8b3..0b92a8979 100644 --- a/zenstore/basicfile.cpp +++ b/zenstore/basicfile.cpp @@ -97,13 +97,19 @@ BasicFile::ReadAll() void BasicFile::StreamFile(std::function&& ChunkFun) +{ + StreamByteRange(0, FileSize(), std::move(ChunkFun)); +} + +void +BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& ChunkFun) { const uint64_t ChunkSize = 128 * 1024; IoBuffer ReadBuffer{ChunkSize}; void* BufferPtr = ReadBuffer.MutableData(); - uint64_t RemainBytes = FileSize(); - uint64_t CurrentOffset = 0; + uint64_t RemainBytes = Size; + uint64_t CurrentOffset = FileOffset; while (RemainBytes) { diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h index a5874cfe4..d4d65b366 100644 --- a/zenstore/include/zenstore/basicfile.h +++ b/zenstore/include/zenstore/basicfile.h @@ -31,6 +31,7 @@ public: void Close(); void Read(void* Data, uint64_t Size, uint64_t FileOffset); void StreamFile(std::function&& ChunkFun); + void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& ChunkFun); void Write(const void* Data, uint64_t Size, uint64_t FileOffset); void Flush(); uint64_t FileSize(); -- cgit v1.2.3 From 22fbe6e86bb6527c9a6214bf921f872cc27ee64c Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 23:30:37 +0200 Subject: Implemented handling of "large" chunks in compact cas scrubbing --- zenstore/compactcas.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp index 0f9349ab0..070ca1503 100644 --- a/zenstore/compactcas.cpp +++ b/zenstore/compactcas.cpp @@ -230,6 +230,16 @@ CasContainerStrategy::Scrub(ScrubContext& Ctx) for (const CasDiskIndexEntry& Entry : BigChunks) { + IoHashStream Hasher; + m_SmallObjectFile.StreamByteRange(Entry.Location.Offset, Entry.Location.Size, [&](const void* Data, uint64_t Size) { + Hasher.Append(Data, Size); + }); + IoHash ComputedHash = Hasher.GetHash(); + + if (Entry.Key != ComputedHash) + { + BadChunks.push_back(Entry); + } } // Deal with bad chunks by removing them from our lookup map -- cgit v1.2.3 From 352aec91f76719492578f36e760a5d268d141010 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 19 Sep 2021 23:31:13 +0200 Subject: Fixed unused variable warnings exposed by xmake build (unclear why I do not receive them in VS, but likely due to vcpkg versioning) --- zenstore/CAS.cpp | 1 + zenstore/filecas.cpp | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp index b3ef7a320..a143230d3 100644 --- a/zenstore/CAS.cpp +++ b/zenstore/CAS.cpp @@ -28,6 +28,7 @@ namespace zen { void ScrubContext::ReportBadChunks(std::span BadChunks) { + ZEN_UNUSED(BadChunks); } /** diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp index 3314beb7e..31991a43e 100644 --- a/zenstore/filecas.cpp +++ b/zenstore/filecas.cpp @@ -342,6 +342,8 @@ FileCasStrategy::IterateChunks(std::function Callback; -- cgit v1.2.3