diff options
| author | Per Larsson <[email protected]> | 2021-09-20 08:54:34 +0200 |
|---|---|---|
| committer | Per Larsson <[email protected]> | 2021-09-20 08:54:34 +0200 |
| commit | e25b4b20d8a5696aa7055c9c167fa47b3739bc7e (patch) | |
| tree | 049654b87096a22e1bf696a385db608a75f229fa | |
| parent | Probe upstream Zen server when initializing upstream cache. (diff) | |
| parent | Fixed unused variable warnings exposed by xmake build (unclear why I do not r... (diff) | |
| download | zen-e25b4b20d8a5696aa7055c9c167fa47b3739bc7e.tar.xz zen-e25b4b20d8a5696aa7055c9c167fa47b3739bc7e.zip | |
Merge branch 'main' of https://github.com/EpicGames/zen
78 files changed, 1778 insertions, 311 deletions
diff --git a/.gitignore b/.gitignore index df6c1a9f8..bf237a7b3 100644 --- a/.gitignore +++ b/.gitignore @@ -204,9 +204,8 @@ ServiceFabricBackup/ *.rptproj.bak - - - +# generated build files +makefile # Python Tools for Visual Studio (PTVS) __pycache__/ diff --git a/vs-chromium-project.txt b/vs-chromium-project.txt index 2b2e15bc7..2bb89a55c 100644 --- a/vs-chromium-project.txt +++ b/vs-chromium-project.txt @@ -3,6 +3,6 @@ [SourceExplorer.ignore] .git/ -.x64/ +x64/ *.suo -**/.x64/ +**/x64/ @@ -36,7 +36,7 @@ if is_mode("debug") then end if is_os("windows") then - add_defines("_CRT_SECURE_NO_WARNINGS") + add_defines("_CRT_SECURE_NO_WARNINGS", "_UNICODE", "UNICODE", "_WIN32_WINNT=0x0A00") end add_defines("USE_SENTRY=1") @@ -64,6 +64,7 @@ set_symbols("debug") includes("zencore", "zencore-test") includes("zenhttp") -includes("zenstore", "zenutil") +includes("zenstore", "zenstore-test") +includes("zenutil") includes("zenserver", "zenserver-test") includes("zen") @@ -11,6 +11,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{4EA55E5B-1 README.md = README.md RESTAPI.md = RESTAPI.md vcpkg.json = vcpkg.json + xmake.lua = xmake.lua EndProjectSection EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zencore", "zencore\zencore.vcxproj", "{D75BF9AB-C61E-4FFF-AD59-1563430F05E2}" @@ -47,6 +48,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zentest-appstub", "zentest- EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zenhttp", "zenhttp\zenhttp.vcxproj", "{8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zenstore-test", "zenstore-test\zenstore-test.vcxproj", "{C001A3DF-B76E-4989-B576-FE2B78AB2580}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -111,6 +114,12 @@ Global {8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Release|x64.ActiveCfg = Release|x64 {8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Release|x64.Build.0 = Release|x64 {8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Release|x86.ActiveCfg = Release|x64 + {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Debug|x64.ActiveCfg = Debug|x64 + {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Debug|x64.Build.0 = Debug|x64 + {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Debug|x86.ActiveCfg = Debug|x64 + {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Release|x64.ActiveCfg = Release|x64 + {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Release|x64.Build.0 = Release|x64 + {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Release|x86.ActiveCfg = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/zen/cmds/copy.cpp b/zen/cmds/copy.cpp index 6b3965f99..947d54e07 100644 --- a/zen/cmds/copy.cpp +++ b/zen/cmds/copy.cpp @@ -97,4 +97,4 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/copy.h b/zen/cmds/copy.h index c2a7fd23b..322cf3f2f 100644 --- a/zen/cmds/copy.h +++ b/zen/cmds/copy.h @@ -25,4 +25,4 @@ private: bool m_NoClone = false; }; -} +} // namespace zen diff --git a/zen/cmds/dedup.cpp b/zen/cmds/dedup.cpp index f95a87518..e71314622 100644 --- a/zen/cmds/dedup.cpp +++ b/zen/cmds/dedup.cpp @@ -293,4 +293,4 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/dedup.h b/zen/cmds/dedup.h index c955d4bbd..7932d10e6 100644 --- a/zen/cmds/dedup.h +++ b/zen/cmds/dedup.h @@ -27,4 +27,4 @@ private: size_t m_SizeThreshold = 1024 * 1024; }; -} +} // namespace zen diff --git a/zen/cmds/deploy.cpp b/zen/cmds/deploy.cpp index faaf0030c..d60392dd5 100644 --- a/zen/cmds/deploy.cpp +++ b/zen/cmds/deploy.cpp @@ -83,4 +83,4 @@ DeployCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/deploy.h b/zen/cmds/deploy.h index 565e11bf3..975caf9e9 100644 --- a/zen/cmds/deploy.h +++ b/zen/cmds/deploy.h @@ -26,4 +26,4 @@ private: bool m_IsClean = false; }; -} +} // namespace zen diff --git a/zen/cmds/hash.cpp b/zen/cmds/hash.cpp index e9484c453..0a7989ffc 100644 --- a/zen/cmds/hash.cpp +++ b/zen/cmds/hash.cpp @@ -126,4 +126,4 @@ HashCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/hash.h b/zen/cmds/hash.h index 8ed8e9fa3..3df9063ea 100644 --- a/zen/cmds/hash.h +++ b/zen/cmds/hash.h @@ -26,4 +26,4 @@ private: std::string m_OutputFile; }; -} +} // namespace zen diff --git a/zen/cmds/run.cpp b/zen/cmds/run.cpp index 711a3a341..ceeb4ddae 100644 --- a/zen/cmds/run.cpp +++ b/zen/cmds/run.cpp @@ -184,4 +184,4 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/run.h b/zen/cmds/run.h index f4eb4ef76..3e1e3f2b2 100644 --- a/zen/cmds/run.h +++ b/zen/cmds/run.h @@ -23,4 +23,4 @@ private: std::string m_ExeTree; }; -} +} // namespace zen diff --git a/zen/cmds/scrub.cpp b/zen/cmds/scrub.cpp index 73d2b45ee..c0fe8ca61 100644 --- a/zen/cmds/scrub.cpp +++ b/zen/cmds/scrub.cpp @@ -31,7 +31,7 @@ GcCommand::~GcCommand() { } -int +int GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_UNUSED(GlobalOptions, argc, argv); @@ -39,4 +39,4 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/scrub.h b/zen/cmds/scrub.h index a3f25c259..561ae578d 100644 --- a/zen/cmds/scrub.h +++ b/zen/cmds/scrub.h @@ -36,4 +36,4 @@ private: cxxopts::Options m_Options{"gc", "Garbage collect zen storage"}; }; -} +} // namespace zen diff --git a/zen/cmds/status.cpp b/zen/cmds/status.cpp index 1050b5b96..10970e3c2 100644 --- a/zen/cmds/status.cpp +++ b/zen/cmds/status.cpp @@ -20,4 +20,4 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/status.h b/zen/cmds/status.h index f2c68b96a..acde280c5 100644 --- a/zen/cmds/status.h +++ b/zen/cmds/status.h @@ -19,4 +19,4 @@ private: cxxopts::Options m_Options{"status", "Show zen status"}; }; -} +} // namespace zen diff --git a/zen/cmds/top.cpp b/zen/cmds/top.cpp index 3ff7edcda..315d8cb38 100644 --- a/zen/cmds/top.cpp +++ b/zen/cmds/top.cpp @@ -2,9 +2,9 @@ #include "top.h" +#include <zencore/fmtutils.h> #include <zencore/logging.h> #include <zencore/uid.h> -#include <zencore/fmtutils.h> #include <zenutil/zenserverprocess.h> #include <memory> @@ -32,7 +32,7 @@ TopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } - int n = 0; + int n = 0; const int HeaderPeriod = 20; for (;;) @@ -42,14 +42,18 @@ TopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("{:>5} {:>6} {:>24}", "port", "pid", "session"); } - State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { ZEN_CONSOLE("{:5} {:6} {:24}", Entry.ListenPort, Entry.Pid, Entry.GetSessionId()); }); + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + ZEN_CONSOLE("{:5} {:6} {:24}", Entry.ListenPort, Entry.Pid, Entry.GetSessionId()); + }); zen::Sleep(1000); - State.Sweep(); + if (!State.IsReadOnly()) + { + State.Sweep(); + } } - return 0; } @@ -79,4 +83,4 @@ PsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/top.h b/zen/cmds/top.h index a842089ff..d8bf91a1c 100644 --- a/zen/cmds/top.h +++ b/zen/cmds/top.h @@ -32,4 +32,4 @@ private: cxxopts::Options m_Options{"ps", "Enumerate running Zen server instances"}; }; -} +} // namespace zen diff --git a/zen/cmds/up.cpp b/zen/cmds/up.cpp index 048133fc9..17cba3794 100644 --- a/zen/cmds/up.cpp +++ b/zen/cmds/up.cpp @@ -101,4 +101,4 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } -} +} // namespace zen diff --git a/zen/cmds/up.h b/zen/cmds/up.h index fda62693a..fe1ed7a0c 100644 --- a/zen/cmds/up.h +++ b/zen/cmds/up.h @@ -32,4 +32,4 @@ private: cxxopts::Options m_Options{"down", "Bring down zen service"}; }; -} +} // namespace zen diff --git a/zen/zen.cpp b/zen/zen.cpp index 6390c832f..9c373face 100644 --- a/zen/zen.cpp +++ b/zen/zen.cpp @@ -42,7 +42,6 @@ ////////////////////////////////////////////////////////////////////////// - class TemplateCommand : public ZenCmdBase { public: @@ -125,7 +124,7 @@ main(int argc, char** argv) ZenCmdBase* Cmd; const char* CmdSummary; } Commands[] = { - // clang-format off + // clang-format off {"chunk", &ChunkCmd, "Perform chunking"}, {"copy", &CopyCmd, "Copy file(s)"}, {"deploy", &DeployCmd, "Deploy data"}, @@ -139,7 +138,7 @@ main(int argc, char** argv) {"top", &TopCmd, "Monitor zen server activity"}, {"up", &UpCmd, "Bring zen server up"}, {"down", &DownCmd, "Bring zen server down"}, - // clang-format on + // clang-format on }; // Build set containing available commands @@ -10,7 +10,6 @@ #include <zencore/refcount.h> #include <zencore/windows.h> -#include <atlfile.h> #include <filesystem> struct ZenCliOptions diff --git a/zen/zen.vcxproj b/zen/zen.vcxproj index ff10f4c59..fb0674e87 100644 --- a/zen/zen.vcxproj +++ b/zen/zen.vcxproj @@ -136,6 +136,9 @@ <Project>{77f8315d-b21d-4db0-9a6f-2d3359f88a70}</Project> </ProjectReference> </ItemGroup> + <ItemGroup> + <None Include="xmake.lua" /> + </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> </ImportGroup> diff --git a/zen/zen.vcxproj.filters b/zen/zen.vcxproj.filters index a38771944..9002f01c2 100644 --- a/zen/zen.vcxproj.filters +++ b/zen/zen.vcxproj.filters @@ -63,4 +63,7 @@ <UniqueIdentifier>{2e06a54c-52be-4260-9275-a4232d01a53c}</UniqueIdentifier> </Filter> </ItemGroup> + <ItemGroup> + <None Include="xmake.lua" /> + </ItemGroup> </Project>
\ No newline at end of file diff --git a/zencore/except.cpp b/zencore/except.cpp index 9bf043f4a..44b8edffb 100644 --- a/zencore/except.cpp +++ b/zencore/except.cpp @@ -7,6 +7,41 @@ namespace zen { #if ZEN_PLATFORM_WINDOWS +class WindowsException : public std::exception +{ +public: + WindowsException(std::string_view Message) + { + m_hResult = HRESULT_FROM_WIN32(GetLastError()); + m_Message = Message; + } + + WindowsException(HRESULT hRes, std::string_view Message) + { + m_hResult = hRes; + m_Message = Message; + } + + WindowsException(HRESULT hRes, const char* Message, const char* Detail) + { + m_hResult = hRes; + + ExtendableStringBuilder<128> msg; + msg.Append(Message); + msg.Append(" (detail: '"); + msg.Append(Detail); + msg.Append("')"); + + m_Message = msg.c_str(); + } + + virtual const char* what() const override { return m_Message.c_str(); } + +private: + std::string m_Message; + HRESULT m_hResult; +}; + void ThrowSystemException([[maybe_unused]] HRESULT hRes, [[maybe_unused]] std::string_view Message) { @@ -28,6 +63,12 @@ ThrowLastError(std::string_view Message) throw std::system_error(std::error_code(zen::GetLastError(), std::system_category()), std::string(Message)); } +void +ThrowSystemError(uint32_t ErrorCode, std::string_view Message) +{ + throw std::system_error(std::error_code(ErrorCode, std::system_category()), std::string(Message)); +} + std::string GetLastErrorAsString() { diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h index 08330e4bc..5cfefb1e2 100644 --- a/zencore/include/zencore/except.h +++ b/zencore/include/zencore/except.h @@ -15,63 +15,20 @@ namespace zen { #if ZEN_PLATFORM_WINDOWS -class WindowsException : public std::exception -{ -public: - WindowsException(std::string_view Message) - { - m_hResult = HRESULT_FROM_WIN32(GetLastError()); - m_Message = Message; - } - - WindowsException(HRESULT hRes, std::string_view Message) - { - m_hResult = hRes; - m_Message = Message; - } - - WindowsException(HRESULT hRes, const char* Message, const char* Detail) - { - m_hResult = hRes; - - ExtendableStringBuilder<128> msg; - msg.Append(Message); - msg.Append(" (detail: '"); - msg.Append(Detail); - msg.Append("')"); - - m_Message = msg.c_str(); - } - - virtual const char* what() const override { return m_Message.c_str(); } - -private: - std::string m_Message; - HRESULT m_hResult; -}; - -ZENCORE_API void ThrowSystemException(HRESULT hRes, std::string_view Message); +ZENCORE_API void ThrowSystemException [[noreturn]] (HRESULT hRes, std::string_view Message); #endif // ZEN_PLATFORM_WINDOWS -ZENCORE_API void ThrowLastError(std::string_view Message); +ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message); #if __cpp_lib_source_location -ZENCORE_API void ThrowLastError(std::string_view Message, const std::source_location& Location); +ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message, const std::source_location& Location); #endif +ZENCORE_API void ThrowSystemError [[noreturn]] (uint32_t ErrorCode, std::string_view Message); + ZENCORE_API std::string GetLastErrorAsString(); ZENCORE_API std::string GetWindowsErrorAsString(uint32_t Win32ErrorCode); -inline void -ThrowSystemException(const char* Message) -{ -#if ZEN_PLATFORM_WINDOWS - throw WindowsException(Message); -#else - ThrowLastError(Message); -#endif -} - inline int32_t GetLastError() { diff --git a/zencore/include/zencore/session.h b/zencore/include/zencore/session.h index 2da41b2c8..dd90197bf 100644 --- a/zencore/include/zencore/session.h +++ b/zencore/include/zencore/session.h @@ -8,6 +8,7 @@ namespace zen { struct Oid; -ZENCORE_API Oid GetSessionId(); +ZENCORE_API [[nodiscard]] Oid GetSessionId(); +ZENCORE_API [[nodiscard]] std::string_view GetSessionIdString(); } // namespace zen diff --git a/zencore/include/zencore/testutils.h b/zencore/include/zencore/testutils.h new file mode 100644 index 000000000..72d985d5c --- /dev/null +++ b/zencore/include/zencore/testutils.h @@ -0,0 +1,31 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <filesystem> + +namespace zen { + +std::filesystem::path CreateTemporaryDirectory(); + +class ScopedTemporaryDirectory +{ +public: + ScopedTemporaryDirectory(); + ~ScopedTemporaryDirectory(); + + std::filesystem::path& Path() { return m_RootPath; } + +private: + std::filesystem::path m_RootPath; +}; + +struct ScopedCurrentDirectoryChange +{ + std::filesystem::path OldPath{std::filesystem::current_path()}; + + ScopedCurrentDirectoryChange() { std::filesystem::current_path(CreateTemporaryDirectory()); } + ~ScopedCurrentDirectoryChange() { std::filesystem::current_path(OldPath); } +}; + +} // namespace zen diff --git a/zencore/include/zencore/thread.h b/zencore/include/zencore/thread.h index e65867ec4..7889682cd 100644 --- a/zencore/include/zencore/thread.h +++ b/zencore/include/zencore/thread.h @@ -4,9 +4,7 @@ #include "zencore.h" -#if !ZEN_PLATFORM_WINDOWS -# include <shared_mutex> -#endif +#include <shared_mutex> #include <vector> @@ -66,11 +64,7 @@ public: }; private: -#if ZEN_PLATFORM_WINDOWS - void* m_Srw = nullptr; -#else std::shared_mutex m_Mutex; -#endif }; /** Basic abstraction of a simple event synchronization mechanism (aka 'binary semaphore') diff --git a/zencore/include/zencore/uid.h b/zencore/include/zencore/uid.h index f095c49ef..f4e9ab65a 100644 --- a/zencore/include/zencore/uid.h +++ b/zencore/include/zencore/uid.h @@ -60,6 +60,7 @@ struct Oid const Oid& Generate(); [[nodiscard]] static Oid FromHexString(const std::string_view String); StringBuilderBase& ToString(StringBuilderBase& OutString) const; + void ToString(char OutString[StringLength]); [[nodiscard]] static Oid FromMemory(const void* Ptr); auto operator<=>(const Oid& rhs) const = default; diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index da17e61e3..310f6c4ed 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -3,7 +3,7 @@ #pragma once #include <cinttypes> -#include <exception> +#include <stdexcept> #include <string> ////////////////////////////////////////////////////////////////////////// @@ -90,37 +90,69 @@ // Assert // +#if ZEN_PLATFORM_WINDOWS +// Tells the compiler to put the decorated function in a certain section (aka. segment) of the executable. +# define ZEN_CODE_SECTION(Name) __declspec(code_seg(Name)) +# define ZEN_FORCENOINLINE __declspec(noinline) /* Force code to NOT be inline */ +#else +# define ZEN_CODE_SECTION(Name) +# define ZEN_FORCENOINLINE +#endif + +#if ZEN_ARCH_ARM64 +// On ARM we can't do this because the executable will require jumps larger +// than the branch instruction can handle. Clang will only generate +// the trampolines in the .text segment of the binary. If the uedbg segment +// is present it will generate code that it cannot link. +# define ZEN_DEBUG_SECTION +#else +// We'll put all assert implementation code into a separate section in the linked +// executable. This code should never execute so using a separate section keeps +// it well off the hot path and hopefully out of the instruction cache. It also +// facilitates reasoning about the makeup of a compiled/linked binary. +# define ZEN_DEBUG_SECTION ZEN_CODE_SECTION(".zcold") +#endif // DO_CHECK || DO_GUARD_SLOW + namespace zen { -class AssertException : public std::exception +class AssertException : public std::logic_error { public: - AssertException(const char* Msg); - ~AssertException(); - - [[nodiscard]] virtual char const* what() const noexcept override { return m_Msg.c_str(); } - -private: - std::string m_Msg; + AssertException(const char* Msg) : std::logic_error(Msg) {} }; } // namespace zen -#define ZEN_ASSERT(x, ...) \ - do \ - { \ - if (x) \ - break; \ - throw ::zen::AssertException{#x}; \ +template<typename RetType = void, class InnerType, typename... ArgTypes> +RetType ZEN_FORCENOINLINE ZEN_DEBUG_SECTION +DispatchAssert(InnerType&& Inner, ArgTypes const&... Args) +{ + return Inner(Args...); +} + +#define ZEN_ASSERT(x, ...) \ + do \ + { \ + if (x) [[unlikely]] \ + break; \ + struct Impl \ + { \ + static void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION ExecThrow [[noreturn]] () { throw ::zen::AssertException{#x}; } \ + }; \ + Impl::ExecThrow(); \ } while (false) #ifndef NDEBUG -# define ZEN_ASSERT_SLOW(x, ...) \ - do \ - { \ - if (x) \ - break; \ - throw ::zen::AssertException{#x}; \ +# define ZEN_ASSERT_SLOW(x, ...) \ + do \ + { \ + if (x) [[unlikely]] \ + break; \ + struct Impl \ + { \ + static void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION ExecThrow [[noreturn]] () { throw ::zen::AssertException{#x}; } \ + }; \ + Impl::ExecThrow(); \ } while (false) #else # define ZEN_ASSERT_SLOW(x, ...) @@ -148,17 +180,19 @@ char (&ZenArrayCountHelper(const T (&)[N]))[N + 1]; #define ZEN_UNUSED(...) ((void)__VA_ARGS__) #define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false, __VA_ARGS__) -#define ZENCORE_API // Placeholder to allow DLL configs in the future +#define ZENCORE_API // Placeholder to allow DLL configs in the future (maybe) namespace zen { ZENCORE_API bool IsPointerToStack(const void* ptr); // Query if pointer is within the stack of the currently executing thread ZENCORE_API bool IsApplicationExitRequested(); ZENCORE_API void RequestApplicationExit(int ExitCode); +ZENCORE_API bool IsDebuggerPresent(); +ZENCORE_API bool IsInteractiveSession(); ZENCORE_API void zencore_forcelinktests(); -} +} // namespace zen ////////////////////////////////////////////////////////////////////////// diff --git a/zencore/session.cpp b/zencore/session.cpp index d57d3685b..ce4bfae1b 100644 --- a/zencore/session.cpp +++ b/zencore/session.cpp @@ -9,14 +9,27 @@ namespace zen { static Oid GlobalSessionId; +static char GlobalSessionString[Oid::StringLength]; static std::once_flag SessionInitFlag; Oid GetSessionId() { - std::call_once(SessionInitFlag, [&] { GlobalSessionId.Generate(); }); + std::call_once(SessionInitFlag, [&] { + GlobalSessionId.Generate(); + GlobalSessionId.ToString(GlobalSessionString); + }); return GlobalSessionId; } +std::string_view +GetSessionIdString() +{ + // Ensure we actually have a generated session identifier + std::ignore = GetSessionId(); + + return std::string_view(GlobalSessionString, Oid::StringLength); +} + } // namespace zen
\ No newline at end of file diff --git a/zencore/testutils.cpp b/zencore/testutils.cpp new file mode 100644 index 000000000..116491950 --- /dev/null +++ b/zencore/testutils.cpp @@ -0,0 +1,33 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencore/testutils.h" +#include <zencore/session.h> +#include "zencore/string.h" + +namespace zen { + +static std::atomic<int> Sequence{0}; + +std::filesystem::path +CreateTemporaryDirectory() +{ + std::error_code Ec; + + std::filesystem::path DirPath = std::filesystem::temp_directory_path() / GetSessionIdString() / IntNum(++Sequence).c_str(); + std::filesystem::remove_all(DirPath, Ec); + std::filesystem::create_directories(DirPath); + + return DirPath; +} + +ScopedTemporaryDirectory::ScopedTemporaryDirectory() : m_RootPath(CreateTemporaryDirectory()) +{ +} + +ScopedTemporaryDirectory::~ScopedTemporaryDirectory() +{ + std::error_code Ec; + std::filesystem::remove_all(m_RootPath, Ec); +} + +} // namespace zen
\ No newline at end of file diff --git a/zencore/thread.cpp b/zencore/thread.cpp index 1c7e4b3ab..d4f101454 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -17,41 +17,25 @@ namespace zen { void RwLock::AcquireShared() { -#if ZEN_PLATFORM_WINDOWS - AcquireSRWLockShared((PSRWLOCK)&m_Srw); -#else m_Mutex.lock_shared(); -#endif } void RwLock::ReleaseShared() { -#if ZEN_PLATFORM_WINDOWS - ReleaseSRWLockShared((PSRWLOCK)&m_Srw); -#else m_Mutex.unlock_shared(); -#endif } void RwLock::AcquireExclusive() { -#if ZEN_PLATFORM_WINDOWS - AcquireSRWLockExclusive((PSRWLOCK)&m_Srw); -#else m_Mutex.lock(); -#endif } void RwLock::ReleaseExclusive() { -#if ZEN_PLATFORM_WINDOWS - ReleaseSRWLockExclusive((PSRWLOCK)&m_Srw); -#else m_Mutex.unlock(); -#endif } Event::Event() @@ -257,7 +241,8 @@ ProcessHandle::Wait(int TimeoutMs) case WAIT_FAILED: // What might go wrong here, and what is meaningful to act on? - throw WindowsException("Process::Wait failed"); + using namespace std::literals; + ThrowLastError("Process::Wait failed"sv); } return false; @@ -333,16 +318,32 @@ ProcessMonitor::IsActive() const bool IsProcessRunning(int pid) { + // This function is arguably not super useful, a pid can be re-used + // by the OS so holding on to a pid and polling it over some time + // period will not necessarily tell you what you probably want to know. + +#if ZEN_PLATFORM_WINDOWS HANDLE hProc = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid); - if (hProc == NULL) + if (!hProc) { - return false; + DWORD Error = zen::GetLastError(); + + if (Error == ERROR_INVALID_PARAMETER) + { + return false; + } + + using namespace fmt::literals; + ThrowSystemError(Error, "failed to open process with pid {}"_format(pid)); } CloseHandle(hProc); return true; +#else + ZEN_NOT_IMPLEMENTED(); +#endif } int diff --git a/zencore/uid.cpp b/zencore/uid.cpp index d946638ec..ed00b1814 100644 --- a/zencore/uid.cpp +++ b/zencore/uid.cpp @@ -91,10 +91,16 @@ Oid::FromMemory(const void* Ptr) return Id; } +void +Oid::ToString(char OutString[StringLength]) +{ + ToHexBytes(reinterpret_cast<const uint8_t*>(OidBits), sizeof(Oid::OidBits), OutString); +} + StringBuilderBase& Oid::ToString(StringBuilderBase& OutString) const { - char str[25]; + char str[StringLength + 1]; ToHexBytes(reinterpret_cast<const uint8_t*>(OidBits), sizeof(Oid::OidBits), str); str[2 * sizeof(Oid)] = '\0'; diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp index 5899f014d..122719d90 100644 --- a/zencore/zencore.cpp +++ b/zencore/zencore.cpp @@ -30,6 +30,8 @@ namespace zen { +////////////////////////////////////////////////////////////////////////// + bool IsPointerToStack(const void* ptr) { @@ -55,12 +57,31 @@ IsPointerToStack(const void* ptr) #endif } -AssertException::AssertException(const char* Msg) : m_Msg(Msg) +bool +IsDebuggerPresent() { +#if ZEN_PLATFORM_WINDOWS + return ::IsDebuggerPresent(); +#else + return false; +#endif } -AssertException::~AssertException() +bool +IsInteractiveSession() { +#if ZEN_PLATFORM_WINDOWS + DWORD dwSessionId = 0; + if (ProcessIdToSessionId(GetCurrentProcessId(), &dwSessionId)) + { + return (dwSessionId != 0); + } + + return false; +#else + // TODO: figure out what makes sense here + return true; +#endif } ////////////////////////////////////////////////////////////////////////// diff --git a/zencore/zencore.vcxproj b/zencore/zencore.vcxproj index 150c42cd6..59b0bafcc 100644 --- a/zencore/zencore.vcxproj +++ b/zencore/zencore.vcxproj @@ -142,6 +142,7 @@ <ClInclude Include="include\zencore\streamutil.h" /> <ClInclude Include="include\zencore\string.h" /> <ClInclude Include="include\zencore\targetver.h" /> + <ClInclude Include="include\zencore\testutils.h" /> <ClInclude Include="include\zencore\thread.h" /> <ClInclude Include="include\zencore\timer.h" /> <ClInclude Include="include\zencore\uid.h" /> @@ -181,6 +182,7 @@ <ClCompile Include="stream.cpp" /> <ClCompile Include="streamutil.cpp" /> <ClCompile Include="string.cpp" /> + <ClCompile Include="testutils.cpp" /> <ClCompile Include="thread.cpp" /> <ClCompile Include="timer.cpp" /> <ClCompile Include="uid.cpp" /> diff --git a/zencore/zencore.vcxproj.filters b/zencore/zencore.vcxproj.filters index ea0f8a912..6c6b308f8 100644 --- a/zencore/zencore.vcxproj.filters +++ b/zencore/zencore.vcxproj.filters @@ -42,6 +42,7 @@ <ClInclude Include="include\zencore\postwindows.h" /> <ClInclude Include="include\zencore\logging.h" /> <ClInclude Include="include\zencore\session.h" /> + <ClInclude Include="include\zencore\testutils.h" /> </ItemGroup> <ItemGroup> <ClCompile Include="snapshot_manifest.cpp" /> @@ -74,6 +75,7 @@ <ClCompile Include="logging.cpp" /> <ClCompile Include="intmath.cpp" /> <ClCompile Include="session.cpp" /> + <ClCompile Include="testutils.cpp" /> </ItemGroup> <ItemGroup> <Filter Include="CAS"> diff --git a/zenhttp/httpnull.cpp b/zenhttp/httpnull.cpp index 57cba13d3..e49051ac5 100644 --- a/zenhttp/httpnull.cpp +++ b/zenhttp/httpnull.cpp @@ -28,8 +28,10 @@ HttpNullServer::Initialize(int BasePort) } void -HttpNullServer::Run(bool TestMode) +HttpNullServer::Run(bool IsInteractiveSession) { + const bool TestMode = !IsInteractiveSession; + if (TestMode == false) { zen::logging::ConsoleLog().info("Zen Server running (null HTTP). Press ESC or Q to quit"); diff --git a/zenhttp/httpnull.h b/zenhttp/httpnull.h index b15b1b123..867bbe4d2 100644 --- a/zenhttp/httpnull.h +++ b/zenhttp/httpnull.h @@ -19,7 +19,7 @@ public: virtual void RegisterService(HttpService& Service) override; virtual void Initialize(int BasePort) override; - virtual void Run(bool TestMode) override; + virtual void Run(bool IsInteractiveSession) override; virtual void RequestExit() override; private: diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index c2d4ef14c..3c5e5d8d3 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -707,29 +707,30 @@ HttpSysServer::StartServer() } void -HttpSysServer::Run(bool TestMode) +HttpSysServer::Run(bool IsInteractive) { - if (TestMode == false) + if (IsInteractive) { zen::logging::ConsoleLog().info("Zen Server running. Press ESC or Q to quit"); } do { - int WaitTimeout = -1; + // int WaitTimeout = -1; + int WaitTimeout = 100; - if (!TestMode) + if (IsInteractive) { WaitTimeout = 1000; - } - - if (!TestMode && _kbhit() != 0) - { - char c = (char)_getch(); - if (c == 27 || c == 'Q' || c == 'q') + if (_kbhit() != 0) { - RequestApplicationExit(0); + char c = (char)_getch(); + + if (c == 27 || c == 'Q' || c == 'q') + { + RequestApplicationExit(0); + } } } diff --git a/zenhttp/httpuws.cpp b/zenhttp/httpuws.cpp index 992809b17..e062e7747 100644 --- a/zenhttp/httpuws.cpp +++ b/zenhttp/httpuws.cpp @@ -37,8 +37,10 @@ HttpUwsServer::Initialize(int BasePort) } void -HttpUwsServer::Run(bool TestMode) +HttpUwsServer::Run(bool IsInteractive) { + const bool TestMode = !IsInteractive; + if (TestMode == false) { zen::logging::ConsoleLog().info("Zen Server running (null HTTP). Press ESC or Q to quit"); diff --git a/zenhttp/httpuws.h b/zenhttp/httpuws.h index ec55ae2fd..5e300202f 100644 --- a/zenhttp/httpuws.h +++ b/zenhttp/httpuws.h @@ -16,7 +16,7 @@ public: virtual void RegisterService(HttpService& Service) override; virtual void Initialize(int BasePort) override; - virtual void Run(bool TestMode) override; + virtual void Run(bool IsInteractiveSession) override; virtual void RequestExit() override; private: diff --git a/zenhttp/include/zenhttp/httpserver.h b/zenhttp/include/zenhttp/httpserver.h index ed6075c92..6a7dc8a70 100644 --- a/zenhttp/include/zenhttp/httpserver.h +++ b/zenhttp/include/zenhttp/httpserver.h @@ -167,7 +167,7 @@ class HttpServer : public RefCounted public: virtual void RegisterService(HttpService& Service) = 0; virtual void Initialize(int BasePort) = 0; - virtual void Run(bool TestMode) = 0; + virtual void Run(bool IsInteractiveSession) = 0; virtual void RequestExit() = 0; }; diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index 7629e2ea1..a556d896e 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -12,6 +12,7 @@ #include <zencore/iohash.h> #include <zencore/logging.h> #include <zencore/memory.h> +#include <zencore/refcount.h> #include <zencore/stream.h> #include <zencore/string.h> #include <zencore/thread.h> @@ -37,6 +38,7 @@ #include <map> #include <random> #include <span> +#include <unordered_map> #include <atlbase.h> #include <process.h> @@ -80,7 +82,7 @@ class HttpClientConnection static HttpClientConnection* This(http_parser* Parser) { return (HttpClientConnection*)Parser->data; }; public: - HttpClientConnection(asio::io_context& IoContext, HttpConnectionPool& Pool, asio::ip::tcp::socket&& InSocket) + HttpClientConnection(asio::io_context& IoContext, zen::Ref<HttpConnectionPool> Pool, asio::ip::tcp::socket&& InSocket) : m_IoContext(IoContext) , m_Pool(Pool) , m_Resolver(IoContext) @@ -89,8 +91,8 @@ public: } ~HttpClientConnection() {} - HttpConnectionPool& ConnectionPool() { return m_Pool; } - void SetKeepAlive(bool NewState) { m_KeepAlive = NewState; } + zen::Ref<HttpConnectionPool> ConnectionPool() { return m_Pool; } + void SetKeepAlive(bool NewState) { m_KeepAlive = NewState; } void Get(const std::string_view Server, int Port, const std::string_view Path) { @@ -227,16 +229,16 @@ private: } private: - asio::io_context& m_IoContext; - HttpConnectionPool& m_Pool; - asio::ip::tcp::resolver m_Resolver; - asio::ip::tcp::socket m_Socket; - std::string m_Uri; - std::string m_RequestBody; // Initial request data - http_parser m_HttpParser{}; - http_parser_settings m_HttpParserSettings{}; - uint8_t m_ResponseIoBuffer[4096]; - asio::mutable_buffer m_ResponseBuffer{m_ResponseIoBuffer, sizeof m_ResponseIoBuffer}; + asio::io_context& m_IoContext; + zen::Ref<HttpConnectionPool> m_Pool; + asio::ip::tcp::resolver m_Resolver; + asio::ip::tcp::socket m_Socket; + std::string m_Uri; + std::string m_RequestBody; // Initial request data + http_parser m_HttpParser{}; + http_parser_settings m_HttpParserSettings{}; + uint8_t m_ResponseIoBuffer[4096]; + asio::mutable_buffer m_ResponseBuffer{m_ResponseIoBuffer, sizeof m_ResponseIoBuffer}; enum class RequestState { @@ -259,7 +261,7 @@ private: ////////////////////////////////////////////////////////////////////////// -class HttpConnectionPool +class HttpConnectionPool : public zen::RefCounted { public: HttpConnectionPool(asio::io_context& Context, std::string_view HostName, uint16_t Port); @@ -322,7 +324,7 @@ HttpConnectionPool::GetConnection() return nullptr; } - return std::make_unique<HttpClientConnection>(m_Context, *this, std::move(Socket)); + return std::make_unique<HttpClientConnection>(m_Context, this, std::move(Socket)); } std::unique_ptr<HttpClientConnection> Connection{m_AvailableConnections.back()}; @@ -347,15 +349,15 @@ public: std::unique_ptr<HttpClientConnection> GetConnection(std::string_view HostName, uint16_t Port) { - return ConnectionPool(HostName, Port).GetConnection(); + return ConnectionPool(HostName, Port)->GetConnection(); } void ReturnConnection(std::unique_ptr<HttpClientConnection> Connection) { - Connection->ConnectionPool().ReturnConnection(std::move(Connection)); + Connection->ConnectionPool()->ReturnConnection(std::move(Connection)); } - HttpConnectionPool& ConnectionPool(std::string_view HostName, uint16_t Port) + zen::Ref<HttpConnectionPool> ConnectionPool(std::string_view HostName, uint16_t Port) { zen::RwLock::ExclusiveLockScope _(m_Lock); ConnectionId ConnId{std::string(HostName), Port}; @@ -364,7 +366,7 @@ public: { // Not found - create new entry - auto In = m_ConnectionPools.insert({ConnId, std::move(HttpConnectionPool(m_Context, HostName, Port))}); + auto In = m_ConnectionPools.emplace(ConnId, new HttpConnectionPool(m_Context, HostName, Port)); return In.first->second; } @@ -393,8 +395,8 @@ private: uint16_t Port; }; - zen::RwLock m_Lock; - std::map<ConnectionId, HttpConnectionPool> m_ConnectionPools; + zen::RwLock m_Lock; + std::map<ConnectionId, zen::Ref<HttpConnectionPool>> m_ConnectionPools; }; ////////////////////////////////////////////////////////////////////////// @@ -670,7 +672,7 @@ main(int argc, char** argv) zen::logging::InitializeLogging(); spdlog::set_level(spdlog::level::debug); - spdlog::set_formatter(std::make_unique< ::logging::full_formatter>("test", std::chrono::system_clock::now())); + spdlog::set_formatter(std::make_unique<::logging::full_formatter>("test", std::chrono::system_clock::now())); std::filesystem::path ProgramBaseDir = std::filesystem::path(argv[0]).parent_path(); std::filesystem::path TestBaseDir = ProgramBaseDir.parent_path().parent_path() / ".test"; @@ -1919,9 +1921,9 @@ public: ZenServerInstance& GetInstance(int Index) { return *m_Instances[Index]; } private: - std::string m_HelperId; - int m_ServerCount = 0; - std::vector<std::unique_ptr<ZenServerInstance> > m_Instances; + std::string m_HelperId; + int m_ServerCount = 0; + std::vector<std::unique_ptr<ZenServerInstance>> m_Instances; }; TEST_CASE("http.basics") diff --git a/zenserver/config.cpp b/zenserver/config.cpp index 30401a52e..164d2a792 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -98,6 +98,21 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z cxxopts::value<std::string>(GlobalOptions.ChildId), "<identifier>"); +#if ZEN_PLATFORM_WINDOWS + options.add_option("lifetime", + "", + "install", + "Install zenserver as a Windows service", + cxxopts::value<bool>(GlobalOptions.InstallService), + ""); + options.add_option("lifetime", + "", + "uninstall", + "Uninstall zenserver as a Windows service", + cxxopts::value<bool>(GlobalOptions.UninstallService), + ""); +#endif + options.add_option("network", "p", "port", diff --git a/zenserver/config.h b/zenserver/config.h index c3f08f84f..6ade1b401 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -9,12 +9,14 @@ struct ZenServerOptions { bool IsDebug = false; bool IsTest = false; - bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements - int BasePort = 1337; // Service listen port (used for both UDP and TCP) - int OwnerPid = 0; // Parent process id (zero for standalone) - std::string ChildId; // Id assigned by parent process (used for lifetime management) - std::string LogId; // Id for tagging log output - std::filesystem::path DataDir; // Root directory for state (used for testing) + bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements + int BasePort = 1337; // Service listen port (used for both UDP and TCP) + int OwnerPid = 0; // Parent process id (zero for standalone) + std::string ChildId; // Id assigned by parent process (used for lifetime management) + bool InstallService = false; // Flag used to initiate service install (temporary) + bool UninstallService = false; // Flag used to initiate service uninstall (temporary) + std::string LogId; // Id for tagging log output + std::filesystem::path DataDir; // Root directory for state (used for testing) }; struct ZenUpstreamJupiterConfig diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp index 4ba4835af..41b140f90 100644 --- a/zenserver/diag/logging.cpp +++ b/zenserver/diag/logging.cpp @@ -9,6 +9,9 @@ #include <spdlog/pattern_formatter.h> #include <spdlog/sinks/ansicolor_sink.h> #include <spdlog/sinks/basic_file_sink.h> +#include <spdlog/sinks/daily_file_sink.h> +#include <spdlog/sinks/msvc_sink.h> +#include <spdlog/sinks/rotating_file_sink.h> #include <spdlog/sinks/stdout_color_sinks.h> #include <spdlog/spdlog.h> #include <zencore/string.h> @@ -207,7 +210,7 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) if (GlobalOptions.IsTest) { LogLevel = spdlog::level::trace; - IsAsync = false; + IsAsync = false; } if (IsAsync) @@ -223,7 +226,19 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) // Sinks auto ConsoleSink = std::make_shared<spdlog::sinks::ansicolor_stdout_sink_mt>(); - auto FileSink = std::make_shared<spdlog::sinks::basic_file_sink_mt>(zen::WideToUtf8(LogPath.c_str()), /* truncate */ true); + +#if 0 + auto FileSink = std::make_shared<spdlog::sinks::daily_file_sink_mt>(zen::WideToUtf8(LogPath.c_str()), + 0, + 0, + /* truncate */ false, + uint16_t(/* max files */ 14)); +#else + auto FileSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::WideToUtf8(LogPath.c_str()), + /* max size */ 128 * 1024 * 1024, + /* max files */ 16, + /* rotate on open */ true); +#endif // Default @@ -234,20 +249,30 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) Sinks.push_back(ConsoleSink); Sinks.push_back(FileSink); +#if ZEN_PLATFORM_WINDOWS + if (zen::IsDebuggerPresent()) + { + auto DebugSink = std::make_shared<spdlog::sinks::msvc_sink_mt>(); + DebugSink->set_level(spdlog::level::debug); + Sinks.push_back(DebugSink); + } +#endif + // Jupiter - only log HTTP traffic to file auto JupiterLogger = std::make_shared<spdlog::logger>("jupiter", FileSink); spdlog::register_logger(JupiterLogger); - JupiterLogger->set_level(LogLevel); + + // Zen - only log HTTP traffic to file auto ZenClientLogger = std::make_shared<spdlog::logger>("zenclient", FileSink); spdlog::register_logger(ZenClientLogger); - ZenClientLogger->set_level(LogLevel); // Configure all registered loggers according to settings spdlog::set_level(LogLevel); spdlog::flush_on(spdlog::level::err); + spdlog::flush_every(std::chrono::seconds{2}); spdlog::set_formatter(std::make_unique<logging::full_formatter>(GlobalOptions.LogId, std::chrono::system_clock::now())); } diff --git a/zenserver/experimental/usnjournal.cpp b/zenserver/experimental/usnjournal.cpp index ab83b8a1c..1e765fbe5 100644 --- a/zenserver/experimental/usnjournal.cpp +++ b/zenserver/experimental/usnjournal.cpp @@ -34,14 +34,14 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) if (!Success) { - zen::ThrowSystemException("GetVolumePathName failed"); + zen::ThrowLastError("GetVolumePathName failed"); } Success = GetVolumeNameForVolumeMountPoint(VolumePathName, VolumeName, ZEN_ARRAY_COUNT(VolumeName)); if (!Success) { - zen::ThrowSystemException("GetVolumeNameForVolumeMountPoint failed"); + zen::ThrowLastError("GetVolumeNameForVolumeMountPoint failed"); } // Chop off trailing slash since we want to open a volume handle, not a handle to the volume root directory @@ -64,7 +64,7 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) if (m_VolumeHandle == INVALID_HANDLE_VALUE) { - ThrowSystemException("Volume handle open failed"); + ThrowLastError("Volume handle open failed"); } // Figure out which file system is in use for volume @@ -86,7 +86,7 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) if (!Success) { - ThrowSystemException("Failed to get volume information"); + ThrowLastError("Failed to get volume information"); } ZEN_DEBUG("File system type is {}", WideToUtf8(FileSystemName)); @@ -173,7 +173,7 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) if (!Success) { - ThrowSystemException("GetFileInformationByHandleEx failed"); + ThrowLastError("GetFileInformationByHandleEx failed"); } const Frn VolumeRootFrn = FileInformation.FileId; diff --git a/zenserver/windows/service.cpp b/zenserver/windows/service.cpp new file mode 100644 index 000000000..017b5f9a7 --- /dev/null +++ b/zenserver/windows/service.cpp @@ -0,0 +1,631 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "service.h" + +#include <zencore/zencore.h> + +#include <stdio.h> +#include <tchar.h> +#include <zencore/windows.h> + +#define SVCNAME L"Zen Store" + +SERVICE_STATUS gSvcStatus; +SERVICE_STATUS_HANDLE gSvcStatusHandle; +HANDLE ghSvcStopEvent = NULL; + +void SvcInstall(void); + +void ReportSvcStatus(DWORD, DWORD, DWORD); +void SvcReportEvent(LPTSTR); + +WindowsService::WindowsService() +{ +} + +WindowsService::~WindowsService() +{ +} + +// +// Purpose: +// Installs a service in the SCM database +// +// Parameters: +// None +// +// Return value: +// None +// +VOID +WindowsService::Install() +{ + SC_HANDLE schSCManager; + SC_HANDLE schService; + TCHAR szPath[MAX_PATH]; + + if (!GetModuleFileName(NULL, szPath, MAX_PATH)) + { + printf("Cannot install service (%d)\n", GetLastError()); + return; + } + + // Get a handle to the SCM database. + + schSCManager = OpenSCManager(NULL, // local computer + NULL, // ServicesActive database + SC_MANAGER_ALL_ACCESS); // full access rights + + if (NULL == schSCManager) + { + printf("OpenSCManager failed (%d)\n", GetLastError()); + return; + } + + // Create the service + + schService = CreateService(schSCManager, // SCM database + SVCNAME, // name of service + SVCNAME, // service name to display + SERVICE_ALL_ACCESS, // desired access + SERVICE_WIN32_OWN_PROCESS, // service type + SERVICE_DEMAND_START, // start type + SERVICE_ERROR_NORMAL, // error control type + szPath, // path to service's binary + NULL, // no load ordering group + NULL, // no tag identifier + NULL, // no dependencies + NULL, // LocalSystem account + NULL); // no password + + if (schService == NULL) + { + printf("CreateService failed (%d)\n", GetLastError()); + CloseServiceHandle(schSCManager); + return; + } + else + printf("Service installed successfully\n"); + + CloseServiceHandle(schService); + CloseServiceHandle(schSCManager); +} + +void +WindowsService::Delete() +{ + SC_HANDLE schSCManager; + SC_HANDLE schService; + + // Get a handle to the SCM database. + + schSCManager = OpenSCManager(NULL, // local computer + NULL, // ServicesActive database + SC_MANAGER_ALL_ACCESS); // full access rights + + if (NULL == schSCManager) + { + printf("OpenSCManager failed (%d)\n", GetLastError()); + return; + } + + // Get a handle to the service. + + schService = OpenService(schSCManager, // SCM database + SVCNAME, // name of service + DELETE); // need delete access + + if (schService == NULL) + { + printf("OpenService failed (%d)\n", GetLastError()); + CloseServiceHandle(schSCManager); + return; + } + + // Delete the service. + + if (!DeleteService(schService)) + { + printf("DeleteService failed (%d)\n", GetLastError()); + } + else + printf("Service deleted successfully\n"); + + CloseServiceHandle(schService); + CloseServiceHandle(schSCManager); +} + +WindowsService* gSvc; + +void WINAPI +CallMain(DWORD, LPSTR*) +{ + gSvc->SvcMain(); +} + +int +WindowsService::ServiceMain() +{ + if (zen::IsInteractiveSession()) + { + // Not actually running as a service + return Run(); + } + else + { + gSvc = this; + + SERVICE_TABLE_ENTRY DispatchTable[] = {{(LPWSTR)SVCNAME, (LPSERVICE_MAIN_FUNCTION)&CallMain}, {NULL, NULL}}; + + // This call returns when the service has stopped. + // The process should simply terminate when the call returns. + + if (!StartServiceCtrlDispatcher(DispatchTable)) + { + SvcReportEvent((LPTSTR)L"StartServiceCtrlDispatcher"); + } + } + + return 0; +} + +int +WindowsService::SvcMain() +{ + // Register the handler function for the service + + gSvcStatusHandle = RegisterServiceCtrlHandler(SVCNAME, SvcCtrlHandler); + + if (!gSvcStatusHandle) + { + SvcReportEvent((LPTSTR)TEXT("RegisterServiceCtrlHandler")); + + return 1; + } + + // These SERVICE_STATUS members remain as set here + + gSvcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS; + gSvcStatus.dwServiceSpecificExitCode = 0; + + // Report initial status to the SCM + + ReportSvcStatus(SERVICE_START_PENDING, NO_ERROR, 3000); + + // Create an event. The control handler function, SvcCtrlHandler, + // signals this event when it receives the stop control code. + + ghSvcStopEvent = CreateEvent(NULL, // default security attributes + TRUE, // manual reset event + FALSE, // not signaled + NULL); // no name + + if (ghSvcStopEvent == NULL) + { + ReportSvcStatus(SERVICE_STOPPED, GetLastError(), 0); + + return 1; + } + + // Report running status when initialization is complete. + + ReportSvcStatus(SERVICE_RUNNING, NO_ERROR, 0); + + int ReturnCode = Run(); + + ReportSvcStatus(SERVICE_STOPPED, NO_ERROR, 0); + + return ReturnCode; +} + +// +// Purpose: +// Retrieves and displays the current service configuration. +// +// Parameters: +// None +// +// Return value: +// None +// +void +DoQuerySvc() +{ + SC_HANDLE schSCManager{}; + SC_HANDLE schService{}; + LPQUERY_SERVICE_CONFIG lpsc{}; + LPSERVICE_DESCRIPTION lpsd{}; + DWORD dwBytesNeeded{}, cbBufSize{}, dwError{}; + + // Get a handle to the SCM database. + + schSCManager = OpenSCManager(NULL, // local computer + NULL, // ServicesActive database + SC_MANAGER_ALL_ACCESS); // full access rights + + if (NULL == schSCManager) + { + printf("OpenSCManager failed (%d)\n", GetLastError()); + return; + } + + // Get a handle to the service. + + schService = OpenService(schSCManager, // SCM database + SVCNAME, // name of service + SERVICE_QUERY_CONFIG); // need query config access + + if (schService == NULL) + { + printf("OpenService failed (%d)\n", GetLastError()); + CloseServiceHandle(schSCManager); + return; + } + + // Get the configuration information. + + if (!QueryServiceConfig(schService, NULL, 0, &dwBytesNeeded)) + { + dwError = GetLastError(); + if (ERROR_INSUFFICIENT_BUFFER == dwError) + { + cbBufSize = dwBytesNeeded; + lpsc = (LPQUERY_SERVICE_CONFIG)LocalAlloc(LMEM_FIXED, cbBufSize); + } + else + { + printf("QueryServiceConfig failed (%d)", dwError); + goto cleanup; + } + } + + if (!QueryServiceConfig(schService, lpsc, cbBufSize, &dwBytesNeeded)) + { + printf("QueryServiceConfig failed (%d)", GetLastError()); + goto cleanup; + } + + if (!QueryServiceConfig2(schService, SERVICE_CONFIG_DESCRIPTION, NULL, 0, &dwBytesNeeded)) + { + dwError = GetLastError(); + if (ERROR_INSUFFICIENT_BUFFER == dwError) + { + cbBufSize = dwBytesNeeded; + lpsd = (LPSERVICE_DESCRIPTION)LocalAlloc(LMEM_FIXED, cbBufSize); + } + else + { + printf("QueryServiceConfig2 failed (%d)", dwError); + goto cleanup; + } + } + + if (!QueryServiceConfig2(schService, SERVICE_CONFIG_DESCRIPTION, (LPBYTE)lpsd, cbBufSize, &dwBytesNeeded)) + { + printf("QueryServiceConfig2 failed (%d)", GetLastError()); + goto cleanup; + } + + // Print the configuration information. + + _tprintf(TEXT("%s configuration: \n"), SVCNAME); + _tprintf(TEXT(" Type: 0x%x\n"), lpsc->dwServiceType); + _tprintf(TEXT(" Start Type: 0x%x\n"), lpsc->dwStartType); + _tprintf(TEXT(" Error Control: 0x%x\n"), lpsc->dwErrorControl); + _tprintf(TEXT(" Binary path: %s\n"), lpsc->lpBinaryPathName); + _tprintf(TEXT(" Account: %s\n"), lpsc->lpServiceStartName); + + if (lpsd->lpDescription != NULL && lstrcmp(lpsd->lpDescription, TEXT("")) != 0) + _tprintf(TEXT(" Description: %s\n"), lpsd->lpDescription); + if (lpsc->lpLoadOrderGroup != NULL && lstrcmp(lpsc->lpLoadOrderGroup, TEXT("")) != 0) + _tprintf(TEXT(" Load order group: %s\n"), lpsc->lpLoadOrderGroup); + if (lpsc->dwTagId != 0) + _tprintf(TEXT(" Tag ID: %d\n"), lpsc->dwTagId); + if (lpsc->lpDependencies != NULL && lstrcmp(lpsc->lpDependencies, TEXT("")) != 0) + _tprintf(TEXT(" Dependencies: %s\n"), lpsc->lpDependencies); + + LocalFree(lpsc); + LocalFree(lpsd); + +cleanup: + CloseServiceHandle(schService); + CloseServiceHandle(schSCManager); +} + +// +// Purpose: +// Disables the service. +// +// Parameters: +// None +// +// Return value: +// None +// +void +DoDisableSvc() +{ + SC_HANDLE schSCManager; + SC_HANDLE schService; + + // Get a handle to the SCM database. + + schSCManager = OpenSCManager(NULL, // local computer + NULL, // ServicesActive database + SC_MANAGER_ALL_ACCESS); // full access rights + + if (NULL == schSCManager) + { + printf("OpenSCManager failed (%d)\n", GetLastError()); + return; + } + + // Get a handle to the service. + + schService = OpenService(schSCManager, // SCM database + SVCNAME, // name of service + SERVICE_CHANGE_CONFIG); // need change config access + + if (schService == NULL) + { + printf("OpenService failed (%d)\n", GetLastError()); + CloseServiceHandle(schSCManager); + return; + } + + // Change the service start type. + + if (!ChangeServiceConfig(schService, // handle of service + SERVICE_NO_CHANGE, // service type: no change + SERVICE_DISABLED, // service start type + SERVICE_NO_CHANGE, // error control: no change + NULL, // binary path: no change + NULL, // load order group: no change + NULL, // tag ID: no change + NULL, // dependencies: no change + NULL, // account name: no change + NULL, // password: no change + NULL)) // display name: no change + { + printf("ChangeServiceConfig failed (%d)\n", GetLastError()); + } + else + printf("Service disabled successfully.\n"); + + CloseServiceHandle(schService); + CloseServiceHandle(schSCManager); +} + +// +// Purpose: +// Enables the service. +// +// Parameters: +// None +// +// Return value: +// None +// +VOID __stdcall DoEnableSvc() +{ + SC_HANDLE schSCManager; + SC_HANDLE schService; + + // Get a handle to the SCM database. + + schSCManager = OpenSCManager(NULL, // local computer + NULL, // ServicesActive database + SC_MANAGER_ALL_ACCESS); // full access rights + + if (NULL == schSCManager) + { + printf("OpenSCManager failed (%d)\n", GetLastError()); + return; + } + + // Get a handle to the service. + + schService = OpenService(schSCManager, // SCM database + SVCNAME, // name of service + SERVICE_CHANGE_CONFIG); // need change config access + + if (schService == NULL) + { + printf("OpenService failed (%d)\n", GetLastError()); + CloseServiceHandle(schSCManager); + return; + } + + // Change the service start type. + + if (!ChangeServiceConfig(schService, // handle of service + SERVICE_NO_CHANGE, // service type: no change + SERVICE_DEMAND_START, // service start type + SERVICE_NO_CHANGE, // error control: no change + NULL, // binary path: no change + NULL, // load order group: no change + NULL, // tag ID: no change + NULL, // dependencies: no change + NULL, // account name: no change + NULL, // password: no change + NULL)) // display name: no change + { + printf("ChangeServiceConfig failed (%d)\n", GetLastError()); + } + else + printf("Service enabled successfully.\n"); + + CloseServiceHandle(schService); + CloseServiceHandle(schSCManager); +} +// +// Purpose: +// Updates the service description to "This is a test description". +// +// Parameters: +// None +// +// Return value: +// None +// +void +DoUpdateSvcDesc() +{ + SC_HANDLE schSCManager; + SC_HANDLE schService; + SERVICE_DESCRIPTION sd; + TCHAR szDesc[] = TEXT("This is a test description"); + + // Get a handle to the SCM database. + + schSCManager = OpenSCManager(NULL, // local computer + NULL, // ServicesActive database + SC_MANAGER_ALL_ACCESS); // full access rights + + if (NULL == schSCManager) + { + printf("OpenSCManager failed (%d)\n", GetLastError()); + return; + } + + // Get a handle to the service. + + schService = OpenService(schSCManager, // SCM database + SVCNAME, // name of service + SERVICE_CHANGE_CONFIG); // need change config access + + if (schService == NULL) + { + printf("OpenService failed (%d)\n", GetLastError()); + CloseServiceHandle(schSCManager); + return; + } + + // Change the service description. + + sd.lpDescription = szDesc; + + if (!ChangeServiceConfig2(schService, // handle to service + SERVICE_CONFIG_DESCRIPTION, // change: description + &sd)) // new description + { + printf("ChangeServiceConfig2 failed\n"); + } + else + printf("Service description updated successfully.\n"); + + CloseServiceHandle(schService); + CloseServiceHandle(schSCManager); +} + +// +// Purpose: +// Sets the current service status and reports it to the SCM. +// +// Parameters: +// dwCurrentState - The current state (see SERVICE_STATUS) +// dwWin32ExitCode - The system error code +// dwWaitHint - Estimated time for pending operation, +// in milliseconds +// +// Return value: +// None +// +VOID +ReportSvcStatus(DWORD dwCurrentState, DWORD dwWin32ExitCode, DWORD dwWaitHint) +{ + static DWORD dwCheckPoint = 1; + + // Fill in the SERVICE_STATUS structure. + + gSvcStatus.dwCurrentState = dwCurrentState; + gSvcStatus.dwWin32ExitCode = dwWin32ExitCode; + gSvcStatus.dwWaitHint = dwWaitHint; + + if (dwCurrentState == SERVICE_START_PENDING) + gSvcStatus.dwControlsAccepted = 0; + else + gSvcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP; + + if ((dwCurrentState == SERVICE_RUNNING) || (dwCurrentState == SERVICE_STOPPED)) + gSvcStatus.dwCheckPoint = 0; + else + gSvcStatus.dwCheckPoint = dwCheckPoint++; + + // Report the status of the service to the SCM. + SetServiceStatus(gSvcStatusHandle, &gSvcStatus); +} + +void +WindowsService::SvcCtrlHandler(DWORD dwCtrl) +{ + // Handle the requested control code. + // + // Called by SCM whenever a control code is sent to the service + // using the ControlService function. + + switch (dwCtrl) + { + case SERVICE_CONTROL_STOP: + ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, 0); + + // Signal the service to stop. + + SetEvent(ghSvcStopEvent); + zen::RequestApplicationExit(0); + + ReportSvcStatus(gSvcStatus.dwCurrentState, NO_ERROR, 0); + return; + + case SERVICE_CONTROL_INTERROGATE: + break; + + default: + break; + } +} + +// +// Purpose: +// Logs messages to the event log +// +// Parameters: +// szFunction - name of function that failed +// +// Return value: +// None +// +// Remarks: +// The service must have an entry in the Application event log. +// +VOID +SvcReportEvent(LPTSTR szFunction) +{ + ZEN_UNUSED(szFunction); + + // HANDLE hEventSource; + // LPCTSTR lpszStrings[2]; + // TCHAR Buffer[80]; + + // hEventSource = RegisterEventSource(NULL, SVCNAME); + + // if (NULL != hEventSource) + //{ + // StringCchPrintf(Buffer, 80, TEXT("%s failed with %d"), szFunction, GetLastError()); + + // lpszStrings[0] = SVCNAME; + // lpszStrings[1] = Buffer; + + // ReportEvent(hEventSource, // event log handle + // EVENTLOG_ERROR_TYPE, // event type + // 0, // event category + // SVC_ERROR, // event identifier + // NULL, // no security identifier + // 2, // size of lpszStrings array + // 0, // no binary data + // lpszStrings, // array of strings + // NULL); // no binary data + + // DeregisterEventSource(hEventSource); + //} +} diff --git a/zenserver/windows/service.h b/zenserver/windows/service.h new file mode 100644 index 000000000..7c9610983 --- /dev/null +++ b/zenserver/windows/service.h @@ -0,0 +1,20 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +class WindowsService +{ +public: + WindowsService(); + ~WindowsService(); + + virtual int Run() = 0; + + int ServiceMain(); + + static void Install(); + static void Delete(); + + int SvcMain(); + static void __stdcall SvcCtrlHandler(unsigned long); +}; diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua index bb70846fa..7a6981fcd 100644 --- a/zenserver/xmake.lua +++ b/zenserver/xmake.lua @@ -14,6 +14,8 @@ target("zenserver") add_ldflags("/MANIFEST:EMBED") add_ldflags("/MANIFESTUAC:level='requireAdministrator'") add_ldflags("/LTCG") + else + del_files("windows/**") end add_options("vfs") diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 63e94ab7d..ea4a2915e 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -40,6 +40,10 @@ #include "config.h" #include "diag/logging.h" +#if ZEN_PLATFORM_WINDOWS +# include "windows/service.h" +#endif + ////////////////////////////////////////////////////////////////////////// // Sentry // @@ -312,7 +316,9 @@ public: __debugbreak(); } - m_Http->Run(m_TestMode); + const bool IsInteractiveMode = zen::IsInteractiveSession() && !m_TestMode; + + m_Http->Run(IsInteractiveMode); ZEN_INFO(ZEN_APP_NAME " exiting"); @@ -432,17 +438,29 @@ private: } // namespace zen -int -main(int argc, char* argv[]) +class ZenWindowsService : public WindowsService { - using namespace zen; +public: + ZenWindowsService(ZenServerOptions& GlobalOptions, ZenServiceConfig& ServiceConfig) + : m_GlobalOptions(GlobalOptions) + , m_ServiceConfig(ServiceConfig) + { + } - mi_version(); + ZenWindowsService(const ZenWindowsService&) = delete; + ZenWindowsService& operator=(const ZenWindowsService&) = delete; - ZenServerOptions GlobalOptions; - ZenServiceConfig ServiceConfig; - ParseGlobalCliOptions(argc, argv, GlobalOptions, ServiceConfig); - InitializeLogging(GlobalOptions); + virtual int Run() override; + +private: + ZenServerOptions& m_GlobalOptions; + ZenServiceConfig& m_ServiceConfig; +}; + +int +ZenWindowsService::Run() +{ + using namespace zen; #if USE_SENTRY // Initialize sentry.io client @@ -451,9 +469,12 @@ main(int argc, char* argv[]) sentry_options_set_dsn(SentryOptions, "https://[email protected]/5919284"); sentry_init(SentryOptions); - auto _ = zen::MakeGuard([&] { sentry_close(); }); + auto _ = zen::MakeGuard([] { sentry_close(); }); #endif + auto& GlobalOptions = m_GlobalOptions; + auto& ServiceConfig = m_ServiceConfig; + try { // Prototype config system, we'll see how this pans out @@ -539,3 +560,35 @@ main(int argc, char* argv[]) return 0; } + +int +main(int argc, char* argv[]) +{ + using namespace zen; + + mi_version(); + + ZenServerOptions GlobalOptions; + ZenServiceConfig ServiceConfig; + ParseGlobalCliOptions(argc, argv, GlobalOptions, ServiceConfig); + InitializeLogging(GlobalOptions); + +#if ZEN_PLATFORM_WINDOWS + if (GlobalOptions.InstallService) + { + WindowsService::Install(); + + std::exit(0); + } + + if (GlobalOptions.UninstallService) + { + WindowsService::Delete(); + + std::exit(0); + } +#endif + + ZenWindowsService App(GlobalOptions, ServiceConfig); + return App.ServiceMain(); +} diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj index aa9d538a5..db657d192 100644 --- a/zenserver/zenserver.vcxproj +++ b/zenserver/zenserver.vcxproj @@ -123,6 +123,7 @@ <ClInclude Include="upstream\upstreamcache.h" /> <ClInclude Include="upstream\zen.h" /> <ClInclude Include="vfs.h" /> + <ClInclude Include="windows\service.h" /> </ItemGroup> <ItemGroup> <ClCompile Include="cache\structuredcache.cpp" /> @@ -142,6 +143,7 @@ <ClCompile Include="upstream\upstreamcache.cpp" /> <ClCompile Include="upstream\zen.cpp" /> <ClCompile Include="vfs.cpp" /> + <ClCompile Include="windows\service.cpp" /> <ClCompile Include="zenserver.cpp" /> </ItemGroup> <ItemGroup> diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters index a86a6d96d..250c55812 100644 --- a/zenserver/zenserver.vcxproj.filters +++ b/zenserver/zenserver.vcxproj.filters @@ -39,6 +39,7 @@ <Filter>upstream</Filter> </ClInclude> <ClInclude Include="testing\httptest.h" /> + <ClInclude Include="windows\service.h" /> </ItemGroup> <ItemGroup> <ClCompile Include="zenserver.cpp" /> @@ -73,6 +74,7 @@ <Filter>upstream</Filter> </ClCompile> <ClCompile Include="testing\httptest.cpp" /> + <ClCompile Include="windows\service.cpp" /> </ItemGroup> <ItemGroup> <Filter Include="cache"> diff --git a/zenstore-test/xmake.lua b/zenstore-test/xmake.lua new file mode 100644 index 000000000..c8995dab2 --- /dev/null +++ b/zenstore-test/xmake.lua @@ -0,0 +1,5 @@ +target("zenstore-test") + set_kind("binary") + add_files("*.cpp") + add_deps("zenstore", "zencore") + add_packages("vcpkg::doctest") diff --git a/zenstore-test/zenstore-test.cpp b/zenstore-test/zenstore-test.cpp new file mode 100644 index 000000000..ed9a12566 --- /dev/null +++ b/zenstore-test/zenstore-test.cpp @@ -0,0 +1,23 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/logging.h> +#include <zencore/zencore.h> +#include <zenstore/zenstore.h> + +#define DOCTEST_CONFIG_IMPLEMENT +#include <doctest/doctest.h> +#undef DOCTEST_CONFIG_IMPLEMENT + +void +forceLinkTests() +{ + zen::zenstore_forcelinktests(); +} + +int +main(int argc, char* argv[]) +{ + zen::logging::InitializeLogging(); + + return doctest::Context(argc, argv).run(); +} diff --git a/zenstore-test/zenstore-test.vcxproj b/zenstore-test/zenstore-test.vcxproj new file mode 100644 index 000000000..201594a25 --- /dev/null +++ b/zenstore-test/zenstore-test.vcxproj @@ -0,0 +1,121 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup Label="ProjectConfigurations"> + <ProjectConfiguration Include="Debug|x64"> + <Configuration>Debug</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|x64"> + <Configuration>Release</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + </ItemGroup> + <PropertyGroup Label="Globals"> + <VCProjectVersion>15.0</VCProjectVersion> + <ProjectGuid>{C001A3DF-B76E-4989-B576-FE2B78AB2580}</ProjectGuid> + <Keyword>Win32Proj</Keyword> + <RootNamespace>zenstoretest</RootNamespace> + <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> + <ConfigurationType>Application</ConfigurationType> + <UseDebugLibraries>true</UseDebugLibraries> + <PlatformToolset>v142</PlatformToolset> + <CharacterSet>Unicode</CharacterSet> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> + <ConfigurationType>Application</ConfigurationType> + <UseDebugLibraries>false</UseDebugLibraries> + <PlatformToolset>v142</PlatformToolset> + <WholeProgramOptimization>false</WholeProgramOptimization> + <CharacterSet>Unicode</CharacterSet> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <ImportGroup Label="ExtensionSettings"> + </ImportGroup> + <ImportGroup Label="Shared"> + </ImportGroup> + <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + <Import Project="..\zenfs_common.props" /> + </ImportGroup> + <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + <Import Project="..\zenfs_common.props" /> + </ImportGroup> + <PropertyGroup Label="UserMacros" /> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <LinkIncremental>true</LinkIncremental> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <LinkIncremental>false</LinkIncremental> + </PropertyGroup> + <PropertyGroup Label="Vcpkg" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <VcpkgEnableManifest>true</VcpkgEnableManifest> + <VcpkgUseStatic>true</VcpkgUseStatic> + <VcpkgAdditionalInstallOptions>--overlay-ports=$(SolutionDir)vcpkg_overlay-ports</VcpkgAdditionalInstallOptions> + </PropertyGroup> + <PropertyGroup Label="Vcpkg" Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <VcpkgEnableManifest>true</VcpkgEnableManifest> + <VcpkgUseStatic>true</VcpkgUseStatic> + <VcpkgAdditionalInstallOptions>--overlay-ports=$(SolutionDir)vcpkg_overlay-ports</VcpkgAdditionalInstallOptions> + </PropertyGroup> + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <Optimization>Disabled</Optimization> + <SDLCheck>true</SDLCheck> + <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <ConformanceMode>true</ConformanceMode> + <LanguageStandard>stdcpplatest</LanguageStandard> + <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <GenerateDebugInformation>true</GenerateDebugInformation> + <AdditionalDependencies>kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories> + </Link> + <ProjectReference /> + </ItemDefinitionGroup> + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <Optimization>MaxSpeed</Optimization> + <FunctionLevelLinking>true</FunctionLevelLinking> + <IntrinsicFunctions>true</IntrinsicFunctions> + <SDLCheck>true</SDLCheck> + <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <ConformanceMode>true</ConformanceMode> + <WholeProgramOptimization>false</WholeProgramOptimization> + <LanguageStandard>stdcpplatest</LanguageStandard> + <RuntimeLibrary>MultiThreaded</RuntimeLibrary> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <EnableCOMDATFolding>true</EnableCOMDATFolding> + <OptimizeReferences>true</OptimizeReferences> + <GenerateDebugInformation>true</GenerateDebugInformation> + <ShowProgress>NotSet</ShowProgress> + </Link> + <ProjectReference /> + </ItemDefinitionGroup> + <ItemGroup> + <ClInclude Include="targetver.h" /> + </ItemGroup> + <ItemGroup> + <ClCompile Include="zenstore-test.cpp" /> + </ItemGroup> + <ItemGroup> + <ProjectReference Include="..\zencore\zencore.vcxproj"> + <Project>{d75bf9ab-c61e-4fff-ad59-1563430f05e2}</Project> + </ProjectReference> + <ProjectReference Include="..\zenstore\zenstore.vcxproj"> + <Project>{26cbbaeb-14c1-4efc-877d-80f48215651c}</Project> + </ProjectReference> + </ItemGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <ImportGroup Label="ExtensionTargets"> + </ImportGroup> +</Project>
\ No newline at end of file diff --git a/zenstore-test/zenstore-test.vcxproj.filters b/zenstore-test/zenstore-test.vcxproj.filters new file mode 100644 index 000000000..000599c79 --- /dev/null +++ b/zenstore-test/zenstore-test.vcxproj.filters @@ -0,0 +1,9 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup> + <ClInclude Include="targetver.h" /> + </ItemGroup> + <ItemGroup> + <ClCompile Include="zenstore-test.cpp" /> + </ItemGroup> +</Project>
\ No newline at end of file diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp index e77c0ed64..a143230d3 100644 --- a/zenstore/CAS.cpp +++ b/zenstore/CAS.cpp @@ -11,6 +11,7 @@ #include <zencore/logging.h> #include <zencore/memory.h> #include <zencore/string.h> +#include <zencore/testutils.h> #include <zencore/thread.h> #include <zencore/uid.h> @@ -20,15 +21,23 @@ #include <functional> #include <unordered_map> -struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive- -#include <atlfile.h> - ////////////////////////////////////////////////////////////////////////// namespace zen { +void +ScrubContext::ReportBadChunks(std::span<IoHash> BadChunks) +{ + ZEN_UNUSED(BadChunks); +} + /** - * Slightly less naive CAS store + * CAS store implementation + * + * Uses a basic strategy of splitting payloads by size, to improve ability to reclaim space + * quickly for unused large chunks and to maintain locality for small chunks which are + * frequently accessed together. + * */ class CasImpl : public CasStore { @@ -41,16 +50,15 @@ public: virtual IoBuffer FindChunk(const IoHash& ChunkHash) override; virtual void FilterChunks(CasChunkSet& InOutChunks) override; virtual void Flush() override; + virtual void Scrub(ScrubContext& Ctx) override; private: - void PickDefaultDirectory(); - CasContainerStrategy m_TinyStrategy; CasContainerStrategy m_SmallStrategy; FileCasStrategy m_LargeStrategy; }; -CasImpl::CasImpl() : m_TinyStrategy(m_Config, m_Stats), m_SmallStrategy(m_Config, m_Stats), m_LargeStrategy(m_Config, m_Stats) +CasImpl::CasImpl() : m_TinyStrategy(m_Config), m_SmallStrategy(m_Config), m_LargeStrategy(m_Config) { } @@ -63,13 +71,16 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) { m_Config = InConfig; - ZEN_INFO("initializing CAS pool at {}", m_Config.RootDirectory); + ZEN_INFO("initializing CAS pool at '{}'", m_Config.RootDirectory); // Ensure root directory exists - create if it doesn't exist already std::filesystem::create_directories(m_Config.RootDirectory); // Open or create manifest + // + // The manifest is not currently fully implemented. The goal is to + // use it for recovery and configuration bool IsNewStore = false; @@ -77,23 +88,22 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) std::filesystem::path ManifestPath = m_Config.RootDirectory; ManifestPath /= ".ucas_root"; - CAtlFile marker; - HRESULT hRes = marker.Create(ManifestPath.c_str(), GENERIC_READ, 0, OPEN_EXISTING); + std::error_code Ec; + BasicFile Marker; + Marker.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec); - if (FAILED(hRes)) + if (Ec) { IsNewStore = true; ExtendableStringBuilder<128> manifest; - manifest.Append("#CAS_ROOT\n"); // TODO: should write something meaningful here + manifest.Append("#CAS_ROOT\n"); manifest.Append("ID="); zen::Oid id = zen::Oid::NewOid(); id.ToString(manifest); - hRes = marker.Create(ManifestPath.c_str(), GENERIC_WRITE, 0, CREATE_ALWAYS); - - if (SUCCEEDED(hRes)) - marker.Write(manifest.c_str(), (DWORD)manifest.Size()); + Marker.Open(ManifestPath.c_str(), /* IsCreate */ true); + Marker.Write(manifest.c_str(), (DWORD)manifest.Size(), 0); } } @@ -101,6 +111,9 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) m_TinyStrategy.Initialize("tobs", 16, IsNewStore); m_SmallStrategy.Initialize("sobs", 4096, IsNewStore); + + ScrubContext Ctx; + Scrub(Ctx); } CasStore::InsertResult @@ -160,6 +173,14 @@ CasImpl::Flush() m_LargeStrategy.Flush(); } +void +CasImpl::Scrub(ScrubContext& Ctx) +{ + m_SmallStrategy.Scrub(Ctx); + m_TinyStrategy.Scrub(Ctx); + m_LargeStrategy.Scrub(Ctx); +} + ////////////////////////////////////////////////////////////////////////// CasStore* @@ -173,18 +194,47 @@ CreateCasStore() // Testing related code follows... // -void -CAS_forcelink() -{ -} - TEST_CASE("CasStore") { + ScopedTemporaryDirectory TempDir; + zen::CasStoreConfiguration config; - config.RootDirectory = "c:\\temp\\test"; + config.RootDirectory = TempDir.Path(); + + std::unique_ptr<zen::CasStore> Store{CreateCasStore()}; + Store->Initialize(config); + + ScrubContext Ctx; + Store->Scrub(Ctx); + + IoBuffer Value1{16}; + memcpy(Value1.MutableData(), "1234567890123456", 16); + IoHash Hash1 = IoHash::HashBuffer(Value1.Data(), Value1.Size()); + CasStore::InsertResult Result1 = Store->InsertChunk(Value1, Hash1); + CHECK(Result1.New); + + IoBuffer Value2{16}; + memcpy(Value2.MutableData(), "ABCDEFGHIJKLMNOP", 16); + IoHash Hash2 = IoHash::HashBuffer(Value2.Data(), Value2.Size()); + CasStore::InsertResult Result2 = Store->InsertChunk(Value2, Hash2); + CHECK(Result2.New); - std::unique_ptr<zen::CasStore> store{CreateCasStore()}; - store->Initialize(config); + CasChunkSet ChunkSet; + ChunkSet.AddChunk(Hash1); + ChunkSet.AddChunk(Hash2); + + Store->FilterChunks(ChunkSet); + CHECK(ChunkSet.GetChunkSet().size() == 0); + + IoBuffer Lookup1 = Store->FindChunk(Hash1); + CHECK(Lookup1); + IoBuffer Lookup2 = Store->FindChunk(Hash2); + CHECK(Lookup2); +} + +void +CAS_forcelink() +{ } } // namespace zen diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp index 35ccdd042..0b92a8979 100644 --- a/zenstore/basicfile.cpp +++ b/zenstore/basicfile.cpp @@ -5,7 +5,9 @@ #include <zencore/except.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> +#include <zencore/testutils.h> +#include <doctest/doctest.h> #include <fmt/format.h> #include <gsl/gsl-lite.hpp> @@ -13,16 +15,54 @@ namespace zen { using namespace fmt::literals; +BasicFile::~BasicFile() +{ + Close(); +} + void -BasicFile::Open(std::filesystem::path FileName, bool isCreate) +BasicFile::Open(std::filesystem::path FileName, bool IsCreate) { - const DWORD dwCreationDisposition = isCreate ? CREATE_ALWAYS : OPEN_EXISTING; + std::error_code Ec; + Open(FileName, IsCreate, Ec); - HRESULT hRes = m_File.Create(FileName.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, dwCreationDisposition); + if (Ec) + { + throw std::system_error(Ec, "failed to open file '{}'"_format(FileName)); + } +} - if (FAILED(hRes)) +void +BasicFile::Open(std::filesystem::path FileName, bool IsCreate, std::error_code& Ec) +{ + const DWORD dwCreationDisposition = IsCreate ? CREATE_ALWAYS : OPEN_EXISTING; + const DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; + const DWORD dwShareMode = FILE_SHARE_READ; + const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; + HANDLE hTemplateFile = nullptr; + + HANDLE FileHandle = CreateFile(FileName.c_str(), + dwDesiredAccess, + dwShareMode, + /* lpSecurityAttributes */ nullptr, + dwCreationDisposition, + dwFlagsAndAttributes, + hTemplateFile); + + if (FileHandle == INVALID_HANDLE_VALUE) { - ThrowSystemException(hRes, "Failed to open bucket sobs file '{}'"_format(FileName)); + Ec = zen::MakeErrorCodeFromLastError(); + } + + m_FileHandle = FileHandle; +} + +void +BasicFile::Close() +{ + if (m_FileHandle) + { + ::CloseHandle(m_FileHandle); } } @@ -34,11 +74,14 @@ BasicFile::Read(void* Data, uint64_t Size, uint64_t Offset) Ovl.Offset = DWORD(Offset & 0xffff'ffffu); Ovl.OffsetHigh = DWORD(Offset >> 32); - HRESULT hRes = m_File.Read(Data, gsl::narrow<DWORD>(Size), &Ovl); + DWORD dwNumberOfBytesToRead = gsl::narrow<DWORD>(Size); + DWORD dwNumberOfBytesRead = 0; + + BOOL Success = ::ReadFile(m_FileHandle, Data, dwNumberOfBytesToRead, &dwNumberOfBytesRead, &Ovl); - if (FAILED(hRes)) + if (!Success) { - ThrowSystemException(hRes, "Failed to read from file '{}'"_format(zen::PathFromHandle(m_File))); + ThrowLastError("Failed to read from file '{}'"_format(zen::PathFromHandle(m_FileHandle))); } } @@ -53,6 +96,35 @@ BasicFile::ReadAll() } void +BasicFile::StreamFile(std::function<void(const void* Data, uint64_t Size)>&& ChunkFun) +{ + StreamByteRange(0, FileSize(), std::move(ChunkFun)); +} + +void +BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun) +{ + const uint64_t ChunkSize = 128 * 1024; + IoBuffer ReadBuffer{ChunkSize}; + void* BufferPtr = ReadBuffer.MutableData(); + + uint64_t RemainBytes = Size; + uint64_t CurrentOffset = FileOffset; + + while (RemainBytes) + { + const uint64_t ThisChunkBytes = zen::Min(ChunkSize, RemainBytes); + + Read(BufferPtr, ThisChunkBytes, CurrentOffset); + + ChunkFun(BufferPtr, ThisChunkBytes); + + CurrentOffset += ThisChunkBytes; + RemainBytes -= ThisChunkBytes; + } +} + +void BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset) { OVERLAPPED Ovl{}; @@ -60,33 +132,46 @@ BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset) Ovl.Offset = DWORD(Offset & 0xffff'ffffu); Ovl.OffsetHigh = DWORD(Offset >> 32); - HRESULT hRes = m_File.Write(Data, gsl::narrow<DWORD>(Size), &Ovl); + DWORD dwNumberOfBytesToWrite = gsl::narrow<DWORD>(Size); + DWORD dwNumberOfBytesWritten = 0; - if (FAILED(hRes)) + BOOL Success = ::WriteFile(m_FileHandle, Data, dwNumberOfBytesToWrite, &dwNumberOfBytesWritten, &Ovl); + + if (!Success) { - ThrowSystemException(hRes, "Failed to write to file '{}'"_format(zen::PathFromHandle(m_File))); + ThrowLastError("Failed to write to file '{}'"_format(zen::PathFromHandle(m_FileHandle))); } } void BasicFile::Flush() { - m_File.Flush(); + FlushFileBuffers(m_FileHandle); } uint64_t BasicFile::FileSize() { - ULONGLONG Sz; - m_File.GetSize(Sz); + ULARGE_INTEGER liFileSize; + liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart); - return uint64_t(Sz); + return uint64_t(liFileSize.QuadPart); +} + +TEST_CASE("BasicFile") +{ + ScopedCurrentDirectoryChange _; + + BasicFile File1; + CHECK_THROWS(File1.Open("zonk", false)); + CHECK_NOTHROW(File1.Open("zonk", true)); + CHECK_NOTHROW(File1.Write("abcd", 4, 0)); + CHECK(File1.FileSize() == 4); } void -BasicFile::Close() +basicfile_forcelink() { - m_File.Close(); } } // namespace zen diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp index 4407d8b08..070ca1503 100644 --- a/zenstore/compactcas.cpp +++ b/zenstore/compactcas.cpp @@ -10,18 +10,22 @@ #include <zencore/thread.h> #include <zencore/uid.h> -#include <gsl/gsl-lite.hpp> - -#include <functional> - -struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive- -#include <atlfile.h> #include <filesystem> +#include <functional> +#include <gsl/gsl-lite.hpp> ////////////////////////////////////////////////////////////////////////// namespace zen { +CasContainerStrategy::CasContainerStrategy(const CasStoreConfiguration& Config) : m_Config(Config) +{ +} + +CasContainerStrategy::~CasContainerStrategy() +{ +} + void CasContainerStrategy::Initialize(const std::string_view ContainerBaseName, uint64_t Alignment, bool IsNewStore) { @@ -43,7 +47,9 @@ CasContainerStrategy::Initialize(const std::string_view ContainerBaseName, uint6 uint64_t MaxFileOffset = 0; { - // This is not technically necessary but may help future static analysis + // This is not technically necessary (nobody should be accessing us from + // another thread at this stage) but may help static analysis + zen::RwLock::ExclusiveLockScope _(m_LocationMapLock); m_CasLog.Replay([&](const CasDiskIndexEntry& Record) { @@ -103,9 +109,8 @@ IoBuffer CasContainerStrategy::FindChunk(const IoHash& ChunkHash) { RwLock::SharedLockScope _(m_LocationMapLock); - auto KeyIt = m_LocationMap.find(ChunkHash); - if (KeyIt != m_LocationMap.end()) + if (auto KeyIt = m_LocationMap.find(ChunkHash); KeyIt != m_LocationMap.end()) { const CasDiskLocation& Location = KeyIt->second; return zen::IoBufferBuilder::MakeFromFileHandle(m_SmallObjectFile.Handle(), Location.Offset, Location.Size); @@ -120,9 +125,8 @@ bool CasContainerStrategy::HaveChunk(const IoHash& ChunkHash) { RwLock::SharedLockScope _(m_LocationMapLock); - auto KeyIt = m_LocationMap.find(ChunkHash); - if (KeyIt != m_LocationMap.end()) + if (auto KeyIt = m_LocationMap.find(ChunkHash); KeyIt != m_LocationMap.end()) { return true; } @@ -133,6 +137,13 @@ CasContainerStrategy::HaveChunk(const IoHash& ChunkHash) void CasContainerStrategy::FilterChunks(CasChunkSet& InOutChunks) { + // This implementation is good enough for relatively small + // chunk sets (in terms of chunk identifiers), but would + // benefit from a better implementation which removes + // items incrementally for large sets, especially when + // we're likely to already have a large proportion of the + // chunks in the set + std::unordered_set<IoHash> HaveSet; for (const IoHash& Hash : InOutChunks.GetChunkSet()) @@ -157,4 +168,113 @@ CasContainerStrategy::Flush() m_SmallObjectFile.Flush(); } +void +CasContainerStrategy::Scrub(ScrubContext& Ctx) +{ + const uint64_t WindowSize = 4 * 1024 * 1024; + uint64_t WindowStart = 0; + uint64_t WindowEnd = WindowSize; + const uint64_t FileSize = m_SmallObjectFile.FileSize(); + + std::vector<CasDiskIndexEntry> BigChunks; + std::vector<CasDiskIndexEntry> BadChunks; + + // We do a read sweep through the payloads file and validate + // any entries that are contained within each segment, with + // the assumption that most entries will be checked in this + // pass. An alternative strategy would be to use memory mapping. + + { + IoBuffer ReadBuffer{WindowSize}; + void* BufferBase = ReadBuffer.MutableData(); + + RwLock::SharedLockScope _(m_LocationMapLock); + + do + { + const uint64_t ChunkSize = zen::Min(WindowSize, FileSize - WindowStart); + m_SmallObjectFile.Read(BufferBase, ChunkSize, WindowStart); + + for (auto& Entry : m_LocationMap) + { + const uint64_t EntryOffset = Entry.second.Offset; + + if ((EntryOffset >= WindowStart) && (EntryOffset < WindowEnd)) + { + const uint64_t EntryEnd = EntryOffset + Entry.second.Size; + + if (EntryEnd >= WindowEnd) + { + BigChunks.push_back({.Key = Entry.first, .Location = Entry.second}); + + continue; + } + + const IoHash ComputedHash = IoHash::HashBuffer(BufferBase, Entry.second.Size); + + if (Entry.first != ComputedHash) + { + // Hash mismatch + + BadChunks.push_back({.Key = Entry.first, .Location = Entry.second}); + } + } + } + + WindowStart += WindowSize; + WindowEnd += WindowSize; + } while (WindowStart < FileSize); + } + + // Deal with large chunks + + for (const CasDiskIndexEntry& Entry : BigChunks) + { + IoHashStream Hasher; + m_SmallObjectFile.StreamByteRange(Entry.Location.Offset, Entry.Location.Size, [&](const void* Data, uint64_t Size) { + Hasher.Append(Data, Size); + }); + IoHash ComputedHash = Hasher.GetHash(); + + if (Entry.Key != ComputedHash) + { + BadChunks.push_back(Entry); + } + } + + // Deal with bad chunks by removing them from our lookup map + + std::vector<IoHash> BadChunkHashes; + + for (const CasDiskIndexEntry& Entry : BadChunks) + { + BadChunkHashes.push_back(Entry.Key); + m_LocationMap.erase(Entry.Key); + } + + // Let whomever it concerns know about the bad chunks. This could + // be used to invalidate higher level data structures more efficiently + // than a full validation pass might be able to do + + Ctx.ReportBadChunks(BadChunkHashes); +} + +void +CasContainerStrategy::MakeSnapshot() +{ + RwLock::SharedLockScope _(m_LocationMapLock); + + std::vector<CasDiskIndexEntry> Entries{m_LocationMap.size()}; + + uint64_t EntryIndex = 0; + for (auto& Entry : m_LocationMap) + { + CasDiskIndexEntry& IndexEntry = Entries[EntryIndex++]; + IndexEntry.Key = Entry.first; + IndexEntry.Location = Entry.second; + } + + m_SmallObjectIndex.Write(Entries.data(), Entries.size() * sizeof(CasDiskIndexEntry), 0); +} + } // namespace zen diff --git a/zenstore/compactcas.h b/zenstore/compactcas.h index 05bbf81f6..101e6b1b7 100644 --- a/zenstore/compactcas.h +++ b/zenstore/compactcas.h @@ -14,9 +14,6 @@ #include <zenstore/cas.h> #include <zenstore/caslog.h> -#include <atlfile.h> -#include <functional> - namespace zen { ////////////////////////////////////////////////////////////////////////// @@ -27,7 +24,10 @@ namespace zen { struct CasDiskLocation { uint64_t Offset; - uint32_t Size; // TODO: Make this more like the IoStore index so we can store larger chunks (should be five bytes) + // If we wanted to be able to store larger chunks using this storage mechanism then + // we could make this more like the IoStore index so we can store larger chunks. + // I.e use five bytes for size and seven for offset + uint32_t Size; }; struct CasDiskIndexEntry @@ -50,7 +50,9 @@ static_assert(sizeof(CasDiskIndexEntry) == 32); struct CasContainerStrategy { - CasContainerStrategy(const CasStoreConfiguration& Config, CasStore::Stats& Stats) : m_Config(Config), m_Stats(Stats) {} + CasContainerStrategy(const CasStoreConfiguration& Config); + ~CasContainerStrategy(); + CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash); CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& chunkHash); IoBuffer FindChunk(const IoHash& ChunkHash); @@ -58,10 +60,10 @@ struct CasContainerStrategy void FilterChunks(CasChunkSet& InOutChunks); void Initialize(const std::string_view ContainerBaseName, uint64_t Alignment, bool IsNewStore); void Flush(); + void Scrub(ScrubContext& Ctx); private: const CasStoreConfiguration& m_Config; - CasStore::Stats& m_Stats; uint64_t m_PayloadAlignment = 1 << 4; bool m_IsInitialized = false; BasicFile m_SmallObjectFile; @@ -73,6 +75,8 @@ private: RwLock m_InsertLock; // used to serialize inserts std::atomic<uint64_t> m_CurrentInsertOffset = 0; std::atomic<uint64_t> m_CurrentIndexOffset = 0; + + void MakeSnapshot(); }; } // namespace zen diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp index 170f13875..31991a43e 100644 --- a/zenstore/filecas.cpp +++ b/zenstore/filecas.cpp @@ -10,6 +10,7 @@ #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/uid.h> +#include <zenstore/basicfile.h> #include <gsl/gsl-lite.hpp> @@ -17,6 +18,7 @@ #include <functional> #include <unordered_map> +// clang-format off #include <zencore/prewindows.h> struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive- @@ -24,13 +26,19 @@ struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax erro #include <zencore/postwindows.h> // clang-format on -// -////////////////////////////////////////////////////////////////////////// namespace zen { using namespace fmt::literals; +FileCasStrategy::FileCasStrategy(const CasStoreConfiguration& Config) : m_Config(Config) +{ +} + +FileCasStrategy::~FileCasStrategy() +{ +} + WideStringBuilderBase& FileCasStrategy::MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHash& ChunkHash, size_t& OutShard2len) { @@ -56,7 +64,7 @@ FileCasStrategy::MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHas OutShard2len = ShardedPath.Size(); ShardedPath.Append('\\'); - ShardedPath.AppendAsciiRange(str + 6, str + 64); + ShardedPath.AppendAsciiRange(str + 5, str + 64); return ShardedPath; } @@ -259,12 +267,9 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize } // We cannot rely on RAII to close the file handle since it would be closed - // *after* the lock is released due to the initialization order. + // *after* the lock is released due to the initialization order PayloadFile.Close(); - AtomicIncrement(m_Stats.PutCount); - AtomicAdd(m_Stats.PutBytes, ChunkSize); - return {.New = true}; } @@ -279,15 +284,7 @@ FileCasStrategy::FindChunk(const IoHash& ChunkHash) RwLock::SharedLockScope _(LockForHash(ChunkHash)); - auto Chunk = IoBufferBuilder::MakeFromFile(ShardedPath.c_str()); - - if (Chunk) - { - AtomicIncrement(m_Stats.GetCount); - AtomicAdd(m_Stats.GetBytes, Chunk.Size()); - } - - return Chunk; + return IoBufferBuilder::MakeFromFile(ShardedPath.c_str()); } bool @@ -338,6 +335,62 @@ FileCasStrategy::FilterChunks(CasChunkSet& InOutChunks) } void +FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, BasicFile& PayloadFile)>&& Callback) +{ + struct Visitor : public FileSystemTraversal::TreeVisitor + { + Visitor(const std::filesystem::path& RootDir) : RootDirectory(RootDir) {} + virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& File, uint64_t FileSize) override + { + ZEN_UNUSED(FileSize); + + std::filesystem::path RelPath = std::filesystem::relative(Parent, RootDirectory); + + std::wstring PathString = RelPath.native(); + + if ((PathString.size() == (3 + 2 + 1)) && (File.size() == (40 - 3 - 2))) + { + if (PathString.at(3) == std::filesystem::path::preferred_separator) + { + PathString.erase(3, 1); + } + PathString.append(File); + + StringBuilder<64> Utf8; + WideToUtf8(PathString, Utf8); + + // TODO: should validate that we're actually dealing with a valid hex string here + + IoHash NameHash = IoHash::FromHexString({Utf8.Data(), Utf8.Size()}); + + BasicFile PayloadFile; + std::error_code Ec; + PayloadFile.Open(Parent / File, false, Ec); + + if (!Ec) + { + Callback(NameHash, PayloadFile); + } + } + } + + virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, + [[maybe_unused]] const std::wstring_view& DirectoryName) + { + return true; + } + + const std::filesystem::path& RootDirectory; + std::function<void(const IoHash& Hash, BasicFile& PayloadFile)> Callback; + } CasVisitor{m_Config.RootDirectory}; + + CasVisitor.Callback = std::move(Callback); + + FileSystemTraversal Traversal; + Traversal.TraverseFileSystem(m_Config.RootDirectory, CasVisitor); +} + +void FileCasStrategy::Flush() { // Since we don't keep files open after writing there's nothing specific @@ -353,6 +406,25 @@ FileCasStrategy::Flush() } void +FileCasStrategy::Scrub(ScrubContext& Ctx) +{ + std::vector<IoHash> BadHashes; + + IterateChunks([&](const IoHash& Hash, BasicFile& Payload) { + IoHashStream Hasher; + Payload.StreamFile([&](const void* Data, size_t Size) { Hasher.Append(Data, Size); }); + IoHash ComputedHash = Hasher.GetHash(); + + if (ComputedHash != Hash) + { + BadHashes.push_back(Hash); + } + }); + + Ctx.ReportBadChunks(BadHashes); +} + +void FileCasStrategy::GarbageCollect(GcContext& GcCtx) { ZEN_UNUSED(GcCtx); diff --git a/zenstore/filecas.h b/zenstore/filecas.h index 448d1a05f..885973810 100644 --- a/zenstore/filecas.h +++ b/zenstore/filecas.h @@ -10,11 +10,20 @@ #include <zencore/thread.h> #include <zenstore/cas.h> +#include <functional> + namespace zen { +class BasicFile; + +/** CAS storage strategy using a file-per-chunk storage strategy +*/ + struct FileCasStrategy { - FileCasStrategy(const CasStoreConfiguration& Config, CasStore::Stats& Stats) : m_Config(Config), m_Stats(Stats) {} + FileCasStrategy(const CasStoreConfiguration& Config); + ~FileCasStrategy(); + CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash); CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash); IoBuffer FindChunk(const IoHash& ChunkHash); @@ -22,15 +31,16 @@ struct FileCasStrategy void FilterChunks(CasChunkSet& InOutChunks); void Flush(); void GarbageCollect(GcContext& GcCtx); + void Scrub(ScrubContext& Ctx); private: const CasStoreConfiguration& m_Config; - CasStore::Stats& m_Stats; RwLock m_Lock; RwLock m_ShardLocks[256]; // TODO: these should be spaced out so they don't share cache lines inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardLocks[Hash.Hash[19]]; } static WideStringBuilderBase& MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHash& ChunkHash, size_t& OutShard2len); + void IterateChunks(std::function<void(const IoHash& Hash, BasicFile& PayloadFile)>&& Callback); }; } // namespace zen diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/CAS.h index b4de533dd..bb310b179 100644 --- a/zenstore/include/zenstore/CAS.h +++ b/zenstore/include/zenstore/CAS.h @@ -2,7 +2,7 @@ #pragma once -#include <zencore/zencore.h> +#include "zenstore.h" #include <zencore/blake3.h> #include <zencore/iobuffer.h> @@ -37,6 +37,14 @@ public: private: }; +class ScrubContext +{ +public: + virtual void ReportBadChunks(std::span<IoHash> BadChunks); + +private: +}; + class CasChunkSet { public: @@ -54,17 +62,7 @@ class CasStore public: virtual ~CasStore() = default; - struct Stats - { - uint64_t PutBytes = 0; - uint64_t PutCount = 0; - - uint64_t GetBytes = 0; - uint64_t GetCount = 0; - }; - const CasStoreConfiguration& Config() { return m_Config; } - const Stats& GetStats() const { return m_Stats; } struct InsertResult { @@ -76,10 +74,10 @@ public: virtual IoBuffer FindChunk(const IoHash& ChunkHash) = 0; virtual void FilterChunks(CasChunkSet& InOutChunks) = 0; virtual void Flush() = 0; + virtual void Scrub(ScrubContext& Ctx) = 0; protected: CasStoreConfiguration m_Config; - Stats m_Stats; }; ZENCORE_API CasStore* CreateCasStore(); diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h index c6f61d466..d4d65b366 100644 --- a/zenstore/include/zenstore/basicfile.h +++ b/zenstore/include/zenstore/basicfile.h @@ -2,34 +2,46 @@ #pragma once -#include <zencore/iobuffer.h> -#include <zencore/zencore.h> +#include "zenstore.h" +#include <zencore/iobuffer.h> #include <zencore/windows.h> -#include <atlfile.h> #include <filesystem> +#include <functional> namespace zen { /** * Probably the most basic file abstraction in the universe + * + * One thing of note is that there is no notion of a "current file position" + * in this API -- all reads and writes are done from explicit offsets in + * the file. This avoids concurrency issues which can occur otherwise. + * */ class BasicFile { public: + BasicFile() = default; + ~BasicFile(); void Open(std::filesystem::path FileName, bool IsCreate); - void Read(void* Data, uint64_t Size, uint64_t Offset); - void Write(const void* Data, uint64_t Size, uint64_t Offset); + void Open(std::filesystem::path FileName, bool IsCreate, std::error_code& Ec); + void Close(); + void Read(void* Data, uint64_t Size, uint64_t FileOffset); + void StreamFile(std::function<void(const void* Data, uint64_t Size)>&& ChunkFun); + void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun); + void Write(const void* Data, uint64_t Size, uint64_t FileOffset); void Flush(); uint64_t FileSize(); - void* Handle() { return m_File; } - void Close(); + void* Handle() { return m_FileHandle; } IoBuffer ReadAll(); private: - CAtlFile m_File; + void* m_FileHandle = nullptr; // This is either null or valid }; +ZENCORE_API void basicfile_forcelink(); + } // namespace zen diff --git a/zenstore/include/zenstore/caslog.h b/zenstore/include/zenstore/caslog.h index aea855e4c..3d558bee0 100644 --- a/zenstore/include/zenstore/caslog.h +++ b/zenstore/include/zenstore/caslog.h @@ -2,7 +2,7 @@ #pragma once -#include <zencore/zencore.h> +#include "zenstore.h" #include <zencore/iobuffer.h> #include <zencore/string.h> diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h index 76a33c915..f023ada40 100644 --- a/zenstore/include/zenstore/cidstore.h +++ b/zenstore/include/zenstore/cidstore.h @@ -2,6 +2,8 @@ #pragma once +#include "zenstore.h" + #include <tsl/robin_map.h> #include <zencore/iohash.h> #include <zenstore/CAS.h> diff --git a/zenstore/include/zenstore/scrub.h b/zenstore/include/zenstore/scrub.h index 5a34d4860..4948afcd5 100644 --- a/zenstore/include/zenstore/scrub.h +++ b/zenstore/include/zenstore/scrub.h @@ -2,6 +2,8 @@ #pragma once +#include "zenstore.h" + #include <zencore/iohash.h> #include <span> diff --git a/zenstore/include/zenstore/zenstore.h b/zenstore/include/zenstore/zenstore.h new file mode 100644 index 000000000..46d62029d --- /dev/null +++ b/zenstore/include/zenstore/zenstore.h @@ -0,0 +1,13 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/zencore.h> + +#define ZENSTORE_API + +namespace zen { + +ZENSTORE_API void zenstore_forcelinktests(); + +} diff --git a/zenstore/zenstore.cpp b/zenstore/zenstore.cpp new file mode 100644 index 000000000..cd16e5634 --- /dev/null +++ b/zenstore/zenstore.cpp @@ -0,0 +1,17 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zenstore/zenstore.h" + +#include <zenstore/CAS.h> +#include <zenstore/basicfile.h> + +namespace zen { + +void +zenstore_forcelinktests() +{ + basicfile_forcelink(); + CAS_forcelink(); +} + +} // namespace zen diff --git a/zenstore/zenstore.vcxproj b/zenstore/zenstore.vcxproj index 8d665f2c3..eb2ecd02b 100644 --- a/zenstore/zenstore.vcxproj +++ b/zenstore/zenstore.vcxproj @@ -19,6 +19,7 @@ <ClCompile Include="filecas.cpp" /> <ClCompile Include="gc.cpp" /> <ClCompile Include="scrub.cpp" /> + <ClCompile Include="zenstore.cpp" /> </ItemGroup> <ItemGroup> <ClInclude Include="compactcas.h" /> @@ -29,12 +30,16 @@ <ClInclude Include="include\zenstore\scrub.h" /> <ClInclude Include="include\zenstore\CAS.h" /> <ClInclude Include="include\zenstore\caslog.h" /> + <ClInclude Include="include\zenstore\zenstore.h" /> </ItemGroup> <ItemGroup> <ProjectReference Include="..\zencore\zencore.vcxproj"> <Project>{d75bf9ab-c61e-4fff-ad59-1563430f05e2}</Project> </ProjectReference> </ItemGroup> + <ItemGroup> + <None Include="xmake.lua" /> + </ItemGroup> <PropertyGroup Label="Globals"> <VCProjectVersion>16.0</VCProjectVersion> <Keyword>Win32Proj</Keyword> diff --git a/zenstore/zenstore.vcxproj.filters b/zenstore/zenstore.vcxproj.filters index 3dfb89dbf..8a52c69f6 100644 --- a/zenstore/zenstore.vcxproj.filters +++ b/zenstore/zenstore.vcxproj.filters @@ -9,6 +9,7 @@ <ClCompile Include="scrub.cpp" /> <ClCompile Include="basicfile.cpp" /> <ClCompile Include="cidstore.cpp" /> + <ClCompile Include="zenstore.cpp" /> </ItemGroup> <ItemGroup> <ClInclude Include="compactcas.h" /> @@ -19,5 +20,9 @@ <ClInclude Include="include\zenstore\scrub.h" /> <ClInclude Include="include\zenstore\basicfile.h" /> <ClInclude Include="include\zenstore\cidstore.h" /> + <ClInclude Include="include\zenstore\zenstore.h" /> + </ItemGroup> + <ItemGroup> + <None Include="xmake.lua" /> </ItemGroup> </Project>
\ No newline at end of file diff --git a/zenutil/include/zenutil/zenserverprocess.h b/zenutil/include/zenutil/zenserverprocess.h index f0924f048..979db349a 100644 --- a/zenutil/include/zenutil/zenserverprocess.h +++ b/zenutil/include/zenutil/zenserverprocess.h @@ -123,12 +123,14 @@ public: ZenServerEntry* Register(int ListenPort); void Sweep(); void Snapshot(std::function<void(const ZenServerEntry&)>&& Callback); + inline bool IsReadOnly() const { return m_IsReadOnly; } private: void* m_hMapFile = nullptr; ZenServerEntry* m_Data = nullptr; int m_MaxEntryCount = 131072 / sizeof(ZenServerEntry); ZenServerEntry* m_OurEntry = nullptr; + bool m_IsReadOnly = true; }; -} +} // namespace zen diff --git a/zenutil/zenserverprocess.cpp b/zenutil/zenserverprocess.cpp index c504ce7da..9e370ae10 100644 --- a/zenutil/zenserverprocess.cpp +++ b/zenutil/zenserverprocess.cpp @@ -40,20 +40,16 @@ namespace zenutil { m_Attributes.nLength = sizeof m_Attributes; m_Attributes.bInheritHandle = false; // Disable inheritance - const BOOL success = InitializeSecurityDescriptor(&m_Sd, SECURITY_DESCRIPTOR_REVISION); + const BOOL Success = InitializeSecurityDescriptor(&m_Sd, SECURITY_DESCRIPTOR_REVISION); - if (success) + if (Success) { - const BOOL bSetOk = SetSecurityDescriptorDacl(&m_Sd, TRUE, (PACL)NULL, FALSE); - - if (bSetOk) - { - m_Attributes.lpSecurityDescriptor = &m_Sd; - } - else + if (!SetSecurityDescriptorDacl(&m_Sd, TRUE, (PACL)NULL, FALSE)) { zen::ThrowLastError("SetSecurityDescriptorDacl failed", std::source_location::current()); } + + m_Attributes.lpSecurityDescriptor = &m_Sd; } } }; @@ -129,7 +125,8 @@ ZenServerState::Initialize() zen::ThrowLastError("Could not map view of Zen server state"); } - m_Data = reinterpret_cast<ZenServerEntry*>(pBuf); + m_Data = reinterpret_cast<ZenServerEntry*>(pBuf); + m_IsReadOnly = false; } bool @@ -221,6 +218,8 @@ ZenServerState::Sweep() return; } + ZEN_ASSERT(m_IsReadOnly == false); + for (int i = 0; i < m_MaxEntryCount; ++i) { ZenServerEntry& Entry = m_Data[i]; |