aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPer Larsson <[email protected]>2021-09-20 08:54:34 +0200
committerPer Larsson <[email protected]>2021-09-20 08:54:34 +0200
commite25b4b20d8a5696aa7055c9c167fa47b3739bc7e (patch)
tree049654b87096a22e1bf696a385db608a75f229fa
parentProbe upstream Zen server when initializing upstream cache. (diff)
parentFixed unused variable warnings exposed by xmake build (unclear why I do not r... (diff)
downloadzen-e25b4b20d8a5696aa7055c9c167fa47b3739bc7e.tar.xz
zen-e25b4b20d8a5696aa7055c9c167fa47b3739bc7e.zip
Merge branch 'main' of https://github.com/EpicGames/zen
-rw-r--r--.gitignore5
-rw-r--r--vs-chromium-project.txt4
-rw-r--r--xmake.lua5
-rw-r--r--zen.sln9
-rw-r--r--zen/cmds/copy.cpp2
-rw-r--r--zen/cmds/copy.h2
-rw-r--r--zen/cmds/dedup.cpp2
-rw-r--r--zen/cmds/dedup.h2
-rw-r--r--zen/cmds/deploy.cpp2
-rw-r--r--zen/cmds/deploy.h2
-rw-r--r--zen/cmds/hash.cpp2
-rw-r--r--zen/cmds/hash.h2
-rw-r--r--zen/cmds/run.cpp2
-rw-r--r--zen/cmds/run.h2
-rw-r--r--zen/cmds/scrub.cpp4
-rw-r--r--zen/cmds/scrub.h2
-rw-r--r--zen/cmds/status.cpp2
-rw-r--r--zen/cmds/status.h2
-rw-r--r--zen/cmds/top.cpp16
-rw-r--r--zen/cmds/top.h2
-rw-r--r--zen/cmds/up.cpp2
-rw-r--r--zen/cmds/up.h2
-rw-r--r--zen/zen.cpp5
-rw-r--r--zen/zen.h1
-rw-r--r--zen/zen.vcxproj3
-rw-r--r--zen/zen.vcxproj.filters3
-rw-r--r--zencore/except.cpp41
-rw-r--r--zencore/include/zencore/except.h53
-rw-r--r--zencore/include/zencore/session.h3
-rw-r--r--zencore/include/zencore/testutils.h31
-rw-r--r--zencore/include/zencore/thread.h8
-rw-r--r--zencore/include/zencore/uid.h1
-rw-r--r--zencore/include/zencore/zencore.h80
-rw-r--r--zencore/session.cpp15
-rw-r--r--zencore/testutils.cpp33
-rw-r--r--zencore/thread.cpp39
-rw-r--r--zencore/uid.cpp8
-rw-r--r--zencore/zencore.cpp25
-rw-r--r--zencore/zencore.vcxproj2
-rw-r--r--zencore/zencore.vcxproj.filters2
-rw-r--r--zenhttp/httpnull.cpp4
-rw-r--r--zenhttp/httpnull.h2
-rw-r--r--zenhttp/httpsys.cpp23
-rw-r--r--zenhttp/httpuws.cpp4
-rw-r--r--zenhttp/httpuws.h2
-rw-r--r--zenhttp/include/zenhttp/httpserver.h2
-rw-r--r--zenserver-test/zenserver-test.cpp52
-rw-r--r--zenserver/config.cpp15
-rw-r--r--zenserver/config.h14
-rw-r--r--zenserver/diag/logging.cpp33
-rw-r--r--zenserver/experimental/usnjournal.cpp10
-rw-r--r--zenserver/windows/service.cpp631
-rw-r--r--zenserver/windows/service.h20
-rw-r--r--zenserver/xmake.lua2
-rw-r--r--zenserver/zenserver.cpp73
-rw-r--r--zenserver/zenserver.vcxproj2
-rw-r--r--zenserver/zenserver.vcxproj.filters2
-rw-r--r--zenstore-test/xmake.lua5
-rw-r--r--zenstore-test/zenstore-test.cpp23
-rw-r--r--zenstore-test/zenstore-test.vcxproj121
-rw-r--r--zenstore-test/zenstore-test.vcxproj.filters9
-rw-r--r--zenstore/CAS.cpp98
-rw-r--r--zenstore/basicfile.cpp119
-rw-r--r--zenstore/compactcas.cpp142
-rw-r--r--zenstore/compactcas.h16
-rw-r--r--zenstore/filecas.cpp104
-rw-r--r--zenstore/filecas.h14
-rw-r--r--zenstore/include/zenstore/CAS.h22
-rw-r--r--zenstore/include/zenstore/basicfile.h28
-rw-r--r--zenstore/include/zenstore/caslog.h2
-rw-r--r--zenstore/include/zenstore/cidstore.h2
-rw-r--r--zenstore/include/zenstore/scrub.h2
-rw-r--r--zenstore/include/zenstore/zenstore.h13
-rw-r--r--zenstore/zenstore.cpp17
-rw-r--r--zenstore/zenstore.vcxproj5
-rw-r--r--zenstore/zenstore.vcxproj.filters5
-rw-r--r--zenutil/include/zenutil/zenserverprocess.h4
-rw-r--r--zenutil/zenserverprocess.cpp19
78 files changed, 1778 insertions, 311 deletions
diff --git a/.gitignore b/.gitignore
index df6c1a9f8..bf237a7b3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -204,9 +204,8 @@ ServiceFabricBackup/
*.rptproj.bak
-
-
-
+# generated build files
+makefile
# Python Tools for Visual Studio (PTVS)
__pycache__/
diff --git a/vs-chromium-project.txt b/vs-chromium-project.txt
index 2b2e15bc7..2bb89a55c 100644
--- a/vs-chromium-project.txt
+++ b/vs-chromium-project.txt
@@ -3,6 +3,6 @@
[SourceExplorer.ignore]
.git/
-.x64/
+x64/
*.suo
-**/.x64/
+**/x64/
diff --git a/xmake.lua b/xmake.lua
index f5a37e12d..8288963e0 100644
--- a/xmake.lua
+++ b/xmake.lua
@@ -36,7 +36,7 @@ if is_mode("debug") then
end
if is_os("windows") then
- add_defines("_CRT_SECURE_NO_WARNINGS")
+ add_defines("_CRT_SECURE_NO_WARNINGS", "_UNICODE", "UNICODE", "_WIN32_WINNT=0x0A00")
end
add_defines("USE_SENTRY=1")
@@ -64,6 +64,7 @@ set_symbols("debug")
includes("zencore", "zencore-test")
includes("zenhttp")
-includes("zenstore", "zenutil")
+includes("zenstore", "zenstore-test")
+includes("zenutil")
includes("zenserver", "zenserver-test")
includes("zen")
diff --git a/zen.sln b/zen.sln
index 2ceb9e58c..052c030ae 100644
--- a/zen.sln
+++ b/zen.sln
@@ -11,6 +11,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{4EA55E5B-1
README.md = README.md
RESTAPI.md = RESTAPI.md
vcpkg.json = vcpkg.json
+ xmake.lua = xmake.lua
EndProjectSection
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zencore", "zencore\zencore.vcxproj", "{D75BF9AB-C61E-4FFF-AD59-1563430F05E2}"
@@ -47,6 +48,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zentest-appstub", "zentest-
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zenhttp", "zenhttp\zenhttp.vcxproj", "{8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}"
EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zenstore-test", "zenstore-test\zenstore-test.vcxproj", "{C001A3DF-B76E-4989-B576-FE2B78AB2580}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64
@@ -111,6 +114,12 @@ Global
{8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Release|x64.ActiveCfg = Release|x64
{8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Release|x64.Build.0 = Release|x64
{8EEB3BE5-7001-46BF-AAFD-EDB7558AC012}.Release|x86.ActiveCfg = Release|x64
+ {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Debug|x64.ActiveCfg = Debug|x64
+ {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Debug|x64.Build.0 = Debug|x64
+ {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Debug|x86.ActiveCfg = Debug|x64
+ {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Release|x64.ActiveCfg = Release|x64
+ {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Release|x64.Build.0 = Release|x64
+ {C001A3DF-B76E-4989-B576-FE2B78AB2580}.Release|x86.ActiveCfg = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
diff --git a/zen/cmds/copy.cpp b/zen/cmds/copy.cpp
index 6b3965f99..947d54e07 100644
--- a/zen/cmds/copy.cpp
+++ b/zen/cmds/copy.cpp
@@ -97,4 +97,4 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return 0;
}
-}
+} // namespace zen
diff --git a/zen/cmds/copy.h b/zen/cmds/copy.h
index c2a7fd23b..322cf3f2f 100644
--- a/zen/cmds/copy.h
+++ b/zen/cmds/copy.h
@@ -25,4 +25,4 @@ private:
bool m_NoClone = false;
};
-}
+} // namespace zen
diff --git a/zen/cmds/dedup.cpp b/zen/cmds/dedup.cpp
index f95a87518..e71314622 100644
--- a/zen/cmds/dedup.cpp
+++ b/zen/cmds/dedup.cpp
@@ -293,4 +293,4 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return 0;
}
-}
+} // namespace zen
diff --git a/zen/cmds/dedup.h b/zen/cmds/dedup.h
index c955d4bbd..7932d10e6 100644
--- a/zen/cmds/dedup.h
+++ b/zen/cmds/dedup.h
@@ -27,4 +27,4 @@ private:
size_t m_SizeThreshold = 1024 * 1024;
};
-}
+} // namespace zen
diff --git a/zen/cmds/deploy.cpp b/zen/cmds/deploy.cpp
index faaf0030c..d60392dd5 100644
--- a/zen/cmds/deploy.cpp
+++ b/zen/cmds/deploy.cpp
@@ -83,4 +83,4 @@ DeployCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return 0;
}
-}
+} // namespace zen
diff --git a/zen/cmds/deploy.h b/zen/cmds/deploy.h
index 565e11bf3..975caf9e9 100644
--- a/zen/cmds/deploy.h
+++ b/zen/cmds/deploy.h
@@ -26,4 +26,4 @@ private:
bool m_IsClean = false;
};
-}
+} // namespace zen
diff --git a/zen/cmds/hash.cpp b/zen/cmds/hash.cpp
index e9484c453..0a7989ffc 100644
--- a/zen/cmds/hash.cpp
+++ b/zen/cmds/hash.cpp
@@ -126,4 +126,4 @@ HashCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return 0;
}
-}
+} // namespace zen
diff --git a/zen/cmds/hash.h b/zen/cmds/hash.h
index 8ed8e9fa3..3df9063ea 100644
--- a/zen/cmds/hash.h
+++ b/zen/cmds/hash.h
@@ -26,4 +26,4 @@ private:
std::string m_OutputFile;
};
-}
+} // namespace zen
diff --git a/zen/cmds/run.cpp b/zen/cmds/run.cpp
index 711a3a341..ceeb4ddae 100644
--- a/zen/cmds/run.cpp
+++ b/zen/cmds/run.cpp
@@ -184,4 +184,4 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return 0;
}
-}
+} // namespace zen
diff --git a/zen/cmds/run.h b/zen/cmds/run.h
index f4eb4ef76..3e1e3f2b2 100644
--- a/zen/cmds/run.h
+++ b/zen/cmds/run.h
@@ -23,4 +23,4 @@ private:
std::string m_ExeTree;
};
-}
+} // namespace zen
diff --git a/zen/cmds/scrub.cpp b/zen/cmds/scrub.cpp
index 73d2b45ee..c0fe8ca61 100644
--- a/zen/cmds/scrub.cpp
+++ b/zen/cmds/scrub.cpp
@@ -31,7 +31,7 @@ GcCommand::~GcCommand()
{
}
-int
+int
GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
ZEN_UNUSED(GlobalOptions, argc, argv);
@@ -39,4 +39,4 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return 0;
}
-}
+} // namespace zen
diff --git a/zen/cmds/scrub.h b/zen/cmds/scrub.h
index a3f25c259..561ae578d 100644
--- a/zen/cmds/scrub.h
+++ b/zen/cmds/scrub.h
@@ -36,4 +36,4 @@ private:
cxxopts::Options m_Options{"gc", "Garbage collect zen storage"};
};
-}
+} // namespace zen
diff --git a/zen/cmds/status.cpp b/zen/cmds/status.cpp
index 1050b5b96..10970e3c2 100644
--- a/zen/cmds/status.cpp
+++ b/zen/cmds/status.cpp
@@ -20,4 +20,4 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return 0;
}
-}
+} // namespace zen
diff --git a/zen/cmds/status.h b/zen/cmds/status.h
index f2c68b96a..acde280c5 100644
--- a/zen/cmds/status.h
+++ b/zen/cmds/status.h
@@ -19,4 +19,4 @@ private:
cxxopts::Options m_Options{"status", "Show zen status"};
};
-}
+} // namespace zen
diff --git a/zen/cmds/top.cpp b/zen/cmds/top.cpp
index 3ff7edcda..315d8cb38 100644
--- a/zen/cmds/top.cpp
+++ b/zen/cmds/top.cpp
@@ -2,9 +2,9 @@
#include "top.h"
+#include <zencore/fmtutils.h>
#include <zencore/logging.h>
#include <zencore/uid.h>
-#include <zencore/fmtutils.h>
#include <zenutil/zenserverprocess.h>
#include <memory>
@@ -32,7 +32,7 @@ TopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return 0;
}
- int n = 0;
+ int n = 0;
const int HeaderPeriod = 20;
for (;;)
@@ -42,14 +42,18 @@ TopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
ZEN_CONSOLE("{:>5} {:>6} {:>24}", "port", "pid", "session");
}
- State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { ZEN_CONSOLE("{:5} {:6} {:24}", Entry.ListenPort, Entry.Pid, Entry.GetSessionId()); });
+ State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) {
+ ZEN_CONSOLE("{:5} {:6} {:24}", Entry.ListenPort, Entry.Pid, Entry.GetSessionId());
+ });
zen::Sleep(1000);
- State.Sweep();
+ if (!State.IsReadOnly())
+ {
+ State.Sweep();
+ }
}
-
return 0;
}
@@ -79,4 +83,4 @@ PsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return 0;
}
-}
+} // namespace zen
diff --git a/zen/cmds/top.h b/zen/cmds/top.h
index a842089ff..d8bf91a1c 100644
--- a/zen/cmds/top.h
+++ b/zen/cmds/top.h
@@ -32,4 +32,4 @@ private:
cxxopts::Options m_Options{"ps", "Enumerate running Zen server instances"};
};
-}
+} // namespace zen
diff --git a/zen/cmds/up.cpp b/zen/cmds/up.cpp
index 048133fc9..17cba3794 100644
--- a/zen/cmds/up.cpp
+++ b/zen/cmds/up.cpp
@@ -101,4 +101,4 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
return 0;
}
-}
+} // namespace zen
diff --git a/zen/cmds/up.h b/zen/cmds/up.h
index fda62693a..fe1ed7a0c 100644
--- a/zen/cmds/up.h
+++ b/zen/cmds/up.h
@@ -32,4 +32,4 @@ private:
cxxopts::Options m_Options{"down", "Bring down zen service"};
};
-}
+} // namespace zen
diff --git a/zen/zen.cpp b/zen/zen.cpp
index 6390c832f..9c373face 100644
--- a/zen/zen.cpp
+++ b/zen/zen.cpp
@@ -42,7 +42,6 @@
//////////////////////////////////////////////////////////////////////////
-
class TemplateCommand : public ZenCmdBase
{
public:
@@ -125,7 +124,7 @@ main(int argc, char** argv)
ZenCmdBase* Cmd;
const char* CmdSummary;
} Commands[] = {
- // clang-format off
+ // clang-format off
{"chunk", &ChunkCmd, "Perform chunking"},
{"copy", &CopyCmd, "Copy file(s)"},
{"deploy", &DeployCmd, "Deploy data"},
@@ -139,7 +138,7 @@ main(int argc, char** argv)
{"top", &TopCmd, "Monitor zen server activity"},
{"up", &UpCmd, "Bring zen server up"},
{"down", &DownCmd, "Bring zen server down"},
- // clang-format on
+ // clang-format on
};
// Build set containing available commands
diff --git a/zen/zen.h b/zen/zen.h
index c90f3169a..1c8d102d3 100644
--- a/zen/zen.h
+++ b/zen/zen.h
@@ -10,7 +10,6 @@
#include <zencore/refcount.h>
#include <zencore/windows.h>
-#include <atlfile.h>
#include <filesystem>
struct ZenCliOptions
diff --git a/zen/zen.vcxproj b/zen/zen.vcxproj
index ff10f4c59..fb0674e87 100644
--- a/zen/zen.vcxproj
+++ b/zen/zen.vcxproj
@@ -136,6 +136,9 @@
<Project>{77f8315d-b21d-4db0-9a6f-2d3359f88a70}</Project>
</ProjectReference>
</ItemGroup>
+ <ItemGroup>
+ <None Include="xmake.lua" />
+ </ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
diff --git a/zen/zen.vcxproj.filters b/zen/zen.vcxproj.filters
index a38771944..9002f01c2 100644
--- a/zen/zen.vcxproj.filters
+++ b/zen/zen.vcxproj.filters
@@ -63,4 +63,7 @@
<UniqueIdentifier>{2e06a54c-52be-4260-9275-a4232d01a53c}</UniqueIdentifier>
</Filter>
</ItemGroup>
+ <ItemGroup>
+ <None Include="xmake.lua" />
+ </ItemGroup>
</Project> \ No newline at end of file
diff --git a/zencore/except.cpp b/zencore/except.cpp
index 9bf043f4a..44b8edffb 100644
--- a/zencore/except.cpp
+++ b/zencore/except.cpp
@@ -7,6 +7,41 @@ namespace zen {
#if ZEN_PLATFORM_WINDOWS
+class WindowsException : public std::exception
+{
+public:
+ WindowsException(std::string_view Message)
+ {
+ m_hResult = HRESULT_FROM_WIN32(GetLastError());
+ m_Message = Message;
+ }
+
+ WindowsException(HRESULT hRes, std::string_view Message)
+ {
+ m_hResult = hRes;
+ m_Message = Message;
+ }
+
+ WindowsException(HRESULT hRes, const char* Message, const char* Detail)
+ {
+ m_hResult = hRes;
+
+ ExtendableStringBuilder<128> msg;
+ msg.Append(Message);
+ msg.Append(" (detail: '");
+ msg.Append(Detail);
+ msg.Append("')");
+
+ m_Message = msg.c_str();
+ }
+
+ virtual const char* what() const override { return m_Message.c_str(); }
+
+private:
+ std::string m_Message;
+ HRESULT m_hResult;
+};
+
void
ThrowSystemException([[maybe_unused]] HRESULT hRes, [[maybe_unused]] std::string_view Message)
{
@@ -28,6 +63,12 @@ ThrowLastError(std::string_view Message)
throw std::system_error(std::error_code(zen::GetLastError(), std::system_category()), std::string(Message));
}
+void
+ThrowSystemError(uint32_t ErrorCode, std::string_view Message)
+{
+ throw std::system_error(std::error_code(ErrorCode, std::system_category()), std::string(Message));
+}
+
std::string
GetLastErrorAsString()
{
diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h
index 08330e4bc..5cfefb1e2 100644
--- a/zencore/include/zencore/except.h
+++ b/zencore/include/zencore/except.h
@@ -15,63 +15,20 @@
namespace zen {
#if ZEN_PLATFORM_WINDOWS
-class WindowsException : public std::exception
-{
-public:
- WindowsException(std::string_view Message)
- {
- m_hResult = HRESULT_FROM_WIN32(GetLastError());
- m_Message = Message;
- }
-
- WindowsException(HRESULT hRes, std::string_view Message)
- {
- m_hResult = hRes;
- m_Message = Message;
- }
-
- WindowsException(HRESULT hRes, const char* Message, const char* Detail)
- {
- m_hResult = hRes;
-
- ExtendableStringBuilder<128> msg;
- msg.Append(Message);
- msg.Append(" (detail: '");
- msg.Append(Detail);
- msg.Append("')");
-
- m_Message = msg.c_str();
- }
-
- virtual const char* what() const override { return m_Message.c_str(); }
-
-private:
- std::string m_Message;
- HRESULT m_hResult;
-};
-
-ZENCORE_API void ThrowSystemException(HRESULT hRes, std::string_view Message);
+ZENCORE_API void ThrowSystemException [[noreturn]] (HRESULT hRes, std::string_view Message);
#endif // ZEN_PLATFORM_WINDOWS
-ZENCORE_API void ThrowLastError(std::string_view Message);
+ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message);
#if __cpp_lib_source_location
-ZENCORE_API void ThrowLastError(std::string_view Message, const std::source_location& Location);
+ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message, const std::source_location& Location);
#endif
+ZENCORE_API void ThrowSystemError [[noreturn]] (uint32_t ErrorCode, std::string_view Message);
+
ZENCORE_API std::string GetLastErrorAsString();
ZENCORE_API std::string GetWindowsErrorAsString(uint32_t Win32ErrorCode);
-inline void
-ThrowSystemException(const char* Message)
-{
-#if ZEN_PLATFORM_WINDOWS
- throw WindowsException(Message);
-#else
- ThrowLastError(Message);
-#endif
-}
-
inline int32_t
GetLastError()
{
diff --git a/zencore/include/zencore/session.h b/zencore/include/zencore/session.h
index 2da41b2c8..dd90197bf 100644
--- a/zencore/include/zencore/session.h
+++ b/zencore/include/zencore/session.h
@@ -8,6 +8,7 @@ namespace zen {
struct Oid;
-ZENCORE_API Oid GetSessionId();
+ZENCORE_API [[nodiscard]] Oid GetSessionId();
+ZENCORE_API [[nodiscard]] std::string_view GetSessionIdString();
} // namespace zen
diff --git a/zencore/include/zencore/testutils.h b/zencore/include/zencore/testutils.h
new file mode 100644
index 000000000..72d985d5c
--- /dev/null
+++ b/zencore/include/zencore/testutils.h
@@ -0,0 +1,31 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <filesystem>
+
+namespace zen {
+
+std::filesystem::path CreateTemporaryDirectory();
+
+class ScopedTemporaryDirectory
+{
+public:
+ ScopedTemporaryDirectory();
+ ~ScopedTemporaryDirectory();
+
+ std::filesystem::path& Path() { return m_RootPath; }
+
+private:
+ std::filesystem::path m_RootPath;
+};
+
+struct ScopedCurrentDirectoryChange
+{
+ std::filesystem::path OldPath{std::filesystem::current_path()};
+
+ ScopedCurrentDirectoryChange() { std::filesystem::current_path(CreateTemporaryDirectory()); }
+ ~ScopedCurrentDirectoryChange() { std::filesystem::current_path(OldPath); }
+};
+
+} // namespace zen
diff --git a/zencore/include/zencore/thread.h b/zencore/include/zencore/thread.h
index e65867ec4..7889682cd 100644
--- a/zencore/include/zencore/thread.h
+++ b/zencore/include/zencore/thread.h
@@ -4,9 +4,7 @@
#include "zencore.h"
-#if !ZEN_PLATFORM_WINDOWS
-# include <shared_mutex>
-#endif
+#include <shared_mutex>
#include <vector>
@@ -66,11 +64,7 @@ public:
};
private:
-#if ZEN_PLATFORM_WINDOWS
- void* m_Srw = nullptr;
-#else
std::shared_mutex m_Mutex;
-#endif
};
/** Basic abstraction of a simple event synchronization mechanism (aka 'binary semaphore')
diff --git a/zencore/include/zencore/uid.h b/zencore/include/zencore/uid.h
index f095c49ef..f4e9ab65a 100644
--- a/zencore/include/zencore/uid.h
+++ b/zencore/include/zencore/uid.h
@@ -60,6 +60,7 @@ struct Oid
const Oid& Generate();
[[nodiscard]] static Oid FromHexString(const std::string_view String);
StringBuilderBase& ToString(StringBuilderBase& OutString) const;
+ void ToString(char OutString[StringLength]);
[[nodiscard]] static Oid FromMemory(const void* Ptr);
auto operator<=>(const Oid& rhs) const = default;
diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h
index da17e61e3..310f6c4ed 100644
--- a/zencore/include/zencore/zencore.h
+++ b/zencore/include/zencore/zencore.h
@@ -3,7 +3,7 @@
#pragma once
#include <cinttypes>
-#include <exception>
+#include <stdexcept>
#include <string>
//////////////////////////////////////////////////////////////////////////
@@ -90,37 +90,69 @@
// Assert
//
+#if ZEN_PLATFORM_WINDOWS
+// Tells the compiler to put the decorated function in a certain section (aka. segment) of the executable.
+# define ZEN_CODE_SECTION(Name) __declspec(code_seg(Name))
+# define ZEN_FORCENOINLINE __declspec(noinline) /* Force code to NOT be inline */
+#else
+# define ZEN_CODE_SECTION(Name)
+# define ZEN_FORCENOINLINE
+#endif
+
+#if ZEN_ARCH_ARM64
+// On ARM we can't do this because the executable will require jumps larger
+// than the branch instruction can handle. Clang will only generate
+// the trampolines in the .text segment of the binary. If the uedbg segment
+// is present it will generate code that it cannot link.
+# define ZEN_DEBUG_SECTION
+#else
+// We'll put all assert implementation code into a separate section in the linked
+// executable. This code should never execute so using a separate section keeps
+// it well off the hot path and hopefully out of the instruction cache. It also
+// facilitates reasoning about the makeup of a compiled/linked binary.
+# define ZEN_DEBUG_SECTION ZEN_CODE_SECTION(".zcold")
+#endif // DO_CHECK || DO_GUARD_SLOW
+
namespace zen {
-class AssertException : public std::exception
+class AssertException : public std::logic_error
{
public:
- AssertException(const char* Msg);
- ~AssertException();
-
- [[nodiscard]] virtual char const* what() const noexcept override { return m_Msg.c_str(); }
-
-private:
- std::string m_Msg;
+ AssertException(const char* Msg) : std::logic_error(Msg) {}
};
} // namespace zen
-#define ZEN_ASSERT(x, ...) \
- do \
- { \
- if (x) \
- break; \
- throw ::zen::AssertException{#x}; \
+template<typename RetType = void, class InnerType, typename... ArgTypes>
+RetType ZEN_FORCENOINLINE ZEN_DEBUG_SECTION
+DispatchAssert(InnerType&& Inner, ArgTypes const&... Args)
+{
+ return Inner(Args...);
+}
+
+#define ZEN_ASSERT(x, ...) \
+ do \
+ { \
+ if (x) [[unlikely]] \
+ break; \
+ struct Impl \
+ { \
+ static void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION ExecThrow [[noreturn]] () { throw ::zen::AssertException{#x}; } \
+ }; \
+ Impl::ExecThrow(); \
} while (false)
#ifndef NDEBUG
-# define ZEN_ASSERT_SLOW(x, ...) \
- do \
- { \
- if (x) \
- break; \
- throw ::zen::AssertException{#x}; \
+# define ZEN_ASSERT_SLOW(x, ...) \
+ do \
+ { \
+ if (x) [[unlikely]] \
+ break; \
+ struct Impl \
+ { \
+ static void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION ExecThrow [[noreturn]] () { throw ::zen::AssertException{#x}; } \
+ }; \
+ Impl::ExecThrow(); \
} while (false)
#else
# define ZEN_ASSERT_SLOW(x, ...)
@@ -148,17 +180,19 @@ char (&ZenArrayCountHelper(const T (&)[N]))[N + 1];
#define ZEN_UNUSED(...) ((void)__VA_ARGS__)
#define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false, __VA_ARGS__)
-#define ZENCORE_API // Placeholder to allow DLL configs in the future
+#define ZENCORE_API // Placeholder to allow DLL configs in the future (maybe)
namespace zen {
ZENCORE_API bool IsPointerToStack(const void* ptr); // Query if pointer is within the stack of the currently executing thread
ZENCORE_API bool IsApplicationExitRequested();
ZENCORE_API void RequestApplicationExit(int ExitCode);
+ZENCORE_API bool IsDebuggerPresent();
+ZENCORE_API bool IsInteractiveSession();
ZENCORE_API void zencore_forcelinktests();
-}
+} // namespace zen
//////////////////////////////////////////////////////////////////////////
diff --git a/zencore/session.cpp b/zencore/session.cpp
index d57d3685b..ce4bfae1b 100644
--- a/zencore/session.cpp
+++ b/zencore/session.cpp
@@ -9,14 +9,27 @@
namespace zen {
static Oid GlobalSessionId;
+static char GlobalSessionString[Oid::StringLength];
static std::once_flag SessionInitFlag;
Oid
GetSessionId()
{
- std::call_once(SessionInitFlag, [&] { GlobalSessionId.Generate(); });
+ std::call_once(SessionInitFlag, [&] {
+ GlobalSessionId.Generate();
+ GlobalSessionId.ToString(GlobalSessionString);
+ });
return GlobalSessionId;
}
+std::string_view
+GetSessionIdString()
+{
+ // Ensure we actually have a generated session identifier
+ std::ignore = GetSessionId();
+
+ return std::string_view(GlobalSessionString, Oid::StringLength);
+}
+
} // namespace zen \ No newline at end of file
diff --git a/zencore/testutils.cpp b/zencore/testutils.cpp
new file mode 100644
index 000000000..116491950
--- /dev/null
+++ b/zencore/testutils.cpp
@@ -0,0 +1,33 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zencore/testutils.h"
+#include <zencore/session.h>
+#include "zencore/string.h"
+
+namespace zen {
+
+static std::atomic<int> Sequence{0};
+
+std::filesystem::path
+CreateTemporaryDirectory()
+{
+ std::error_code Ec;
+
+ std::filesystem::path DirPath = std::filesystem::temp_directory_path() / GetSessionIdString() / IntNum(++Sequence).c_str();
+ std::filesystem::remove_all(DirPath, Ec);
+ std::filesystem::create_directories(DirPath);
+
+ return DirPath;
+}
+
+ScopedTemporaryDirectory::ScopedTemporaryDirectory() : m_RootPath(CreateTemporaryDirectory())
+{
+}
+
+ScopedTemporaryDirectory::~ScopedTemporaryDirectory()
+{
+ std::error_code Ec;
+ std::filesystem::remove_all(m_RootPath, Ec);
+}
+
+} // namespace zen \ No newline at end of file
diff --git a/zencore/thread.cpp b/zencore/thread.cpp
index 1c7e4b3ab..d4f101454 100644
--- a/zencore/thread.cpp
+++ b/zencore/thread.cpp
@@ -17,41 +17,25 @@ namespace zen {
void
RwLock::AcquireShared()
{
-#if ZEN_PLATFORM_WINDOWS
- AcquireSRWLockShared((PSRWLOCK)&m_Srw);
-#else
m_Mutex.lock_shared();
-#endif
}
void
RwLock::ReleaseShared()
{
-#if ZEN_PLATFORM_WINDOWS
- ReleaseSRWLockShared((PSRWLOCK)&m_Srw);
-#else
m_Mutex.unlock_shared();
-#endif
}
void
RwLock::AcquireExclusive()
{
-#if ZEN_PLATFORM_WINDOWS
- AcquireSRWLockExclusive((PSRWLOCK)&m_Srw);
-#else
m_Mutex.lock();
-#endif
}
void
RwLock::ReleaseExclusive()
{
-#if ZEN_PLATFORM_WINDOWS
- ReleaseSRWLockExclusive((PSRWLOCK)&m_Srw);
-#else
m_Mutex.unlock();
-#endif
}
Event::Event()
@@ -257,7 +241,8 @@ ProcessHandle::Wait(int TimeoutMs)
case WAIT_FAILED:
// What might go wrong here, and what is meaningful to act on?
- throw WindowsException("Process::Wait failed");
+ using namespace std::literals;
+ ThrowLastError("Process::Wait failed"sv);
}
return false;
@@ -333,16 +318,32 @@ ProcessMonitor::IsActive() const
bool
IsProcessRunning(int pid)
{
+ // This function is arguably not super useful, a pid can be re-used
+ // by the OS so holding on to a pid and polling it over some time
+ // period will not necessarily tell you what you probably want to know.
+
+#if ZEN_PLATFORM_WINDOWS
HANDLE hProc = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid);
- if (hProc == NULL)
+ if (!hProc)
{
- return false;
+ DWORD Error = zen::GetLastError();
+
+ if (Error == ERROR_INVALID_PARAMETER)
+ {
+ return false;
+ }
+
+ using namespace fmt::literals;
+ ThrowSystemError(Error, "failed to open process with pid {}"_format(pid));
}
CloseHandle(hProc);
return true;
+#else
+ ZEN_NOT_IMPLEMENTED();
+#endif
}
int
diff --git a/zencore/uid.cpp b/zencore/uid.cpp
index d946638ec..ed00b1814 100644
--- a/zencore/uid.cpp
+++ b/zencore/uid.cpp
@@ -91,10 +91,16 @@ Oid::FromMemory(const void* Ptr)
return Id;
}
+void
+Oid::ToString(char OutString[StringLength])
+{
+ ToHexBytes(reinterpret_cast<const uint8_t*>(OidBits), sizeof(Oid::OidBits), OutString);
+}
+
StringBuilderBase&
Oid::ToString(StringBuilderBase& OutString) const
{
- char str[25];
+ char str[StringLength + 1];
ToHexBytes(reinterpret_cast<const uint8_t*>(OidBits), sizeof(Oid::OidBits), str);
str[2 * sizeof(Oid)] = '\0';
diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp
index 5899f014d..122719d90 100644
--- a/zencore/zencore.cpp
+++ b/zencore/zencore.cpp
@@ -30,6 +30,8 @@
namespace zen {
+//////////////////////////////////////////////////////////////////////////
+
bool
IsPointerToStack(const void* ptr)
{
@@ -55,12 +57,31 @@ IsPointerToStack(const void* ptr)
#endif
}
-AssertException::AssertException(const char* Msg) : m_Msg(Msg)
+bool
+IsDebuggerPresent()
{
+#if ZEN_PLATFORM_WINDOWS
+ return ::IsDebuggerPresent();
+#else
+ return false;
+#endif
}
-AssertException::~AssertException()
+bool
+IsInteractiveSession()
{
+#if ZEN_PLATFORM_WINDOWS
+ DWORD dwSessionId = 0;
+ if (ProcessIdToSessionId(GetCurrentProcessId(), &dwSessionId))
+ {
+ return (dwSessionId != 0);
+ }
+
+ return false;
+#else
+ // TODO: figure out what makes sense here
+ return true;
+#endif
}
//////////////////////////////////////////////////////////////////////////
diff --git a/zencore/zencore.vcxproj b/zencore/zencore.vcxproj
index 150c42cd6..59b0bafcc 100644
--- a/zencore/zencore.vcxproj
+++ b/zencore/zencore.vcxproj
@@ -142,6 +142,7 @@
<ClInclude Include="include\zencore\streamutil.h" />
<ClInclude Include="include\zencore\string.h" />
<ClInclude Include="include\zencore\targetver.h" />
+ <ClInclude Include="include\zencore\testutils.h" />
<ClInclude Include="include\zencore\thread.h" />
<ClInclude Include="include\zencore\timer.h" />
<ClInclude Include="include\zencore\uid.h" />
@@ -181,6 +182,7 @@
<ClCompile Include="stream.cpp" />
<ClCompile Include="streamutil.cpp" />
<ClCompile Include="string.cpp" />
+ <ClCompile Include="testutils.cpp" />
<ClCompile Include="thread.cpp" />
<ClCompile Include="timer.cpp" />
<ClCompile Include="uid.cpp" />
diff --git a/zencore/zencore.vcxproj.filters b/zencore/zencore.vcxproj.filters
index ea0f8a912..6c6b308f8 100644
--- a/zencore/zencore.vcxproj.filters
+++ b/zencore/zencore.vcxproj.filters
@@ -42,6 +42,7 @@
<ClInclude Include="include\zencore\postwindows.h" />
<ClInclude Include="include\zencore\logging.h" />
<ClInclude Include="include\zencore\session.h" />
+ <ClInclude Include="include\zencore\testutils.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="snapshot_manifest.cpp" />
@@ -74,6 +75,7 @@
<ClCompile Include="logging.cpp" />
<ClCompile Include="intmath.cpp" />
<ClCompile Include="session.cpp" />
+ <ClCompile Include="testutils.cpp" />
</ItemGroup>
<ItemGroup>
<Filter Include="CAS">
diff --git a/zenhttp/httpnull.cpp b/zenhttp/httpnull.cpp
index 57cba13d3..e49051ac5 100644
--- a/zenhttp/httpnull.cpp
+++ b/zenhttp/httpnull.cpp
@@ -28,8 +28,10 @@ HttpNullServer::Initialize(int BasePort)
}
void
-HttpNullServer::Run(bool TestMode)
+HttpNullServer::Run(bool IsInteractiveSession)
{
+ const bool TestMode = !IsInteractiveSession;
+
if (TestMode == false)
{
zen::logging::ConsoleLog().info("Zen Server running (null HTTP). Press ESC or Q to quit");
diff --git a/zenhttp/httpnull.h b/zenhttp/httpnull.h
index b15b1b123..867bbe4d2 100644
--- a/zenhttp/httpnull.h
+++ b/zenhttp/httpnull.h
@@ -19,7 +19,7 @@ public:
virtual void RegisterService(HttpService& Service) override;
virtual void Initialize(int BasePort) override;
- virtual void Run(bool TestMode) override;
+ virtual void Run(bool IsInteractiveSession) override;
virtual void RequestExit() override;
private:
diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp
index c2d4ef14c..3c5e5d8d3 100644
--- a/zenhttp/httpsys.cpp
+++ b/zenhttp/httpsys.cpp
@@ -707,29 +707,30 @@ HttpSysServer::StartServer()
}
void
-HttpSysServer::Run(bool TestMode)
+HttpSysServer::Run(bool IsInteractive)
{
- if (TestMode == false)
+ if (IsInteractive)
{
zen::logging::ConsoleLog().info("Zen Server running. Press ESC or Q to quit");
}
do
{
- int WaitTimeout = -1;
+ // int WaitTimeout = -1;
+ int WaitTimeout = 100;
- if (!TestMode)
+ if (IsInteractive)
{
WaitTimeout = 1000;
- }
-
- if (!TestMode && _kbhit() != 0)
- {
- char c = (char)_getch();
- if (c == 27 || c == 'Q' || c == 'q')
+ if (_kbhit() != 0)
{
- RequestApplicationExit(0);
+ char c = (char)_getch();
+
+ if (c == 27 || c == 'Q' || c == 'q')
+ {
+ RequestApplicationExit(0);
+ }
}
}
diff --git a/zenhttp/httpuws.cpp b/zenhttp/httpuws.cpp
index 992809b17..e062e7747 100644
--- a/zenhttp/httpuws.cpp
+++ b/zenhttp/httpuws.cpp
@@ -37,8 +37,10 @@ HttpUwsServer::Initialize(int BasePort)
}
void
-HttpUwsServer::Run(bool TestMode)
+HttpUwsServer::Run(bool IsInteractive)
{
+ const bool TestMode = !IsInteractive;
+
if (TestMode == false)
{
zen::logging::ConsoleLog().info("Zen Server running (null HTTP). Press ESC or Q to quit");
diff --git a/zenhttp/httpuws.h b/zenhttp/httpuws.h
index ec55ae2fd..5e300202f 100644
--- a/zenhttp/httpuws.h
+++ b/zenhttp/httpuws.h
@@ -16,7 +16,7 @@ public:
virtual void RegisterService(HttpService& Service) override;
virtual void Initialize(int BasePort) override;
- virtual void Run(bool TestMode) override;
+ virtual void Run(bool IsInteractiveSession) override;
virtual void RequestExit() override;
private:
diff --git a/zenhttp/include/zenhttp/httpserver.h b/zenhttp/include/zenhttp/httpserver.h
index ed6075c92..6a7dc8a70 100644
--- a/zenhttp/include/zenhttp/httpserver.h
+++ b/zenhttp/include/zenhttp/httpserver.h
@@ -167,7 +167,7 @@ class HttpServer : public RefCounted
public:
virtual void RegisterService(HttpService& Service) = 0;
virtual void Initialize(int BasePort) = 0;
- virtual void Run(bool TestMode) = 0;
+ virtual void Run(bool IsInteractiveSession) = 0;
virtual void RequestExit() = 0;
};
diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp
index 7629e2ea1..a556d896e 100644
--- a/zenserver-test/zenserver-test.cpp
+++ b/zenserver-test/zenserver-test.cpp
@@ -12,6 +12,7 @@
#include <zencore/iohash.h>
#include <zencore/logging.h>
#include <zencore/memory.h>
+#include <zencore/refcount.h>
#include <zencore/stream.h>
#include <zencore/string.h>
#include <zencore/thread.h>
@@ -37,6 +38,7 @@
#include <map>
#include <random>
#include <span>
+#include <unordered_map>
#include <atlbase.h>
#include <process.h>
@@ -80,7 +82,7 @@ class HttpClientConnection
static HttpClientConnection* This(http_parser* Parser) { return (HttpClientConnection*)Parser->data; };
public:
- HttpClientConnection(asio::io_context& IoContext, HttpConnectionPool& Pool, asio::ip::tcp::socket&& InSocket)
+ HttpClientConnection(asio::io_context& IoContext, zen::Ref<HttpConnectionPool> Pool, asio::ip::tcp::socket&& InSocket)
: m_IoContext(IoContext)
, m_Pool(Pool)
, m_Resolver(IoContext)
@@ -89,8 +91,8 @@ public:
}
~HttpClientConnection() {}
- HttpConnectionPool& ConnectionPool() { return m_Pool; }
- void SetKeepAlive(bool NewState) { m_KeepAlive = NewState; }
+ zen::Ref<HttpConnectionPool> ConnectionPool() { return m_Pool; }
+ void SetKeepAlive(bool NewState) { m_KeepAlive = NewState; }
void Get(const std::string_view Server, int Port, const std::string_view Path)
{
@@ -227,16 +229,16 @@ private:
}
private:
- asio::io_context& m_IoContext;
- HttpConnectionPool& m_Pool;
- asio::ip::tcp::resolver m_Resolver;
- asio::ip::tcp::socket m_Socket;
- std::string m_Uri;
- std::string m_RequestBody; // Initial request data
- http_parser m_HttpParser{};
- http_parser_settings m_HttpParserSettings{};
- uint8_t m_ResponseIoBuffer[4096];
- asio::mutable_buffer m_ResponseBuffer{m_ResponseIoBuffer, sizeof m_ResponseIoBuffer};
+ asio::io_context& m_IoContext;
+ zen::Ref<HttpConnectionPool> m_Pool;
+ asio::ip::tcp::resolver m_Resolver;
+ asio::ip::tcp::socket m_Socket;
+ std::string m_Uri;
+ std::string m_RequestBody; // Initial request data
+ http_parser m_HttpParser{};
+ http_parser_settings m_HttpParserSettings{};
+ uint8_t m_ResponseIoBuffer[4096];
+ asio::mutable_buffer m_ResponseBuffer{m_ResponseIoBuffer, sizeof m_ResponseIoBuffer};
enum class RequestState
{
@@ -259,7 +261,7 @@ private:
//////////////////////////////////////////////////////////////////////////
-class HttpConnectionPool
+class HttpConnectionPool : public zen::RefCounted
{
public:
HttpConnectionPool(asio::io_context& Context, std::string_view HostName, uint16_t Port);
@@ -322,7 +324,7 @@ HttpConnectionPool::GetConnection()
return nullptr;
}
- return std::make_unique<HttpClientConnection>(m_Context, *this, std::move(Socket));
+ return std::make_unique<HttpClientConnection>(m_Context, this, std::move(Socket));
}
std::unique_ptr<HttpClientConnection> Connection{m_AvailableConnections.back()};
@@ -347,15 +349,15 @@ public:
std::unique_ptr<HttpClientConnection> GetConnection(std::string_view HostName, uint16_t Port)
{
- return ConnectionPool(HostName, Port).GetConnection();
+ return ConnectionPool(HostName, Port)->GetConnection();
}
void ReturnConnection(std::unique_ptr<HttpClientConnection> Connection)
{
- Connection->ConnectionPool().ReturnConnection(std::move(Connection));
+ Connection->ConnectionPool()->ReturnConnection(std::move(Connection));
}
- HttpConnectionPool& ConnectionPool(std::string_view HostName, uint16_t Port)
+ zen::Ref<HttpConnectionPool> ConnectionPool(std::string_view HostName, uint16_t Port)
{
zen::RwLock::ExclusiveLockScope _(m_Lock);
ConnectionId ConnId{std::string(HostName), Port};
@@ -364,7 +366,7 @@ public:
{
// Not found - create new entry
- auto In = m_ConnectionPools.insert({ConnId, std::move(HttpConnectionPool(m_Context, HostName, Port))});
+ auto In = m_ConnectionPools.emplace(ConnId, new HttpConnectionPool(m_Context, HostName, Port));
return In.first->second;
}
@@ -393,8 +395,8 @@ private:
uint16_t Port;
};
- zen::RwLock m_Lock;
- std::map<ConnectionId, HttpConnectionPool> m_ConnectionPools;
+ zen::RwLock m_Lock;
+ std::map<ConnectionId, zen::Ref<HttpConnectionPool>> m_ConnectionPools;
};
//////////////////////////////////////////////////////////////////////////
@@ -670,7 +672,7 @@ main(int argc, char** argv)
zen::logging::InitializeLogging();
spdlog::set_level(spdlog::level::debug);
- spdlog::set_formatter(std::make_unique< ::logging::full_formatter>("test", std::chrono::system_clock::now()));
+ spdlog::set_formatter(std::make_unique<::logging::full_formatter>("test", std::chrono::system_clock::now()));
std::filesystem::path ProgramBaseDir = std::filesystem::path(argv[0]).parent_path();
std::filesystem::path TestBaseDir = ProgramBaseDir.parent_path().parent_path() / ".test";
@@ -1919,9 +1921,9 @@ public:
ZenServerInstance& GetInstance(int Index) { return *m_Instances[Index]; }
private:
- std::string m_HelperId;
- int m_ServerCount = 0;
- std::vector<std::unique_ptr<ZenServerInstance> > m_Instances;
+ std::string m_HelperId;
+ int m_ServerCount = 0;
+ std::vector<std::unique_ptr<ZenServerInstance>> m_Instances;
};
TEST_CASE("http.basics")
diff --git a/zenserver/config.cpp b/zenserver/config.cpp
index 30401a52e..164d2a792 100644
--- a/zenserver/config.cpp
+++ b/zenserver/config.cpp
@@ -98,6 +98,21 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
cxxopts::value<std::string>(GlobalOptions.ChildId),
"<identifier>");
+#if ZEN_PLATFORM_WINDOWS
+ options.add_option("lifetime",
+ "",
+ "install",
+ "Install zenserver as a Windows service",
+ cxxopts::value<bool>(GlobalOptions.InstallService),
+ "");
+ options.add_option("lifetime",
+ "",
+ "uninstall",
+ "Uninstall zenserver as a Windows service",
+ cxxopts::value<bool>(GlobalOptions.UninstallService),
+ "");
+#endif
+
options.add_option("network",
"p",
"port",
diff --git a/zenserver/config.h b/zenserver/config.h
index c3f08f84f..6ade1b401 100644
--- a/zenserver/config.h
+++ b/zenserver/config.h
@@ -9,12 +9,14 @@ struct ZenServerOptions
{
bool IsDebug = false;
bool IsTest = false;
- bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements
- int BasePort = 1337; // Service listen port (used for both UDP and TCP)
- int OwnerPid = 0; // Parent process id (zero for standalone)
- std::string ChildId; // Id assigned by parent process (used for lifetime management)
- std::string LogId; // Id for tagging log output
- std::filesystem::path DataDir; // Root directory for state (used for testing)
+ bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements
+ int BasePort = 1337; // Service listen port (used for both UDP and TCP)
+ int OwnerPid = 0; // Parent process id (zero for standalone)
+ std::string ChildId; // Id assigned by parent process (used for lifetime management)
+ bool InstallService = false; // Flag used to initiate service install (temporary)
+ bool UninstallService = false; // Flag used to initiate service uninstall (temporary)
+ std::string LogId; // Id for tagging log output
+ std::filesystem::path DataDir; // Root directory for state (used for testing)
};
struct ZenUpstreamJupiterConfig
diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp
index 4ba4835af..41b140f90 100644
--- a/zenserver/diag/logging.cpp
+++ b/zenserver/diag/logging.cpp
@@ -9,6 +9,9 @@
#include <spdlog/pattern_formatter.h>
#include <spdlog/sinks/ansicolor_sink.h>
#include <spdlog/sinks/basic_file_sink.h>
+#include <spdlog/sinks/daily_file_sink.h>
+#include <spdlog/sinks/msvc_sink.h>
+#include <spdlog/sinks/rotating_file_sink.h>
#include <spdlog/sinks/stdout_color_sinks.h>
#include <spdlog/spdlog.h>
#include <zencore/string.h>
@@ -207,7 +210,7 @@ InitializeLogging(const ZenServerOptions& GlobalOptions)
if (GlobalOptions.IsTest)
{
LogLevel = spdlog::level::trace;
- IsAsync = false;
+ IsAsync = false;
}
if (IsAsync)
@@ -223,7 +226,19 @@ InitializeLogging(const ZenServerOptions& GlobalOptions)
// Sinks
auto ConsoleSink = std::make_shared<spdlog::sinks::ansicolor_stdout_sink_mt>();
- auto FileSink = std::make_shared<spdlog::sinks::basic_file_sink_mt>(zen::WideToUtf8(LogPath.c_str()), /* truncate */ true);
+
+#if 0
+ auto FileSink = std::make_shared<spdlog::sinks::daily_file_sink_mt>(zen::WideToUtf8(LogPath.c_str()),
+ 0,
+ 0,
+ /* truncate */ false,
+ uint16_t(/* max files */ 14));
+#else
+ auto FileSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::WideToUtf8(LogPath.c_str()),
+ /* max size */ 128 * 1024 * 1024,
+ /* max files */ 16,
+ /* rotate on open */ true);
+#endif
// Default
@@ -234,20 +249,30 @@ InitializeLogging(const ZenServerOptions& GlobalOptions)
Sinks.push_back(ConsoleSink);
Sinks.push_back(FileSink);
+#if ZEN_PLATFORM_WINDOWS
+ if (zen::IsDebuggerPresent())
+ {
+ auto DebugSink = std::make_shared<spdlog::sinks::msvc_sink_mt>();
+ DebugSink->set_level(spdlog::level::debug);
+ Sinks.push_back(DebugSink);
+ }
+#endif
+
// Jupiter - only log HTTP traffic to file
auto JupiterLogger = std::make_shared<spdlog::logger>("jupiter", FileSink);
spdlog::register_logger(JupiterLogger);
- JupiterLogger->set_level(LogLevel);
+
+ // Zen - only log HTTP traffic to file
auto ZenClientLogger = std::make_shared<spdlog::logger>("zenclient", FileSink);
spdlog::register_logger(ZenClientLogger);
- ZenClientLogger->set_level(LogLevel);
// Configure all registered loggers according to settings
spdlog::set_level(LogLevel);
spdlog::flush_on(spdlog::level::err);
+ spdlog::flush_every(std::chrono::seconds{2});
spdlog::set_formatter(std::make_unique<logging::full_formatter>(GlobalOptions.LogId, std::chrono::system_clock::now()));
}
diff --git a/zenserver/experimental/usnjournal.cpp b/zenserver/experimental/usnjournal.cpp
index ab83b8a1c..1e765fbe5 100644
--- a/zenserver/experimental/usnjournal.cpp
+++ b/zenserver/experimental/usnjournal.cpp
@@ -34,14 +34,14 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath)
if (!Success)
{
- zen::ThrowSystemException("GetVolumePathName failed");
+ zen::ThrowLastError("GetVolumePathName failed");
}
Success = GetVolumeNameForVolumeMountPoint(VolumePathName, VolumeName, ZEN_ARRAY_COUNT(VolumeName));
if (!Success)
{
- zen::ThrowSystemException("GetVolumeNameForVolumeMountPoint failed");
+ zen::ThrowLastError("GetVolumeNameForVolumeMountPoint failed");
}
// Chop off trailing slash since we want to open a volume handle, not a handle to the volume root directory
@@ -64,7 +64,7 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath)
if (m_VolumeHandle == INVALID_HANDLE_VALUE)
{
- ThrowSystemException("Volume handle open failed");
+ ThrowLastError("Volume handle open failed");
}
// Figure out which file system is in use for volume
@@ -86,7 +86,7 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath)
if (!Success)
{
- ThrowSystemException("Failed to get volume information");
+ ThrowLastError("Failed to get volume information");
}
ZEN_DEBUG("File system type is {}", WideToUtf8(FileSystemName));
@@ -173,7 +173,7 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath)
if (!Success)
{
- ThrowSystemException("GetFileInformationByHandleEx failed");
+ ThrowLastError("GetFileInformationByHandleEx failed");
}
const Frn VolumeRootFrn = FileInformation.FileId;
diff --git a/zenserver/windows/service.cpp b/zenserver/windows/service.cpp
new file mode 100644
index 000000000..017b5f9a7
--- /dev/null
+++ b/zenserver/windows/service.cpp
@@ -0,0 +1,631 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "service.h"
+
+#include <zencore/zencore.h>
+
+#include <stdio.h>
+#include <tchar.h>
+#include <zencore/windows.h>
+
+#define SVCNAME L"Zen Store"
+
+SERVICE_STATUS gSvcStatus;
+SERVICE_STATUS_HANDLE gSvcStatusHandle;
+HANDLE ghSvcStopEvent = NULL;
+
+void SvcInstall(void);
+
+void ReportSvcStatus(DWORD, DWORD, DWORD);
+void SvcReportEvent(LPTSTR);
+
+WindowsService::WindowsService()
+{
+}
+
+WindowsService::~WindowsService()
+{
+}
+
+//
+// Purpose:
+// Installs a service in the SCM database
+//
+// Parameters:
+// None
+//
+// Return value:
+// None
+//
+VOID
+WindowsService::Install()
+{
+ SC_HANDLE schSCManager;
+ SC_HANDLE schService;
+ TCHAR szPath[MAX_PATH];
+
+ if (!GetModuleFileName(NULL, szPath, MAX_PATH))
+ {
+ printf("Cannot install service (%d)\n", GetLastError());
+ return;
+ }
+
+ // Get a handle to the SCM database.
+
+ schSCManager = OpenSCManager(NULL, // local computer
+ NULL, // ServicesActive database
+ SC_MANAGER_ALL_ACCESS); // full access rights
+
+ if (NULL == schSCManager)
+ {
+ printf("OpenSCManager failed (%d)\n", GetLastError());
+ return;
+ }
+
+ // Create the service
+
+ schService = CreateService(schSCManager, // SCM database
+ SVCNAME, // name of service
+ SVCNAME, // service name to display
+ SERVICE_ALL_ACCESS, // desired access
+ SERVICE_WIN32_OWN_PROCESS, // service type
+ SERVICE_DEMAND_START, // start type
+ SERVICE_ERROR_NORMAL, // error control type
+ szPath, // path to service's binary
+ NULL, // no load ordering group
+ NULL, // no tag identifier
+ NULL, // no dependencies
+ NULL, // LocalSystem account
+ NULL); // no password
+
+ if (schService == NULL)
+ {
+ printf("CreateService failed (%d)\n", GetLastError());
+ CloseServiceHandle(schSCManager);
+ return;
+ }
+ else
+ printf("Service installed successfully\n");
+
+ CloseServiceHandle(schService);
+ CloseServiceHandle(schSCManager);
+}
+
+void
+WindowsService::Delete()
+{
+ SC_HANDLE schSCManager;
+ SC_HANDLE schService;
+
+ // Get a handle to the SCM database.
+
+ schSCManager = OpenSCManager(NULL, // local computer
+ NULL, // ServicesActive database
+ SC_MANAGER_ALL_ACCESS); // full access rights
+
+ if (NULL == schSCManager)
+ {
+ printf("OpenSCManager failed (%d)\n", GetLastError());
+ return;
+ }
+
+ // Get a handle to the service.
+
+ schService = OpenService(schSCManager, // SCM database
+ SVCNAME, // name of service
+ DELETE); // need delete access
+
+ if (schService == NULL)
+ {
+ printf("OpenService failed (%d)\n", GetLastError());
+ CloseServiceHandle(schSCManager);
+ return;
+ }
+
+ // Delete the service.
+
+ if (!DeleteService(schService))
+ {
+ printf("DeleteService failed (%d)\n", GetLastError());
+ }
+ else
+ printf("Service deleted successfully\n");
+
+ CloseServiceHandle(schService);
+ CloseServiceHandle(schSCManager);
+}
+
+WindowsService* gSvc;
+
+void WINAPI
+CallMain(DWORD, LPSTR*)
+{
+ gSvc->SvcMain();
+}
+
+int
+WindowsService::ServiceMain()
+{
+ if (zen::IsInteractiveSession())
+ {
+ // Not actually running as a service
+ return Run();
+ }
+ else
+ {
+ gSvc = this;
+
+ SERVICE_TABLE_ENTRY DispatchTable[] = {{(LPWSTR)SVCNAME, (LPSERVICE_MAIN_FUNCTION)&CallMain}, {NULL, NULL}};
+
+ // This call returns when the service has stopped.
+ // The process should simply terminate when the call returns.
+
+ if (!StartServiceCtrlDispatcher(DispatchTable))
+ {
+ SvcReportEvent((LPTSTR)L"StartServiceCtrlDispatcher");
+ }
+ }
+
+ return 0;
+}
+
+int
+WindowsService::SvcMain()
+{
+ // Register the handler function for the service
+
+ gSvcStatusHandle = RegisterServiceCtrlHandler(SVCNAME, SvcCtrlHandler);
+
+ if (!gSvcStatusHandle)
+ {
+ SvcReportEvent((LPTSTR)TEXT("RegisterServiceCtrlHandler"));
+
+ return 1;
+ }
+
+ // These SERVICE_STATUS members remain as set here
+
+ gSvcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS;
+ gSvcStatus.dwServiceSpecificExitCode = 0;
+
+ // Report initial status to the SCM
+
+ ReportSvcStatus(SERVICE_START_PENDING, NO_ERROR, 3000);
+
+ // Create an event. The control handler function, SvcCtrlHandler,
+ // signals this event when it receives the stop control code.
+
+ ghSvcStopEvent = CreateEvent(NULL, // default security attributes
+ TRUE, // manual reset event
+ FALSE, // not signaled
+ NULL); // no name
+
+ if (ghSvcStopEvent == NULL)
+ {
+ ReportSvcStatus(SERVICE_STOPPED, GetLastError(), 0);
+
+ return 1;
+ }
+
+ // Report running status when initialization is complete.
+
+ ReportSvcStatus(SERVICE_RUNNING, NO_ERROR, 0);
+
+ int ReturnCode = Run();
+
+ ReportSvcStatus(SERVICE_STOPPED, NO_ERROR, 0);
+
+ return ReturnCode;
+}
+
+//
+// Purpose:
+// Retrieves and displays the current service configuration.
+//
+// Parameters:
+// None
+//
+// Return value:
+// None
+//
+void
+DoQuerySvc()
+{
+ SC_HANDLE schSCManager{};
+ SC_HANDLE schService{};
+ LPQUERY_SERVICE_CONFIG lpsc{};
+ LPSERVICE_DESCRIPTION lpsd{};
+ DWORD dwBytesNeeded{}, cbBufSize{}, dwError{};
+
+ // Get a handle to the SCM database.
+
+ schSCManager = OpenSCManager(NULL, // local computer
+ NULL, // ServicesActive database
+ SC_MANAGER_ALL_ACCESS); // full access rights
+
+ if (NULL == schSCManager)
+ {
+ printf("OpenSCManager failed (%d)\n", GetLastError());
+ return;
+ }
+
+ // Get a handle to the service.
+
+ schService = OpenService(schSCManager, // SCM database
+ SVCNAME, // name of service
+ SERVICE_QUERY_CONFIG); // need query config access
+
+ if (schService == NULL)
+ {
+ printf("OpenService failed (%d)\n", GetLastError());
+ CloseServiceHandle(schSCManager);
+ return;
+ }
+
+ // Get the configuration information.
+
+ if (!QueryServiceConfig(schService, NULL, 0, &dwBytesNeeded))
+ {
+ dwError = GetLastError();
+ if (ERROR_INSUFFICIENT_BUFFER == dwError)
+ {
+ cbBufSize = dwBytesNeeded;
+ lpsc = (LPQUERY_SERVICE_CONFIG)LocalAlloc(LMEM_FIXED, cbBufSize);
+ }
+ else
+ {
+ printf("QueryServiceConfig failed (%d)", dwError);
+ goto cleanup;
+ }
+ }
+
+ if (!QueryServiceConfig(schService, lpsc, cbBufSize, &dwBytesNeeded))
+ {
+ printf("QueryServiceConfig failed (%d)", GetLastError());
+ goto cleanup;
+ }
+
+ if (!QueryServiceConfig2(schService, SERVICE_CONFIG_DESCRIPTION, NULL, 0, &dwBytesNeeded))
+ {
+ dwError = GetLastError();
+ if (ERROR_INSUFFICIENT_BUFFER == dwError)
+ {
+ cbBufSize = dwBytesNeeded;
+ lpsd = (LPSERVICE_DESCRIPTION)LocalAlloc(LMEM_FIXED, cbBufSize);
+ }
+ else
+ {
+ printf("QueryServiceConfig2 failed (%d)", dwError);
+ goto cleanup;
+ }
+ }
+
+ if (!QueryServiceConfig2(schService, SERVICE_CONFIG_DESCRIPTION, (LPBYTE)lpsd, cbBufSize, &dwBytesNeeded))
+ {
+ printf("QueryServiceConfig2 failed (%d)", GetLastError());
+ goto cleanup;
+ }
+
+ // Print the configuration information.
+
+ _tprintf(TEXT("%s configuration: \n"), SVCNAME);
+ _tprintf(TEXT(" Type: 0x%x\n"), lpsc->dwServiceType);
+ _tprintf(TEXT(" Start Type: 0x%x\n"), lpsc->dwStartType);
+ _tprintf(TEXT(" Error Control: 0x%x\n"), lpsc->dwErrorControl);
+ _tprintf(TEXT(" Binary path: %s\n"), lpsc->lpBinaryPathName);
+ _tprintf(TEXT(" Account: %s\n"), lpsc->lpServiceStartName);
+
+ if (lpsd->lpDescription != NULL && lstrcmp(lpsd->lpDescription, TEXT("")) != 0)
+ _tprintf(TEXT(" Description: %s\n"), lpsd->lpDescription);
+ if (lpsc->lpLoadOrderGroup != NULL && lstrcmp(lpsc->lpLoadOrderGroup, TEXT("")) != 0)
+ _tprintf(TEXT(" Load order group: %s\n"), lpsc->lpLoadOrderGroup);
+ if (lpsc->dwTagId != 0)
+ _tprintf(TEXT(" Tag ID: %d\n"), lpsc->dwTagId);
+ if (lpsc->lpDependencies != NULL && lstrcmp(lpsc->lpDependencies, TEXT("")) != 0)
+ _tprintf(TEXT(" Dependencies: %s\n"), lpsc->lpDependencies);
+
+ LocalFree(lpsc);
+ LocalFree(lpsd);
+
+cleanup:
+ CloseServiceHandle(schService);
+ CloseServiceHandle(schSCManager);
+}
+
+//
+// Purpose:
+// Disables the service.
+//
+// Parameters:
+// None
+//
+// Return value:
+// None
+//
+void
+DoDisableSvc()
+{
+ SC_HANDLE schSCManager;
+ SC_HANDLE schService;
+
+ // Get a handle to the SCM database.
+
+ schSCManager = OpenSCManager(NULL, // local computer
+ NULL, // ServicesActive database
+ SC_MANAGER_ALL_ACCESS); // full access rights
+
+ if (NULL == schSCManager)
+ {
+ printf("OpenSCManager failed (%d)\n", GetLastError());
+ return;
+ }
+
+ // Get a handle to the service.
+
+ schService = OpenService(schSCManager, // SCM database
+ SVCNAME, // name of service
+ SERVICE_CHANGE_CONFIG); // need change config access
+
+ if (schService == NULL)
+ {
+ printf("OpenService failed (%d)\n", GetLastError());
+ CloseServiceHandle(schSCManager);
+ return;
+ }
+
+ // Change the service start type.
+
+ if (!ChangeServiceConfig(schService, // handle of service
+ SERVICE_NO_CHANGE, // service type: no change
+ SERVICE_DISABLED, // service start type
+ SERVICE_NO_CHANGE, // error control: no change
+ NULL, // binary path: no change
+ NULL, // load order group: no change
+ NULL, // tag ID: no change
+ NULL, // dependencies: no change
+ NULL, // account name: no change
+ NULL, // password: no change
+ NULL)) // display name: no change
+ {
+ printf("ChangeServiceConfig failed (%d)\n", GetLastError());
+ }
+ else
+ printf("Service disabled successfully.\n");
+
+ CloseServiceHandle(schService);
+ CloseServiceHandle(schSCManager);
+}
+
+//
+// Purpose:
+// Enables the service.
+//
+// Parameters:
+// None
+//
+// Return value:
+// None
+//
+VOID __stdcall DoEnableSvc()
+{
+ SC_HANDLE schSCManager;
+ SC_HANDLE schService;
+
+ // Get a handle to the SCM database.
+
+ schSCManager = OpenSCManager(NULL, // local computer
+ NULL, // ServicesActive database
+ SC_MANAGER_ALL_ACCESS); // full access rights
+
+ if (NULL == schSCManager)
+ {
+ printf("OpenSCManager failed (%d)\n", GetLastError());
+ return;
+ }
+
+ // Get a handle to the service.
+
+ schService = OpenService(schSCManager, // SCM database
+ SVCNAME, // name of service
+ SERVICE_CHANGE_CONFIG); // need change config access
+
+ if (schService == NULL)
+ {
+ printf("OpenService failed (%d)\n", GetLastError());
+ CloseServiceHandle(schSCManager);
+ return;
+ }
+
+ // Change the service start type.
+
+ if (!ChangeServiceConfig(schService, // handle of service
+ SERVICE_NO_CHANGE, // service type: no change
+ SERVICE_DEMAND_START, // service start type
+ SERVICE_NO_CHANGE, // error control: no change
+ NULL, // binary path: no change
+ NULL, // load order group: no change
+ NULL, // tag ID: no change
+ NULL, // dependencies: no change
+ NULL, // account name: no change
+ NULL, // password: no change
+ NULL)) // display name: no change
+ {
+ printf("ChangeServiceConfig failed (%d)\n", GetLastError());
+ }
+ else
+ printf("Service enabled successfully.\n");
+
+ CloseServiceHandle(schService);
+ CloseServiceHandle(schSCManager);
+}
+//
+// Purpose:
+// Updates the service description to "This is a test description".
+//
+// Parameters:
+// None
+//
+// Return value:
+// None
+//
+void
+DoUpdateSvcDesc()
+{
+ SC_HANDLE schSCManager;
+ SC_HANDLE schService;
+ SERVICE_DESCRIPTION sd;
+ TCHAR szDesc[] = TEXT("This is a test description");
+
+ // Get a handle to the SCM database.
+
+ schSCManager = OpenSCManager(NULL, // local computer
+ NULL, // ServicesActive database
+ SC_MANAGER_ALL_ACCESS); // full access rights
+
+ if (NULL == schSCManager)
+ {
+ printf("OpenSCManager failed (%d)\n", GetLastError());
+ return;
+ }
+
+ // Get a handle to the service.
+
+ schService = OpenService(schSCManager, // SCM database
+ SVCNAME, // name of service
+ SERVICE_CHANGE_CONFIG); // need change config access
+
+ if (schService == NULL)
+ {
+ printf("OpenService failed (%d)\n", GetLastError());
+ CloseServiceHandle(schSCManager);
+ return;
+ }
+
+ // Change the service description.
+
+ sd.lpDescription = szDesc;
+
+ if (!ChangeServiceConfig2(schService, // handle to service
+ SERVICE_CONFIG_DESCRIPTION, // change: description
+ &sd)) // new description
+ {
+ printf("ChangeServiceConfig2 failed\n");
+ }
+ else
+ printf("Service description updated successfully.\n");
+
+ CloseServiceHandle(schService);
+ CloseServiceHandle(schSCManager);
+}
+
+//
+// Purpose:
+// Sets the current service status and reports it to the SCM.
+//
+// Parameters:
+// dwCurrentState - The current state (see SERVICE_STATUS)
+// dwWin32ExitCode - The system error code
+// dwWaitHint - Estimated time for pending operation,
+// in milliseconds
+//
+// Return value:
+// None
+//
+VOID
+ReportSvcStatus(DWORD dwCurrentState, DWORD dwWin32ExitCode, DWORD dwWaitHint)
+{
+ static DWORD dwCheckPoint = 1;
+
+ // Fill in the SERVICE_STATUS structure.
+
+ gSvcStatus.dwCurrentState = dwCurrentState;
+ gSvcStatus.dwWin32ExitCode = dwWin32ExitCode;
+ gSvcStatus.dwWaitHint = dwWaitHint;
+
+ if (dwCurrentState == SERVICE_START_PENDING)
+ gSvcStatus.dwControlsAccepted = 0;
+ else
+ gSvcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP;
+
+ if ((dwCurrentState == SERVICE_RUNNING) || (dwCurrentState == SERVICE_STOPPED))
+ gSvcStatus.dwCheckPoint = 0;
+ else
+ gSvcStatus.dwCheckPoint = dwCheckPoint++;
+
+ // Report the status of the service to the SCM.
+ SetServiceStatus(gSvcStatusHandle, &gSvcStatus);
+}
+
+void
+WindowsService::SvcCtrlHandler(DWORD dwCtrl)
+{
+ // Handle the requested control code.
+ //
+ // Called by SCM whenever a control code is sent to the service
+ // using the ControlService function.
+
+ switch (dwCtrl)
+ {
+ case SERVICE_CONTROL_STOP:
+ ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, 0);
+
+ // Signal the service to stop.
+
+ SetEvent(ghSvcStopEvent);
+ zen::RequestApplicationExit(0);
+
+ ReportSvcStatus(gSvcStatus.dwCurrentState, NO_ERROR, 0);
+ return;
+
+ case SERVICE_CONTROL_INTERROGATE:
+ break;
+
+ default:
+ break;
+ }
+}
+
+//
+// Purpose:
+// Logs messages to the event log
+//
+// Parameters:
+// szFunction - name of function that failed
+//
+// Return value:
+// None
+//
+// Remarks:
+// The service must have an entry in the Application event log.
+//
+VOID
+SvcReportEvent(LPTSTR szFunction)
+{
+ ZEN_UNUSED(szFunction);
+
+ // HANDLE hEventSource;
+ // LPCTSTR lpszStrings[2];
+ // TCHAR Buffer[80];
+
+ // hEventSource = RegisterEventSource(NULL, SVCNAME);
+
+ // if (NULL != hEventSource)
+ //{
+ // StringCchPrintf(Buffer, 80, TEXT("%s failed with %d"), szFunction, GetLastError());
+
+ // lpszStrings[0] = SVCNAME;
+ // lpszStrings[1] = Buffer;
+
+ // ReportEvent(hEventSource, // event log handle
+ // EVENTLOG_ERROR_TYPE, // event type
+ // 0, // event category
+ // SVC_ERROR, // event identifier
+ // NULL, // no security identifier
+ // 2, // size of lpszStrings array
+ // 0, // no binary data
+ // lpszStrings, // array of strings
+ // NULL); // no binary data
+
+ // DeregisterEventSource(hEventSource);
+ //}
+}
diff --git a/zenserver/windows/service.h b/zenserver/windows/service.h
new file mode 100644
index 000000000..7c9610983
--- /dev/null
+++ b/zenserver/windows/service.h
@@ -0,0 +1,20 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+class WindowsService
+{
+public:
+ WindowsService();
+ ~WindowsService();
+
+ virtual int Run() = 0;
+
+ int ServiceMain();
+
+ static void Install();
+ static void Delete();
+
+ int SvcMain();
+ static void __stdcall SvcCtrlHandler(unsigned long);
+};
diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua
index bb70846fa..7a6981fcd 100644
--- a/zenserver/xmake.lua
+++ b/zenserver/xmake.lua
@@ -14,6 +14,8 @@ target("zenserver")
add_ldflags("/MANIFEST:EMBED")
add_ldflags("/MANIFESTUAC:level='requireAdministrator'")
add_ldflags("/LTCG")
+ else
+ del_files("windows/**")
end
add_options("vfs")
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 63e94ab7d..ea4a2915e 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -40,6 +40,10 @@
#include "config.h"
#include "diag/logging.h"
+#if ZEN_PLATFORM_WINDOWS
+# include "windows/service.h"
+#endif
+
//////////////////////////////////////////////////////////////////////////
// Sentry
//
@@ -312,7 +316,9 @@ public:
__debugbreak();
}
- m_Http->Run(m_TestMode);
+ const bool IsInteractiveMode = zen::IsInteractiveSession() && !m_TestMode;
+
+ m_Http->Run(IsInteractiveMode);
ZEN_INFO(ZEN_APP_NAME " exiting");
@@ -432,17 +438,29 @@ private:
} // namespace zen
-int
-main(int argc, char* argv[])
+class ZenWindowsService : public WindowsService
{
- using namespace zen;
+public:
+ ZenWindowsService(ZenServerOptions& GlobalOptions, ZenServiceConfig& ServiceConfig)
+ : m_GlobalOptions(GlobalOptions)
+ , m_ServiceConfig(ServiceConfig)
+ {
+ }
- mi_version();
+ ZenWindowsService(const ZenWindowsService&) = delete;
+ ZenWindowsService& operator=(const ZenWindowsService&) = delete;
- ZenServerOptions GlobalOptions;
- ZenServiceConfig ServiceConfig;
- ParseGlobalCliOptions(argc, argv, GlobalOptions, ServiceConfig);
- InitializeLogging(GlobalOptions);
+ virtual int Run() override;
+
+private:
+ ZenServerOptions& m_GlobalOptions;
+ ZenServiceConfig& m_ServiceConfig;
+};
+
+int
+ZenWindowsService::Run()
+{
+ using namespace zen;
#if USE_SENTRY
// Initialize sentry.io client
@@ -451,9 +469,12 @@ main(int argc, char* argv[])
sentry_options_set_dsn(SentryOptions, "https://[email protected]/5919284");
sentry_init(SentryOptions);
- auto _ = zen::MakeGuard([&] { sentry_close(); });
+ auto _ = zen::MakeGuard([] { sentry_close(); });
#endif
+ auto& GlobalOptions = m_GlobalOptions;
+ auto& ServiceConfig = m_ServiceConfig;
+
try
{
// Prototype config system, we'll see how this pans out
@@ -539,3 +560,35 @@ main(int argc, char* argv[])
return 0;
}
+
+int
+main(int argc, char* argv[])
+{
+ using namespace zen;
+
+ mi_version();
+
+ ZenServerOptions GlobalOptions;
+ ZenServiceConfig ServiceConfig;
+ ParseGlobalCliOptions(argc, argv, GlobalOptions, ServiceConfig);
+ InitializeLogging(GlobalOptions);
+
+#if ZEN_PLATFORM_WINDOWS
+ if (GlobalOptions.InstallService)
+ {
+ WindowsService::Install();
+
+ std::exit(0);
+ }
+
+ if (GlobalOptions.UninstallService)
+ {
+ WindowsService::Delete();
+
+ std::exit(0);
+ }
+#endif
+
+ ZenWindowsService App(GlobalOptions, ServiceConfig);
+ return App.ServiceMain();
+}
diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj
index aa9d538a5..db657d192 100644
--- a/zenserver/zenserver.vcxproj
+++ b/zenserver/zenserver.vcxproj
@@ -123,6 +123,7 @@
<ClInclude Include="upstream\upstreamcache.h" />
<ClInclude Include="upstream\zen.h" />
<ClInclude Include="vfs.h" />
+ <ClInclude Include="windows\service.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="cache\structuredcache.cpp" />
@@ -142,6 +143,7 @@
<ClCompile Include="upstream\upstreamcache.cpp" />
<ClCompile Include="upstream\zen.cpp" />
<ClCompile Include="vfs.cpp" />
+ <ClCompile Include="windows\service.cpp" />
<ClCompile Include="zenserver.cpp" />
</ItemGroup>
<ItemGroup>
diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters
index a86a6d96d..250c55812 100644
--- a/zenserver/zenserver.vcxproj.filters
+++ b/zenserver/zenserver.vcxproj.filters
@@ -39,6 +39,7 @@
<Filter>upstream</Filter>
</ClInclude>
<ClInclude Include="testing\httptest.h" />
+ <ClInclude Include="windows\service.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="zenserver.cpp" />
@@ -73,6 +74,7 @@
<Filter>upstream</Filter>
</ClCompile>
<ClCompile Include="testing\httptest.cpp" />
+ <ClCompile Include="windows\service.cpp" />
</ItemGroup>
<ItemGroup>
<Filter Include="cache">
diff --git a/zenstore-test/xmake.lua b/zenstore-test/xmake.lua
new file mode 100644
index 000000000..c8995dab2
--- /dev/null
+++ b/zenstore-test/xmake.lua
@@ -0,0 +1,5 @@
+target("zenstore-test")
+ set_kind("binary")
+ add_files("*.cpp")
+ add_deps("zenstore", "zencore")
+ add_packages("vcpkg::doctest")
diff --git a/zenstore-test/zenstore-test.cpp b/zenstore-test/zenstore-test.cpp
new file mode 100644
index 000000000..ed9a12566
--- /dev/null
+++ b/zenstore-test/zenstore-test.cpp
@@ -0,0 +1,23 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/logging.h>
+#include <zencore/zencore.h>
+#include <zenstore/zenstore.h>
+
+#define DOCTEST_CONFIG_IMPLEMENT
+#include <doctest/doctest.h>
+#undef DOCTEST_CONFIG_IMPLEMENT
+
+void
+forceLinkTests()
+{
+ zen::zenstore_forcelinktests();
+}
+
+int
+main(int argc, char* argv[])
+{
+ zen::logging::InitializeLogging();
+
+ return doctest::Context(argc, argv).run();
+}
diff --git a/zenstore-test/zenstore-test.vcxproj b/zenstore-test/zenstore-test.vcxproj
new file mode 100644
index 000000000..201594a25
--- /dev/null
+++ b/zenstore-test/zenstore-test.vcxproj
@@ -0,0 +1,121 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <VCProjectVersion>15.0</VCProjectVersion>
+ <ProjectGuid>{C001A3DF-B76E-4989-B576-FE2B78AB2580}</ProjectGuid>
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>zenstoretest</RootNamespace>
+ <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <PlatformToolset>v142</PlatformToolset>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <PlatformToolset>v142</PlatformToolset>
+ <WholeProgramOptimization>false</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="Shared">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ <Import Project="..\zenfs_common.props" />
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ <Import Project="..\zenfs_common.props" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <LinkIncremental>true</LinkIncremental>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <LinkIncremental>false</LinkIncremental>
+ </PropertyGroup>
+ <PropertyGroup Label="Vcpkg" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <VcpkgEnableManifest>true</VcpkgEnableManifest>
+ <VcpkgUseStatic>true</VcpkgUseStatic>
+ <VcpkgAdditionalInstallOptions>--overlay-ports=$(SolutionDir)vcpkg_overlay-ports</VcpkgAdditionalInstallOptions>
+ </PropertyGroup>
+ <PropertyGroup Label="Vcpkg" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <VcpkgEnableManifest>true</VcpkgEnableManifest>
+ <VcpkgUseStatic>true</VcpkgUseStatic>
+ <VcpkgAdditionalInstallOptions>--overlay-ports=$(SolutionDir)vcpkg_overlay-ports</VcpkgAdditionalInstallOptions>
+ </PropertyGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ <ClCompile>
+ <PrecompiledHeader>NotUsing</PrecompiledHeader>
+ <Optimization>Disabled</Optimization>
+ <SDLCheck>true</SDLCheck>
+ <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <ConformanceMode>true</ConformanceMode>
+ <LanguageStandard>stdcpplatest</LanguageStandard>
+ <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <AdditionalDependencies>kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+ </Link>
+ <ProjectReference />
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ <ClCompile>
+ <PrecompiledHeader>NotUsing</PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <SDLCheck>true</SDLCheck>
+ <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <ConformanceMode>true</ConformanceMode>
+ <WholeProgramOptimization>false</WholeProgramOptimization>
+ <LanguageStandard>stdcpplatest</LanguageStandard>
+ <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <ShowProgress>NotSet</ShowProgress>
+ </Link>
+ <ProjectReference />
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClInclude Include="targetver.h" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="zenstore-test.cpp" />
+ </ItemGroup>
+ <ItemGroup>
+ <ProjectReference Include="..\zencore\zencore.vcxproj">
+ <Project>{d75bf9ab-c61e-4fff-ad59-1563430f05e2}</Project>
+ </ProjectReference>
+ <ProjectReference Include="..\zenstore\zenstore.vcxproj">
+ <Project>{26cbbaeb-14c1-4efc-877d-80f48215651c}</Project>
+ </ProjectReference>
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project> \ No newline at end of file
diff --git a/zenstore-test/zenstore-test.vcxproj.filters b/zenstore-test/zenstore-test.vcxproj.filters
new file mode 100644
index 000000000..000599c79
--- /dev/null
+++ b/zenstore-test/zenstore-test.vcxproj.filters
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup>
+ <ClInclude Include="targetver.h" />
+ </ItemGroup>
+ <ItemGroup>
+ <ClCompile Include="zenstore-test.cpp" />
+ </ItemGroup>
+</Project> \ No newline at end of file
diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp
index e77c0ed64..a143230d3 100644
--- a/zenstore/CAS.cpp
+++ b/zenstore/CAS.cpp
@@ -11,6 +11,7 @@
#include <zencore/logging.h>
#include <zencore/memory.h>
#include <zencore/string.h>
+#include <zencore/testutils.h>
#include <zencore/thread.h>
#include <zencore/uid.h>
@@ -20,15 +21,23 @@
#include <functional>
#include <unordered_map>
-struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive-
-#include <atlfile.h>
-
//////////////////////////////////////////////////////////////////////////
namespace zen {
+void
+ScrubContext::ReportBadChunks(std::span<IoHash> BadChunks)
+{
+ ZEN_UNUSED(BadChunks);
+}
+
/**
- * Slightly less naive CAS store
+ * CAS store implementation
+ *
+ * Uses a basic strategy of splitting payloads by size, to improve ability to reclaim space
+ * quickly for unused large chunks and to maintain locality for small chunks which are
+ * frequently accessed together.
+ *
*/
class CasImpl : public CasStore
{
@@ -41,16 +50,15 @@ public:
virtual IoBuffer FindChunk(const IoHash& ChunkHash) override;
virtual void FilterChunks(CasChunkSet& InOutChunks) override;
virtual void Flush() override;
+ virtual void Scrub(ScrubContext& Ctx) override;
private:
- void PickDefaultDirectory();
-
CasContainerStrategy m_TinyStrategy;
CasContainerStrategy m_SmallStrategy;
FileCasStrategy m_LargeStrategy;
};
-CasImpl::CasImpl() : m_TinyStrategy(m_Config, m_Stats), m_SmallStrategy(m_Config, m_Stats), m_LargeStrategy(m_Config, m_Stats)
+CasImpl::CasImpl() : m_TinyStrategy(m_Config), m_SmallStrategy(m_Config), m_LargeStrategy(m_Config)
{
}
@@ -63,13 +71,16 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig)
{
m_Config = InConfig;
- ZEN_INFO("initializing CAS pool at {}", m_Config.RootDirectory);
+ ZEN_INFO("initializing CAS pool at '{}'", m_Config.RootDirectory);
// Ensure root directory exists - create if it doesn't exist already
std::filesystem::create_directories(m_Config.RootDirectory);
// Open or create manifest
+ //
+ // The manifest is not currently fully implemented. The goal is to
+ // use it for recovery and configuration
bool IsNewStore = false;
@@ -77,23 +88,22 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig)
std::filesystem::path ManifestPath = m_Config.RootDirectory;
ManifestPath /= ".ucas_root";
- CAtlFile marker;
- HRESULT hRes = marker.Create(ManifestPath.c_str(), GENERIC_READ, 0, OPEN_EXISTING);
+ std::error_code Ec;
+ BasicFile Marker;
+ Marker.Open(ManifestPath.c_str(), /* IsCreate */ false, Ec);
- if (FAILED(hRes))
+ if (Ec)
{
IsNewStore = true;
ExtendableStringBuilder<128> manifest;
- manifest.Append("#CAS_ROOT\n"); // TODO: should write something meaningful here
+ manifest.Append("#CAS_ROOT\n");
manifest.Append("ID=");
zen::Oid id = zen::Oid::NewOid();
id.ToString(manifest);
- hRes = marker.Create(ManifestPath.c_str(), GENERIC_WRITE, 0, CREATE_ALWAYS);
-
- if (SUCCEEDED(hRes))
- marker.Write(manifest.c_str(), (DWORD)manifest.Size());
+ Marker.Open(ManifestPath.c_str(), /* IsCreate */ true);
+ Marker.Write(manifest.c_str(), (DWORD)manifest.Size(), 0);
}
}
@@ -101,6 +111,9 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig)
m_TinyStrategy.Initialize("tobs", 16, IsNewStore);
m_SmallStrategy.Initialize("sobs", 4096, IsNewStore);
+
+ ScrubContext Ctx;
+ Scrub(Ctx);
}
CasStore::InsertResult
@@ -160,6 +173,14 @@ CasImpl::Flush()
m_LargeStrategy.Flush();
}
+void
+CasImpl::Scrub(ScrubContext& Ctx)
+{
+ m_SmallStrategy.Scrub(Ctx);
+ m_TinyStrategy.Scrub(Ctx);
+ m_LargeStrategy.Scrub(Ctx);
+}
+
//////////////////////////////////////////////////////////////////////////
CasStore*
@@ -173,18 +194,47 @@ CreateCasStore()
// Testing related code follows...
//
-void
-CAS_forcelink()
-{
-}
-
TEST_CASE("CasStore")
{
+ ScopedTemporaryDirectory TempDir;
+
zen::CasStoreConfiguration config;
- config.RootDirectory = "c:\\temp\\test";
+ config.RootDirectory = TempDir.Path();
+
+ std::unique_ptr<zen::CasStore> Store{CreateCasStore()};
+ Store->Initialize(config);
+
+ ScrubContext Ctx;
+ Store->Scrub(Ctx);
+
+ IoBuffer Value1{16};
+ memcpy(Value1.MutableData(), "1234567890123456", 16);
+ IoHash Hash1 = IoHash::HashBuffer(Value1.Data(), Value1.Size());
+ CasStore::InsertResult Result1 = Store->InsertChunk(Value1, Hash1);
+ CHECK(Result1.New);
+
+ IoBuffer Value2{16};
+ memcpy(Value2.MutableData(), "ABCDEFGHIJKLMNOP", 16);
+ IoHash Hash2 = IoHash::HashBuffer(Value2.Data(), Value2.Size());
+ CasStore::InsertResult Result2 = Store->InsertChunk(Value2, Hash2);
+ CHECK(Result2.New);
- std::unique_ptr<zen::CasStore> store{CreateCasStore()};
- store->Initialize(config);
+ CasChunkSet ChunkSet;
+ ChunkSet.AddChunk(Hash1);
+ ChunkSet.AddChunk(Hash2);
+
+ Store->FilterChunks(ChunkSet);
+ CHECK(ChunkSet.GetChunkSet().size() == 0);
+
+ IoBuffer Lookup1 = Store->FindChunk(Hash1);
+ CHECK(Lookup1);
+ IoBuffer Lookup2 = Store->FindChunk(Hash2);
+ CHECK(Lookup2);
+}
+
+void
+CAS_forcelink()
+{
}
} // namespace zen
diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp
index 35ccdd042..0b92a8979 100644
--- a/zenstore/basicfile.cpp
+++ b/zenstore/basicfile.cpp
@@ -5,7 +5,9 @@
#include <zencore/except.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
+#include <zencore/testutils.h>
+#include <doctest/doctest.h>
#include <fmt/format.h>
#include <gsl/gsl-lite.hpp>
@@ -13,16 +15,54 @@ namespace zen {
using namespace fmt::literals;
+BasicFile::~BasicFile()
+{
+ Close();
+}
+
void
-BasicFile::Open(std::filesystem::path FileName, bool isCreate)
+BasicFile::Open(std::filesystem::path FileName, bool IsCreate)
{
- const DWORD dwCreationDisposition = isCreate ? CREATE_ALWAYS : OPEN_EXISTING;
+ std::error_code Ec;
+ Open(FileName, IsCreate, Ec);
- HRESULT hRes = m_File.Create(FileName.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, dwCreationDisposition);
+ if (Ec)
+ {
+ throw std::system_error(Ec, "failed to open file '{}'"_format(FileName));
+ }
+}
- if (FAILED(hRes))
+void
+BasicFile::Open(std::filesystem::path FileName, bool IsCreate, std::error_code& Ec)
+{
+ const DWORD dwCreationDisposition = IsCreate ? CREATE_ALWAYS : OPEN_EXISTING;
+ const DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE;
+ const DWORD dwShareMode = FILE_SHARE_READ;
+ const DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL;
+ HANDLE hTemplateFile = nullptr;
+
+ HANDLE FileHandle = CreateFile(FileName.c_str(),
+ dwDesiredAccess,
+ dwShareMode,
+ /* lpSecurityAttributes */ nullptr,
+ dwCreationDisposition,
+ dwFlagsAndAttributes,
+ hTemplateFile);
+
+ if (FileHandle == INVALID_HANDLE_VALUE)
{
- ThrowSystemException(hRes, "Failed to open bucket sobs file '{}'"_format(FileName));
+ Ec = zen::MakeErrorCodeFromLastError();
+ }
+
+ m_FileHandle = FileHandle;
+}
+
+void
+BasicFile::Close()
+{
+ if (m_FileHandle)
+ {
+ ::CloseHandle(m_FileHandle);
}
}
@@ -34,11 +74,14 @@ BasicFile::Read(void* Data, uint64_t Size, uint64_t Offset)
Ovl.Offset = DWORD(Offset & 0xffff'ffffu);
Ovl.OffsetHigh = DWORD(Offset >> 32);
- HRESULT hRes = m_File.Read(Data, gsl::narrow<DWORD>(Size), &Ovl);
+ DWORD dwNumberOfBytesToRead = gsl::narrow<DWORD>(Size);
+ DWORD dwNumberOfBytesRead = 0;
+
+ BOOL Success = ::ReadFile(m_FileHandle, Data, dwNumberOfBytesToRead, &dwNumberOfBytesRead, &Ovl);
- if (FAILED(hRes))
+ if (!Success)
{
- ThrowSystemException(hRes, "Failed to read from file '{}'"_format(zen::PathFromHandle(m_File)));
+ ThrowLastError("Failed to read from file '{}'"_format(zen::PathFromHandle(m_FileHandle)));
}
}
@@ -53,6 +96,35 @@ BasicFile::ReadAll()
}
void
+BasicFile::StreamFile(std::function<void(const void* Data, uint64_t Size)>&& ChunkFun)
+{
+ StreamByteRange(0, FileSize(), std::move(ChunkFun));
+}
+
+void
+BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun)
+{
+ const uint64_t ChunkSize = 128 * 1024;
+ IoBuffer ReadBuffer{ChunkSize};
+ void* BufferPtr = ReadBuffer.MutableData();
+
+ uint64_t RemainBytes = Size;
+ uint64_t CurrentOffset = FileOffset;
+
+ while (RemainBytes)
+ {
+ const uint64_t ThisChunkBytes = zen::Min(ChunkSize, RemainBytes);
+
+ Read(BufferPtr, ThisChunkBytes, CurrentOffset);
+
+ ChunkFun(BufferPtr, ThisChunkBytes);
+
+ CurrentOffset += ThisChunkBytes;
+ RemainBytes -= ThisChunkBytes;
+ }
+}
+
+void
BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset)
{
OVERLAPPED Ovl{};
@@ -60,33 +132,46 @@ BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset)
Ovl.Offset = DWORD(Offset & 0xffff'ffffu);
Ovl.OffsetHigh = DWORD(Offset >> 32);
- HRESULT hRes = m_File.Write(Data, gsl::narrow<DWORD>(Size), &Ovl);
+ DWORD dwNumberOfBytesToWrite = gsl::narrow<DWORD>(Size);
+ DWORD dwNumberOfBytesWritten = 0;
- if (FAILED(hRes))
+ BOOL Success = ::WriteFile(m_FileHandle, Data, dwNumberOfBytesToWrite, &dwNumberOfBytesWritten, &Ovl);
+
+ if (!Success)
{
- ThrowSystemException(hRes, "Failed to write to file '{}'"_format(zen::PathFromHandle(m_File)));
+ ThrowLastError("Failed to write to file '{}'"_format(zen::PathFromHandle(m_FileHandle)));
}
}
void
BasicFile::Flush()
{
- m_File.Flush();
+ FlushFileBuffers(m_FileHandle);
}
uint64_t
BasicFile::FileSize()
{
- ULONGLONG Sz;
- m_File.GetSize(Sz);
+ ULARGE_INTEGER liFileSize;
+ liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart);
- return uint64_t(Sz);
+ return uint64_t(liFileSize.QuadPart);
+}
+
+TEST_CASE("BasicFile")
+{
+ ScopedCurrentDirectoryChange _;
+
+ BasicFile File1;
+ CHECK_THROWS(File1.Open("zonk", false));
+ CHECK_NOTHROW(File1.Open("zonk", true));
+ CHECK_NOTHROW(File1.Write("abcd", 4, 0));
+ CHECK(File1.FileSize() == 4);
}
void
-BasicFile::Close()
+basicfile_forcelink()
{
- m_File.Close();
}
} // namespace zen
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index 4407d8b08..070ca1503 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -10,18 +10,22 @@
#include <zencore/thread.h>
#include <zencore/uid.h>
-#include <gsl/gsl-lite.hpp>
-
-#include <functional>
-
-struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive-
-#include <atlfile.h>
#include <filesystem>
+#include <functional>
+#include <gsl/gsl-lite.hpp>
//////////////////////////////////////////////////////////////////////////
namespace zen {
+CasContainerStrategy::CasContainerStrategy(const CasStoreConfiguration& Config) : m_Config(Config)
+{
+}
+
+CasContainerStrategy::~CasContainerStrategy()
+{
+}
+
void
CasContainerStrategy::Initialize(const std::string_view ContainerBaseName, uint64_t Alignment, bool IsNewStore)
{
@@ -43,7 +47,9 @@ CasContainerStrategy::Initialize(const std::string_view ContainerBaseName, uint6
uint64_t MaxFileOffset = 0;
{
- // This is not technically necessary but may help future static analysis
+ // This is not technically necessary (nobody should be accessing us from
+ // another thread at this stage) but may help static analysis
+
zen::RwLock::ExclusiveLockScope _(m_LocationMapLock);
m_CasLog.Replay([&](const CasDiskIndexEntry& Record) {
@@ -103,9 +109,8 @@ IoBuffer
CasContainerStrategy::FindChunk(const IoHash& ChunkHash)
{
RwLock::SharedLockScope _(m_LocationMapLock);
- auto KeyIt = m_LocationMap.find(ChunkHash);
- if (KeyIt != m_LocationMap.end())
+ if (auto KeyIt = m_LocationMap.find(ChunkHash); KeyIt != m_LocationMap.end())
{
const CasDiskLocation& Location = KeyIt->second;
return zen::IoBufferBuilder::MakeFromFileHandle(m_SmallObjectFile.Handle(), Location.Offset, Location.Size);
@@ -120,9 +125,8 @@ bool
CasContainerStrategy::HaveChunk(const IoHash& ChunkHash)
{
RwLock::SharedLockScope _(m_LocationMapLock);
- auto KeyIt = m_LocationMap.find(ChunkHash);
- if (KeyIt != m_LocationMap.end())
+ if (auto KeyIt = m_LocationMap.find(ChunkHash); KeyIt != m_LocationMap.end())
{
return true;
}
@@ -133,6 +137,13 @@ CasContainerStrategy::HaveChunk(const IoHash& ChunkHash)
void
CasContainerStrategy::FilterChunks(CasChunkSet& InOutChunks)
{
+ // This implementation is good enough for relatively small
+ // chunk sets (in terms of chunk identifiers), but would
+ // benefit from a better implementation which removes
+ // items incrementally for large sets, especially when
+ // we're likely to already have a large proportion of the
+ // chunks in the set
+
std::unordered_set<IoHash> HaveSet;
for (const IoHash& Hash : InOutChunks.GetChunkSet())
@@ -157,4 +168,113 @@ CasContainerStrategy::Flush()
m_SmallObjectFile.Flush();
}
+void
+CasContainerStrategy::Scrub(ScrubContext& Ctx)
+{
+ const uint64_t WindowSize = 4 * 1024 * 1024;
+ uint64_t WindowStart = 0;
+ uint64_t WindowEnd = WindowSize;
+ const uint64_t FileSize = m_SmallObjectFile.FileSize();
+
+ std::vector<CasDiskIndexEntry> BigChunks;
+ std::vector<CasDiskIndexEntry> BadChunks;
+
+ // We do a read sweep through the payloads file and validate
+ // any entries that are contained within each segment, with
+ // the assumption that most entries will be checked in this
+ // pass. An alternative strategy would be to use memory mapping.
+
+ {
+ IoBuffer ReadBuffer{WindowSize};
+ void* BufferBase = ReadBuffer.MutableData();
+
+ RwLock::SharedLockScope _(m_LocationMapLock);
+
+ do
+ {
+ const uint64_t ChunkSize = zen::Min(WindowSize, FileSize - WindowStart);
+ m_SmallObjectFile.Read(BufferBase, ChunkSize, WindowStart);
+
+ for (auto& Entry : m_LocationMap)
+ {
+ const uint64_t EntryOffset = Entry.second.Offset;
+
+ if ((EntryOffset >= WindowStart) && (EntryOffset < WindowEnd))
+ {
+ const uint64_t EntryEnd = EntryOffset + Entry.second.Size;
+
+ if (EntryEnd >= WindowEnd)
+ {
+ BigChunks.push_back({.Key = Entry.first, .Location = Entry.second});
+
+ continue;
+ }
+
+ const IoHash ComputedHash = IoHash::HashBuffer(BufferBase, Entry.second.Size);
+
+ if (Entry.first != ComputedHash)
+ {
+ // Hash mismatch
+
+ BadChunks.push_back({.Key = Entry.first, .Location = Entry.second});
+ }
+ }
+ }
+
+ WindowStart += WindowSize;
+ WindowEnd += WindowSize;
+ } while (WindowStart < FileSize);
+ }
+
+ // Deal with large chunks
+
+ for (const CasDiskIndexEntry& Entry : BigChunks)
+ {
+ IoHashStream Hasher;
+ m_SmallObjectFile.StreamByteRange(Entry.Location.Offset, Entry.Location.Size, [&](const void* Data, uint64_t Size) {
+ Hasher.Append(Data, Size);
+ });
+ IoHash ComputedHash = Hasher.GetHash();
+
+ if (Entry.Key != ComputedHash)
+ {
+ BadChunks.push_back(Entry);
+ }
+ }
+
+ // Deal with bad chunks by removing them from our lookup map
+
+ std::vector<IoHash> BadChunkHashes;
+
+ for (const CasDiskIndexEntry& Entry : BadChunks)
+ {
+ BadChunkHashes.push_back(Entry.Key);
+ m_LocationMap.erase(Entry.Key);
+ }
+
+ // Let whomever it concerns know about the bad chunks. This could
+ // be used to invalidate higher level data structures more efficiently
+ // than a full validation pass might be able to do
+
+ Ctx.ReportBadChunks(BadChunkHashes);
+}
+
+void
+CasContainerStrategy::MakeSnapshot()
+{
+ RwLock::SharedLockScope _(m_LocationMapLock);
+
+ std::vector<CasDiskIndexEntry> Entries{m_LocationMap.size()};
+
+ uint64_t EntryIndex = 0;
+ for (auto& Entry : m_LocationMap)
+ {
+ CasDiskIndexEntry& IndexEntry = Entries[EntryIndex++];
+ IndexEntry.Key = Entry.first;
+ IndexEntry.Location = Entry.second;
+ }
+
+ m_SmallObjectIndex.Write(Entries.data(), Entries.size() * sizeof(CasDiskIndexEntry), 0);
+}
+
} // namespace zen
diff --git a/zenstore/compactcas.h b/zenstore/compactcas.h
index 05bbf81f6..101e6b1b7 100644
--- a/zenstore/compactcas.h
+++ b/zenstore/compactcas.h
@@ -14,9 +14,6 @@
#include <zenstore/cas.h>
#include <zenstore/caslog.h>
-#include <atlfile.h>
-#include <functional>
-
namespace zen {
//////////////////////////////////////////////////////////////////////////
@@ -27,7 +24,10 @@ namespace zen {
struct CasDiskLocation
{
uint64_t Offset;
- uint32_t Size; // TODO: Make this more like the IoStore index so we can store larger chunks (should be five bytes)
+ // If we wanted to be able to store larger chunks using this storage mechanism then
+ // we could make this more like the IoStore index so we can store larger chunks.
+ // I.e use five bytes for size and seven for offset
+ uint32_t Size;
};
struct CasDiskIndexEntry
@@ -50,7 +50,9 @@ static_assert(sizeof(CasDiskIndexEntry) == 32);
struct CasContainerStrategy
{
- CasContainerStrategy(const CasStoreConfiguration& Config, CasStore::Stats& Stats) : m_Config(Config), m_Stats(Stats) {}
+ CasContainerStrategy(const CasStoreConfiguration& Config);
+ ~CasContainerStrategy();
+
CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash);
CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& chunkHash);
IoBuffer FindChunk(const IoHash& ChunkHash);
@@ -58,10 +60,10 @@ struct CasContainerStrategy
void FilterChunks(CasChunkSet& InOutChunks);
void Initialize(const std::string_view ContainerBaseName, uint64_t Alignment, bool IsNewStore);
void Flush();
+ void Scrub(ScrubContext& Ctx);
private:
const CasStoreConfiguration& m_Config;
- CasStore::Stats& m_Stats;
uint64_t m_PayloadAlignment = 1 << 4;
bool m_IsInitialized = false;
BasicFile m_SmallObjectFile;
@@ -73,6 +75,8 @@ private:
RwLock m_InsertLock; // used to serialize inserts
std::atomic<uint64_t> m_CurrentInsertOffset = 0;
std::atomic<uint64_t> m_CurrentIndexOffset = 0;
+
+ void MakeSnapshot();
};
} // namespace zen
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 170f13875..31991a43e 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -10,6 +10,7 @@
#include <zencore/string.h>
#include <zencore/thread.h>
#include <zencore/uid.h>
+#include <zenstore/basicfile.h>
#include <gsl/gsl-lite.hpp>
@@ -17,6 +18,7 @@
#include <functional>
#include <unordered_map>
+// clang-format off
#include <zencore/prewindows.h>
struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax error: 'identifier' was unexpected here" when using /permissive-
@@ -24,13 +26,19 @@ struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax erro
#include <zencore/postwindows.h>
// clang-format on
-//
-//////////////////////////////////////////////////////////////////////////
namespace zen {
using namespace fmt::literals;
+FileCasStrategy::FileCasStrategy(const CasStoreConfiguration& Config) : m_Config(Config)
+{
+}
+
+FileCasStrategy::~FileCasStrategy()
+{
+}
+
WideStringBuilderBase&
FileCasStrategy::MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHash& ChunkHash, size_t& OutShard2len)
{
@@ -56,7 +64,7 @@ FileCasStrategy::MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHas
OutShard2len = ShardedPath.Size();
ShardedPath.Append('\\');
- ShardedPath.AppendAsciiRange(str + 6, str + 64);
+ ShardedPath.AppendAsciiRange(str + 5, str + 64);
return ShardedPath;
}
@@ -259,12 +267,9 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize
}
// We cannot rely on RAII to close the file handle since it would be closed
- // *after* the lock is released due to the initialization order.
+ // *after* the lock is released due to the initialization order
PayloadFile.Close();
- AtomicIncrement(m_Stats.PutCount);
- AtomicAdd(m_Stats.PutBytes, ChunkSize);
-
return {.New = true};
}
@@ -279,15 +284,7 @@ FileCasStrategy::FindChunk(const IoHash& ChunkHash)
RwLock::SharedLockScope _(LockForHash(ChunkHash));
- auto Chunk = IoBufferBuilder::MakeFromFile(ShardedPath.c_str());
-
- if (Chunk)
- {
- AtomicIncrement(m_Stats.GetCount);
- AtomicAdd(m_Stats.GetBytes, Chunk.Size());
- }
-
- return Chunk;
+ return IoBufferBuilder::MakeFromFile(ShardedPath.c_str());
}
bool
@@ -338,6 +335,62 @@ FileCasStrategy::FilterChunks(CasChunkSet& InOutChunks)
}
void
+FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, BasicFile& PayloadFile)>&& Callback)
+{
+ struct Visitor : public FileSystemTraversal::TreeVisitor
+ {
+ Visitor(const std::filesystem::path& RootDir) : RootDirectory(RootDir) {}
+ virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& File, uint64_t FileSize) override
+ {
+ ZEN_UNUSED(FileSize);
+
+ std::filesystem::path RelPath = std::filesystem::relative(Parent, RootDirectory);
+
+ std::wstring PathString = RelPath.native();
+
+ if ((PathString.size() == (3 + 2 + 1)) && (File.size() == (40 - 3 - 2)))
+ {
+ if (PathString.at(3) == std::filesystem::path::preferred_separator)
+ {
+ PathString.erase(3, 1);
+ }
+ PathString.append(File);
+
+ StringBuilder<64> Utf8;
+ WideToUtf8(PathString, Utf8);
+
+ // TODO: should validate that we're actually dealing with a valid hex string here
+
+ IoHash NameHash = IoHash::FromHexString({Utf8.Data(), Utf8.Size()});
+
+ BasicFile PayloadFile;
+ std::error_code Ec;
+ PayloadFile.Open(Parent / File, false, Ec);
+
+ if (!Ec)
+ {
+ Callback(NameHash, PayloadFile);
+ }
+ }
+ }
+
+ virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent,
+ [[maybe_unused]] const std::wstring_view& DirectoryName)
+ {
+ return true;
+ }
+
+ const std::filesystem::path& RootDirectory;
+ std::function<void(const IoHash& Hash, BasicFile& PayloadFile)> Callback;
+ } CasVisitor{m_Config.RootDirectory};
+
+ CasVisitor.Callback = std::move(Callback);
+
+ FileSystemTraversal Traversal;
+ Traversal.TraverseFileSystem(m_Config.RootDirectory, CasVisitor);
+}
+
+void
FileCasStrategy::Flush()
{
// Since we don't keep files open after writing there's nothing specific
@@ -353,6 +406,25 @@ FileCasStrategy::Flush()
}
void
+FileCasStrategy::Scrub(ScrubContext& Ctx)
+{
+ std::vector<IoHash> BadHashes;
+
+ IterateChunks([&](const IoHash& Hash, BasicFile& Payload) {
+ IoHashStream Hasher;
+ Payload.StreamFile([&](const void* Data, size_t Size) { Hasher.Append(Data, Size); });
+ IoHash ComputedHash = Hasher.GetHash();
+
+ if (ComputedHash != Hash)
+ {
+ BadHashes.push_back(Hash);
+ }
+ });
+
+ Ctx.ReportBadChunks(BadHashes);
+}
+
+void
FileCasStrategy::GarbageCollect(GcContext& GcCtx)
{
ZEN_UNUSED(GcCtx);
diff --git a/zenstore/filecas.h b/zenstore/filecas.h
index 448d1a05f..885973810 100644
--- a/zenstore/filecas.h
+++ b/zenstore/filecas.h
@@ -10,11 +10,20 @@
#include <zencore/thread.h>
#include <zenstore/cas.h>
+#include <functional>
+
namespace zen {
+class BasicFile;
+
+/** CAS storage strategy using a file-per-chunk storage strategy
+*/
+
struct FileCasStrategy
{
- FileCasStrategy(const CasStoreConfiguration& Config, CasStore::Stats& Stats) : m_Config(Config), m_Stats(Stats) {}
+ FileCasStrategy(const CasStoreConfiguration& Config);
+ ~FileCasStrategy();
+
CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash);
CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash);
IoBuffer FindChunk(const IoHash& ChunkHash);
@@ -22,15 +31,16 @@ struct FileCasStrategy
void FilterChunks(CasChunkSet& InOutChunks);
void Flush();
void GarbageCollect(GcContext& GcCtx);
+ void Scrub(ScrubContext& Ctx);
private:
const CasStoreConfiguration& m_Config;
- CasStore::Stats& m_Stats;
RwLock m_Lock;
RwLock m_ShardLocks[256]; // TODO: these should be spaced out so they don't share cache lines
inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardLocks[Hash.Hash[19]]; }
static WideStringBuilderBase& MakeShardedPath(WideStringBuilderBase& ShardedPath, const IoHash& ChunkHash, size_t& OutShard2len);
+ void IterateChunks(std::function<void(const IoHash& Hash, BasicFile& PayloadFile)>&& Callback);
};
} // namespace zen
diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/CAS.h
index b4de533dd..bb310b179 100644
--- a/zenstore/include/zenstore/CAS.h
+++ b/zenstore/include/zenstore/CAS.h
@@ -2,7 +2,7 @@
#pragma once
-#include <zencore/zencore.h>
+#include "zenstore.h"
#include <zencore/blake3.h>
#include <zencore/iobuffer.h>
@@ -37,6 +37,14 @@ public:
private:
};
+class ScrubContext
+{
+public:
+ virtual void ReportBadChunks(std::span<IoHash> BadChunks);
+
+private:
+};
+
class CasChunkSet
{
public:
@@ -54,17 +62,7 @@ class CasStore
public:
virtual ~CasStore() = default;
- struct Stats
- {
- uint64_t PutBytes = 0;
- uint64_t PutCount = 0;
-
- uint64_t GetBytes = 0;
- uint64_t GetCount = 0;
- };
-
const CasStoreConfiguration& Config() { return m_Config; }
- const Stats& GetStats() const { return m_Stats; }
struct InsertResult
{
@@ -76,10 +74,10 @@ public:
virtual IoBuffer FindChunk(const IoHash& ChunkHash) = 0;
virtual void FilterChunks(CasChunkSet& InOutChunks) = 0;
virtual void Flush() = 0;
+ virtual void Scrub(ScrubContext& Ctx) = 0;
protected:
CasStoreConfiguration m_Config;
- Stats m_Stats;
};
ZENCORE_API CasStore* CreateCasStore();
diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h
index c6f61d466..d4d65b366 100644
--- a/zenstore/include/zenstore/basicfile.h
+++ b/zenstore/include/zenstore/basicfile.h
@@ -2,34 +2,46 @@
#pragma once
-#include <zencore/iobuffer.h>
-#include <zencore/zencore.h>
+#include "zenstore.h"
+#include <zencore/iobuffer.h>
#include <zencore/windows.h>
-#include <atlfile.h>
#include <filesystem>
+#include <functional>
namespace zen {
/**
* Probably the most basic file abstraction in the universe
+ *
+ * One thing of note is that there is no notion of a "current file position"
+ * in this API -- all reads and writes are done from explicit offsets in
+ * the file. This avoids concurrency issues which can occur otherwise.
+ *
*/
class BasicFile
{
public:
+ BasicFile() = default;
+ ~BasicFile();
void Open(std::filesystem::path FileName, bool IsCreate);
- void Read(void* Data, uint64_t Size, uint64_t Offset);
- void Write(const void* Data, uint64_t Size, uint64_t Offset);
+ void Open(std::filesystem::path FileName, bool IsCreate, std::error_code& Ec);
+ void Close();
+ void Read(void* Data, uint64_t Size, uint64_t FileOffset);
+ void StreamFile(std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
+ void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
+ void Write(const void* Data, uint64_t Size, uint64_t FileOffset);
void Flush();
uint64_t FileSize();
- void* Handle() { return m_File; }
- void Close();
+ void* Handle() { return m_FileHandle; }
IoBuffer ReadAll();
private:
- CAtlFile m_File;
+ void* m_FileHandle = nullptr; // This is either null or valid
};
+ZENCORE_API void basicfile_forcelink();
+
} // namespace zen
diff --git a/zenstore/include/zenstore/caslog.h b/zenstore/include/zenstore/caslog.h
index aea855e4c..3d558bee0 100644
--- a/zenstore/include/zenstore/caslog.h
+++ b/zenstore/include/zenstore/caslog.h
@@ -2,7 +2,7 @@
#pragma once
-#include <zencore/zencore.h>
+#include "zenstore.h"
#include <zencore/iobuffer.h>
#include <zencore/string.h>
diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h
index 76a33c915..f023ada40 100644
--- a/zenstore/include/zenstore/cidstore.h
+++ b/zenstore/include/zenstore/cidstore.h
@@ -2,6 +2,8 @@
#pragma once
+#include "zenstore.h"
+
#include <tsl/robin_map.h>
#include <zencore/iohash.h>
#include <zenstore/CAS.h>
diff --git a/zenstore/include/zenstore/scrub.h b/zenstore/include/zenstore/scrub.h
index 5a34d4860..4948afcd5 100644
--- a/zenstore/include/zenstore/scrub.h
+++ b/zenstore/include/zenstore/scrub.h
@@ -2,6 +2,8 @@
#pragma once
+#include "zenstore.h"
+
#include <zencore/iohash.h>
#include <span>
diff --git a/zenstore/include/zenstore/zenstore.h b/zenstore/include/zenstore/zenstore.h
new file mode 100644
index 000000000..46d62029d
--- /dev/null
+++ b/zenstore/include/zenstore/zenstore.h
@@ -0,0 +1,13 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#define ZENSTORE_API
+
+namespace zen {
+
+ZENSTORE_API void zenstore_forcelinktests();
+
+}
diff --git a/zenstore/zenstore.cpp b/zenstore/zenstore.cpp
new file mode 100644
index 000000000..cd16e5634
--- /dev/null
+++ b/zenstore/zenstore.cpp
@@ -0,0 +1,17 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zenstore/zenstore.h"
+
+#include <zenstore/CAS.h>
+#include <zenstore/basicfile.h>
+
+namespace zen {
+
+void
+zenstore_forcelinktests()
+{
+ basicfile_forcelink();
+ CAS_forcelink();
+}
+
+} // namespace zen
diff --git a/zenstore/zenstore.vcxproj b/zenstore/zenstore.vcxproj
index 8d665f2c3..eb2ecd02b 100644
--- a/zenstore/zenstore.vcxproj
+++ b/zenstore/zenstore.vcxproj
@@ -19,6 +19,7 @@
<ClCompile Include="filecas.cpp" />
<ClCompile Include="gc.cpp" />
<ClCompile Include="scrub.cpp" />
+ <ClCompile Include="zenstore.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="compactcas.h" />
@@ -29,12 +30,16 @@
<ClInclude Include="include\zenstore\scrub.h" />
<ClInclude Include="include\zenstore\CAS.h" />
<ClInclude Include="include\zenstore\caslog.h" />
+ <ClInclude Include="include\zenstore\zenstore.h" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\zencore\zencore.vcxproj">
<Project>{d75bf9ab-c61e-4fff-ad59-1563430f05e2}</Project>
</ProjectReference>
</ItemGroup>
+ <ItemGroup>
+ <None Include="xmake.lua" />
+ </ItemGroup>
<PropertyGroup Label="Globals">
<VCProjectVersion>16.0</VCProjectVersion>
<Keyword>Win32Proj</Keyword>
diff --git a/zenstore/zenstore.vcxproj.filters b/zenstore/zenstore.vcxproj.filters
index 3dfb89dbf..8a52c69f6 100644
--- a/zenstore/zenstore.vcxproj.filters
+++ b/zenstore/zenstore.vcxproj.filters
@@ -9,6 +9,7 @@
<ClCompile Include="scrub.cpp" />
<ClCompile Include="basicfile.cpp" />
<ClCompile Include="cidstore.cpp" />
+ <ClCompile Include="zenstore.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="compactcas.h" />
@@ -19,5 +20,9 @@
<ClInclude Include="include\zenstore\scrub.h" />
<ClInclude Include="include\zenstore\basicfile.h" />
<ClInclude Include="include\zenstore\cidstore.h" />
+ <ClInclude Include="include\zenstore\zenstore.h" />
+ </ItemGroup>
+ <ItemGroup>
+ <None Include="xmake.lua" />
</ItemGroup>
</Project> \ No newline at end of file
diff --git a/zenutil/include/zenutil/zenserverprocess.h b/zenutil/include/zenutil/zenserverprocess.h
index f0924f048..979db349a 100644
--- a/zenutil/include/zenutil/zenserverprocess.h
+++ b/zenutil/include/zenutil/zenserverprocess.h
@@ -123,12 +123,14 @@ public:
ZenServerEntry* Register(int ListenPort);
void Sweep();
void Snapshot(std::function<void(const ZenServerEntry&)>&& Callback);
+ inline bool IsReadOnly() const { return m_IsReadOnly; }
private:
void* m_hMapFile = nullptr;
ZenServerEntry* m_Data = nullptr;
int m_MaxEntryCount = 131072 / sizeof(ZenServerEntry);
ZenServerEntry* m_OurEntry = nullptr;
+ bool m_IsReadOnly = true;
};
-}
+} // namespace zen
diff --git a/zenutil/zenserverprocess.cpp b/zenutil/zenserverprocess.cpp
index c504ce7da..9e370ae10 100644
--- a/zenutil/zenserverprocess.cpp
+++ b/zenutil/zenserverprocess.cpp
@@ -40,20 +40,16 @@ namespace zenutil {
m_Attributes.nLength = sizeof m_Attributes;
m_Attributes.bInheritHandle = false; // Disable inheritance
- const BOOL success = InitializeSecurityDescriptor(&m_Sd, SECURITY_DESCRIPTOR_REVISION);
+ const BOOL Success = InitializeSecurityDescriptor(&m_Sd, SECURITY_DESCRIPTOR_REVISION);
- if (success)
+ if (Success)
{
- const BOOL bSetOk = SetSecurityDescriptorDacl(&m_Sd, TRUE, (PACL)NULL, FALSE);
-
- if (bSetOk)
- {
- m_Attributes.lpSecurityDescriptor = &m_Sd;
- }
- else
+ if (!SetSecurityDescriptorDacl(&m_Sd, TRUE, (PACL)NULL, FALSE))
{
zen::ThrowLastError("SetSecurityDescriptorDacl failed", std::source_location::current());
}
+
+ m_Attributes.lpSecurityDescriptor = &m_Sd;
}
}
};
@@ -129,7 +125,8 @@ ZenServerState::Initialize()
zen::ThrowLastError("Could not map view of Zen server state");
}
- m_Data = reinterpret_cast<ZenServerEntry*>(pBuf);
+ m_Data = reinterpret_cast<ZenServerEntry*>(pBuf);
+ m_IsReadOnly = false;
}
bool
@@ -221,6 +218,8 @@ ZenServerState::Sweep()
return;
}
+ ZEN_ASSERT(m_IsReadOnly == false);
+
for (int i = 0; i < m_MaxEntryCount; ++i)
{
ZenServerEntry& Entry = m_Data[i];