aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPer Larsson <[email protected]>2021-09-28 15:09:15 +0200
committerPer Larsson <[email protected]>2021-09-28 15:09:15 +0200
commit141317786f9d59e95da8316ce40cf30e4dfd7b53 (patch)
tree3c863384c6ca68a30e82989994408c5f40159273
parentRemoved using the bucket name to detect binary cache records and store conten... (diff)
parentapply: Re-enabled environment variable setup for child processes (diff)
downloadzen-141317786f9d59e95da8316ce40cf30e4dfd7b53.tar.xz
zen-141317786f9d59e95da8316ce40cf30e4dfd7b53.zip
Merge branch 'main' of https://github.com/EpicGames/zen
-rw-r--r--vs-chromium-project.txt1
-rw-r--r--zen/cmds/run.cpp2
-rw-r--r--zencore/compactbinaryvalidation.cpp2
-rw-r--r--zencore/except.cpp6
-rw-r--r--zencore/filesystem.cpp287
-rw-r--r--zencore/include/zencore/compactbinaryvalue.h2
-rw-r--r--zencore/include/zencore/endian.h4
-rw-r--r--zencore/include/zencore/except.h6
-rw-r--r--zencore/include/zencore/filesystem.h10
-rw-r--r--zencore/include/zencore/intmath.h8
-rw-r--r--zencore/include/zencore/iobuffer.h8
-rw-r--r--zencore/include/zencore/logging.h60
-rw-r--r--zencore/include/zencore/mpscqueue.h2
-rw-r--r--zencore/include/zencore/zencore.h1
-rw-r--r--zencore/intmath.cpp5
-rw-r--r--zencore/iobuffer.cpp152
-rw-r--r--zencore/testutils.cpp2
-rw-r--r--zencore/thread.cpp18
-rw-r--r--zencore/zencore.cpp36
-rw-r--r--zenhttp/httpsys.cpp60
-rw-r--r--zenhttp/httpsys.h1
-rw-r--r--zenserver/admin/admin.cpp44
-rw-r--r--zenserver/admin/admin.h10
-rw-r--r--zenserver/compute/apply.cpp4
-rw-r--r--zenserver/config.cpp4
-rw-r--r--zenserver/config.h9
-rw-r--r--zenserver/testing/httptest.cpp11
-rw-r--r--zenserver/testing/httptest.h5
-rw-r--r--zenserver/windows/service.cpp33
-rw-r--r--zenserver/zenserver.cpp4
-rw-r--r--zenserver/zenserver.vcxproj1
-rw-r--r--zenstore/cidstore.cpp2
-rw-r--r--zenstore/include/zenstore/cidstore.h4
33 files changed, 633 insertions, 171 deletions
diff --git a/vs-chromium-project.txt b/vs-chromium-project.txt
index 2bb89a55c..0a1075662 100644
--- a/vs-chromium-project.txt
+++ b/vs-chromium-project.txt
@@ -3,6 +3,7 @@
[SourceExplorer.ignore]
.git/
+vcpkg_installed/
x64/
*.suo
**/x64/
diff --git a/zen/cmds/run.cpp b/zen/cmds/run.cpp
index ceeb4ddae..4ffbf820c 100644
--- a/zen/cmds/run.cpp
+++ b/zen/cmds/run.cpp
@@ -82,7 +82,7 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
Visitor(const std::filesystem::path& RootPath) : m_RootPath(RootPath) {}
- virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& FileName, uint64_t FileSize) override
+ virtual void VisitFile(const std::filesystem::path& Parent, const path_view& FileName, uint64_t FileSize) override
{
std::filesystem::path FullPath = Parent / FileName;
diff --git a/zencore/compactbinaryvalidation.cpp b/zencore/compactbinaryvalidation.cpp
index 3d72148f9..a787e88ab 100644
--- a/zencore/compactbinaryvalidation.cpp
+++ b/zencore/compactbinaryvalidation.cpp
@@ -17,7 +17,7 @@ namespace CbValidationPrivate {
template<typename T>
static constexpr inline T ReadUnaligned(const void* const Memory)
{
-#if PLATFORM_SUPPORTS_UNALIGNED_LOADS
+#if ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS
return *static_cast<const T*>(Memory);
#else
T Value;
diff --git a/zencore/except.cpp b/zencore/except.cpp
index 44b8edffb..ebaecf815 100644
--- a/zencore/except.cpp
+++ b/zencore/except.cpp
@@ -72,13 +72,13 @@ ThrowSystemError(uint32_t ErrorCode, std::string_view Message)
std::string
GetLastErrorAsString()
{
- return GetWindowsErrorAsString(zen::GetLastError());
+ return GetSystemErrorAsString(zen::GetLastError());
}
std::string
-GetWindowsErrorAsString(uint32_t Win32ErrorCode)
+GetSystemErrorAsString(uint32_t ErrorCode)
{
- return std::error_code(Win32ErrorCode, std::system_category()).message();
+ return std::error_code(ErrorCode, std::system_category()).message();
}
#if __cpp_lib_source_location
diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp
index f6e410ee2..d1b8b7aeb 100644
--- a/zencore/filesystem.cpp
+++ b/zencore/filesystem.cpp
@@ -7,14 +7,25 @@
#include <zencore/iobuffer.h>
#include <zencore/logging.h>
#include <zencore/string.h>
-#include <zencore/windows.h>
+#include <zencore/testing.h>
-#include <atlbase.h>
-#include <atlfile.h>
-#include <winioctl.h>
-#include <winnt.h>
-#include <filesystem>
+#if ZEN_PLATFORM_WINDOWS
+# include <zencore/windows.h>
+#endif
+
+#if ZEN_PLATFORM_WINDOWS
+# include <atlbase.h>
+# include <atlfile.h>
+# include <winioctl.h>
+# include <winnt.h>
+#else
+# include <dirent.h>
+# include <fcntl.h>
+# include <sys/stat.h>
+# include <unistd.h>
+#endif
+#include <filesystem>
#include <gsl/gsl-lite.hpp>
namespace zen {
@@ -22,6 +33,7 @@ namespace zen {
using namespace std::literals;
#if ZEN_PLATFORM_WINDOWS
+
static bool
DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag)
{
@@ -48,6 +60,12 @@ DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag)
return false;
}
+bool
+CreateDirectories(const wchar_t* Dir)
+{
+ return std::filesystem::create_directories(Dir);
+}
+
// Erase all files and directories in a given directory, leaving an empty directory
// behind
@@ -126,12 +144,11 @@ WipeDirectory(const wchar_t* DirPath)
return true;
}
-#endif
bool
-CreateDirectories(const std::filesystem::path& Dir)
+DeleteDirectories(const wchar_t* DirPath)
{
- return std::filesystem::create_directories(Dir);
+ return WipeDirectory(DirPath) && RemoveDirectoryW(DirPath) == TRUE;
}
bool
@@ -145,16 +162,46 @@ CleanDirectory(const wchar_t* DirPath)
return CreateDirectories(DirPath);
}
+#endif // ZEN_PLATFORM_WINDOWS
+
+bool
+CreateDirectories(const std::filesystem::path& Dir)
+{
+ return std::filesystem::create_directories(Dir);
+}
+
bool
DeleteDirectories(const std::filesystem::path& Dir)
{
- return WipeDirectory(Dir.c_str()) && RemoveDirectoryW(Dir.c_str()) == TRUE;
+#if ZEN_PLATFORM_WINDOWS
+ return DeleteDirectories(Dir.c_str());
+#else
+ std::error_code ErrorCode;
+ return std::filesystem::remove_all(Dir, ErrorCode);
+#endif
}
bool
CleanDirectory(const std::filesystem::path& Dir)
{
+#if ZEN_PLATFORM_WINDOWS
return CleanDirectory(Dir.c_str());
+#else
+ if (std::filesystem::exists(Dir))
+ {
+ bool Success = true;
+
+ std::error_code ErrorCode;
+ for (const auto& Item : std::filesystem::directory_iterator(Dir))
+ {
+ Success &= std::filesystem::remove_all(Item, ErrorCode);
+ }
+
+ return Success;
+ }
+
+ return CreateDirectories(Dir);
+#endif
}
//////////////////////////////////////////////////////////////////////////
@@ -191,12 +238,13 @@ SupportsBlockRefCounting(std::filesystem::path Path)
return true;
#else
return false;
-#endif
+#endif // ZEN_PLATFORM_WINDOWS
}
bool
CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
{
+#if ZEN_PLATFORM_WINDOWS
ATL::CHandle FromFile(CreateFileW(FromPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
if (FromFile == INVALID_HANDLE_VALUE)
{
@@ -351,11 +399,16 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
const bool AllOk = (TRUE == SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition));
return AllOk;
+#else
+ ZEN_ERROR("CloneFile() is not implemented on this platform");
+ return false;
+#endif // ZEN_PLATFORM_WINDOWS
}
bool
CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options)
{
+#if ZEN_PLATFORM_WINDOWS
bool Success = false;
if (Options.EnableClone)
@@ -387,6 +440,10 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
}
return Success;
+#else
+ ZEN_ERROR("CopyFile() is not implemented on this platform");
+ return false;
+#endif // ZEN_PLATFORM_WINDOWS
}
void
@@ -394,6 +451,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer
{
using namespace fmt::literals;
+#if ZEN_PLATFORM_WINDOWS
CAtlFile Outfile;
HRESULT hRes = Outfile.Create(Path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS);
if (hRes == HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND))
@@ -408,6 +466,20 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer
ThrowSystemException(hRes, "File open failed for '{}'"_format(Path).c_str());
}
+#else
+ int Fd = open(Path.c_str(), O_WRONLY);
+ if (Fd < 0)
+ {
+ zen::CreateDirectories(Path.parent_path());
+ Fd = open(Path.c_str(), O_WRONLY);
+ }
+
+ if (Fd < 0)
+ {
+ ThrowLastError("File open failed for '{}'"_format(Path));
+ }
+#endif
+
// TODO: this should be block-enlightened
for (size_t i = 0; i < BufferCount; ++i)
@@ -419,17 +491,27 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer
{
const uint64_t ChunkSize = Min<uint64_t>(WriteSize, uint64_t(2) * 1024 * 1024 * 1024);
+#if ZEN_PLATFORM_WINDOWS
hRes = Outfile.Write(DataPtr, gsl::narrow_cast<uint32_t>(WriteSize));
-
if (FAILED(hRes))
{
ThrowSystemException(hRes, "File write failed for '{}'"_format(Path).c_str());
}
+#else
+ if (write(Fd, DataPtr, WriteSize) != WriteSize)
+ {
+ ThrowLastError("File write failed for '{}'"_format(Path));
+ }
+#endif // ZEN_PLATFORM_WINDOWS
WriteSize -= ChunkSize;
DataPtr = reinterpret_cast<const uint8_t*>(DataPtr) + ChunkSize;
}
}
+
+#if !ZEN_PLATFORM_WINDOWS
+ close(Fd);
+#endif
}
void
@@ -443,6 +525,10 @@ WriteFile(std::filesystem::path Path, IoBuffer Data)
FileContents
ReadFile(std::filesystem::path Path)
{
+ uint64_t FileSizeBytes;
+ void* Handle;
+
+#if ZEN_PLATFORM_WINDOWS
ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
if (FromFile == INVALID_HANDLE_VALUE)
{
@@ -456,18 +542,32 @@ ReadFile(std::filesystem::path Path)
return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())};
}
- const uint64_t FileSizeBytes = FileSize.EndOfFile.QuadPart;
+ FileSizeBytes = FileSize.EndOfFile.QuadPart;
+ Handle = FromFile.Detach();
+#else
+ int Fd = open(Path.c_str(), O_RDONLY);
+ if (Fd < 0)
+ {
+ return FileContents{.ErrorCode = std::error_code(zen::GetLastError(), std::system_category())};
+ }
- FileContents Contents;
+ static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
+ struct stat Stat;
+ fstat(Fd, &Stat);
- Contents.Data.emplace_back(IoBuffer(IoBuffer::File, FromFile.Detach(), 0, FileSizeBytes));
+ FileSizeBytes = Stat.st_size;
+ Handle = (void*)uintptr_t(Fd);
+#endif
+ FileContents Contents;
+ Contents.Data.emplace_back(IoBuffer(IoBuffer::File, Handle, 0, FileSizeBytes));
return Contents;
}
bool
ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc)
{
+#if ZEN_PLATFORM_WINDOWS
ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
if (FromFile == INVALID_HANDLE_VALUE)
{
@@ -494,30 +594,34 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<voi
}
return true;
+#else
+ ZEN_ERROR("ScanFile() is not implemented on this platform");
+ return false;
+#endif // ZEN_PLATFORM_WINDOWS
}
std::string
ToUtf8(const std::filesystem::path& Path)
{
+#if ZEN_PLATFORM_WINDOWS
return WideToUtf8(Path.native().c_str());
+#else
+ return Path.string();
+#endif
}
void
FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor)
{
+#if ZEN_PLATFORM_WINDOWS
uint64_t FileInfoBuffer[8 * 1024];
FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo;
bool Continue = true;
- std::wstring RootDirPath = RootDir.native();
-
CAtlFile RootDirHandle;
- HRESULT hRes = RootDirHandle.Create(RootDirPath.c_str(),
- GENERIC_READ,
- FILE_SHARE_READ | FILE_SHARE_WRITE,
- OPEN_EXISTING,
- FILE_FLAG_BACKUP_SEMANTICS);
+ HRESULT hRes =
+ RootDirHandle.Create(RootDir.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS);
if (FAILED(hRes))
{
@@ -573,12 +677,12 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
}
else if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DEVICE)
{
- ZEN_WARN("encountered device node during file system traversal: {} found in {}", WideToUtf8(FileName), RootDir);
+ ZEN_WARN("encountered device node during file system traversal: {} found in {}",
+ WideToUtf8(FileName),
+ WideToUtf8(RootDir.c_str()));
}
else
{
- std::filesystem::path FullPath = RootDir / FileName;
-
Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart);
}
@@ -592,11 +696,57 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr
EntryOffset += NextOffset;
}
}
+#else
+ using namespace fmt::literals;
+
+ /* Could also implement this using Linux's getdents() syscall */
+
+ DIR* Dir = opendir(RootDir.c_str());
+ if (Dir == nullptr)
+ {
+ ThrowLastError("Failed to open directory for traversal: {}"_format(RootDir.c_str()));
+ }
+
+ for (struct dirent* Entry; Entry = readdir(Dir);)
+ {
+ const char* FileName = Entry->d_name;
+
+ struct stat Stat;
+ std::filesystem::path FullPath = RootDir / FileName;
+ stat(FullPath.c_str(), &Stat);
+
+ if (S_ISDIR(Stat.st_mode))
+ {
+ if (strcmp(FileName, ".") == 0 || strcmp(FileName, "..") == 0)
+ {
+ /* nop */
+ }
+ else if (Visitor.VisitDirectory(RootDir, FileName))
+ {
+ TraverseFileSystem(FullPath, Visitor);
+ }
+ }
+ else if (S_ISREG(Stat.st_mode))
+ {
+ Visitor.VisitFile(RootDir, FileName, Stat.st_size);
+ }
+ else
+ {
+ ZEN_WARN("encountered non-regular file during file system traversal ({}): {} found in {}",
+ Stat.st_mode,
+ FileName,
+ RootDir.c_str());
+ }
+ }
+
+ closedir(Dir);
+#endif // ZEN_PLATFORM_WINDOWS
}
std::filesystem::path
PathFromHandle(void* NativeHandle)
{
+#if ZEN_PLATFORM_WINDOWS
if (NativeHandle == nullptr || NativeHandle == INVALID_HANDLE_VALUE)
{
return std::filesystem::path();
@@ -612,15 +762,102 @@ PathFromHandle(void* NativeHandle)
ZEN_UNUSED(FinalLength);
return FullPath;
+#elif ZEN_PLATFORM_LINUX
+ char Buffer[256];
+ sprintf(Buffer, "/proc/%d/fd/%d", getpid(), int(uintptr_t(NativeHandle)));
+ ssize_t BytesRead = readlink(Buffer, Buffer, sizeof(Buffer) - 1);
+ if (BytesRead <= 0)
+ return std::filesystem::path();
+
+ Buffer[BytesRead] = '\0';
+ return Buffer;
+#else
+# error Unimplemented platform
+#endif // ZEN_PLATFORM_WINDOWS
}
std::filesystem::path
GetRunningExecutablePath()
{
+#if ZEN_PLATFORM_WINDOWS
TCHAR ExePath[MAX_PATH];
DWORD PathLength = GetModuleFileName(NULL, ExePath, ZEN_ARRAY_COUNT(ExePath));
return {std::wstring_view(ExePath, PathLength)};
+#elif ZEN_PLATFORM_LINUX
+ char Buffer[256];
+ sprintf(Buffer, "/proc/%d/exe", getpid());
+ ssize_t BytesRead = readlink(Buffer, Buffer, sizeof(Buffer) - 1);
+ if (BytesRead < 0)
+ return {};
+
+ Buffer[BytesRead] = '\0';
+ return Buffer;
+#else
+# error Unimplemented platform
+#endif // ZEN_PLATFORM_WINDOWS
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Testing related code follows...
+//
+
+#if ZEN_WITH_TESTS
+
+void
+filesystem_forcelink()
+{
}
+TEST_CASE("filesystem")
+{
+ using namespace std::filesystem;
+
+ // GetExePath
+ path BinPath = GetRunningExecutablePath();
+ CHECK(BinPath.stem() == "zencore-test");
+ CHECK(is_regular_file(BinPath));
+
+ // PathFromHandle
+ void* Handle;
+# if ZEN_PLATFORM_WINDOWS
+ Handle = CreateFileW(BinPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr);
+ CHECK(Handle != INVALID_HANDLE_VALUE);
+# else
+ int Fd = open(BinPath.c_str(), O_RDONLY);
+ CHECK(Fd >= 0);
+ Handle = (void*)uintptr_t(Fd);
+# endif
+
+ auto FromHandle = PathFromHandle((void*)uintptr_t(Handle));
+ CHECK(equivalent(FromHandle, BinPath));
+
+# if ZEN_PLATFORM_WINDOWS
+ CloseHandle(Handle);
+# else
+ close(int(uintptr_t(Handle)));
+# endif
+
+ // Traversal
+ struct : public FileSystemTraversal::TreeVisitor
+ {
+ virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t) override
+ {
+ bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected);
+ }
+
+ virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
+
+ bool bFoundExpected = false;
+ std::filesystem::path Expected;
+ } Visitor;
+ Visitor.Expected = BinPath;
+
+ FileSystemTraversal().TraverseFileSystem(BinPath.parent_path().parent_path(), Visitor);
+ CHECK(Visitor.bFoundExpected);
+}
+
+#endif
+
} // namespace zen
diff --git a/zencore/include/zencore/compactbinaryvalue.h b/zencore/include/zencore/compactbinaryvalue.h
index 5795ef957..0124a8983 100644
--- a/zencore/include/zencore/compactbinaryvalue.h
+++ b/zencore/include/zencore/compactbinaryvalue.h
@@ -15,7 +15,7 @@ namespace CompactBinaryPrivate {
template<typename T>
static constexpr inline T ReadUnaligned(const void* const Memory)
{
-#if PLATFORM_SUPPORTS_UNALIGNED_LOADS
+#if ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS
return *static_cast<const T*>(Memory);
#else
T Value;
diff --git a/zencore/include/zencore/endian.h b/zencore/include/zencore/endian.h
index d44a27b01..7a9e6b44c 100644
--- a/zencore/include/zencore/endian.h
+++ b/zencore/include/zencore/endian.h
@@ -2,6 +2,10 @@
#pragma once
+#include "zencore.h"
+
+#include <cstdint>
+
namespace zen {
inline uint16_t
diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h
index 5cfefb1e2..e0e4aaae0 100644
--- a/zencore/include/zencore/except.h
+++ b/zencore/include/zencore/except.h
@@ -27,7 +27,7 @@ ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message, const st
ZENCORE_API void ThrowSystemError [[noreturn]] (uint32_t ErrorCode, std::string_view Message);
ZENCORE_API std::string GetLastErrorAsString();
-ZENCORE_API std::string GetWindowsErrorAsString(uint32_t Win32ErrorCode);
+ZENCORE_API std::string GetSystemErrorAsString(uint32_t Win32ErrorCode);
inline int32_t
GetLastError()
@@ -40,9 +40,9 @@ GetLastError()
}
inline std::error_code
-MakeWin32ErrorCode(uint32_t Win32ErrorCode) noexcept
+MakeErrorCode(uint32_t ErrorCode) noexcept
{
- return std::error_code(Win32ErrorCode, std::system_category());
+ return std::error_code(ErrorCode, std::system_category());
}
inline std::error_code
diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h
index 16d6ede53..6678528f6 100644
--- a/zencore/include/zencore/filesystem.h
+++ b/zencore/include/zencore/filesystem.h
@@ -66,13 +66,19 @@ class FileSystemTraversal
public:
struct TreeVisitor
{
- virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& File, uint64_t FileSize) = 0;
+ using path_view = std::basic_string_view<std::filesystem::path::value_type>;
+
+ virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) = 0;
// This should return true if we should recurse into the directory
- virtual bool VisitDirectory(const std::filesystem::path& Parent, const std::wstring_view& DirectoryName) = 0;
+ virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName) = 0;
};
void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor);
};
+//////////////////////////////////////////////////////////////////////////
+
+void filesystem_forcelink(); // internal
+
} // namespace zen
diff --git a/zencore/include/zencore/intmath.h b/zencore/include/zencore/intmath.h
index b127c8993..7619e1950 100644
--- a/zencore/include/zencore/intmath.h
+++ b/zencore/include/zencore/intmath.h
@@ -21,7 +21,7 @@ _BitScanReverse(unsigned long* Index, uint32_t Mask)
return 0;
}
- *Index = __builtin_clz(Mask);
+ *Index = 31 - __builtin_clz(Mask);
return 1;
}
@@ -33,8 +33,8 @@ _BitScanReverse64(unsigned long* Index, uint64_t Mask)
return 0;
}
- *Index = __builtin_clzll(Mask);
- return 0;
+ *Index = 63 - __builtin_clzll(Mask);
+ return 1;
}
inline uint8_t
@@ -46,7 +46,7 @@ _BitScanForward64(unsigned long* Index, uint64_t Mask)
}
*Index = __builtin_ctzll(Mask);
- return 0;
+ return 1;
}
#endif
diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h
index 21751cd40..54801f9ac 100644
--- a/zencore/include/zencore/iobuffer.h
+++ b/zencore/include/zencore/iobuffer.h
@@ -7,6 +7,8 @@
#include "refcount.h"
#include "zencore.h"
+#include <filesystem>
+
namespace zen {
struct IoHash;
@@ -371,9 +373,11 @@ private:
class IoBufferBuilder
{
+ using path_char_t = std::filesystem::path::value_type;
+
public:
- ZENCORE_API static IoBuffer MakeFromFile(const wchar_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull);
- ZENCORE_API static IoBuffer MakeFromTemporaryFile(const wchar_t* FileName);
+ ZENCORE_API static IoBuffer MakeFromFile(const path_char_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull);
+ ZENCORE_API static IoBuffer MakeFromTemporaryFile(const path_char_t* FileName);
ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull);
inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); }
};
diff --git a/zencore/include/zencore/logging.h b/zencore/include/zencore/logging.h
index e98509bf8..0838cfe80 100644
--- a/zencore/include/zencore/logging.h
+++ b/zencore/include/zencore/logging.h
@@ -39,46 +39,46 @@ using zen::Log;
// Helper macros for logging
-#define ZEN_TRACE(fmtstr, ...) \
- do \
- { \
- using namespace std::literals; \
- Log().trace(fmtstr##sv, __VA_ARGS__); \
+#define ZEN_TRACE(fmtstr, ...) \
+ do \
+ { \
+ using namespace std::literals; \
+ Log().trace(fmtstr##sv, ##__VA_ARGS__); \
} while (false)
-#define ZEN_DEBUG(fmtstr, ...) \
- do \
- { \
- using namespace std::literals; \
- Log().debug(fmtstr##sv, __VA_ARGS__); \
+#define ZEN_DEBUG(fmtstr, ...) \
+ do \
+ { \
+ using namespace std::literals; \
+ Log().debug(fmtstr##sv, ##__VA_ARGS__); \
} while (false)
-#define ZEN_INFO(fmtstr, ...) \
- do \
- { \
- using namespace std::literals; \
- Log().info(fmtstr##sv, __VA_ARGS__); \
+#define ZEN_INFO(fmtstr, ...) \
+ do \
+ { \
+ using namespace std::literals; \
+ Log().info(fmtstr##sv, ##__VA_ARGS__); \
} while (false)
-#define ZEN_WARN(fmtstr, ...) \
- do \
- { \
- using namespace std::literals; \
- Log().warn(fmtstr##sv, __VA_ARGS__); \
+#define ZEN_WARN(fmtstr, ...) \
+ do \
+ { \
+ using namespace std::literals; \
+ Log().warn(fmtstr##sv, ##__VA_ARGS__); \
} while (false)
-#define ZEN_ERROR(fmtstr, ...) \
- do \
- { \
- using namespace std::literals; \
- Log().error(fmtstr##sv, __VA_ARGS__); \
+#define ZEN_ERROR(fmtstr, ...) \
+ do \
+ { \
+ using namespace std::literals; \
+ Log().error(fmtstr##sv, ##__VA_ARGS__); \
} while (false)
-#define ZEN_CRITICAL(fmtstr, ...) \
- do \
- { \
- using namespace std::literals; \
- Log().critical(fmtstr##sv, __VA_ARGS__); \
+#define ZEN_CRITICAL(fmtstr, ...) \
+ do \
+ { \
+ using namespace std::literals; \
+ Log().critical(fmtstr##sv, ##__VA_ARGS__); \
} while (false)
#define ZEN_CONSOLE(fmtstr, ...) \
diff --git a/zencore/include/zencore/mpscqueue.h b/zencore/include/zencore/mpscqueue.h
index bb558bb5a..e3359852a 100644
--- a/zencore/include/zencore/mpscqueue.h
+++ b/zencore/include/zencore/mpscqueue.h
@@ -24,7 +24,7 @@ struct TypeCompatibleStorage
ElementType* Data() { return (ElementType*)this; }
const ElementType* Data() const { return (const ElementType*)this; }
- char alignas(ElementType) DataMember;
+ alignas(ElementType) char DataMember;
};
/** Fast multi-producer/single-consumer unbounded concurrent queue.
diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h
index 4b9c1af1b..b5b47d076 100644
--- a/zencore/include/zencore/zencore.h
+++ b/zencore/include/zencore/zencore.h
@@ -198,6 +198,7 @@ ZENCORE_API bool IsPointerToStack(const void* ptr); // Query if pointer is with
ZENCORE_API bool IsApplicationExitRequested();
ZENCORE_API void RequestApplicationExit(int ExitCode);
ZENCORE_API bool IsDebuggerPresent();
+ZENCORE_API void SetIsInteractiveSession(bool Value);
ZENCORE_API bool IsInteractiveSession();
ZENCORE_API void zencore_forcelinktests();
diff --git a/zencore/intmath.cpp b/zencore/intmath.cpp
index f11f5e8aa..5a686dc8e 100644
--- a/zencore/intmath.cpp
+++ b/zencore/intmath.cpp
@@ -1,5 +1,6 @@
// Copyright Epic Games, Inc. All Rights Reserved.
+#include <zencore/endian.h>
#include <zencore/intmath.h>
#include <zencore/testing.h>
@@ -53,6 +54,10 @@ TEST_CASE("intmath")
CHECK(CountTrailingZeros64(0x0000'0000'0000'0001ull) == 0);
CHECK(CountTrailingZeros64(0x0000'0000'8000'0000ull) == 31);
CHECK(CountTrailingZeros64(0x0000'0001'0000'0000ull) == 32);
+
+ CHECK(ByteSwap(uint16_t(0x6d72)) == 0x726d);
+ CHECK(ByteSwap(uint32_t(0x2741'3965)) == 0x6539'4127);
+ CHECK(ByteSwap(uint64_t(0x214d'6172'7469'6e21ull)) == 0x216e'6974'7261'4d21ull);
}
#endif
diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp
index dc998d5ea..a730a316f 100644
--- a/zencore/iobuffer.cpp
+++ b/zencore/iobuffer.cpp
@@ -2,6 +2,7 @@
#include <zencore/iobuffer.h>
+#include <zencore/except.h>
#include <zencore/filesystem.h>
#include <zencore/fmtutils.h>
#include <zencore/iohash.h>
@@ -13,7 +14,13 @@
#include <memory.h>
#include <system_error>
-#include <atlfile.h>
+#if ZEN_PLATFORM_WINDOWS
+# include <atlfile.h>
+#else
+# include <sys/stat.h>
+# include <sys/mman.h>
+#endif
+
#include <gsl/gsl-lite.hpp>
namespace zen {
@@ -23,12 +30,14 @@ namespace zen {
void*
IoBufferCore::AllocateBuffer(size_t InSize, size_t Alignment)
{
+#if ZEN_PLATFORM_WINDOWS
if (((InSize & 0xffFF) == 0) && (Alignment == 0x10000))
{
m_Flags |= kLowLevelAlloc;
return VirtualAlloc(nullptr, InSize, MEM_COMMIT, PAGE_READWRITE);
}
else
+#endif // ZEN_PLATFORM_WINDOWS
{
return Memory::Alloc(InSize, Alignment);
}
@@ -37,11 +46,13 @@ IoBufferCore::AllocateBuffer(size_t InSize, size_t Alignment)
void
IoBufferCore::FreeBuffer()
{
+#if ZEN_PLATFORM_WINDOWS
if (m_Flags & kLowLevelAlloc)
{
VirtualFree(const_cast<void*>(m_DataPtr), 0, MEM_DECOMMIT);
}
else
+#endif // ZEN_PLATFORM_WINDOWS
{
return Memory::Free(const_cast<void*>(m_DataPtr));
}
@@ -153,17 +164,29 @@ IoBufferExtendedCore::~IoBufferExtendedCore()
{
if (m_MappedPointer)
{
+#if ZEN_PLATFORM_WINDOWS
UnmapViewOfFile(m_MappedPointer);
+#else
+ uint64_t MapSize = ~uint64_t(uintptr_t(m_MmapHandle));
+ munmap(m_MappedPointer, MapSize);
+#endif
}
+#if ZEN_PLATFORM_WINDOWS
if (m_Flags & kOwnsMmap)
{
CloseHandle(m_MmapHandle);
}
+#endif
if (m_Flags & kOwnsFile)
{
+#if ZEN_PLATFORM_WINDOWS
BOOL Success = CloseHandle(m_FileHandle);
+#else
+ int Fd = int(uintptr_t(m_FileHandle));
+ bool Success = (close(Fd) == 0);
+#endif
if (!Success)
{
@@ -174,7 +197,7 @@ IoBufferExtendedCore::~IoBufferExtendedCore()
m_DataPtr = nullptr;
}
-RwLock g_MappingLock;
+static RwLock g_MappingLock;
void
IoBufferExtendedCore::Materialize() const
@@ -193,6 +216,11 @@ IoBufferExtendedCore::Materialize() const
if (m_MmapHandle)
return;
+ const uint64_t MapOffset = m_FileOffset & ~0xffffull;
+ const uint64_t MappedOffsetDisplacement = m_FileOffset - MapOffset;
+ const uint64_t MapSize = m_DataBytes + MappedOffsetDisplacement;
+
+#if ZEN_PLATFORM_WINDOWS
m_MmapHandle = CreateFileMapping(m_FileHandle,
/* lpFileMappingAttributes */ nullptr,
/* flProtect */ PAGE_READONLY,
@@ -208,20 +236,28 @@ IoBufferExtendedCore::Materialize() const
m_Flags |= kOwnsMmap;
- const uint64_t MapOffset = m_FileOffset & ~0xffffull;
- const uint64_t MappedOffsetDisplacement = m_FileOffset - MapOffset;
- const uint64_t MapSize = m_DataBytes + MappedOffsetDisplacement;
-
void* MappedBase = MapViewOfFile(m_MmapHandle,
/* dwDesiredAccess */ FILE_MAP_READ,
/* FileOffsetHigh */ uint32_t(MapOffset >> 32),
/* FileOffsetLow */ uint32_t(MapOffset & 0xffFFffFFu),
/* dwNumberOfBytesToMap */ MapSize);
+#else
+ m_MmapHandle = (void*)uintptr_t(~MapSize); // ~ so it's never null (assuming MapSize >= 0)
+ m_Flags |= kOwnsMmap;
+
+ void* MappedBase = mmap(
+ /* addr */ nullptr,
+ /* length */ MapSize,
+ /* prot */ PROT_READ,
+ /* flags */ MAP_SHARED | MAP_NORESERVE,
+ /* fd */ int(uintptr_t(m_FileHandle)),
+ /* offset */ MapOffset);
+#endif // ZEN_PLATFORM_WINDOWS
if (MappedBase == nullptr)
{
throw std::system_error(
- std::error_code(::GetLastError(), std::system_category()),
+ std::error_code(zen::GetLastError(), std::system_category()),
"MapViewOfFile failed (offset {:#x}, size {:#x}) file: '{}'"_format(MapOffset, MapSize, zen::PathFromHandle(m_FileHandle)));
}
@@ -317,49 +353,72 @@ IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t
}
IoBuffer
-IoBufferBuilder::MakeFromFile(const wchar_t* FileName, uint64_t Offset, uint64_t Size)
+IoBufferBuilder::MakeFromFile(const path_char_t* FileName, uint64_t Offset, uint64_t Size)
{
+ uint64_t FileSize;
+
+#if ZEN_PLATFORM_WINDOWS
CAtlFile DataFile;
HRESULT hRes = DataFile.Create(FileName, GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING);
- if (SUCCEEDED(hRes))
+ if (FAILED(hRes))
{
- ULONGLONG FileSize;
- DataFile.GetSize(FileSize);
+ return {};
+ }
+
+ DataFile.GetSize((ULONGLONG&)FileSize);
+#else
+ int Fd = open(FileName, O_RDONLY);
+ if (Fd < 0)
+ {
+ return {};
+ }
+
+ static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
+ struct stat Stat;
+ fstat(Fd, &Stat);
+ FileSize = Stat.st_size;
+#endif // ZEN_PLATFORM_WINDOWS
- // TODO: should validate that offset is in range
+ // TODO: should validate that offset is in range
- if (Size == ~0ull)
+ if (Size == ~0ull)
+ {
+ Size = FileSize - Offset;
+ }
+ else
+ {
+ // Clamp size
+ if ((Offset + Size) > FileSize)
{
Size = FileSize - Offset;
}
- else
- {
- // Clamp size
- if ((Offset + Size) > FileSize)
- {
- Size = FileSize - Offset;
- }
- }
+ }
- if (Size)
- {
- return IoBuffer(IoBuffer::File, DataFile.Detach(), Offset, Size);
- }
- else
- {
- // For an empty file, we may as well just return an empty memory IoBuffer
- return IoBuffer(IoBuffer::Wrap, "", 0);
- }
+ if (Size)
+ {
+#if ZEN_PLATFORM_WINDOWS
+ void* Fd = DataFile.Detach();
+#endif
+ return IoBuffer(IoBuffer::File, (void*)uintptr_t(Fd), Offset, Size);
}
- return {};
+#if !ZEN_PLATFORM_WINDOWS
+ close(Fd);
+#endif
+
+ // For an empty file, we may as well just return an empty memory IoBuffer
+ return IoBuffer(IoBuffer::Wrap, "", 0);
}
IoBuffer
-IoBufferBuilder::MakeFromTemporaryFile(const wchar_t* FileName)
+IoBufferBuilder::MakeFromTemporaryFile(const path_char_t* FileName)
{
+ uint64_t FileSize;
+ void* Handle;
+
+#if ZEN_PLATFORM_WINDOWS
CAtlFile DataFile;
// We need to open with DELETE since this is used for the case
@@ -368,18 +427,33 @@ IoBufferBuilder::MakeFromTemporaryFile(const wchar_t* FileName)
HRESULT hRes = DataFile.Create(FileName, GENERIC_READ | DELETE, FILE_SHARE_READ | FILE_SHARE_DELETE, OPEN_EXISTING);
- if (SUCCEEDED(hRes))
+ if (FAILED(hRes))
{
- ULONGLONG FileSize;
- DataFile.GetSize(FileSize);
+ return {};
+ }
- IoBuffer Iob(IoBuffer::File, DataFile.Detach(), 0, FileSize);
- Iob.m_Core->SetIsWholeFile(true);
+ DataFile.GetSize((ULONGLONG&)FileSize);
- return Iob;
+ Handle = DataFile.Detach();
+#else
+ int Fd = open(FileName, O_RDONLY);
+ if (Fd < 0)
+ {
+ return {};
}
- return {};
+ static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
+ struct stat Stat;
+ fstat(Fd, &Stat);
+ FileSize = Stat.st_size;
+
+ Handle = (void*)uintptr_t(Fd);
+#endif // ZEN_PLATFORM_WINDOWS
+
+ IoBuffer Iob(IoBuffer::File, Handle, 0, FileSize);
+ Iob.m_Core->SetIsWholeFile(true);
+
+ return Iob;
}
IoHash
diff --git a/zencore/testutils.cpp b/zencore/testutils.cpp
index 116491950..78957cbcf 100644
--- a/zencore/testutils.cpp
+++ b/zencore/testutils.cpp
@@ -4,6 +4,8 @@
#include <zencore/session.h>
#include "zencore/string.h"
+#include <atomic>
+
namespace zen {
static std::atomic<int> Sequence{0};
diff --git a/zencore/thread.cpp b/zencore/thread.cpp
index d4f101454..20ab19f56 100644
--- a/zencore/thread.cpp
+++ b/zencore/thread.cpp
@@ -38,6 +38,10 @@ RwLock::ReleaseExclusive()
m_Mutex.unlock();
}
+//////////////////////////////////////////////////////////////////////////
+
+#if ZEN_PLATFORM_WINDOWS
+
Event::Event()
{
m_EventHandle = CreateEvent(nullptr, true, false, nullptr);
@@ -84,6 +88,8 @@ Event::Wait(int TimeoutMs)
return (Result == WAIT_OBJECT_0);
}
+//////////////////////////////////////////////////////////////////////////
+
NamedEvent::NamedEvent(std::u8string_view EventName) : Event(nullptr)
{
using namespace std::literals;
@@ -151,6 +157,12 @@ NamedMutex::Exists(std::string_view MutexName)
return true;
}
+#endif // ZEN_PLATFORM_WINDOWS
+
+#if ZEN_PLATFORM_WINDOWS
+
+//////////////////////////////////////////////////////////////////////////
+
ProcessHandle::ProcessHandle() = default;
void
@@ -248,8 +260,12 @@ ProcessHandle::Wait(int TimeoutMs)
return false;
}
+#endif // ZEN_PLATFORM_WINDOWS
+
//////////////////////////////////////////////////////////////////////////
+#if ZEN_PLATFORM_WINDOWS
+
ProcessMonitor::ProcessMonitor()
{
}
@@ -313,6 +329,8 @@ ProcessMonitor::IsActive() const
return m_ProcessHandles.empty() == false;
}
+#endif // ZEN_PLATFORM_WINDOWS
+
//////////////////////////////////////////////////////////////////////////
bool
diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp
index 1ea8ceb37..0b94f3bea 100644
--- a/zencore/zencore.cpp
+++ b/zencore/zencore.cpp
@@ -16,6 +16,8 @@
#include <zencore/compactbinarypackage.h>
#include <zencore/compositebuffer.h>
#include <zencore/compress.h>
+#include <zencore/filesystem.h>
+#include <zencore/intmath.h>
#include <zencore/iobuffer.h>
#include <zencore/memory.h>
#include <zencore/mpscqueue.h>
@@ -67,21 +69,36 @@ IsDebuggerPresent()
#endif
}
+std::optional<bool> InteractiveSessionFlag;
+
+void
+SetIsInteractiveSession(bool Value)
+{
+ InteractiveSessionFlag = Value;
+}
+
bool
IsInteractiveSession()
{
-#if ZEN_PLATFORM_WINDOWS
- DWORD dwSessionId = 0;
- if (ProcessIdToSessionId(GetCurrentProcessId(), &dwSessionId))
+ if (!InteractiveSessionFlag.has_value())
{
- return (dwSessionId != 0);
- }
-
- return false;
+#if ZEN_PLATFORM_WINDOWS
+ DWORD dwSessionId = 0;
+ if (ProcessIdToSessionId(GetCurrentProcessId(), &dwSessionId))
+ {
+ InteractiveSessionFlag = (dwSessionId != 0);
+ }
+ else
+ {
+ InteractiveSessionFlag = false;
+ }
#else
- // TODO: figure out what makes sense here
- return true;
+ // TODO: figure out what actually makes sense here
+ InteractiveSessionFlag = true;
#endif
+ }
+
+ return InteractiveSessionFlag.value();
}
//////////////////////////////////////////////////////////////////////////
@@ -109,6 +126,7 @@ zencore_forcelinktests()
zen::blake3_forcelink();
zen::compositebuffer_forcelink();
zen::compress_forcelink();
+ zen::filesystem_forcelink();
zen::intmath_forcelink();
zen::iobuffer_forcelink();
zen::memory_forcelink();
diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp
index 2a50388e3..bccff24ab 100644
--- a/zenhttp/httpsys.cpp
+++ b/zenhttp/httpsys.cpp
@@ -42,7 +42,7 @@ UTF8_to_wstring(const char* in)
if (((*in & 0xc0) != 0x80) && (codepoint <= 0x10ffff))
{
- if (sizeof(wchar_t) > 2)
+ if constexpr (sizeof(wchar_t) > 2)
{
out.append(1, static_cast<wchar_t>(codepoint));
}
@@ -417,7 +417,7 @@ HttpMessageResponseRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfB
if (IoResult != NO_ERROR)
{
- ZEN_WARN("response aborted due to error: '{}'", GetWindowsErrorAsString(IoResult));
+ ZEN_WARN("response aborted due to error: '{}'", GetSystemErrorAsString(IoResult));
// if one transmit failed there's really no need to go on
return nullptr;
@@ -465,7 +465,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode)
HttpSendHttpResponse function as well.
*/
- ULONG SendFlags = 0;
+ ULONG SendFlags = HTTP_SEND_RESPONSE_FLAG_BUFFER_DATA;
if (m_RemainingChunkCount)
{
@@ -484,6 +484,23 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode)
HttpResponse.EntityChunkCount = USHORT(ThisRequestChunkCount);
HttpResponse.pEntityChunks = m_HttpDataChunks.data() + ThisRequestChunkOffset;
+ // Server header
+ //
+ // By default this will also add a suffix " Microsoft-HTTPAPI/2.0" to this header
+ //
+ // This is controlled via a registry key 'DisableServerHeader', at:
+ //
+ // Computer\HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\HTTP\Parameters
+ //
+ // Set DisableServerHeader to 1 to disable suffix, or 2 to disable the header altogether
+ //
+ // (reference https://docs.microsoft.com/en-us/archive/blogs/dsnotes/wswcf-remove-server-header)
+ //
+
+ PHTTP_KNOWN_HEADER ServerHeader = &HttpResponse.Headers.KnownHeaders[HttpHeaderServer];
+ ServerHeader->pRawValue = "Zen";
+ ServerHeader->RawValueLength = (USHORT)3;
+
// Content-length header
char ContentLengthString[32];
@@ -545,24 +562,30 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode)
);
}
- if ((SendResult != NO_ERROR) // Synchronous completion, but the completion event will still be posted to IOCP
- && (SendResult != ERROR_IO_PENDING) // Asynchronous completion
- )
+ if (SendResult == NO_ERROR)
+ {
+ // Synchronous completion, but the completion event will still be posted to IOCP
+
+ ErrorCode.clear();
+ }
+ else if (SendResult == ERROR_IO_PENDING)
+ {
+ // Asynchronous completion, a completion notification will be posted to IOCP
+
+ ErrorCode.clear();
+ }
+ else
{
- // Some error occurred, no completion will be posted
+ // An error occurred, no completion will be posted to IOCP
CancelThreadpoolIo(Iocp);
ZEN_ERROR("failed to send HTTP response (error: '{}'), request URL: '{}', request id: {}",
- GetWindowsErrorAsString(SendResult),
+ GetSystemErrorAsString(SendResult),
HttpReq->pRawUrl,
HttpReq->RequestId);
- ErrorCode = MakeWin32ErrorCode(SendResult);
- }
- else
- {
- ErrorCode = {};
+ ErrorCode = MakeErrorCode(SendResult);
}
}
@@ -1244,21 +1267,14 @@ InitialRequestHandler::IssueRequest(std::error_code& ErrorCode)
{
CancelThreadpoolIo(Iocp);
- if (HttpApiResult == ERROR_MORE_DATA)
- {
- // ProcessReceiveAndPostResponse(pIoRequest, pServerContext->Io, ERROR_MORE_DATA);
- }
-
- // CleanupHttpIoRequest(pIoRequest);
-
- ErrorCode = MakeWin32ErrorCode(HttpApiResult);
+ ErrorCode = MakeErrorCode(HttpApiResult);
ZEN_ERROR("HttpReceiveHttpRequest failed, error {}", ErrorCode.message());
return;
}
- ErrorCode = std::error_code();
+ ErrorCode.clear();
}
HttpSysRequestHandler*
diff --git a/zenhttp/httpsys.h b/zenhttp/httpsys.h
index 2e51c538f..a8395b283 100644
--- a/zenhttp/httpsys.h
+++ b/zenhttp/httpsys.h
@@ -17,7 +17,6 @@
# include <zencore/windows.h>
# include "iothreadpool.h"
-# include <atlbase.h>
# include <http.h>
namespace spdlog {
diff --git a/zenserver/admin/admin.cpp b/zenserver/admin/admin.cpp
new file mode 100644
index 000000000..07211cbeb
--- /dev/null
+++ b/zenserver/admin/admin.cpp
@@ -0,0 +1,44 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "admin.h"
+
+#include <zencore/compactbinarybuilder.h>
+
+namespace zen {
+
+HttpAdminService::HttpAdminService()
+{
+ m_Router.RegisterRoute(
+ "hello",
+ [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK); },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
+ "health",
+ [this](HttpRouterRequest& Req) {
+ CbObjectWriter Obj;
+ Obj.AddBool("ok", true);
+ Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
+ },
+ HttpVerb::kGet);
+}
+
+HttpAdminService::~HttpAdminService()
+{
+}
+
+const char*
+HttpAdminService::BaseUri() const
+{
+ return "/admin/";
+}
+
+void
+HttpAdminService::HandleRequest(zen::HttpServerRequest& Request)
+{
+ m_Router.HandleRequest(Request);
+}
+
+} // namespace zen
diff --git a/zenserver/admin/admin.h b/zenserver/admin/admin.h
index 3554b1005..6257f0998 100644
--- a/zenserver/admin/admin.h
+++ b/zenserver/admin/admin.h
@@ -9,14 +9,14 @@ namespace zen {
class HttpAdminService : public zen::HttpService
{
public:
- HttpAdminService() = default;
- ~HttpAdminService() = default;
+ HttpAdminService();
+ ~HttpAdminService();
- virtual const char* BaseUri() const override { return "/admin/"; }
-
- virtual void HandleRequest(zen::HttpServerRequest& Request) override { ZEN_UNUSED(Request); }
+ virtual const char* BaseUri() const override;
+ virtual void HandleRequest(zen::HttpServerRequest& Request) override;
private:
+ HttpRequestRouter m_Router;
};
} // namespace zen
diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp
index 15d9e0141..a522aa35b 100644
--- a/zenserver/compute/apply.cpp
+++ b/zenserver/compute/apply.cpp
@@ -767,8 +767,8 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action)
lpThreadAttributes,
bInheritHandles,
dwCreationFlags,
- nullptr, // (LPVOID)EnvironmentBlock.c_str(), // Environment block
- SandboxPath.c_str(), // Current directory
+ (LPVOID)EnvironmentBlock.Data(), // Environment block
+ SandboxPath.c_str(), // Current directory
&StartupInfo,
/* out */ &ProcessInformation);
diff --git a/zenserver/config.cpp b/zenserver/config.cpp
index 91fb80747..42f59b26c 100644
--- a/zenserver/config.cpp
+++ b/zenserver/config.cpp
@@ -122,12 +122,14 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z
cxxopts::value<int>(GlobalOptions.BasePort)->default_value("1337"),
"<port number>");
+#if ZEN_ENABLE_MESH
options.add_option("network",
"m",
"mesh",
"Enable mesh network",
cxxopts::value<bool>(ServiceConfig.MeshEnabled)->default_value("false"),
"");
+#endif
options.add_option("diagnostics",
"",
@@ -311,7 +313,9 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv
throw std::runtime_error("failed to run global config script ('{}'): {}"_format(ConfigScript, e.what()).c_str());
}
+#if ZEN_ENABLE_MESH
ServiceConfig.MeshEnabled = lua["mesh"]["enable"].get_or(ServiceConfig.MeshEnabled);
+#endif
auto UpdateStringValueFromConfig = [](const sol::table& Table, std::string_view Key, std::string& OutValue) {
// Update the specified config value unless it has been set, i.e. from command line
diff --git a/zenserver/config.h b/zenserver/config.h
index ce059bdb2..75c19d690 100644
--- a/zenserver/config.h
+++ b/zenserver/config.h
@@ -56,10 +56,11 @@ struct ZenUpstreamCacheConfig
struct ZenServiceConfig
{
- bool StructuredCacheEnabled = true;
- bool ShouldCrash = false; // Option for testing crash handling
- bool MeshEnabled = false; // Experimental p2p mesh discovery
- std::string FlockId; // Id for grouping test instances into sets
+ bool StructuredCacheEnabled = true;
+ bool ShouldCrash = false; // Option for testing crash handling
+#if ZEN_ENABLE_MESH
+ bool MeshEnabled = false; // Experimental p2p mesh discovery
+#endif
ZenUpstreamCacheConfig UpstreamCacheConfig;
};
diff --git a/zenserver/testing/httptest.cpp b/zenserver/testing/httptest.cpp
index c4fd6003c..18d63a6ef 100644
--- a/zenserver/testing/httptest.cpp
+++ b/zenserver/testing/httptest.cpp
@@ -2,6 +2,7 @@
#include "httptest.h"
+#include <zencore/compactbinarybuilder.h>
#include <zencore/compactbinarypackage.h>
namespace zen {
@@ -14,6 +15,16 @@ HttpTestingService::HttpTestingService()
HttpVerb::kGet);
m_Router.RegisterRoute(
+ "json",
+ [this](HttpRouterRequest& Req) {
+ CbObjectWriter Obj;
+ Obj.AddBool("ok", true);
+ Obj.AddInteger("counter", ++m_Counter);
+ Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save());
+ },
+ HttpVerb::kGet);
+
+ m_Router.RegisterRoute(
"echo",
[this](HttpRouterRequest& Req) {
IoBuffer Body = Req.ServerRequest().ReadPayload();
diff --git a/zenserver/testing/httptest.h b/zenserver/testing/httptest.h
index b445fb450..f55780d05 100644
--- a/zenserver/testing/httptest.h
+++ b/zenserver/testing/httptest.h
@@ -5,6 +5,8 @@
#include <zencore/logging.h>
#include <zenhttp/httpserver.h>
+#include <atomic>
+
namespace zen {
/**
@@ -37,7 +39,8 @@ public:
};
private:
- HttpRequestRouter m_Router;
+ HttpRequestRouter m_Router;
+ std::atomic<uint32_t> m_Counter{0};
RwLock m_RwLock;
std::unordered_map<uint32_t, Ref<PackageHandler>> m_HandlerMap;
diff --git a/zenserver/windows/service.cpp b/zenserver/windows/service.cpp
index 017b5f9a7..b7b3b9bc1 100644
--- a/zenserver/windows/service.cpp
+++ b/zenserver/windows/service.cpp
@@ -3,6 +3,7 @@
#include "service.h"
#include <zencore/zencore.h>
+#include <zencore/except.h>
#include <stdio.h>
#include <tchar.h>
@@ -146,26 +147,34 @@ CallMain(DWORD, LPSTR*)
int
WindowsService::ServiceMain()
{
- if (zen::IsInteractiveSession())
- {
- // Not actually running as a service
- return Run();
- }
- else
+ gSvc = this;
+
+ SERVICE_TABLE_ENTRY DispatchTable[] = {{(LPWSTR)SVCNAME, (LPSERVICE_MAIN_FUNCTION)&CallMain}, {NULL, NULL}};
+
+ // This call returns when the service has stopped.
+ // The process should simply terminate when the call returns.
+
+ if (!StartServiceCtrlDispatcher(DispatchTable))
{
- gSvc = this;
+ const DWORD dwError = zen::GetLastError();
- SERVICE_TABLE_ENTRY DispatchTable[] = {{(LPWSTR)SVCNAME, (LPSERVICE_MAIN_FUNCTION)&CallMain}, {NULL, NULL}};
+ if (dwError == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT)
+ {
+ // Not actually running as a service
+ gSvc = nullptr;
- // This call returns when the service has stopped.
- // The process should simply terminate when the call returns.
+ zen::SetIsInteractiveSession(true);
- if (!StartServiceCtrlDispatcher(DispatchTable))
+ return Run();
+ }
+ else
{
- SvcReportEvent((LPTSTR)L"StartServiceCtrlDispatcher");
+ zen::ThrowSystemError(dwError, "StartServiceCtrlDispatcher failed");
}
}
+ zen::SetIsInteractiveSession(false);
+
return 0;
}
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index fe4f41ab5..b45df9fef 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -261,6 +261,7 @@ public:
ZEN_INFO("NOT instantiating structured cache service");
}
+#if ZEN_ENABLE_MESH
if (ServiceConfig.MeshEnabled)
{
StartMesh(BasePort);
@@ -269,6 +270,7 @@ public:
{
ZEN_INFO("NOT starting mesh");
}
+#endif
m_Http = zen::CreateHttpServer();
m_Http->Initialize(BasePort);
@@ -302,11 +304,13 @@ public:
}
}
+#if ZEN_ENABLE_MESH
void StartMesh(int BasePort)
{
ZEN_INFO("initializing mesh discovery");
m_ZenMesh.Start(uint16_t(BasePort));
}
+#endif
void Run()
{
diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj
index 29436d840..bcb7ea028 100644
--- a/zenserver/zenserver.vcxproj
+++ b/zenserver/zenserver.vcxproj
@@ -126,6 +126,7 @@
<ClInclude Include="windows\service.h" />
</ItemGroup>
<ItemGroup>
+ <ClCompile Include="admin\admin.cpp" />
<ClCompile Include="cache\structuredcache.cpp" />
<ClCompile Include="cache\structuredcachestore.cpp" />
<ClCompile Include="compute\apply.cpp" />
diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp
index d76058bd1..df5c32d25 100644
--- a/zenstore/cidstore.cpp
+++ b/zenstore/cidstore.cpp
@@ -46,7 +46,7 @@ struct CidStore::Impl
RwLock::ExclusiveLockScope _(m_Lock);
- auto It = m_CidMap.try_emplace(DecompressedId, Compressed);
+ auto It = m_CidMap.try_emplace(DecompressedId, Compressed);
if (!It.second)
{
if (It.first.value() != Compressed)
diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h
index 2eea04164..5f567e7fc 100644
--- a/zenstore/include/zenstore/cidstore.h
+++ b/zenstore/include/zenstore/cidstore.h
@@ -25,9 +25,9 @@ class IoBuffer;
* be used to deal with other kinds of indirections in the future. For example, if we want
* to support chunking then a CID may represent a list of chunks which could be concatenated
* to form the referenced chunk.
- *
+ *
* It would likely be possible to implement this mapping in a more efficient way if we
- * integrate it into the CAS store itself, so we can avoid maintaining copies of large
+ * integrate it into the CAS store itself, so we can avoid maintaining copies of large
* hashes in multiple locations. This would also allow us to consolidate commit logs etc
* which would be more resilient than the current split log scheme
*