diff options
| author | Stefan Boberg <[email protected]> | 2021-09-15 19:49:20 +0200 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2021-09-15 19:49:20 +0200 |
| commit | 83ccd52321a23c8f1c8a3228cbbf34b8f199a22b (patch) | |
| tree | 9cf1fb68651f616aef2fa28000e4f328ef9204d8 | |
| parent | Added GetSize/GetData functions to reduce cognitive load and bridge the gap b... (diff) | |
| parent | Tweaked logging to streamline access, and simplified setup code for new loggers (diff) | |
| download | zen-83ccd52321a23c8f1c8a3228cbbf34b8f199a22b.tar.xz zen-83ccd52321a23c8f1c8a3228cbbf34b8f199a22b.zip | |
Merge branch 'main' into cbpackage-update
86 files changed, 1043 insertions, 659 deletions
diff --git a/3rdparty/BLAKE3/lib/Linux_x64/libblake3.a b/3rdparty/BLAKE3/lib/Linux_x64/libblake3.a Binary files differnew file mode 100644 index 000000000..b956e22cb --- /dev/null +++ b/3rdparty/BLAKE3/lib/Linux_x64/libblake3.a diff --git a/3rdparty/Oodle/lib/Linux_x64/liboo2corelinux64.a b/3rdparty/Oodle/lib/Linux_x64/liboo2corelinux64.a Binary files differnew file mode 100755 index 000000000..dee0353e5 --- /dev/null +++ b/3rdparty/Oodle/lib/Linux_x64/liboo2corelinux64.a diff --git a/3rdparty/Oodle/lib/Linux_x64/liboo2corelinux64.so.8 b/3rdparty/Oodle/lib/Linux_x64/liboo2corelinux64.so.8 Binary files differnew file mode 100755 index 000000000..425ada44d --- /dev/null +++ b/3rdparty/Oodle/lib/Linux_x64/liboo2corelinux64.so.8 diff --git a/scripts/deploybuild.py b/scripts/deploybuild.py index 1bb052ef7..971f34ff9 100644 --- a/scripts/deploybuild.py +++ b/scripts/deploybuild.py @@ -107,7 +107,7 @@ crashpadtarget = os.path.join(target_bin_dir, "crashpad_handler.exe") try: shutil.copy(os.path.join(zenroot, "x64\Release\zenserver.exe"), os.path.join(target_bin_dir, "zenserver.exe")) shutil.copy(os.path.join(zenroot, "x64\Release\zenserver.pdb"), os.path.join(target_bin_dir, "zenserver.pdb")) - shutil.copy(os.path.join(zenroot, r'vcpkg_installed\x64-windows-static\tools\sentry-native\crashpad_handler.exe'), crashpadtarget) + shutil.copy(os.path.join(zenroot, r'vcpkg_installed\x64-windows-static\x64-windows-static\tools\sentry-native\crashpad_handler.exe'), crashpadtarget) P4.add(crashpadtarget).run() except Exception as e: print(f"Caught exception while copying: {e.args}") @@ -35,6 +35,10 @@ if is_mode("debug") then add_defines("DEBUG") end +if is_os("windows") then + add_defines("_CRT_SECURE_NO_WARNINGS") +end + add_defines("USE_SENTRY=1") option("vfs") diff --git a/zen/chunk/chunk.cpp b/zen/chunk/chunk.cpp index bc1b17d2f..a5f010dbe 100644 --- a/zen/chunk/chunk.cpp +++ b/zen/chunk/chunk.cpp @@ -7,6 +7,7 @@ #include <zencore/filesystem.h> #include <zencore/iohash.h> +#include <zencore/logging.h> #include <zencore/refcount.h> #include <zencore/scopeguard.h> #include <zencore/sha1.h> @@ -18,7 +19,6 @@ #include "../internalfile.h" #include <lz4.h> -#include <spdlog/spdlog.h> #include <zstd.h> #include <ppl.h> @@ -589,7 +589,7 @@ public: Concurrency::task_group ChunkProcessTasks; - spdlog::info("Chunking {} ({})", RelativePath.generic_string(), zen::NiceBytes(File.FileSize)); + ZEN_INFO("Chunking {} ({})", RelativePath.generic_string(), zen::NiceBytes(File.FileSize)); zen::RefPtr<InternalFile> Zfile = new InternalFile; Zfile->OpenRead(File.Path); @@ -776,7 +776,7 @@ public: { std::filesystem::path RelativePath{std::filesystem::relative(File.Path.generic_string(), m_RootPath)}; - spdlog::info("Chunking {} ({})", RelativePath.generic_string(), zen::NiceBytes(File.FileSize)); + ZEN_INFO("Chunking {} ({})", RelativePath.generic_string(), zen::NiceBytes(File.FileSize)); zen::RefPtr<InternalFile> Zfile = new InternalFile; Zfile->OpenRead(File.Path); @@ -897,11 +897,11 @@ public: auto ChunkCount = Boundaries.size() + 1; - spdlog::info("Split {} ({}) into {} chunks, avg size {}", - RelativePath.generic_string(), - zen::NiceBytes(File.FileSize), - ChunkCount, - File.FileSize / ChunkCount); + ZEN_INFO("Split {} ({}) into {} chunks, avg size {}", + RelativePath.generic_string(), + zen::NiceBytes(File.FileSize), + ChunkCount, + File.FileSize / ChunkCount); }; private: @@ -953,7 +953,7 @@ ChunkCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) // Gather list of files to process - spdlog::info("Gathering files from {}", m_ScanDirectory); + ZEN_INFO("Gathering files from {}", m_ScanDirectory); std::filesystem::path RootPath{m_ScanDirectory}; DirectoryScanner Scanner; @@ -966,33 +966,33 @@ ChunkCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return Lhs.FileSize < Rhs.FileSize; }); - spdlog::info("Gathered {} files, total size {}", Files.size(), zen::NiceBytes(FileBytes)); + ZEN_INFO("Gathered {} files, total size {}", Files.size(), zen::NiceBytes(FileBytes)); auto ReportSummary = [&](BaseChunker& Chunker, uint64_t ElapsedMs) { const BaseChunker::StatsBlock& Stats = Chunker.SumStats(); const size_t TotalChunkCount = Stats.TotalChunks; - spdlog::info("Scanned {} files in {}, generated {} chunks", Files.size(), zen::NiceTimeSpanMs(ElapsedMs), TotalChunkCount); + ZEN_INFO("Scanned {} files in {}, generated {} chunks", Files.size(), zen::NiceTimeSpanMs(ElapsedMs), TotalChunkCount); const size_t TotalByteCount = Stats.TotalBytes; const size_t TotalCompressedBytes = Stats.TotalCompressed; - spdlog::info("Total bytes {} ({}), compresses into {}", - zen::NiceBytes(TotalByteCount), - zen::NiceByteRate(TotalByteCount, ElapsedMs), - zen::NiceBytes(TotalCompressedBytes)); + ZEN_INFO("Total bytes {} ({}), compresses into {}", + zen::NiceBytes(TotalByteCount), + zen::NiceByteRate(TotalByteCount, ElapsedMs), + zen::NiceBytes(TotalCompressedBytes)); const size_t TotalUniqueBytes = Stats.UniqueBytes; const size_t TotalUniqueCompressedBytes = Stats.UniqueCompressed; const size_t TotalDuplicateBytes = Stats.DuplicateBytes; - spdlog::info("Chunksize average {}, unique bytes = {} (compressed {}), dup bytes = {}", - TotalByteCount / TotalChunkCount, - zen::NiceBytes(TotalUniqueBytes), - zen::NiceBytes(TotalUniqueCompressedBytes), - zen::NiceBytes(TotalDuplicateBytes)); + ZEN_INFO("Chunksize average {}, unique bytes = {} (compressed {}), dup bytes = {}", + TotalByteCount / TotalChunkCount, + zen::NiceBytes(TotalUniqueBytes), + zen::NiceBytes(TotalUniqueCompressedBytes), + zen::NiceBytes(TotalDuplicateBytes)); - spdlog::info("New to CAS: {} chunks, {}", Stats.NewCasChunks, zen::NiceBytes(Stats.NewCasBytes)); + ZEN_INFO("New to CAS: {} chunks, {}", Stats.NewCasChunks, zen::NiceBytes(Stats.NewCasBytes)); }; // Process them as quickly as possible @@ -1037,7 +1037,7 @@ ChunkCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { zen::ExtendableStringBuilder<256> Path8; zen::WideToUtf8(ThisFile.Path.c_str(), Path8); - spdlog::warn("Caught exception while chunking '{}': {}", Path8, ex.what()); + ZEN_WARN("Caught exception while chunking '{}': {}", Path8, ex.what()); } }); @@ -1143,12 +1143,12 @@ TEST_CASE("chunking") double Avg = double(BoundarySum) / BoundaryCount; const uint64_t ElapsedTimeMs = timer.getElapsedTimeMs(); - spdlog::info("{:9} : Avg {:9} - {:2.5} ({:6}, {})", - i, - Avg, - double(i / Avg), - NiceTimeSpanMs(ElapsedTimeMs), - NiceByteRate(chunker.BytesScanned(), ElapsedTimeMs)); + ZEN_INFO("{:9} : Avg {:9} - {:2.5} ({:6}, {})", + i, + Avg, + double(i / Avg), + NiceTimeSpanMs(ElapsedTimeMs), + NiceByteRate(chunker.BytesScanned(), ElapsedTimeMs)); } }; diff --git a/zen/cmds/cache.cpp b/zen/cmds/cache.cpp index 3f4a4cdc3..69e500293 100644 --- a/zen/cmds/cache.cpp +++ b/zen/cmds/cache.cpp @@ -3,10 +3,10 @@ #include "cache.h" #include <zencore/filesystem.h> +#include <zencore/logging.h> #include <zenhttp/httpcommon.h> #include <zenserverprocess.h> -#include <spdlog/spdlog.h> #include <memory> // cpr //////////////////////////////////////////////////////////////////// @@ -39,7 +39,7 @@ DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_Options.parse_positional({"bucket", "positional"}); m_Options.parse(argc, argv); - spdlog::info("Dropping cache bucket '{}' from '{}'", m_BucketName, m_HostName); + ZEN_INFO("Dropping cache bucket '{}' from '{}'", m_BucketName, m_HostName); cpr::Session Session; Session.SetUrl({"{}/z$/{}"_format(m_HostName, m_BucketName)}); @@ -47,18 +47,18 @@ DropCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (zen::IsHttpSuccessCode(Result.status_code)) { - spdlog::info("OK: dropped cache bucket '{}' from '{}'", m_BucketName, m_HostName); + ZEN_INFO("OK: dropped cache bucket '{}' from '{}'", m_BucketName, m_HostName); return 0; } if (Result.status_code) { - spdlog::error("Drop failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); + ZEN_ERROR("Drop failed: {}: {} ({})", Result.status_code, Result.reason, Result.text); } else { - spdlog::error("Drop failed: {}", Result.error.message); + ZEN_ERROR("Drop failed: {}", Result.error.message); } return 1; diff --git a/zen/cmds/copy.cpp b/zen/cmds/copy.cpp index 571a9e092..4ce09c982 100644 --- a/zen/cmds/copy.cpp +++ b/zen/cmds/copy.cpp @@ -3,11 +3,10 @@ #include "copy.h" #include <zencore/filesystem.h> +#include <zencore/logging.h> #include <zencore/string.h> #include <zencore/timer.h> -#include <spdlog/spdlog.h> - CopyCommand::CopyCommand() { m_Options.add_options()("h,help", "Print help"); @@ -38,10 +37,10 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) // Validate arguments if (m_CopySource.empty()) - throw std::exception("No source specified"); + throw std::runtime_error("No source specified"); if (m_CopyTarget.empty()) - throw std::exception("No target specified"); + throw std::runtime_error("No target specified"); std::filesystem::path FromPath; std::filesystem::path ToPath; @@ -54,12 +53,12 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!IsFileCopy && !IsDirCopy) { - throw std::exception("Invalid source specification (neither directory nor file)"); + throw std::runtime_error("Invalid source specification (neither directory nor file)"); } if (IsFileCopy && IsDirCopy) { - throw std::exception("Invalid source specification (both directory AND file!?)"); + throw std::runtime_error("Invalid source specification (both directory AND file!?)"); } if (IsDirCopy) @@ -71,7 +70,7 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (std::filesystem::is_regular_file(ToPath)) { - throw std::exception("Attempted copy of directory into file"); + throw std::runtime_error("Attempted copy of directory into file"); } } } @@ -90,7 +89,7 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) CopyOptions.EnableClone = !m_NoClone; zen::CopyFile(FromPath, ToPath, CopyOptions); - spdlog::info("Copy completed in {}", zen::NiceTimeSpanMs(Timer.getElapsedTimeMs())); + ZEN_INFO("Copy completed in {}", zen::NiceTimeSpanMs(Timer.getElapsedTimeMs())); } return 0; diff --git a/zen/cmds/dedup.cpp b/zen/cmds/dedup.cpp index 0684da30c..90a4fea76 100644 --- a/zen/cmds/dedup.cpp +++ b/zen/cmds/dedup.cpp @@ -5,12 +5,12 @@ #include <zencore/blake3.h> #include <zencore/filesystem.h> #include <zencore/iobuffer.h> +#include <zencore/logging.h> #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/timer.h> #include <ppl.h> -#include <spdlog/spdlog.h> DedupCommand::DedupCommand() { @@ -46,22 +46,22 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!SourceGood) { - spdlog::info("Source directory '{}' does not support deduplication", m_DedupSource); + ZEN_INFO("Source directory '{}' does not support deduplication", m_DedupSource); return 0; } if (!TargetGood) { - spdlog::info("Target directory '{}' does not support deduplication", m_DedupTarget); + ZEN_INFO("Target directory '{}' does not support deduplication", m_DedupTarget); return 0; } - spdlog::info("Performing dedup operation between {} and {}, size threshold {}", - m_DedupSource, - m_DedupTarget, - zen::NiceBytes(m_SizeThreshold)); + ZEN_INFO("Performing dedup operation between {} and {}, size threshold {}", + m_DedupSource, + m_DedupTarget, + zen::NiceBytes(m_SizeThreshold)); using DirEntryList_t = std::list<std::filesystem::directory_entry>; @@ -93,8 +93,8 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::string_view c_str() { return std::string_view(Path8); }; }; - spdlog::info("Gathering file info from source: '{}'", m_DedupSource); - spdlog::info("Gathering file info from target: '{}'", m_DedupTarget); + ZEN_INFO("Gathering file info from source: '{}'", m_DedupSource); + ZEN_INFO("Gathering file info from target: '{}'", m_DedupTarget); { zen::Stopwatch Timer; @@ -113,13 +113,13 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); - spdlog::info("Gathered {} candidates across {} size buckets. Elapsed: {}", - CandidateCount, - FileSizeMap.size(), - zen::NiceTimeSpanMs(Timer.getElapsedTimeMs())); + ZEN_INFO("Gathered {} candidates across {} size buckets. Elapsed: {}", + CandidateCount, + FileSizeMap.size(), + zen::NiceTimeSpanMs(Timer.getElapsedTimeMs())); } - spdlog::info("Sorting buckets by size"); + ZEN_INFO("Sorting buckets by size"); zen::Stopwatch Timer; @@ -147,7 +147,7 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::sort(begin(SizeLists), end(SizeLists), [](const SizeList& Lhs, const SizeList& Rhs) { return Lhs.Size > Rhs.Size; }); - spdlog::info("Bucket summary:"); + ZEN_INFO("Bucket summary:"); std::vector<size_t> BucketId; std::vector<size_t> BucketOffsets; @@ -204,10 +204,10 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) for (int i = 0; i < BucketOffsets.size(); ++i) { - spdlog::info(" Bucket {} : {}, {} candidates", zen::NiceBytes(BucketId[i]), zen::NiceBytes(BucketSizes[i]), BucketFileCounts[i]); + ZEN_INFO(" Bucket {} : {}, {} candidates", zen::NiceBytes(BucketId[i]), zen::NiceBytes(BucketSizes[i]), BucketFileCounts[i]); } - spdlog::info("Total : {}, {} candidates", zen::NiceBytes(TotalFileSizes), TotalFileCount); + ZEN_INFO("Total : {}, {} candidates", zen::NiceBytes(TotalFileSizes), TotalFileCount); std::string CurrentNice; @@ -218,7 +218,7 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (CurNice != CurrentNice) { CurrentNice = CurNice; - spdlog::info("Now scanning bucket: {}", CurrentNice); + ZEN_INFO("Now scanning bucket: {}", CurrentNice); } std::unordered_map<zen::BLAKE3, const std::filesystem::directory_entry*, zen::BLAKE3::Hasher> DedupMap; @@ -264,10 +264,10 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) FileB = std::wstring(FileB.begin(), Its.first.base()) + L"..."; } - spdlog::info("{} {} <-> {}", - zen::NiceBytes(Entry.file_size()).c_str(), - Utf8Helper(FileA.c_str()).c_str(), - Utf8Helper(FileB.c_str()).c_str()); + ZEN_INFO("{} {} <-> {}", + zen::NiceBytes(Entry.file_size()).c_str(), + Utf8Helper(FileA.c_str()).c_str(), + Utf8Helper(FileB.c_str()).c_str()); zen::CopyFileOptions Options; Options.EnableClone = true; @@ -286,7 +286,7 @@ DedupCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Size.DirEntries->clear(); } - spdlog::info("Elapsed: {} Deduped: {}", zen::NiceTimeSpanMs(Timer.getElapsedTimeMs()), zen::NiceBytes(DupeBytes)); + ZEN_INFO("Elapsed: {} Deduped: {}", zen::NiceTimeSpanMs(Timer.getElapsedTimeMs()), zen::NiceBytes(DupeBytes)); return 0; } diff --git a/zen/cmds/deploy.cpp b/zen/cmds/deploy.cpp index 3e5b77c20..b8879fefb 100644 --- a/zen/cmds/deploy.cpp +++ b/zen/cmds/deploy.cpp @@ -2,10 +2,9 @@ #include "deploy.h" +#include <zencore/logging.h> #include <zencore/string.h> -#include <spdlog/spdlog.h> - DeployCommand::DeployCommand() { m_Options.add_options()("h,help", "Print help"); @@ -39,10 +38,10 @@ DeployCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) // Validate arguments if (m_CopySource.empty()) - throw std::exception("No source specified"); + throw std::runtime_error("No source specified"); if (m_CopyTarget.empty()) - throw std::exception("No target specified"); + throw std::runtime_error("No target specified"); std::filesystem::path ToPath; @@ -53,7 +52,7 @@ DeployCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!IsTargetNew && !IsTargetDir) { - throw std::exception("Invalid target specification (needs to be a directory)"); + throw std::runtime_error("Invalid target specification (needs to be a directory)"); } zen::ExtendableStringBuilder<128> Path8; @@ -61,7 +60,7 @@ DeployCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (IsTargetNew == false && m_IsClean) { - spdlog::info("Clean deploy -- deleting directory {}", Path8.c_str()); + ZEN_INFO("Clean deploy -- deleting directory {}", Path8.c_str()); std::filesystem::remove_all(ToPath); @@ -70,12 +69,12 @@ DeployCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (IsTargetNew) { - spdlog::info("Creating directory {}", Path8.c_str()); + ZEN_INFO("Creating directory {}", Path8.c_str()); std::filesystem::create_directories(ToPath); } - spdlog::info("Starting deploy operation..."); + ZEN_INFO("Starting deploy operation..."); // TODO: implement! diff --git a/zen/cmds/hash.cpp b/zen/cmds/hash.cpp index 45506ebee..b6276dbc1 100644 --- a/zen/cmds/hash.cpp +++ b/zen/cmds/hash.cpp @@ -3,11 +3,11 @@ #include "hash.h" #include <zencore/blake3.h> +#include <zencore/logging.h> #include <zencore/string.h> #include <zencore/timer.h> #include <ppl.h> -#include <spdlog/spdlog.h> HashCommand::HashCommand() { @@ -33,7 +33,7 @@ HashCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) // Gather list of files to process - spdlog::info("Gathering files from {}", m_ScanDirectory); + ZEN_INFO("Gathering files from {}", m_ScanDirectory); struct FileEntry { @@ -55,7 +55,7 @@ HashCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - spdlog::info("Gathered {} files, total size {}", FileList.size(), zen::NiceBytes(FileBytes)); + ZEN_INFO("Gathered {} files, total size {}", FileList.size(), zen::NiceBytes(FileBytes)); Concurrency::combinable<uint64_t> TotalBytes; @@ -88,8 +88,8 @@ HashCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) TotalBytes.combine_each([&](size_t Total) { TotalByteCount += Total; }); const uint64_t ElapsedMs = Timer.getElapsedTimeMs(); - spdlog::info("Scanned {} files in {}", FileList.size(), zen::NiceTimeSpanMs(ElapsedMs)); - spdlog::info("Total bytes {} ({})", zen::NiceBytes(TotalByteCount), zen::NiceByteRate(TotalByteCount, ElapsedMs)); + ZEN_INFO("Scanned {} files in {}", FileList.size(), zen::NiceTimeSpanMs(ElapsedMs)); + ZEN_INFO("Total bytes {} ({})", zen::NiceBytes(TotalByteCount), zen::NiceByteRate(TotalByteCount, ElapsedMs)); InternalFile Output; diff --git a/zen/cmds/run.cpp b/zen/cmds/run.cpp index 3427a6efc..e5210f695 100644 --- a/zen/cmds/run.cpp +++ b/zen/cmds/run.cpp @@ -9,11 +9,11 @@ #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/iohash.h> +#include <zencore/logging.h> #include <zencore/string.h> #include <zencore/timer.h> #include <zenserverprocess.h> -#include <spdlog/spdlog.h> #include <filesystem> // cpr //////////////////////////////////////////////////////////////////// @@ -89,7 +89,7 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) zen::IoHash Hash = Ios.GetHash(); std::wstring RelativePath = FullPath.lexically_relative(m_RootPath).native(); - // spdlog::info("File: {:32} => {} ({})", zen::WideToUtf8(RelativePath), Hash, FileSize); + // ZEN_INFO("File: {:32} => {} ({})", zen::WideToUtf8(RelativePath), Hash, FileSize); FileEntry& Entry = m_Files[RelativePath]; Entry.Hash = Hash; @@ -164,12 +164,12 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (CasResponse.status_code >= 300) { - spdlog::error("CAS put failed with {}", CasResponse.status_code); + ZEN_ERROR("CAS put failed with {}", CasResponse.status_code); } } else { - spdlog::error("unknown hash in 'need' list: {}", NeedHash); + ZEN_ERROR("unknown hash in 'need' list: {}", NeedHash); } } } @@ -177,7 +177,7 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) cpr::Response JobResponse = cpr::Post(cpr::Url("http://localhost:13337/exec/jobs"), cpr::Body((const char*)MemOut.Data(), MemOut.Size())); - spdlog::info("job exec: {}", JobResponse.status_code); + ZEN_INFO("job exec: {}", JobResponse.status_code); return 0; } diff --git a/zen/cmds/status.cpp b/zen/cmds/status.cpp index b55c08101..6741ab9e9 100644 --- a/zen/cmds/status.cpp +++ b/zen/cmds/status.cpp @@ -2,7 +2,7 @@ #include "status.h" -#include <spdlog/spdlog.h> +#include <zencore/logging.h> #include <memory> StatusCommand::StatusCommand() diff --git a/zen/cmds/top.cpp b/zen/cmds/top.cpp index 2ec7db66a..5bb11d0a0 100644 --- a/zen/cmds/top.cpp +++ b/zen/cmds/top.cpp @@ -2,9 +2,9 @@ #include "top.h" +#include <zencore/logging.h> #include <zenserverprocess.h> -#include <spdlog/spdlog.h> #include <memory> ////////////////////////////////////////////////////////////////////////// @@ -23,10 +23,10 @@ TopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZenServerState State; if (!State.InitializeReadOnly()) { - spdlog::info("no Zen state found"); + ZEN_INFO("no Zen state found"); } - State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { spdlog::info("Port {} : pid {}", Entry.ListenPort, Entry.Pid); }); + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { ZEN_INFO("Port {} : pid {}", Entry.ListenPort, Entry.Pid); }); return 0; } @@ -47,10 +47,10 @@ PsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZenServerState State; if (!State.InitializeReadOnly()) { - spdlog::info("no Zen state found"); + ZEN_INFO("no Zen state found"); } - State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { spdlog::info("Port {} : pid {}", Entry.ListenPort, Entry.Pid); }); + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { ZEN_INFO("Port {} : pid {}", Entry.ListenPort, Entry.Pid); }); return 0; } diff --git a/zen/cmds/up.cpp b/zen/cmds/up.cpp index ff5fc3e3b..a1047fd57 100644 --- a/zen/cmds/up.cpp +++ b/zen/cmds/up.cpp @@ -3,9 +3,9 @@ #include "up.h" #include <zencore/filesystem.h> +#include <zencore/logging.h> #include <zenserverprocess.h> -#include <spdlog/spdlog.h> #include <memory> UpCommand::UpCommand() @@ -30,11 +30,11 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!Server.WaitUntilReady(Timeout)) { - spdlog::error("zen server launch failed (timed out)"); + ZEN_ERROR("zen server launch failed (timed out)"); } else { - spdlog::info("zen server up"); + ZEN_INFO("zen server up"); } return 0; @@ -63,7 +63,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!Entry) { - spdlog::warn("no zen server to bring down"); + ZEN_WARN("no zen server to bring down"); return 0; } @@ -77,17 +77,17 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZenServerInstance Server(ServerEnvironment); Server.AttachToRunningServer(ListenPort); - spdlog::info("attached to server on port {}, requesting shutdown", ListenPort); + ZEN_INFO("attached to server on port {}, requesting shutdown", ListenPort); Server.Shutdown(); - spdlog::info("shutdown complete"); + ZEN_INFO("shutdown complete"); return 0; } catch (std::exception& Ex) { - spdlog::debug("Exception caught when requesting shutdown: {}", Ex.what()); + ZEN_DEBUG("Exception caught when requesting shutdown: {}", Ex.what()); } // Since we cannot obtain a handle to the process we are unable to block on the process diff --git a/zen/internalfile.cpp b/zen/internalfile.cpp index b88ec51dc..3ad5a7d05 100644 --- a/zen/internalfile.cpp +++ b/zen/internalfile.cpp @@ -4,10 +4,9 @@ #include <zencore/filesystem.h> #include <zencore/fmtutils.h> +#include <zencore/logging.h> #include <zencore/windows.h> -#include <spdlog/spdlog.h> - #include <gsl/gsl-lite.hpp> #define ZEN_USE_SLIST ZEN_PLATFORM_WINDOWS diff --git a/zen/xmake.lua b/zen/xmake.lua index 5e2fce4b6..6fbdd88b5 100644 --- a/zen/xmake.lua +++ b/zen/xmake.lua @@ -1,7 +1,7 @@ target("zen") set_kind("binary") add_files("**.cpp") - add_deps("zencore", "zenstore", "zenutil") + add_deps("zencore", "zenstore", "zenhttp", "zenutil") add_includedirs(".") set_symbols("debug") diff --git a/zen/zen.cpp b/zen/zen.cpp index 67ff50105..4b8d3adc1 100644 --- a/zen/zen.cpp +++ b/zen/zen.cpp @@ -18,6 +18,7 @@ #include "cmds/top.h" #include "cmds/up.h" +#include <zencore/logging.h> #include <zencore/scopeguard.h> #include <zencore/string.h> #include <zenstore/cas.h> @@ -35,7 +36,6 @@ # pragma comment(lib, "userenv.lib") #endif -#include <spdlog/spdlog.h> #include <gsl/gsl-lite.hpp> #include <mimalloc-new-delete.h> diff --git a/zencore/compress.cpp b/zencore/compress.cpp index 2b2c4dd0b..12a7b9ef8 100644 --- a/zencore/compress.cpp +++ b/zencore/compress.cpp @@ -8,11 +8,14 @@ #include <zencore/endian.h> #include "../3rdparty/Oodle/include/oodle2.h" -#pragma comment(lib, "oo2core_win64.lib") +#if ZEN_PLATFORM_WINDOWS +# pragma comment(lib, "oo2core_win64.lib") +#endif #include <doctest/doctest.h> #include <lz4.h> #include <functional> +#include <limits> namespace zen::detail { @@ -100,7 +103,7 @@ struct BufferHeader constexpr uint64_t MethodOffset = offsetof(BufferHeader, Method); for (MemoryView View = HeaderView + MethodOffset; const uint64_t ViewSize = View.GetSize();) { - const int32_t Size = static_cast<int32_t>(zen::Min<uint64_t>(ViewSize, INT_MAX)); + const int32_t Size = static_cast<int32_t>(zen::Min<uint64_t>(ViewSize, /* INT_MAX */ 2147483647u)); Crc32 = zen::MemCrc32(View.GetData(), Size, Crc32); View += Size; } @@ -208,7 +211,7 @@ private: CompositeBuffer BlockEncoder::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize) const { - ZEN_ASSERT(IsPow2(BlockSize) && BlockSize <= (1 << 31)); + ZEN_ASSERT(IsPow2(BlockSize) && (BlockSize <= (1u << 31))); const uint64_t RawSize = RawData.GetSize(); BLAKE3Stream RawHash; @@ -589,7 +592,7 @@ protected: const int Size = LZ4_decompress_safe(static_cast<const char*>(CompressedData.GetData()), static_cast<char*>(RawData.GetData()), static_cast<int>(CompressedData.GetSize()), - static_cast<int>(zen::Min<uint64_t>(RawData.GetSize(), LZ4_MAX_INPUT_SIZE))); + static_cast<int>(zen::Min<uint64_t>(RawData.GetSize(), uint64_t(LZ4_MAX_INPUT_SIZE)))); return static_cast<uint64_t>(Size) == RawData.GetSize(); } return false; diff --git a/zencore/except.cpp b/zencore/except.cpp index 9bd447308..834585522 100644 --- a/zencore/except.cpp +++ b/zencore/except.cpp @@ -2,10 +2,11 @@ #include <fmt/format.h> #include <zencore/except.h> -#include <system_error> namespace zen { +#if ZEN_PLATFORM_WINDOWS + void ThrowSystemException([[maybe_unused]] HRESULT hRes, [[maybe_unused]] std::string_view Message) { @@ -19,16 +20,18 @@ ThrowSystemException([[maybe_unused]] HRESULT hRes, [[maybe_unused]] std::string } } +#endif // ZEN_PLATFORM_WINDOWS + void ThrowLastError(std::string_view Message) { - throw std::system_error(std::error_code(::GetLastError(), std::system_category()), std::string(Message)); + throw std::system_error(std::error_code(zen::GetLastError(), std::system_category()), std::string(Message)); } std::string GetLastErrorAsString() { - return GetWindowsErrorAsString(::GetLastError()); + return GetWindowsErrorAsString(zen::GetLastError()); } std::string @@ -37,12 +40,14 @@ GetWindowsErrorAsString(uint32_t Win32ErrorCode) return std::error_code(Win32ErrorCode, std::system_category()).message(); } +#if __cpp_lib_source_location void ThrowLastError(std::string_view Message, const std::source_location& Location) { using namespace fmt::literals; - throw std::system_error(std::error_code(::GetLastError(), std::system_category()), + throw std::system_error(std::error_code(zen::GetLastError(), std::system_category()), "{}({}): {}"_format(Location.file_name(), Location.line(), Message)); } +#endif } // namespace zen diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index 09c2e7386..59300b7ad 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -5,6 +5,7 @@ #include <zencore/except.h> #include <zencore/fmtutils.h> #include <zencore/iobuffer.h> +#include <zencore/logging.h> #include <zencore/string.h> #include <zencore/windows.h> @@ -14,8 +15,6 @@ #include <winnt.h> #include <filesystem> -#include <spdlog/spdlog.h> - #include <gsl/gsl-lite.hpp> namespace zen { @@ -582,7 +581,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr } else if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DEVICE) { - spdlog::warn("encountered device node during file system traversal: {} found in {}", WideToUtf8(FileName), RootDir); + ZEN_WARN("encountered device node during file system traversal: {} found in {}", WideToUtf8(FileName), RootDir); } else { @@ -618,6 +617,8 @@ PathFromHandle(void* NativeHandle) const DWORD FinalLength = GetFinalPathNameByHandleW(NativeHandle, FullPath.data(), RequiredLengthIncludingNul, FILE_NAME_OPENED); + ZEN_UNUSED(FinalLength); + return FullPath; } diff --git a/zencore/include/zencore/atomic.h b/zencore/include/zencore/atomic.h index 457128bd4..7e261771b 100644 --- a/zencore/include/zencore/atomic.h +++ b/zencore/include/zencore/atomic.h @@ -2,7 +2,14 @@ #pragma once +#include <zencore/zencore.h> + +#if ZEN_COMPILER_MSC #include <intrin.h> +#else +#include <atomic> +#endif + #include <cinttypes> namespace zen { @@ -10,34 +17,58 @@ namespace zen { inline uint32_t AtomicIncrement(volatile uint32_t& value) { +#if ZEN_COMPILER_MSC return _InterlockedIncrement((long volatile*)&value); +#else + return ((std::atomic<uint32_t>*)(&value))->fetch_add(1, std::memory_order_seq_cst) + 1; +#endif } inline uint32_t AtomicDecrement(volatile uint32_t& value) { +#if ZEN_COMPILER_MSC return _InterlockedDecrement((long volatile*)&value); +#else + return ((std::atomic<uint32_t>*)(&value))->fetch_sub(1, std::memory_order_seq_cst) - 1; +#endif } inline uint64_t AtomicIncrement(volatile uint64_t& value) { +#if ZEN_COMPILER_MSC return _InterlockedIncrement64((__int64 volatile*)&value); +#else + return ((std::atomic<uint64_t>*)(&value))->fetch_add(1, std::memory_order_seq_cst) + 1; +#endif } inline uint64_t AtomicDecrement(volatile uint64_t& value) { +#if ZEN_COMPILER_MSC return _InterlockedDecrement64((__int64 volatile*)&value); +#else + return ((std::atomic<uint64_t>*)(&value))->fetch_sub(1, std::memory_order_seq_cst) - 1; +#endif } inline uint32_t AtomicAdd(volatile uint32_t& value, uint32_t amount) { +#if ZEN_COMPILER_MSC return _InterlockedExchangeAdd((long volatile*)&value, amount); +#else + return ((std::atomic<uint32_t>*)(&value))->fetch_add(amount, std::memory_order_seq_cst); +#endif } inline uint64_t AtomicAdd(volatile uint64_t& value, uint64_t amount) { +#if ZEN_COMPILER_MSC return _InterlockedExchangeAdd64((__int64 volatile*)&value, amount); +#else + return ((std::atomic<uint64_t>*)(&value))->fetch_add(amount, std::memory_order_seq_cst); +#endif } } // namespace zen diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h index 09619be8b..4fce129ea 100644 --- a/zencore/include/zencore/compactbinary.h +++ b/zencore/include/zencore/compactbinary.h @@ -748,6 +748,8 @@ private: friend class CbFieldViewIterator; + friend class CbFieldIterator; + /** Pointer to the first byte past the end of the last field. Set to null at the end. */ const void* FieldsEnd = nullptr; }; diff --git a/zencore/include/zencore/compactbinarybuilder.h b/zencore/include/zencore/compactbinarybuilder.h index f7f2bbfd3..5f6d9fd0c 100644 --- a/zencore/include/zencore/compactbinarybuilder.h +++ b/zencore/include/zencore/compactbinarybuilder.h @@ -533,7 +533,7 @@ operator<<(CbWriter& Writer, const CbArray& Value) } inline CbWriter& -operator<<(CbWriter& Writer, nullptr_t) +operator<<(CbWriter& Writer, std::nullptr_t) { Writer.AddNull(); return Writer; diff --git a/zencore/include/zencore/endian.h b/zencore/include/zencore/endian.h index 1f79a59e8..d44a27b01 100644 --- a/zencore/include/zencore/endian.h +++ b/zencore/include/zencore/endian.h @@ -7,19 +7,31 @@ namespace zen { inline uint16_t ByteSwap(uint16_t x) { +#if ZEN_COMPILER_MSC return _byteswap_ushort(x); +#else + return __builtin_bswap16(x); +#endif } inline uint32_t ByteSwap(uint32_t x) { +#if ZEN_COMPILER_MSC return _byteswap_ulong(x); +#else + return __builtin_bswap32(x); +#endif } inline uint64_t ByteSwap(uint64_t x) { +#if ZEN_COMPILER_MSC return _byteswap_uint64(x); +#else + return __builtin_bswap64(x); +#endif } inline uint16_t diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h index 8625f01d0..36cca895f 100644 --- a/zencore/include/zencore/except.h +++ b/zencore/include/zencore/except.h @@ -3,12 +3,18 @@ #pragma once #include <zencore/string.h> -#include <zencore/windows.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#else +# include <errno.h> +#endif #include <source_location> #include <string> +#include <system_error> namespace zen { +#if ZEN_PLATFORM_WINDOWS class WindowsException : public std::exception { public: @@ -45,17 +51,36 @@ private: }; ZENCORE_API void ThrowSystemException(HRESULT hRes, std::string_view Message); +#endif // ZEN_PLATFORM_WINDOWS + +ZENCORE_API void ThrowLastError(std::string_view Message); + +#if __cpp_lib_source_location +ZENCORE_API void ThrowLastError(std::string_view Message, const std::source_location& Location); +#endif + +ZENCORE_API std::string GetLastErrorAsString(); +ZENCORE_API std::string GetWindowsErrorAsString(uint32_t Win32ErrorCode); inline void ThrowSystemException(const char* Message) { +#if ZEN_PLATFORM_WINDOWS throw WindowsException(Message); +#else + ThrowLastError(Message); +#endif } -ZENCORE_API void ThrowLastError(std::string_view Message); -ZENCORE_API void ThrowLastError(std::string_view Message, const std::source_location& Location); -ZENCORE_API std::string GetLastErrorAsString(); -ZENCORE_API std::string GetWindowsErrorAsString(uint32_t Win32ErrorCode); +inline int32_t +GetLastError() +{ +#if ZEN_PLATFORM_WINDOWS + return ::GetLastError(); +#else + return errno; +#endif +} inline std::error_code MakeWin32ErrorCode(uint32_t Win32ErrorCode) noexcept @@ -66,7 +91,7 @@ MakeWin32ErrorCode(uint32_t Win32ErrorCode) noexcept inline std::error_code MakeErrorCodeFromLastError() noexcept { - return std::error_code(::GetLastError(), std::system_category()); + return std::error_code(zen::GetLastError(), std::system_category()); } } // namespace zen diff --git a/zencore/include/zencore/fmtutils.h b/zencore/include/zencore/fmtutils.h index fb5a08d56..29e2ae78a 100644 --- a/zencore/include/zencore/fmtutils.h +++ b/zencore/include/zencore/fmtutils.h @@ -43,7 +43,7 @@ struct fmt::formatter<std::filesystem::path> : formatter<string_view> auto format(const std::filesystem::path& Path, FormatContext& ctx) { zen::ExtendableStringBuilder<128> String; - WideToUtf8(Path.c_str(), String); + String << Path.u8string(); return formatter<string_view>::format(String.ToView(), ctx); } }; diff --git a/zencore/include/zencore/intmath.h b/zencore/include/zencore/intmath.h index 792b8b2b4..814a03df4 100644 --- a/zencore/include/zencore/intmath.h +++ b/zencore/include/zencore/intmath.h @@ -9,8 +9,46 @@ ////////////////////////////////////////////////////////////////////////// -#pragma intrinsic(_BitScanReverse) -#pragma intrinsic(_BitScanReverse64) +#if ZEN_COMPILER_MSC || ZEN_PLATFORM_WINDOWS +# pragma intrinsic(_BitScanReverse) +# pragma intrinsic(_BitScanReverse64) +#else +inline uint8_t +_BitScanReverse(unsigned long* Index, uint32_t Mask) +{ + if (Mask == 0) + { + return 0; + } + + *Index = __builtin_clz(Mask); + return 1; +} + +inline uint8_t +_BitScanReverse64(unsigned long* Index, uint64_t Mask) +{ + if (Mask == 0) + { + return 0; + } + + *Index = __builtin_clzll(Mask); + return 0; +} + +inline uint8_t +_BitScanForward64(unsigned long* Index, uint64_t Mask) +{ + if (Mask == 0) + { + return 0; + } + + *Index = __builtin_ctzll(Mask); + return 0; +} +#endif namespace zen { @@ -121,8 +159,10 @@ IsPointerAligned(const void* Ptr, uint64_t Alignment) ////////////////////////////////////////////////////////////////////////// -#ifdef min -# error "Looks like you did #include <windows.h> -- use <zencore/windows.h> instead" +#if ZEN_PLATFORM_WINDOWS +# ifdef min +# error "Looks like you did #include <windows.h> -- use <zencore/windows.h> instead" +# endif #endif constexpr auto @@ -137,4 +177,8 @@ Max(auto x, auto y) return x > y ? x : y; } +////////////////////////////////////////////////////////////////////////// + +void intmath_forcelink(); // internal + } // namespace zen diff --git a/zencore/include/zencore/iohash.h b/zencore/include/zencore/iohash.h index aaa638684..fd0f4b2a7 100644 --- a/zencore/include/zencore/iohash.h +++ b/zencore/include/zencore/iohash.h @@ -8,6 +8,7 @@ #include <zencore/blake3.h> #include <zencore/memory.h> +#include <compare> #include <string_view> namespace zen { diff --git a/zencore/include/zencore/logging.h b/zencore/include/zencore/logging.h index a2404a5e9..4eee20414 100644 --- a/zencore/include/zencore/logging.h +++ b/zencore/include/zencore/logging.h @@ -13,6 +13,7 @@ namespace zen::logging { spdlog::logger& Default(); +void SetDefault(std::shared_ptr<spdlog::logger> NewDefaultLogger); spdlog::logger& ConsoleLog(); spdlog::logger& Get(std::string_view Name); @@ -20,3 +21,29 @@ void InitializeLogging(); void ShutdownLogging(); } // namespace zen::logging + +namespace zen { +extern spdlog::logger* TheDefaultLogger; + +inline spdlog::logger& +Log() +{ + return *TheDefaultLogger; +} + +using logging::ConsoleLog; +} // namespace zen + +using zen::ConsoleLog; +using zen::Log; + +// Helper macros for logging + +using namespace std::literals; + +#define ZEN_TRACE(fmtstr, ...) Log().trace(fmtstr##sv, __VA_ARGS__) +#define ZEN_DEBUG(fmtstr, ...) Log().debug(fmtstr##sv, __VA_ARGS__) +#define ZEN_INFO(fmtstr, ...) Log().info(fmtstr##sv, __VA_ARGS__) +#define ZEN_WARN(fmtstr, ...) Log().warn(fmtstr##sv, __VA_ARGS__) +#define ZEN_ERROR(fmtstr, ...) Log().error(fmtstr##sv, __VA_ARGS__) +#define ZEN_CRITICAL(fmtstr, ...) Log().critical(fmtstr##sv, __VA_ARGS__) diff --git a/zencore/include/zencore/memory.h b/zencore/include/zencore/memory.h index 5a324955d..9d6339595 100644 --- a/zencore/include/zencore/memory.h +++ b/zencore/include/zencore/memory.h @@ -8,6 +8,8 @@ #include <zencore/thread.h> #include <algorithm> +#include <cstddef> +#include <cstring> #include <span> #include <vector> @@ -81,7 +83,7 @@ struct MutableMemoryView { } - inline bool IsEmpty() const { return m_Data == m_DataEnd; } + inline bool IsEmpty() const { return m_Data == m_DataEnd; } void* GetData() const { return m_Data; } void* GetDataEnd() const { return m_DataEnd; } size_t GetSize() const { return reinterpret_cast<uint8_t*>(m_DataEnd) - reinterpret_cast<uint8_t*>(m_Data); } @@ -111,7 +113,7 @@ struct MutableMemoryView } /** Modifies the view by chopping the given number of bytes from the left. */ - inline constexpr void RightChopInline(uint64_t InSize) + inline void RightChopInline(uint64_t InSize) { const uint64_t Offset = zen::Min(GetSize(), InSize); m_Data = GetDataAtOffsetNoCheck(Offset); @@ -151,7 +153,7 @@ struct MutableMemoryView return View; } - inline constexpr MutableMemoryView& operator+=(size_t InSize) + inline MutableMemoryView& operator+=(size_t InSize) { RightChopInline(InSize); return *this; @@ -192,10 +194,10 @@ struct MemoryView { } - inline bool Contains(const MemoryView& Other) const { return (m_Data <= Other.m_Data) && (m_DataEnd >= Other.m_DataEnd); } - inline bool IsEmpty() const { return m_Data == m_DataEnd; } - const void* GetData() const { return m_Data; } - const void* GetDataEnd() const { return m_DataEnd; } + inline bool Contains(const MemoryView& Other) const { return (m_Data <= Other.m_Data) && (m_DataEnd >= Other.m_DataEnd); } + inline bool IsEmpty() const { return m_Data == m_DataEnd; } + const void* GetData() const { return m_Data; } + const void* GetDataEnd() const { return m_DataEnd; } size_t GetSize() const { return reinterpret_cast<const uint8_t*>(m_DataEnd) - reinterpret_cast<const uint8_t*>(m_Data); } inline bool operator==(const MemoryView& Rhs) const { return m_Data == Rhs.m_Data && m_DataEnd == Rhs.m_DataEnd; } @@ -206,14 +208,14 @@ struct MemoryView return Size == InView.GetSize() && (memcmp(m_Data, InView.GetData(), Size) == 0); } - inline constexpr MemoryView& operator+=(size_t InSize) + inline MemoryView& operator+=(size_t InSize) { RightChopInline(InSize); return *this; } /** Modifies the view by chopping the given number of bytes from the left. */ - inline constexpr void RightChopInline(uint64_t InSize) + inline void RightChopInline(uint64_t InSize) { const uint64_t Offset = std::min(GetSize(), InSize); m_Data = GetDataAtOffsetNoCheck(Offset); @@ -244,7 +246,7 @@ struct MemoryView } /** Returns the left-most part of the view by taking the given number of bytes from the left. */ - constexpr inline MemoryView Left(uint64_t InSize) const + inline MemoryView Left(uint64_t InSize) const { MemoryView View(*this); View.LeftInline(InSize); @@ -252,7 +254,7 @@ struct MemoryView } /** Modifies the view to be the given number of bytes from the left. */ - constexpr inline void LeftInline(uint64_t InSize) + inline void LeftInline(uint64_t InSize) { InSize = zen::Min(GetSize(), InSize); m_DataEnd = std::min(m_DataEnd, m_Data + InSize); @@ -296,28 +298,28 @@ MutableMemoryView::CopyFrom(MemoryView InView) const } /** Advances the start of the view by an offset, which is clamped to stay within the view. */ -constexpr inline MemoryView +inline MemoryView operator+(const MemoryView& View, uint64_t Offset) { return MemoryView(View) += Offset; } /** Advances the start of the view by an offset, which is clamped to stay within the view. */ -constexpr inline MemoryView +inline MemoryView operator+(uint64_t Offset, const MemoryView& View) { return MemoryView(View) += Offset; } /** Advances the start of the view by an offset, which is clamped to stay within the view. */ -constexpr inline MutableMemoryView +inline MutableMemoryView operator+(const MutableMemoryView& View, uint64_t Offset) { return MutableMemoryView(View) += Offset; } /** Advances the start of the view by an offset, which is clamped to stay within the view. */ -constexpr inline MutableMemoryView +inline MutableMemoryView operator+(uint64_t Offset, const MutableMemoryView& View) { return MutableMemoryView(View) += Offset; diff --git a/zencore/include/zencore/sharedbuffer.h b/zencore/include/zencore/sharedbuffer.h index b230c05b1..640c3fe74 100644 --- a/zencore/include/zencore/sharedbuffer.h +++ b/zencore/include/zencore/sharedbuffer.h @@ -12,6 +12,8 @@ namespace zen { +class SharedBuffer; + /** * Reference to a memory buffer with a single owner * diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h index de2ca71d9..2b5f20f86 100644 --- a/zencore/include/zencore/string.h +++ b/zencore/include/zencore/string.h @@ -560,15 +560,17 @@ NiceRate(uint64_t Num, uint32_t DurationMilliseconds, const char* Unit = "B") if (DurationMilliseconds) { - NiceNumToBuffer(Num * 1000 / DurationMilliseconds, Buffer); + // Leave a little of 'Buffer' for the "Unit/s" suffix + std::span<char> BufferSpan(Buffer, sizeof(Buffer) - 8); + NiceNumToBuffer(Num * 1000 / DurationMilliseconds, BufferSpan); } else { - strcpy_s(Buffer, "0"); + strcpy(Buffer, "0"); } - strcat_s(Buffer, Unit); - strcat_s(Buffer, "/s"); + strncat(Buffer, Unit, 4); + strcat(Buffer, "/s"); return Buffer; } diff --git a/zencore/include/zencore/thread.h b/zencore/include/zencore/thread.h index 30d0c0897..b18da6031 100644 --- a/zencore/include/zencore/thread.h +++ b/zencore/include/zencore/thread.h @@ -4,6 +4,10 @@ #include "zencore.h" +#if !ZEN_PLATFORM_WINDOWS +# include <shared_mutex> +#endif + namespace zen { /** @@ -60,7 +64,11 @@ public: }; private: +#if ZEN_PLATFORM_WINDOWS void* m_Srw = nullptr; +#else + std::shared_mutex m_Mutex; +#endif }; /** Basic abstraction of a simple event synchronization mechanism (aka 'binary semaphore') diff --git a/zencore/include/zencore/timer.h b/zencore/include/zencore/timer.h index c9122eb44..eb284eaee 100644 --- a/zencore/include/zencore/timer.h +++ b/zencore/include/zencore/timer.h @@ -2,10 +2,16 @@ #pragma once -#include <intrin.h> -#include <stdint.h> #include "zencore.h" +#if ZEN_COMPILER_MSC +# include <intrin.h> +#elif ZEN_ARCH_X64 +# include <x86intrin.h> +#endif + +#include <stdint.h> + namespace zen { // High frequency timers diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index 206046f0b..73446b447 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -10,24 +10,65 @@ // Platform // -#define ZEN_PLATFORM_WINDOWS 1 -#define ZEN_PLATFORM_LINUX 0 -#define ZEN_PLATFORM_MACOS 0 +#define ZEN_PLATFORM_WINDOWS 0 +#define ZEN_PLATFORM_LINUX 0 +#define ZEN_PLATFORM_MACOS 0 + +#ifdef _WIN32 +# undef ZEN_PLATFORM_WINDOWS +# define ZEN_PLATFORM_WINDOWS 1 +#elif defined(__linux__) +# undef ZEN_PLATFORM_LINUX +# define ZEN_PLATFORM_LINUX 1 +#elif defined(__APPLE__) +# undef ZEN_PLATFORM_MACOS +# define ZEN_PLATFORM_MACOS 1 +#endif ////////////////////////////////////////////////////////////////////////// // Compiler // -#ifdef _MSC_VER -# define ZEN_COMPILER_MSC 1 +#define ZEN_COMPILER_CLANG 0 +#define ZEN_COMPILER_MSC 0 +#define ZEN_COMPILER_GCC 0 + +// Clang can define __GNUC__ and/or _MSC_VER so we check for Clang first +#ifdef __clang__ +# undef ZEN_COMPILER_CLANG +# define ZEN_COMPILER_CLANG 1 +#elif defined(_MSC_VER) +# undef ZEN_COMPILER_MSC +# define ZEN_COMPILER_MSC 1 +#elif defined(__GNUC__) +# undef ZEN_COMPILER_GCC +# define ZEN_COMPILER_GCC 1 +#else +# error Unknown compiler #endif -#ifndef ZEN_COMPILER_MSC -# define ZEN_COMPILER_MSC 0 +// Check for C++20 support +#if !ZEN_PLATFORM_WINDOWS +# if ZEN_COMPILER_CLANG && __clang_major__ < 14 +# error LLVM-14 onwards required for complete C++20 support +# elif ZEN_COMPILER_GCC && __GNUC__ < 11 +# error GCC-11 onwards required for complete C++20 support +# endif #endif -#ifndef ZEN_COMPILER_CLANG -# define ZEN_COMPILER_CLANG 0 + +////////////////////////////////////////////////////////////////////////// +// Architecture +// + +#if defined(__amd64__) || defined(_M_X64) +# define ZEN_ARCH_X64 1 +# define ZEN_ARCH_ARM64 0 +#elif defined(__arm64__) || defined(_M_ARM64) +# define ZEN_ARCH_X64 0 +# define ZEN_ARCH_ARM64 1 +#else +# error Unknown architecture #endif ////////////////////////////////////////////////////////////////////////// @@ -58,7 +99,7 @@ public: AssertException(const char* Msg); ~AssertException(); - [[nodiscard]] virtual char const* what() const override { return m_Msg.c_str(); } + [[nodiscard]] virtual char const* what() const noexcept override { return m_Msg.c_str(); } private: std::string m_Msg; @@ -100,7 +141,12 @@ char (&ZenArrayCountHelper(const T (&)[N]))[N + 1]; ////////////////////////////////////////////////////////////////////////// -#define ZEN_NOINLINE __declspec(noinline) +#if ZEN_COMPILER_MSC +# define ZEN_NOINLINE __declspec(noinline) +#else +# define ZEN_NOINLINE __attribute__((noinline)) +#endif + #define ZEN_UNUSED(...) ((void)__VA_ARGS__) #define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false) #define ZENCORE_API // Placeholder to allow DLL configs in the future diff --git a/zencore/intmath.cpp b/zencore/intmath.cpp new file mode 100644 index 000000000..4039a3b39 --- /dev/null +++ b/zencore/intmath.cpp @@ -0,0 +1,56 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zencore/intmath.h> + +#include <doctest/doctest.h> + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +void +intmath_forcelink() +{ +} + +TEST_CASE("intmath") +{ + CHECK(FloorLog2(0x00) == 0); + CHECK(FloorLog2(0x01) == 0); + CHECK(FloorLog2(0x0f) == 3); + CHECK(FloorLog2(0x10) == 4); + CHECK(FloorLog2(0x11) == 4); + CHECK(FloorLog2(0x12) == 4); + CHECK(FloorLog2(0x22) == 5); + CHECK(FloorLog2(0x0001'0000) == 16); + CHECK(FloorLog2(0x0001'000f) == 16); + CHECK(FloorLog2(0x8000'0000) == 31); + + CHECK(FloorLog2_64(0x00ull) == 0); + CHECK(FloorLog2_64(0x01ull) == 0); + CHECK(FloorLog2_64(0x0full) == 3); + CHECK(FloorLog2_64(0x10ull) == 4); + CHECK(FloorLog2_64(0x11ull) == 4); + CHECK(FloorLog2_64(0x0001'0000ull) == 16); + CHECK(FloorLog2_64(0x0001'000full) == 16); + CHECK(FloorLog2_64(0x8000'0000ull) == 31); + CHECK(FloorLog2_64(0x0000'0001'0000'0000ull) == 32); + CHECK(FloorLog2_64(0x8000'0000'0000'0000ull) == 63); + + CHECK(CountLeadingZeros64(0x8000'0000'0000'0000ull) == 0); + CHECK(CountLeadingZeros64(0x0000'0000'0000'0000ull) == 64); + CHECK(CountLeadingZeros64(0x0000'0000'0000'0001ull) == 63); + CHECK(CountLeadingZeros64(0x0000'0000'8000'0000ull) == 32); + CHECK(CountLeadingZeros64(0x0000'0001'0000'0000ull) == 31); + + CHECK(CountTrailingZeros64(0x8000'0000'0000'0000ull) == 63); + CHECK(CountTrailingZeros64(0x0000'0000'0000'0000ull) == 64); + CHECK(CountTrailingZeros64(0x0000'0000'0000'0001ull) == 0); + CHECK(CountTrailingZeros64(0x0000'0000'8000'0000ull) == 31); + CHECK(CountTrailingZeros64(0x0000'0001'0000'0000ull) == 32); +} + +} // namespace zen diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index 9c7781663..758cf539c 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -6,12 +6,12 @@ #include <memory.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> +#include <zencore/logging.h> #include <zencore/memory.h> #include <zencore/thread.h> #include <system_error> #include <atlfile.h> -#include <spdlog/spdlog.h> #include <gsl/gsl-lite.hpp> namespace zen { @@ -165,7 +165,7 @@ IoBufferExtendedCore::~IoBufferExtendedCore() if (!Success) { - spdlog::warn("Error reported on file handle close!"); + ZEN_WARN("Error reported on file handle close!"); } } @@ -220,7 +220,7 @@ IoBufferExtendedCore::Materialize() const { throw std::system_error( std::error_code(::GetLastError(), std::system_category()), - "MapViewOfFile failed (offset {#x}, size {#x}) file: '{}'"_format(MapOffset, MapSize, zen::PathFromHandle(m_FileHandle))); + "MapViewOfFile failed (offset {:#x}, size {:#x}) file: '{}'"_format(MapOffset, MapSize, zen::PathFromHandle(m_FileHandle))); } m_MappedPointer = MappedBase; diff --git a/zencore/logging.cpp b/zencore/logging.cpp index 6441fc3bc..c5c0b6446 100644 --- a/zencore/logging.cpp +++ b/zencore/logging.cpp @@ -4,25 +4,36 @@ #include <spdlog/sinks/stdout_color_sinks.h> +namespace zen { + +// We shadow the underlying spdlog default logger, in order to avoid a bunch of overhead +spdlog::logger* TheDefaultLogger; + +} // namespace zen + namespace zen::logging { spdlog::logger& Default() { - return *spdlog::default_logger(); + return *TheDefaultLogger; +} + +void +SetDefault(std::shared_ptr<spdlog::logger> NewDefaultLogger) +{ + spdlog::set_default_logger(NewDefaultLogger); + TheDefaultLogger = spdlog::default_logger_raw(); } spdlog::logger& Get(std::string_view Name) { std::shared_ptr<spdlog::logger> Logger = spdlog::get(std::string(Name)); + if (!Logger) { - Logger = std::make_shared<spdlog::logger>(std::string(Name), - begin(spdlog::default_logger()->sinks()), - end(spdlog::default_logger()->sinks())); - - Logger->set_level(spdlog::default_logger()->level()); + Logger = Default().clone(std::string(Name)); spdlog::register_logger(Logger); } @@ -47,6 +58,7 @@ ConsoleLog() void InitializeLogging() { + TheDefaultLogger = spdlog::default_logger_raw(); } void diff --git a/zencore/memory.cpp b/zencore/memory.cpp index 63d61f5e1..9c7fb8333 100644 --- a/zencore/memory.cpp +++ b/zencore/memory.cpp @@ -1,15 +1,47 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include <malloc.h> #include <zencore/intmath.h> #include <zencore/memory.h> +#ifdef ZEN_PLATFORM_WINDOWS +# include <malloc.h> +#else +# include <cstdlib> +#endif + #include <doctest/doctest.h> namespace zen { ////////////////////////////////////////////////////////////////////////// +static void* AlignedAllocImpl(size_t size, size_t alignment) +{ +#if ZEN_PLATFORM_WINDOWS + // return _aligned_malloc(size, alignment); // MSVC alternative + return _mm_malloc(size, alignment); +#else + // posix_memalign(&Ret, Alignment, Size); // Apple, AndroidApi<28 + return std::aligned_alloc(alignment, size); +#endif +} + +void AlignedFreeImpl(void* ptr) +{ + if (ptr == nullptr) + return; + +#if ZEN_PLATFORM_WINDOWS + // _aligned_free(ptr); MSVC alternative + _mm_free(ptr); +#else + // free(ptr) // Apple :) + std::free(ptr); +#endif +} + +////////////////////////////////////////////////////////////////////////// + MemoryArena::MemoryArena() { } @@ -21,14 +53,13 @@ MemoryArena::~MemoryArena() void* MemoryArena::Alloc(size_t size, size_t alignment) { - return _mm_malloc(size, alignment); + return AlignedAllocImpl(size, alignment); } void MemoryArena::Free(void* ptr) { - if (ptr) - _mm_free(ptr); + AlignedFreeImpl(ptr); } ////////////////////////////////////////////////////////////////////////// @@ -36,14 +67,13 @@ MemoryArena::Free(void* ptr) void* Memory::Alloc(size_t size, size_t alignment) { - return _mm_malloc(size, alignment); + return AlignedAllocImpl(size, alignment); } void Memory::Free(void* ptr) { - if (ptr) - _mm_free(ptr); + AlignedFreeImpl(ptr); } ////////////////////////////////////////////////////////////////////////// diff --git a/zencore/sharedbuffer.cpp b/zencore/sharedbuffer.cpp index a14da1bf7..2761d0b4d 100644 --- a/zencore/sharedbuffer.cpp +++ b/zencore/sharedbuffer.cpp @@ -5,7 +5,6 @@ #include <doctest/doctest.h> #include <memory.h> -#include <atlfile.h> #include <gsl/gsl-lite.hpp> namespace zen { diff --git a/zencore/snapshot_manifest.cpp b/zencore/snapshot_manifest.cpp index 7d0769d13..87625fb7f 100644 --- a/zencore/snapshot_manifest.cpp +++ b/zencore/snapshot_manifest.cpp @@ -13,7 +13,9 @@ // Used for getting My Documents for default snapshot dir #include <ShlObj.h> -#pragma comment(lib, "shell32.lib") +#if ZEN_PLATFORM_WINDOWS +# pragma comment(lib, "shell32.lib") +#endif namespace zen { diff --git a/zencore/stream.cpp b/zencore/stream.cpp index bd925ebc4..8687d5501 100644 --- a/zencore/stream.cpp +++ b/zencore/stream.cpp @@ -5,7 +5,7 @@ #include <zencore/memory.h> #include <zencore/stream.h> #include <algorithm> -#include <exception> +#include <stdexcept> namespace zen { @@ -22,7 +22,7 @@ MemoryInStream::Read(void* buffer, size_t byteCount, uint64_t offset) const size_t needEnd = offset + byteCount; if (needEnd > m_Buffer.size()) - throw std::exception("read past end of file!"); // TODO: better exception + throw std::runtime_error("read past end of file!"); // TODO: better exception memcpy(buffer, m_Buffer.data() + offset, byteCount); } @@ -95,7 +95,11 @@ TextWriter& operator<<(TextWriter& writer, int8_t value) { char buffer[16]; +#if ZEN_PLATFORM_WINDOWS _itoa_s(value, buffer, 10); +#else + sprintf(buffer, "%d", value); +#endif writer << buffer; return writer; } @@ -104,7 +108,11 @@ TextWriter& operator<<(TextWriter& writer, int16_t value) { char buffer[16]; +#if ZEN_PLATFORM_WINDOWS _itoa_s(value, buffer, 10); +#else + sprintf(buffer, "%d", value); +#endif writer << buffer; return writer; } @@ -113,7 +121,11 @@ TextWriter& operator<<(TextWriter& writer, int32_t value) { char buffer[16]; +#if ZEN_PLATFORM_WINDOWS _itoa_s(value, buffer, 10); +#else + sprintf(buffer, "%d", value); +#endif writer << buffer; return writer; } @@ -122,7 +134,11 @@ TextWriter& operator<<(TextWriter& writer, int64_t value) { char buffer[32]; +#if ZEN_PLATFORM_WINDOWS _i64toa_s(value, buffer, sizeof buffer, 10); +#else + sprintf(buffer, "%" PRId64, value); +#endif writer << buffer; return writer; } @@ -131,7 +147,11 @@ TextWriter& operator<<(TextWriter& writer, uint8_t value) { char buffer[16]; +#if ZEN_PLATFORM_WINDOWS _ultoa_s(value, buffer, 10); +#else + sprintf(buffer, "%u", value); +#endif writer << buffer; return writer; } @@ -140,7 +160,11 @@ TextWriter& operator<<(TextWriter& writer, uint16_t value) { char buffer[16]; +#if ZEN_PLATFORM_WINDOWS _ultoa_s(value, buffer, 10); +#else + sprintf(buffer, "%u", value); +#endif writer << buffer; return writer; } @@ -149,7 +173,11 @@ TextWriter& operator<<(TextWriter& writer, uint32_t value) { char buffer[16]; +#if ZEN_PLATFORM_WINDOWS _ultoa_s(value, buffer, 10); +#else + sprintf(buffer, "%u", value); +#endif writer << buffer; return writer; } @@ -158,7 +186,11 @@ TextWriter& operator<<(TextWriter& writer, uint64_t value) { char buffer[32]; +#if ZEN_PLATFORM_WINDOWS _ui64toa_s(value, buffer, sizeof buffer, 10); +#else + sprintf(buffer, "%" PRIu64, value); +#endif writer << buffer; return writer; } diff --git a/zencore/string.cpp b/zencore/string.cpp index 21ba5b204..8ea10d2a3 100644 --- a/zencore/string.cpp +++ b/zencore/string.cpp @@ -2,11 +2,13 @@ #include <doctest/doctest.h> #include <inttypes.h> +#include <math.h> #include <stdio.h> #include <zencore/memory.h> #include <zencore/string.h> #include <exception> #include <ostream> +#include <stdexcept> #include <utf8.h> @@ -34,14 +36,14 @@ namespace zen { bool ToString(std::span<char> Buffer, uint64_t Num) { - snprintf(Buffer.data(), Buffer.size(), "%I64u", Num); + snprintf(Buffer.data(), Buffer.size(), "%" PRIu64, Num); return true; } bool ToString(std::span<char> Buffer, int64_t Num) { - snprintf(Buffer.data(), Buffer.size(), "%I64d", Num); + snprintf(Buffer.data(), Buffer.size(), "%" PRId64, Num); return true; } @@ -231,12 +233,12 @@ NiceNumGeneral(uint64_t Num, std::span<char> Buffer, NicenumFormat Format) switch (Format) { case kNicenumRaw: - return snprintf(Buffer.data(), Buffer.size(), "%llu", (uint64_t)Num); + return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64, (uint64_t)Num); case kNicenumRawTime: if (Num > 0) { - return snprintf(Buffer.data(), Buffer.size(), "%llu", (uint64_t)Num); + return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64, (uint64_t)Num); } else { @@ -275,7 +277,7 @@ NiceNumGeneral(uint64_t Num, std::span<char> Buffer, NicenumFormat Format) * If this is an even multiple of the base, always display * without any decimal precision. */ - return snprintf(Buffer.data(), Buffer.size(), "%llu%s", (uint64_t)n, u); + return snprintf(Buffer.data(), Buffer.size(), "%" PRIu64 "%s", (uint64_t)n, u); } else { @@ -437,7 +439,7 @@ template<typename C> [[noreturn]] void StringBuilderImpl<C>::Fail(const char* reason) { - throw std::exception(reason); + throw std::runtime_error(reason); } // Instantiate templates once diff --git a/zencore/thread.cpp b/zencore/thread.cpp index fa9da0258..598466bb4 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -5,33 +5,53 @@ #include <fmt/format.h> #include <zencore/except.h> #include <zencore/string.h> -#include <zencore/windows.h> -#include <thread> + +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#elif ZEN_PLATFORM_LINUX +# include <unistd.h> +#endif namespace zen { void RwLock::AcquireShared() { +#if ZEN_PLATFORM_WINDOWS AcquireSRWLockShared((PSRWLOCK)&m_Srw); +#else + m_Mutex.lock_shared(); +#endif } void RwLock::ReleaseShared() { +#if ZEN_PLATFORM_WINDOWS ReleaseSRWLockShared((PSRWLOCK)&m_Srw); +#else + m_Mutex.unlock_shared(); +#endif } void RwLock::AcquireExclusive() { +#if ZEN_PLATFORM_WINDOWS AcquireSRWLockExclusive((PSRWLOCK)&m_Srw); +#else + m_Mutex.lock(); +#endif } void RwLock::ReleaseExclusive() { +#if ZEN_PLATFORM_WINDOWS ReleaseSRWLockExclusive((PSRWLOCK)&m_Srw); +#else + m_Mutex.unlock(); +#endif } Event::Event() @@ -253,13 +273,21 @@ IsProcessRunning(int pid) int GetCurrentProcessId() { +#if ZEN_PLATFORM_WINDOWS return ::GetCurrentProcessId(); +#else + return getpid(); +#endif } void Sleep(int ms) { +#if ZEN_PLATFORM_WINDOWS ::Sleep(ms); +#else + usleep(ms * 1000U); +#endif } ////////////////////////////////////////////////////////////////////////// diff --git a/zencore/timer.cpp b/zencore/timer.cpp index ee8e1cf9c..08b5e06d2 100644 --- a/zencore/timer.cpp +++ b/zencore/timer.cpp @@ -3,26 +3,41 @@ #include <doctest/doctest.h> #include <zencore/thread.h> #include <zencore/timer.h> -#include <zencore/windows.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#elif ZEN_PLATFORM_LINUX +# include <time.h> +# include <unistd.h> +#endif namespace zen { uint64_t GetHifreqTimerValue() { +#if ZEN_PLATFORM_WINDOWS LARGE_INTEGER li; QueryPerformanceCounter(&li); return li.QuadPart; +#else + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t(ts.tv_sec) * 1000000ull) + (uint64_t(ts.tv_nsec) / 1000ull); +#endif } uint64_t internalGetHifreqTimerFrequency() { +#if ZEN_PLATFORM_WINDOWS LARGE_INTEGER li; QueryPerformanceFrequency(&li); return li.QuadPart; +#else + return 1000000ull; +#endif } static uint64_t qpcFreq = internalGetHifreqTimerFrequency(); @@ -56,7 +71,7 @@ TEST_CASE("Timer") { uint64_t s0 = GetHifreqTimerValue(); uint64_t t0 = GetCpuTimerValue(); - Sleep(1000); + zen::Sleep(1000); uint64_t s1 = GetHifreqTimerValue(); uint64_t t1 = GetCpuTimerValue(); // double r = double(t1 - t0) / (s1 - s0); diff --git a/zencore/uid.cpp b/zencore/uid.cpp index 644d0aa77..acf9f9790 100644 --- a/zencore/uid.cpp +++ b/zencore/uid.cpp @@ -100,6 +100,7 @@ TEST_CASE("Oid") SUBCASE("Basic") { Oid id1 = Oid::NewOid(); + ZEN_UNUSED(id1); std::vector<Oid> ids; std::set<Oid> idset; diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp index c2a57b653..c53fd218f 100644 --- a/zencore/zencore.cpp +++ b/zencore/zencore.cpp @@ -2,7 +2,13 @@ #include <zencore/zencore.h> +#if ZEN_PLATFORM_WINDOWS #include <zencore/windows.h> +#endif + +#if ZEN_PLATFORM_LINUX +#include <pthread.h> +#endif #include <zencore/blake3.h> #include <zencore/compactbinary.h> @@ -25,12 +31,26 @@ bool IsPointerToStack(const void* ptr) { +#if ZEN_PLATFORM_WINDOWS ULONG_PTR low, high; GetCurrentThreadStackLimits(&low, &high); const uintptr_t intPtr = reinterpret_cast<uintptr_t>(ptr); return (intPtr - low) < (high - low); +#elif ZEN_PLATFORM_LINUX + pthread_t self = pthread_self(); + + pthread_attr_t attr; + pthread_getattr_np(self, &attr); + + void* low; + size_t size; + pthread_attr_getstack(&attr, &low, &size); + + return (uintptr_t(ptr) - uintptr_t(low)) < uintptr_t(size); +#elif 0 +#endif } zen::AssertException::AssertException(const char* Msg) : m_Msg(Msg) @@ -65,6 +85,7 @@ zencore_forcelinktests() zen::blake3_forcelink(); zen::compositebuffer_forcelink(); zen::compress_forcelink(); + zen::intmath_forcelink(); zen::iobuffer_forcelink(); zen::memory_forcelink(); zen::refcount_forcelink(); diff --git a/zencore/zencore.vcxproj b/zencore/zencore.vcxproj index 4b5bba185..4f1e63670 100644 --- a/zencore/zencore.vcxproj +++ b/zencore/zencore.vcxproj @@ -160,6 +160,7 @@ <ClCompile Include="crc32.cpp" /> <ClCompile Include="except.cpp" /> <ClCompile Include="filesystem.cpp" /> + <ClCompile Include="intmath.cpp" /> <ClCompile Include="iohash.cpp" /> <ClCompile Include="logging.cpp" /> <ClCompile Include="md5.cpp" /> diff --git a/zencore/zencore.vcxproj.filters b/zencore/zencore.vcxproj.filters index 70c1cfb01..de3d915b8 100644 --- a/zencore/zencore.vcxproj.filters +++ b/zencore/zencore.vcxproj.filters @@ -71,6 +71,7 @@ <ClCompile Include="compositebuffer.cpp" /> <ClCompile Include="crc32.cpp" /> <ClCompile Include="logging.cpp" /> + <ClCompile Include="intmath.cpp" /> </ItemGroup> <ItemGroup> <Filter Include="CAS"> diff --git a/zenfs_common.props b/zenfs_common.props index f550ff929..67894b9e8 100644 --- a/zenfs_common.props +++ b/zenfs_common.props @@ -11,7 +11,7 @@ <WarningLevel>Level4</WarningLevel> <MultiProcessorCompilation>true</MultiProcessorCompilation> <TreatWarningAsError>true</TreatWarningAsError> - <PreprocessorDefinitions>_UNICODE;UNICODE;_WIN32_WINNT=0x0A00 ;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;_UNICODE;UNICODE;_WIN32_WINNT=0x0A00 ;%(PreprocessorDefinitions)</PreprocessorDefinitions> </ClCompile> <Link> <IgnoreSpecificDefaultLibraries>msvcrt</IgnoreSpecificDefaultLibraries> diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index 9ee004c5c..8ae4fa602 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -418,7 +418,7 @@ HttpMessageResponseRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfB if (IoResult) { - spdlog::warn("response aborted due to error: '{}'", GetWindowsErrorAsString(IoResult)); + ZEN_WARN("response aborted due to error: '{}'", GetWindowsErrorAsString(IoResult)); // if one transmit failed there's really no need to go on return nullptr; @@ -554,7 +554,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) CancelThreadpoolIo(Iocp); - spdlog::error("failed to send HTTP response (error: '{}'), request URL: {}"sv, SendResult, HttpReq->pRawUrl); + ZEN_ERROR("failed to send HTTP response (error: '{}'), request URL: {}", SendResult, HttpReq->pRawUrl); ErrorCode = MakeWin32ErrorCode(SendResult); } @@ -605,7 +605,7 @@ HttpSysServer::Initialize(const wchar_t* UrlPath) if (Result != NO_ERROR) { - spdlog::error("Failed to create server session for '{}': {x}"sv, WideToUtf8(UrlPath), Result); + ZEN_ERROR("Failed to create server session for '{}': {:#x}", WideToUtf8(UrlPath), Result); return; } @@ -614,7 +614,7 @@ HttpSysServer::Initialize(const wchar_t* UrlPath) if (Result != NO_ERROR) { - spdlog::error("Failed to create URL group for '{}': {x}"sv, WideToUtf8(UrlPath), Result); + ZEN_ERROR("Failed to create URL group for '{}': {:#x}", WideToUtf8(UrlPath), Result); return; } @@ -625,7 +625,7 @@ HttpSysServer::Initialize(const wchar_t* UrlPath) if (Result != NO_ERROR) { - spdlog::error("Failed to add base URL to URL group for '{}': {x}"sv, WideToUtf8(UrlPath), Result); + ZEN_ERROR("Failed to add base URL to URL group for '{}': {:#x}", WideToUtf8(UrlPath), Result); return; } @@ -640,7 +640,7 @@ HttpSysServer::Initialize(const wchar_t* UrlPath) if (Result != NO_ERROR) { - spdlog::error("Failed to create request queue for '{}': {x}"sv, WideToUtf8(UrlPath), Result); + ZEN_ERROR("Failed to create request queue for '{}': {:#x}", WideToUtf8(UrlPath), Result); return; } @@ -652,7 +652,7 @@ HttpSysServer::Initialize(const wchar_t* UrlPath) if (Result != NO_ERROR) { - spdlog::error("Failed to set server binding property for '{}': {x}"sv, WideToUtf8(UrlPath), Result); + ZEN_ERROR("Failed to set server binding property for '{}': {:#x}", WideToUtf8(UrlPath), Result); return; } @@ -664,7 +664,7 @@ HttpSysServer::Initialize(const wchar_t* UrlPath) if (ErrorCode) { - spdlog::error("Failed to create IOCP for '{}': {}"sv, WideToUtf8(UrlPath), ErrorCode.message()); + ZEN_ERROR("Failed to create IOCP for '{}': {}", WideToUtf8(UrlPath), ErrorCode.message()); } else { @@ -803,7 +803,7 @@ HttpSysServer::RegisterService(const char* UrlPath, HttpService& Service) if (Result != NO_ERROR) { - spdlog::error("HttpAddUrlToUrlGroup failed with result {}"sv, Result); + ZEN_ERROR("HttpAddUrlToUrlGroup failed with result {}", Result); return; } @@ -829,7 +829,7 @@ HttpSysServer::UnregisterService(const char* UrlPath, HttpService& Service) if (Result != NO_ERROR) { - spdlog::error("HttpRemoveUrlFromUrlGroup failed with result {}"sv, Result); + ZEN_ERROR("HttpRemoveUrlFromUrlGroup failed with result {}", Result); } } @@ -917,7 +917,7 @@ HttpSysTransaction::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTran if (ErrorCode) { - spdlog::error("IssueRequest() failed {}"sv, ErrorCode.message()); + ZEN_ERROR("IssueRequest() failed {}", ErrorCode.message()); } else { @@ -926,7 +926,7 @@ HttpSysTransaction::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTran } catch (std::exception& Ex) { - spdlog::error("exception caught from IssueRequest(): {}"sv, Ex.what()); + ZEN_ERROR("exception caught from IssueRequest(): {}", Ex.what()); // something went wrong, no request is pending } @@ -1233,7 +1233,7 @@ InitialRequestHandler::IssueRequest(std::error_code& ErrorCode) ErrorCode = MakeWin32ErrorCode(HttpApiResult); - spdlog::error("HttpReceiveHttpRequest failed, error {}", ErrorCode.message()); + ZEN_ERROR("HttpReceiveHttpRequest failed, error {}", ErrorCode.message()); return; } @@ -1273,45 +1273,45 @@ InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesT { const HTTP_REQUEST_TIMING_INFO* TimingInfo = reinterpret_cast<HTTP_REQUEST_TIMING_INFO*>(ReqInfo.pInfo); - spdlog::info(""); + ZEN_INFO(""); } break; case HttpRequestInfoTypeAuth: - spdlog::info(""); + ZEN_INFO(""); break; case HttpRequestInfoTypeChannelBind: - spdlog::info(""); + ZEN_INFO(""); break; case HttpRequestInfoTypeSslProtocol: - spdlog::info(""); + ZEN_INFO(""); break; case HttpRequestInfoTypeSslTokenBindingDraft: - spdlog::info(""); + ZEN_INFO(""); break; case HttpRequestInfoTypeSslTokenBinding: - spdlog::info(""); + ZEN_INFO(""); break; case HttpRequestInfoTypeTcpInfoV0: { const TCP_INFO_v0* TcpInfo = reinterpret_cast<const TCP_INFO_v0*>(ReqInfo.pInfo); - spdlog::info(""); + ZEN_INFO(""); } break; case HttpRequestInfoTypeRequestSizing: { const HTTP_REQUEST_SIZING_INFO* SizingInfo = reinterpret_cast<const HTTP_REQUEST_SIZING_INFO*>(ReqInfo.pInfo); - spdlog::info(""); + ZEN_INFO(""); } break; case HttpRequestInfoTypeQuicStats: - spdlog::info(""); + ZEN_INFO(""); break; case HttpRequestInfoTypeTcpInfoV1: { const TCP_INFO_v1* TcpInfo = reinterpret_cast<const TCP_INFO_v1*>(ReqInfo.pInfo); - spdlog::info(""); + ZEN_INFO(""); } break; } diff --git a/zenserver-test/projectclient.cpp b/zenserver-test/projectclient.cpp index 14d7140ec..2700ae9da 100644 --- a/zenserver-test/projectclient.cpp +++ b/zenserver-test/projectclient.cpp @@ -3,11 +3,11 @@ #include "projectclient.h" #include <zencore/compactbinary.h> +#include <zencore/logging.h> #include <zencore/sharedbuffer.h> #include <zencore/string.h> #include <zencore/zencore.h> -#include <spdlog/spdlog.h> #include <asio.hpp> #include <gsl/gsl-lite.hpp> @@ -39,7 +39,7 @@ struct ProjectClientConnection if (hPipe == INVALID_HANDLE_VALUE) { - spdlog::warn("failed while creating named pipe {}", WideToUtf8(PipeName)); + ZEN_WARN("failed while creating named pipe {}", WideToUtf8(PipeName)); throw std::system_error(GetLastError(), std::system_category(), "Failed to open named pipe '{}'"_format(WideToUtf8(PipeName))); } diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index efcbf5da8..df17cc353 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -10,6 +10,7 @@ #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/iohash.h> +#include <zencore/logging.h> #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/timer.h> @@ -26,8 +27,6 @@ #endif #include <cpr/cpr.h> -#include <spdlog/spdlog.h> - #include <ppl.h> #include <atomic> #include <filesystem> @@ -123,7 +122,7 @@ public: private: void Reset() {} - void OnError(const std::error_code& Error) { spdlog::error("HTTP client error! '{}'", Error.message()); } + void OnError(const std::error_code& Error) { ZEN_ERROR("HTTP client error! '{}'", Error.message()); } int OnHeader(const char* Data, size_t Bytes) { @@ -137,7 +136,7 @@ private: } int OnHeadersComplete() { - spdlog::debug("Headers complete"); + ZEN_DEBUG("Headers complete"); return 0; } int OnMessageComplete() @@ -198,7 +197,7 @@ private: { // Something bad! - spdlog::error("parse error {}", (uint32_t)m_HttpParser.http_errno); + ZEN_ERROR("parse error {}", (uint32_t)m_HttpParser.http_errno); } switch (m_RequestState) @@ -613,7 +612,7 @@ main() if (Result.status_code != 200) { - spdlog::warn("request response: {}", Result.status_code); + ZEN_WARN("request response: {}", Result.status_code); } } }; @@ -631,7 +630,7 @@ main() // cpr::Response r = cpr::Get(cpr::Url{ "http://localhost:1337/test/hello" }); - spdlog::info("{} requests in {} ({})", + ZEN_INFO("{} requests in {} ({})", RequestCount, zen::NiceTimeSpanMs(timer.getElapsedTimeMs()), zen::NiceRate(RequestCount, (uint32_t)timer.getElapsedTimeMs(), "req")); @@ -668,7 +667,7 @@ main(int argc, char** argv) TestEnv.InitializeForTest(ProgramBaseDir, TestBaseDir); - spdlog::info("Running tests...(base dir: '{}')", TestBaseDir); + ZEN_INFO("Running tests...(base dir: '{}')", TestBaseDir); return doctest::Context(argc, argv).run(); } @@ -681,7 +680,7 @@ TEST_CASE("asio.http") Instance.SetTestDir(TestDir); Instance.SpawnServer(13337); - spdlog::info("Waiting..."); + ZEN_INFO("Waiting..."); Instance.WaitUntilReady(); @@ -704,20 +703,20 @@ TEST_CASE("default.single") Instance.SetTestDir(TestDir); Instance.SpawnServer(13337); - spdlog::info("Waiting..."); + ZEN_INFO("Waiting..."); Instance.WaitUntilReady(); std::atomic<uint64_t> RequestCount{0}; std::atomic<uint64_t> BatchCounter{0}; - spdlog::info("Running single server test..."); + ZEN_INFO("Running single server test..."); auto IssueTestRequests = [&] { const uint64_t BatchNo = BatchCounter.fetch_add(1); const DWORD ThreadId = GetCurrentThreadId(); - spdlog::info("query batch {} started (thread {})", BatchNo, ThreadId); + ZEN_INFO("query batch {} started (thread {})", BatchNo, ThreadId); cpr::Session cli; cli.SetUrl(cpr::Url{"http://localhost:13337/test/hello"}); @@ -726,7 +725,7 @@ TEST_CASE("default.single") auto res = cli.Get(); ++RequestCount; } - spdlog::info("query batch {} ended (thread {})", BatchNo, ThreadId); + ZEN_INFO("query batch {} ended (thread {})", BatchNo, ThreadId); }; auto fun10 = [&] { @@ -758,10 +757,7 @@ TEST_CASE("default.single") uint64_t Elapsed = timer.getElapsedTimeMs(); - spdlog::info("{} requests in {} ({})", - RequestCount, - zen::NiceTimeSpanMs(Elapsed), - zen::NiceRate(RequestCount, (uint32_t)Elapsed, "req")); + ZEN_INFO("{} requests in {} ({})", RequestCount, zen::NiceTimeSpanMs(Elapsed), zen::NiceRate(RequestCount, (uint32_t)Elapsed, "req")); } TEST_CASE("multi.basic") @@ -776,7 +772,7 @@ TEST_CASE("multi.basic") Instance2.SetTestDir(TestDir2); Instance2.SpawnServer(13338); - spdlog::info("Waiting..."); + ZEN_INFO("Waiting..."); Instance1.WaitUntilReady(); Instance2.WaitUntilReady(); @@ -788,7 +784,7 @@ TEST_CASE("multi.basic") const uint64_t BatchNo = BatchCounter.fetch_add(1); const DWORD ThreadId = GetCurrentThreadId(); - spdlog::info("query batch {} started (thread {}) for port {}", BatchNo, ThreadId, PortNumber); + ZEN_INFO("query batch {} started (thread {}) for port {}", BatchNo, ThreadId, PortNumber); cpr::Session cli; cli.SetUrl(cpr::Url{"http://localhost:{}/test/hello"_format(PortNumber)}); @@ -798,12 +794,12 @@ TEST_CASE("multi.basic") auto res = cli.Get(); ++RequestCount; } - spdlog::info("query batch {} ended (thread {})", BatchNo, ThreadId); + ZEN_INFO("query batch {} ended (thread {})", BatchNo, ThreadId); }; zen::Stopwatch timer; - spdlog::info("Running multi-server test..."); + ZEN_INFO("Running multi-server test..."); Concurrency::parallel_invoke([&] { IssueTestRequests(13337); }, [&] { IssueTestRequests(13338); }, @@ -812,10 +808,7 @@ TEST_CASE("multi.basic") uint64_t Elapsed = timer.getElapsedTimeMs(); - spdlog::info("{} requests in {} ({})", - RequestCount, - zen::NiceTimeSpanMs(Elapsed), - zen::NiceRate(RequestCount, (uint32_t)Elapsed, "req")); + ZEN_INFO("{} requests in {} ({})", RequestCount, zen::NiceTimeSpanMs(Elapsed), zen::NiceRate(RequestCount, (uint32_t)Elapsed, "req")); } TEST_CASE("cas.basic") @@ -888,10 +881,10 @@ TEST_CASE("cas.basic") uint64_t Elapsed = timer.getElapsedTimeMs(); - spdlog::info("{} requests in {} ({})", - RequestCount, - zen::NiceTimeSpanMs(Elapsed), - zen::NiceRate(RequestCount, (uint32_t)Elapsed, "req")); + ZEN_INFO("{} requests in {} ({})", + RequestCount, + zen::NiceTimeSpanMs(Elapsed), + zen::NiceRate(RequestCount, (uint32_t)Elapsed, "req")); } // Verify that the data persists between process runs (the previous server has exited at this point) @@ -1058,17 +1051,14 @@ TEST_CASE("project.basic") CHECK(Response.text.size() == 10); } - spdlog::info("+++++++"); + ZEN_INFO("+++++++"); } - SUBCASE("build store op commit") { spdlog::info("-------"); } + SUBCASE("build store op commit") { ZEN_INFO("-------"); } } const uint64_t Elapsed = timer.getElapsedTimeMs(); - spdlog::info("{} requests in {} ({})", - RequestCount, - zen::NiceTimeSpanMs(Elapsed), - zen::NiceRate(RequestCount, (uint32_t)Elapsed, "req")); + ZEN_INFO("{} requests in {} ({})", RequestCount, zen::NiceTimeSpanMs(Elapsed), zen::NiceRate(RequestCount, (uint32_t)Elapsed, "req")); } # if 0 // this is extremely WIP @@ -1239,12 +1229,12 @@ struct RemoteExecutionRequest if (CasResponse.status_code >= 300) { - spdlog::error("CAS put failed with {}", CasResponse.status_code); + ZEN_ERROR("CAS put failed with {}", CasResponse.status_code); } } else { - spdlog::error("unknown hash in 'need' list: {}", NeedHash); + ZEN_ERROR("unknown hash in 'need' list: {}", NeedHash); } } } @@ -1260,7 +1250,7 @@ struct RemoteExecutionRequest return zen::LoadCompactBinaryObject(std::move(Payload)); } - spdlog::info("job exec: {}", JobResponse.status_code); + ZEN_INFO("job exec: {}", JobResponse.status_code); return {}; } @@ -1280,7 +1270,7 @@ private: zen::IoHash Hash = Ios.GetHash(); std::wstring RelativePath = FullPath.lexically_relative(m_RootPath).native(); - // spdlog::info("File: {:32} => {} ({})", zen::WideToUtf8(RelativePath), Hash, FileSize); + // ZEN_INFO("File: {:32} => {} ({})", zen::WideToUtf8(RelativePath), Hash, FileSize); FileEntry& Entry = m_Files[RelativePath]; Entry.Hash = Hash; @@ -1360,7 +1350,7 @@ TEST_CASE("mesh.basic") const int kInstanceCount = 4; - spdlog::info("spawning {} instances", kInstanceCount); + ZEN_INFO("spawning {} instances", kInstanceCount); std::unique_ptr<ZenServerInstance> Instances[kInstanceCount]; @@ -1395,7 +1385,7 @@ public: void SpawnServers(auto&& Callback) { - spdlog::info("{}: spawning {} server instances", m_HelperId, m_ServerCount); + ZEN_INFO("{}: spawning {} server instances", m_HelperId, m_ServerCount); m_Instances.resize(m_ServerCount); diff --git a/zenserver/cache/cachestore.cpp b/zenserver/cache/cachestore.cpp index 9180835d9..2fc253a07 100644 --- a/zenserver/cache/cachestore.cpp +++ b/zenserver/cache/cachestore.cpp @@ -4,6 +4,7 @@ #include <zencore/crc32.h> #include <zencore/except.h> +#include <zencore/logging.h> #include <zencore/windows.h> #include <zencore/filesystem.h> @@ -16,7 +17,6 @@ #include <zenstore/caslog.h> #include <fmt/core.h> -#include <spdlog/spdlog.h> #include <concepts> #include <filesystem> #include <gsl/gsl-lite.hpp> @@ -85,7 +85,7 @@ FileCacheStore::FileCacheStore(const char* RootDir, const char* ReadRootDir) { // Ensure root directory exists - create if it doesn't exist already - spdlog::info("Initializing FileCacheStore at '{}'", std::string_view(RootDir)); + ZEN_INFO("Initializing FileCacheStore at '{}'", std::string_view(RootDir)); m_RootDir = RootDir; @@ -98,7 +98,7 @@ FileCacheStore::FileCacheStore(const char* RootDir, const char* ReadRootDir) ExtendableStringBuilder<256> Name; WideToUtf8(m_RootDir.c_str(), Name); - spdlog::error("Could not open file cache directory '{}' for writing ({})", Name.c_str(), ErrorCode.message()); + ZEN_ERROR("Could not open file cache directory '{}' for writing ({})", Name.c_str(), ErrorCode.message()); m_IsOk = false; } @@ -109,7 +109,7 @@ FileCacheStore::FileCacheStore(const char* RootDir, const char* ReadRootDir) if (std::filesystem::exists(m_ReadRootDir, ErrorCode)) { - spdlog::info("FileCacheStore will use additional read tree at '{}'", std::string_view(ReadRootDir)); + ZEN_INFO("FileCacheStore will use additional read tree at '{}'", std::string_view(ReadRootDir)); m_ReadRootIsValid = true; } @@ -145,7 +145,7 @@ FileCacheStore::Get(std::string_view Key, CacheValue& OutValue) if (FAILED(hRes)) { - spdlog::debug("GET MISS {}", Key); + ZEN_DEBUG("GET MISS {}", Key); return false; } @@ -162,7 +162,7 @@ FileCacheStore::Get(std::string_view Key, CacheValue& OutValue) OutValue.Value = IoBuffer(IoBuffer::File, File.Detach(), 0, FileSize); - spdlog::debug("GET HIT {}", Key); + ZEN_DEBUG("GET HIT {}", Key); return true; } @@ -180,7 +180,7 @@ FileCacheStore::Put(std::string_view Key, const CacheValue& Value) CAtlTemporaryFile File; - spdlog::debug("PUT {}", Key); + ZEN_DEBUG("PUT {}", Key); HRESULT hRes = File.Create(m_RootDir.c_str()); @@ -205,11 +205,11 @@ FileCacheStore::Put(std::string_view Key, const CacheValue& Value) if (FAILED(hRes)) { - spdlog::warn("Failed to rename temp file for key '{}' - deleting temporary file", Key); + ZEN_WARN("Failed to rename temp file for key '{}' - deleting temporary file", Key); if (!DeleteFile(File.TempFileName())) { - spdlog::warn("Temp file for key '{}' could not be deleted - no value persisted", Key); + ZEN_WARN("Temp file for key '{}' could not be deleted - no value persisted", Key); } } } diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index b2f8d191c..9600c5f8a 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -4,6 +4,7 @@ #include <zencore/compactbinaryvalidation.h> #include <zencore/compress.h> #include <zencore/fmtutils.h> +#include <zencore/logging.h> #include <zencore/timer.h> #include <zenhttp/httpserver.h> @@ -14,7 +15,6 @@ #include "upstream/zen.h" #include "zenstore/cidstore.h" -#include <spdlog/spdlog.h> #include <algorithm> #include <atomic> #include <filesystem> @@ -31,18 +31,18 @@ HttpStructuredCacheService::HttpStructuredCacheService(::ZenCacheStore& InC zen::CasStore& InStore, zen::CidStore& InCidStore, std::unique_ptr<UpstreamCache> UpstreamCache) -: m_Log("cache", begin(spdlog::default_logger()->sinks()), end(spdlog::default_logger()->sinks())) +: m_Log(zen::logging::Get("cache")) , m_CacheStore(InCacheStore) , m_CasStore(InStore) , m_CidStore(InCidStore) , m_UpstreamCache(std::move(UpstreamCache)) { - m_Log.set_level(spdlog::level::debug); + // m_Log.set_level(spdlog::level::debug); } HttpStructuredCacheService::~HttpStructuredCacheService() { - spdlog::info("closing structured cache"); + ZEN_INFO("closing structured cache"); } const char* @@ -173,7 +173,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req { Value.Value = IoBuffer(); Success = false; - m_Log.warn("Upstream cache record '{}/{}' failed validation", Ref.BucketSegment, Ref.HashKey); + ZEN_WARN("Upstream cache record '{}/{}' failed validation", Ref.BucketSegment, Ref.HashKey); } } @@ -186,7 +186,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req if (!Success) { - m_Log.debug("MISS - '{}/{}'", Ref.BucketSegment, Ref.HashKey); + ZEN_DEBUG("MISS - '{}/{}'", Ref.BucketSegment, Ref.HashKey); return Request.WriteResponse(zen::HttpResponseCode::NotFound); } @@ -196,12 +196,12 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req Request.SetSuppressResponseBody(); } - m_Log.debug("HIT - '{}/{}' ({} bytes {}) ({})", - Ref.BucketSegment, - Ref.HashKey, - Value.Value.Size(), - Value.Value.GetContentType(), - InUpstreamCache ? "upstream" : "local"); + ZEN_DEBUG("HIT - '{}/{}' ({} bytes {}) ({})", + Ref.BucketSegment, + Ref.HashKey, + Value.Value.Size(), + Value.Value.GetContentType(), + InUpstreamCache ? "upstream" : "local"); return Request.WriteResponse(zen::HttpResponseCode::OK, Value.Value.GetContentType(), Value.Value); } @@ -239,11 +239,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req { // TODO: create a cache record and put value in CAS? m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = Body}); - m_Log.debug("PUT (binary) - '{}/{}' ({} bytes, {})", - Ref.BucketSegment, - Ref.HashKey, - Body.Size(), - Body.GetContentType()); + ZEN_DEBUG("PUT (binary) - '{}/{}' ({} bytes, {})", Ref.BucketSegment, Ref.HashKey, Body.Size(), Body.GetContentType()); if (m_UpstreamCache) { @@ -260,7 +256,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req if (ValidationResult != CbValidateError::None) { - m_Log.warn("Payload for key '{}/{}' ({} bytes) failed validation", Ref.BucketSegment, Ref.HashKey, Body.Size()); + ZEN_WARN("Payload for key '{}/{}' ({} bytes) failed validation", Ref.BucketSegment, Ref.HashKey, Body.Size()); // TODO: add details in response, kText || kCbObject? return Request.WriteResponse(HttpResponseCode::BadRequest, @@ -299,13 +295,13 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, CacheValue); - m_Log.debug("PUT (cache record) - '{}/{}' ({} bytes, {}, ({}/{} refs/missing))", - Ref.BucketSegment, - Ref.HashKey, - CacheValue.Value.Size(), - CacheValue.Value.GetContentType(), - References.size(), - MissingRefs.size()); + ZEN_DEBUG("PUT (cache record) - '{}/{}' ({} bytes, {}, ({}/{} refs/missing))", + Ref.BucketSegment, + Ref.HashKey, + CacheValue.Value.Size(), + CacheValue.Value.GetContentType(), + References.size(), + MissingRefs.size()); if (MissingRefs.empty()) { @@ -327,7 +323,7 @@ HttpStructuredCacheService::HandleCacheRecordRequest(zen::HttpServerRequest& Req for (const IoHash& MissingRef : MissingRefs) { Response.AddHash(MissingRef); - m_Log.debug("cache record '{}/{}' is missing reference '{}'", Ref.BucketSegment, Ref.HashKey, MissingRef); + ZEN_DEBUG("cache record '{}/{}' is missing reference '{}'", Ref.BucketSegment, Ref.HashKey, MissingRef); } Response.EndArray(); @@ -380,24 +376,24 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re } else { - m_Log.warn("got uncompressed upstream cache payload"); + ZEN_WARN("got uncompressed upstream cache payload"); } } } if (!Payload) { - m_Log.debug("MISS - '{}/{}/{}'", Ref.BucketSegment, Ref.HashKey, Ref.PayloadId); + ZEN_DEBUG("MISS - '{}/{}/{}'", Ref.BucketSegment, Ref.HashKey, Ref.PayloadId); return Request.WriteResponse(zen::HttpResponseCode::NotFound); } - m_Log.debug("HIT - '{}/{}/{}' ({} bytes, {}) ({})", - Ref.BucketSegment, - Ref.HashKey, - Ref.PayloadId, - Payload.Size(), - Payload.GetContentType(), - InUpstreamCache ? "upstream" : "local"); + ZEN_DEBUG("HIT - '{}/{}/{}' ({} bytes, {}) ({})", + Ref.BucketSegment, + Ref.HashKey, + Ref.PayloadId, + Payload.Size(), + Payload.GetContentType(), + InUpstreamCache ? "upstream" : "local"); if (Verb == kHead) { @@ -442,13 +438,13 @@ HttpStructuredCacheService::HandleCachePayloadRequest(zen::HttpServerRequest& Re m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash); - m_Log.debug("PUT ({}) - '{}/{}/{}' ({} bytes, {})", - Result.New ? "NEW" : "OLD", - Ref.BucketSegment, - Ref.HashKey, - Ref.PayloadId, - Body.Size(), - Body.GetContentType()); + ZEN_DEBUG("PUT ({}) - '{}/{}/{}' ({} bytes, {})", + Result.New ? "NEW" : "OLD", + Ref.BucketSegment, + Ref.HashKey, + Ref.PayloadId, + Body.Size(), + Body.GetContentType()); if (Result.New) { diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h index d4bb94c52..c8c959569 100644 --- a/zenserver/cache/structuredcache.h +++ b/zenserver/cache/structuredcache.h @@ -4,9 +4,12 @@ #include <zenhttp/httpserver.h> -#include <spdlog/spdlog.h> #include <memory> +namespace spdlog { +class logger; +} + class ZenCacheStore; namespace zen { @@ -71,7 +74,7 @@ private: void HandleCachePayloadRequest(zen::HttpServerRequest& Request, CacheRef& Ref); void HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket); - spdlog::logger m_Log; + spdlog::logger& m_Log; ZenCacheStore& m_CacheStore; zen::CasStore& m_CasStore; zen::CidStore& m_CidStore; diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index a07fccf98..018955e65 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -9,6 +9,7 @@ #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/iobuffer.h> +#include <zencore/logging.h> #include <zencore/string.h> #include <zencore/thread.h> #include <zenstore/basicfile.h> @@ -16,7 +17,6 @@ #include <zenstore/caslog.h> #include <fmt/core.h> -#include <spdlog/spdlog.h> #include <concepts> #include <filesystem> #include <gsl/gsl-lite.hpp> @@ -31,7 +31,7 @@ using namespace fmt::literals; ZenCacheStore::ZenCacheStore(zen::CasStore& Cas, const std::filesystem::path& RootDir) : m_DiskLayer{Cas, RootDir} { - spdlog::info("initializing structured cache at '{}'", RootDir); + ZEN_INFO("initializing structured cache at '{}'", RootDir); zen::CreateDirectories(RootDir); } @@ -85,7 +85,7 @@ ZenCacheStore::Put(std::string_view InBucket, const zen::IoHash& HashKey, const bool ZenCacheStore::DropBucket(std::string_view Bucket) { - spdlog::info("dropping bucket '{}'", Bucket); + ZEN_INFO("dropping bucket '{}'", Bucket); // TODO: should ensure this is done atomically across all layers @@ -93,7 +93,7 @@ ZenCacheStore::DropBucket(std::string_view Bucket) const bool DiskDropped = m_DiskLayer.DropBucket(Bucket); const bool AnyDropped = MemDropped || DiskDropped; - spdlog::info("bucket '{}' was {}", Bucket, AnyDropped ? "dropped" : "not found"); + ZEN_INFO("bucket '{}' was {}", Bucket, AnyDropped ? "dropped" : "not found"); return AnyDropped; } diff --git a/zenserver/casstore.cpp b/zenserver/casstore.cpp index 6f1e4873b..b36dcc09f 100644 --- a/zenserver/casstore.cpp +++ b/zenserver/casstore.cpp @@ -2,9 +2,9 @@ #include "casstore.h" +#include <zencore/logging.h> #include <zencore/streamutil.h> -#include <spdlog/spdlog.h> #include <gsl/gsl-lite.hpp> namespace zen { @@ -63,7 +63,7 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store) "{cas}", [this](HttpRouterRequest& Req) { IoHash Hash = IoHash::FromHexString(Req.GetCapture(1)); - spdlog::debug("CAS request for {}", Hash); + ZEN_DEBUG("CAS request for {}", Hash); HttpServerRequest& ServerRequest = Req.ServerRequest(); @@ -123,7 +123,7 @@ HttpCasService::HandleRequest(zen::HttpServerRequest& Request) IoBuffer Payload = Request.ReadPayload(); IoHash PayloadHash = IoHash::HashBuffer(Payload.Data(), Payload.Size()); - spdlog::debug("CAS POST request for {} ({} bytes)", PayloadHash, Payload.Size()); + ZEN_DEBUG("CAS POST request for {} ({} bytes)", PayloadHash, Payload.Size()); auto InsertResult = m_CasStore.InsertChunk(Payload, PayloadHash); diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index b46945d88..c3d83b2b5 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -89,7 +89,7 @@ BasicFunctionJob::SpawnJob(std::filesystem::path ExePath, std::wstring CommandLi m_ProcessHandle.Attach(ProcessInfo.hProcess); ::CloseHandle(ProcessInfo.hThread); - spdlog::info("Created process {}", m_ProcessId); + ZEN_INFO("Created process {}", m_ProcessId); return true; } @@ -114,7 +114,7 @@ BasicFunctionJob::Wait(uint32_t TimeoutMs) return true; } - throw std::exception("Failed wait on process handle"); + throw std::runtime_error("Failed wait on process handle"); } int @@ -125,12 +125,12 @@ BasicFunctionJob::ExitCode() if (!Success) { - spdlog::warn("failed getting exit code"); + ZEN_WARN("failed getting exit code"); } if (Ec == STILL_ACTIVE) { - spdlog::warn("getting exit code but process is STILL_ACTIVE"); + ZEN_WARN("getting exit code but process is STILL_ACTIVE"); } return gsl::narrow_cast<int>(Ec); @@ -231,7 +231,7 @@ SandboxedFunctionJob::Initialize(std::string_view AppContainerId) if (FAILED(hRes)) { - spdlog::error("Failed creating app container SID"); + ZEN_ERROR("Failed creating app container SID"); } } @@ -240,12 +240,12 @@ SandboxedFunctionJob::Initialize(std::string_view AppContainerId) PWSTR Str = nullptr; ::ConvertSidToStringSid(m_AppContainerSid, &Str); - spdlog::info("AppContainer SID : '{}'", WideToUtf8(Str)); + ZEN_INFO("AppContainer SID : '{}'", WideToUtf8(Str)); PWSTR Path = nullptr; if (SUCCEEDED(::GetAppContainerFolderPath(Str, &Path))) { - spdlog::info("AppContainer folder: '{}'", WideToUtf8(Path)); + ZEN_INFO("AppContainer folder: '{}'", WideToUtf8(Path)); ::CoTaskMemFree(Path); } @@ -321,13 +321,13 @@ SandboxedFunctionJob::SpawnJob(std::filesystem::path ExePath) return false; } - spdlog::info("Created process {}", ProcessInfo.dwProcessId); + ZEN_INFO("Created process {}", ProcessInfo.dwProcessId); return true; } HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, const std::filesystem::path& BaseDir) -: m_Log("apply", begin(spdlog::default_logger()->sinks()), end(spdlog::default_logger()->sinks())) +: m_Log(logging::Get("apply")) , m_CasStore(Store) , m_CidStore(InCidStore) , m_SandboxPath(BaseDir / "scratch") @@ -378,6 +378,10 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, ChunkSet.AddChunk(Hash); }); + // Note that we store executables uncompressed to make it + // more straightforward and efficient to materialize them, hence + // the CAS lookup here instead of CID for the input payloads + m_CasStore.FilterChunks(ChunkSet); if (ChunkSet.IsEmpty()) @@ -386,7 +390,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, m_WorkerMap.insert_or_assign(WorkerId, WorkerDesc{FunctionSpec}); - spdlog::debug("worker {}: all attachments already available", WorkerId); + ZEN_DEBUG("worker {}: all attachments already available", WorkerId); return HttpReq.WriteResponse(HttpResponseCode::NoContent); } @@ -397,14 +401,14 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, for (const IoHash& Hash : ChunkSet.GetChunkSet()) { - spdlog::debug("worker {}: need chunk {}", WorkerId, Hash); + ZEN_DEBUG("worker {}: need chunk {}", WorkerId, Hash); ResponseWriter.AddHash(Hash); } ResponseWriter.EndArray(); - spdlog::debug("worker {}: need {} attachments", WorkerId, ChunkSet.GetChunkSet().size()); + ZEN_DEBUG("worker {}: need {} attachments", WorkerId, ChunkSet.GetChunkSet().size()); return HttpReq.WriteResponse(HttpResponseCode::NotFound, ResponseWriter.Save()); } @@ -426,31 +430,35 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, for (const CbAttachment& Attachment : Attachments) { - ZEN_ASSERT(Attachment.IsBinary()); + ZEN_ASSERT(Attachment.IsCompressedBinary()); - const IoHash DataHash = Attachment.GetHash(); - CompressedBuffer DataView = Attachment.AsCompressedBinary(); + const IoHash DataHash = Attachment.GetHash(); + CompressedBuffer DataView = Attachment.AsCompressedBinary(); + SharedBuffer Decompressed = DataView.Decompress(); + const uint64_t DecompressedSize = DataView.GetRawSize(); - TotalAttachmentBytes += DataView.GetCompressedSize(); + TotalAttachmentBytes += DecompressedSize; ++AttachmentCount; - IoBuffer Payload = DataView.GetCompressed().Flatten().AsIoBuffer(); + // Note that we store executables uncompressed to make it + // more straightforward and efficient to materialize them - CasStore::InsertResult InsertResult = m_CasStore.InsertChunk(Payload, DataHash); + const CasStore::InsertResult InsertResult = + m_CasStore.InsertChunk(Decompressed.AsIoBuffer(), IoHash::FromBLAKE3(DataView.GetRawHash())); if (InsertResult.New) { - TotalNewBytes += Payload.Size(); + TotalNewBytes += DecompressedSize; ++NewAttachmentCount; } } - spdlog::debug("worker {}: {} in {} attachments, {} in {} new attachments", - WorkerId, - zen::NiceBytes(TotalAttachmentBytes), - AttachmentCount, - zen::NiceBytes(TotalNewBytes), - NewAttachmentCount); + ZEN_DEBUG("worker {}: {} in {} attachments, {} in {} new attachments", + WorkerId, + zen::NiceBytes(TotalAttachmentBytes), + AttachmentCount, + zen::NiceBytes(TotalNewBytes), + NewAttachmentCount); RwLock::ExclusiveLockScope _(m_WorkerLock); @@ -526,7 +534,7 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, RequestObject.IterateAttachments([&](CbFieldView Field) { const IoHash FileHash = Field.AsHash(); - if (!m_CasStore.FindChunk(FileHash)) + if (!m_CidStore.ContainsChunk(FileHash)) { NeedList.push_back(FileHash); } @@ -570,28 +578,30 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, for (const CbAttachment& Attachment : Attachments) { - ZEN_ASSERT(Attachment.IsBinary()); + ZEN_ASSERT(Attachment.IsCompressedBinary()); + + const IoHash DataHash = Attachment.GetHash(); + CompressedBuffer DataView = Attachment.AsCompressedBinary(); - const IoHash DataHash = Attachment.GetHash(); - SharedBuffer DataView = Attachment.AsBinary(); + const uint64_t CompressedSize = DataView.GetCompressedSize(); - TotalAttachmentBytes += DataView.GetSize(); + TotalAttachmentBytes += CompressedSize; ++AttachmentCount; - CasStore::InsertResult InsertResult = m_CasStore.InsertChunk(DataView.AsIoBuffer(), DataHash); + const CasStore::InsertResult InsertResult = m_CidStore.AddChunk(DataView); if (InsertResult.New) { - TotalNewBytes += DataView.GetSize(); + TotalNewBytes += CompressedSize; ++NewAttachmentCount; } } - spdlog::debug("new action: {}B in {} attachments. {}B new ({} attachments)", - zen::NiceBytes(TotalAttachmentBytes), - AttachmentCount, - zen::NiceBytes(TotalNewBytes), - NewAttachmentCount); + ZEN_DEBUG("new action: {} in {} attachments. {} new ({} attachments)", + zen::NiceBytes(TotalAttachmentBytes), + AttachmentCount, + zen::NiceBytes(TotalNewBytes), + NewAttachmentCount); CbPackage Output = ExecAction(Worker, ActionObj); @@ -603,90 +613,6 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, } }, HttpVerb::kPost); - - // This is just for reference - m_Router.RegisterRoute( - "jobs/noop", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - - switch (HttpReq.RequestVerb()) - { - case HttpVerb::kGet: - break; - - case HttpVerb::kPost: - { - IoBuffer Payload = HttpReq.ReadPayload(); - CbObject RequestObject = LoadCompactBinaryObject(Payload); - - bool AllOk = true; - - std::vector<IoHash> NeedList; - - std::filesystem::path SandboxDir{CreateNewSandbox()}; - - spdlog::debug("setting up job in sandbox '{}'", SandboxDir); - - zen::DeleteDirectories(SandboxDir); - zen::CreateDirectories(SandboxDir); - - for (auto Entry : RequestObject["files"sv]) - { - CbObjectView Ob = Entry.AsObjectView(); - - std::string_view FileName = Ob["file"sv].AsString(); - const IoHash FileHash = Ob["hash"sv].AsHash(); - uint64_t FileSize = Ob["size"sv].AsUInt64(); - - if (IoBuffer Chunk = m_CasStore.FindChunk(FileHash); !Chunk) - { - spdlog::debug("MISSING: {} {} {}", FileHash, FileName, FileSize); - AllOk = false; - - NeedList.push_back(FileHash); - } - else - { - std::filesystem::path FullPath = SandboxDir / FileName; - - const IoBuffer* Chunks[] = {&Chunk}; - - zen::WriteFile(FullPath, Chunks, 1); - } - } - - if (!AllOk) - { - // TODO: Could report all the missing pieces in the response here - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - - std::string Executable8{RequestObject["cmd"].AsString()}; - std::string Args8{RequestObject["args"].AsString()}; - - std::wstring Executable = Utf8ToWide(Executable8); - std::wstring Args = Utf8ToWide(Args8); - - spdlog::debug("spawning job in sandbox '{}': '{}' '{}'", SandboxDir, Executable8, Args8); - - std::filesystem::path ExeName = SandboxDir / Executable; - - BasicFunctionJob Job; - Job.SetWorkingDirectory(SandboxDir); - Job.SpawnJob(ExeName, Args); - Job.Wait(); - - CbObjectWriter Response; - - Response << "exitcode" << Job.ExitCode(); - - return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); - } - break; - } - }, - HttpVerb::kGet | HttpVerb::kPost); } HttpFunctionService::~HttpFunctionService() @@ -704,7 +630,7 @@ HttpFunctionService::HandleRequest(HttpServerRequest& Request) { if (m_Router.HandleRequest(Request) == false) { - m_Log.warn("No route found for {0}", Request.RelativeUri()); + ZEN_WARN("No route found for {0}", Request.RelativeUri()); } } @@ -741,7 +667,7 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action) if (!DataBuffer) { - throw std::exception("Chunk missing" /* ADD CONTEXT */); + throw std::runtime_error("Chunk missing" /* ADD CONTEXT */); } zen::WriteFile(FilePath, DataBuffer); @@ -767,7 +693,7 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action) if (!DataBuffer) { - throw std::exception("Chunk missing" /* ADD CONTEXT */); + throw std::runtime_error("Chunk missing" /* ADD CONTEXT */); } zen::WriteFile(FilePath, DataBuffer); @@ -780,18 +706,16 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action) // Manifest inputs in sandbox Action.IterateAttachments([&](CbFieldView Field) { - const IoHash Hash = Field.AsHash(); - std::filesystem::path FilePath{SandboxPath / "Inputs" / Hash.ToHexString()}; - IoBuffer DataBuffer = m_CasStore.FindChunk(Hash); + const IoHash Cid = Field.AsHash(); + std::filesystem::path FilePath{SandboxPath / "Inputs" / Cid.ToHexString()}; + IoBuffer DataBuffer = m_CidStore.FindChunkByCid(Cid); if (!DataBuffer) { - throw std::exception("Chunk missing" /* ADD CONTEXT */); + throw std::runtime_error("Chunk missing" /* ADD CONTEXT */); } - CompressedBuffer Buffer = CompressedBuffer::Compress(SharedBuffer(std::move(DataBuffer))); - - zen::WriteFile(FilePath, Buffer.GetCompressed().Flatten().AsIoBuffer()); + zen::WriteFile(FilePath, DataBuffer); }); // Set up environment variables @@ -884,7 +808,7 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action) ZEN_ASSERT(OutputData.Data.size() == 1); - CbAttachment Attachment(SharedBuffer(ChunkData.Data[0]), Hash); + CbAttachment Attachment(CompressedBuffer::FromCompressed(SharedBuffer(ChunkData.Data[0]))); OutputPackage.AddAttachment(Attachment); }); diff --git a/zenserver/compute/apply.h b/zenserver/compute/apply.h index 695dc2e6e..474156a5e 100644 --- a/zenserver/compute/apply.h +++ b/zenserver/compute/apply.h @@ -28,7 +28,7 @@ public: virtual void HandleRequest(HttpServerRequest& Request) override; private: - spdlog::logger m_Log; + spdlog::logger& m_Log; HttpRequestRouter m_Router; CasStore& m_CasStore; CidStore& m_CidStore; diff --git a/zenserver/config.cpp b/zenserver/config.cpp index ddef83c02..578a3a202 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -14,7 +14,7 @@ #pragma warning(pop) #include <fmt/format.h> -#include <spdlog/spdlog.h> +#include <zencore/logging.h> #include <sol/sol.hpp> #if ZEN_PLATFORM_WINDOWS @@ -59,6 +59,9 @@ void ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, ZenServiceConfig& ServiceConfig) { cxxopts::Options options("zenserver", "Zen Server"); + options.add_options()("dedicated", + "Enable dedicated server mode", + cxxopts::value<bool>(GlobalOptions.IsDedicated)->default_value("false")); options.add_options()("d, debug", "Enable debugging", cxxopts::value<bool>(GlobalOptions.IsDebug)->default_value("false")); options.add_options()("help", "Show command line help"); options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(GlobalOptions.IsTest)->default_value("false")); @@ -237,9 +240,9 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv } catch (std::exception& e) { - spdlog::error("config script failure: {}", e.what()); + ZEN_ERROR("config script failure: {}", e.what()); - throw std::exception("fatal zen global config script ({}) failure: {}"_format(ConfigScript, e.what()).c_str()); + throw std::runtime_error("fatal zen global config script ({}) failure: {}"_format(ConfigScript, e.what()).c_str()); } ServiceConfig.MeshEnabled = lua["mesh"]["enable"].get_or(ServiceConfig.MeshEnabled); diff --git a/zenserver/config.h b/zenserver/config.h index 92b7c9e31..80ec86905 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -7,13 +7,14 @@ struct ZenServerOptions { - bool IsDebug = false; - bool IsTest = false; - int BasePort = 1337; // Service listen port (used for both UDP and TCP) - int OwnerPid = 0; // Parent process id (zero for standalone) - std::string ChildId; // Id assigned by parent process (used for lifetime management) - std::string LogId; // Id for tagging log output - std::filesystem::path DataDir; // Root directory for state (used for testing) + bool IsDebug = false; + bool IsTest = false; + bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements + int BasePort = 1337; // Service listen port (used for both UDP and TCP) + int OwnerPid = 0; // Parent process id (zero for standalone) + std::string ChildId; // Id assigned by parent process (used for lifetime management) + std::string LogId; // Id for tagging log output + std::filesystem::path DataDir; // Root directory for state (used for testing) }; struct ZenUpstreamJupiterConfig diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp index 5782ce582..48eda7512 100644 --- a/zenserver/diag/logging.cpp +++ b/zenserver/diag/logging.cpp @@ -211,7 +211,7 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) spdlog::init_thread_pool(QueueSize, ThreadCount); auto AsyncLogger = spdlog::create_async<spdlog::sinks::ansicolor_stdout_sink_mt>("main"); - spdlog::set_default_logger(AsyncLogger); + zen::logging::SetDefault(AsyncLogger); } // Sinks @@ -220,16 +220,16 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) auto FileSink = std::make_shared<spdlog::sinks::basic_file_sink_mt>(zen::WideToUtf8(LogPath.c_str()), /* truncate */ true); // Default - auto DefaultLogger = spdlog::default_logger(); - auto& Sinks = spdlog::default_logger()->sinks(); - Sinks.clear(); + auto& DefaultLogger = zen::logging::Default(); + auto& Sinks = DefaultLogger.sinks(); + + Sinks.clear(); Sinks.push_back(ConsoleSink); Sinks.push_back(FileSink); - DefaultLogger->set_level(LogLevel); - DefaultLogger->flush_on(spdlog::level::err); // Jupiter - only log HTTP traffic to file + auto JupiterLogger = std::make_shared<spdlog::logger>("jupiter", FileSink); spdlog::register_logger(JupiterLogger); JupiterLogger->set_level(LogLevel); @@ -238,7 +238,10 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) spdlog::register_logger(ZenClientLogger); ZenClientLogger->set_level(LogLevel); + // Configure all registered loggers according to settings + spdlog::set_level(LogLevel); + spdlog::flush_on(spdlog::level::err); spdlog::set_formatter(std::make_unique<logging::full_formatter>(GlobalOptions.LogId, std::chrono::system_clock::now())); } diff --git a/zenserver/experimental/usnjournal.cpp b/zenserver/experimental/usnjournal.cpp index 0553945dd..ab83b8a1c 100644 --- a/zenserver/experimental/usnjournal.cpp +++ b/zenserver/experimental/usnjournal.cpp @@ -3,11 +3,10 @@ #include "usnjournal.h" #include <zencore/except.h> +#include <zencore/logging.h> #include <zencore/timer.h> #include <zencore/zencore.h> -#include <spdlog/spdlog.h> - #include <atlfile.h> #include <filesystem> @@ -90,7 +89,7 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) ThrowSystemException("Failed to get volume information"); } - spdlog::debug("File system type is {}", WideToUtf8(FileSystemName)); + ZEN_DEBUG("File system type is {}", WideToUtf8(FileSystemName)); if (wcscmp(L"ReFS", FileSystemName) == 0) { @@ -143,7 +142,7 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) switch (DWORD Error = GetLastError()) { case ERROR_JOURNAL_NOT_ACTIVE: - spdlog::info("No USN journal active on drive"); + ZEN_INFO("No USN journal active on drive"); // TODO: optionally activate USN journal on drive? @@ -183,7 +182,7 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) if (m_FileSystemType == FileSystemType::NTFS) { - spdlog::info("Enumerating MFT for {}", WideToUtf8(VolumePathName)); + ZEN_INFO("Enumerating MFT for {}", WideToUtf8(VolumePathName)); zen::Stopwatch Timer; uint64_t MftBytesProcessed = 0; @@ -262,10 +261,10 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) const auto ElapsedMs = Timer.getElapsedTimeMs(); - spdlog::info("MFT enumeration of {} completed after {} ({})", - zen::NiceBytes(MftBytesProcessed), - zen::NiceTimeSpanMs(ElapsedMs), - zen::NiceByteRate(MftBytesProcessed, ElapsedMs)); + ZEN_INFO("MFT enumeration of {} completed after {} ({})", + zen::NiceBytes(MftBytesProcessed), + zen::NiceTimeSpanMs(ElapsedMs), + zen::NiceByteRate(MftBytesProcessed, ElapsedMs)); } // Populate by traversal diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index 2bbc1dce3..404484edf 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -7,6 +7,7 @@ #include <zencore/compactbinaryvalidation.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> +#include <zencore/logging.h> #include <zencore/stream.h> #include <zencore/string.h> #include <zencore/timer.h> @@ -22,11 +23,8 @@ # include <rocksdb/db.h> #endif -#include <ppl.h> -#include <spdlog/spdlog.h> #include <xxh3.h> #include <asio.hpp> -#include <future> #include <latch> #include <string> @@ -66,7 +64,7 @@ struct ProjectStore::OplogStorage : public RefCounted ~OplogStorage() { - Log().info("closing oplog storage at {}", m_OplogStoragePath); + ZEN_INFO("closing oplog storage at {}", m_OplogStoragePath); Flush(); #if USE_ROCKSDB @@ -82,7 +80,7 @@ struct ProjectStore::OplogStorage : public RefCounted if (!Status.ok()) { - Log().warn("db close error reported for '{}' : '{}'", m_OplogStoragePath, Status.getState()); + ZEN_WARN("db close error reported for '{}' : '{}'", m_OplogStoragePath, Status.getState()); } } #endif @@ -98,7 +96,7 @@ struct ProjectStore::OplogStorage : public RefCounted void Open(bool IsCreate) { - Log().info("initializing oplog storage at '{}'", m_OplogStoragePath); + ZEN_INFO("initializing oplog storage at '{}'", m_OplogStoragePath); if (IsCreate) { @@ -118,7 +116,7 @@ struct ProjectStore::OplogStorage : public RefCounted { std::string RocksdbPath = WideToUtf8((m_OplogStoragePath / "ops.rdb").native().c_str()); - Log().debug("opening rocksdb db at '{}'", RocksdbPath); + ZEN_DEBUG("opening rocksdb db at '{}'", RocksdbPath); rocksdb::DB* Db; rocksdb::DBOptions Options; @@ -144,14 +142,14 @@ struct ProjectStore::OplogStorage : public RefCounted } else { - throw std::exception("column family iteration failed for '{}': '{}'"_format(RocksdbPath, Status.getState()).c_str()); + throw std::runtime_error("column family iteration failed for '{}': '{}'"_format(RocksdbPath, Status.getState()).c_str()); } Status = rocksdb::DB::Open(Options, RocksdbPath, ColumnDescriptors, &m_RocksDbColumnHandles, &Db); if (!Status.ok()) { - throw std::exception("database open failed for '{}': '{}'"_format(RocksdbPath, Status.getState()).c_str()); + throw std::runtime_error("database open failed for '{}': '{}'"_format(RocksdbPath, Status.getState()).c_str()); } m_RocksDb.reset(Db); @@ -163,7 +161,7 @@ struct ProjectStore::OplogStorage : public RefCounted { // This could use memory mapping or do something clever but for now it just reads the file sequentially - spdlog::info("replaying log for '{}'", m_OplogStoragePath); + ZEN_INFO("replaying log for '{}'", m_OplogStoragePath); Stopwatch Timer; @@ -179,7 +177,7 @@ struct ProjectStore::OplogStorage : public RefCounted if (OpCoreHash != LogEntry.OpCoreHash) { - Log().warn("skipping oplog entry with bad checksum!"); + ZEN_WARN("skipping oplog entry with bad checksum!"); return; } @@ -192,10 +190,10 @@ struct ProjectStore::OplogStorage : public RefCounted Handler(Op, LogEntry); }); - spdlog::info("Oplog replay completed in {} - Max LSN# {}, Next offset: {}", - NiceTimeSpanMs(Timer.getElapsedTimeMs()), - m_MaxLsn, - m_NextOpsOffset); + ZEN_INFO("Oplog replay completed in {} - Max LSN# {}, Next offset: {}", + NiceTimeSpanMs(Timer.getElapsedTimeMs()), + m_MaxLsn, + m_NextOpsOffset); } void ReplayLog(const std::vector<OplogEntryAddress>& Entries, std::function<void(CbObject)>&& Handler) @@ -466,7 +464,7 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry AddChunkMapping(PackageId, PackageHash); - Log().debug("package data {} -> {}", PackageId, PackageHash); + ZEN_DEBUG("package data {} -> {}", PackageId, PackageHash); } for (CbFieldView& Entry : Core["bulkdata"sv]) @@ -478,7 +476,7 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry AddChunkMapping(BulkDataId, BulkDataHash); - Log().debug("bulkdata {} -> {}", BulkDataId, BulkDataHash); + ZEN_DEBUG("bulkdata {} -> {}", BulkDataId, BulkDataHash); } if (Core["files"sv]) @@ -500,11 +498,11 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry } else { - Log().warn("invalid file"); + ZEN_WARN("invalid file"); } } - Log().debug("added {} file(s) in {}", FileCount, NiceTimeSpanMs(Timer.getElapsedTimeMs())); + ZEN_DEBUG("added {} file(s) in {}", FileCount, NiceTimeSpanMs(Timer.getElapsedTimeMs())); } for (CbFieldView& Entry : Core["meta"sv]) @@ -516,7 +514,7 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry AddMetaMapping(MetaId, MetaDataHash); - Log().debug("meta data ({}) {} -> {}", NameString, MetaId, MetaDataHash); + ZEN_DEBUG("meta data ({}) {} -> {}", NameString, MetaId, MetaDataHash); } m_OpAddressMap.emplace(OpEntry.OpLsn, OplogEntryAddress{.Offset = OpEntry.OpCoreOffset, .Size = OpEntry.OpCoreSize}); @@ -579,10 +577,10 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbPackage OpPackage) const uint32_t EntryId = RegisterOplogEntry(Core, OpEntry, kUpdateNewEntry); - Log().debug("oplog entry #{} attachments: {}B new, {}B total", - EntryId, - zen::NiceBytes(NewAttachmentBytes), - zen::NiceBytes(AttachmentBytes)); + ZEN_DEBUG("oplog entry #{} attachments: {} new, {} total", + EntryId, + zen::NiceBytes(NewAttachmentBytes), + zen::NiceBytes(AttachmentBytes)); return EntryId; } @@ -611,7 +609,7 @@ ProjectStore::Project::Read() { std::filesystem::path ProjectStateFilePath = m_OplogStoragePath / "Project.zcb"; - spdlog::info("reading config for project '{}' from {}", Identifier, ProjectStateFilePath); + ZEN_INFO("reading config for project '{}' from {}", Identifier, ProjectStateFilePath); BasicFile Blob; Blob.Open(ProjectStateFilePath, false); @@ -630,7 +628,7 @@ ProjectStore::Project::Read() } else { - spdlog::error("validation error {} hit for '{}'", int(ValidationError), ProjectStateFilePath); + ZEN_ERROR("validation error {} hit for '{}'", int(ValidationError), ProjectStateFilePath); } } @@ -652,7 +650,7 @@ ProjectStore::Project::Write() std::filesystem::path ProjectStateFilePath = m_OplogStoragePath / "Project.zcb"; - spdlog::info("persisting config for project '{}' to {}", Identifier, ProjectStateFilePath); + ZEN_INFO("persisting config for project '{}' to {}", Identifier, ProjectStateFilePath); BasicFile Blob; Blob.Open(ProjectStateFilePath, true); @@ -729,7 +727,7 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId) } catch (std::exception& ex) { - spdlog::error("failed to open oplog '{}' @ '{}': {}", OplogId, OplogBasePath, ex.what()); + ZEN_ERROR("failed to open oplog '{}' @ '{}': {}", OplogId, OplogBasePath, ex.what()); m_Oplogs.erase(std::string{OplogId}); } @@ -785,17 +783,17 @@ ProjectStore::Project::Flush() ////////////////////////////////////////////////////////////////////////// ProjectStore::ProjectStore(CasStore& Store, std::filesystem::path BasePath) -: m_Log("project", begin(spdlog::default_logger()->sinks()), end(spdlog::default_logger()->sinks())) +: m_Log(zen::logging::Get("project")) , m_ProjectBasePath(BasePath) , m_CasStore(Store) { - m_Log.info("initializing project store at '{}'", BasePath); - m_Log.set_level(spdlog::level::debug); + ZEN_INFO("initializing project store at '{}'", BasePath); + // m_Log.set_level(spdlog::level::debug); } ProjectStore::~ProjectStore() { - m_Log.info("closing project store ('{}')", m_ProjectBasePath); + ZEN_INFO("closing project store ('{}')", m_ProjectBasePath); } std::filesystem::path @@ -839,7 +837,7 @@ ProjectStore::OpenProject(std::string_view ProjectId) { try { - Log().info("opening project {} @ {}", ProjectId, ProjectBasePath); + ZEN_INFO("opening project {} @ {}", ProjectId, ProjectBasePath); ProjectStore::Project& Prj = m_Projects.try_emplace(std::string{ProjectId}, this, m_CasStore, ProjectBasePath).first->second; Prj.Identifier = ProjectId; @@ -848,7 +846,7 @@ ProjectStore::OpenProject(std::string_view ProjectId) } catch (std::exception& e) { - Log().warn("failed to open {} @ {} ({})", ProjectId, ProjectBasePath, e.what()); + ZEN_WARN("failed to open {} @ {} ({})", ProjectId, ProjectBasePath, e.what()); m_Projects.erase(std::string{ProjectId}); } } @@ -880,7 +878,7 @@ ProjectStore::DeleteProject(std::string_view ProjectId) { std::filesystem::path ProjectBasePath = BasePathForProject(ProjectId); - Log().info("deleting project {} @ {}", ProjectId, ProjectBasePath); + ZEN_INFO("deleting project {} @ {}", ProjectId, ProjectBasePath); m_Projects.erase(std::string{ProjectId}); @@ -908,7 +906,7 @@ ProjectStore::OpenProjectOplog(std::string_view ProjectId, std::string_view Oplo HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) : m_CasStore(Store) -, m_Log("project", begin(spdlog::default_logger()->sinks()), end(spdlog::default_logger()->sinks())) +, m_Log(logging::Get("project")) , m_ProjectStore(Projects) { using namespace std::literals; @@ -1163,7 +1161,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) } } - m_Log.debug("chunk - {} / {} / {}", ProjectId, OplogId, ChunkId); + ZEN_DEBUG("chunk - {} / {} / {}", ProjectId, OplogId, ChunkId); ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId); @@ -1256,7 +1254,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) } } - m_Log.debug("oplog hash - {} / {} / {}", ProjectId, OplogId, HashString); + ZEN_DEBUG("oplog hash - {} / {} / {}", ProjectId, OplogId, HashString); IoHash Hash = IoHash::FromHexString(HashString); IoBuffer Value = m_CasStore.FindChunk(Hash); @@ -1317,7 +1315,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (!m_CasStore.FindChunk(FileHash)) { - spdlog::debug("NEED: {}", FileHash); + ZEN_DEBUG("NEED: {}", FileHash); NeedList.push_back(FileHash); } @@ -1365,7 +1363,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) return HttpReq.WriteResponse(HttpResponseCode::NotFound); } - ProjectStore::Oplog& Log = *FoundLog; + ProjectStore::Oplog& Oplog = *FoundLog; IoBuffer Payload = HttpReq.ReadPayload(); @@ -1389,7 +1387,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) AttachmentId = Hash; } - std::filesystem::path AttachmentPath = Log.TempPath() / AttachmentId.ToHexString(); + std::filesystem::path AttachmentPath = Oplog.TempPath() / AttachmentId.ToHexString(); if (IoBuffer Data = m_CasStore.FindChunk(Hash)) { @@ -1412,7 +1410,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver)) { - m_Log.error("Received malformed package!"); + ZEN_ERROR("Received malformed package!"); return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"); } @@ -1433,14 +1431,14 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) // Write core to oplog - const uint32_t OpLsn = Log.AppendNewOplogEntry(Package); + const uint32_t OpLsn = Oplog.AppendNewOplogEntry(Package); if (OpLsn == ProjectStore::Oplog::kInvalidOp) { return HttpReq.WriteResponse(HttpResponseCode::BadRequest); } - m_Log.info("new op #{:4} - {}/{} ({:>6}) {}", OpLsn, ProjectId, OplogId, NiceBytes(Payload.Size()), Core["key"sv].AsString()); + ZEN_INFO("new op #{:4} - {}/{} ({:>6}) {}", OpLsn, ProjectId, OplogId, NiceBytes(Payload.Size()), Core["key"sv].AsString()); HttpReq.WriteResponse(HttpResponseCode::Created); }, @@ -1510,7 +1508,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) return Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError); } - m_Log.info("established oplog {} / {}", ProjectId, OplogId); + ZEN_INFO("established oplog {} / {}", ProjectId, OplogId); return Req.ServerRequest().WriteResponse(HttpResponseCode::Created); } @@ -1524,7 +1522,7 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) case HttpVerb::kDelete: { - spdlog::info("deleting oplog {}/{}", ProjectId, OplogId); + ZEN_INFO("deleting oplog {}/{}", ProjectId, OplogId); ProjectIt->DeleteOplog(OplogId); @@ -1603,12 +1601,12 @@ HttpProjectService::HttpProjectService(CasStore& Store, ProjectStore* Projects) const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot); - m_Log.info("established project - {} (id: '{}', roots: '{}', '{}', '{}')", - ProjectId, - Id, - Root, - EngineRoot, - ProjectRoot); + ZEN_INFO("established project - {} (id: '{}', roots: '{}', '{}', '{}')", + ProjectId, + Id, + Root, + EngineRoot, + ProjectRoot); Req.ServerRequest().WriteResponse(HttpResponseCode::Created); } @@ -1672,7 +1670,7 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request) { if (m_Router.HandleRequest(Request) == false) { - m_Log.warn("No route found for {0}", Request.RelativeUri()); + ZEN_WARN("No route found for {0}", Request.RelativeUri()); } } @@ -1738,7 +1736,7 @@ struct LocalProjectService::LocalProjectImpl } catch (std::exception& ex) { - spdlog::error("exception caught in pipe project service loop: {}", ex.what()); + ZEN_ERROR("exception caught in pipe project service loop: {}", ex.what()); } m_ShutdownLatch.count_down(); @@ -1823,7 +1821,7 @@ private: if (hPipe == INVALID_HANDLE_VALUE) { - spdlog::warn("failed while creating named pipe {}", PipeName.c_str()); + ZEN_WARN("failed while creating named pipe {}", PipeName.c_str()); // TODO: error - how to best handle? } @@ -1870,13 +1868,13 @@ private: return; } - spdlog::warn("pipe connection error: {}", Ec.message()); + ZEN_WARN("pipe connection error: {}", Ec.message()); // TODO: should disconnect and issue a new connect return; } - spdlog::debug("pipe connection established"); + ZEN_DEBUG("pipe connection established"); IssueRead(); } @@ -1898,13 +1896,13 @@ private: return; } - spdlog::warn("pipe read error: {}", Ec.message()); + ZEN_WARN("pipe read error: {}", Ec.message()); // TODO: should disconnect and issue a new connect return; } - spdlog::debug("received message: {} bytes", Bytes); + ZEN_DEBUG("received message: {} bytes", Bytes); // TODO: Actually process request @@ -1926,7 +1924,7 @@ private: return; } - spdlog::warn("pipe write error: {}", Ec.message()); + ZEN_WARN("pipe write error: {}", Ec.message()); // TODO: should disconnect and issue a new connect return; diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h index 8fe189ab9..3d2247305 100644 --- a/zenserver/projectstore.h +++ b/zenserver/projectstore.h @@ -8,8 +8,8 @@ #include <zenstore/cas.h> #include <zenstore/caslog.h> -#include <spdlog/spdlog.h> #include <tsl/robin_map.h> +#include <zencore/logging.h> #include <filesystem> #include <map> #include <optional> @@ -182,7 +182,7 @@ public: const std::filesystem::path& BasePath() const { return m_ProjectBasePath; } private: - spdlog::logger m_Log; + spdlog::logger& m_Log; CasStore& m_CasStore; std::filesystem::path m_ProjectBasePath; RwLock m_ProjectsLock; @@ -221,9 +221,11 @@ public: private: CasStore& m_CasStore; - spdlog::logger m_Log; + spdlog::logger& m_Log; HttpRequestRouter m_Router; Ref<ProjectStore> m_ProjectStore; + + inline spdlog::logger& Log() { return m_Log; } }; /** Project store interface for local clients diff --git a/zenserver/sos/sos.cpp b/zenserver/sos/sos.cpp index d52a346ae..5fa6ffaae 100644 --- a/zenserver/sos/sos.cpp +++ b/zenserver/sos/sos.cpp @@ -2,8 +2,11 @@ #include "sos.h" -HttpCommonStructuredObjectStore::HttpCommonStructuredObjectStore() -: m_Log("sos", begin(spdlog::default_logger()->sinks()), end(spdlog::default_logger()->sinks())) +#include <zencore/logging.h> + +namespace zen { + +HttpCommonStructuredObjectStore::HttpCommonStructuredObjectStore() : m_Log(logging::Get("sos")) { m_Router.AddPattern("ns", "([[:alnum:]_-.]+)"); m_Router.AddPattern("bucket", "([[:alnum:]_-.]+)"); @@ -25,3 +28,5 @@ HttpCommonStructuredObjectStore::HandleRequest(zen::HttpServerRequest& HttpServi { ZEN_UNUSED(HttpServiceRequest); } + +} // namespace zen diff --git a/zenserver/sos/sos.h b/zenserver/sos/sos.h index da9064262..e602df8c4 100644 --- a/zenserver/sos/sos.h +++ b/zenserver/sos/sos.h @@ -4,7 +4,9 @@ #include <zenhttp/httpserver.h> -#include <spdlog/spdlog.h> +#include <zencore/logging.h> + +namespace zen { /** Simple Object Store API * @@ -25,6 +27,8 @@ public: virtual void HandleRequest(zen::HttpServerRequest& HttpServiceRequest) override; private: - spdlog::logger m_Log; + spdlog::logger& m_Log; zen::HttpRequestRouter m_Router; }; + +} // namespace zen diff --git a/zenserver/testing/httptest.h b/zenserver/testing/httptest.h index 5809d4e2e..b445fb450 100644 --- a/zenserver/testing/httptest.h +++ b/zenserver/testing/httptest.h @@ -2,10 +2,9 @@ #pragma once +#include <zencore/logging.h> #include <zenhttp/httpserver.h> -#include <spdlog/spdlog.h> - namespace zen { /** diff --git a/zenserver/testing/launch.cpp b/zenserver/testing/launch.cpp index b031193d5..55695ac9c 100644 --- a/zenserver/testing/launch.cpp +++ b/zenserver/testing/launch.cpp @@ -8,6 +8,7 @@ #include <zencore/fmtutils.h> #include <zencore/iobuffer.h> #include <zencore/iohash.h> +#include <zencore/logging.h> #include <zencore/windows.h> #include <zenstore/CAS.h> @@ -79,7 +80,7 @@ BasicJob::SpawnJob(std::filesystem::path ExePath, std::wstring CommandLine) m_ProcessHandle.Attach(ProcessInfo.hProcess); ::CloseHandle(ProcessInfo.hThread); - spdlog::info("Created process {}", m_ProcessId); + ZEN_INFO("Created process {}", m_ProcessId); return true; } @@ -104,7 +105,7 @@ BasicJob::Wait(uint32_t TimeoutMs) return true; } - throw std::exception("Failed wait on process handle"); + throw std::runtime_error("Failed wait on process handle"); } int @@ -115,12 +116,12 @@ BasicJob::ExitCode() if (!Success) { - spdlog::warn("failed getting exit code"); + ZEN_WARN("failed getting exit code"); } if (Ec == STILL_ACTIVE) { - spdlog::warn("getting exit code but process is STILL_ACTIVE"); + ZEN_WARN("getting exit code but process is STILL_ACTIVE"); } return gsl::narrow_cast<int>(Ec); @@ -221,7 +222,7 @@ SandboxedJob::Initialize(std::string_view AppContainerId) if (FAILED(hRes)) { - spdlog::error("Failed creating app container SID"); + ZEN_ERROR("Failed creating app container SID"); } } @@ -230,12 +231,12 @@ SandboxedJob::Initialize(std::string_view AppContainerId) PWSTR Str = nullptr; ::ConvertSidToStringSid(m_AppContainerSid, &Str); - spdlog::info("AppContainer SID : '{}'", WideToUtf8(Str)); + ZEN_INFO("AppContainer SID : '{}'", WideToUtf8(Str)); PWSTR Path = nullptr; if (SUCCEEDED(::GetAppContainerFolderPath(Str, &Path))) { - spdlog::info("AppContainer folder: '{}'", WideToUtf8(Path)); + ZEN_INFO("AppContainer folder: '{}'", WideToUtf8(Path)); ::CoTaskMemFree(Path); } @@ -311,13 +312,13 @@ SandboxedJob::SpawnJob(std::filesystem::path ExePath) return false; } - spdlog::info("Created process {}", ProcessInfo.dwProcessId); + ZEN_INFO("Created process {}", ProcessInfo.dwProcessId); return true; } HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::path& SandboxBaseDir) -: m_Log("exec", begin(spdlog::default_logger()->sinks()), end(spdlog::default_logger()->sinks())) +: m_Log(logging::Get("exec")) , m_CasStore(Store) , m_SandboxPath(SandboxBaseDir) { @@ -392,7 +393,7 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat if (!m_CasStore.FindChunk(FileHash)) { - spdlog::debug("NEED: {} {} {}", FileHash, Ob["file"sv].AsString(), Ob["size"sv].AsUInt64()); + ZEN_DEBUG("NEED: {} {} {}", FileHash, Ob["file"sv].AsString(), Ob["size"sv].AsUInt64()); NeedList.push_back(FileHash); } @@ -437,7 +438,7 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat std::filesystem::path SandboxDir{CreateNewSandbox()}; - spdlog::debug("setting up job in sandbox '{}'", SandboxDir); + ZEN_DEBUG("setting up job in sandbox '{}'", SandboxDir); zen::DeleteDirectories(SandboxDir); zen::CreateDirectories(SandboxDir); @@ -452,7 +453,7 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat if (IoBuffer Chunk = m_CasStore.FindChunk(FileHash); !Chunk) { - spdlog::debug("MISSING: {} {} {}", FileHash, FileName, FileSize); + ZEN_DEBUG("MISSING: {} {} {}", FileHash, FileName, FileSize); AllOk = false; NeedList.push_back(FileHash); @@ -479,7 +480,7 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat std::wstring Executable = Utf8ToWide(Executable8); std::wstring Args = Utf8ToWide(Args8); - spdlog::debug("spawning job in sandbox '{}': '{}' '{}'", SandboxDir, Executable8, Args8); + ZEN_DEBUG("spawning job in sandbox '{}': '{}' '{}'", SandboxDir, Executable8, Args8); std::filesystem::path ExeName = SandboxDir / Executable; @@ -515,7 +516,7 @@ HttpLaunchService::HandleRequest(HttpServerRequest& Request) { if (m_Router.HandleRequest(Request) == false) { - m_Log.warn("No route found for {0}", Request.RelativeUri()); + ZEN_WARN("No route found for {0}", Request.RelativeUri()); } } diff --git a/zenserver/testing/launch.h b/zenserver/testing/launch.h index a6eb137d2..49f12e2ec 100644 --- a/zenserver/testing/launch.h +++ b/zenserver/testing/launch.h @@ -2,9 +2,9 @@ #pragma once +#include <zencore/logging.h> #include <zenhttp/httpserver.h> -#include <spdlog/spdlog.h> #include <filesystem> namespace zen { @@ -24,7 +24,9 @@ public: virtual void HandleRequest(HttpServerRequest& Request) override; private: - spdlog::logger m_Log; + inline spdlog::logger& Log() { return m_Log; } + + spdlog::logger& m_Log; HttpRequestRouter m_Router; CasStore& m_CasStore; std::filesystem::path m_SandboxPath; diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index ba6300c65..0af92da6d 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -80,7 +80,7 @@ CloudCacheSession::GetDerivedData(std::string_view BucketId, std::string_view Ke Session.SetOption(cpr::Header{{"Authorization", Auth}}); cpr::Response Response = Session.Get(); - m_Log.debug("GET {}", Response); + ZEN_DEBUG("GET {}", Response); const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); @@ -111,7 +111,7 @@ CloudCacheSession::GetRef(std::string_view BucketId, const IoHash& Key, ZenConte Session.SetOption(cpr::Header{{"Authorization", Auth}, {"Accept", ContentType}}); cpr::Response Response = Session.Get(); - m_Log.debug("GET {}", Response); + ZEN_DEBUG("GET {}", Response); const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); @@ -134,7 +134,7 @@ CloudCacheSession::GetCompressedBlob(const IoHash& Key) Session.SetOption(cpr::Header{{"Authorization", Auth}, {"Accept", "application/x-ue-comp"}}); cpr::Response Response = Session.Get(); - m_Log.debug("GET {}", Response); + ZEN_DEBUG("GET {}", Response); const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); @@ -161,7 +161,7 @@ CloudCacheSession::PutDerivedData(std::string_view BucketId, std::string_view Ke Session.SetBody(cpr::Body{(const char*)DerivedData.Data(), DerivedData.Size()}); cpr::Response Response = Session.Put(); - m_Log.debug("PUT {}", Response); + ZEN_DEBUG("PUT {}", Response); return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } @@ -192,7 +192,7 @@ CloudCacheSession::PutRef(std::string_view BucketId, const IoHash& Key, IoBuffer Session.SetBody(cpr::Body{(const char*)Ref.Data(), Ref.Size()}); cpr::Response Response = Session.Put(); - m_Log.debug("PUT {}", Response); + ZEN_DEBUG("PUT {}", Response); return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } @@ -213,7 +213,7 @@ CloudCacheSession::PutCompressedBlob(const IoHash& Key, IoBuffer Blob) Session.SetBody(cpr::Body{(const char*)Blob.Data(), Blob.Size()}); cpr::Response Response = Session.Put(); - m_Log.debug("PUT {}", Response); + ZEN_DEBUG("PUT {}", Response); return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } @@ -268,7 +268,7 @@ CloudCacheClient::CloudCacheClient(const CloudCacheClientOptions& Options) { if (!Options.OAuthProvider.starts_with("http://"sv) && !Options.OAuthProvider.starts_with("https://"sv)) { - m_Log.warn("bad provider specification: '{}' - must be fully qualified"_format(Options.OAuthProvider).c_str()); + ZEN_WARN("bad provider specification: '{}' - must be fully qualified", Options.OAuthProvider); m_IsValid = false; return; @@ -280,7 +280,7 @@ CloudCacheClient::CloudCacheClient(const CloudCacheClientOptions& Options) if (SchemePos == std::string::npos) { - m_Log.warn("Bad service URL passed to cloud cache client: '{}'", Options.ServiceUrl); + ZEN_WARN("Bad service URL passed to cloud cache client: '{}'", Options.ServiceUrl); m_IsValid = false; return; @@ -290,7 +290,7 @@ CloudCacheClient::CloudCacheClient(const CloudCacheClientOptions& Options) if (DomainEnd == std::string::npos) { - m_Log.warn("Bad service URL passed to cloud cache client: '{}' no path delimiter found", Options.ServiceUrl); + ZEN_WARN("Bad service URL passed to cloud cache client: '{}' no path delimiter found", Options.ServiceUrl); m_IsValid = false; return; @@ -347,7 +347,7 @@ CloudCacheClient::AcquireAccessToken(std::string& AuthorizationHeaderValue) json11::Json JsonResponse = json11::Json::parse(Body, /* out */ JsonError); if (!JsonError.empty()) { - spdlog::warn("failed to parse OAuth response: '{}'", JsonError); + ZEN_WARN("failed to parse OAuth response: '{}'", JsonError); return false; } diff --git a/zenserver/upstream/jupiter.h b/zenserver/upstream/jupiter.h index 9f36704fa..5535ba000 100644 --- a/zenserver/upstream/jupiter.h +++ b/zenserver/upstream/jupiter.h @@ -2,12 +2,11 @@ #pragma once +#include <zencore/logging.h> #include <zencore/refcount.h> #include <zencore/thread.h> #include <zenhttp/httpserver.h> -#include <spdlog/spdlog.h> - #include <atomic> #include <list> #include <memory> @@ -75,6 +74,8 @@ public: std::vector<IoHash> Filter(std::string_view BucketId, const std::vector<IoHash>& ChunkHashes); private: + inline spdlog::logger& Log() { return m_Log; } + spdlog::logger& m_Log; RefPtr<CloudCacheClient> m_CacheClient; detail::CloudCacheSessionState* m_SessionState; diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index aef9c0bb3..97b222a68 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -419,17 +419,19 @@ private: ZEN_UNUSED(Endpoint); - m_Log.info("{} Endpoint: {}, Bytes: {:.2f} MB, Time: {:.2f} s, Speed: {:.2f} MB/s, Avg: {:.2f} ms/request, Samples: {}", - What, - Kv.first->DisplayName(), - TotalMb, - Counters.Seconds, - TotalMb / Counters.Seconds, - (Counters.Seconds * 1000.0) / double(Counters.Count), - Counters.Count); + ZEN_INFO("{} Endpoint: {}, Bytes: {:.2f} MB, Time: {:.2f} s, Speed: {:.2f} MB/s, Avg: {:.2f} ms/request, Samples: {}", + What, + Kv.first->DisplayName(), + TotalMb, + Counters.Seconds, + TotalMb / Counters.Seconds, + (Counters.Seconds * 1000.0) / double(Counters.Count), + Counters.Count); } } + spdlog::logger& Log() { return m_Log; } + spdlog::logger& m_Log; EndpointStats m_UpStats; EndpointStats m_DownStats; @@ -455,7 +457,7 @@ public: { auto NewEnd = std::remove_if(std::begin(m_Endpoints), std::end(m_Endpoints), [this](auto& Endpoint) { const bool Ok = Endpoint->Initialize(); - m_Log.info("{} [{}]", Endpoint->DisplayName(), Ok ? "OK" : "FAILED"); + ZEN_INFO("{} [{}]", Endpoint->DisplayName(), Ok ? "OK" : "FAILED"); return !Ok; }); @@ -522,9 +524,9 @@ private: if (!m_CacheStore.Get(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, CacheValue)) { - m_Log.warn("process upstream FAILED, '{}/{}', cache record doesn't exist", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash); + ZEN_WARN("process upstream FAILED, '{}/{}', cache record doesn't exist", + CacheRecord.CacheKey.Bucket, + CacheRecord.CacheKey.Hash); return; } @@ -536,10 +538,10 @@ private: } else { - m_Log.warn("process upstream FAILED, '{}/{}/{}', payload doesn't exist in CAS", - CacheRecord.CacheKey.Bucket, - CacheRecord.CacheKey.Hash, - PayloadId); + ZEN_WARN("process upstream FAILED, '{}/{}/{}', payload doesn't exist in CAS", + CacheRecord.CacheKey.Bucket, + CacheRecord.CacheKey.Hash, + PayloadId); return; } } @@ -567,7 +569,7 @@ private: } catch (std::exception& e) { - m_Log.warn("process upstream ({}/{}) FAILED '{}'", CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, e.what()); + ZEN_WARN("process upstream ({}/{}) FAILED '{}'", CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, e.what()); } } @@ -597,6 +599,8 @@ private: using UpstreamQueue = detail::BlockingQueue<UpstreamCacheRecord>; + spdlog::logger& Log() { return m_Log; } + spdlog::logger& m_Log; UpstreamCacheOptions m_Options; ::ZenCacheStore& m_CacheStore; diff --git a/zenserver/upstream/zen.cpp b/zenserver/upstream/zen.cpp index 7cdaa0036..eef92bab4 100644 --- a/zenserver/upstream/zen.cpp +++ b/zenserver/upstream/zen.cpp @@ -137,7 +137,7 @@ Mesh::EnqueueTick() { if (m_State != kExiting) { - spdlog::warn("Mesh timer error: {}", Ec.message()); + ZEN_WARN("Mesh timer error: {}", Ec.message()); } } }); @@ -198,12 +198,12 @@ Mesh::BroadcastPacket(CbObjectWriter& Obj) if (ErrorCode) { - spdlog::warn("packet broadcast failed: {}", ErrorCode.message()); + ZEN_WARN("packet broadcast failed: {}", ErrorCode.message()); } } else { - spdlog::warn("failed to open broadcast socket: {}", ErrorCode.message()); + ZEN_WARN("failed to open broadcast socket: {}", ErrorCode.message()); } } @@ -260,7 +260,7 @@ Mesh::IssueReceive() const uint16_t Port = (++It)->AsUInt16(m_SenderEndpoint.port()); const uint32_t Lsn = (++It)->AsUInt32(); - spdlog::info("received hey from {} ({})", SenderIp, SessionId); + ZEN_INFO("received hey from {} ({})", SenderIp, SessionId); RwLock::ExclusiveLockScope _(m_SessionsLock); @@ -279,7 +279,7 @@ Mesh::IssueReceive() { Oid SessionId = Field.AsObjectId(); - spdlog::info("received bye from {} ({})", SenderIp, SessionId); + ZEN_INFO("received bye from {} ({})", SenderIp, SessionId); // We could verify that it's sent from a known IP before erasing the // session, if we want to be paranoid @@ -295,13 +295,13 @@ Mesh::IssueReceive() } else { - spdlog::warn("received malformed message from {}", SenderIp); + ZEN_WARN("received malformed message from {}", SenderIp); } } break; default: - spdlog::warn("received malformed data from {}", SenderIp); + ZEN_WARN("received malformed data from {}", SenderIp); break; } } @@ -392,7 +392,7 @@ ZenStructuredCacheSession::GetCacheRecord(std::string_view BucketId, const IoHas Session.SetHeader(cpr::Header{{"Accept", Type == ZenContentType::kCbObject ? "application/x-ue-cb" : "application/octet-stream"}}); cpr::Response Response = Session.Get(); - m_Log.debug("GET {}", Response); + ZEN_DEBUG("GET {}", Response); const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); @@ -412,7 +412,7 @@ ZenStructuredCacheSession::GetCachePayload(std::string_view BucketId, const IoHa Session.SetHeader(cpr::Header{{"Accept", "application/x-ue-comp"}}); cpr::Response Response = Session.Get(); - m_Log.debug("GET {}", Response); + ZEN_DEBUG("GET {}", Response); const bool Success = Response.status_code == 200; const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); @@ -434,7 +434,7 @@ ZenStructuredCacheSession::PutCacheRecord(std::string_view BucketId, const IoHas Session.SetBody(cpr::Body{static_cast<const char*>(Value.Data()), Value.Size()}); cpr::Response Response = Session.Put(); - m_Log.debug("PUT {}", Response); + ZEN_DEBUG("PUT {}", Response); return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } @@ -452,7 +452,7 @@ ZenStructuredCacheSession::PutCachePayload(std::string_view BucketId, const IoHa Session.SetBody(cpr::Body{static_cast<const char*>(Payload.Data()), Payload.Size()}); cpr::Response Response = Session.Put(); - m_Log.debug("PUT {}", Response); + ZEN_DEBUG("PUT {}", Response); return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; } diff --git a/zenserver/upstream/zen.h b/zenserver/upstream/zen.h index c4bff8980..541495818 100644 --- a/zenserver/upstream/zen.h +++ b/zenserver/upstream/zen.h @@ -111,6 +111,8 @@ public: ZenCacheResult PutCachePayload(std::string_view BucketId, const IoHash& Key, const IoHash& PayloadId, IoBuffer Payload); private: + inline spdlog::logger& Log() { return m_Log; } + spdlog::logger& m_Log; ZenStructuredCacheClient& m_Client; detail::ZenCacheSessionState* m_SessionState; diff --git a/zenserver/vfs.cpp b/zenserver/vfs.cpp index 18d8f1842..86e265b20 100644 --- a/zenserver/vfs.cpp +++ b/zenserver/vfs.cpp @@ -8,13 +8,13 @@ # include <zencore/snapshot_manifest.h> # include <zencore/stream.h> # include <zencore/windows.h> +# include <zencore/logging.h> # include <zenstore/CAS.h> # include <map> # include <atlfile.h> # include <projectedfslib.h> -# include <spdlog/spdlog.h> # pragma comment(lib, "projectedfslib.lib") @@ -54,7 +54,7 @@ public: HRESULT hRes = ManifestFile.Create(ManifestSpec.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); if (FAILED(hRes)) { - spdlog::error("MANIFEST NOT FOUND!"); // TODO: add context + ZEN_ERROR("MANIFEST NOT FOUND!"); // TODO: add context return hRes; } @@ -588,11 +588,11 @@ retry: if (SUCCEEDED(hRes)) { - spdlog::info("Successfully mounted snapshot at '{}'!", WideToUtf8(RootPath.c_str())); + ZEN_INFO("Successfully mounted snapshot at '{}'!", WideToUtf8(RootPath.c_str())); } else { - spdlog::info("Failed mounting snapshot at '{}'!", WideToUtf8(RootPath.c_str())); + ZEN_INFO("Failed mounting snapshot at '{}'!", WideToUtf8(RootPath.c_str())); } return hRes; diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index c9f74daa4..3b56d8683 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -3,6 +3,7 @@ #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/iobuffer.h> +#include <zencore/logging.h> #include <zencore/refcount.h> #include <zencore/scopeguard.h> #include <zencore/string.h> @@ -17,7 +18,6 @@ #include <fmt/format.h> #include <mimalloc-new-delete.h> #include <mimalloc.h> -#include <spdlog/spdlog.h> #include <asio.hpp> #include <exception> #include <list> @@ -88,7 +88,7 @@ public: void Initialize(ZenServiceConfig& ServiceConfig, int BasePort, int ParentPid) { using namespace fmt::literals; - spdlog::info(ZEN_APP_NAME " initializing"); + ZEN_INFO(ZEN_APP_NAME " initializing"); m_DebugOptionForcedCrash = ServiceConfig.ShouldCrash; @@ -98,11 +98,11 @@ public: if (!m_Process.IsValid()) { - spdlog::warn("Unable to initialize process handle for specified parent pid #{}", ParentPid); + ZEN_WARN("Unable to initialize process handle for specified parent pid #{}", ParentPid); } else { - spdlog::info("Using parent pid #{} to control process lifetime", ParentPid); + ZEN_INFO("Using parent pid #{} to control process lifetime", ParentPid); } } @@ -112,12 +112,12 @@ public: if (zen::NamedMutex::Exists(MutexName) || (m_ServerMutex.Create(MutexName) == false)) { - throw std::exception("Failed to create mutex '{}' - is another instance already running?"_format(MutexName).c_str()); + throw std::runtime_error("Failed to create mutex '{}' - is another instance already running?"_format(MutexName).c_str()); } // Ok so now we're configured, let's kick things off - spdlog::info("initializing storage"); + ZEN_INFO("initializing storage"); zen::CasStoreConfiguration Config; Config.RootDirectory = m_DataRoot / "cas"; @@ -126,13 +126,13 @@ public: m_CidStore = std::make_unique<zen::CidStore>(*m_CasStore, m_DataRoot / "cid"); - spdlog::info("instantiating project service"); + ZEN_INFO("instantiating project service"); m_ProjectStore = new zen::ProjectStore(*m_CasStore, m_DataRoot / "projects"); m_HttpProjectService.reset(new zen::HttpProjectService{*m_CasStore, m_ProjectStore}); m_LocalProjectService = zen::LocalProjectService::New(*m_CasStore, m_ProjectStore); - spdlog::info("instantiating compute services"); + ZEN_INFO("instantiating compute services"); std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox"; zen::CreateDirectories(SandboxDir); @@ -147,7 +147,7 @@ public: using namespace std::literals; auto ValueOrDefault = [](std::string_view Value, std::string_view Default) { return Value.empty() ? Default : Value; }; - spdlog::info("instantiating structured cache service"); + ZEN_INFO("instantiating structured cache service"); m_CacheStore = std::make_unique<ZenCacheStore>(*m_CasStore, m_DataRoot / "cache"); std::unique_ptr<zen::UpstreamCache> UpstreamCache; @@ -201,12 +201,12 @@ public: if (UpstreamCache->Initialize()) { - spdlog::info("upstream cache active"); + ZEN_INFO("upstream cache active"); } else { UpstreamCache.reset(); - spdlog::info("NOT using upstream cache"); + ZEN_INFO("NOT using upstream cache"); } } @@ -215,7 +215,7 @@ public: } else { - spdlog::info("NOT instantiating structured cache service"); + ZEN_INFO("NOT instantiating structured cache service"); } if (ServiceConfig.MeshEnabled) @@ -224,7 +224,7 @@ public: } else { - spdlog::info("NOT starting mesh"); + ZEN_INFO("NOT starting mesh"); } m_Http = zen::CreateHttpServer(); @@ -261,7 +261,7 @@ public: void StartMesh(int BasePort) { - spdlog::info("initializing mesh discovery"); + ZEN_INFO("initializing mesh discovery"); m_ZenMesh.Start(uint16_t(BasePort)); } @@ -274,15 +274,15 @@ public: if (!m_TestMode) { - spdlog::info("__________ _________ __ "); - spdlog::info("\\____ /____ ____ / _____// |_ ___________ ____ "); - spdlog::info(" / // __ \\ / \\ \\_____ \\\\ __\\/ _ \\_ __ \\_/ __ \\ "); - spdlog::info(" / /\\ ___/| | \\ / \\| | ( <_> ) | \\/\\ ___/ "); - spdlog::info("/_______ \\___ >___| / /_______ /|__| \\____/|__| \\___ >"); - spdlog::info(" \\/ \\/ \\/ \\/ \\/ "); + ZEN_INFO("__________ _________ __ "); + ZEN_INFO("\\____ /____ ____ / _____// |_ ___________ ____ "); + ZEN_INFO(" / // __ \\ / \\ \\_____ \\\\ __\\/ _ \\_ __ \\_/ __ \\ "); + ZEN_INFO(" / /\\ ___/| | \\ / \\| | ( <_> ) | \\/\\ ___/ "); + ZEN_INFO("/_______ \\___ >___| / /_______ /|__| \\____/|__| \\___ >"); + ZEN_INFO(" \\/ \\/ \\/ \\/ \\/ "); } - spdlog::info(ZEN_APP_NAME " now running"); + ZEN_INFO(ZEN_APP_NAME " now running"); #if USE_SENTRY sentry_clear_modulecache(); @@ -295,7 +295,7 @@ public: m_Http->Run(m_TestMode); - spdlog::info(ZEN_APP_NAME " exiting"); + ZEN_INFO(ZEN_APP_NAME " exiting"); m_IoContext.stop(); @@ -308,8 +308,9 @@ public: m_Http->RequestExit(); } - void Cleanup() { spdlog::info(ZEN_APP_NAME " cleaning up"); } + void Cleanup() { ZEN_INFO(ZEN_APP_NAME " cleaning up"); } + void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; } void SetTestMode(bool State) { m_TestMode = State; } void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; } @@ -337,7 +338,7 @@ public: } else { - spdlog::info(ZEN_APP_NAME " exiting since parent process id {} is gone", m_Process.Pid()); + ZEN_INFO(ZEN_APP_NAME " exiting since parent process id {} is gone", m_Process.Pid()); RequestExit(0); } @@ -359,7 +360,8 @@ public: } private: - bool m_TestMode = false; + bool m_IsDedicatedMode = false; + bool m_TestMode = false; std::filesystem::path m_DataRoot; std::jthread m_IoRunner; asio::io_context m_IoContext; @@ -413,7 +415,7 @@ main(int argc, char* argv[]) ParseServiceConfig(GlobalOptions.DataDir, /* out */ ServiceConfig); - spdlog::info("zen cache server starting on port {}", GlobalOptions.BasePort); + ZEN_INFO("zen cache server starting on port {}", GlobalOptions.BasePort); try { @@ -425,7 +427,7 @@ main(int argc, char* argv[]) { // Instance already running for this port? Should double check pid - spdlog::warn("Looks like there is already a process listening to this port (pid: {})", Entry->Pid); + ZEN_WARN("Looks like there is already a process listening to this port (pid: {})", Entry->Pid); } else { @@ -442,14 +444,15 @@ main(int argc, char* argv[]) ZenServer Server; Server.SetDataRoot(GlobalOptions.DataDir); Server.SetTestMode(GlobalOptions.IsTest); + Server.SetDedicatedMode(GlobalOptions.IsDedicated); Server.Initialize(ServiceConfig, GlobalOptions.BasePort, GlobalOptions.OwnerPid); // Monitor shutdown signals ShutdownThread.reset(new std::thread{[&] { - spdlog::info("shutdown monitor thread waiting for shutdown signal '{}'", ShutdownEventName); + ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}'", ShutdownEventName); ShutdownEvent->Wait(); - spdlog::info("shutdown signal received"); + ZEN_INFO("shutdown signal received"); Server.RequestExit(0); }}); diff --git a/zenstore/CAS.cpp b/zenstore/CAS.cpp index c05e3703d..e77c0ed64 100644 --- a/zenstore/CAS.cpp +++ b/zenstore/CAS.cpp @@ -8,13 +8,12 @@ #include <doctest/doctest.h> #include <zencore/except.h> #include <zencore/fmtutils.h> +#include <zencore/logging.h> #include <zencore/memory.h> #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/uid.h> -#include <spdlog/spdlog.h> - #include <gsl/gsl-lite.hpp> #include <filesystem> @@ -64,7 +63,7 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) { m_Config = InConfig; - spdlog::info("initializing CAS pool at {}", m_Config.RootDirectory); + ZEN_INFO("initializing CAS pool at {}", m_Config.RootDirectory); // Ensure root directory exists - create if it doesn't exist already diff --git a/zenstore/caslog.cpp b/zenstore/caslog.cpp index 169629053..70bcf4669 100644 --- a/zenstore/caslog.cpp +++ b/zenstore/caslog.cpp @@ -7,6 +7,7 @@ #include <zencore/except.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> +#include <zencore/logging.h> #include <zencore/memory.h> #include <zencore/string.h> #include <zencore/thread.h> @@ -14,7 +15,6 @@ #include <xxhash.h> -#include <spdlog/spdlog.h> #include <gsl/gsl-lite.hpp> #include <functional> @@ -81,7 +81,7 @@ CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, bool IsCreat if ((0 != memcmp(Header.Magic, FileHeader::MagicSequence, sizeof Header.Magic)) || (Header.Checksum != Header.ComputeChecksum())) { // TODO: provide more context! - throw std::exception("Mangled log header"); + throw std::runtime_error("Mangled log header"); } ULONGLONG Sz; diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp index 4e5188f1c..e6c7f98ee 100644 --- a/zenstore/cidstore.cpp +++ b/zenstore/cidstore.cpp @@ -2,11 +2,12 @@ #include "zenstore/cidstore.h" +#include <zencore/compress.h> #include <zencore/filesystem.h> +#include <zencore/logging.h> #include <zenstore/CAS.h> #include <zenstore/caslog.h> -#include <spdlog/spdlog.h> #include <filesystem> namespace zen { @@ -27,10 +28,25 @@ struct CidStore::CidState RwLock m_Lock; tsl::robin_map<IoHash, IoHash> m_CidMap; + CasStore::InsertResult AddChunk(CompressedBuffer& ChunkData) + { + IoBuffer Payload = ChunkData.GetCompressed().Flatten().AsIoBuffer(); + IoHash CompressedHash = IoHash::HashBuffer(Payload.Data(), Payload.Size()); + + CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, CompressedHash); + AddCompressedCid(IoHash::FromBLAKE3(ChunkData.GetRawHash()), CompressedHash); + + return Result; + } + void AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed) { RwLock::ExclusiveLockScope _(m_Lock); m_CidMap.insert_or_assign(DecompressedId, Compressed); + // TODO: it's pretty wasteful to log even idempotent updates + // however we can't simply use the boolean returned by insert_or_assign + // since there's not a 1:1 mapping between compressed and uncompressed + // so if we want a last-write-wins policy then we have to log each update m_LogFile.Append({.Uncompressed = DecompressedId, .Compressed = Compressed}); } @@ -70,7 +86,7 @@ struct CidStore::CidState m_LogFile.Replay([&](const IndexEntry& Ie) { m_CidMap.insert_or_assign(Ie.Uncompressed, Ie.Compressed); }); - spdlog::debug("CID index initialized: {} entries found", m_CidMap.size()); + ZEN_DEBUG("CID index initialized: {} entries found", m_CidMap.size()); } void Flush() { m_LogFile.Flush(); } @@ -87,6 +103,12 @@ CidStore::~CidStore() { } +CasStore::InsertResult +CidStore::AddChunk(CompressedBuffer& ChunkData) +{ + return m_Impl->AddChunk(ChunkData); +} + void CidStore::AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed) { diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp index 56d25e729..170f13875 100644 --- a/zenstore/filecas.cpp +++ b/zenstore/filecas.cpp @@ -5,12 +5,12 @@ #include <zencore/except.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> +#include <zencore/logging.h> #include <zencore/memory.h> #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/uid.h> -#include <spdlog/spdlog.h> #include <gsl/gsl-lite.hpp> #include <filesystem> @@ -84,7 +84,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) if (!Success) { - spdlog::warn("Failed to flag temporary payload file for deletion: '{}'", PathFromHandle(FileRef.FileHandle)); + ZEN_WARN("Failed to flag temporary payload file for deletion: '{}'", PathFromHandle(FileRef.FileHandle)); } }; @@ -173,9 +173,9 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) return CasStore::InsertResult{.New = true}; } - spdlog::warn("rename of CAS payload file failed ('{}'), falling back to regular write for insert of {}", - GetLastErrorAsString(), - ChunkHash); + ZEN_WARN("rename of CAS payload file failed ('{}'), falling back to regular write for insert of {}", + GetLastErrorAsString(), + ChunkHash); DeletePayloadFileOnClose(); } @@ -224,7 +224,7 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize if ((hRes != HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND)) && (hRes != HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND))) { - spdlog::warn("Unexpected error code when opening shard file for read: {:#x}", uint32_t(hRes)); + ZEN_WARN("Unexpected error code when opening shard file for read: {:#x}", uint32_t(hRes)); } auto InternalCreateFile = [&] { return PayloadFile.Create(ShardedPath.c_str(), GENERIC_WRITE, FILE_SHARE_DELETE, CREATE_ALWAYS); }; diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h index 2c2b395a5..62d642ad1 100644 --- a/zenstore/include/zenstore/cidstore.h +++ b/zenstore/include/zenstore/cidstore.h @@ -4,6 +4,7 @@ #include <tsl/robin_map.h> #include <zencore/iohash.h> +#include <zenstore/CAS.h> namespace std::filesystem { class path; @@ -12,6 +13,7 @@ class path; namespace zen { class CasStore; +class CompressedBuffer; class IoBuffer; /** Content Store @@ -29,10 +31,13 @@ public: CidStore(CasStore& InCasStore, const std::filesystem::path& RootDir); ~CidStore(); - void AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed); - IoBuffer FindChunkByCid(const IoHash& DecompressedId); - bool ContainsChunk(const IoHash& DecompressedId); - void Flush(); + CasStore::InsertResult AddChunk(CompressedBuffer& ChunkData); + void AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed); + IoBuffer FindChunkByCid(const IoHash& DecompressedId); + bool ContainsChunk(const IoHash& DecompressedId); + void Flush(); + + // TODO: add batch filter support private: struct CidState; diff --git a/zenutil/include/zenserverprocess.h b/zenutil/include/zenserverprocess.h index f7d911a87..b81d61c25 100644 --- a/zenutil/include/zenserverprocess.h +++ b/zenutil/include/zenserverprocess.h @@ -3,9 +3,9 @@ #pragma once #include <zencore/enumflags.h> +#include <zencore/logging.h> #include <zencore/thread.h> -#include <spdlog/spdlog.h> #include <gsl/gsl-lite.hpp> #include <atomic> diff --git a/zenutil/zenserverprocess.cpp b/zenutil/zenserverprocess.cpp index 093f18f6a..00f5d4c0d 100644 --- a/zenutil/zenserverprocess.cpp +++ b/zenutil/zenserverprocess.cpp @@ -5,11 +5,11 @@ #include <zencore/except.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> +#include <zencore/logging.h> #include <zencore/string.h> #include <atlbase.h> #include <shellapi.h> -#include <spdlog/spdlog.h> #include <source_location> #include <zencore/windows.h> @@ -223,7 +223,7 @@ ZenServerState::Sweep() { if (zen::IsProcessRunning(Entry.Pid) == false) { - spdlog::debug("Sweep - pid {} not running, reclaiming entry (port {})", Entry.Pid, Entry.ListenPort); + ZEN_DEBUG("Sweep - pid {} not running, reclaiming entry (port {})", Entry.Pid, Entry.ListenPort); Entry.Reset(); } @@ -281,7 +281,7 @@ ZenServerEnvironment::Initialize(std::filesystem::path ProgramBaseDir) { m_ProgramBaseDir = ProgramBaseDir; - spdlog::debug("Program base dir is '{}'", ProgramBaseDir); + ZEN_DEBUG("Program base dir is '{}'", ProgramBaseDir); m_IsInitialized = true; } @@ -292,8 +292,8 @@ ZenServerEnvironment::InitializeForTest(std::filesystem::path ProgramBaseDir, st m_ProgramBaseDir = ProgramBaseDir; m_TestBaseDir = TestBaseDir; - spdlog::info("Program base dir is '{}'", ProgramBaseDir); - spdlog::info("Cleaning test base dir '{}'", TestBaseDir); + ZEN_INFO("Program base dir is '{}'", ProgramBaseDir); + ZEN_INFO("Cleaning test base dir '{}'", TestBaseDir); zen::DeleteDirectories(TestBaseDir.c_str()); m_IsTestInstance = true; @@ -310,7 +310,7 @@ ZenServerEnvironment::CreateNewTestDir() std::filesystem::path TestPath = m_TestBaseDir / TestDir.c_str(); - spdlog::info("Creating new test dir @ '{}'", TestPath); + ZEN_INFO("Creating new test dir @ '{}'", TestPath); zen::CreateDirectories(TestPath.c_str()); @@ -354,7 +354,7 @@ ZenServerInstance::Shutdown() { if (m_Terminate) { - spdlog::info("Terminating zenserver process"); + ZEN_INFO("Terminating zenserver process"); m_Process.Terminate(111); } else @@ -419,7 +419,7 @@ ZenServerInstance::SpawnServer(int BasePort) std::filesystem::path CurrentDirectory = std::filesystem::current_path(); - spdlog::debug("Spawning server '{}'", LogId); + ZEN_DEBUG("Spawning server '{}'", LogId); PROCESS_INFORMATION ProcessInfo{}; STARTUPINFO StartupInfo{.cb = sizeof(STARTUPINFO)}; @@ -463,7 +463,7 @@ ZenServerInstance::SpawnServer(int BasePort) { // Try launching elevated process - spdlog::debug("Regular spawn failed - spawning elevated server"); + ZEN_DEBUG("Regular spawn failed - spawning elevated server"); SHELLEXECUTEINFO ShellExecuteInfo; ZeroMemory(&ShellExecuteInfo, sizeof(ShellExecuteInfo)); @@ -486,14 +486,14 @@ ZenServerInstance::SpawnServer(int BasePort) { std::error_code err(WinError, std::system_category()); - spdlog::error("Server spawn failed: {}", err.message()); + ZEN_ERROR("Server spawn failed: {}", err.message()); throw std::system_error(err, "failed to create server process"); } } } - spdlog::debug("Server '{}' spawned OK", LogId); + ZEN_DEBUG("Server '{}' spawned OK", LogId); if (IsTest) { @@ -524,7 +524,7 @@ ZenServerInstance::AttachToRunningServer(int BasePort) if (!State.InitializeReadOnly()) { // TODO: return success/error code instead? - throw std::exception("No zen state found"); + throw std::runtime_error("No zen state found"); } const ZenServerState::ZenServerEntry* Entry = nullptr; @@ -541,7 +541,7 @@ ZenServerInstance::AttachToRunningServer(int BasePort) if (!Entry) { // TODO: return success/error code instead? - throw std::exception("No server found"); + throw std::runtime_error("No server found"); } m_Process.Initialize(Entry->Pid); |