switch to xmake for package management (#611)

This change removes our dependency on vcpkg for package management, in favour of bringing some code in-tree in the `thirdparty` folder as well as using the xmake build-in package management feature. For the latter, all the package definitions are maintained in the zen repo itself, in the `repo` folder.

It should now also be easier to build the project as it will no longer depend on having the right version of vcpkg installed, which has been a common problem for new people coming in to the codebase. Now you should only need xmake to build.

* Bumps xmake requirement on github runners to 2.9.9 to resolve an issue where xmake on Windows invokes cmake with `v144` toolchain which does not exist
* BLAKE3 is now in-tree at `thirdparty/blake3`
* cpr is now in-tree at `thirdparty/cpr`
* cxxopts is now in-tree at `thirdparty/cxxopts`
* fmt is now in-tree at `thirdparty/fmt`
* robin-map is now in-tree at `thirdparty/robin-map`
* ryml is now in-tree at `thirdparty/ryml`
* sol2 is now in-tree at `thirdparty/sol2`
* spdlog is now in-tree at `thirdparty/spdlog`
* utfcpp is now in-tree at `thirdparty/utfcpp`
* xmake package repo definitions is in `repo`
* implemented support for sanitizers. ASAN is supported on windows, TSAN, UBSAN, MSAN etc are supported on Linux/MacOS though I have not yet tested it extensively on MacOS
* the zencore encryption implementation also now supports using mbedTLS which is used on MacOS, though for now we still use openssl on Linux
* crashpad
* bumps libcurl to 8.11.0 (from 8.8.0) which should address a rare build upload bug

5 files changed, 389 insertions, 0 deletions

diff --git a/thirdparty/ryml/tools/amalgamate.py b/thirdparty/ryml/tools/amalgamate.py
new file mode 100644
index 000000000..221397f03
--- /dev/null
+++ b/thirdparty/ryml/tools/amalgamate.py
@@ -0,0 +1,130 @@
+import re
+import os
+from os.path import abspath, dirname
+import sys
+import subprocess
+import argparse
+
+
+projdir = abspath(dirname(dirname(__file__)))
+sys.path.insert(0, f"{projdir}/ext/c4core/cmake")
+import amalgamate_utils as am
+sys.path.insert(0, f"{projdir}/ext/c4core/tools")
+import amalgamate as am_c4core
+
+ryml_defmacro = "RYML_SINGLE_HDR_DEFINE_NOW"
+c4core_defmacro = "C4CORE_SINGLE_HDR_DEFINE_NOW"
+exports_def_code = f""" // shared library: export when defining
+#if defined(RYML_SHARED) && defined({ryml_defmacro}) && !defined(RYML_EXPORTS)
+#define RYML_EXPORTS
+#endif
+"""
+c4core_def_code = f""" // propagate defines to c4core
+#if defined({ryml_defmacro}) && !defined({c4core_defmacro})
+#define {c4core_defmacro}
+#endif
+
+#if defined(RYML_EXPORTS) && !defined(C4CORE_EXPORTS)
+#define C4CORE_EXPORTS
+#endif
+
+#if defined(RYML_SHARED) && !defined(C4CORE_SHARED)
+#define C4CORE_SHARED
+#endif
+
+// workaround for include removal while amalgamating
+// resulting in <stdarg.h> missing in arm-none-eabi-g++
+// https://github.com/biojppm/rapidyaml/issues/193
+#include <stdarg.h>
+"""
+
+
+def amalgamate_ryml(filename: str,
+                    with_c4core: bool,
+                    with_fastfloat: bool,
+                    with_stl: bool):
+    c4core_amalgamated = ""
+    if with_c4core:
+        c4core_amalgamated = "src/c4/c4core_all.hpp"
+        am_c4core.amalgamate_c4core(f"{projdir}/{c4core_amalgamated}",
+                                    with_fastfloat=with_fastfloat,
+                                    with_stl=with_stl)
+    repo = "https://github.com/biojppm/rapidyaml"
+    defmacro = ryml_defmacro
+    srcfiles = [
+        am.cmttext(f"""
+Rapid YAML - a library to parse and emit YAML, and do it fast.
+
+{repo}
+
+DO NOT EDIT. This file is generated automatically.
+This is an amalgamated single-header version of the library.
+
+INSTRUCTIONS:
+  - Include at will in any header of your project
+  - In one (and only one) of your project source files,
+    #define {defmacro} and then include this header.
+    This will enable the function and class definitions in
+    the header file.
+  - To compile into a shared library, just define the
+    preprocessor symbol RYML_SHARED . This will take
+    care of symbol export/import.
+"""),
+        am.cmtfile("LICENSE.txt"),
+        am.injcode(exports_def_code),
+        am.onlyif(with_c4core, am.injcode(c4core_def_code)),
+        am.onlyif(with_c4core, c4core_amalgamated),
+        "src/c4/yml/export.hpp",
+        "src/c4/yml/common.hpp",
+        "src/c4/yml/tree.hpp",
+        "src/c4/yml/node.hpp",
+        "src/c4/yml/writer.hpp",
+        "src/c4/yml/detail/parser_dbg.hpp",
+        am.injcode("#define C4_YML_EMIT_DEF_HPP_"),
+        "src/c4/yml/emit.hpp",
+        "src/c4/yml/emit.def.hpp",
+        "src/c4/yml/detail/stack.hpp",
+        "src/c4/yml/parse.hpp",
+        am.onlyif(with_stl, "src/c4/yml/std/map.hpp"),
+        am.onlyif(with_stl, "src/c4/yml/std/string.hpp"),
+        am.onlyif(with_stl, "src/c4/yml/std/vector.hpp"),
+        am.onlyif(with_stl, "src/c4/yml/std/std.hpp"),
+        "src/c4/yml/common.cpp",
+        "src/c4/yml/tree.cpp",
+        "src/c4/yml/parse.cpp",
+        "src/c4/yml/node.cpp",
+        "src/c4/yml/preprocess.hpp",
+        "src/c4/yml/preprocess.cpp",
+        "src/c4/yml/detail/checks.hpp",
+        "src/c4/yml/detail/print.hpp",
+        "src/c4/yml/yml.hpp",
+        "src/ryml.hpp",
+    ]
+    result = am.catfiles(srcfiles,
+                         projdir,
+                         # comment out lines with these patterns:
+                         include_regexes=[
+                             re.compile(r'^\s*#\s*include "(c4/yml/.*)".*$'),
+                             re.compile(r'^\s*#\s*include <(c4/yml/.*)>.*$'),
+                             re.compile(r'^\s*#\s*include "(c4/.*)".*$'),
+                             re.compile(r'^\s*#\s*include <(c4/.*)>.*$'),
+                         ],
+                         definition_macro=defmacro,
+                         repo=repo,
+                         result_incguard="_RYML_SINGLE_HEADER_AMALGAMATED_HPP_")
+    result_with_only_first_includes = am.include_only_first(result)
+    am.file_put_contents(filename, result_with_only_first_includes)
+
+
+def mkparser():
+    return am.mkparser(c4core=(True, "amalgamate c4core together with ryml"),
+                       fastfloat=(True, "enable fastfloat library"),
+                       stl=(True, "enable stl interop"))
+
+
+if __name__ == "__main__":
+    args = mkparser().parse_args()
+    amalgamate_ryml(filename=args.output,
+                    with_c4core=args.c4core,
+                    with_fastfloat=args.fastfloat,
+                    with_stl=args.stl)
diff --git a/thirdparty/ryml/tools/parse_emit.cpp b/thirdparty/ryml/tools/parse_emit.cpp
new file mode 100644
index 000000000..6ae807964
--- /dev/null
+++ b/thirdparty/ryml/tools/parse_emit.cpp
@@ -0,0 +1,116 @@
+#ifdef RYML_SINGLE_HEADER
+#include <ryml_all.hpp>
+#else
+#include <c4/yml/std/std.hpp>
+#include <c4/yml/parse.hpp>
+#include <c4/yml/emit.hpp>
+#endif
+#include <c4/fs/fs.hpp>
+
+#include <cstdio>
+#include <chrono>
+
+
+using namespace c4;
+
+
+//-----------------------------------------------------------------------------
+
+struct timed_section
+{
+    using myclock = std::chrono::steady_clock;
+    using msecs = std::chrono::duration<double, std::milli>;
+
+    csubstr name;
+    myclock::time_point start;
+
+    msecs since() const { return myclock::now() - start; }
+    timed_section(csubstr n) : name(n), start(myclock::now()) {}
+    ~timed_section()
+    {
+        fprintf(stderr, "%.6fms: %.*s\n", since().count(), (int)name.len, name.str);
+        fflush(stderr);
+    }
+};
+
+#define TS(name) timed_section name##__##__LINE__(#name)
+
+
+//-----------------------------------------------------------------------------
+//-----------------------------------------------------------------------------
+//-----------------------------------------------------------------------------
+
+int main(int argc, const char *argv[])
+{
+    bool print_emitted_to_stdout = true;
+    csubstr file;
+    // LCOV_EXCL_START
+    auto show_usage = [argv]{
+        printf("usage: %s [-s] <path/to/file.yaml>\n", argv[0]);
+    };
+    if(argc == 2)
+    {
+        file = to_csubstr(argv[1]);
+    }
+    else if(argc > 2)
+    {
+        file = to_csubstr(argv[2]);
+        csubstr arg = to_csubstr(argv[1]);
+        if(arg == "-s")
+        {
+            print_emitted_to_stdout = false;
+        }
+        else
+        {
+            show_usage();
+            return 1;
+        }
+    }
+    else
+    {
+        show_usage();
+        return 1;
+    }
+    // LCOV_EXCL_STOP
+
+    TS(TOTAL);
+
+    C4_CHECK_MSG(fs::path_exists(file.str), "cannot find file: %s (cwd=%s)", file.str, fs::cwd<std::string>().c_str());
+
+    {
+        TS(objects);
+        std::string contents, output;
+        yml::Tree tree;
+        {
+            TS(read_file);
+            fs::file_get_contents(file.str, &contents);
+        }
+        {
+            TS(tree_reserve);
+            size_t nlines;
+            {
+                TS(count_lines);
+                nlines = to_csubstr(contents).count('\n');
+            }
+            fprintf(stderr, "reserving #lines=%zu\n", nlines);
+            tree.reserve(nlines);
+        }
+        {
+            TS(parse_yml);
+            yml::parse_in_place(file, to_substr(contents), &tree);
+        }
+        {
+            TS(emit_to_buffer);
+            output.resize(contents.size()); // resize, not just reserve
+            yml::emitrs_yaml(tree, &output);
+        }
+        if(print_emitted_to_stdout)
+        {
+            TS(print_stdout);
+            fwrite(output.data(), 1, output.size(), stdout);
+            putchar('\n');
+        }
+    }
+
+    return 0;
+}
diff --git a/thirdparty/ryml/tools/test_suite/Dockerfile b/thirdparty/ryml/tools/test_suite/Dockerfile
new file mode 100644
index 000000000..fc5b51d65
--- /dev/null
+++ b/thirdparty/ryml/tools/test_suite/Dockerfile
@@ -0,0 +1,33 @@
+FROM alpine:latest
+
+# to run:
+#    docker run --rm -it -v $PWD:/host alpine sh
+#    docker run --rm -it -v $PWD:/host -w /host -v /tmp/bash_history:/root/.bash_history yts-importing bash
+#
+
+RUN apk add build-base \
+ && apk add \
+        bash \
+        curl \
+        fortune \
+        git \
+        jq \
+        perl \
+        perl-app-cpanminus \
+        tig \
+        vim \
+        wget \
+        python \
+        cmake \
+        ninja \
+ && true
+
+RUN cpanm -n \
+        boolean \
+        Capture::Tiny \
+        XXX \
+        YAML::PP \
+ && true
+
+
+ENV PYTHONPATH=/python/lib/python3.7/site-packages
diff --git a/thirdparty/ryml/tools/test_suite/run_test_suite.sh b/thirdparty/ryml/tools/test_suite/run_test_suite.sh
new file mode 100755
index 000000000..4f955f67b
--- /dev/null
+++ b/thirdparty/ryml/tools/test_suite/run_test_suite.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -x
+set -e
+
+d=$1
+[ "$d" == "" ] && d=.
+if [ ! -d $d ] ; then
+    echo "$d is not a directory"
+    exit 1
+fi
+d=$(cd $d ; pwd) # get absolute path
+
+cd $d/yaml-test-runtimes
+make force build
+cd $d/yaml-test-suite
+make clean run-tests export
+xsel -b <export.tsv
diff --git a/thirdparty/ryml/tools/yaml_events.cpp b/thirdparty/ryml/tools/yaml_events.cpp
new file mode 100644
index 000000000..0d3031f64
--- /dev/null
+++ b/thirdparty/ryml/tools/yaml_events.cpp
@@ -0,0 +1,92 @@
+#ifdef RYML_SINGLE_HEADER
+#include <ryml_all.hpp>
+#else
+#include <c4/yml/std/std.hpp>
+#include <c4/yml/parse.hpp>
+#endif
+#include <test_suite/test_suite_events.hpp>
+#include <c4/fs/fs.hpp>
+#include <cstdio>
+#include <stdexcept>
+
+using namespace c4;
+using namespace c4::yml;
+
+void usage(const char *exename);
+std::string load_file(csubstr filename);
+void report_error(const char* msg, size_t length, Location loc, FILE *f);
+
+
+int main(int argc, const char *argv[])
+{
+    if(argc < 2)
+    {
+        usage(argv[0]);
+        return 1;
+    }
+    Callbacks callbacks = {};
+    callbacks.m_error = [](const char *msg, size_t msg_len, Location location, void *)
+    {
+        report_error(msg, msg_len, location, stderr);
+        throw std::runtime_error({msg, msg_len});
+    };
+    try {
+        Tree tree(callbacks);
+        csubstr filename = to_csubstr(argv[1]);
+        std::string evt, src = load_file(filename);
+        tree.reserve(to_substr(src).count('\n'));
+        parse_in_place(filename, to_substr(src), &tree);
+        emit_events(&evt, tree);
+        std::fwrite(evt.data(), 1, evt.size(), stdout);
+    }
+    catch(std::exception const&)
+    {
+        return 1;
+    }
+    return 0;
+}
+
+
+//-----------------------------------------------------------------------------
+
+void usage(const char *exename)
+{
+    std::printf(R"(usage:
+%s -          # read from stdin
+%s <file>     # read from file
+)", exename, exename);
+}
+
+std::string load_file(csubstr filename)
+{
+    if(filename == "-") // read from stdin
+    {
+        std::string buf;
+        for(int c = std::getchar(); c != EOF; c = std::getchar())
+        {
+            buf.push_back((char)c);
+            if(buf.size() == buf.capacity())
+                buf.reserve(2u * (buf.capacity() >= 128u ? buf.capacity() : 128u));
+        }
+        return buf;
+    }
+    C4_CHECK_MSG(fs::path_exists(filename.str), "cannot find file: %s (cwd=%s)", filename.str, fs::cwd<std::string>().c_str());
+    return fs::file_get_contents<std::string>(filename.str);
+}
+
+void report_error(const char* msg, size_t length, Location loc, FILE *f)
+{
+    if(!loc.name.empty())
+    {
+        fwrite(loc.name.str, 1, loc.name.len, f);
+        fputc(':', f);
+    }
+    fprintf(f, "%zu:", loc.line);
+    if(loc.col)
+        fprintf(f, "%zu:", loc.col);
+    if(loc.offset)
+        fprintf(f, " (%zuB):", loc.offset);
+    fputc(' ', f);
+    fprintf(f, "%.*s\n", (int)length, msg);
+    fflush(f);
+}