switch to xmake for package management (#611)

This change removes our dependency on vcpkg for package management, in favour of bringing some code in-tree in the `thirdparty` folder as well as using the xmake build-in package management feature. For the latter, all the package definitions are maintained in the zen repo itself, in the `repo` folder.

It should now also be easier to build the project as it will no longer depend on having the right version of vcpkg installed, which has been a common problem for new people coming in to the codebase. Now you should only need xmake to build.

* Bumps xmake requirement on github runners to 2.9.9 to resolve an issue where xmake on Windows invokes cmake with `v144` toolchain which does not exist
* BLAKE3 is now in-tree at `thirdparty/blake3`
* cpr is now in-tree at `thirdparty/cpr`
* cxxopts is now in-tree at `thirdparty/cxxopts`
* fmt is now in-tree at `thirdparty/fmt`
* robin-map is now in-tree at `thirdparty/robin-map`
* ryml is now in-tree at `thirdparty/ryml`
* sol2 is now in-tree at `thirdparty/sol2`
* spdlog is now in-tree at `thirdparty/spdlog`
* utfcpp is now in-tree at `thirdparty/utfcpp`
* xmake package repo definitions is in `repo`
* implemented support for sanitizers. ASAN is supported on windows, TSAN, UBSAN, MSAN etc are supported on Linux/MacOS though I have not yet tested it extensively on MacOS
* the zencore encryption implementation also now supports using mbedTLS which is used on MacOS, though for now we still use openssl on Linux
* crashpad
* bumps libcurl to 8.11.0 (from 8.8.0) which should address a rare build upload bug

1 files changed, 662 insertions, 0 deletions

diff --git a/thirdparty/ryml/api/ryml.i b/thirdparty/ryml/api/ryml.i
new file mode 100644
index 000000000..ac226142a
--- /dev/null
+++ b/thirdparty/ryml/api/ryml.i
@@ -0,0 +1,662 @@
+
+%module ryml
+
+
+//-----------------------------------------------------------------------------
+// this block will be pasted verbatim in the generated C++ source file
+
+%{
+// specifies that the resulting C file should be built as a python
+// extension, inserting the module init code
+#define SWIG_FILE_WITH_INIT
+
+#include <c4/yml/yml.hpp>
+
+namespace c4 {
+namespace yml {
+
+using substr = c4::substr;
+using csubstr = c4::csubstr;
+
+} /* namespace yml */
+} /* namespace c4 */
+
+%}
+
+//-----------------------------------------------------------------------------
+
+
+%apply (const char *STRING, size_t LENGTH) { (const char *str, size_t len) };
+%apply (char *STRING, size_t LENGTH) { (char *str, size_t len) };
+%newobject emit_malloc;
+
+%typemap(in) c4::substr {
+#if defined(SWIGPYTHON)
+  Py_buffer view;
+  int ok = PyObject_CheckBuffer($input);
+  if(ok)
+  {
+      ok = (0 == PyObject_GetBuffer($input, &view, PyBUF_SIMPLE|PyBUF_WRITABLE));
+  }
+  if(ok)
+  {
+      $1 = c4::substr((char*)view.buf, view.len);
+      PyBuffer_Release(&view);
+  }
+  else
+  {
+      PyErr_SetString(PyExc_TypeError, "could not get mutable memory for c4::csubstr - have you passed a str?");
+      SWIG_fail;
+  }
+#else
+#error no "in" typemap defined for this export language
+#endif
+};
+
+%typemap(in) c4::csubstr {
+#if defined(SWIGPYTHON)
+  Py_buffer view;
+  view.buf = nullptr;
+  int ok = PyObject_CheckBuffer($input);
+  if(ok)
+  {
+      ok = (0 == PyObject_GetBuffer($input, &view, PyBUF_CONTIG_RO));
+  }
+  if(ok)
+  {
+      $1 = c4::csubstr((const char*)view.buf, view.len);
+      PyBuffer_Release(&view);
+  }
+  else
+  {
+      // https://stackoverflow.com/questions/36098984/python-3-3-c-api-and-utf-8-strings
+      Py_ssize_t sz = 0;
+      const char *buf = PyUnicode_AsUTF8AndSize($input, &sz);
+      if(buf || sz == 0)
+      {
+          $1 = c4::csubstr(buf, sz);
+      }
+      else
+      {
+          PyErr_SetString(PyExc_TypeError, "c4::csubstr: could not get readonly memory from python object");
+          SWIG_fail;
+      }
+  }
+#else
+#error no "in" typemap defined for this export language
+#endif
+};
+// Copy the typecheck code for "char *".
+%typemap(typecheck) c4::substr = char *;
+%typemap(typecheck) c4::csubstr = const char *;
+
+
+%typemap(out) c4::csubstr {
+#if defined(SWIGPYTHON)
+  if($1.str == nullptr) {
+    $result = Py_None;
+    Py_INCREF($result);
+  } else {
+    PyObject *obj = PyMemoryView_FromMemory((char*)$1.str, $1.len, PyBUF_READ);
+    if( ! obj)
+    {
+        PyErr_SetString(PyExc_TypeError, "could not get readonly memory from c4::csubstr - have you passed a str?");
+        SWIG_fail;
+    }
+    $result = obj;
+  }
+#else
+#error no "out" typemap defined for this export language
+#endif
+};
+
+
+%inline %{
+
+void parse_csubstr(c4::csubstr s, c4::yml::Tree *t)
+{
+    c4::yml::parse_in_arena(s, t);
+}
+
+void parse_substr(c4::substr s, c4::yml::Tree *t)
+{
+    c4::yml::parse_in_place(s, t);
+}
+
+char * emit_yaml_malloc(c4::yml::Tree const& t, size_t id)
+{
+    c4::substr buf;
+    c4::substr ret = c4::yml::emit_yaml(t, id, buf, /*error_on_excess*/false);
+    if(ret.str == nullptr && ret.len > 0)
+    {
+        // Use new[] to parse with delete[] in SWIG.
+        char * alloc = new char[ret.len + 1]; // we'll return a c-string and not a csubstr
+        c4::substr alloced_buf(alloc, ret.len);
+        ret = c4::yml::emit_yaml(t, id, alloced_buf, /*error_on_excess*/true);
+        ret.str[ret.len] = 0;
+    }
+    return ret.str;
+}
+
+char * emit_json_malloc(c4::yml::Tree const& t, size_t id)
+{
+    c4::substr buf;
+    c4::substr ret = c4::yml::emit_json(t, id, buf, /*error_on_excess*/false);
+    if(ret.str == nullptr && ret.len > 0)
+    {
+        // Use new[] to parse with delete[] in SWIG.
+        char * alloc = new char[ret.len + 1]; // we'll return a c-string and not a csubstr
+        c4::substr alloced_buf(alloc, ret.len);
+        ret = c4::yml::emit_json(t, id, alloced_buf, /*error_on_excess*/true);
+        ret.str[ret.len] = 0;
+    }
+    return ret.str;
+}
+
+size_t emit_yaml_length(const c4::yml::Tree &t, size_t id)
+{
+    c4::substr buf;
+    c4::substr ret = c4::yml::emit_yaml(t, id, buf, /*error_on_excess*/false);
+    return ret.len;
+}
+
+size_t emit_json_length(const c4::yml::Tree &t, size_t id)
+{
+    c4::substr buf;
+    c4::substr ret = c4::yml::emit_json(t, id, buf, /*error_on_excess*/false);
+    return ret.len;
+}
+
+bool emit_yaml_to_substr(const c4::yml::Tree &t, size_t id, c4::substr s, size_t *OUTPUT)
+{
+    c4::substr result = c4::yml::emit_yaml(t, id, s, /*error_on_excess*/false);
+    *OUTPUT = result.len;
+    return result.str == nullptr;
+}
+
+bool emit_json_to_substr(const c4::yml::Tree &t, size_t id, c4::substr s, size_t *OUTPUT)
+{
+    c4::substr result = c4::yml::emit_json(t, id, s, /*error_on_excess*/false);
+    *OUTPUT = result.len;
+    return result.str == nullptr;
+}
+
+
+// force a roundtrip to C++, which triggers a conversion to csubstr and returns it as a memoryview
+c4::csubstr _get_as_csubstr(c4::csubstr s)
+{
+    //printf("_get_as_csubstr: %p[%zu]'%.*s'\n", s.str, s.len, (int)s.len, s.str);
+    return s;
+}
+
+c4::csubstr  _get_as_substr(c4::substr s)
+{
+    //printf("_get_as_substr: %p[%zu]'%.*s'\n", s.str, s.len, (int)s.len, s.str);
+    return s;
+}
+
+
+// utilities for testing
+bool _same_ptr(c4::csubstr l, c4::csubstr r)
+{
+    return l.str == r.str;
+}
+
+bool _same_mem(c4::csubstr l, c4::csubstr r)
+{
+    return l.str == r.str && l.len == r.len;
+}
+
+
+%}
+
+
+//-----------------------------------------------------------------------------
+
+%pythoncode %{
+
+from deprecation import deprecated
+
+
+def as_csubstr(s):
+    return _get_as_csubstr(s)
+
+def as_substr(s):
+    return _get_as_substr(s)
+
+def u(memview):
+    return str(memview, "utf8")
+
+
+def children(tree, node=None):
+    assert tree is not None
+    if node is None:
+        node = tree.root_id()
+    ch = tree.first_child(node)
+    while ch != NONE:
+        yield ch
+        ch = tree.next_sibling(ch)
+
+
+def siblings(tree, node):
+    assert tree is not None
+    if node is None:
+        return
+    ch = tree.first_sibling(node)
+    while ch != NONE:
+        yield ch
+        ch = tree.next_sibling(ch)
+
+
+def walk(tree, node=None, indentation_level=0):
+    assert tree is not None
+    if node is None: node = tree.root_id()
+    yield node, indentation_level
+    ch = tree.first_child(node)
+    while ch != NONE:
+       for gc, il in walk(tree, ch, indentation_level + 1):
+           yield gc, il
+       ch = tree.next_sibling(ch)
+
+
+@deprecated(deprecated_in="0.5.0", details="Use parse_in_arena() instead")
+def parse(buf, **kwargs):
+    return parse_in_arena(tree, id)
+def parse_in_arena(buf, **kwargs):
+    return _call_parse(parse_csubstr, buf, **kwargs)
+def parse_in_place(buf, **kwargs):
+    _check_valid_for_in_situ(buf)
+    return _call_parse(parse_substr, buf, **kwargs)
+
+
+
+def _call_parse(parse_fn, buf, **kwargs):
+    tree = kwargs.get("tree", Tree())
+    parse_fn(buf, tree)
+    return tree
+
+
+def _check_valid_for_in_situ(obj):
+    if type(obj) in (str, bytes):
+        raise TypeError("cannot parse in situ: " + type(obj).__name__)
+
+
+
+@deprecated(deprecated_in="0.5.0", details="Use emit_yaml() instead")
+def emit(tree, id=None):
+    return emit_yaml(tree, id)
+def emit_yaml(tree, id=None):
+    if id is None:
+        id = tree.root_id()
+    return emit_yaml_malloc(tree, id)
+def emit_json(tree, id=None):
+    if id is None:
+        id = tree.root_id()
+    return emit_json_malloc(tree, id)
+
+
+@deprecated(deprecated_in="0.5.0", details="Use compute_emit_yaml_length() instead")
+def compute_emit_length(tree, id=None):
+    return compute_emit_yaml_length(tree, id)
+def compute_emit_yaml_length(tree, id=None):
+    if id is None:
+        id = tree.root_id()
+    return emit_yaml_length(tree, id)
+def compute_emit_json_length(tree, id=None):
+    if id is None:
+        id = tree.root_id()
+    return emit_json_length(tree, id)
+
+
+@deprecated(deprecated_in="0.5.0", details="Use emit_yaml_in_place() instead")
+def emit_in_place(tree, buf, id=None):
+    return emit_yaml_in_place(tree, buf, id)
+def emit_yaml_in_place(tree, buf, id=None):
+    return _emit_fn_in_place(tree, buf, id, emit_yaml_to_substr)
+def emit_json_in_place(tree, buf, id=None):
+    return _emit_fn_in_place(tree, buf, id, emit_json_to_substr)
+def _emit_fn_in_place(tree, buf, id, fn):
+    if id is None:
+        id = tree.root_id()
+    (failed, expected_size) = fn(tree, id, buf)
+    if failed:
+        raise IndexError("Output buffer has {} bytes, but emit requires {} bytes".format(
+            len(buf), expected_size))
+    return memoryview(buf)[:expected_size]
+
+%}
+
+//-----------------------------------------------------------------------------
+
+namespace c4 {
+namespace yml {
+
+constexpr const size_t NONE = (size_t)-1;
+
+typedef enum {
+    NOTYPE  = 0,          ///< no type is set
+    VAL     = (1<<0),     ///< a leaf node, has a (possibly empty) value
+    KEY     = (1<<1),     ///< is member of a map, must have non-empty key
+    MAP     = (1<<2),     ///< a map: a parent of keyvals
+    SEQ     = (1<<3),     ///< a seq: a parent of vals
+    DOC     = (1<<4),     ///< a document
+    STREAM  = (1<<5)|SEQ, ///< a stream: a seq of docs
+    KEYREF  = (1<<6),     ///< a *reference: the key references an &anchor
+    VALREF  = (1<<7),     ///< a *reference: the val references an &anchor
+    KEYANCH = (1<<8),     ///< the key has an &anchor
+    VALANCH = (1<<9),     ///< the val has an &anchor
+    KEYTAG  = (1<<10),    ///< the key has an explicit tag/type
+    VALTAG  = (1<<11),    ///< the val has an explicit tag/type
+} NodeType_e;
+
+
+struct NodeType
+{
+    NodeType_e type;
+
+    NodeType();
+    NodeType(NodeType_e t);
+    ~NodeType();
+
+    const char *type_str();
+    static const char* type_str(NodeType_e t);
+
+    void set(NodeType_e t);
+    void add(NodeType_e t);
+    void rem(NodeType_e t);
+
+    bool is_stream() const;
+    bool is_doc() const;
+    bool is_container() const;
+    bool is_map() const;
+    bool is_seq() const;
+    bool has_val() const;
+    bool has_key() const;
+    bool is_val() const;
+    bool is_keyval() const;
+    bool has_key_tag() const;
+    bool has_val_tag() const;
+    bool has_key_anchor() const;
+    bool has_val_anchor() const;
+    bool has_anchor() const;
+    bool is_key_ref() const;
+    bool is_val_ref() const;
+    bool is_ref() const;
+    bool is_anchor_or_ref() const;
+    bool is_key_quoted() const;
+    bool is_val_quoted() const;
+    bool is_quoted() const;
+};
+
+
+struct Tree
+{
+    Tree();
+    ~Tree();
+
+    void reserve(size_t node_capacity);
+    void reserve_arena(size_t node_capacity);
+    void clear();
+    void clear_arena();
+
+    size_t size() const;
+    size_t capacity() const;
+    size_t slack() const;
+
+    size_t arena_size() const;
+    size_t arena_capacity() const;
+    size_t arena_slack() const;
+
+    void resolve();
+
+public:
+
+    // getters
+
+    NodeType_e  type(size_t node) const;
+    const char* type_str(size_t node) const;
+
+    c4::csubstr key       (size_t node) const;
+    c4::csubstr key_tag   (size_t node) const;
+    c4::csubstr key_ref   (size_t node) const;
+    c4::csubstr key_anchor(size_t node) const;
+    c4::yml::NodeScalar keysc(size_t node) const;
+
+    c4::csubstr val       (size_t node) const;
+    c4::csubstr val_tag   (size_t node) const;
+    c4::csubstr val_ref   (size_t node) const;
+    c4::csubstr val_anchor(size_t node) const;
+    c4::yml::NodeScalar valsc(size_t node) const;
+
+public:
+
+    // node predicates
+
+    bool is_root(size_t node) const;
+    bool is_stream(size_t node) const;
+    bool is_doc(size_t node) const;
+    bool is_container(size_t node) const;
+    bool is_map(size_t node) const;
+    bool is_seq(size_t node) const;
+    bool has_val(size_t node) const;
+    bool has_key(size_t node) const;
+    bool is_val(size_t node) const;
+    bool is_keyval(size_t node) const;
+    bool has_key_tag(size_t node) const;
+    bool has_val_tag(size_t node) const;
+    bool has_key_anchor(size_t node) const;
+    bool has_val_anchor(size_t node) const;
+    bool is_key_ref(size_t node) const;
+    bool is_val_ref(size_t node) const;
+    bool is_ref(size_t node) const;
+    bool is_anchor_or_ref(size_t node) const;
+    bool is_key_quoted(size_t node) const;
+    bool is_val_quoted(size_t node) const;
+    bool is_quoted(size_t node) const;
+    bool is_anchor(size_t node) const;
+    bool parent_is_seq(size_t node) const;
+    bool parent_is_map(size_t node) const;
+    bool empty(size_t node) const;
+    bool has_anchor(size_t node, c4::csubstr a) const;
+
+public:
+
+    // hierarchy predicates
+
+    bool has_parent(size_t node) const;
+    bool has_child(size_t node, c4::csubstr key) const;
+    //bool has_child(size_t node, size_t ch) const;
+    bool has_children(size_t node) const;
+    bool has_sibling(size_t node, c4::csubstr key) const;
+    //bool has_sibling(size_t node, size_t sib) const;
+    bool has_other_siblings(size_t node) const;
+
+public:
+
+    // hierarchy getters
+
+    size_t root_id() const;
+
+    size_t parent(size_t node) const;
+    size_t prev_sibling(size_t node) const;
+    size_t next_sibling(size_t node) const;
+    size_t num_children(size_t node) const;
+    size_t child_pos(size_t node, size_t ch) const;
+    size_t first_child(size_t node) const;
+    size_t last_child(size_t node) const;
+    size_t child(size_t node, size_t pos) const;
+    size_t find_child(size_t node, c4::csubstr key) const;
+    size_t num_siblings(size_t node) const;
+    size_t num_other_siblings(size_t node) const;
+    size_t sibling_pos(size_t node, size_t sib) const;
+    size_t first_sibling(size_t node) const;
+    size_t last_sibling(size_t node) const;
+    size_t sibling(size_t node, size_t pos) const;
+    size_t find_sibling(size_t node, c4::csubstr key) const;
+
+public:
+
+    void to_keyval(size_t node, c4::csubstr key, c4::csubstr val, int more_flags=0);
+    void to_map(size_t node, c4::csubstr key, int more_flags=0);
+    void to_seq(size_t node, c4::csubstr key, int more_flags=0);
+    void to_val(size_t node, c4::csubstr val, int more_flags=0);
+    void to_stream(size_t node, int more_flags=0);
+    void to_map(size_t node, int more_flags=0);
+    void to_seq(size_t node, int more_flags=0);
+    void to_doc(size_t node, int more_flags=0);
+
+    void set_key_tag(size_t node, c4::csubstr tag);
+    void set_key_anchor(size_t node, c4::csubstr anchor);
+    void set_val_anchor(size_t node, c4::csubstr anchor);
+    void set_key_ref   (size_t node, c4::csubstr ref   );
+    void set_val_ref   (size_t node, c4::csubstr ref   );
+
+    void _set_key(size_t node, c4::csubstr key, int more_flags=0);
+    void _set_val(size_t node, c4::csubstr val, int more_flags=0);
+
+    void set_val_tag(size_t node, c4::csubstr tag);
+    void rem_key_anchor(size_t node);
+    void rem_val_anchor(size_t node);
+    void rem_key_ref   (size_t node);
+    void rem_val_ref   (size_t node);
+    void rem_anchor_ref(size_t node);
+
+public:
+
+    /** create and insert a new child of "parent". insert after the (to-be)
+     * sibling "after", which must be a child of "parent". To insert as the
+     * first child, set after to NONE */
+    size_t insert_child(size_t parent, size_t after);
+    size_t prepend_child(size_t parent);
+    size_t  append_child(size_t parent);
+
+public:
+
+    //! create and insert a new sibling of n. insert after "after"
+    size_t insert_sibling(size_t node, size_t after);
+    size_t prepend_sibling(size_t node);
+    size_t  append_sibling(size_t node);
+
+public:
+
+    //! remove an entire branch at once: ie remove the children and the node itself
+    void remove(size_t node);
+
+    //! remove all the node's children, but keep the node itself
+    void remove_children(size_t node);
+
+public:
+
+    void reorder();
+
+    /** change the node's position in the parent */
+    void move(size_t node, size_t after);
+
+    /** change the node's parent and position */
+    void move(size_t node, size_t new_parent, size_t after);
+    /** change the node's parent and position */
+    size_t move(Tree * src, size_t node, size_t new_parent, size_t after);
+
+    /** recursively duplicate the node */
+    size_t duplicate(size_t node, size_t new_parent, size_t after);
+    /** recursively duplicate a node from a different tree */
+    size_t duplicate(Tree const* src, size_t node, size_t new_parent, size_t after);
+
+    /** recursively duplicate the node's children (but not the node) */
+    void duplicate_children(size_t node, size_t parent, size_t after);
+    /** recursively duplicate the node's children (but not the node), where the node is from a different tree */
+    void duplicate_children(Tree const* src, size_t node, size_t parent, size_t after);
+
+    void duplicate_contents(size_t node, size_t where);
+
+    /** duplicate the node's children (but not the node) in a new parent, but
+     * omit repetitions where a duplicated node has the same key (in maps) or
+     * value (in seqs). If one of the duplicated children has the same key
+     * (in maps) or value (in seqs) as one of the parent's children, the one
+     * that is placed closest to the end will prevail. */
+    void duplicate_children_no_rep(size_t node, size_t parent, size_t after);
+
+};
+
+/*
+%extend Tree {
+
+    bool has_anchor(size_t node, const char *str, size_t len) const
+    {
+        return $self->has_anchor(node, c4::csubstr(str, len));
+    }
+
+    bool has_child(size_t node, const char *str, size_t len) const
+    {
+        return $self->has_child(node, c4::csubstr(str, len));
+    }
+
+    bool has_sibling(size_t node, const char *str, size_t len) const
+    {
+        return $self->has_sibling(node, c4::csubstr(str, len));
+    }
+
+    size_t find_child(size_t node, const char *str, size_t len) const
+    {
+        return $self->find_child(node, c4::csubstr(str, len));
+    }
+
+    size_t find_sibling(size_t node, const char *str, size_t len) const
+    {
+        return $self->find_sibling(node, c4::csubstr(str, len));
+    }
+
+    void to_keyval(size_t node, const char *keystr, size_t keylen, const char *valstr, size_t vallen, int more_flags=0)
+    {
+        return $self->to_keyval(node, c4::csubstr(keystr, keylen), c4::csubstr(valstr, vallen), more_flags);
+    }
+
+    void to_map(size_t node, const char *keystr, size_t keylen, int more_flags=0)
+    {
+        return $self->to_map(node, c4::csubstr(keystr, keylen), more_flags);
+    }
+
+    void to_seq(size_t node, const char *keystr, size_t keylen, int more_flags=0)
+    {
+        return $self->to_seq(node, c4::csubstr(keystr, keylen), more_flags);
+    }
+
+    void to_val(size_t node, const char *valstr, size_t vallen, int more_flags=0)
+    {
+        return $self->to_val(node, c4::csubstr(valstr, vallen), more_flags);
+    }
+
+    void set_key_tag(size_t node, const char *str, size_t len)
+    {
+        return $self->set_key_tag(node, c4::csubstr(str, len));
+    }
+    void set_val_tag(size_t node, const char *str, size_t len)
+    {
+        return $self->set_val_tag(node, c4::csubstr(str, len));
+    }
+
+    void set_key_anchor(size_t node, const char *str, size_t len)
+    {
+        return $self->set_key_anchor(node, c4::csubstr(str, len));
+    }
+    void set_val_anchor(size_t node, const char *str, size_t len)
+    {
+        return $self->set_val_anchor(node, c4::csubstr(str, len));
+    }
+
+    void set_key_ref(size_t node, const char *str, size_t len)
+    {
+        return $self->set_key_ref(node, c4::csubstr(str, len));
+    }
+    void set_val_ref(size_t node, const char *str, size_t len)
+    {
+        return $self->set_val_ref(node, c4::csubstr(str, len));
+    }
+
+};
+*/
+
+} // namespace yml
+} // namespace c4
+
+//-----------------------------------------------------------------------------