switch to xmake for package management (#611)

This change removes our dependency on vcpkg for package management, in favour of bringing some code in-tree in the `thirdparty` folder as well as using the xmake build-in package management feature. For the latter, all the package definitions are maintained in the zen repo itself, in the `repo` folder.

It should now also be easier to build the project as it will no longer depend on having the right version of vcpkg installed, which has been a common problem for new people coming in to the codebase. Now you should only need xmake to build.

* Bumps xmake requirement on github runners to 2.9.9 to resolve an issue where xmake on Windows invokes cmake with `v144` toolchain which does not exist
* BLAKE3 is now in-tree at `thirdparty/blake3`
* cpr is now in-tree at `thirdparty/cpr`
* cxxopts is now in-tree at `thirdparty/cxxopts`
* fmt is now in-tree at `thirdparty/fmt`
* robin-map is now in-tree at `thirdparty/robin-map`
* ryml is now in-tree at `thirdparty/ryml`
* sol2 is now in-tree at `thirdparty/sol2`
* spdlog is now in-tree at `thirdparty/spdlog`
* utfcpp is now in-tree at `thirdparty/utfcpp`
* xmake package repo definitions is in `repo`
* implemented support for sanitizers. ASAN is supported on windows, TSAN, UBSAN, MSAN etc are supported on Linux/MacOS though I have not yet tested it extensively on MacOS
* the zencore encryption implementation also now supports using mbedTLS which is used on MacOS, though for now we still use openssl on Linux
* crashpad
* bumps libcurl to 8.11.0 (from 8.8.0) which should address a rare build upload bug

6 files changed, 967 insertions, 0 deletions

diff --git a/thirdparty/ryml/api/python/.gitignore b/thirdparty/ryml/api/python/.gitignore
new file mode 100644
index 000000000..830e7e4d6
--- /dev/null
+++ b/thirdparty/ryml/api/python/.gitignore
@@ -0,0 +1,141 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Version file generated by setuptools_scm
+version.py
+
+# SWIG produced file
+ryml/ryml.py
diff --git a/thirdparty/ryml/api/python/Makefile b/thirdparty/ryml/api/python/Makefile
new file mode 100644
index 000000000..bdb5a6ffd
--- /dev/null
+++ b/thirdparty/ryml/api/python/Makefile
@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: MIT
+
+# Use bash even on Windows
+SHELL := /bin/bash
+
+# On Windows the activate script is stored in a different location.
+ACTIVATE_SCRIPT := venv/bin/activate
+ifeq ($(OS),Windows_NT)
+ACTIVATE_SCRIPT := venv/Scripts/activate
+endif
+
+# How to invoke python
+PYTHON := python
+# How to invoke pytest
+PYTEST := $(PYTHON) -m pytest -vvv
+
+ACTIVATE=[[ -e $(ACTIVATE_SCRIPT) ]] && source $(ACTIVATE_SCRIPT);
+
+.PHONY: clean
+clean:
+	rm -rf dist *.egg-info
+	rm -rf ../../build ../../.egg*
+	rm -rf ryml/*.so ryml/ryml.py ryml/include ryml/lib
+
+.PHONY: venv-clean
+venv-clean:
+	rm -rf venv
+
+
+$(ACTIVATE_SCRIPT): requirements.txt Makefile
+	make venv
+	@touch $(ACTIVATE_SCRIPT)
+
+.PHONY: venv
+venv:
+	virtualenv --python=python3 --always-copy venv
+	# Packaging tooling.
+	${ACTIVATE} pip install -U pip
+	# Setup requirements.
+	${ACTIVATE} pip install -v -r requirements.txt
+	${ACTIVATE} pip install -v -e ../..
+	@${ACTIVATE} $(PYTHON) -c "from ryml.version import version as v; print('Installed version:', v)"
+
+.PHONY: build-sdist
+build-sdist: | $(ACTIVATE_SCRIPT)
+	${ACTIVATE} (cd ../..; $(PYTHON) -m build --sdist --outdir $(PWD)/dist)
+
+
+.PHONY: build-wheel
+build-wheel: | $(ACTIVATE_SCRIPT)
+	rm -rf dist
+	$(MAKE) build-sdist
+	@ls -l dist/*.tar.gz
+	${ACTIVATE} pip wheel -v dist/*.tar.gz --wheel-dir $(PWD)/dist
+
+.PHONY: build
+build:
+	rm -rf build dist
+	$(MAKE) build-sdist
+	$(MAKE) build-wheel
+
+# PYPI_TEST = --repository-url https://test.pypi.org/legacy/
+PYPI_TEST = --repository testpypi
+
+.PHONY: upload-test
+upload-test: | $(ACTIVATE_SCRIPT)
+	make clean
+	make build-sdist
+	${ACTIVATE} twine upload ${PYPI_TEST} dist/*
+
+.PHONY: upload
+upload: | $(ACTIVATE_SCRIPT)
+	make clean
+	make build-sdist
+	${ACTIVATE} twine upload --verbose dist/*
+
+.PHONY: check
+check: | $(ACTIVATE_SCRIPT)
+	make clean
+	make build-wheel
+	${ACTIVATE} twine check dist/*.whl
+
+.PHONY: install
+install: | $(ACTIVATE_SCRIPT)
+	${ACTIVATE} $(PYTHON) setup.py install
+
+.PHONY: test
+test: | $(ACTIVATE_SCRIPT)
+	${ACTIVATE} $(PYTEST) tests
+
+.PHONY: version
+version: | $(ACTIVATE_SCRIPT)
+	${ACTIVATE} $(PYTHON) setup.py --version
diff --git a/thirdparty/ryml/api/python/bm/bm_parse.py b/thirdparty/ryml/api/python/bm/bm_parse.py
new file mode 100644
index 000000000..294be679b
--- /dev/null
+++ b/thirdparty/ryml/api/python/bm/bm_parse.py
@@ -0,0 +1,237 @@
+import ryml
+import ruamel.yaml
+import yaml
+import timeit
+import time
+import copy
+import prettytable
+import os.path
+from collections import OrderedDict as odict
+
+
+def _nodbg(*args, **kwargs):
+    pass
+
+
+def _dbg(*args, **kwargs):
+    print(*args, **kwargs, file=sys.stderr, flush=True)
+
+
+dbg = _dbg
+
+
+class RunResults:
+
+    __slots__ = ('name', 'time_ms', 'count', 'avg', 'MBps')
+
+    def __init__(self, name, time_ms, count, num_bytes):
+        self.name = name
+        self.time_ms = time_ms
+        self.count = count
+        self.avg = time_ms / count
+        num_megabytes = count * num_bytes / 1.0e6
+        num_seconds = time_ms / 1000.0
+        self.MBps = num_megabytes / num_seconds
+
+    def __str__(self):
+        fmt = "{}: count={} time={:.3f}ms avg={:.3f}ms MB/s={:.3f}"
+        fmt = fmt.format(self.name, self.count, self.time_ms, self.avg, self.MBps)
+        return fmt
+
+
+class BmCase:
+
+    def __init__(self, filename):
+        with open(filename, "r") as f:
+            src = f.read()
+        self.filename = filename
+        self.src_as_str = src
+        self.src_as_bytes = bytes(src, "utf8")
+        self.src_as_bytearray = bytearray(src, "utf8")
+        self.src_as_bytearray_orig = copy.copy(self.src_as_bytearray)
+        self.emittree = ryml.parse_in_arena(self.src_as_bytearray)
+        self.emitbuf = bytearray(4 * len(self.src_as_str))  # should be enough
+
+    def run(self, bm_method_name, cls):
+        def run_bm(obj, subject):
+            obj.count = 0
+            t = timeit.Timer(subject)
+            delta = time.time()
+            result = t.autorange() #lambda number, time_taken: time_taken > 1.0)
+            delta = 1000. * (time.time() - delta)
+            return delta, obj.count
+        obj = cls(self)
+        if not hasattr(obj, bm_method_name):
+            return None
+        name = bm_method_name + ":" + cls.__name__
+        dbg(name, "...")
+        method = getattr(obj, bm_method_name)
+        reset_name = 'reset_' + bm_method_name
+        reset_fn = getattr(obj, reset_name, None)
+        def bm_fn():
+            method(self)
+            obj.count += 1
+            if reset_fn is not None:
+                reset_fn(self)
+        delta, count = run_bm(obj, bm_fn)
+        # correct the benchmark to account for the time spent
+        # resetting
+        if reset_fn is not None:
+            # find out how much it takes to reset the bytearray
+            if not hasattr(obj, 'bm_reset_done'):
+                def bm_reset():
+                    reset_fn(self)
+                    obj.count += 1
+                rdelta, rcount = run_bm(obj, bm_reset)
+                obj.bm_reset_time_per_iteration = rdelta / rcount
+                dbg(name, "reset_time_per_iteration={:.3f}us".format(obj.bm_reset_time_per_iteration * 1000.0))
+                obj.bm_reset_done = True
+            reset_correction = count * obj.bm_reset_time_per_iteration
+            dbg(name, "delta={:.3f}ms".format(delta), "reset_correction={:.3f}ms({:.2f}%)".format(reset_correction, 100.0 * reset_correction / delta))
+            delta -= reset_correction
+        ret = RunResults(name, delta, count, len(self.src_as_str))
+        dbg(name, "ok:", ret)
+        return ret
+
+
+def run(case, benchmarks, approaches):
+    for bm in benchmarks:
+        results = odict()
+        for cls in approaches:
+            r = case.run(bm, cls)
+            if r is None:
+                continue
+            results[r.name] = r
+        table = prettytable.PrettyTable()
+        name = os.path.basename(case.filename)
+        table.field_names = [name, "count", "time(ms)", "avg(ms)", "avg(MB/s)"]
+        table.align[name] = "l"
+        def i(v): return "{:5d}".format(v)
+        def f(v): return "{:8.3f}".format(v)
+        for v in results.values():
+            table.add_row([v.name, i(v.count), f(v.time_ms), f(v.avg), f(v.MBps)])
+        print(table)
+
+
+class BmCaseRun:
+    def __init__(self, case):
+        self.reset_bytearray = False
+
+
+class RymlParseInArena(BmCaseRun):
+
+    def parse(self, case):
+        _ = ryml.parse_in_arena(case.src_as_bytearray)
+
+
+class RymlParseInArenaReuse(BmCaseRun):
+
+    def __init__(self, case):
+        self.tree = ryml.Tree()
+
+    def parse(self, case):
+        ryml.parse_in_arena(case.src_as_bytearray, tree=self.tree)
+
+    def reset_parse(self, case):
+        self.tree.clear()
+        self.tree.clear_arena()
+
+
+class RymlParseInPlace(BmCaseRun):
+
+    def parse(self, case):
+        _ = ryml.parse_in_place(case.src_as_bytearray)
+
+    def reset_parse(self, case):
+        case.src_as_bytearray = copy.copy(case.src_as_bytearray_orig)
+
+
+class RymlParseInPlaceReuse(BmCaseRun):
+
+    def __init__(self, case):
+        self.tree = ryml.Tree()
+
+    def parse(self, case):
+        ryml.parse_in_place(case.src_as_bytearray, tree=self.tree)
+
+    def reset_parse(self, case):
+        self.tree.clear()
+        self.tree.clear_arena()
+        case.src_as_bytearray = copy.copy(case.src_as_bytearray_orig)
+
+
+class RuamelYamlParse(BmCaseRun):
+
+    def parse(self, case):
+        _ = ruamel.yaml.load(case.src_as_str, Loader=ruamel.yaml.Loader)
+
+
+class PyYamlParse(BmCaseRun):
+
+    def parse(self, case):
+        _ = yaml.safe_load(case.src_as_str)
+
+
+class RymlEmitToNewBuffer(BmCaseRun):
+
+    def emit_yaml(self, case):
+        _ = ryml.emit_yaml(case.emittree)
+
+    def emit_json(self, case):
+        _ = ryml.emit_json(case.emittree)
+
+
+class RymlEmitReuse(BmCaseRun):
+
+    def emit_yaml(self, case):
+        _ = ryml.emit_yaml_in_place(case.emittree, case.emitbuf)
+
+    def emit_json(self, case):
+        _ = ryml.emit_json_in_place(case.emittree, case.emitbuf)
+
+
+class RuamelYamlEmit:
+
+    def __init__(self, case):
+        case.ruamel_emittree = ruamel.yaml.load(case.src_as_str, Loader=ruamel.yaml.Loader)
+
+    def emit_yaml(self, case):
+        # https://stackoverflow.com/a/47617341/5875572
+        class MyToStr:
+            def __init__(self, *args, **kwargs):
+                self.s = b""
+            def write(self, s):
+                self.s += s
+        dumper = MyToStr()
+        ruamel.yaml.YAML().dump(case.ruamel_emittree, MyToStr())
+
+
+class PyYamlEmit:
+
+    def __init__(self, case):
+        case.pyyaml_emittree = yaml.load(case.src_as_str, Loader=yaml.Loader)
+
+    def emit_yaml(self, case):
+        _ = yaml.dump(case.pyyaml_emittree)
+
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) < 2:
+        raise Exception("")
+    filename = sys.argv[1]
+    if filename.endswith("outer1000_inner1000.yml"):  # this one is too heavy for the Python libs
+        exit(0)
+    case = BmCase(filename)
+    run(case, benchmarks=('parse', ),
+        approaches=(RuamelYamlParse,
+                    PyYamlParse,
+                    RymlParseInArena,
+                    RymlParseInArenaReuse,
+                    RymlParseInPlace,
+                    RymlParseInPlaceReuse))
+    run(case, benchmarks=('emit_yaml', 'emit_json', ),
+        approaches=(RuamelYamlEmit,
+                    PyYamlEmit,
+                    RymlEmitToNewBuffer,
+                    RymlEmitReuse))
diff --git a/thirdparty/ryml/api/python/requirements.txt b/thirdparty/ryml/api/python/requirements.txt
new file mode 100644
index 000000000..86c126420
--- /dev/null
+++ b/thirdparty/ryml/api/python/requirements.txt
@@ -0,0 +1,5 @@
+ruamel.yaml
+ninja
+pyyaml
+prettytable
+pytest
diff --git a/thirdparty/ryml/api/python/ryml/__init__.py b/thirdparty/ryml/api/python/ryml/__init__.py
new file mode 100644
index 000000000..8bbb62199
--- /dev/null
+++ b/thirdparty/ryml/api/python/ryml/__init__.py
@@ -0,0 +1,2 @@
+from ryml.ryml import *
+from .version import *
diff --git a/thirdparty/ryml/api/python/tests/test_parse.py b/thirdparty/ryml/api/python/tests/test_parse.py
new file mode 100644
index 000000000..bb7973cb3
--- /dev/null
+++ b/thirdparty/ryml/api/python/tests/test_parse.py
@@ -0,0 +1,488 @@
+import ryml
+from ryml.ryml import _same_ptr, _same_mem
+import unittest
+
+
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+class TestSubstrInterop(unittest.TestCase):
+
+    # ------------------------------------------------
+    # str
+
+    # CAN create c4::csubstr from string object
+    def test11_str2csubstr(self):
+        s = "asdasd"
+        m = ryml.as_csubstr(s)
+        self.assertTrue(_same_ptr(s, m))
+        self.assertTrue(_same_mem(s, m))
+        self.assertEqual(s, ryml.u(m))
+        #
+        m = ryml.as_csubstr(m)
+        self.assertTrue(_same_ptr(s, m))
+        self.assertTrue(_same_mem(s, m))
+        self.assertEqual(s, ryml.u(m))
+
+    # CANNOT create c4::substr from string object
+    def test12_str2substr(self):
+        s = ""
+        with self.assertRaises(TypeError) as context:
+            _ = ryml.as_substr(s)
+        self.assertTrue(type(context.exception), TypeError)
+
+    # ------------------------------------------------
+    # bytes
+
+    # CAN create c4::csubstr from string object
+    def test21_bytes2csubstr(self):
+        s = b"foo21"
+        m = ryml.as_csubstr(s)
+        self.assertTrue(_same_ptr(s, m))
+        self.assertTrue(_same_mem(s, m))
+        self.assertEqual(s, m)
+        #
+        m = ryml.as_csubstr(m)
+        self.assertTrue(_same_ptr(s, m))
+        self.assertTrue(_same_mem(s, m))
+        self.assertEqual(s, m)
+
+    # CANNOT create c4::csubstr from string object
+    def test22_bytes2substr(self):
+        s = b"foo22"
+        with self.assertRaises(TypeError) as context:
+            _ = ryml.as_substr(s)
+        self.assertTrue(type(context.exception), TypeError)
+
+    # ------------------------------------------------
+    # bytearray
+
+    # CAN create c4::csubstr from string object
+    def test31_bytes2csubstr(self):
+        s = bytearray("foo31", "utf8")
+        m = ryml.as_csubstr(s)
+        self.assertTrue(_same_ptr(s, m))
+        self.assertTrue(_same_mem(s, m))
+        self.assertEqual(s, m)
+        #
+        m = ryml.as_csubstr(m)
+        self.assertTrue(_same_ptr(s, m))
+        self.assertTrue(_same_mem(s, m))
+        self.assertEqual(s, m)
+
+    # CANNOT create c4::csubstr from string object
+    def test32_bytes2substr(self):
+        s = bytearray("foo31", "utf8")
+        m = ryml.as_csubstr(s)
+        self.assertTrue(_same_ptr(s, m))
+        self.assertTrue(_same_mem(s, m))
+        self.assertEqual(s, m)
+        #
+        m = ryml.as_csubstr(m)
+        self.assertTrue(_same_ptr(s, m))
+        self.assertTrue(_same_mem(s, m))
+        self.assertEqual(s, m)
+
+
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+
+def _addmap(t, node, k=None):
+    m = t.append_child(node)
+    if k is None:
+        t.to_map(m)
+    else:
+        t.to_map(m, k)
+    return m
+
+
+def _addseq(t, node, k=None):
+    m = t.append_child(node)
+    if k is None:
+        t.to_seq(m)
+    else:
+        t.to_seq(m, k)
+    return m
+
+
+def _addval(t, node, k, v=None):
+    ch = t.append_child(node)
+    if v is None:
+        t.to_val(ch, k)
+    else:
+        t.to_keyval(ch, k, v)
+    return ch
+
+
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+def check_tree_mod(ut, t):
+    # some convenient shorthands
+    eq = ut.assertEqual
+    def _addval_and_check(node, k, v=None):
+        ch = _addval(t, node, k, v)
+        pos = t.child_pos(node, ch)
+        eq(t.child(node, pos), ch)
+        if v is not None:
+            eq(t.find_child(node, k), ch)
+            eq(t.child(node, pos), t.find_child(node, k))
+        return ch
+    def _addseq_and_check(node, k):
+        ch = _addseq(t, node, k)
+        eq(t.find_child(node, k), ch)
+        return ch
+    def _addmap_and_check(node, k):
+        ch = _addmap(t, node, k)
+        eq(t.find_child(node, k), ch)
+        return ch
+    m = _addmap_and_check(t.root_id(), "check_tree_mod_map")
+    _addval_and_check(m, "k1", "v1")
+    _addval_and_check(m, "k2", "v2")
+    _addval_and_check(m, "k3", "v3")
+    eq(t.num_children(m), 3)
+    eq(t.num_siblings(t.first_child(m)), 3)
+    s = _addseq_and_check(t.root_id(), "check_tree_mod_seq")
+    _addval_and_check(s, "v1")
+    _addval_and_check(s, "v2")
+    _addval_and_check(s, "v3")
+    eq(t.num_children(s), 3)
+    eq(t.num_siblings(t.first_child(m)), 3)
+
+
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+class SimpleTestCase:
+
+    yaml = "{'HELLO': a, foo: \"b\", bar: c, baz: d, seq: [0, 1, 2, 3]}"
+
+    def check(self, ut, t, is_json=False):
+        # some convenient shorthands
+        eq = ut.assertEqual
+        ne = ut.assertNotEqual
+        fs = ut.assertFalse
+        tr = ut.assertTrue
+        #
+        eq(t.size(), 10)
+        tr(t.is_root(0))
+        eq(t.num_children(0), 5)
+        eq(t.find_child(0, b"HELLO"), 1)
+        eq(t.find_child(0, b"foo"), 2)
+        eq(t.find_child(0, b"bar"), 3)
+        eq(t.find_child(0, b"baz"), 4)
+        eq(t.find_child(0, b"seq"), 5)
+        eq(t.parent(0), ryml.NONE)
+        eq(t.parent(1), 0)
+        eq(t.parent(2), 0)
+        eq(t.parent(3), 0)
+        eq(t.parent(4), 0)
+        eq(t.parent(5), 0)
+        fs(t.is_root(1))
+        fs(t.is_root(2))
+        fs(t.is_root(3))
+        fs(t.is_root(4))
+        fs(t.is_root(5))
+        fs(t.has_child(0, b"foozzie"))
+        fs(t.has_child(0, b"bark"))
+        fs(t.has_child(0, b"bart"))
+        fs(t.has_child(0, b"bazk"))
+        eq(t.next_sibling(0), ryml.NONE)
+        eq(t.prev_sibling(0), ryml.NONE)
+        eq(t.prev_sibling(1), ryml.NONE)
+        eq(t.next_sibling(5), ryml.NONE)
+        tr(t.has_child(0, b"HELLO"))
+        tr(t.has_child(0, b"foo"))
+        tr(t.has_child(0, b"bar"))
+        tr(t.has_child(0, b"baz"))
+        eq(t.key(1), b"HELLO")
+        eq(t.key(2), b"foo")
+        eq(t.key(3), b"bar")
+        eq(t.key(4), b"baz")
+        eq(t.key(5), b"seq")
+        eq(t.val(1), b"a")
+        eq(t.val(2), b"b")
+        eq(t.val(3), b"c")
+        eq(t.val(4), b"d")
+        eq(t.val(6), b"0")
+        eq(t.val(7), b"1")
+        eq(t.val(8), b"2")
+        eq(t.val(9), b"3")
+        if not is_json:
+            tr(t.is_key_quoted(1))
+            fs(t.is_key_quoted(2))
+            fs(t.is_key_quoted(3))
+            fs(t.is_key_quoted(4))
+            fs(t.is_key_quoted(5))
+        else:
+            tr(t.is_key_quoted(1))
+            tr(t.is_key_quoted(2))
+            tr(t.is_key_quoted(3))
+            tr(t.is_key_quoted(4))
+            tr(t.is_key_quoted(5))
+        if not is_json:
+            fs(t.is_val_quoted(1))
+            tr(t.is_val_quoted(2))
+            fs(t.is_val_quoted(3))
+            fs(t.is_val_quoted(4))
+            fs(t.is_val_quoted(5))
+            fs(t.is_val_quoted(6))
+            fs(t.is_val_quoted(7))
+            fs(t.is_val_quoted(8))
+            fs(t.is_val_quoted(9))
+        else:
+            tr(t.is_val_quoted(1))
+            tr(t.is_val_quoted(2))
+            tr(t.is_val_quoted(3))
+            tr(t.is_val_quoted(4))
+            fs(t.is_val_quoted(5))
+            fs(t.is_val_quoted(6))
+            fs(t.is_val_quoted(7))
+            fs(t.is_val_quoted(8))
+            fs(t.is_val_quoted(9))
+        if not is_json:
+            tr(t.is_quoted(1))
+            tr(t.is_quoted(2))
+            fs(t.is_quoted(3))
+            fs(t.is_quoted(4))
+            fs(t.is_quoted(5))
+            fs(t.is_quoted(6))
+            fs(t.is_quoted(7))
+            fs(t.is_quoted(8))
+            fs(t.is_quoted(9))
+        else:
+            tr(t.is_quoted(1))
+            tr(t.is_quoted(2))
+            tr(t.is_quoted(3))
+            tr(t.is_quoted(4))
+            tr(t.is_quoted(5))
+            fs(t.is_quoted(6))
+            fs(t.is_quoted(7))
+            fs(t.is_quoted(8))
+            fs(t.is_quoted(9))
+        tr(t.has_sibling(1, b"bar"))
+        tr(t.has_sibling(1, b"baz"))
+        tr(t.has_sibling(2, b"foo"))
+        tr(t.has_sibling(2, b"baz"))
+        tr(t.has_sibling(3, b"foo"))
+        tr(t.has_sibling(3, b"bar"))
+        for i in (1, 2, 3, 4, 5):
+            eq(t.find_sibling(i, b"HELLO"), 1)
+            eq(t.find_sibling(i, b"foo"), 2)
+            eq(t.find_sibling(i, b"bar"), 3)
+            eq(t.find_sibling(i, b"baz"), 4)
+            eq(t.find_sibling(i, b"seq"), 5)
+        #
+        num = 0
+        for id in ryml.children(t):
+            num += 1
+            eq(id, num)
+        eq(num, t.num_children(t.root_id()))
+        eq(num, t.num_siblings(t.first_child(t.root_id())))
+        #
+        num = 0
+        for id in ryml.children(t, 1):
+            num += 1
+        eq(num, 0)
+        #
+        num = 0
+        for id in ryml.siblings(t, 1):
+            num += 1
+            eq(id, num)
+        eq(num, t.num_children(t.root_id()))
+        eq(num, t.num_siblings(t.first_child(t.root_id())))
+        #
+        num = 0
+        for id in ryml.siblings(t, 3):
+            num += 1
+            eq(id, num)
+        eq(num, 5)
+        eq(num, t.num_siblings(t.first_child(t.root_id())))
+        #
+        for i, ch in enumerate(ryml.children(t, 5)):
+            eq(t.val(ch), [b"0", b"1", b"2", b"3"][i])
+        sibs = [b"HELLO", b"foo", b"bar", b"baz", b"seq"]
+        sibs_s = ["HELLO", "foo", "bar", "baz", "seq"]
+        for i, sib in enumerate(ryml.siblings(t, 5)):
+            k = t.key(sib)
+            k_s = str(k, "utf8")
+            eq(k, sibs[i])
+            eq(k_s, sibs_s[i])
+            ne(k, sibs_s[i])
+            ne(k_s, sibs[i])
+            k_s = str(k)
+            ne(k_s, sibs_s[i])
+            ne(k_s, sibs[i])
+        num = 0
+        for id in ryml.siblings(t, 0):
+            num += 1
+        eq(num, 1)
+        #
+        num = 0
+        for id, level in ryml.walk(t):
+            num += 1
+            if t.is_root(id):
+                eq(id, 0)
+                eq(level, 0)
+            if t.is_map(id):
+                eq(id, 0)
+                eq(level, 0)
+            if t.is_seq(id):
+                eq(id, 5)
+                eq(level, 1)
+            if t.is_keyval(id):
+                tr(id > 0 and id < 5)
+            if t.is_val(id):
+                tr(id > 5)
+                eq(level, 2)
+        eq(num, t.size())
+        #
+        num = 0
+        for id in ryml.walk(t, 5):
+            num += 1
+        eq(num, 5)
+        #
+        num = 0
+        for id in ryml.walk(t, 9):
+            num += 1
+        eq(num, 1)
+        check_tree_mod(ut, t)
+
+
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+class TestRunner(unittest.TestCase):
+
+    def setUp(self):
+        self._setUp(SimpleTestCase())
+
+    # allow creating this class with different cases
+    # if they are added
+    def _setUp(self, case):
+        self.case = case
+        self.src_as_str = str(case.yaml)
+        self.src_as_bytes = bytes(case.yaml, "utf8")
+        self.src_as_bytearray = bytearray(case.yaml, "utf8")
+
+    # ----------------------------------------------------------
+    def test11_str__arena(self):  # cannot read string buffers (or can we?)
+        tree = ryml.parse_in_arena(self.src_as_str)
+        self.case.check(self, tree)
+
+    def test12_str__arena__reuse_tree(self):  # cannot read string buffers (or can we?)
+        t = ryml.Tree()
+        ryml.parse_in_arena(self.src_as_str, tree=t)
+        self.case.check(self, t)
+
+    def test13_str__inplace(self):  # cannot mutate string buffers (or can we?)
+        with self.assertRaises(TypeError) as context:
+            ryml.parse_in_place(self.src_as_str)
+        self.assertTrue(type(context.exception), TypeError)
+
+    # ----------------------------------------------------------
+    def test21_bytes__arena(self):
+        tree = ryml.parse_in_arena(self.src_as_bytes)
+        self.case.check(self, tree)
+
+    def test22_bytes__arena__reuse_tree(self):
+        t = ryml.Tree()
+        r = ryml.parse_in_arena(self.src_as_bytes, tree=t)
+        self.assertTrue(r is t)
+        self.case.check(self, t)
+
+    def test23_bytes__inplace(self):  # cannot mutate bytes buffers
+        with self.assertRaises(TypeError) as context:
+            ryml.parse_in_place(self.src_as_bytes)
+        self.assertTrue(type(context.exception), TypeError)
+
+    # ----------------------------------------------------------
+    def test31_bytearray__arena(self):
+        tree = ryml.parse_in_arena(self.src_as_bytearray)
+        self.case.check(self, tree)
+
+    def test32_bytearray__arena__reuse_tree(self):
+        t = ryml.Tree()
+        r = ryml.parse_in_arena(self.src_as_bytearray, tree=t)
+        self.assertTrue(r is t)
+        self.case.check(self, t)
+
+    def test33_bytearray__inplace(self):  # bytearray buffers are mutable
+        tree = ryml.parse_in_place(self.src_as_bytearray)
+        self.case.check(self, tree)
+
+    def test34_bytearray__inplace__reuse_tree(self):  # bytearray buffers are mutable
+        t = ryml.Tree()
+        r = ryml.parse_in_place(self.src_as_bytearray, tree=t)
+        self.assertTrue(r is t)
+        self.case.check(self, t)
+
+    # ----------------------------------------------------------
+    def test41_emit_yaml(self):
+        tree = ryml.parse_in_arena(self.src_as_bytearray)
+        yaml = ryml.emit_yaml(tree)
+        output_tree = ryml.parse_in_arena(yaml)
+        self.case.check(self, output_tree)
+
+    def test41_emit_json(self):
+        tree = ryml.parse_in_arena(self.src_as_bytearray)
+        json = ryml.emit_json(tree)
+        output_tree = ryml.parse_in_arena(json)
+        self.case.check(self, output_tree, is_json=True)
+
+    def test42_compute_emit_yaml_length(self):
+        tree = ryml.parse_in_arena(self.src_as_bytearray)
+        yaml = ryml.emit_yaml(tree)
+        length = ryml.compute_emit_yaml_length(tree)
+        self.assertEqual(len(yaml), length)
+
+    def test42_compute_emit_json_length(self):
+        tree = ryml.parse_in_arena(self.src_as_bytearray)
+        json = ryml.emit_json(tree)
+        length = ryml.compute_emit_json_length(tree)
+        self.assertEqual(len(json), length)
+
+    def test43_emit_yaml_inplace(self):
+        tree = ryml.parse_in_arena(self.src_as_bytearray)
+        yaml = ryml.emit_yaml(tree)
+        length = ryml.compute_emit_yaml_length(tree)
+        self.assertEqual(len(yaml), length)
+        buf = bytearray(length)
+        s = ryml.emit_yaml_in_place(tree, buf)
+        self.assertEqual(len(s), length)
+        self.assertTrue(s.tobytes().decode('utf-8') == yaml)
+        self.assertTrue(buf.decode('utf-8') == yaml)
+
+    def test43_emit_json_inplace(self):
+        tree = ryml.parse_in_arena(self.src_as_bytearray)
+        json = ryml.emit_json(tree)
+        length = ryml.compute_emit_json_length(tree)
+        self.assertEqual(len(json), length)
+        buf = bytearray(length)
+        s = ryml.emit_json_in_place(tree, buf)
+        self.assertEqual(len(s), length)
+        self.assertTrue(s.tobytes().decode('utf-8') == json)
+        self.assertTrue(buf.decode('utf-8') == json)
+
+    def test44_emit_yaml_short_buf(self):
+        tree = ryml.parse_in_arena(self.src_as_bytearray)
+        length = ryml.compute_emit_yaml_length(tree)
+        buf = bytearray(length-1)
+        with self.assertRaises(IndexError):
+            ryml.emit_yaml_in_place(tree, buf)
+
+    def test44_emit_json_short_buf(self):
+        tree = ryml.parse_in_arena(self.src_as_bytearray)
+        length = ryml.compute_emit_json_length(tree)
+        buf = bytearray(length-1)
+        with self.assertRaises(IndexError):
+            ryml.emit_json_in_place(tree, buf)
+
+
+
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+if __name__ == "__main__":
+    unittest.main()