From 24e43a913f29ac3b314354e8ce5175f135bcc64f Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 7 Nov 2025 14:49:13 +0100 Subject: switch to xmake for package management (#611) This change removes our dependency on vcpkg for package management, in favour of bringing some code in-tree in the `thirdparty` folder as well as using the xmake build-in package management feature. For the latter, all the package definitions are maintained in the zen repo itself, in the `repo` folder. It should now also be easier to build the project as it will no longer depend on having the right version of vcpkg installed, which has been a common problem for new people coming in to the codebase. Now you should only need xmake to build. * Bumps xmake requirement on github runners to 2.9.9 to resolve an issue where xmake on Windows invokes cmake with `v144` toolchain which does not exist * BLAKE3 is now in-tree at `thirdparty/blake3` * cpr is now in-tree at `thirdparty/cpr` * cxxopts is now in-tree at `thirdparty/cxxopts` * fmt is now in-tree at `thirdparty/fmt` * robin-map is now in-tree at `thirdparty/robin-map` * ryml is now in-tree at `thirdparty/ryml` * sol2 is now in-tree at `thirdparty/sol2` * spdlog is now in-tree at `thirdparty/spdlog` * utfcpp is now in-tree at `thirdparty/utfcpp` * xmake package repo definitions is in `repo` * implemented support for sanitizers. ASAN is supported on windows, TSAN, UBSAN, MSAN etc are supported on Linux/MacOS though I have not yet tested it extensively on MacOS * the zencore encryption implementation also now supports using mbedTLS which is used on MacOS, though for now we still use openssl on Linux * crashpad * bumps libcurl to 8.11.0 (from 8.8.0) which should address a rare build upload bug --- thirdparty/ryml/api/python/.gitignore | 141 +++++++ thirdparty/ryml/api/python/Makefile | 94 +++++ thirdparty/ryml/api/python/bm/bm_parse.py | 237 ++++++++++++ thirdparty/ryml/api/python/requirements.txt | 5 + thirdparty/ryml/api/python/ryml/__init__.py | 2 + thirdparty/ryml/api/python/tests/test_parse.py | 488 +++++++++++++++++++++++++ 6 files changed, 967 insertions(+) create mode 100644 thirdparty/ryml/api/python/.gitignore create mode 100644 thirdparty/ryml/api/python/Makefile create mode 100644 thirdparty/ryml/api/python/bm/bm_parse.py create mode 100644 thirdparty/ryml/api/python/requirements.txt create mode 100644 thirdparty/ryml/api/python/ryml/__init__.py create mode 100644 thirdparty/ryml/api/python/tests/test_parse.py (limited to 'thirdparty/ryml/api/python') diff --git a/thirdparty/ryml/api/python/.gitignore b/thirdparty/ryml/api/python/.gitignore new file mode 100644 index 000000000..830e7e4d6 --- /dev/null +++ b/thirdparty/ryml/api/python/.gitignore @@ -0,0 +1,141 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Version file generated by setuptools_scm +version.py + +# SWIG produced file +ryml/ryml.py diff --git a/thirdparty/ryml/api/python/Makefile b/thirdparty/ryml/api/python/Makefile new file mode 100644 index 000000000..bdb5a6ffd --- /dev/null +++ b/thirdparty/ryml/api/python/Makefile @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT + +# Use bash even on Windows +SHELL := /bin/bash + +# On Windows the activate script is stored in a different location. +ACTIVATE_SCRIPT := venv/bin/activate +ifeq ($(OS),Windows_NT) +ACTIVATE_SCRIPT := venv/Scripts/activate +endif + +# How to invoke python +PYTHON := python +# How to invoke pytest +PYTEST := $(PYTHON) -m pytest -vvv + +ACTIVATE=[[ -e $(ACTIVATE_SCRIPT) ]] && source $(ACTIVATE_SCRIPT); + +.PHONY: clean +clean: + rm -rf dist *.egg-info + rm -rf ../../build ../../.egg* + rm -rf ryml/*.so ryml/ryml.py ryml/include ryml/lib + +.PHONY: venv-clean +venv-clean: + rm -rf venv + + +$(ACTIVATE_SCRIPT): requirements.txt Makefile + make venv + @touch $(ACTIVATE_SCRIPT) + +.PHONY: venv +venv: + virtualenv --python=python3 --always-copy venv + # Packaging tooling. + ${ACTIVATE} pip install -U pip + # Setup requirements. + ${ACTIVATE} pip install -v -r requirements.txt + ${ACTIVATE} pip install -v -e ../.. + @${ACTIVATE} $(PYTHON) -c "from ryml.version import version as v; print('Installed version:', v)" + +.PHONY: build-sdist +build-sdist: | $(ACTIVATE_SCRIPT) + ${ACTIVATE} (cd ../..; $(PYTHON) -m build --sdist --outdir $(PWD)/dist) + + +.PHONY: build-wheel +build-wheel: | $(ACTIVATE_SCRIPT) + rm -rf dist + $(MAKE) build-sdist + @ls -l dist/*.tar.gz + ${ACTIVATE} pip wheel -v dist/*.tar.gz --wheel-dir $(PWD)/dist + +.PHONY: build +build: + rm -rf build dist + $(MAKE) build-sdist + $(MAKE) build-wheel + +# PYPI_TEST = --repository-url https://test.pypi.org/legacy/ +PYPI_TEST = --repository testpypi + +.PHONY: upload-test +upload-test: | $(ACTIVATE_SCRIPT) + make clean + make build-sdist + ${ACTIVATE} twine upload ${PYPI_TEST} dist/* + +.PHONY: upload +upload: | $(ACTIVATE_SCRIPT) + make clean + make build-sdist + ${ACTIVATE} twine upload --verbose dist/* + +.PHONY: check +check: | $(ACTIVATE_SCRIPT) + make clean + make build-wheel + ${ACTIVATE} twine check dist/*.whl + +.PHONY: install +install: | $(ACTIVATE_SCRIPT) + ${ACTIVATE} $(PYTHON) setup.py install + +.PHONY: test +test: | $(ACTIVATE_SCRIPT) + ${ACTIVATE} $(PYTEST) tests + +.PHONY: version +version: | $(ACTIVATE_SCRIPT) + ${ACTIVATE} $(PYTHON) setup.py --version diff --git a/thirdparty/ryml/api/python/bm/bm_parse.py b/thirdparty/ryml/api/python/bm/bm_parse.py new file mode 100644 index 000000000..294be679b --- /dev/null +++ b/thirdparty/ryml/api/python/bm/bm_parse.py @@ -0,0 +1,237 @@ +import ryml +import ruamel.yaml +import yaml +import timeit +import time +import copy +import prettytable +import os.path +from collections import OrderedDict as odict + + +def _nodbg(*args, **kwargs): + pass + + +def _dbg(*args, **kwargs): + print(*args, **kwargs, file=sys.stderr, flush=True) + + +dbg = _dbg + + +class RunResults: + + __slots__ = ('name', 'time_ms', 'count', 'avg', 'MBps') + + def __init__(self, name, time_ms, count, num_bytes): + self.name = name + self.time_ms = time_ms + self.count = count + self.avg = time_ms / count + num_megabytes = count * num_bytes / 1.0e6 + num_seconds = time_ms / 1000.0 + self.MBps = num_megabytes / num_seconds + + def __str__(self): + fmt = "{}: count={} time={:.3f}ms avg={:.3f}ms MB/s={:.3f}" + fmt = fmt.format(self.name, self.count, self.time_ms, self.avg, self.MBps) + return fmt + + +class BmCase: + + def __init__(self, filename): + with open(filename, "r") as f: + src = f.read() + self.filename = filename + self.src_as_str = src + self.src_as_bytes = bytes(src, "utf8") + self.src_as_bytearray = bytearray(src, "utf8") + self.src_as_bytearray_orig = copy.copy(self.src_as_bytearray) + self.emittree = ryml.parse_in_arena(self.src_as_bytearray) + self.emitbuf = bytearray(4 * len(self.src_as_str)) # should be enough + + def run(self, bm_method_name, cls): + def run_bm(obj, subject): + obj.count = 0 + t = timeit.Timer(subject) + delta = time.time() + result = t.autorange() #lambda number, time_taken: time_taken > 1.0) + delta = 1000. * (time.time() - delta) + return delta, obj.count + obj = cls(self) + if not hasattr(obj, bm_method_name): + return None + name = bm_method_name + ":" + cls.__name__ + dbg(name, "...") + method = getattr(obj, bm_method_name) + reset_name = 'reset_' + bm_method_name + reset_fn = getattr(obj, reset_name, None) + def bm_fn(): + method(self) + obj.count += 1 + if reset_fn is not None: + reset_fn(self) + delta, count = run_bm(obj, bm_fn) + # correct the benchmark to account for the time spent + # resetting + if reset_fn is not None: + # find out how much it takes to reset the bytearray + if not hasattr(obj, 'bm_reset_done'): + def bm_reset(): + reset_fn(self) + obj.count += 1 + rdelta, rcount = run_bm(obj, bm_reset) + obj.bm_reset_time_per_iteration = rdelta / rcount + dbg(name, "reset_time_per_iteration={:.3f}us".format(obj.bm_reset_time_per_iteration * 1000.0)) + obj.bm_reset_done = True + reset_correction = count * obj.bm_reset_time_per_iteration + dbg(name, "delta={:.3f}ms".format(delta), "reset_correction={:.3f}ms({:.2f}%)".format(reset_correction, 100.0 * reset_correction / delta)) + delta -= reset_correction + ret = RunResults(name, delta, count, len(self.src_as_str)) + dbg(name, "ok:", ret) + return ret + + +def run(case, benchmarks, approaches): + for bm in benchmarks: + results = odict() + for cls in approaches: + r = case.run(bm, cls) + if r is None: + continue + results[r.name] = r + table = prettytable.PrettyTable() + name = os.path.basename(case.filename) + table.field_names = [name, "count", "time(ms)", "avg(ms)", "avg(MB/s)"] + table.align[name] = "l" + def i(v): return "{:5d}".format(v) + def f(v): return "{:8.3f}".format(v) + for v in results.values(): + table.add_row([v.name, i(v.count), f(v.time_ms), f(v.avg), f(v.MBps)]) + print(table) + + +class BmCaseRun: + def __init__(self, case): + self.reset_bytearray = False + + +class RymlParseInArena(BmCaseRun): + + def parse(self, case): + _ = ryml.parse_in_arena(case.src_as_bytearray) + + +class RymlParseInArenaReuse(BmCaseRun): + + def __init__(self, case): + self.tree = ryml.Tree() + + def parse(self, case): + ryml.parse_in_arena(case.src_as_bytearray, tree=self.tree) + + def reset_parse(self, case): + self.tree.clear() + self.tree.clear_arena() + + +class RymlParseInPlace(BmCaseRun): + + def parse(self, case): + _ = ryml.parse_in_place(case.src_as_bytearray) + + def reset_parse(self, case): + case.src_as_bytearray = copy.copy(case.src_as_bytearray_orig) + + +class RymlParseInPlaceReuse(BmCaseRun): + + def __init__(self, case): + self.tree = ryml.Tree() + + def parse(self, case): + ryml.parse_in_place(case.src_as_bytearray, tree=self.tree) + + def reset_parse(self, case): + self.tree.clear() + self.tree.clear_arena() + case.src_as_bytearray = copy.copy(case.src_as_bytearray_orig) + + +class RuamelYamlParse(BmCaseRun): + + def parse(self, case): + _ = ruamel.yaml.load(case.src_as_str, Loader=ruamel.yaml.Loader) + + +class PyYamlParse(BmCaseRun): + + def parse(self, case): + _ = yaml.safe_load(case.src_as_str) + + +class RymlEmitToNewBuffer(BmCaseRun): + + def emit_yaml(self, case): + _ = ryml.emit_yaml(case.emittree) + + def emit_json(self, case): + _ = ryml.emit_json(case.emittree) + + +class RymlEmitReuse(BmCaseRun): + + def emit_yaml(self, case): + _ = ryml.emit_yaml_in_place(case.emittree, case.emitbuf) + + def emit_json(self, case): + _ = ryml.emit_json_in_place(case.emittree, case.emitbuf) + + +class RuamelYamlEmit: + + def __init__(self, case): + case.ruamel_emittree = ruamel.yaml.load(case.src_as_str, Loader=ruamel.yaml.Loader) + + def emit_yaml(self, case): + # https://stackoverflow.com/a/47617341/5875572 + class MyToStr: + def __init__(self, *args, **kwargs): + self.s = b"" + def write(self, s): + self.s += s + dumper = MyToStr() + ruamel.yaml.YAML().dump(case.ruamel_emittree, MyToStr()) + + +class PyYamlEmit: + + def __init__(self, case): + case.pyyaml_emittree = yaml.load(case.src_as_str, Loader=yaml.Loader) + + def emit_yaml(self, case): + _ = yaml.dump(case.pyyaml_emittree) + + +if __name__ == "__main__": + import sys + if len(sys.argv) < 2: + raise Exception("") + filename = sys.argv[1] + if filename.endswith("outer1000_inner1000.yml"): # this one is too heavy for the Python libs + exit(0) + case = BmCase(filename) + run(case, benchmarks=('parse', ), + approaches=(RuamelYamlParse, + PyYamlParse, + RymlParseInArena, + RymlParseInArenaReuse, + RymlParseInPlace, + RymlParseInPlaceReuse)) + run(case, benchmarks=('emit_yaml', 'emit_json', ), + approaches=(RuamelYamlEmit, + PyYamlEmit, + RymlEmitToNewBuffer, + RymlEmitReuse)) diff --git a/thirdparty/ryml/api/python/requirements.txt b/thirdparty/ryml/api/python/requirements.txt new file mode 100644 index 000000000..86c126420 --- /dev/null +++ b/thirdparty/ryml/api/python/requirements.txt @@ -0,0 +1,5 @@ +ruamel.yaml +ninja +pyyaml +prettytable +pytest diff --git a/thirdparty/ryml/api/python/ryml/__init__.py b/thirdparty/ryml/api/python/ryml/__init__.py new file mode 100644 index 000000000..8bbb62199 --- /dev/null +++ b/thirdparty/ryml/api/python/ryml/__init__.py @@ -0,0 +1,2 @@ +from ryml.ryml import * +from .version import * diff --git a/thirdparty/ryml/api/python/tests/test_parse.py b/thirdparty/ryml/api/python/tests/test_parse.py new file mode 100644 index 000000000..bb7973cb3 --- /dev/null +++ b/thirdparty/ryml/api/python/tests/test_parse.py @@ -0,0 +1,488 @@ +import ryml +from ryml.ryml import _same_ptr, _same_mem +import unittest + + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +class TestSubstrInterop(unittest.TestCase): + + # ------------------------------------------------ + # str + + # CAN create c4::csubstr from string object + def test11_str2csubstr(self): + s = "asdasd" + m = ryml.as_csubstr(s) + self.assertTrue(_same_ptr(s, m)) + self.assertTrue(_same_mem(s, m)) + self.assertEqual(s, ryml.u(m)) + # + m = ryml.as_csubstr(m) + self.assertTrue(_same_ptr(s, m)) + self.assertTrue(_same_mem(s, m)) + self.assertEqual(s, ryml.u(m)) + + # CANNOT create c4::substr from string object + def test12_str2substr(self): + s = "" + with self.assertRaises(TypeError) as context: + _ = ryml.as_substr(s) + self.assertTrue(type(context.exception), TypeError) + + # ------------------------------------------------ + # bytes + + # CAN create c4::csubstr from string object + def test21_bytes2csubstr(self): + s = b"foo21" + m = ryml.as_csubstr(s) + self.assertTrue(_same_ptr(s, m)) + self.assertTrue(_same_mem(s, m)) + self.assertEqual(s, m) + # + m = ryml.as_csubstr(m) + self.assertTrue(_same_ptr(s, m)) + self.assertTrue(_same_mem(s, m)) + self.assertEqual(s, m) + + # CANNOT create c4::csubstr from string object + def test22_bytes2substr(self): + s = b"foo22" + with self.assertRaises(TypeError) as context: + _ = ryml.as_substr(s) + self.assertTrue(type(context.exception), TypeError) + + # ------------------------------------------------ + # bytearray + + # CAN create c4::csubstr from string object + def test31_bytes2csubstr(self): + s = bytearray("foo31", "utf8") + m = ryml.as_csubstr(s) + self.assertTrue(_same_ptr(s, m)) + self.assertTrue(_same_mem(s, m)) + self.assertEqual(s, m) + # + m = ryml.as_csubstr(m) + self.assertTrue(_same_ptr(s, m)) + self.assertTrue(_same_mem(s, m)) + self.assertEqual(s, m) + + # CANNOT create c4::csubstr from string object + def test32_bytes2substr(self): + s = bytearray("foo31", "utf8") + m = ryml.as_csubstr(s) + self.assertTrue(_same_ptr(s, m)) + self.assertTrue(_same_mem(s, m)) + self.assertEqual(s, m) + # + m = ryml.as_csubstr(m) + self.assertTrue(_same_ptr(s, m)) + self.assertTrue(_same_mem(s, m)) + self.assertEqual(s, m) + + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- + +def _addmap(t, node, k=None): + m = t.append_child(node) + if k is None: + t.to_map(m) + else: + t.to_map(m, k) + return m + + +def _addseq(t, node, k=None): + m = t.append_child(node) + if k is None: + t.to_seq(m) + else: + t.to_seq(m, k) + return m + + +def _addval(t, node, k, v=None): + ch = t.append_child(node) + if v is None: + t.to_val(ch, k) + else: + t.to_keyval(ch, k, v) + return ch + + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +def check_tree_mod(ut, t): + # some convenient shorthands + eq = ut.assertEqual + def _addval_and_check(node, k, v=None): + ch = _addval(t, node, k, v) + pos = t.child_pos(node, ch) + eq(t.child(node, pos), ch) + if v is not None: + eq(t.find_child(node, k), ch) + eq(t.child(node, pos), t.find_child(node, k)) + return ch + def _addseq_and_check(node, k): + ch = _addseq(t, node, k) + eq(t.find_child(node, k), ch) + return ch + def _addmap_and_check(node, k): + ch = _addmap(t, node, k) + eq(t.find_child(node, k), ch) + return ch + m = _addmap_and_check(t.root_id(), "check_tree_mod_map") + _addval_and_check(m, "k1", "v1") + _addval_and_check(m, "k2", "v2") + _addval_and_check(m, "k3", "v3") + eq(t.num_children(m), 3) + eq(t.num_siblings(t.first_child(m)), 3) + s = _addseq_and_check(t.root_id(), "check_tree_mod_seq") + _addval_and_check(s, "v1") + _addval_and_check(s, "v2") + _addval_and_check(s, "v3") + eq(t.num_children(s), 3) + eq(t.num_siblings(t.first_child(m)), 3) + + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +class SimpleTestCase: + + yaml = "{'HELLO': a, foo: \"b\", bar: c, baz: d, seq: [0, 1, 2, 3]}" + + def check(self, ut, t, is_json=False): + # some convenient shorthands + eq = ut.assertEqual + ne = ut.assertNotEqual + fs = ut.assertFalse + tr = ut.assertTrue + # + eq(t.size(), 10) + tr(t.is_root(0)) + eq(t.num_children(0), 5) + eq(t.find_child(0, b"HELLO"), 1) + eq(t.find_child(0, b"foo"), 2) + eq(t.find_child(0, b"bar"), 3) + eq(t.find_child(0, b"baz"), 4) + eq(t.find_child(0, b"seq"), 5) + eq(t.parent(0), ryml.NONE) + eq(t.parent(1), 0) + eq(t.parent(2), 0) + eq(t.parent(3), 0) + eq(t.parent(4), 0) + eq(t.parent(5), 0) + fs(t.is_root(1)) + fs(t.is_root(2)) + fs(t.is_root(3)) + fs(t.is_root(4)) + fs(t.is_root(5)) + fs(t.has_child(0, b"foozzie")) + fs(t.has_child(0, b"bark")) + fs(t.has_child(0, b"bart")) + fs(t.has_child(0, b"bazk")) + eq(t.next_sibling(0), ryml.NONE) + eq(t.prev_sibling(0), ryml.NONE) + eq(t.prev_sibling(1), ryml.NONE) + eq(t.next_sibling(5), ryml.NONE) + tr(t.has_child(0, b"HELLO")) + tr(t.has_child(0, b"foo")) + tr(t.has_child(0, b"bar")) + tr(t.has_child(0, b"baz")) + eq(t.key(1), b"HELLO") + eq(t.key(2), b"foo") + eq(t.key(3), b"bar") + eq(t.key(4), b"baz") + eq(t.key(5), b"seq") + eq(t.val(1), b"a") + eq(t.val(2), b"b") + eq(t.val(3), b"c") + eq(t.val(4), b"d") + eq(t.val(6), b"0") + eq(t.val(7), b"1") + eq(t.val(8), b"2") + eq(t.val(9), b"3") + if not is_json: + tr(t.is_key_quoted(1)) + fs(t.is_key_quoted(2)) + fs(t.is_key_quoted(3)) + fs(t.is_key_quoted(4)) + fs(t.is_key_quoted(5)) + else: + tr(t.is_key_quoted(1)) + tr(t.is_key_quoted(2)) + tr(t.is_key_quoted(3)) + tr(t.is_key_quoted(4)) + tr(t.is_key_quoted(5)) + if not is_json: + fs(t.is_val_quoted(1)) + tr(t.is_val_quoted(2)) + fs(t.is_val_quoted(3)) + fs(t.is_val_quoted(4)) + fs(t.is_val_quoted(5)) + fs(t.is_val_quoted(6)) + fs(t.is_val_quoted(7)) + fs(t.is_val_quoted(8)) + fs(t.is_val_quoted(9)) + else: + tr(t.is_val_quoted(1)) + tr(t.is_val_quoted(2)) + tr(t.is_val_quoted(3)) + tr(t.is_val_quoted(4)) + fs(t.is_val_quoted(5)) + fs(t.is_val_quoted(6)) + fs(t.is_val_quoted(7)) + fs(t.is_val_quoted(8)) + fs(t.is_val_quoted(9)) + if not is_json: + tr(t.is_quoted(1)) + tr(t.is_quoted(2)) + fs(t.is_quoted(3)) + fs(t.is_quoted(4)) + fs(t.is_quoted(5)) + fs(t.is_quoted(6)) + fs(t.is_quoted(7)) + fs(t.is_quoted(8)) + fs(t.is_quoted(9)) + else: + tr(t.is_quoted(1)) + tr(t.is_quoted(2)) + tr(t.is_quoted(3)) + tr(t.is_quoted(4)) + tr(t.is_quoted(5)) + fs(t.is_quoted(6)) + fs(t.is_quoted(7)) + fs(t.is_quoted(8)) + fs(t.is_quoted(9)) + tr(t.has_sibling(1, b"bar")) + tr(t.has_sibling(1, b"baz")) + tr(t.has_sibling(2, b"foo")) + tr(t.has_sibling(2, b"baz")) + tr(t.has_sibling(3, b"foo")) + tr(t.has_sibling(3, b"bar")) + for i in (1, 2, 3, 4, 5): + eq(t.find_sibling(i, b"HELLO"), 1) + eq(t.find_sibling(i, b"foo"), 2) + eq(t.find_sibling(i, b"bar"), 3) + eq(t.find_sibling(i, b"baz"), 4) + eq(t.find_sibling(i, b"seq"), 5) + # + num = 0 + for id in ryml.children(t): + num += 1 + eq(id, num) + eq(num, t.num_children(t.root_id())) + eq(num, t.num_siblings(t.first_child(t.root_id()))) + # + num = 0 + for id in ryml.children(t, 1): + num += 1 + eq(num, 0) + # + num = 0 + for id in ryml.siblings(t, 1): + num += 1 + eq(id, num) + eq(num, t.num_children(t.root_id())) + eq(num, t.num_siblings(t.first_child(t.root_id()))) + # + num = 0 + for id in ryml.siblings(t, 3): + num += 1 + eq(id, num) + eq(num, 5) + eq(num, t.num_siblings(t.first_child(t.root_id()))) + # + for i, ch in enumerate(ryml.children(t, 5)): + eq(t.val(ch), [b"0", b"1", b"2", b"3"][i]) + sibs = [b"HELLO", b"foo", b"bar", b"baz", b"seq"] + sibs_s = ["HELLO", "foo", "bar", "baz", "seq"] + for i, sib in enumerate(ryml.siblings(t, 5)): + k = t.key(sib) + k_s = str(k, "utf8") + eq(k, sibs[i]) + eq(k_s, sibs_s[i]) + ne(k, sibs_s[i]) + ne(k_s, sibs[i]) + k_s = str(k) + ne(k_s, sibs_s[i]) + ne(k_s, sibs[i]) + num = 0 + for id in ryml.siblings(t, 0): + num += 1 + eq(num, 1) + # + num = 0 + for id, level in ryml.walk(t): + num += 1 + if t.is_root(id): + eq(id, 0) + eq(level, 0) + if t.is_map(id): + eq(id, 0) + eq(level, 0) + if t.is_seq(id): + eq(id, 5) + eq(level, 1) + if t.is_keyval(id): + tr(id > 0 and id < 5) + if t.is_val(id): + tr(id > 5) + eq(level, 2) + eq(num, t.size()) + # + num = 0 + for id in ryml.walk(t, 5): + num += 1 + eq(num, 5) + # + num = 0 + for id in ryml.walk(t, 9): + num += 1 + eq(num, 1) + check_tree_mod(ut, t) + + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +class TestRunner(unittest.TestCase): + + def setUp(self): + self._setUp(SimpleTestCase()) + + # allow creating this class with different cases + # if they are added + def _setUp(self, case): + self.case = case + self.src_as_str = str(case.yaml) + self.src_as_bytes = bytes(case.yaml, "utf8") + self.src_as_bytearray = bytearray(case.yaml, "utf8") + + # ---------------------------------------------------------- + def test11_str__arena(self): # cannot read string buffers (or can we?) + tree = ryml.parse_in_arena(self.src_as_str) + self.case.check(self, tree) + + def test12_str__arena__reuse_tree(self): # cannot read string buffers (or can we?) + t = ryml.Tree() + ryml.parse_in_arena(self.src_as_str, tree=t) + self.case.check(self, t) + + def test13_str__inplace(self): # cannot mutate string buffers (or can we?) + with self.assertRaises(TypeError) as context: + ryml.parse_in_place(self.src_as_str) + self.assertTrue(type(context.exception), TypeError) + + # ---------------------------------------------------------- + def test21_bytes__arena(self): + tree = ryml.parse_in_arena(self.src_as_bytes) + self.case.check(self, tree) + + def test22_bytes__arena__reuse_tree(self): + t = ryml.Tree() + r = ryml.parse_in_arena(self.src_as_bytes, tree=t) + self.assertTrue(r is t) + self.case.check(self, t) + + def test23_bytes__inplace(self): # cannot mutate bytes buffers + with self.assertRaises(TypeError) as context: + ryml.parse_in_place(self.src_as_bytes) + self.assertTrue(type(context.exception), TypeError) + + # ---------------------------------------------------------- + def test31_bytearray__arena(self): + tree = ryml.parse_in_arena(self.src_as_bytearray) + self.case.check(self, tree) + + def test32_bytearray__arena__reuse_tree(self): + t = ryml.Tree() + r = ryml.parse_in_arena(self.src_as_bytearray, tree=t) + self.assertTrue(r is t) + self.case.check(self, t) + + def test33_bytearray__inplace(self): # bytearray buffers are mutable + tree = ryml.parse_in_place(self.src_as_bytearray) + self.case.check(self, tree) + + def test34_bytearray__inplace__reuse_tree(self): # bytearray buffers are mutable + t = ryml.Tree() + r = ryml.parse_in_place(self.src_as_bytearray, tree=t) + self.assertTrue(r is t) + self.case.check(self, t) + + # ---------------------------------------------------------- + def test41_emit_yaml(self): + tree = ryml.parse_in_arena(self.src_as_bytearray) + yaml = ryml.emit_yaml(tree) + output_tree = ryml.parse_in_arena(yaml) + self.case.check(self, output_tree) + + def test41_emit_json(self): + tree = ryml.parse_in_arena(self.src_as_bytearray) + json = ryml.emit_json(tree) + output_tree = ryml.parse_in_arena(json) + self.case.check(self, output_tree, is_json=True) + + def test42_compute_emit_yaml_length(self): + tree = ryml.parse_in_arena(self.src_as_bytearray) + yaml = ryml.emit_yaml(tree) + length = ryml.compute_emit_yaml_length(tree) + self.assertEqual(len(yaml), length) + + def test42_compute_emit_json_length(self): + tree = ryml.parse_in_arena(self.src_as_bytearray) + json = ryml.emit_json(tree) + length = ryml.compute_emit_json_length(tree) + self.assertEqual(len(json), length) + + def test43_emit_yaml_inplace(self): + tree = ryml.parse_in_arena(self.src_as_bytearray) + yaml = ryml.emit_yaml(tree) + length = ryml.compute_emit_yaml_length(tree) + self.assertEqual(len(yaml), length) + buf = bytearray(length) + s = ryml.emit_yaml_in_place(tree, buf) + self.assertEqual(len(s), length) + self.assertTrue(s.tobytes().decode('utf-8') == yaml) + self.assertTrue(buf.decode('utf-8') == yaml) + + def test43_emit_json_inplace(self): + tree = ryml.parse_in_arena(self.src_as_bytearray) + json = ryml.emit_json(tree) + length = ryml.compute_emit_json_length(tree) + self.assertEqual(len(json), length) + buf = bytearray(length) + s = ryml.emit_json_in_place(tree, buf) + self.assertEqual(len(s), length) + self.assertTrue(s.tobytes().decode('utf-8') == json) + self.assertTrue(buf.decode('utf-8') == json) + + def test44_emit_yaml_short_buf(self): + tree = ryml.parse_in_arena(self.src_as_bytearray) + length = ryml.compute_emit_yaml_length(tree) + buf = bytearray(length-1) + with self.assertRaises(IndexError): + ryml.emit_yaml_in_place(tree, buf) + + def test44_emit_json_short_buf(self): + tree = ryml.parse_in_arena(self.src_as_bytearray) + length = ryml.compute_emit_json_length(tree) + buf = bytearray(length-1) + with self.assertRaises(IndexError): + ryml.emit_json_in_place(tree, buf) + + + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3