switch to xmake for package management (#611)

This change removes our dependency on vcpkg for package management, in favour of bringing some code in-tree in the `thirdparty` folder as well as using the xmake build-in package management feature. For the latter, all the package definitions are maintained in the zen repo itself, in the `repo` folder.

It should now also be easier to build the project as it will no longer depend on having the right version of vcpkg installed, which has been a common problem for new people coming in to the codebase. Now you should only need xmake to build.

* Bumps xmake requirement on github runners to 2.9.9 to resolve an issue where xmake on Windows invokes cmake with `v144` toolchain which does not exist
* BLAKE3 is now in-tree at `thirdparty/blake3`
* cpr is now in-tree at `thirdparty/cpr`
* cxxopts is now in-tree at `thirdparty/cxxopts`
* fmt is now in-tree at `thirdparty/fmt`
* robin-map is now in-tree at `thirdparty/robin-map`
* ryml is now in-tree at `thirdparty/ryml`
* sol2 is now in-tree at `thirdparty/sol2`
* spdlog is now in-tree at `thirdparty/spdlog`
* utfcpp is now in-tree at `thirdparty/utfcpp`
* xmake package repo definitions is in `repo`
* implemented support for sanitizers. ASAN is supported on windows, TSAN, UBSAN, MSAN etc are supported on Linux/MacOS though I have not yet tested it extensively on MacOS
* the zencore encryption implementation also now supports using mbedTLS which is used on MacOS, though for now we still use openssl on Linux
* crashpad
* bumps libcurl to 8.11.0 (from 8.8.0) which should address a rare build upload bug

1 files changed, 366 insertions, 0 deletions

diff --git a/thirdparty/ryml/.github/workflows/release.yml b/thirdparty/ryml/.github/workflows/release.yml
new file mode 100644
index 000000000..3797359b8
--- /dev/null
+++ b/thirdparty/ryml/.github/workflows/release.yml
@@ -0,0 +1,366 @@
+name: release
+
+defaults:
+  run:
+    # Use a bash shell so we can use the same syntax for environment variable
+    # access regardless of the host operating system
+    shell: bash -e -x {0}
+
+on:
+  # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662
+  workflow_dispatch:
+  push:
+    tags:
+      - v0.*
+      - v1.*
+      - v2.*
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+env:
+  PROJ_PKG_NAME: rapidyaml-
+  PROJ_PFX_TARGET: ryml-
+  PROJ_PFX_CMAKE: RYML_
+  CMAKE_FLAGS: -DRYML_TEST_SUITE=OFF
+  NUM_JOBS_BUILD: # 4
+
+
+# useful to iterate when fixing the release:
+# ver=0.2.1 ; ( set -x ; git tag -d v$ver ; git push origin :v$ver ) ; (set -x ; set -e ; tbump --only-patch --non-interactive $ver ; git add -u ; git commit --amend --no-edit ; git tag --annotate --message "v$ver" "v$ver" ; git push -f --tags origin )
+
+jobs:
+
+  gettag:
+    if: |
+      (!contains(github.event.head_commit.message, 'skip all')) ||
+      (!contains(github.event.head_commit.message, 'skip release')) ||
+      contains(github.event.head_commit.message, 'only release')
+    runs-on: ubuntu-latest
+    steps:
+      # use fetch-depth to ensure all tags are fetched
+      - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}}
+      - name: Variables (from tag)
+        if: contains(github.ref, 'tags/v')
+        run: |
+          # https://github.community/t/how-to-get-just-the-tag-name/16241/11
+          SRC_TAG=${GITHUB_REF#refs/tags/}
+          SRC_VERSION=${GITHUB_REF#refs/tags/v}
+          cat <<EOF > vars.sh
+          export SRC_TAG=$SRC_TAG
+          export SRC_VERSION=$SRC_VERSION
+          EOF
+      - name: Variables (from commit, no tag)
+        if: ${{ !contains(github.ref, 'tags/v') }}
+        run: |
+          set -x
+          branch_name=${GITHUB_REF#refs/heads/}
+          # builds triggered from PRs have the branch_name like this: refs/pull/150/merge
+          # so filter to eg pr0150_merge
+          branch_name=`echo $branch_name | sed "s:refs/pull/\([0-9]*\)/\(.*\):pr0\1_\2:"`
+          # sanitize the branch name; eg merge/foo-bar -> merge_foo_bar
+          branch_name=`echo $branch_name | sed 's:[/.-]:_:g'`
+          SRC_TAG=$(git describe || git rev-parse --short HEAD)  # eg v0.2.0-110-gda837e0
+          SRC_VERSION="${branch_name}-${SRC_TAG}"
+          cat <<EOF > vars.sh
+          export SRC_TAG=$SRC_TAG
+          export SRC_VERSION=$SRC_VERSION
+          EOF
+      - name: Verify vars.sh
+        run: cat vars.sh ; source vars.sh ; echo $SRC_TAG ; echo $SRC_VERSION
+      - name: Save vars.sh
+        uses: actions/upload-artifact@v3
+        with: {name: vars.sh, path: ./vars.sh}
+
+  #----------------------------------------------------------------------------
+  # create source packages
+  src:
+    if: |
+      (!contains(github.event.head_commit.message, 'skip all')) ||
+      (!contains(github.event.head_commit.message, 'skip release')) ||
+      contains(github.event.head_commit.message, 'only release')
+    needs: gettag
+    runs-on: ubuntu-latest
+    steps:
+      - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}}
+      - name: Download vars.sh
+        uses: actions/download-artifact@v3
+        with: {name: vars.sh, path: ./}
+      - name: Install python 3.9
+        uses: actions/setup-python@v4
+        with: { python-version: 3.9 }
+      - name: Install requirements
+        run: |
+          sudo -E pip install git-archive-all
+      - name: Create source packages
+        run: |
+          pwd
+          ls -lFhp
+          source vars.sh
+          echo SRC_TAG=$SRC_TAG
+          echo SRC_VERSION=$SRC_VERSION
+          id=${PROJ_PKG_NAME}${SRC_VERSION}
+          name=${id}-src
+          mkdir -p assets
+          git-archive-all --prefix $name assets/$name.tgz
+          git-archive-all --prefix $name assets/$name.zip
+          python --version
+          python tools/amalgamate.py assets/$id.hpp
+      - name: Save source artifacts
+        uses: actions/upload-artifact@v3
+        with: {name: assets, path: assets}
+
+  #----------------------------------------------------------------------------
+  # create c++ packages
+  cpp:
+    if: |
+      (!contains(github.event.head_commit.message, 'skip all')) ||
+      (!contains(github.event.head_commit.message, 'skip release')) ||
+      contains(github.event.head_commit.message, 'only release')
+    name: cpp/${{matrix.config.os}}/${{matrix.config.gen}}
+    needs: gettag
+    runs-on: ${{matrix.config.os}}
+    env: {DEV: OFF, BT: Release, OS: "${{matrix.config.os}}", CXX_: "${{matrix.config.cxx}}", GEN: "${{matrix.config.gen}}"}
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          #  name of the artifact    | suffix (gen)    | suffix (package)         | cpack gen | mime type                      | os              | cxx
+          - {name: Ubuntu 20.04 deb  , sfxg: unix64.deb, sfxp: ubuntu-20.04.deb   , gen: DEB  , mime: vnd.debian.binary-package, os: ubuntu-20.04             }
+          #- {name: Ubuntu 18.04 deb  , sfxg: unix64.deb, sfxp: ubuntu-18.04.deb   , gen: DEB  , mime: vnd.debian.binary-package, os: ubuntu-18.04             }
+          - {name: Windows VS2019 zip, sfxg: win64.zip , sfxp: windows-vs2019.zip , gen: ZIP  , mime: zip                      , os: windows-2019, cxx: vs2019}
+          - {name: MacOSX sh         , sfxg: apple64.sh, sfxp: macosx-xcode.sh    , gen: STGZ , mime: x-sh                     , os: macos-11.0  , cxx: xcode }
+    steps:
+      - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}}
+      - name: Download vars.sh
+        uses: actions/download-artifact@v3
+        with: {name: vars.sh, path: ./}
+      - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS}
+      - {name: show info, run: source .github/setenv.sh && c4_show_info }
+      - name: shared64-configure---------------------------------------------------
+        run: source .github/setenv.sh && c4_cfg_test shared64
+      - {name: shared64-build, run: source .github/setenv.sh && c4_build_target shared64}
+      - name: shared64-pack
+        run: source .github/setenv.sh && c4_package shared64 $GEN
+      - name: shared64-normalize
+        run: |
+          set -x
+          source vars.sh
+          mkdir -p assets
+          asset_src=`ls -1 ./build/shared64/*-${{matrix.config.sfxg}}`
+          asset_dst=./assets/${PROJ_PKG_NAME}${SRC_VERSION}-${{matrix.config.sfxp}}
+          [ ! -f $asset_src ] && exit 1
+          cp -fav $asset_src $asset_dst
+      - name: Save artifacts
+        uses: actions/upload-artifact@v3
+        with: {name: assets, path: assets}
+
+  #----------------------------------------------------------------------------
+  # create python packages
+  # adapted from https://github.com/pikepdf/pikepdf/blob/master/.github/workflows/build_wheels.yml
+
+  python_src:
+    if: |
+      (!contains(github.event.head_commit.message, 'skip all')) ||
+      (!contains(github.event.head_commit.message, 'skip release')) ||
+      contains(github.event.head_commit.message, 'only release')
+    name: python/src
+    runs-on: ubuntu-latest
+    steps:
+      # use fetch-depth to ensure all tags are fetched
+      - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}}
+      - name: install python 3.9
+        uses: actions/setup-python@v4
+        with: { python-version: 3.9 }
+      - name: package python src packages
+        run: |
+          python --version
+          pip install -v -r requirements.txt
+          python setup.py sdist --formats=zip
+      - name: normalize src package names
+        run: |
+          sdist_orig=`find dist -type f -name 'rapidyaml-*.zip'`
+          [ ! -f $sdist_orig ] && exit 1
+          sdist=`echo $sdist_orig | sed 's:\.zip:-python_src.zip:'`
+          mv -fv $sdist_orig $sdist
+      - name: Save artifacts
+        uses: actions/upload-artifact@v3
+        with: {name: dist, path: dist}
+
+  python_wheels:
+    if: |
+      (!contains(github.event.head_commit.message, 'skip all')) ||
+      (!contains(github.event.head_commit.message, 'skip release')) ||
+      contains(github.event.head_commit.message, 'only release')
+    name: python/${{matrix.config.cibw_pyv}}/${{matrix.config.osname}}/${{matrix.config.cibw_arch}}
+    runs-on: ${{matrix.config.os}}
+    env:
+      CMAKE_FLAGS: "${{matrix.config.cmakeflags}} -DRYML_DEV=OFF -DRYML_BUILD_API=ON -DRYML_API_TESTS=OFF -DRYML_API_BENCHMARKS=OFF"
+      CIBW_BUILD: "cp${{matrix.config.cibw_pyv}}-${{matrix.config.cibw_platform}}"
+      CIBW_ARCHS: "${{matrix.config.cibw_arch}}"
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          # the 3-digit versions NEED to be quoted to prevent the version being read as float. (!)
+          - {pythonv: '3.11', cibw_pyv: 311, cibw_arch: x86_64, cibw_platform: manylinux_x86_64, osname: linux, os: ubuntu-20.04}
+          - {pythonv: '3.11', cibw_pyv: 311, cibw_arch: i686  , cibw_platform: manylinux_i686  , osname: linux, os: ubuntu-20.04}
+          - {pythonv: '3.10', cibw_pyv: 310, cibw_arch: x86_64, cibw_platform: manylinux_x86_64, osname: linux, os: ubuntu-20.04}
+          - {pythonv: '3.10', cibw_pyv: 310, cibw_arch: i686  , cibw_platform: manylinux_i686  , osname: linux, os: ubuntu-20.04}
+          - {pythonv:  3.9  , cibw_pyv: 39 , cibw_arch: x86_64, cibw_platform: manylinux_x86_64, osname: linux, os: ubuntu-20.04}
+          - {pythonv:  3.9  , cibw_pyv: 39 , cibw_arch: i686  , cibw_platform: manylinux_i686  , osname: linux, os: ubuntu-20.04}
+          - {pythonv:  3.8  , cibw_pyv: 38 , cibw_arch: x86_64, cibw_platform: manylinux_x86_64, osname: linux, os: ubuntu-20.04}
+          - {pythonv:  3.8  , cibw_pyv: 38 , cibw_arch: i686  , cibw_platform: manylinux_i686  , osname: linux, os: ubuntu-20.04}
+          - {pythonv:  3.7  , cibw_pyv: 37 , cibw_arch: x86_64, cibw_platform: manylinux_x86_64, osname: linux, os: ubuntu-20.04}
+          - {pythonv:  3.7  , cibw_pyv: 37 , cibw_arch: i686  , cibw_platform: manylinux_i686  , osname: linux, os: ubuntu-20.04}
+          - {pythonv:  3.6  , cibw_pyv: 36 , cibw_arch: x86_64, cibw_platform: manylinux_x86_64, osname: linux, os: ubuntu-20.04}
+          - {pythonv:  3.6  , cibw_pyv: 36 , cibw_arch: i686  , cibw_platform: manylinux_i686  , osname: linux, os: ubuntu-20.04}
+          # the windows builds are disabled because they are causing problems and preventing the release.
+          # the problems are related to CMakeExtension forcing the use of Ninja
+          # which does not play well with the -G 'Visual Studio...' option used below.
+          # fixing this looks like it will be time-intensive.
+          #- {pythonv: '3.11', cibw_pyv: 311, cibw_arch: AMD64 , cibw_platform: win_amd64, osname: win  , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A x64'}
+          #- {pythonv: '3.11', cibw_pyv: 311, cibw_arch: x86   , cibw_platform: win32    , osname: win  , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A Win32'}
+          #- {pythonv: '3.10', cibw_pyv: 310, cibw_arch: AMD64 , cibw_platform: win_amd64, osname: win  , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A x64'}
+          #- {pythonv: '3.10', cibw_pyv: 310, cibw_arch: x86   , cibw_platform: win32    , osname: win  , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A Win32'}
+          #- {pythonv:  3.9  , cibw_pyv: 39 , cibw_arch: AMD64 , cibw_platform: win_amd64, osname: win  , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A x64'}
+          #- {pythonv:  3.9  , cibw_pyv: 39 , cibw_arch: x86   , cibw_platform: win32    , osname: win  , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A Win32'}
+          #- {pythonv:  3.8  , cibw_pyv: 38 , cibw_arch: AMD64 , cibw_platform: win_amd64, osname: win  , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A x64'}
+          #- {pythonv:  3.8  , cibw_pyv: 38 , cibw_arch: x86   , cibw_platform: win32    , osname: win  , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A Win32'}
+          - {pythonv:  3.7  , cibw_pyv: 37 , cibw_arch: AMD64 , cibw_platform: win_amd64, osname: win  , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A x64'}
+          #- {pythonv:  3.7  , cibw_pyv: 37 , cibw_arch: x86   , cibw_platform: win32    , osname: win  , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A Win32'}
+          - {pythonv:  3.6  , cibw_pyv: 36 , cibw_arch: AMD64 , cibw_platform: win_amd64, osname: win  , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A x64'}
+          #- {pythonv:  3.6  , cibw_pyv: 36 , cibw_arch: x86   , cibw_platform: win32    , osname: win  , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A Win32'}
+          ## macosx builds are generating a SIGSEGV when importing. (!)
+          ## https://github.com/biojppm/rapidyaml/actions/runs/3062528713/jobs/4943611397#step:7:269
+          #- {pythonv: '3.11', cibw_pyv: 311, cibw_arch: x86_64, cibw_platform: macosx_x86_64, osname: macos, os: macos-10.15}
+          #- {pythonv: '3.10', cibw_pyv: 310, cibw_arch: x86_64, cibw_platform: macosx_x86_64, osname: macos, os: macos-10.15}
+          #- {pythonv:  3.9  , cibw_pyv: 39 , cibw_arch: x86_64, cibw_platform: macosx_x86_64, osname: macos, os: macos-10.15}
+          #- {pythonv:  3.8  , cibw_pyv: 38 , cibw_arch: x86_64, cibw_platform: macosx_x86_64, osname: macos, os: macos-10.15}
+          #- {pythonv:  3.7  , cibw_pyv: 37 , cibw_arch: x86_64, cibw_platform: macosx_x86_64, osname: macos, os: macos-10.15}
+          #- {pythonv:  3.6  , cibw_pyv: 36 , cibw_arch: x86_64, cibw_platform: macosx_x86_64, osname: macos, os: macos-10.15}
+    steps:
+      # use fetch-depth to ensure all tags are fetched
+      - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}}
+      - name: create wheel
+        uses: joerick/[email protected]
+      - name: rename wheelhouse -> dist
+        run: |
+          mv -fv wheelhouse dist
+          ls -lFhp dist/
+      - name: Save artifacts for publishing to PyPI
+        uses: actions/upload-artifact@v3
+        with: {name: dist, path: dist}
+      # run the tests
+      - name: install python ${{matrix.config.pythonv}}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '${{matrix.config.pythonv}}'
+      - name: test with python ${{matrix.config.pythonv}}
+        run: |
+          set -x
+          echo "python ${{matrix.config.pythonv}} ${{matrix.config.py_arch}} ${{matrix.config.cibw_arch}}"
+          # skip 32 bit tests, as Python 32 bits are not available in ubuntu
+          arch="${{matrix.config.cibw_arch}}"
+          if [ "$arch" == "x86" ] || [ "$arch" == "i686" ] ; then
+            exit 0
+          fi
+          python --version
+          python -c 'import sys ; import struct ; print("python:", sys.version, struct.calcsize("P") * 8, "bits")'
+          pip --version
+          pip install -v -r requirements.txt
+          pip install -v -r api/python/requirements.txt
+          for whl in dist/* ; do
+            pip install -v $whl
+            pip show -f rapidyaml
+            python -c 'import ryml ; print("ryml", ryml.version, ryml.version_tuple)'
+            python -c 'import ryml ; tree = ryml.parse_in_arena(b"{foo: bar}") ; assert tree.key(1) == b"foo" ; assert tree.val(1) == b"bar" ; print(str(tree.key(1), "utf8")) ; print(str(tree.val(1), "utf8"))'
+            python -m pytest -vvv api/python/tests
+            pip uninstall -y -v rapidyaml
+          done
+
+
+  #----------------------------------------------------------------------------
+  release:
+    if: |
+      (!contains(github.event.head_commit.message, 'skip all')) ||
+      (!contains(github.event.head_commit.message, 'skip release')) ||
+      contains(github.event.head_commit.message, 'only release')
+    runs-on: ubuntu-latest
+    needs:
+      - src
+      - cpp
+      - python_src
+      - python_wheels
+    steps:
+      - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}}
+      - name: Gather artifacts - ./assets
+        uses: actions/download-artifact@v3
+        with: {name: assets, path: assets}
+      - name: Gather artifacts - ./dist
+        uses: actions/download-artifact@v3
+        with: {name: dist, path: dist}
+      - name: Verify existing artifacts
+        run: |
+          ls -lFhp assets/
+          ls -lFhp dist/
+      #
+      # Github
+      - name: Restore vars.sh
+        if: contains(github.ref, 'tags/v')
+        uses: actions/download-artifact@v3
+        with: {name: vars.sh, path: ./}
+      - name: Save vars for following steps
+        if: contains(github.ref, 'tags/v')
+        id: vars
+        run: |
+          source vars.sh
+          version_body=${{github.workspace}}/changelog/$SRC_VERSION.md
+          if [ ! -f $version_body ] ; then
+            echo "version body file was not found: $version_body"
+            exit 1
+          fi
+          echo "VERSION=$SRC_VERSION >> $GITHUB_OUTPUT"
+          echo "VERSION_BODY=$version_body >> $GITHUB_OUTPUT"
+      - name: Move Python packages to assets folder
+        if: contains(github.ref, 'tags/v')
+        run: mv -fv dist/*src.zip assets/.
+      - name: Create Github Release
+        if: contains(github.ref, 'tags/v')
+        id: create_release
+        uses: actions/create-release@v1
+        env: { GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" }
+        with:
+          tag_name: ${{github.ref}}
+          release_name: Release ${{steps.vars.outputs.VERSION}}
+          body_path: ${{steps.vars.outputs.VERSION_BODY}}
+          draft: true
+          prerelease: ${{contains(github.ref, 'rc')}}
+      - name: Upload assets to Github Release
+        if: contains(github.ref, 'tags/v')
+        uses: dwenegar/upload-release-assets@v1
+        env: { GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" }
+        with:
+          release_id: ${{steps.create_release.outputs.id}}
+          assets_path: ./assets/
+      #
+      # PyPI (test)
+      - name: Publish python packages to test PyPI
+        uses: pypa/[email protected]
+        with:
+          repository_url: https://test.pypi.org/legacy/
+          user: __token__
+          password: ${{secrets.PYPI_TOKEN_TEST}}
+          verbose: true
+          skip_existing: true
+      #
+      # PyPI (production)
+      - name: Publish python packages to production PyPI
+        if: contains(github.ref, 'tags/v')
+        uses: pypa/[email protected]
+        with:
+          user: __token__
+          password: ${{secrets.PYPI_TOKEN}}
+          verbose: true