diff options
416 files changed, 11154 insertions, 4737 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 7dcf9388b..6722c1fbe 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -11,7 +11,7 @@ environment: QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21' QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' - VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' + VCPKG_COMMIT_ID: 'f3f329a048eaff759c1992c458f2e12351486bc7' install: # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. # - cmd: pip install zmq diff --git a/.cirrus.yml b/.cirrus.yml index 446d3e35a..1c9e1a69e 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -5,7 +5,7 @@ container: # https://cirrus-ci.org/faq/#are-there-any-limits # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel cpu: 2 - memory: 6G # https://cirrus-ci.org/guide/linux/#linux-containers + memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers env: PACKAGE_MANAGER_INSTALL : "apt-get update && apt-get install -y" MAKEJOBS: "-j4" @@ -56,7 +56,10 @@ task: << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal + cpu: 4 # Double CPU and Memory to avoid timeout + memory: 16G env: + MAKEJOBS: "-j8" FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" task: @@ -66,3 +69,11 @@ task: image: ubuntu:focal env: FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" + +task: + name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, fuzzers under valgrind]' + << : *GLOBAL_TASK_TEMPLATE + container: + image: ubuntu:focal + env: + FILE_ENV: "./ci/test/00_setup_env_native_fuzz_with_valgrind.sh" diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.md b/.github/ISSUE_TEMPLATE/good_first_issue.md index 8be78a1f6..d32e22d36 100644 --- a/.github/ISSUE_TEMPLATE/good_first_issue.md +++ b/.github/ISSUE_TEMPLATE/good_first_issue.md @@ -2,11 +2,13 @@ name: Good first issue about: '(Regular devs only): Suggest a new good first issue' title: '' -labels: good first issue +labels: '' assignees: '' --- +<!-- Needs the label "good first issue" assigned manually before or after opening --> + <!-- A good first issue is an uncontroversial issue, that has a relatively unique and obvious solution --> <!-- Motivate the issue and explain the solution briefly --> diff --git a/.gitignore b/.gitignore index 3e5d284aa..5726b1892 100644 --- a/.gitignore +++ b/.gitignore @@ -119,7 +119,10 @@ releases /*.info test_bitcoin.coverage/ total.coverage/ +fuzz.coverage/ coverage_percent.txt +/cov_tool_wrapper.sh +qa-assets/ #build tests linux-coverage-build diff --git a/.travis.yml b/.travis.yml index f9932cfac..8f8fba310 100644 --- a/.travis.yml +++ b/.travis.yml @@ -80,7 +80,7 @@ jobs: QEMU_USER_CMD="" - stage: test - name: 'Win64 [GOAL: deploy] [unit tests, no gui, no functional tests]' + name: 'Win64 [GOAL: deploy] [unit tests, no gui, no boost::process, no functional tests]' env: >- FILE_ENV="./ci/test/00_setup_env_win64.sh" @@ -110,11 +110,6 @@ jobs: FILE_ENV="./ci/test/00_setup_env_native_multiprocess.sh" - stage: test - name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, fuzzers under valgrind]' - env: >- - FILE_ENV="./ci/test/00_setup_env_native_fuzz_with_valgrind.sh" - - - stage: test name: 'x86_64 Linux [GOAL: install] [xenial] [no wallet]' env: >- FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 11a0f4bac..2e1147438 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -158,7 +158,7 @@ the pull request affects. Valid areas as: Examples: consensus: Add new opcode for BIP-XXXX OP_CHECKAWESOMESIG - net: Automatically create hidden service, listen on Tor + net: Automatically create onion service, listen on Tor qt: Add feed bump button log: Fix typo in log message diff --git a/Makefile.am b/Makefile.am index 75a164f49..c8af4228f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -65,10 +65,10 @@ OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_FANCY_PLIST) $(OSX_INSTALLER_ICONS) \ $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \ $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh -COVERAGE_INFO = baseline.info \ +COVERAGE_INFO = $(COV_TOOL_WRAPPER) baseline.info \ test_bitcoin_filtered.info total_coverage.info \ baseline_filtered.info functional_test.info functional_test_filtered.info \ - test_bitcoin_coverage.info test_bitcoin.info fuzz.info fuzz_coverage.info + test_bitcoin_coverage.info test_bitcoin.info fuzz.info fuzz_filtered.info fuzz_coverage.info dist-hook: -$(GIT) archive --format=tar HEAD -- src/clientversion.cpp | $(AMTAR) -C $(top_distdir) -xf - @@ -181,6 +181,7 @@ $(BITCOIN_WALLET_BIN): FORCE if USE_LCOV LCOV_FILTER_PATTERN = \ + -p "/usr/local/" \ -p "/usr/include/" \ -p "/usr/lib/" \ -p "/usr/lib64/" \ @@ -192,7 +193,13 @@ LCOV_FILTER_PATTERN = \ -p "src/secp256k1" \ -p "depends" -baseline.info: +DIR_FUZZ_SEED_CORPUS ?= qa-assets/fuzz_seed_corpus + +$(COV_TOOL_WRAPPER): + @echo 'exec $(COV_TOOL) "$$@"' > $(COV_TOOL_WRAPPER) + @chmod +x $(COV_TOOL_WRAPPER) + +baseline.info: $(COV_TOOL_WRAPPER) $(LCOV) -c -i -d $(abs_builddir)/src -o $@ baseline_filtered.info: baseline.info @@ -200,7 +207,7 @@ baseline_filtered.info: baseline.info $(LCOV) -a $@ $(LCOV_OPTS) -o $@ fuzz.info: baseline_filtered.info - @TIMEOUT=15 test/fuzz/test_runner.py qa-assets/fuzz_seed_corpus -l DEBUG + @TIMEOUT=15 test/fuzz/test_runner.py $(DIR_FUZZ_SEED_CORPUS) -l DEBUG $(LCOV) -c $(LCOV_OPTS) -d $(abs_builddir)/src --t fuzz-tests -o $@ $(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src diff --git a/build-aux/m4/ax_boost_process.m4 b/build-aux/m4/ax_boost_process.m4 new file mode 100644 index 000000000..5d20e6746 --- /dev/null +++ b/build-aux/m4/ax_boost_process.m4 @@ -0,0 +1,121 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_boost_process.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_BOOST_PROCESS +# +# DESCRIPTION +# +# Test for Process library from the Boost C++ libraries. The macro +# requires a preceding call to AX_BOOST_BASE. Further documentation is +# available at <http://randspringer.de/boost/index.html>. +# +# This macro calls: +# +# AC_SUBST(BOOST_PROCESS_LIB) +# +# And sets: +# +# HAVE_BOOST_PROCESS +# +# LICENSE +# +# Copyright (c) 2008 Thomas Porschberg <[email protected]> +# Copyright (c) 2008 Michael Tindal +# Copyright (c) 2008 Daniel Casimiro <[email protected]> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 2 + +AC_DEFUN([AX_BOOST_PROCESS], +[ + AC_ARG_WITH([boost-process], + AS_HELP_STRING([--with-boost-process@<:@=special-lib@:>@], + [use the Process library from boost - it is possible to specify a certain library for the linker + e.g. --with-boost-process=boost_process-gcc-mt ]), + [ + if test "$withval" = "no"; then + want_boost_process="no" + elif test "$withval" = "yes"; then + want_boost_process="yes" + ax_boost_user_process_lib="" + else + want_boost_process="yes" + ax_boost_user_process_lib="$withval" + fi + ], + [want_boost_process="yes"] + ) + + if test "x$want_boost_process" = "xyes"; then + AC_REQUIRE([AC_PROG_CC]) + AC_REQUIRE([AC_CANONICAL_BUILD]) + CPPFLAGS_SAVED="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + + LDFLAGS_SAVED="$LDFLAGS" + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_CACHE_CHECK(whether the Boost::Process library is available, + ax_cv_boost_process, + [AC_LANG_PUSH([C++]) + CXXFLAGS_SAVE=$CXXFLAGS + CXXFLAGS= + + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <boost/process.hpp>]], + [[boost::process::child* child = new boost::process::child; delete child;]])], + ax_cv_boost_process=yes, ax_cv_boost_process=no) + CXXFLAGS=$CXXFLAGS_SAVE + AC_LANG_POP([C++]) + ]) + if test "x$ax_cv_boost_process" = "xyes"; then + AC_SUBST(BOOST_CPPFLAGS) + + AC_DEFINE(HAVE_BOOST_PROCESS,,[define if the Boost::Process library is available]) + BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` + + LDFLAGS_SAVE=$LDFLAGS + if test "x$ax_boost_user_process_lib" = "x"; then + for libextension in `ls -r $BOOSTLIBDIR/libboost_process* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'` ; do + ax_lib=${libextension} + AC_CHECK_LIB($ax_lib, exit, + [BOOST_PROCESS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROCESS_LIB) link_process="yes"; break], + [link_process="no"]) + done + if test "x$link_process" != "xyes"; then + for libextension in `ls -r $BOOSTLIBDIR/boost_process* 2>/dev/null | sed 's,.*/,,' | sed -e 's,\..*,,'` ; do + ax_lib=${libextension} + AC_CHECK_LIB($ax_lib, exit, + [BOOST_PROCESS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROCESS_LIB) link_process="yes"; break], + [link_process="no"]) + done + fi + + else + for ax_lib in $ax_boost_user_process_lib boost_process-$ax_boost_user_process_lib; do + AC_CHECK_LIB($ax_lib, exit, + [BOOST_PROCESS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROCESS_LIB) link_process="yes"; break], + [link_process="no"]) + done + + fi + if test "x$ax_lib" = "x"; then + AC_MSG_ERROR(Could not find a version of the Boost::Process library!) + fi + if test "x$link_process" = "xno"; then + AC_MSG_ERROR(Could not link against $ax_lib !) + fi + fi + + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" + fi +]) diff --git a/build-aux/m4/ax_boost_thread.m4 b/build-aux/m4/ax_boost_thread.m4 index e9dea4353..75e80e6e7 100644 --- a/build-aux/m4/ax_boost_thread.m4 +++ b/build-aux/m4/ax_boost_thread.m4 @@ -30,7 +30,7 @@ # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 32 +#serial 33 AC_DEFUN([AX_BOOST_THREAD], [ @@ -67,13 +67,24 @@ AC_DEFUN([AX_BOOST_THREAD], [AC_LANG_PUSH([C++]) CXXFLAGS_SAVE=$CXXFLAGS - if test "x$host_os" = "xsolaris" ; then - CXXFLAGS="-pthreads $CXXFLAGS" - elif test "x$host_os" = "xmingw32" ; then - CXXFLAGS="-mthreads $CXXFLAGS" - else - CXXFLAGS="-pthread $CXXFLAGS" - fi + case "x$host_os" in + xsolaris ) + CXXFLAGS="-pthreads $CXXFLAGS" + break; + ;; + xmingw32 ) + CXXFLAGS="-mthreads $CXXFLAGS" + break; + ;; + *android* ) + break; + ;; + * ) + CXXFLAGS="-pthread $CXXFLAGS" + break; + ;; + esac + AC_COMPILE_IFELSE([ AC_LANG_PROGRAM( [[@%:@include <boost/thread/thread.hpp>]], @@ -84,13 +95,23 @@ AC_DEFUN([AX_BOOST_THREAD], AC_LANG_POP([C++]) ]) if test "x$ax_cv_boost_thread" = "xyes"; then - if test "x$host_os" = "xsolaris" ; then - BOOST_CPPFLAGS="-pthreads $BOOST_CPPFLAGS" - elif test "x$host_os" = "xmingw32" ; then - BOOST_CPPFLAGS="-mthreads $BOOST_CPPFLAGS" - else - BOOST_CPPFLAGS="-pthread $BOOST_CPPFLAGS" - fi + case "x$host_os" in + xsolaris ) + BOOST_CPPFLAGS="-pthreads $BOOST_CPPFLAGS" + break; + ;; + xmingw32 ) + BOOST_CPPFLAGS="-mthreads $BOOST_CPPFLAGS" + break; + ;; + *android* ) + break; + ;; + * ) + BOOST_CPPFLAGS="-pthread $BOOST_CPPFLAGS" + break; + ;; + esac AC_SUBST(BOOST_CPPFLAGS) @@ -148,6 +169,9 @@ AC_DEFUN([AX_BOOST_THREAD], xmingw32 ) break; ;; + *android* ) + break; + ;; * ) BOOST_THREAD_LIB="$BOOST_THREAD_LIB -lpthread" break; diff --git a/build-aux/m4/ax_pthread.m4 b/build-aux/m4/ax_pthread.m4 index 4c4051ea3..1598d077f 100644 --- a/build-aux/m4/ax_pthread.m4 +++ b/build-aux/m4/ax_pthread.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_pthread.html +# https://www.gnu.org/software/autoconf-archive/ax_pthread.html # =========================================================================== # # SYNOPSIS @@ -55,6 +55,7 @@ # # Copyright (c) 2008 Steven G. Johnson <[email protected]> # Copyright (c) 2011 Daniel Richard G. <[email protected]> +# Copyright (c) 2019 Marc Stevens <[email protected]> # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the @@ -67,7 +68,7 @@ # Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. +# with this program. If not, see <https://www.gnu.org/licenses/>. # # As a special exception, the respective Autoconf Macro's copyright owner # gives unlimited permission to copy, distribute and modify the configure @@ -82,7 +83,7 @@ # modified version of the Autoconf Macro, you may extend this special # exception to the GPL to apply to your modified version as well. -#serial 23 +#serial 27 AU_ALIAS([ACX_PTHREAD], [AX_PTHREAD]) AC_DEFUN([AX_PTHREAD], [ @@ -123,10 +124,12 @@ fi # (e.g. DEC) have both -lpthread and -lpthreads, where one of the # libraries is broken (non-POSIX). -# Create a list of thread flags to try. Items starting with a "-" are -# C compiler flags, and other items are library names, except for "none" -# which indicates that we try without any flags at all, and "pthread-config" -# which is a program returning the flags for the Pth emulation library. +# Create a list of thread flags to try. Items with a "," contain both +# C compiler flags (before ",") and linker flags (after ","). Other items +# starting with a "-" are C compiler flags, and remaining items are +# library names, except for "none" which indicates that we try without +# any flags at all, and "pthread-config" which is a program returning +# the flags for the Pth emulation library. ax_pthread_flags="pthreads none -Kthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config" @@ -194,14 +197,47 @@ case $host_os in # that too in a future libc.) So we'll check first for the # standard Solaris way of linking pthreads (-mt -lpthread). - ax_pthread_flags="-mt,pthread pthread $ax_pthread_flags" + ax_pthread_flags="-mt,-lpthread pthread $ax_pthread_flags" ;; esac +# Are we compiling with Clang? + +AC_CACHE_CHECK([whether $CC is Clang], + [ax_cv_PTHREAD_CLANG], + [ax_cv_PTHREAD_CLANG=no + # Note that Autoconf sets GCC=yes for Clang as well as GCC + if test "x$GCC" = "xyes"; then + AC_EGREP_CPP([AX_PTHREAD_CC_IS_CLANG], + [/* Note: Clang 2.7 lacks __clang_[a-z]+__ */ +# if defined(__clang__) && defined(__llvm__) + AX_PTHREAD_CC_IS_CLANG +# endif + ], + [ax_cv_PTHREAD_CLANG=yes]) + fi + ]) +ax_pthread_clang="$ax_cv_PTHREAD_CLANG" + + # GCC generally uses -pthread, or -pthreads on some platforms (e.g. SPARC) +# Note that for GCC and Clang -pthread generally implies -lpthread, +# except when -nostdlib is passed. +# This is problematic using libtool to build C++ shared libraries with pthread: +# [1] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=25460 +# [2] https://bugzilla.redhat.com/show_bug.cgi?id=661333 +# [3] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=468555 +# To solve this, first try -pthread together with -lpthread for GCC + AS_IF([test "x$GCC" = "xyes"], - [ax_pthread_flags="-pthread -pthreads $ax_pthread_flags"]) + [ax_pthread_flags="-pthread,-lpthread -pthread -pthreads $ax_pthread_flags"]) + +# Clang takes -pthread (never supported any other flag), but we'll try with -lpthread first + +AS_IF([test "x$ax_pthread_clang" = "xyes"], + [ax_pthread_flags="-pthread,-lpthread -pthread"]) + # The presence of a feature test macro requesting re-entrant function # definitions is, on some systems, a strong hint that pthreads support is @@ -224,25 +260,86 @@ AS_IF([test "x$ax_pthread_check_macro" = "x--"], [ax_pthread_check_cond=0], [ax_pthread_check_cond="!defined($ax_pthread_check_macro)"]) -# Are we compiling with Clang? -AC_CACHE_CHECK([whether $CC is Clang], - [ax_cv_PTHREAD_CLANG], - [ax_cv_PTHREAD_CLANG=no - # Note that Autoconf sets GCC=yes for Clang as well as GCC - if test "x$GCC" = "xyes"; then - AC_EGREP_CPP([AX_PTHREAD_CC_IS_CLANG], - [/* Note: Clang 2.7 lacks __clang_[a-z]+__ */ -# if defined(__clang__) && defined(__llvm__) - AX_PTHREAD_CC_IS_CLANG -# endif - ], - [ax_cv_PTHREAD_CLANG=yes]) - fi - ]) -ax_pthread_clang="$ax_cv_PTHREAD_CLANG" +if test "x$ax_pthread_ok" = "xno"; then +for ax_pthread_try_flag in $ax_pthread_flags; do + + case $ax_pthread_try_flag in + none) + AC_MSG_CHECKING([whether pthreads work without any flags]) + ;; + + *,*) + PTHREAD_CFLAGS=`echo $ax_pthread_try_flag | sed "s/^\(.*\),\(.*\)$/\1/"` + PTHREAD_LIBS=`echo $ax_pthread_try_flag | sed "s/^\(.*\),\(.*\)$/\2/"` + AC_MSG_CHECKING([whether pthreads work with "$PTHREAD_CFLAGS" and "$PTHREAD_LIBS"]) + ;; + + -*) + AC_MSG_CHECKING([whether pthreads work with $ax_pthread_try_flag]) + PTHREAD_CFLAGS="$ax_pthread_try_flag" + ;; + + pthread-config) + AC_CHECK_PROG([ax_pthread_config], [pthread-config], [yes], [no]) + AS_IF([test "x$ax_pthread_config" = "xno"], [continue]) + PTHREAD_CFLAGS="`pthread-config --cflags`" + PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`" + ;; + + *) + AC_MSG_CHECKING([for the pthreads library -l$ax_pthread_try_flag]) + PTHREAD_LIBS="-l$ax_pthread_try_flag" + ;; + esac + + ax_pthread_save_CFLAGS="$CFLAGS" + ax_pthread_save_LIBS="$LIBS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + LIBS="$PTHREAD_LIBS $LIBS" + + # Check for various functions. We must include pthread.h, + # since some functions may be macros. (On the Sequent, we + # need a special flag -Kthread to make this header compile.) + # We check for pthread_join because it is in -lpthread on IRIX + # while pthread_create is in libc. We check for pthread_attr_init + # due to DEC craziness with -lpthreads. We check for + # pthread_cleanup_push because it is one of the few pthread + # functions on Solaris that doesn't have a non-functional libc stub. + # We try pthread_create on general principles. + + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <pthread.h> +# if $ax_pthread_check_cond +# error "$ax_pthread_check_macro must be defined" +# endif + static void *some_global = NULL; + static void routine(void *a) + { + /* To avoid any unused-parameter or + unused-but-set-parameter warning. */ + some_global = a; + } + static void *start_routine(void *a) { return a; }], + [pthread_t th; pthread_attr_t attr; + pthread_create(&th, 0, start_routine, 0); + pthread_join(th, 0); + pthread_attr_init(&attr); + pthread_cleanup_push(routine, 0); + pthread_cleanup_pop(0) /* ; */])], + [ax_pthread_ok=yes], + []) + + CFLAGS="$ax_pthread_save_CFLAGS" + LIBS="$ax_pthread_save_LIBS" + + AC_MSG_RESULT([$ax_pthread_ok]) + AS_IF([test "x$ax_pthread_ok" = "xyes"], [break]) + + PTHREAD_LIBS="" + PTHREAD_CFLAGS="" +done +fi -ax_pthread_clang_warning=no # Clang needs special handling, because older versions handle the -pthread # option in a rather... idiosyncratic way @@ -261,11 +358,6 @@ if test "x$ax_pthread_clang" = "xyes"; then # -pthread does define _REENTRANT, and while the Darwin headers # ignore this macro, third-party headers might not.) - PTHREAD_CFLAGS="-pthread" - PTHREAD_LIBS= - - ax_pthread_ok=yes - # However, older versions of Clang make a point of warning the user # that, in an invocation where only linking and no compilation is # taking place, the -pthread option has no effect ("argument unused @@ -320,78 +412,7 @@ if test "x$ax_pthread_clang" = "xyes"; then fi # $ax_pthread_clang = yes -if test "x$ax_pthread_ok" = "xno"; then -for ax_pthread_try_flag in $ax_pthread_flags; do - - case $ax_pthread_try_flag in - none) - AC_MSG_CHECKING([whether pthreads work without any flags]) - ;; - - -mt,pthread) - AC_MSG_CHECKING([whether pthreads work with -mt -lpthread]) - PTHREAD_CFLAGS="-mt" - PTHREAD_LIBS="-lpthread" - ;; - - -*) - AC_MSG_CHECKING([whether pthreads work with $ax_pthread_try_flag]) - PTHREAD_CFLAGS="$ax_pthread_try_flag" - ;; - - pthread-config) - AC_CHECK_PROG([ax_pthread_config], [pthread-config], [yes], [no]) - AS_IF([test "x$ax_pthread_config" = "xno"], [continue]) - PTHREAD_CFLAGS="`pthread-config --cflags`" - PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`" - ;; - *) - AC_MSG_CHECKING([for the pthreads library -l$ax_pthread_try_flag]) - PTHREAD_LIBS="-l$ax_pthread_try_flag" - ;; - esac - - ax_pthread_save_CFLAGS="$CFLAGS" - ax_pthread_save_LIBS="$LIBS" - CFLAGS="$CFLAGS $PTHREAD_CFLAGS" - LIBS="$PTHREAD_LIBS $LIBS" - - # Check for various functions. We must include pthread.h, - # since some functions may be macros. (On the Sequent, we - # need a special flag -Kthread to make this header compile.) - # We check for pthread_join because it is in -lpthread on IRIX - # while pthread_create is in libc. We check for pthread_attr_init - # due to DEC craziness with -lpthreads. We check for - # pthread_cleanup_push because it is one of the few pthread - # functions on Solaris that doesn't have a non-functional libc stub. - # We try pthread_create on general principles. - - AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <pthread.h> -# if $ax_pthread_check_cond -# error "$ax_pthread_check_macro must be defined" -# endif - static void routine(void *a) { a = 0; } - static void *start_routine(void *a) { return a; }], - [pthread_t th; pthread_attr_t attr; - pthread_create(&th, 0, start_routine, 0); - pthread_join(th, 0); - pthread_attr_init(&attr); - pthread_cleanup_push(routine, 0); - pthread_cleanup_pop(0) /* ; */])], - [ax_pthread_ok=yes], - []) - - CFLAGS="$ax_pthread_save_CFLAGS" - LIBS="$ax_pthread_save_LIBS" - - AC_MSG_RESULT([$ax_pthread_ok]) - AS_IF([test "x$ax_pthread_ok" = "xyes"], [break]) - - PTHREAD_LIBS="" - PTHREAD_CFLAGS="" -done -fi # Various other checks: if test "x$ax_pthread_ok" = "xyes"; then @@ -438,7 +459,8 @@ if test "x$ax_pthread_ok" = "xyes"; then AC_CACHE_CHECK([for PTHREAD_PRIO_INHERIT], [ax_cv_PTHREAD_PRIO_INHERIT], [AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <pthread.h>]], - [[int i = PTHREAD_PRIO_INHERIT;]])], + [[int i = PTHREAD_PRIO_INHERIT; + return i;]])], [ax_cv_PTHREAD_PRIO_INHERIT=yes], [ax_cv_PTHREAD_PRIO_INHERIT=no]) ]) diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index e171238cb..6c7665830 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -72,18 +72,32 @@ AC_DEFUN([BITCOIN_QT_INIT],[ AC_ARG_WITH([qtdbus], [AS_HELP_STRING([--with-qtdbus], - [enable DBus support (default is yes if qt is enabled and QtDBus is found)])], + [enable DBus support (default is yes if qt is enabled and QtDBus is found, except on Android)])], [use_dbus=$withval], [use_dbus=auto]) + dnl Android doesn't support D-Bus and certainly doesn't use it for notifications + case $host in + *android*) + if test "x$use_dbus" != xyes; then + use_dbus=no + fi + ;; + esac + AC_SUBST(QT_TRANSLATION_DIR,$qt_translation_path) ]) dnl Find Qt libraries and includes. +dnl +dnl BITCOIN_QT_CONFIGURE([MINIMUM-VERSION]) +dnl dnl Outputs: See _BITCOIN_QT_FIND_LIBS dnl Outputs: Sets variables for all qt-related tools. dnl Outputs: bitcoin_enable_qt, bitcoin_enable_qt_dbus, bitcoin_enable_qt_test AC_DEFUN([BITCOIN_QT_CONFIGURE],[ + qt_version=">= $1" + qt_lib_prefix="Qt5" BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS]) dnl This is ugly and complicated. Yuck. Works as follows: @@ -221,7 +235,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ bitcoin_enable_qt=no ]) if test x$bitcoin_enable_qt = xyes; then - AC_MSG_RESULT([$bitcoin_enable_qt ($QT_LIB_PREFIX)]) + AC_MSG_RESULT([$bitcoin_enable_qt ($qt_lib_prefix)]) else AC_MSG_RESULT([$bitcoin_enable_qt]) fi @@ -295,25 +309,19 @@ AC_DEFUN([_BITCOIN_QT_FIND_STATIC_PLUGINS],[ if test -d "$qt_plugin_path/platforms/android"; then QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms/android -lqtfreetype -lEGL" fi - m4_ifdef([PKG_CHECK_MODULES],[ - if test x$bitcoin_cv_qt58 = xno; then - PKG_CHECK_MODULES([QTPLATFORM], [Qt5PlatformSupport], [QT_LIBS="$QTPLATFORM_LIBS $QT_LIBS"]) - else - PKG_CHECK_MODULES([QTFONTDATABASE], [Qt5FontDatabaseSupport], [QT_LIBS="-lQt5FontDatabaseSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTEVENTDISPATCHER], [Qt5EventDispatcherSupport], [QT_LIBS="-lQt5EventDispatcherSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTTHEME], [Qt5ThemeSupport], [QT_LIBS="-lQt5ThemeSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTDEVICEDISCOVERY], [Qt5DeviceDiscoverySupport], [QT_LIBS="-lQt5DeviceDiscoverySupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTACCESSIBILITY], [Qt5AccessibilitySupport], [QT_LIBS="-lQt5AccessibilitySupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTFB], [Qt5FbSupport], [QT_LIBS="-lQt5FbSupport $QT_LIBS"]) - fi - if test "x$TARGET_OS" = xlinux; then - PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) - elif test "x$TARGET_OS" = xdarwin; then - PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport], [QT_LIBS="-lQt5ClipboardSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTGRAPHICS], [Qt5GraphicsSupport], [QT_LIBS="-lQt5GraphicsSupport $QT_LIBS"]) - PKG_CHECK_MODULES([QTCGL], [Qt5CglSupport], [QT_LIBS="-lQt5CglSupport $QT_LIBS"]) - fi - ]) + PKG_CHECK_MODULES([QTFONTDATABASE], [Qt5FontDatabaseSupport], [QT_LIBS="-lQt5FontDatabaseSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTEVENTDISPATCHER], [Qt5EventDispatcherSupport], [QT_LIBS="-lQt5EventDispatcherSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTTHEME], [Qt5ThemeSupport], [QT_LIBS="-lQt5ThemeSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTDEVICEDISCOVERY], [Qt5DeviceDiscoverySupport], [QT_LIBS="-lQt5DeviceDiscoverySupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTACCESSIBILITY], [Qt5AccessibilitySupport], [QT_LIBS="-lQt5AccessibilitySupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTFB], [Qt5FbSupport], [QT_LIBS="-lQt5FbSupport $QT_LIBS"]) + if test "x$TARGET_OS" = xlinux; then + PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) + elif test "x$TARGET_OS" = xdarwin; then + PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport], [QT_LIBS="-lQt5ClipboardSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTGRAPHICS], [Qt5GraphicsSupport], [QT_LIBS="-lQt5GraphicsSupport $QT_LIBS"]) + PKG_CHECK_MODULES([QTCGL], [Qt5CglSupport], [QT_LIBS="-lQt5CglSupport $QT_LIBS"]) + fi fi ]) @@ -321,23 +329,29 @@ dnl Internal. Find Qt libraries using pkg-config. dnl Outputs: All necessary QT_* variables are set. dnl Outputs: have_qt_test and have_qt_dbus are set (if applicable) to yes|no. AC_DEFUN([_BITCOIN_QT_FIND_LIBS],[ - m4_ifdef([PKG_CHECK_MODULES],[ - QT_LIB_PREFIX=Qt5 - qt5_modules="Qt5Core Qt5Gui Qt5Network Qt5Widgets" - BITCOIN_QT_CHECK([ - PKG_CHECK_MODULES([QT5], [$qt5_modules], [QT_INCLUDES="$QT5_CFLAGS"; QT_LIBS="$QT5_LIBS" have_qt=yes],[have_qt=no]) + BITCOIN_QT_CHECK([ + PKG_CHECK_MODULES([QT_CORE], [${qt_lib_prefix}Core $qt_version], [], + [BITCOIN_QT_FAIL([${qt_lib_prefix}Core $qt_version not found])]) + ]) + BITCOIN_QT_CHECK([ + PKG_CHECK_MODULES([QT_GUI], [${qt_lib_prefix}Gui $qt_version], [], + [BITCOIN_QT_FAIL([${qt_lib_prefix}Gui $qt_version not found])]) + ]) + BITCOIN_QT_CHECK([ + PKG_CHECK_MODULES([QT_WIDGETS], [${qt_lib_prefix}Widgets $qt_version], [], + [BITCOIN_QT_FAIL([${qt_lib_prefix}Widgets $qt_version not found])]) + ]) + BITCOIN_QT_CHECK([ + PKG_CHECK_MODULES([QT_NETWORK], [${qt_lib_prefix}Network $qt_version], [], + [BITCOIN_QT_FAIL([${qt_lib_prefix}Network $qt_version not found])]) + ]) + QT_INCLUDES="$QT_CORE_CFLAGS $QT_GUI_CFLAGS $QT_WIDGETS_CFLAGS $QT_NETWORK_CFLAGS" + QT_LIBS="$QT_CORE_LIBS $QT_GUI_LIBS $QT_WIDGETS_LIBS $QT_NETWORK_LIBS" - if test "x$have_qt" != xyes; then - have_qt=no - BITCOIN_QT_FAIL([Qt dependencies not found]) - fi - ]) - BITCOIN_QT_CHECK([ - PKG_CHECK_MODULES([QT_TEST], [${QT_LIB_PREFIX}Test], [QT_TEST_INCLUDES="$QT_TEST_CFLAGS"; have_qt_test=yes], [have_qt_test=no]) - if test "x$use_dbus" != xno; then - PKG_CHECK_MODULES([QT_DBUS], [${QT_LIB_PREFIX}DBus], [QT_DBUS_INCLUDES="$QT_DBUS_CFLAGS"; have_qt_dbus=yes], [have_qt_dbus=no]) - fi - ]) + BITCOIN_QT_CHECK([ + PKG_CHECK_MODULES([QT_TEST], [${qt_lib_prefix}Test $qt_version], [QT_TEST_INCLUDES="$QT_TEST_CFLAGS"; have_qt_test=yes], [have_qt_test=no]) + if test "x$use_dbus" != xno; then + PKG_CHECK_MODULES([QT_DBUS], [${qt_lib_prefix}DBus $qt_version], [QT_DBUS_INCLUDES="$QT_DBUS_CFLAGS"; have_qt_dbus=yes], [have_qt_dbus=no]) + fi ]) - true; dnl ]) diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index fbbe1a215..9d0b50a0b 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -47,6 +47,9 @@ /* define if the Boost::Filesystem library is available */ #define HAVE_BOOST_FILESYSTEM /**/ +/* define if the Boost::Process library is available */ +#define HAVE_BOOST_PROCESS /**/ + /* define if the Boost::System library is available */ #define HAVE_BOOST_SYSTEM /**/ diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index 4fd516fff..a080fd2aa 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -110,7 +110,7 @@ <AdditionalOptions>/utf-8 /std:c++17 %(AdditionalOptions)</AdditionalOptions> <DisableSpecificWarnings>4018;4221;4244;4267;4334;4715;4805;4834</DisableSpecificWarnings> <TreatWarningAsError>true</TreatWarningAsError> - <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions> <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> diff --git a/build_msvc/vcpkg-packages.txt b/build_msvc/vcpkg-packages.txt index 307f295f0..edce8576c 100644 --- a/build_msvc/vcpkg-packages.txt +++ b/build_msvc/vcpkg-packages.txt @@ -1 +1 @@ -berkeleydb boost-filesystem boost-multi-index boost-signals2 boost-test boost-thread libevent[thread] zeromq double-conversion
\ No newline at end of file +berkeleydb boost-filesystem boost-multi-index boost-process boost-signals2 boost-test boost-thread libevent[thread] zeromq double-conversion diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 2413cfca9..702e88186 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -35,6 +35,10 @@ export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} +# By how much to scale the test_runner timeouts (option --timeout-factor). +# This is needed because some ci machines have slow CPU or disk, so sanitizers +# might be slow or a reindex might be waiting on disk IO. +export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-4} export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export EXPECTED_TESTS_DURATION_IN_SECONDS=${EXPECTED_TESTS_DURATION_IN_SECONDS:-1000} diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index b70a58153..2e445c126 100644 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -25,4 +25,4 @@ export RUN_FUNCTIONAL_TESTS=true export GOAL="install" # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi --enable-werror" +export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi --enable-werror --with-boost-process" diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 5688799f9..e58003ab1 100644 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -11,5 +11,5 @@ export CONTAINER_NAME=ci_i686_centos_7 export DOCKER_NAME_TAG=centos:7 export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python36-zmq which patch lbzip2 dash" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports" +export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports --with-boost-process" export CONFIG_SHELL="/bin/dash" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index 7ec66eeb4..b62f1603f 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -14,4 +14,4 @@ export XCODE_BUILD_ID=11C505 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-werror" +export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-werror --with-boost-process" diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index b8a9ccaae..5fb127b76 100644 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -10,7 +10,7 @@ export HOST=x86_64-apple-darwin16 export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well) export PIP_PACKAGES="zmq" export GOAL="install" -export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-werror" +export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-werror --with-boost-process" export NO_DEPENDS=1 export OSX_SDK="" export CCACHE_SIZE=300M diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index d57c67306..251ece798 100644 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -10,6 +10,5 @@ export CONTAINER_NAME=ci_native_asan export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" export DOCKER_NAME_TAG=ubuntu:20.04 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--timeout-factor=4" # Increase timeout because sanitizers slow down export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++ --with-boost-process" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 31f437f0e..a32de4a6b 100644 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -14,5 +14,5 @@ export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++ --with-boost-process" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh index 786f0f927..522a5d9fc 100644 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -11,5 +11,5 @@ export DOCKER_NAME_TAG=ubuntu:20.04 export PACKAGES="cmake python3" export DEP_OPTS="MULTIPROCESS=1" export GOAL="install" -export BITCOIN_CONFIG="" +export BITCOIN_CONFIG="--with-boost-process" export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index 1a0b14b62..0a09bfe23 100644 --- a/ci/test/00_setup_env_native_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -11,4 +11,4 @@ export DOCKER_NAME_TAG=ubuntu:16.04 # Use xenial to have one config run the tes export PACKAGES="python3-zmq clang-3.8 llvm-3.8" # Use clang-3.8 to test C++11 compatibility, see doc/dependencies.md export DEP_OPTS="NO_WALLET=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CC=clang-3.8 CXX=clang++-3.8" +export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CC=clang-3.8 CXX=clang++-3.8 --with-boost-process" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 6e2ff729a..6f2e39429 100644 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -15,5 +15,5 @@ export RUN_SECURITY_TESTS="true" export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" -export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.1 v0.18.1 v0.19.1" -export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-c++17 --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" +export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1" +export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-c++17 --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" --with-boost-process" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 5695c43ec..b14a46562 100644 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -10,6 +10,6 @@ export CONTAINER_NAME=ci_native_tsan export DOCKER_NAME_TAG=ubuntu:20.04 export PACKAGES="clang llvm libc++abi-dev libc++-dev python3-zmq" export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'" -export TEST_RUNNER_EXTRA="--exclude feature_block --timeout-factor=4" # Increase timeout because sanitizers slow down. Low memory on Travis machines, exclude feature_block. +export TEST_RUNNER_EXTRA="--exclude feature_block" # Low memory on Travis machines, exclude feature_block. export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-gui=no CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++'" +export BITCOIN_CONFIG="--enable-zmq --with-gui=no CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++' --with-boost-process" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 0041122f1..710d9e101 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -10,6 +10,6 @@ export CONTAINER_NAME=ci_native_valgrind export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--exclude rpc_bind --timeout-factor=4" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index c180d023d..fe330920d 100644 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -22,4 +22,4 @@ export DOCKER_NAME_TAG="debian:buster" export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-reduce-exports --with-incompatible-bdb" +export BITCOIN_CONFIG="--enable-reduce-exports --with-incompatible-bdb --with-boost-process" diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index eb8b870dd..2b351dff6 100644 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -13,4 +13,4 @@ export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64" export RUN_FUNCTIONAL_TESTS=false export RUN_SECURITY_TESTS="true" export GOAL="deploy" -export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests" +export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests --without-boost-process" diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 131ea2167..8ce839fc0 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -48,6 +48,6 @@ if [ -z "$NO_DEPENDS" ]; then fi if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then BEGIN_FOLD previous-versions - DOCKER_EXEC contrib/devtools/previous_release.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" + DOCKER_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" END_FOLD fi diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 96d44328b..6c14a3dfb 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -35,7 +35,7 @@ fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then BEGIN_FOLD functional-tests - DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast + DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --quiet --failfast END_FOLD fi diff --git a/configure.ac b/configure.ac index f11f2b205..fbf56443f 100644 --- a/configure.ac +++ b/configure.ac @@ -105,6 +105,7 @@ AC_PATH_TOOL(AR, ar) AC_PATH_TOOL(RANLIB, ranlib) AC_PATH_TOOL(STRIP, strip) AC_PATH_TOOL(GCOV, gcov) +AC_PATH_TOOL(LLVM_COV, llvm-cov) AC_PATH_PROG(LCOV, lcov) dnl Python 3.5 is specified in .python-version and should be used if available, see doc/dependencies.md AC_PATH_PROGS([PYTHON], [python3.5 python3.6 python3.7 python3.8 python3 python]) @@ -376,6 +377,7 @@ if test "x$enable_werror" = "xyes"; then AX_CHECK_COMPILE_FLAG([-Werror=shadow-field],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=shadow-field"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=switch],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=switch"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=thread-safety],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=range-loop-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=range-loop-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=unused-variable],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unused-variable"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=date-time],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=date-time"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=return-type],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=return-type"],,[[$CXXFLAG_WERROR]]) @@ -404,6 +406,10 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wdate-time],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wsign-compare],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsign-compare"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wduplicated-branches],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-branches"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wduplicated-cond],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-cond"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wlogical-op],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wlogical-op"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Woverloaded-virtual],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Woverloaded-virtual"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wsuggest-override],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"],,[[$CXXFLAG_WERROR]], [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) AX_CHECK_COMPILE_FLAG([-Wunreachable-code-loop-increment],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) @@ -598,7 +604,7 @@ case $host in AC_MSG_ERROR("windres not found") fi - CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -DBOOST_THREAD_USE_LIB -D_WIN32_WINNT=0x0601" + CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -DBOOST_THREAD_USE_LIB -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" dnl libtool insists upon adding -nostdlib and a list of objects/libs to link against. dnl That breaks our ability to build dll's with static libgcc/libstdc++/libssp. Override @@ -680,16 +686,37 @@ if test x$use_lcov = xyes; then if test x$LCOV = x; then AC_MSG_ERROR("lcov testing requested but lcov not found") fi - if test x$GCOV = x; then - AC_MSG_ERROR("lcov testing requested but gcov not found") - fi if test x$PYTHON = x; then AC_MSG_ERROR("lcov testing requested but python not found") fi if test x$GENHTML = x; then AC_MSG_ERROR("lcov testing requested but genhtml not found") fi - LCOV="$LCOV --gcov-tool=$GCOV" + + AC_MSG_CHECKING([whether compiler is Clang]) + AC_PREPROC_IFELSE([AC_LANG_SOURCE([[ + #if defined(__clang__) && defined(__llvm__) + // Compiler is Clang + #else + # error Compiler is not Clang + #endif + ]])],[ + AC_MSG_RESULT([yes]) + if test x$LLVM_COV = x; then + AC_MSG_ERROR([lcov testing requested but llvm-cov not found]) + fi + COV_TOOL="$LLVM_COV gcov" + ],[ + AC_MSG_RESULT([no]) + if test x$GCOV = x; then + AC_MSG_ERROR([lcov testing requested but gcov not found]) + fi + COV_TOOL="$GCOV" + ]) + AC_SUBST(COV_TOOL) + AC_SUBST(COV_TOOL_WRAPPER, "cov_tool_wrapper.sh") + LCOV="$LCOV --gcov-tool $(pwd)/$COV_TOOL_WRAPPER" + AX_CHECK_LINK_FLAG([[--coverage]], [LDFLAGS="$LDFLAGS --coverage"], [AC_MSG_ERROR("lcov testing requested but --coverage linker flag does not work")]) AX_CHECK_COMPILE_FLAG([--coverage],[CXXFLAGS="$CXXFLAGS --coverage"], @@ -768,6 +795,14 @@ if test x$use_hardening != xno; then AX_CHECK_COMPILE_FLAG([-Wstack-protector],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) AX_CHECK_COMPILE_FLAG([-fstack-protector-all],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) + AX_CHECK_COMPILE_FLAG([-fcf-protection=full],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"]) + + dnl stack-clash-protection does not work properly when building for Windows. + dnl We use the test case from https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90458 + dnl to determine if it can be enabled. + AX_CHECK_COMPILE_FLAG([-fstack-clash-protection],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-clash-protection"],[],["-O0"], + [AC_LANG_SOURCE([[class D {public: unsigned char buf[32768];}; int main() {D d; return 0;}]])]) + dnl When enable_debug is yes, all optimizations are disabled. dnl However, FORTIFY_SOURCE requires that there is some level of optimization, otherwise it does nothing and just creates a compiler warning. dnl Since FORTIFY_SOURCE is a no-op without optimizations, do not enable it when enable_debug is yes. @@ -1012,13 +1047,13 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <stdint.h> [ AC_MSG_RESULT(no)] ) -dnl LevelDB platform checks AC_MSG_CHECKING(for fdatasync) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h>]], [[ fdatasync(0); ]])], [ AC_MSG_RESULT(yes); HAVE_FDATASYNC=1 ], [ AC_MSG_RESULT(no); HAVE_FDATASYNC=0 ] ) +AC_DEFINE_UNQUOTED([HAVE_FDATASYNC], [$HAVE_FDATASYNC], [Define to 1 if fdatasync is available.]) AC_MSG_CHECKING(for F_FULLFSYNC) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <fcntl.h>]], @@ -1134,7 +1169,7 @@ else BITCOIN_QT_INIT dnl sets $bitcoin_enable_qt, $bitcoin_enable_qt_test, $bitcoin_enable_qt_dbus - BITCOIN_QT_CONFIGURE + BITCOIN_QT_CONFIGURE([5.5.1]) fi if test x$enable_wallet != xno; then @@ -1180,9 +1215,9 @@ fi if test x$use_boost = xyes; then dnl Minimum required Boost version -define(MINIMUM_REQUIRED_BOOST, 1.47.0) +define(MINIMUM_REQUIRED_BOOST, 1.58.0) -dnl Check for boost libs +dnl Check for Boost libs AX_BOOST_BASE([MINIMUM_REQUIRED_BOOST]) if test x$want_boost = xno; then AC_MSG_ERROR([[only libbitcoinconsensus can be built without boost]]) @@ -1191,30 +1226,15 @@ AX_BOOST_SYSTEM AX_BOOST_FILESYSTEM AX_BOOST_THREAD +dnl Opt-in to boost-process +AS_IF([ test x$with_boost_process != x ], [ AX_BOOST_PROCESS ], [ ax_cv_boost_process=no ] ) + dnl Boost 1.56 through 1.62 allow using std::atomic instead of its own atomic dnl counter implementations. In 1.63 and later the std::atomic approach is default. m4_pattern_allow(DBOOST_AC_USE_STD_ATOMIC) dnl otherwise it's treated like a macro BOOST_CPPFLAGS="-DBOOST_SP_USE_STD_ATOMIC -DBOOST_AC_USE_STD_ATOMIC $BOOST_CPPFLAGS" -if test x$use_reduce_exports = xyes; then - AC_MSG_CHECKING([for working boost reduced exports]) - TEMP_CPPFLAGS="$CPPFLAGS" - CPPFLAGS="$BOOST_CPPFLAGS $CPPFLAGS" - AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ - @%:@include <boost/version.hpp> - ]], [[ - #if BOOST_VERSION >= 104900 - // Everything is okay - #else - # error Boost version is too old - #endif - ]])],[ - AC_MSG_RESULT(yes) - ],[ - AC_MSG_ERROR([boost versions < 1.49 are known to be broken with reduced exports. Use --disable-reduce-exports.]) - ]) - CPPFLAGS="$TEMP_CPPFLAGS" -fi +BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_THREAD_LIB" fi if test x$use_reduce_exports = xyes; then @@ -1228,7 +1248,6 @@ if test x$use_tests = xyes; then AC_MSG_ERROR(hexdump is required for tests) fi - if test x$use_boost = xyes; then AX_BOOST_UNIT_TEST_FRAMEWORK @@ -1254,48 +1273,6 @@ if test x$use_tests = xyes; then fi fi -if test x$use_boost = xyes; then - -BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_THREAD_LIB" - - -dnl If boost (prior to 1.57) was built without c++11, it emulated scoped enums -dnl using c++98 constructs. Unfortunately, this implementation detail leaked into -dnl the abi. This was fixed in 1.57. - -dnl When building against that installed version using c++11, the headers pick up -dnl on the native c++11 scoped enum support and enable it, however it will fail to -dnl link. This can be worked around by disabling c++11 scoped enums if linking will -dnl fail. -dnl BOOST_NO_SCOPED_ENUMS was changed to BOOST_NO_CXX11_SCOPED_ENUMS in 1.51. - -TEMP_LIBS="$LIBS" -LIBS="$BOOST_LIBS $LIBS" -TEMP_CPPFLAGS="$CPPFLAGS" -CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" -AC_MSG_CHECKING([for mismatched boost c++11 scoped enums]) -AC_LINK_IFELSE([AC_LANG_PROGRAM([[ - #include <boost/config.hpp> - #include <boost/version.hpp> - #if !defined(BOOST_NO_SCOPED_ENUMS) && !defined(BOOST_NO_CXX11_SCOPED_ENUMS) && BOOST_VERSION < 105700 - #define BOOST_NO_SCOPED_ENUMS - #define BOOST_NO_CXX11_SCOPED_ENUMS - #define CHECK - #endif - #include <boost/filesystem.hpp> - ]],[[ - #if defined(CHECK) - boost::filesystem::copy_file("foo", "bar"); - #else - choke; - #endif - ]])], - [AC_MSG_RESULT(mismatched); BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_NO_SCOPED_ENUMS -DBOOST_NO_CXX11_SCOPED_ENUMS"], [AC_MSG_RESULT(ok)]) -LIBS="$TEMP_LIBS" -CPPFLAGS="$TEMP_CPPFLAGS" - -fi - dnl libevent check if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then @@ -1683,6 +1660,7 @@ esac echo echo "Options used to compile and link:" +echo " boost process = $ax_cv_boost_process" echo " multiprocess = $build_multiprocess" echo " with wallet = $enable_wallet" echo " with gui / qt = $bitcoin_enable_qt" @@ -1706,10 +1684,10 @@ echo " target os = $TARGET_OS" echo " build os = $build_os" echo echo " CC = $CC" -echo " CFLAGS = $CFLAGS" +echo " CFLAGS = $PTHREAD_CFLAGS $CFLAGS" echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS" echo " CXX = $CXX" echo " CXXFLAGS = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS" -echo " LDFLAGS = $PTHREAD_CFLAGS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" +echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 0ed1e16f7..e86ff8379 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -7,31 +7,29 @@ suites: architectures: - "amd64" packages: -- "curl" -- "g++-aarch64-linux-gnu" -- "g++-8-aarch64-linux-gnu" -- "gcc-8-aarch64-linux-gnu" -- "binutils-aarch64-linux-gnu" -- "g++-arm-linux-gnueabihf" -- "g++-8-arm-linux-gnueabihf" -- "gcc-8-arm-linux-gnueabihf" -- "binutils-arm-linux-gnueabihf" -- "g++-riscv64-linux-gnu" -- "g++-8-riscv64-linux-gnu" -- "gcc-8-riscv64-linux-gnu" -- "binutils-riscv64-linux-gnu" -- "g++-8-multilib" -- "gcc-8-multilib" -- "binutils-gold" -- "git" -- "pkg-config" +# Common dependencies. - "autoconf" -- "libtool" - "automake" -- "faketime" +- "binutils" - "bsdmainutils" - "ca-certificates" +- "curl" +- "faketime" +- "git" +- "libtool" +- "patch" +- "pkg-config" - "python3" +# Cross compilation HOSTS: +# - arm-linux-gnueabihf +- "binutils-arm-linux-gnueabihf" +- "g++-8-arm-linux-gnueabihf" +# - aarch64-linux-gnu +- "binutils-aarch64-linux-gnu" +- "g++-8-aarch64-linux-gnu" +# - riscv64-linux-gnu +- "binutils-riscv64-linux-gnu" +- "g++-8-riscv64-linux-gnu" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" @@ -93,45 +91,11 @@ script: | create_per-host_faketime_wrappers "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} - EXTRA_INCLUDES_BASE=$WRAP_DIR/extra_includes - mkdir -p $EXTRA_INCLUDES_BASE - - # x86 needs /usr/include/i386-linux-gnu/asm pointed to /usr/include/x86_64-linux-gnu/asm, - # but we can't write there. Instead, create a link here and force it to be included in the - # search paths by wrapping gcc/g++. - - mkdir -p $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu - rm -f $WRAP_DIR/extra_includes/i686-pc-linux-gnu/asm - ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu/asm - - for prog in gcc g++; do - rm -f ${WRAP_DIR}/${prog} - cat << EOF > ${WRAP_DIR}/${prog} - #!/usr/bin/env bash - REAL="$(which -a ${prog}-8 | grep -v ${WRAP_DIR}/${prog} | head -1)" - for var in "\$@" - do - if [ "\$var" = "-m32" ]; then - export C_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" - export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" - break - fi - done - \$REAL \$@ - EOF - chmod +x ${WRAP_DIR}/${prog} - done - cd bitcoin BASEPREFIX="${PWD}/depends" # Build dependencies for each host for i in $HOSTS; do - EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" - if [ -d "$EXTRA_INCLUDES" ]; then - export HOST_ID_SALT="$EXTRA_INCLUDES" - fi make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" - unset HOST_ID_SALT done # Faketime for binaries diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index d2652119b..ece02dc24 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -123,7 +123,6 @@ Memcheck:Cond ... fun:_ZN5boost10filesystem6detail11unique_pathERKNS0_4pathEPNS_6system10error_codeE - fun:unique_path } { Suppress boost warning diff --git a/contrib/zmq/zmq_sub.py b/contrib/zmq/zmq_sub.py index 06893407f..06893407f 100644..100755 --- a/contrib/zmq/zmq_sub.py +++ b/contrib/zmq/zmq_sub.py diff --git a/depends/funcs.mk b/depends/funcs.mk index 6fc20543b..81ecbedf5 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -157,12 +157,17 @@ ifneq ($($(1)_ldflags),) $(1)_autoconf += LDFLAGS="$$($(1)_ldflags)" endif -$(1)_cmake=cmake -DCMAKE_INSTALL_PREFIX=$($($(1)_type)_prefix) +$(1)_cmake=env CC="$$($(1)_cc)" \ + CFLAGS="$$($(1)_cppflags) $$($(1)_cflags)" \ + CXX="$$($(1)_cxx)" \ + CXXFLAGS="$$($(1)_cppflags) $$($(1)_cxxflags)" \ + LDFLAGS="$$($(1)_ldflags)" \ + cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" ifneq ($($(1)_type),build) ifneq ($(host),$(build)) -$(1)_cmake += -DCMAKE_SYSTEM_NAME=$($(host_os)_cmake_system) -DCMAKE_SYSROOT=$(host_prefix) -$(1)_cmake += -DCMAKE_C_COMPILER_TARGET=$(host) -DCMAKE_C_COMPILER=$(firstword $($($(1)_type)_CC)) -DCMAKE_C_FLAGS="$(wordlist 2,1000,$($($(1)_type)_CC))" -$(1)_cmake += -DCMAKE_CXX_COMPILER_TARGET=$(host) -DCMAKE_CXX_COMPILER=$(firstword $($($(1)_type)_CXX)) -DCMAKE_CXX_FLAGS="$(wordlist 2,1000,$($($(1)_type)_CXX))" +$(1)_cmake += -DCMAKE_SYSTEM_NAME=$($(host_os)_cmake_system) +$(1)_cmake += -DCMAKE_C_COMPILER_TARGET=$(host) +$(1)_cmake += -DCMAKE_CXX_COMPILER_TARGET=$(host) endif endif endef diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index b679438c6..06cf974f7 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -4,6 +4,7 @@ $(package)_download_path=https://download.oracle.com/berkeley-db $(package)_file_name=db-$($(package)_version).NC.tar.gz $(package)_sha256_hash=12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef $(package)_build_subdir=build_unix +$(package)_patches=clang_cxx_11.patch define $(package)_set_vars $(package)_config_opts=--disable-shared --enable-cxx --disable-replication --enable-option-checking @@ -14,8 +15,7 @@ $(package)_cppflags_mingw32=-DUNICODE -D_UNICODE endef define $(package)_preprocess_cmds - sed -i.old 's/__atomic_compare_exchange/__atomic_compare_exchange_db/' dbinc/atomic.h && \ - sed -i.old 's/atomic_init/atomic_init_db/' dbinc/atomic.h mp/mp_region.c mp/mp_mvcc.c mp/mp_fget.c mutex/mut_method.c mutex/mut_tas.c && \ + patch -p1 < $($(package)_patch_dir)/clang_cxx_11.patch && \ cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub dist endef diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 3a7e605b4..d8bce108b 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -3,6 +3,7 @@ $(package)_version=1_70_0 $(package)_download_path=https://dl.bintray.com/boostorg/release/1.70.0/source/ $(package)_file_name=boost_$($(package)_version).tar.bz2 $(package)_sha256_hash=430ae8354789de4fd19ee52f3b1f739e1fba576f0aded0897c3c2bc00fb38778 +$(package)_patches=unused_var_in_process.patch define $(package)_set_vars $(package)_config_opts_release=variant=release @@ -32,6 +33,7 @@ $(package)_cxxflags_android=-fPIC endef define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/unused_var_in_process.patch && \ echo "using $($(package)_toolset_$(host_os)) : : $($(package)_cxx) : <cxxflags>\"$($(package)_cxxflags) $($(package)_cppflags)\" <linkflags>\"$($(package)_ldflags)\" <archiver>\"$($(package)_archiver_$(host_os))\" <striper>\"$(host_STRIP)\" <ranlib>\"$(host_RANLIB)\" <rc>\"$(host_WINDRES)\" : ;" > user-config.jam endef diff --git a/depends/packages/fontconfig.mk b/depends/packages/fontconfig.mk index 128599ba7..0d5f94f38 100644 --- a/depends/packages/fontconfig.mk +++ b/depends/packages/fontconfig.mk @@ -4,23 +4,23 @@ $(package)_download_path=https://www.freedesktop.org/software/fontconfig/release $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=b449a3e10c47e1d1c7a6ec6e2016cca73d3bd68fbbd4f0ae5cc6b573f7d6c7f3 $(package)_dependencies=freetype expat +$(package)_patches=remove_char_width_usage.patch gperf_header_regen.patch define $(package)_set_vars $(package)_config_opts=--disable-docs --disable-static --disable-libxml2 --disable-iconv $(package)_config_opts += --disable-dependency-tracking --enable-option-checking endef +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/remove_char_width_usage.patch && \ + patch -p1 < $($(package)_patch_dir)/gperf_header_regen.patch +endef + define $(package)_config_cmds $($(package)_autoconf) endef -# 2.12.1 uses CHAR_WIDTH which is reserved and clashes with some glibc versions, but newer versions of fontconfig -# have broken makefiles which needlessly attempt to re-generate headers with gperf. -# Instead, change all uses of CHAR_WIDTH, and disable the rule that forces header re-generation. -# This can be removed once the upstream build is fixed. define $(package)_build_cmds - sed -i 's/CHAR_WIDTH/CHARWIDTH/g' fontconfig/fontconfig.h src/fcobjshash.gperf src/fcobjs.h src/fcobjshash.h && \ - sed -i 's/fcobjshash.h: fcobjshash.gperf/fcobjshash.h:/' src/Makefile && \ $(MAKE) endef diff --git a/depends/packages/miniupnpc.mk b/depends/packages/miniupnpc.mk index fdbe22cda..49a584e46 100644 --- a/depends/packages/miniupnpc.mk +++ b/depends/packages/miniupnpc.mk @@ -3,6 +3,7 @@ $(package)_version=2.0.20180203 $(package)_download_path=https://miniupnp.tuxfamily.org/files/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=90dda8c7563ca6cd4a83e23b3c66dbbea89603a1675bfdb852897c2c9cc220b7 +$(package)_patches=dont_use_wingen.patch define $(package)_set_vars $(package)_build_opts=CC="$($(package)_cc)" @@ -14,7 +15,7 @@ endef define $(package)_preprocess_cmds mkdir dll && \ sed -e 's|MINIUPNPC_VERSION_STRING \"version\"|MINIUPNPC_VERSION_STRING \"$($(package)_version)\"|' -e 's|OS/version|$(host)|' miniupnpcstrings.h.in > miniupnpcstrings.h && \ - sed -i.old "s|miniupnpcstrings.h: miniupnpcstrings.h.in wingenminiupnpcstrings|miniupnpcstrings.h: miniupnpcstrings.h.in|" Makefile.mingw + patch -p1 < $($(package)_patch_dir)/dont_use_wingen.patch endef define $(package)_build_cmds diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 5022ed980..d56b63669 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -4,6 +4,8 @@ $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive $(package)_file_name=$($(package)_version).tar.gz $(package)_sha256_hash=e51995a843533a3dac155dd0c71362dd471597a2d23f13dff194c6285362f875 $(package)_build_subdir=cctools +$(package)_patches=ld64_disable_threading.patch + ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) $(package)_clang_version=8.0.0 $(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version) @@ -78,7 +80,7 @@ endef define $(package)_preprocess_cmds CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/build.sh && \ CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/install.sh && \ - sed -i.old "/define HAVE_PTHREADS/d" $($(package)_build_subdir)/ld64/src/ld/InputFiles.h + patch -p1 < $($(package)_patch_dir)/ld64_disable_threading.patch endef define $(package)_config_cmds diff --git a/depends/packages/native_cdrkit.mk b/depends/packages/native_cdrkit.mk index 14c37a0fc..7bdf2d7df 100644 --- a/depends/packages/native_cdrkit.mk +++ b/depends/packages/native_cdrkit.mk @@ -12,7 +12,7 @@ endef # Starting with 10.1, GCC defaults to -fno-common, resulting in linking errors. # Pass -fcommon to retain the legacy behaviour. define $(package)_config_cmds - cmake -DCMAKE_INSTALL_PREFIX=$(build_prefix) -DCMAKE_C_FLAGS="-fcommon" + $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -fcommon" endef define $(package)_build_cmds diff --git a/depends/packages/native_libdmg-hfsplus.mk b/depends/packages/native_libdmg-hfsplus.mk index c0f0ce74d..035b76718 100644 --- a/depends/packages/native_libdmg-hfsplus.mk +++ b/depends/packages/native_libdmg-hfsplus.mk @@ -12,7 +12,7 @@ define $(package)_preprocess_cmds endef define $(package)_config_cmds - cmake -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_C_FLAGS="-Wl,--build-id=none" .. + $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -Wl,--build-id=none" .. endef define $(package)_build_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 500881e44..f560099b6 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -8,7 +8,10 @@ $(package)_dependencies=zlib $(package)_linux_dependencies=freetype fontconfig libxcb $(package)_build_subdir=qtbase $(package)_qt_libs=corelib network widgets gui plugins testlib -$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch fix_android_qmake_conf.patch fix_android_jni_static.patch +$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch +$(package)_patches+= fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch no-xlib.patch +$(package)_patches+= fix_android_qmake_conf.patch fix_android_jni_static.patch dont_hardcode_pwd.patch +$(package)_patches+= freetype_back_compat.patch drop_lrelease_dependency.patch # Update OSX_QT_TRANSLATIONS when this is updated $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) @@ -190,11 +193,10 @@ define $(package)_extract_cmds endef define $(package)_preprocess_cmds - sed -i.old "s|FT_Get_Font_Format|FT_Get_X11_Font_Format|" qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp && \ + patch -p1 -i $($(package)_patch_dir)/freetype_back_compat.patch && \ sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \ - sed -i.old "/updateqm.depends =/d" qttranslations/translations/translations.pro && \ - sed -i.old "s/src_plugins.depends = src_sql src_network/src_plugins.depends = src_network/" qtbase/src/src.pro && \ - sed -i.old -e 's/if \[ "$$$$XPLATFORM_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/if \[ "$$$$BUILD_ON_MAC" = "yes" \]; then xspecvals=$$$$(macSDKify/' -e 's|/bin/pwd|pwd|' qtbase/configure && \ + patch -p1 -i $($(package)_patch_dir)/drop_lrelease_dependency.patch && \ + patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch &&\ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/Info.plist.lib qtbase/mkspecs/macx-clang-linux/ &&\ cp -f qtbase/mkspecs/macx-clang/Info.plist.app qtbase/mkspecs/macx-clang-linux/ &&\ diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index 6f35ede24..c93aa1a74 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -3,7 +3,7 @@ $(package)_version=4.3.1 $(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d835cd21eb -$(package)_patches=0001-fix-build-with-older-mingw64.patch 0002-disable-pthread_set_name_np.patch +$(package)_patches=remove_libstd_link.patch define $(package)_set_vars $(package)_config_opts=--without-docs --disable-shared --disable-curve --disable-curve-keygen --disable-perf @@ -16,9 +16,8 @@ define $(package)_set_vars endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/0001-fix-build-with-older-mingw64.patch && \ - patch -p1 < $($(package)_patch_dir)/0002-disable-pthread_set_name_np.patch && \ - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config + patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch && \ + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config endef define $(package)_config_cmds @@ -34,6 +33,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - sed -i.old "s/ -lstdc++//" lib/pkgconfig/libzmq.pc && \ rm -rf bin share lib/*.la endef diff --git a/depends/patches/bdb/clang_cxx_11.patch b/depends/patches/bdb/clang_cxx_11.patch new file mode 100644 index 000000000..58f7ddc7d --- /dev/null +++ b/depends/patches/bdb/clang_cxx_11.patch @@ -0,0 +1,147 @@ +commit 3311d68f11d1697565401eee6efc85c34f022ea7 +Author: fanquake <[email protected]> +Date: Mon Aug 17 20:03:56 2020 +0800 + + Fix C++11 compatibility + +diff --git a/dbinc/atomic.h b/dbinc/atomic.h +index 0034dcc..7c11d4a 100644 +--- a/dbinc/atomic.h ++++ b/dbinc/atomic.h +@@ -70,7 +70,7 @@ typedef struct { + * These have no memory barriers; the caller must include them when necessary. + */ + #define atomic_read(p) ((p)->value) +-#define atomic_init(p, val) ((p)->value = (val)) ++#define atomic_init_db(p, val) ((p)->value = (val)) + + #ifdef HAVE_ATOMIC_SUPPORT + +@@ -144,7 +144,7 @@ typedef LONG volatile *interlocked_val; + #define atomic_inc(env, p) __atomic_inc(p) + #define atomic_dec(env, p) __atomic_dec(p) + #define atomic_compare_exchange(env, p, o, n) \ +- __atomic_compare_exchange((p), (o), (n)) ++ __atomic_compare_exchange_db((p), (o), (n)) + static inline int __atomic_inc(db_atomic_t *p) + { + int temp; +@@ -176,7 +176,7 @@ static inline int __atomic_dec(db_atomic_t *p) + * http://gcc.gnu.org/onlinedocs/gcc-4.1.0/gcc/Atomic-Builtins.html + * which configure could be changed to use. + */ +-static inline int __atomic_compare_exchange( ++static inline int __atomic_compare_exchange_db( + db_atomic_t *p, atomic_value_t oldval, atomic_value_t newval) + { + atomic_value_t was; +@@ -206,7 +206,7 @@ static inline int __atomic_compare_exchange( + #define atomic_dec(env, p) (--(p)->value) + #define atomic_compare_exchange(env, p, oldval, newval) \ + (DB_ASSERT(env, atomic_read(p) == (oldval)), \ +- atomic_init(p, (newval)), 1) ++ atomic_init_db(p, (newval)), 1) + #else + #define atomic_inc(env, p) __atomic_inc(env, p) + #define atomic_dec(env, p) __atomic_dec(env, p) +diff --git a/mp/mp_fget.c b/mp/mp_fget.c +index 5fdee5a..0b75f57 100644 +--- a/mp/mp_fget.c ++++ b/mp/mp_fget.c +@@ -617,7 +617,7 @@ alloc: /* Allocate a new buffer header and data space. */ + + /* Initialize enough so we can call __memp_bhfree. */ + alloc_bhp->flags = 0; +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + #ifdef DIAGNOSTIC + if ((uintptr_t)alloc_bhp->buf & (sizeof(size_t) - 1)) { + __db_errx(env, +@@ -911,7 +911,7 @@ alloc: /* Allocate a new buffer header and data space. */ + MVCC_MPROTECT(bhp->buf, mfp->stat.st_pagesize, + PROT_READ); + +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + MUTEX_LOCK(env, alloc_bhp->mtx_buf); + alloc_bhp->priority = bhp->priority; + alloc_bhp->pgno = bhp->pgno; +diff --git a/mp/mp_mvcc.c b/mp/mp_mvcc.c +index 34467d2..f05aa0c 100644 +--- a/mp/mp_mvcc.c ++++ b/mp/mp_mvcc.c +@@ -276,7 +276,7 @@ __memp_bh_freeze(dbmp, infop, hp, bhp, need_frozenp) + #else + memcpy(frozen_bhp, bhp, SSZA(BH, buf)); + #endif +- atomic_init(&frozen_bhp->ref, 0); ++ atomic_init_db(&frozen_bhp->ref, 0); + if (mutex != MUTEX_INVALID) + frozen_bhp->mtx_buf = mutex; + else if ((ret = __mutex_alloc(env, MTX_MPOOL_BH, +@@ -428,7 +428,7 @@ __memp_bh_thaw(dbmp, infop, hp, frozen_bhp, alloc_bhp) + #endif + alloc_bhp->mtx_buf = mutex; + MUTEX_LOCK(env, alloc_bhp->mtx_buf); +- atomic_init(&alloc_bhp->ref, 1); ++ atomic_init_db(&alloc_bhp->ref, 1); + F_CLR(alloc_bhp, BH_FROZEN); + } + +diff --git a/mp/mp_region.c b/mp/mp_region.c +index e6cece9..ddbe906 100644 +--- a/mp/mp_region.c ++++ b/mp/mp_region.c +@@ -224,7 +224,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) + MTX_MPOOL_FILE_BUCKET, 0, &htab[i].mtx_hash)) != 0) + return (ret); + SH_TAILQ_INIT(&htab[i].hash_bucket); +- atomic_init(&htab[i].hash_page_dirty, 0); ++ atomic_init_db(&htab[i].hash_page_dirty, 0); + } + + /* +@@ -269,7 +269,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) + hp->mtx_hash = (mtx_base == MUTEX_INVALID) ? MUTEX_INVALID : + mtx_base + i; + SH_TAILQ_INIT(&hp->hash_bucket); +- atomic_init(&hp->hash_page_dirty, 0); ++ atomic_init_db(&hp->hash_page_dirty, 0); + #ifdef HAVE_STATISTICS + hp->hash_io_wait = 0; + hp->hash_frozen = hp->hash_thawed = hp->hash_frozen_freed = 0; +diff --git a/mutex/mut_method.c b/mutex/mut_method.c +index 2588763..5c6d516 100644 +--- a/mutex/mut_method.c ++++ b/mutex/mut_method.c +@@ -426,7 +426,7 @@ atomic_compare_exchange(env, v, oldval, newval) + MUTEX_LOCK(env, mtx); + ret = atomic_read(v) == oldval; + if (ret) +- atomic_init(v, newval); ++ atomic_init_db(v, newval); + MUTEX_UNLOCK(env, mtx); + + return (ret); +diff --git a/mutex/mut_tas.c b/mutex/mut_tas.c +index f3922e0..e40fcdf 100644 +--- a/mutex/mut_tas.c ++++ b/mutex/mut_tas.c +@@ -46,7 +46,7 @@ __db_tas_mutex_init(env, mutex, flags) + + #ifdef HAVE_SHARED_LATCHES + if (F_ISSET(mutexp, DB_MUTEX_SHARED)) +- atomic_init(&mutexp->sharecount, 0); ++ atomic_init_db(&mutexp->sharecount, 0); + else + #endif + if (MUTEX_INIT(&mutexp->tas)) { +@@ -486,7 +486,7 @@ __db_tas_mutex_unlock(env, mutex) + F_CLR(mutexp, DB_MUTEX_LOCKED); + /* Flush flag update before zeroing count */ + MEMBAR_EXIT(); +- atomic_init(&mutexp->sharecount, 0); ++ atomic_init_db(&mutexp->sharecount, 0); + } else { + DB_ASSERT(env, sharecount > 0); + MEMBAR_EXIT(); diff --git a/depends/patches/boost/unused_var_in_process.patch b/depends/patches/boost/unused_var_in_process.patch new file mode 100644 index 000000000..722f7bb5e --- /dev/null +++ b/depends/patches/boost/unused_var_in_process.patch @@ -0,0 +1,22 @@ +commit dbd95cdaefdea95307d004f019a1c394cf9389f0 +Author: fanquake <[email protected]> +Date: Mon Aug 17 20:15:17 2020 +0800 + + Remove unused variable in Boost Process + + This causes issues with our linters / CI. + + Can be removed once depends Boost is 1.71.0 or later. + +diff --git a/boost/process/detail/posix/wait_group.hpp b/boost/process/detail/posix/wait_group.hpp +index 9dc249803..2502d9772 100644 +--- a/boost/process/detail/posix/wait_group.hpp ++++ b/boost/process/detail/posix/wait_group.hpp +@@ -137,7 +137,6 @@ inline bool wait_until( + + do + { +- int ret_sig = 0; + int status; + if ((::waitpid(timeout_pid, &status, WNOHANG) != 0) + && (WIFEXITED(status) || WIFSIGNALED(status))) diff --git a/depends/patches/fontconfig/gperf_header_regen.patch b/depends/patches/fontconfig/gperf_header_regen.patch new file mode 100644 index 000000000..7401b83d8 --- /dev/null +++ b/depends/patches/fontconfig/gperf_header_regen.patch @@ -0,0 +1,24 @@ +commit 7b6eb33ecd88768b28c67ce5d2d68a7eed5936b6 +Author: fanquake <[email protected]> +Date: Tue Aug 25 14:34:53 2020 +0800 + + Remove rule that causes inadvertant header regeneration + + Otherwise the makefile will needlessly attempt to re-generate the + headers with gperf. This can be dropped once the upstream build is fixed. + + See #10851. + +diff --git a/src/Makefile.in b/src/Makefile.in +index f4626ad..4ae1b00 100644 +--- a/src/Makefile.in ++++ b/src/Makefile.in +@@ -903,7 +903,7 @@ fcobjshash.gperf: fcobjshash.gperf.h fcobjs.h + ' - > [email protected] && \ + mv -f [email protected] $@ || ( $(RM) [email protected] && false ) + +-fcobjshash.h: fcobjshash.gperf ++fcobjshash.h: + $(AM_V_GEN) $(GPERF) -m 100 $< > [email protected] && \ + mv -f [email protected] $@ || ( $(RM) [email protected] && false ) + diff --git a/depends/patches/fontconfig/remove_char_width_usage.patch b/depends/patches/fontconfig/remove_char_width_usage.patch new file mode 100644 index 000000000..9f6908189 --- /dev/null +++ b/depends/patches/fontconfig/remove_char_width_usage.patch @@ -0,0 +1,62 @@ +commit 28165a9b078583dc8e9e5c344510e37582284cef +Author: fanquake <[email protected]> +Date: Mon Aug 17 20:35:42 2020 +0800 + + Remove usage of CHAR_WIDTH + + CHAR_WIDTH which is reserved and clashes with glibc 2.25+ + + See #10851. + +diff --git a/fontconfig/fontconfig.h b/fontconfig/fontconfig.h +index 5c72b22..843c532 100644 +--- a/fontconfig/fontconfig.h ++++ b/fontconfig/fontconfig.h +@@ -128,7 +128,7 @@ typedef int FcBool; + #define FC_USER_CACHE_FILE ".fonts.cache-" FC_CACHE_VERSION + + /* Adjust outline rasterizer */ +-#define FC_CHAR_WIDTH "charwidth" /* Int */ ++#define FC_CHARWIDTH "charwidth" /* Int */ + #define FC_CHAR_HEIGHT "charheight"/* Int */ + #define FC_MATRIX "matrix" /* FcMatrix */ + +diff --git a/src/fcobjs.h b/src/fcobjs.h +index 1fc4f65..d27864b 100644 +--- a/src/fcobjs.h ++++ b/src/fcobjs.h +@@ -51,7 +51,7 @@ FC_OBJECT (DPI, FcTypeDouble, NULL) + FC_OBJECT (RGBA, FcTypeInteger, NULL) + FC_OBJECT (SCALE, FcTypeDouble, NULL) + FC_OBJECT (MINSPACE, FcTypeBool, NULL) +-FC_OBJECT (CHAR_WIDTH, FcTypeInteger, NULL) ++FC_OBJECT (CHARWIDTH, FcTypeInteger, NULL) + FC_OBJECT (CHAR_HEIGHT, FcTypeInteger, NULL) + FC_OBJECT (MATRIX, FcTypeMatrix, NULL) + FC_OBJECT (CHARSET, FcTypeCharSet, FcCompareCharSet) +diff --git a/src/fcobjshash.gperf b/src/fcobjshash.gperf +index 80a0237..eb4ad84 100644 +--- a/src/fcobjshash.gperf ++++ b/src/fcobjshash.gperf +@@ -44,7 +44,7 @@ int id; + "rgba",FC_RGBA_OBJECT + "scale",FC_SCALE_OBJECT + "minspace",FC_MINSPACE_OBJECT +-"charwidth",FC_CHAR_WIDTH_OBJECT ++"charwidth",FC_CHARWIDTH_OBJECT + "charheight",FC_CHAR_HEIGHT_OBJECT + "matrix",FC_MATRIX_OBJECT + "charset",FC_CHARSET_OBJECT +diff --git a/src/fcobjshash.h b/src/fcobjshash.h +index 5a4d1ea..4e66bb0 100644 +--- a/src/fcobjshash.h ++++ b/src/fcobjshash.h +@@ -284,7 +284,7 @@ FcObjectTypeLookup (register const char *str, register unsigned int len) + {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str43,FC_CHARSET_OBJECT}, + {-1}, + #line 47 "fcobjshash.gperf" +- {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str45,FC_CHAR_WIDTH_OBJECT}, ++ {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str45,FC_CHARWIDTH_OBJECT}, + #line 48 "fcobjshash.gperf" + {(int)(long)&((struct FcObjectTypeNamePool_t *)0)->FcObjectTypeNamePool_str46,FC_CHAR_HEIGHT_OBJECT}, + #line 55 "fcobjshash.gperf" diff --git a/depends/patches/miniupnpc/dont_use_wingen.patch b/depends/patches/miniupnpc/dont_use_wingen.patch new file mode 100644 index 000000000..a1cc9b50d --- /dev/null +++ b/depends/patches/miniupnpc/dont_use_wingen.patch @@ -0,0 +1,26 @@ +commit e8077044df239bcf0d9e9980b0e1afb9f1f5c446 +Author: fanquake <[email protected]> +Date: Tue Aug 18 20:50:19 2020 +0800 + + Don't use wingenminiupnpcstrings when generating miniupnpcstrings.h + + The wingenminiupnpcstrings tool is used on Windows to generate version + information. This information is irrelevant for us, and trying to use + wingenminiupnpcstrings would cause builds to fail, so just don't use it. + + We should be able to drop this once we are using 2.1 or later. See + upstream commit: 9663c55c61408fdcc39a82987d2243f816b22932. + +diff --git a/Makefile.mingw b/Makefile.mingw +index 574720e..fcc17bb 100644 +--- a/Makefile.mingw ++++ b/Makefile.mingw +@@ -74,7 +74,7 @@ wingenminiupnpcstrings: wingenminiupnpcstrings.o + + wingenminiupnpcstrings.o: wingenminiupnpcstrings.c + +-miniupnpcstrings.h: miniupnpcstrings.h.in wingenminiupnpcstrings ++miniupnpcstrings.h: miniupnpcstrings.h.in + wingenminiupnpcstrings $< $@ + + minixml.o: minixml.c minixml.h diff --git a/depends/patches/native_cctools/ld64_disable_threading.patch b/depends/patches/native_cctools/ld64_disable_threading.patch new file mode 100644 index 000000000..d6c58c102 --- /dev/null +++ b/depends/patches/native_cctools/ld64_disable_threading.patch @@ -0,0 +1,26 @@ +commit 584668415039adeed073decee7e04de28248afd3 +Author: fanquake <[email protected]> +Date: Tue Aug 18 01:20:24 2020 +0000 + + Disable threading to fix non-determinism + + A bug in the file parser can cause dependencies to be calculated + differently based on which files have already been parsed. This is more + likely to occur on systems with more CPUs. + + Just disable threading for now. There is no noticable slowdown. + + See #9891. + +diff --git a/cctools/ld64/src/ld/InputFiles.h b/cctools/ld64/src/ld/InputFiles.h +index ef9c756..90a70b6 100644 +--- a/cctools/ld64/src/ld/InputFiles.h ++++ b/cctools/ld64/src/ld/InputFiles.h +@@ -25,7 +25,6 @@ + #ifndef __INPUT_FILES_H__ + #define __INPUT_FILES_H__ + +-#define HAVE_PTHREADS 1 + + #include <stdlib.h> + #include <sys/types.h> diff --git a/depends/patches/qt/dont_hardcode_pwd.patch b/depends/patches/qt/dont_hardcode_pwd.patch new file mode 100644 index 000000000..a74e9cb09 --- /dev/null +++ b/depends/patches/qt/dont_hardcode_pwd.patch @@ -0,0 +1,27 @@ +commit 0e953866fc4672486e29e1ba6d83b4207e7b2f0b +Author: fanquake <[email protected]> +Date: Tue Aug 18 15:09:06 2020 +0800 + + Don't hardcode pwd path + + Let a man use his builtins if he wants to! Also, removes the unnecessary + assumption that pwd lives under /bin/pwd. + + See #15581. + +diff --git a/qtbase/configure b/qtbase/configure +index 08b49a8d..faea5b55 100755 +--- a/qtbase/configure ++++ b/qtbase/configure +@@ -36,9 +36,9 @@ + relconf=`basename $0` + # the directory of this script is the "source tree" + relpath=`dirname $0` +-relpath=`(cd "$relpath"; /bin/pwd)` ++relpath=`(cd "$relpath"; pwd)` + # the current directory is the "build tree" or "object tree" +-outpath=`/bin/pwd` ++outpath=`pwd` + + WHICH="which" + diff --git a/depends/patches/qt/drop_lrelease_dependency.patch b/depends/patches/qt/drop_lrelease_dependency.patch new file mode 100644 index 000000000..f6b2c9fc8 --- /dev/null +++ b/depends/patches/qt/drop_lrelease_dependency.patch @@ -0,0 +1,20 @@ +commit 67b3ed7406e1d0762188dbad2c44a06824ba0778 +Author: fanquake <[email protected]> +Date: Tue Aug 18 15:24:01 2020 +0800 + + Drop dependency on lrelease + + Qts buildsystem insists on using the installed lrelease, but gets + confused about how to find it. Since we manually control the build + order, just drop the dependency. + + See #9469 + +diff --git a/qttranslations/translations/translations.pro b/qttranslations/translations/translations.pro +index 694544c..eff339d 100644 +--- a/qttranslations/translations/translations.pro ++++ b/qttranslations/translations/translations.pro +@@ -109,3 +109,2 @@ updateqm.commands = $$LRELEASE ${QMAKE_FILE_IN} -qm ${QMAKE_FILE_OUT} + silent:updateqm.commands = @echo lrelease ${QMAKE_FILE_IN} && $$updateqm.commands +-updateqm.depends = $$LRELEASE_EXE + updateqm.name = LRELEASE ${QMAKE_FILE_IN} diff --git a/depends/patches/qt/freetype_back_compat.patch b/depends/patches/qt/freetype_back_compat.patch new file mode 100644 index 000000000..b0f1c98aa --- /dev/null +++ b/depends/patches/qt/freetype_back_compat.patch @@ -0,0 +1,28 @@ +commit 14bc77db61bf9d56f9b6c8b84aa02573605c19c6 +Author: fanquake <[email protected]> +Date: Tue Aug 18 15:15:08 2020 +0800 + + Fix backwards compatibility with older Freetype versions at runtime + + A few years ago, libfreetype introduced FT_Get_Font_Format() as an alias + for FT_Get_X11_Font_Format(), but FT_Get_X11_Font_Format() was kept for abi + backwards-compatibility. + + Qt 5.9 introduced a call to FT_Get_Font_Format(). Replace it with FT_Get_X11_Font_Format() + in order to remain compatibile with older freetype, which is still used by e.g. Ubuntu Trusty. + + See #14348. + +diff --git a/qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp b/qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp +index 3f543755..8ecc1c8c 100644 +--- a/qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp ++++ b/qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp +@@ -898,7 +898,7 @@ bool QFontEngineFT::init(FaceId faceId, bool antialias, GlyphFormat format, + } + } + #if defined(FT_FONT_FORMATS_H) +- const char *fmt = FT_Get_Font_Format(face); ++ const char *fmt = FT_Get_X11_Font_Format(face); + if (fmt && qstrncmp(fmt, "CFF", 4) == 0) { + FT_Bool no_stem_darkening = true; + FT_Error err = FT_Property_Get(qt_getFreetype(), "cff", "no-stem-darkening", &no_stem_darkening); diff --git a/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch b/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch deleted file mode 100644 index b911ac567..000000000 --- a/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch +++ /dev/null @@ -1,30 +0,0 @@ -From f6866b0f166ad168618aae64c7fbee8775d3eb23 Mon Sep 17 00:00:00 2001 -From: mruddy <[email protected]> -Date: Sat, 30 Jun 2018 09:44:58 -0400 -Subject: [PATCH] fix build with older mingw64 - ---- - src/windows.hpp | 7 +++++++ - 1 file changed, 7 insertions(+) - -diff --git a/src/windows.hpp b/src/windows.hpp -index 6c3839fd..2c32ec79 100644 ---- a/src/windows.hpp -+++ b/src/windows.hpp -@@ -58,6 +58,13 @@ - #include <winsock2.h> - #include <windows.h> - #include <mswsock.h> -+ -+#if defined __MINGW64_VERSION_MAJOR && __MINGW64_VERSION_MAJOR < 4 -+// Workaround for mingw-w64 < v4.0 which did not include ws2ipdef.h in iphlpapi.h. -+// Fixed in mingw-w64 by 9bd8fe9148924840d315b4c915dd099955ea89d1. -+#include <ws2def.h> -+#include <ws2ipdef.h> -+#endif - #include <iphlpapi.h> - - #if !defined __MINGW32__ --- -2.17.1 - diff --git a/depends/patches/zeromq/0002-disable-pthread_set_name_np.patch b/depends/patches/zeromq/0002-disable-pthread_set_name_np.patch deleted file mode 100644 index b1c6f78a7..000000000 --- a/depends/patches/zeromq/0002-disable-pthread_set_name_np.patch +++ /dev/null @@ -1,35 +0,0 @@ -From c9bbdd6581d07acfe8971e4bcebe278a3676cf03 Mon Sep 17 00:00:00 2001 -From: mruddy <[email protected]> -Date: Sat, 30 Jun 2018 09:57:18 -0400 -Subject: [PATCH] disable pthread_set_name_np - -pthread_set_name_np adds a Glibc requirement on >= 2.12. ---- - src/thread.cpp | 4 +++- - 1 file changed, 3 insertions(+), 1 deletion(-) - -diff --git a/src/thread.cpp b/src/thread.cpp -index a1086b0c..9943f354 100644 ---- a/src/thread.cpp -+++ b/src/thread.cpp -@@ -308,7 +308,7 @@ void zmq::thread_t::setThreadName (const char *name_) - */ - if (!name_) - return; -- -+#if 0 - #if defined(ZMQ_HAVE_PTHREAD_SETNAME_1) - int rc = pthread_setname_np (name_); - if (rc) -@@ -324,6 +324,8 @@ void zmq::thread_t::setThreadName (const char *name_) - #elif defined(ZMQ_HAVE_PTHREAD_SET_NAME) - pthread_set_name_np (_descriptor, name_); - #endif -+#endif -+ return; - } - - #endif --- -2.17.1 - diff --git a/depends/patches/zeromq/remove_libstd_link.patch b/depends/patches/zeromq/remove_libstd_link.patch new file mode 100644 index 000000000..ddf91e6ab --- /dev/null +++ b/depends/patches/zeromq/remove_libstd_link.patch @@ -0,0 +1,25 @@ +commit 47d4cd12a2c051815ddda78adebdb3923b260d8a +Author: fanquake <[email protected]> +Date: Tue Aug 18 14:45:40 2020 +0800 + + Remove needless linking against libstdc++ + + This is broken for a number of reasons, including: + - g++ understands "static-libstdc++ -lstdc++" to mean "link against + whatever libstdc++ exists, probably shared", which in itself is buggy. + - another stdlib (libc++ for example) may be in use + + See #11981. + +diff --git a/src/libzmq.pc.in b/src/libzmq.pc.in +index 233bc3a..3c2bf0d 100644 +--- a/src/libzmq.pc.in ++++ b/src/libzmq.pc.in +@@ -7,6 +7,6 @@ Name: libzmq + Description: 0MQ c++ library + Version: @VERSION@ + Libs: -L${libdir} -lzmq +-Libs.private: -lstdc++ @pkg_config_libs_private@ ++Libs.private: @pkg_config_libs_private@ + Requires.private: @pkg_config_names_private@ + Cflags: -I${includedir} @pkg_config_defines@ diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 7e307ab7c..2f7916821 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -2073,7 +2073,7 @@ INCLUDE_FILE_PATTERNS = # recursively expanded use the := operator instead of the = operator. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -PREDEFINED = +PREDEFINED = HAVE_BOOST_PROCESS # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this # tag can be used to specify a list of macro names that should be expanded. The diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index a0cfe84a3..40d8e330e 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -60,7 +60,7 @@ RPC interface will be abused. are sent as clear text that can be read by anyone on your network path. Additionally, the RPC interface has not been hardened to withstand arbitrary Internet traffic, so changing the above settings - to expose it to the Internet (even using something like a Tor hidden + to expose it to the Internet (even using something like a Tor onion service) could expose you to unconsidered vulnerabilities. See `bitcoind -help` for more information about these settings and other settings described in this document. diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 53c647ae3..2b051c078 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -1,10 +1,8 @@ OpenBSD build guide ====================== -(updated for OpenBSD 6.4) +(updated for OpenBSD 6.7) -This guide describes how to build bitcoind and command-line utilities on OpenBSD. - -OpenBSD is most commonly used as a server OS, so this guide does not contain instructions for building the GUI. +This guide describes how to build bitcoind, bitcoin-qt, and command-line utilities on OpenBSD. Preparation ------------- @@ -13,9 +11,10 @@ Run the following as root to install the base dependencies for building: ```bash pkg_add git gmake libevent libtool boost +pkg_add qt5 # (optional for enabling the GUI) pkg_add autoconf # (select highest version, e.g. 2.69) pkg_add automake # (select highest version, e.g. 1.16) -pkg_add python # (select highest version, e.g. 3.6) +pkg_add python # (select highest version, e.g. 3.8) git clone https://github.com/bitcoin/bitcoin.git ``` @@ -23,10 +22,10 @@ git clone https://github.com/bitcoin/bitcoin.git See [dependencies.md](dependencies.md) for a complete overview. **Important**: From OpenBSD 6.2 onwards a C++11-supporting clang compiler is -part of the base image, and while building it is necessary to make sure that this -compiler is used and not ancient g++ 4.2.1. This is done by appending -`CC=cc CXX=c++` to configuration commands. Mixing different compilers -within the same executable will result in linker errors. +part of the base image, and while building it is necessary to make sure that +this compiler is used and not ancient g++ 4.2.1. This is done by appending +`CC=cc CC_FOR_BUILD=cc CXX=c++` to configuration commands. Mixing different +compilers within the same executable will result in errors. ### Building BerkeleyDB @@ -77,7 +76,15 @@ To configure with wallet: To configure without wallet: ```bash -./configure --disable-wallet --with-gui=no CC=cc CXX=c++ MAKE=gmake +./configure --disable-wallet --with-gui=no CC=cc CC_FOR_BUILD=cc CXX=c++ MAKE=gmake +``` + +To configure with GUI: +```bash +./configure --with-gui=yes CC=cc CXX=c++ \ + BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ + BDB_CFLAGS="-I${BDB_PREFIX}/include" \ + MAKE=gmake ``` Build and run the tests: diff --git a/doc/dependencies.md b/doc/dependencies.md index 0cb5311e8..92dea6530 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -6,7 +6,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | Dependency | Version used | Minimum required | CVEs | Shared | [Bundled Qt library](https://doc.qt.io/qt-5/configure-options.html#third-party-libraries) | | --- | --- | --- | --- | --- | --- | | Berkeley DB | [4.8.30](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 4.8.x | No | | | -| Boost | [1.70.0](https://www.boost.org/users/download/) | [1.47.0](https://github.com/bitcoin/bitcoin/pull/8920) | No | | | +| Boost | [1.70.0](https://www.boost.org/users/download/) | [1.58.0](https://github.com/bitcoin/bitcoin/pull/19667) | No | | | | Clang | | [3.3+](https://releases.llvm.org/download.html) (C++11 support) | | | | | Expat | [2.2.7](https://libexpat.github.io/) | | No | Yes | | | fontconfig | [2.12.1](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 6ae7e770e..ef9ecbb08 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -746,6 +746,72 @@ the upper cycle, etc. Threads and synchronization ---------------------------- +- Prefer `Mutex` type to `RecursiveMutex` one + +- Consistently use [Clang Thread Safety Analysis](https://clang.llvm.org/docs/ThreadSafetyAnalysis.html) annotations to + get compile-time warnings about potential race conditions in code. Combine annotations in function declarations with + run-time asserts in function definitions: + +```C++ +// txmempool.h +class CTxMemPool +{ +public: + ... + mutable RecursiveMutex cs; + ... + void UpdateTransactionsFromBlock(...) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, cs); + ... +} + +// txmempool.cpp +void CTxMemPool::UpdateTransactionsFromBlock(...) +{ + AssertLockHeld(::cs_main); + AssertLockHeld(cs); + ... +} +``` + +```C++ +// validation.h +class ChainstateManager +{ +public: + ... + bool ProcessNewBlock(...) EXCLUSIVE_LOCKS_REQUIRED(!::cs_main); + ... +} + +// validation.cpp +bool ChainstateManager::ProcessNewBlock(...) +{ + AssertLockNotHeld(::cs_main); + ... + LOCK(::cs_main); + ... +} +``` + +- When Clang Thread Safety Analysis is unable to determine if a mutex is locked, use `LockAssertion` class instances: + +```C++ +// net_processing.h +void RelayTransaction(...) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + +// net_processing.cpp +void RelayTransaction(...) +{ + AssertLockHeld(::cs_main); + + connman.ForEachNode([&txid, &wtxid](CNode* pnode) { + LockAssertion lock(::cs_main); + ... + }); +} + +``` + - Build and run tests with `-DDEBUG_LOCKORDER` to verify that no potential deadlocks are introduced. As of 0.12, this is defined by default when configuring with `--enable-debug`. diff --git a/doc/files.md b/doc/files.md index 547582632..52e094a60 100644 --- a/doc/files.md +++ b/doc/files.md @@ -56,7 +56,7 @@ Subdirectory | File(s) | Description `./` | `fee_estimates.dat` | Stores statistics used to estimate minimum transaction fees and priorities required for confirmation `./` | `guisettings.ini.bak` | Backup of former [GUI settings](#gui-settings) after `-resetguisettings` option is used `./` | `mempool.dat` | Dump of the mempool's transactions -`./` | `onion_private_key` | Cached Tor hidden service private key for `-listenonion` option +`./` | `onion_private_key` | Cached Tor onion service private key for `-listenonion` option `./` | `peers.dat` | Peer IP address database (custom format) `./` | `settings.json` | Read-write settings set through GUI or RPC interfaces, augmenting manual settings from [bitcoin.conf](bitcoin-conf.md). File is created automatically if read-write settings storage is not disabled with `-nosettings` option. Path can be specified with `-settings` option `./` | `.cookie` | Session RPC authentication cookie; if used, created at start and deleted on shutdown; can be specified by `-rpccookiefile` option diff --git a/doc/release-notes-14582.md b/doc/release-notes-14582.md new file mode 100644 index 000000000..28b0abecd --- /dev/null +++ b/doc/release-notes-14582.md @@ -0,0 +1,14 @@ +Configuration +------------- + +A new configuration flag `-maxapsfee` has been added, which sets the max allowed +avoid partial spends (APS) fee. It defaults to 0 (i.e. fee is the same with +and without APS). Setting it to -1 will disable APS, unless `-avoidpartialspends` +is set. (#14582) + +Wallet +------ + +The wallet will now avoid partial spends (APS) by default, if this does not result +in a difference in fees compared to the non-APS variant. The allowed fee threshold +can be adjusted using the new `-maxapsfee` configuration option. (#14582) diff --git a/doc/release-notes-15454.md b/doc/release-notes-15454.md new file mode 100644 index 000000000..00c847a8d --- /dev/null +++ b/doc/release-notes-15454.md @@ -0,0 +1,6 @@ +Wallet +------ + +Bitcoin Core will no longer create an unnamed `""` wallet by default when no wallet is specified on the command line or in the configuration files. +For backwards compatibility, if an unnamed `""` wallet already exists and would have been loaded previously, then it will still be loaded. +Users without an unnamed `""` wallet and without any other wallets to be loaded on startup will be prompted to either choose a wallet to load, or to create a new wallet. diff --git a/doc/release-notes-15937.md b/doc/release-notes-15937.md new file mode 100644 index 000000000..1ab817b0e --- /dev/null +++ b/doc/release-notes-15937.md @@ -0,0 +1,15 @@ +Configuration +------------- + +Wallets created or loaded in the GUI will now be automatically loaded on +startup, so they don't need to be manually reloaded next time Bitcoin is +started. The list of wallets to load on startup is stored in +`\<datadir\>/settings.json` and augments any command line or `bitcoin.conf` +`-wallet=` settings that specify more wallets to load. Wallets that are +unloaded in the GUI get removed from the settings list so they won't load again +automatically next startup. (#19754) + +The `createwallet`, `loadwallet`, and `unloadwallet` RPCs now accept +`load_on_startup` options to modify the settings list. Unless these options are +explicitly set to true or false, the list is not modified, so the RPC methods +remain backwards compatible. (#15937) diff --git a/doc/release-notes-16378.md b/doc/release-notes-16378.md new file mode 100644 index 000000000..b006ea1a5 --- /dev/null +++ b/doc/release-notes-16378.md @@ -0,0 +1,5 @@ +RPC +--- +- A new `send` RPC with similar syntax to `walletcreatefundedpsbt`, including + support for coin selection and a custom fee rate. Using the new `send` method + is encouraged: `sendmany` and `sendtoaddress` may be deprecated in a future release. diff --git a/doc/release-notes-18244.md b/doc/release-notes-18244.md new file mode 100644 index 000000000..625fbaf7a --- /dev/null +++ b/doc/release-notes-18244.md @@ -0,0 +1,7 @@ +Updated RPCs +------------ + +- `fundrawtransaction` and `walletcreatefundedpsbt` when used with the `lockUnspents` + argument now lock manually selected coins, in addition to automatically selected + coins. Note that locked coins are never used in automatic coin selection, but + can still be manually selected. diff --git a/doc/release-notes-19405.md b/doc/release-notes-19405.md new file mode 100644 index 000000000..14f2a81c7 --- /dev/null +++ b/doc/release-notes-19405.md @@ -0,0 +1,12 @@ +## Updated RPCs + +- `getnetworkinfo` now returns two new fields, `connections_in` and + `connections_out`, that provide the number of inbound and outbound peer + connections. These new fields are in addition to the existing `connections` + field, which returns the total number of peer connections. (#19405) + +## CLI + +- The `connections` field of `bitcoin-cli -getinfo` is expanded to return a JSON + object with `in`, `out` and `total` numbers of peer connections. It previously + returned a single integer value for the total number of peer connections. (#19405) diff --git a/doc/release-notes-19671.md b/doc/release-notes-19671.md new file mode 100644 index 000000000..fb2d56d9a --- /dev/null +++ b/doc/release-notes-19671.md @@ -0,0 +1,6 @@ +Wallet +------ + +* The `-zapwallettxes` startup option has been removed and its functionality removed from the wallet. + This option was originally intended to allow for the fee bumping of transactions that did not + signal RBF. This functionality has been superseded with the abandon transaction feature. diff --git a/doc/release-notes-19731.md b/doc/release-notes-19731.md new file mode 100644 index 000000000..abe38e06a --- /dev/null +++ b/doc/release-notes-19731.md @@ -0,0 +1,6 @@ +Updated RPCs +------------ + +- The `getpeerinfo` RPC now has additional `last_block` and `last_transaction` + fields that return the UNIX epoch time of the last block and the last valid + transaction received from each peer. (#19731) diff --git a/doc/release-notes.md b/doc/release-notes.md index 23983dcd7..0ce2b32c6 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -102,6 +102,9 @@ will trigger BIP 125 (replace-by-fee) opt-in. (#11413) option `-deprecatedrpc=banscore` is used. The `banscore` field will be fully removed in the next major release. (#19469) +- The `testmempoolaccept` RPC returns `vsize` and a `fee` object with the `base` fee + if the transaction passes validation. (#19940) + - The `walletcreatefundedpsbt` RPC call will now fail with `Insufficient funds` when inputs are manually selected but are not enough to cover the outputs and fee. Additional inputs can automatically be added through the @@ -115,6 +118,10 @@ Changes to Wallet or GUI related RPCs can be found in the GUI or Wallet section New RPCs -------- +- The `getindexinfo` RPC returns the actively running indices of the node, + including their current sync status and height. It also accepts an `index_name` + to specify returning only the status of that index. (#19550) + Build System ------------ @@ -135,6 +142,10 @@ Updated settings in future releases. Refer to the help of the affected settings `-whitebind` and `-whitelist` for more details. (#19191) +- Netmasks that contain 1-bits after 0-bits (the 1-bits are not contiguous on + the left side, e.g. 255.0.255.255) are no longer accepted. They are invalid + according to RFC 4632. + Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. Tools and Utilities diff --git a/doc/tor.md b/doc/tor.md index 2c54e32f8..17807856e 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -1,6 +1,6 @@ # TOR SUPPORT IN BITCOIN -It is possible to run Bitcoin Core as a Tor hidden service, and connect to such services. +It is possible to run Bitcoin Core as a Tor onion service, and connect to such services. The following directions assume you have a Tor proxy running on port 9050. Many distributions default to having a SOCKS proxy listening on port 9050, but others may not. In particular, the Tor Browser Bundle defaults to listening on port 9150. See [Tor Project FAQ:TBBSocksPort](https://www.torproject.org/docs/faq.html.en#TBBSocksPort) for how to properly configure Tor. @@ -14,12 +14,12 @@ outgoing connections, but more is possible. -proxy=ip:port Set the proxy server. If SOCKS5 is selected (default), this proxy server will be used to try to reach .onion addresses as well. - -onion=ip:port Set the proxy server to use for Tor hidden services. You do not + -onion=ip:port Set the proxy server to use for Tor onion services. You do not need to set this if it's the same as -proxy. You can use -noonion - to explicitly disable access to hidden services. + to explicitly disable access to onion services. -listen When using -proxy, listening is disabled by default. If you want - to run a hidden service (see next section), you'll need to enable + to run an onion service (see next section), you'll need to enable it explicitly. -connect=X When behind a Tor proxy, you can specify .onion addresses instead @@ -94,11 +94,11 @@ for normal IPv4/IPv6 communication, use: ## 3. Automatically listen on Tor Starting with Tor version 0.2.7.1 it is possible, through Tor's control socket -API, to create and destroy 'ephemeral' hidden services programmatically. +API, to create and destroy 'ephemeral' onion services programmatically. Bitcoin Core has been updated to make use of this. This means that if Tor is running (and proper authentication has been configured), -Bitcoin Core automatically creates a hidden service to listen on. This will positively +Bitcoin Core automatically creates an onion service to listen on. This will positively affect the number of available .onion nodes. This new feature is enabled by default if Bitcoin Core is listening (`-listen`), and @@ -110,7 +110,7 @@ Connecting to Tor's control socket API requires one of two authentication method configured. It also requires the control socket to be enabled, e.g. put `ControlPort 9051` in `torrc` config file. For cookie authentication the user running bitcoind must have read access to the `CookieAuthFile` specified in Tor configuration. In some cases this is -preconfigured and the creation of a hidden service is automatic. If permission problems +preconfigured and the creation of an onion service is automatic. If permission problems are seen with `-debug=tor` they can be resolved by adding both the user running Tor and the user running bitcoind to the same group and setting permissions appropriately. On Debian-based systems the user running bitcoind can be added to the debian-tor group, @@ -127,8 +127,8 @@ in the tor configuration file. The hashed password can be obtained with the comm ## 4. Privacy recommendations -- Do not add anything but Bitcoin Core ports to the hidden service created in section 2. - If you run a web service too, create a new hidden service for that. +- Do not add anything but Bitcoin Core ports to the onion service created in section 2. + If you run a web service too, create a new onion service for that. Otherwise it is trivial to link them, which may reduce privacy. Hidden services created automatically (as in section 3) always have only one port open. diff --git a/doc/zmq.md b/doc/zmq.md index 3a1194de1..835dea23f 100644 --- a/doc/zmq.md +++ b/doc/zmq.md @@ -1,6 +1,6 @@ # Block and Transaction Broadcasting with ZeroMQ -[ZeroMQ](http://zeromq.org/) is a lightweight wrapper around TCP +[ZeroMQ](https://zeromq.org/) is a lightweight wrapper around TCP connections, inter-process communication, and shared-memory, providing various message-oriented semantics such as publish/subscribe, request/reply, and push/pull. @@ -39,8 +39,9 @@ For version information, see [dependencies.md](dependencies.md). Typically, it is packaged by distributions as something like *libzmq3-dev*. The C++ wrapper for ZeroMQ is *not* needed. -In order to run the example Python client scripts in contrib/ one must -also install *python3-zmq*, though this is not necessary for daemon +In order to run the example Python client scripts in the `contrib/zmq/` +directory, one must also install [PyZMQ](https://github.com/zeromq/pyzmq) +(generally with `pip install pyzmq`), though this is not necessary for daemon operation. ## Enabling @@ -98,6 +99,20 @@ ZMQ_SUBSCRIBE option set to one or either of these prefixes (for instance, just `hash`); without doing so will result in no messages arriving. Please see [`contrib/zmq/zmq_sub.py`](/contrib/zmq/zmq_sub.py) for a working example. +The ZMQ_PUB socket's ZMQ_TCP_KEEPALIVE option is enabled. This means that +the underlying SO_KEEPALIVE option is enabled when using a TCP transport. +The effective TCP keepalive values are managed through the underlying +operating system configuration and must be configured prior to connection establishment. + +For example, when running on GNU/Linux, one might use the following +to lower the keepalive setting to 10 minutes: + +sudo sysctl -w net.ipv4.tcp_keepalive_time=600 + +Setting the keepalive values appropriately for your operating environment may +improve connectivity in situations where long-lived connections are silently +dropped by network middle boxes. + ## Remarks From the perspective of bitcoind, the ZeroMQ socket is write-only; PUB diff --git a/src/Makefile.am b/src/Makefile.am index cd3cc9570..b23bf062c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -4,10 +4,11 @@ DIST_SUBDIRS = secp256k1 univalue -AM_LDFLAGS = $(PTHREAD_CFLAGS) $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) +AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) AM_LIBTOOLFLAGS = --preserve-dup-deps +PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) EXTRA_LIBRARIES = if EMBEDDED_UNIVALUE @@ -140,6 +141,7 @@ BITCOIN_CORE_H = \ httpserver.h \ index/base.h \ index/blockfilterindex.h \ + index/disktxpos.h \ index/txindex.h \ indirectmap.h \ init.h \ @@ -261,10 +263,10 @@ BITCOIN_CORE_H = \ walletinitinterface.h \ warnings.h \ zmq/zmqabstractnotifier.h \ - zmq/zmqconfig.h\ zmq/zmqnotificationinterface.h \ zmq/zmqpublishnotifier.h \ - zmq/zmqrpc.h + zmq/zmqrpc.h \ + zmq/zmqutil.h obj/build.h: FORCE @@ -343,7 +345,8 @@ libbitcoin_zmq_a_SOURCES = \ zmq/zmqabstractnotifier.cpp \ zmq/zmqnotificationinterface.cpp \ zmq/zmqpublishnotifier.cpp \ - zmq/zmqrpc.cpp + zmq/zmqrpc.cpp \ + zmq/zmqutil.cpp endif @@ -402,6 +405,8 @@ crypto_libbitcoin_crypto_base_a_SOURCES = \ crypto/sha1.h \ crypto/sha256.cpp \ crypto/sha256.h \ + crypto/sha3.cpp \ + crypto/sha3.h \ crypto/sha512.cpp \ crypto/sha512.h \ crypto/siphash.cpp \ @@ -563,7 +568,7 @@ nodist_libbitcoin_util_a_SOURCES = $(srcdir)/obj/build.h bitcoin_daemon_sources = bitcoind.cpp bitcoin_bin_cppflags = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) bitcoin_bin_cxxflags = $(AM_CXXFLAGS) $(PIE_FLAGS) -bitcoin_bin_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +bitcoin_bin_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) if TARGET_WINDOWS bitcoin_daemon_sources += bitcoind-res.rc @@ -600,7 +605,7 @@ bitcoin_node_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) bitcoin_cli_SOURCES = bitcoin-cli.cpp bitcoin_cli_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CFLAGS) bitcoin_cli_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -bitcoin_cli_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +bitcoin_cli_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) if TARGET_WINDOWS bitcoin_cli_SOURCES += bitcoin-cli-res.rc @@ -619,7 +624,7 @@ bitcoin_cli_LDADD += $(BOOST_LIBS) $(EVENT_LIBS) bitcoin_tx_SOURCES = bitcoin-tx.cpp bitcoin_tx_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) bitcoin_tx_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -bitcoin_tx_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +bitcoin_tx_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) if TARGET_WINDOWS bitcoin_tx_SOURCES += bitcoin-tx-res.rc @@ -676,12 +681,18 @@ CLEANFILES = $(EXTRA_LIBRARIES) CLEANFILES += *.gcda *.gcno CLEANFILES += compat/*.gcda compat/*.gcno CLEANFILES += consensus/*.gcda consensus/*.gcno +CLEANFILES += crc32c/src/*.gcda crc32c/src/*.gcno CLEANFILES += crypto/*.gcda crypto/*.gcno +CLEANFILES += index/*.gcda index/*.gcno +CLEANFILES += interfaces/*.gcda interfaces/*.gcno +CLEANFILES += node/*.gcda node/*.gcno CLEANFILES += policy/*.gcda policy/*.gcno CLEANFILES += primitives/*.gcda primitives/*.gcno +CLEANFILES += rpc/*.gcda rpc/*.gcno CLEANFILES += script/*.gcda script/*.gcno CLEANFILES += support/*.gcda support/*.gcno CLEANFILES += univalue/*.gcda univalue/*.gcno +CLEANFILES += util/*.gcda util/*.gcno CLEANFILES += wallet/*.gcda wallet/*.gcno CLEANFILES += wallet/test/*.gcda wallet/test/*.gcno CLEANFILES += zmq/*.gcda zmq/*.gcno diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index c224ca7bf..bd9143a38 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -75,7 +75,7 @@ bench_bench_bitcoin_SOURCES += bench/wallet_balance.cpp endif bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) -bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES) diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 848053e84..69ff0f025 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -322,7 +322,7 @@ endif bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) -bitcoin_qt_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +bitcoin_qt_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX qt_bitcoin_qt_CPPFLAGS = $(bitcoin_qt_cppflags) diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 8c47fabad..d300398b2 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -57,7 +57,7 @@ qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBIT $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) -qt_test_test_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +qt_test_test_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) qt_test_test_bitcoin_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) CLEAN_BITCOIN_QT_TEST = $(TEST_QT_MOC_CPP) qt/test/*.gcda qt/test/*.gcno diff --git a/src/Makefile.test.include b/src/Makefile.test.include index b961f8d5b..77e50cf22 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -70,6 +70,7 @@ FUZZ_TARGETS = \ test/fuzz/message \ test/fuzz/messageheader_deserialize \ test/fuzz/multiplication_overflow \ + test/fuzz/net \ test/fuzz/net_permissions \ test/fuzz/netaddr_deserialize \ test/fuzz/netaddress \ @@ -132,6 +133,8 @@ FUZZ_TARGETS = \ test/fuzz/script_sigcache \ test/fuzz/script_sign \ test/fuzz/scriptnum_ops \ + test/fuzz/secp256k1_ec_seckey_import_export_der \ + test/fuzz/secp256k1_ecdsa_signature_parse_der_lax \ test/fuzz/service_deserialize \ test/fuzz/signature_checker \ test/fuzz/snapshotmetadata_deserialize \ @@ -265,6 +268,7 @@ BITCOIN_TESTS =\ test/skiplist_tests.cpp \ test/streams_tests.cpp \ test/sync_tests.cpp \ + test/system_tests.cpp \ test/util_threadnames_tests.cpp \ test/timedata_tests.cpp \ test/torcontrol_tests.cpp \ @@ -311,7 +315,7 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_C test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) -test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static +test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) -static if ENABLE_ZMQ test_test_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) @@ -319,886 +323,906 @@ endif if ENABLE_FUZZ +FUZZ_SUITE_LDFLAGS_COMMON = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) + test_fuzz_addition_overflow_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_addition_overflow_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_addition_overflow_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_addition_overflow_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_addition_overflow_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_addition_overflow_SOURCES = test/fuzz/addition_overflow.cpp test_fuzz_addr_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDR_INFO_DESERIALIZE=1 test_fuzz_addr_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_addr_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_addr_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_addr_info_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_addr_info_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_addrdb_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_addrdb_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_addrdb_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_addrdb_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_addrdb_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_addrdb_SOURCES = test/fuzz/addrdb.cpp test_fuzz_address_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRESS_DESERIALIZE=1 test_fuzz_address_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_address_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_address_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_address_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_address_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_addrman_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRMAN_DESERIALIZE=1 test_fuzz_addrman_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_addrman_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_addrman_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_addrman_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_addrman_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_asmap_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_asmap_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_asmap_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_asmap_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_asmap_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_asmap_SOURCES = test/fuzz/asmap.cpp test_fuzz_asmap_direct_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_asmap_direct_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_asmap_direct_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_asmap_direct_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_asmap_direct_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_asmap_direct_SOURCES = test/fuzz/asmap_direct.cpp test_fuzz_autofile_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_autofile_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_autofile_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_autofile_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_autofile_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_autofile_SOURCES = test/fuzz/autofile.cpp test_fuzz_banentry_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBANENTRY_DESERIALIZE=1 test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_banentry_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_banentry_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_banentry_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_banman_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_banman_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_banman_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_banman_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_banman_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_banman_SOURCES = test/fuzz/banman.cpp test_fuzz_base_encode_decode_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_base_encode_decode_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_base_encode_decode_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_base_encode_decode_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_base_encode_decode_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_base_encode_decode_SOURCES = test/fuzz/base_encode_decode.cpp test_fuzz_bech32_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_bech32_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_bech32_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_bech32_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_bech32_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_bech32_SOURCES = test/fuzz/bech32.cpp test_fuzz_block_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_block_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_block_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_block_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_block_SOURCES = test/fuzz/block.cpp test_fuzz_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_DESERIALIZE=1 test_fuzz_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_block_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_block_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_block_file_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_FILE_INFO_DESERIALIZE=1 test_fuzz_block_file_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_block_file_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_block_file_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_file_info_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_block_file_info_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_block_filter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_FILTER_DESERIALIZE=1 test_fuzz_block_filter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_block_filter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_block_filter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_filter_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_block_filter_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_block_header_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_block_header_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_block_header_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_block_header_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_header_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_block_header_SOURCES = test/fuzz/block_header.cpp test_fuzz_block_header_and_short_txids_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_HEADER_AND_SHORT_TXIDS_DESERIALIZE=1 test_fuzz_block_header_and_short_txids_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_block_header_and_short_txids_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_block_header_and_short_txids_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_block_header_and_short_txids_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_block_header_and_short_txids_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_blockfilter_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_blockfilter_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockfilter_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blockfilter_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blockfilter_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_blockfilter_SOURCES = test/fuzz/blockfilter.cpp test_fuzz_blockheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKHEADER_DESERIALIZE=1 test_fuzz_blockheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blockheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blockheader_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_blockheader_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_blocklocator_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKLOCATOR_DESERIALIZE=1 test_fuzz_blocklocator_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocklocator_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blocklocator_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blocklocator_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_blocklocator_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_blockmerkleroot_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKMERKLEROOT=1 test_fuzz_blockmerkleroot_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockmerkleroot_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blockmerkleroot_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blockmerkleroot_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_blockmerkleroot_SOURCES = test/fuzz/deserialize.cpp test_fuzz_blocktransactions_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONS_DESERIALIZE=1 test_fuzz_blocktransactions_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocktransactions_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blocktransactions_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blocktransactions_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_blocktransactions_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_blocktransactionsrequest_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONSREQUEST_DESERIALIZE=1 test_fuzz_blocktransactionsrequest_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocktransactionsrequest_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blocktransactionsrequest_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blocktransactionsrequest_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_blocktransactionsrequest_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_blockundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKUNDO_DESERIALIZE=1 test_fuzz_blockundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blockundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_blockundo_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_blockundo_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_bloom_filter_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_bloom_filter_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_bloom_filter_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_bloom_filter_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_bloom_filter_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_bloom_filter_SOURCES = test/fuzz/bloom_filter.cpp test_fuzz_bloomfilter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOOMFILTER_DESERIALIZE=1 test_fuzz_bloomfilter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_bloomfilter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_bloomfilter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_bloomfilter_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_bloomfilter_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_buffered_file_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_buffered_file_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_buffered_file_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_buffered_file_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_buffered_file_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_buffered_file_SOURCES = test/fuzz/buffered_file.cpp test_fuzz_chain_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_chain_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_chain_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_chain_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_chain_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_chain_SOURCES = test/fuzz/chain.cpp test_fuzz_checkqueue_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_checkqueue_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_checkqueue_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_checkqueue_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_checkqueue_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_checkqueue_SOURCES = test/fuzz/checkqueue.cpp test_fuzz_coins_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DCOINS_DESERIALIZE=1 test_fuzz_coins_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_coins_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_coins_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_coins_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_coins_view_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_coins_view_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_coins_view_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_coins_view_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_coins_view_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_coins_view_SOURCES = test/fuzz/coins_view.cpp test_fuzz_crypto_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_crypto_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_crypto_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_crypto_SOURCES = test/fuzz/crypto.cpp test_fuzz_crypto_aes256_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_crypto_aes256_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_crypto_aes256_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_aes256_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_aes256_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_crypto_aes256_SOURCES = test/fuzz/crypto_aes256.cpp test_fuzz_crypto_aes256cbc_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_crypto_aes256cbc_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_crypto_aes256cbc_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_aes256cbc_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_aes256cbc_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_crypto_aes256cbc_SOURCES = test/fuzz/crypto_aes256cbc.cpp test_fuzz_crypto_chacha20_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_crypto_chacha20_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_crypto_chacha20_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_chacha20_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_chacha20_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_crypto_chacha20_SOURCES = test/fuzz/crypto_chacha20.cpp test_fuzz_crypto_chacha20_poly1305_aead_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_crypto_chacha20_poly1305_aead_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_crypto_chacha20_poly1305_aead_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_chacha20_poly1305_aead_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_chacha20_poly1305_aead_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_crypto_chacha20_poly1305_aead_SOURCES = test/fuzz/crypto_chacha20_poly1305_aead.cpp test_fuzz_crypto_common_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_crypto_common_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_crypto_common_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_common_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_common_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_crypto_common_SOURCES = test/fuzz/crypto_common.cpp test_fuzz_crypto_hkdf_hmac_sha256_l32_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_crypto_hkdf_hmac_sha256_l32_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_crypto_hkdf_hmac_sha256_l32_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_hkdf_hmac_sha256_l32_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_hkdf_hmac_sha256_l32_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_crypto_hkdf_hmac_sha256_l32_SOURCES = test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp test_fuzz_crypto_poly1305_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_crypto_poly1305_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_crypto_poly1305_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_crypto_poly1305_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_crypto_poly1305_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_crypto_poly1305_SOURCES = test/fuzz/crypto_poly1305.cpp test_fuzz_cuckoocache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_cuckoocache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_cuckoocache_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_cuckoocache_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_cuckoocache_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_cuckoocache_SOURCES = test/fuzz/cuckoocache.cpp test_fuzz_decode_tx_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_decode_tx_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_decode_tx_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_decode_tx_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_decode_tx_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_decode_tx_SOURCES = test/fuzz/decode_tx.cpp test_fuzz_descriptor_parse_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_descriptor_parse_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_descriptor_parse_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_descriptor_parse_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_descriptor_parse_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_descriptor_parse_SOURCES = test/fuzz/descriptor_parse.cpp test_fuzz_diskblockindex_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DDISKBLOCKINDEX_DESERIALIZE=1 test_fuzz_diskblockindex_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_diskblockindex_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_diskblockindex_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_diskblockindex_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_diskblockindex_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_eval_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_eval_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_eval_script_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_eval_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_eval_script_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_eval_script_SOURCES = test/fuzz/eval_script.cpp test_fuzz_fee_rate_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_fee_rate_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fee_rate_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_fee_rate_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_fee_rate_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_fee_rate_SOURCES = test/fuzz/fee_rate.cpp test_fuzz_fee_rate_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DFEE_RATE_DESERIALIZE=1 test_fuzz_fee_rate_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fee_rate_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_fee_rate_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_fee_rate_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_fee_rate_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_fees_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_fees_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fees_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_fees_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_fees_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_fees_SOURCES = test/fuzz/fees.cpp test_fuzz_flat_file_pos_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DFLAT_FILE_POS_DESERIALIZE=1 test_fuzz_flat_file_pos_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_flat_file_pos_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_flat_file_pos_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_flat_file_pos_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_flat_file_pos_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_flatfile_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_flatfile_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_flatfile_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_flatfile_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_flatfile_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_flatfile_SOURCES = test/fuzz/flatfile.cpp test_fuzz_float_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_float_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_float_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_float_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_float_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_float_SOURCES = test/fuzz/float.cpp test_fuzz_golomb_rice_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_golomb_rice_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_golomb_rice_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_golomb_rice_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_golomb_rice_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_golomb_rice_SOURCES = test/fuzz/golomb_rice.cpp test_fuzz_hex_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_hex_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_hex_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_hex_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_hex_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_hex_SOURCES = test/fuzz/hex.cpp test_fuzz_http_request_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_http_request_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_http_request_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_http_request_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_http_request_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_http_request_SOURCES = test/fuzz/http_request.cpp test_fuzz_integer_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_integer_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_integer_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_integer_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_integer_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_integer_SOURCES = test/fuzz/integer.cpp test_fuzz_inv_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DINV_DESERIALIZE=1 test_fuzz_inv_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_inv_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_inv_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_inv_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_inv_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_key_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_key_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_key_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_key_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_key_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_key_SOURCES = test/fuzz/key.cpp test_fuzz_key_io_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_key_io_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_key_io_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_key_io_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_key_io_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_key_io_SOURCES = test/fuzz/key_io.cpp test_fuzz_key_origin_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DKEY_ORIGIN_INFO_DESERIALIZE=1 test_fuzz_key_origin_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_key_origin_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_key_origin_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_key_origin_info_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_key_origin_info_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_kitchen_sink_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_kitchen_sink_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_kitchen_sink_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_kitchen_sink_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_kitchen_sink_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_kitchen_sink_SOURCES = test/fuzz/kitchen_sink.cpp test_fuzz_load_external_block_file_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_load_external_block_file_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_load_external_block_file_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_load_external_block_file_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_load_external_block_file_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_load_external_block_file_SOURCES = test/fuzz/load_external_block_file.cpp test_fuzz_locale_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_locale_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_locale_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_locale_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_locale_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_locale_SOURCES = test/fuzz/locale.cpp test_fuzz_merkle_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMERKLE_BLOCK_DESERIALIZE=1 test_fuzz_merkle_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_merkle_block_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_merkle_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_merkle_block_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_merkle_block_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_merkleblock_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_merkleblock_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_merkleblock_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_merkleblock_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_merkleblock_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_merkleblock_SOURCES = test/fuzz/merkleblock.cpp test_fuzz_message_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_message_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_message_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_message_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_message_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_message_SOURCES = test/fuzz/message.cpp test_fuzz_messageheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGEHEADER_DESERIALIZE=1 test_fuzz_messageheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_messageheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_messageheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_messageheader_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_messageheader_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_multiplication_overflow_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_multiplication_overflow_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_multiplication_overflow_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_multiplication_overflow_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_multiplication_overflow_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_multiplication_overflow_SOURCES = test/fuzz/multiplication_overflow.cpp +test_fuzz_net_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_net_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_net_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_net_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) +test_fuzz_net_SOURCES = test/fuzz/net.cpp + test_fuzz_net_permissions_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_net_permissions_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_net_permissions_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_net_permissions_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_net_permissions_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_net_permissions_SOURCES = test/fuzz/net_permissions.cpp test_fuzz_netaddr_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DNETADDR_DESERIALIZE=1 test_fuzz_netaddr_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_netaddr_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_netaddr_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_netaddr_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_netaddr_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_netaddress_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_netaddress_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_netaddress_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_netaddress_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_netaddress_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_netaddress_SOURCES = test/fuzz/netaddress.cpp test_fuzz_out_point_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DOUT_POINT_DESERIALIZE=1 test_fuzz_out_point_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_out_point_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_out_point_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_out_point_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_out_point_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_p2p_transport_deserializer_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_p2p_transport_deserializer_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_p2p_transport_deserializer_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_p2p_transport_deserializer_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_p2p_transport_deserializer_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_p2p_transport_deserializer_SOURCES = test/fuzz/p2p_transport_deserializer.cpp test_fuzz_parse_hd_keypath_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_parse_hd_keypath_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_parse_hd_keypath_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_parse_hd_keypath_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_hd_keypath_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_parse_hd_keypath_SOURCES = test/fuzz/parse_hd_keypath.cpp test_fuzz_parse_iso8601_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_parse_iso8601_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_parse_iso8601_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_parse_iso8601_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_iso8601_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_parse_iso8601_SOURCES = test/fuzz/parse_iso8601.cpp test_fuzz_parse_numbers_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_parse_numbers_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_parse_numbers_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_parse_numbers_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_numbers_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_parse_numbers_SOURCES = test/fuzz/parse_numbers.cpp test_fuzz_parse_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_parse_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_parse_script_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_parse_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_script_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_parse_script_SOURCES = test/fuzz/parse_script.cpp test_fuzz_parse_univalue_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_parse_univalue_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_parse_univalue_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_parse_univalue_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_parse_univalue_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_parse_univalue_SOURCES = test/fuzz/parse_univalue.cpp test_fuzz_prevector_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_prevector_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_prevector_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_prevector_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_prevector_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_prevector_SOURCES = test/fuzz/prevector.cpp test_fuzz_partial_merkle_tree_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPARTIAL_MERKLE_TREE_DESERIALIZE=1 test_fuzz_partial_merkle_tree_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_partial_merkle_tree_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_partial_merkle_tree_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_partial_merkle_tree_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_partial_merkle_tree_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_partially_signed_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPARTIALLY_SIGNED_TRANSACTION_DESERIALIZE=1 test_fuzz_partially_signed_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_partially_signed_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_partially_signed_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_partially_signed_transaction_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_partially_signed_transaction_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_policy_estimator_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_policy_estimator_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_policy_estimator_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_policy_estimator_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_policy_estimator_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_policy_estimator_SOURCES = test/fuzz/policy_estimator.cpp test_fuzz_policy_estimator_io_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_policy_estimator_io_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_policy_estimator_io_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_policy_estimator_io_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_policy_estimator_io_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_policy_estimator_io_SOURCES = test/fuzz/policy_estimator_io.cpp test_fuzz_pow_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_pow_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_pow_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_pow_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_pow_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_pow_SOURCES = test/fuzz/pow.cpp test_fuzz_prefilled_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPREFILLED_TRANSACTION_DESERIALIZE=1 test_fuzz_prefilled_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_prefilled_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_prefilled_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_prefilled_transaction_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_prefilled_transaction_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_primitives_transaction_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_primitives_transaction_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_primitives_transaction_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_primitives_transaction_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_primitives_transaction_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_primitives_transaction_SOURCES = test/fuzz/primitives_transaction.cpp test_fuzz_process_messages_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_process_messages_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_messages_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_messages_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_messages_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_messages_SOURCES = test/fuzz/process_messages.cpp test_fuzz_process_message_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_process_message_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_addr_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=addr test_fuzz_process_message_addr_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_addr_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_addr_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_addr_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_addr_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_block_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=block test_fuzz_process_message_block_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_block_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_block_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_block_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_block_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_blocktxn_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=blocktxn test_fuzz_process_message_blocktxn_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_blocktxn_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_blocktxn_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_blocktxn_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_blocktxn_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_cmpctblock_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=cmpctblock test_fuzz_process_message_cmpctblock_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_cmpctblock_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_cmpctblock_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_cmpctblock_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_cmpctblock_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_feefilter_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=feefilter test_fuzz_process_message_feefilter_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_feefilter_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_feefilter_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_feefilter_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_feefilter_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_filteradd_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=filteradd test_fuzz_process_message_filteradd_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_filteradd_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_filteradd_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_filteradd_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_filteradd_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_filterclear_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=filterclear test_fuzz_process_message_filterclear_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_filterclear_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_filterclear_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_filterclear_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_filterclear_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_filterload_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=filterload test_fuzz_process_message_filterload_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_filterload_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_filterload_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_filterload_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_filterload_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_getaddr_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=getaddr test_fuzz_process_message_getaddr_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_getaddr_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_getaddr_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_getaddr_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_getaddr_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_getblocks_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=getblocks test_fuzz_process_message_getblocks_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_getblocks_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_getblocks_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_getblocks_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_getblocks_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_getblocktxn_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=getblocktxn test_fuzz_process_message_getblocktxn_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_getblocktxn_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_getblocktxn_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_getblocktxn_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_getblocktxn_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_getdata_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=getdata test_fuzz_process_message_getdata_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_getdata_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_getdata_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_getdata_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_getdata_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_getheaders_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=getheaders test_fuzz_process_message_getheaders_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_getheaders_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_getheaders_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_getheaders_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_getheaders_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_headers_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=headers test_fuzz_process_message_headers_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_headers_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_headers_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_headers_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_headers_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_inv_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=inv test_fuzz_process_message_inv_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_inv_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_inv_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_inv_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_inv_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_mempool_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=mempool test_fuzz_process_message_mempool_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_mempool_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_mempool_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_mempool_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_mempool_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_notfound_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=notfound test_fuzz_process_message_notfound_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_notfound_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_notfound_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_notfound_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_notfound_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_ping_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=ping test_fuzz_process_message_ping_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_ping_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_ping_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_ping_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_ping_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_pong_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=pong test_fuzz_process_message_pong_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_pong_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_pong_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_pong_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_pong_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_sendcmpct_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=sendcmpct test_fuzz_process_message_sendcmpct_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_sendcmpct_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_sendcmpct_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_sendcmpct_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_sendcmpct_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_sendheaders_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=sendheaders test_fuzz_process_message_sendheaders_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_sendheaders_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_sendheaders_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_sendheaders_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_sendheaders_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_tx_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=tx test_fuzz_process_message_tx_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_tx_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_tx_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_tx_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_tx_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_verack_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=verack test_fuzz_process_message_verack_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_verack_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_verack_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_verack_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_verack_SOURCES = test/fuzz/process_message.cpp test_fuzz_process_message_version_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGE_TYPE=version test_fuzz_process_message_version_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_version_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_process_message_version_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_message_version_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_process_message_version_SOURCES = test/fuzz/process_message.cpp test_fuzz_protocol_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_protocol_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_protocol_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_protocol_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_protocol_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_protocol_SOURCES = test/fuzz/protocol.cpp test_fuzz_psbt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_psbt_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_psbt_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_psbt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_psbt_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_psbt_SOURCES = test/fuzz/psbt.cpp test_fuzz_psbt_input_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPSBT_INPUT_DESERIALIZE=1 test_fuzz_psbt_input_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_psbt_input_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_psbt_input_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_psbt_input_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_psbt_input_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_psbt_output_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPSBT_OUTPUT_DESERIALIZE=1 test_fuzz_psbt_output_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_psbt_output_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_psbt_output_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_psbt_output_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_psbt_output_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_pub_key_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPUB_KEY_DESERIALIZE=1 test_fuzz_pub_key_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_pub_key_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_pub_key_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_pub_key_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_pub_key_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_random_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_random_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_random_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_random_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_random_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_random_SOURCES = test/fuzz/random.cpp test_fuzz_rbf_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_rbf_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_rbf_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_rbf_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_rbf_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_rbf_SOURCES = test/fuzz/rbf.cpp test_fuzz_rolling_bloom_filter_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_rolling_bloom_filter_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_rolling_bloom_filter_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_rolling_bloom_filter_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_rolling_bloom_filter_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_rolling_bloom_filter_SOURCES = test/fuzz/rolling_bloom_filter.cpp test_fuzz_script_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_script_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_script_SOURCES = test/fuzz/script.cpp test_fuzz_script_bitcoin_consensus_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_bitcoin_consensus_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_script_bitcoin_consensus_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_bitcoin_consensus_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_bitcoin_consensus_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_script_bitcoin_consensus_SOURCES = test/fuzz/script_bitcoin_consensus.cpp test_fuzz_script_descriptor_cache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_descriptor_cache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_script_descriptor_cache_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_descriptor_cache_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_descriptor_cache_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_script_descriptor_cache_SOURCES = test/fuzz/script_descriptor_cache.cpp test_fuzz_script_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSCRIPT_DESERIALIZE=1 test_fuzz_script_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_script_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_script_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_script_flags_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_flags_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_script_flags_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_flags_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_flags_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_script_flags_SOURCES = test/fuzz/script_flags.cpp test_fuzz_script_interpreter_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_interpreter_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_script_interpreter_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_interpreter_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_interpreter_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_script_interpreter_SOURCES = test/fuzz/script_interpreter.cpp test_fuzz_script_ops_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_ops_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_script_ops_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_ops_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_ops_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_script_ops_SOURCES = test/fuzz/script_ops.cpp test_fuzz_script_sigcache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_sigcache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_script_sigcache_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_sigcache_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_sigcache_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_script_sigcache_SOURCES = test/fuzz/script_sigcache.cpp test_fuzz_script_sign_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_script_sign_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_script_sign_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_script_sign_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_sign_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_script_sign_SOURCES = test/fuzz/script_sign.cpp test_fuzz_scriptnum_ops_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_scriptnum_ops_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_scriptnum_ops_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_scriptnum_ops_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_scriptnum_ops_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_scriptnum_ops_SOURCES = test/fuzz/scriptnum_ops.cpp +test_fuzz_secp256k1_ec_seckey_import_export_der_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_secp256k1_ec_seckey_import_export_der_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_secp256k1_ec_seckey_import_export_der_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_secp256k1_ec_seckey_import_export_der_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) +test_fuzz_secp256k1_ec_seckey_import_export_der_SOURCES = test/fuzz/secp256k1_ec_seckey_import_export_der.cpp + +test_fuzz_secp256k1_ecdsa_signature_parse_der_lax_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_secp256k1_ecdsa_signature_parse_der_lax_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_secp256k1_ecdsa_signature_parse_der_lax_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_secp256k1_ecdsa_signature_parse_der_lax_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) +test_fuzz_secp256k1_ecdsa_signature_parse_der_lax_SOURCES = test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp + test_fuzz_service_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSERVICE_DESERIALIZE=1 test_fuzz_service_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_service_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_service_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_service_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_service_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_signature_checker_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_signature_checker_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_signature_checker_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_signature_checker_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_signature_checker_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_signature_checker_SOURCES = test/fuzz/signature_checker.cpp test_fuzz_snapshotmetadata_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSNAPSHOTMETADATA_DESERIALIZE=1 test_fuzz_snapshotmetadata_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_snapshotmetadata_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_snapshotmetadata_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_snapshotmetadata_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_snapshotmetadata_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_span_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_span_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_span_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_span_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_span_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_span_SOURCES = test/fuzz/span.cpp test_fuzz_spanparsing_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_spanparsing_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_spanparsing_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_spanparsing_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_spanparsing_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_spanparsing_SOURCES = test/fuzz/spanparsing.cpp test_fuzz_string_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_string_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_string_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_string_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_string_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_string_SOURCES = test/fuzz/string.cpp test_fuzz_strprintf_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_strprintf_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_strprintf_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_strprintf_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_strprintf_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_strprintf_SOURCES = test/fuzz/strprintf.cpp test_fuzz_sub_net_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSUB_NET_DESERIALIZE=1 test_fuzz_sub_net_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_sub_net_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_sub_net_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_sub_net_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_sub_net_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_system_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_system_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_system_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_system_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_system_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_system_SOURCES = test/fuzz/system.cpp test_fuzz_timedata_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_timedata_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_timedata_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_timedata_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_timedata_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_timedata_SOURCES = test/fuzz/timedata.cpp test_fuzz_transaction_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_transaction_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_transaction_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_transaction_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_transaction_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_transaction_SOURCES = test/fuzz/transaction.cpp test_fuzz_tx_in_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_tx_in_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_tx_in_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_tx_in_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_tx_in_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_tx_in_SOURCES = test/fuzz/tx_in.cpp test_fuzz_tx_in_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTX_IN_DESERIALIZE=1 test_fuzz_tx_in_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_tx_in_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_tx_in_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_tx_in_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_tx_in_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_tx_out_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_tx_out_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_tx_out_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_tx_out_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_tx_out_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_tx_out_SOURCES = test/fuzz/tx_out.cpp test_fuzz_txoutcompressor_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXOUTCOMPRESSOR_DESERIALIZE=1 test_fuzz_txoutcompressor_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_txoutcompressor_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_txoutcompressor_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_txoutcompressor_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_txoutcompressor_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_txundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXUNDO_DESERIALIZE=1 test_fuzz_txundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_txundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_txundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_txundo_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_txundo_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_uint160_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DUINT160_DESERIALIZE=1 test_fuzz_uint160_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_uint160_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_uint160_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_uint160_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_uint160_deserialize_SOURCES = test/fuzz/deserialize.cpp test_fuzz_uint256_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DUINT256_DESERIALIZE=1 test_fuzz_uint256_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_uint256_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_uint256_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_uint256_deserialize_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) test_fuzz_uint256_deserialize_SOURCES = test/fuzz/deserialize.cpp endif # ENABLE_FUZZ @@ -1207,7 +1231,7 @@ nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) $(BITCOIN_TESTS): $(GENERATED_TEST_FILES) -CLEAN_BITCOIN_TEST = test/*.gcda test/*.gcno $(GENERATED_TEST_FILES) $(BITCOIN_TESTS:=.log) +CLEAN_BITCOIN_TEST = test/*.gcda test/*.gcno test/fuzz/*.gcda test/fuzz/*.gcno test/util/*.gcda test/util/*.gcno $(GENERATED_TEST_FILES) $(BITCOIN_TESTS:=.log) CLEANFILES += $(CLEAN_BITCOIN_TEST) diff --git a/src/addrman.cpp b/src/addrman.cpp index 7aba340d9..7636c6bad 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -479,11 +479,15 @@ int CAddrMan::Check_() } #endif -void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr) +void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size_t max_pct) { - unsigned int nNodes = ADDRMAN_GETADDR_MAX_PCT * vRandom.size() / 100; - if (nNodes > ADDRMAN_GETADDR_MAX) - nNodes = ADDRMAN_GETADDR_MAX; + size_t nNodes = vRandom.size(); + if (max_pct != 0) { + nNodes = max_pct * nNodes / 100; + } + if (max_addresses != 0) { + nNodes = std::min(nNodes, max_addresses); + } // gather a list of random nodes, skipping those of low quality for (unsigned int n = 0; n < vRandom.size(); n++) { diff --git a/src/addrman.h b/src/addrman.h index 9e742339d..ca045b91c 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -153,12 +153,6 @@ public: //! how recent a successful connection should be before we allow an address to be evicted from tried #define ADDRMAN_REPLACEMENT_HOURS 4 -//! the maximum percentage of nodes to return in a getaddr call -#define ADDRMAN_GETADDR_MAX_PCT 23 - -//! the maximum number of nodes to return in a getaddr call -#define ADDRMAN_GETADDR_MAX 1000 - //! Convenience #define ADDRMAN_TRIED_BUCKET_COUNT (1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2) #define ADDRMAN_NEW_BUCKET_COUNT (1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2) @@ -261,7 +255,7 @@ protected: #endif //! Select several addresses at once. - void GetAddr_(std::vector<CAddress> &vAddr) EXCLUSIVE_LOCKS_REQUIRED(cs); + void GetAddr_(std::vector<CAddress> &vAddr, size_t max_addresses, size_t max_pct) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Mark an entry as currently-connected-to. void Connected_(const CService &addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); @@ -638,13 +632,13 @@ public: } //! Return a bunch of addresses, selected at random. - std::vector<CAddress> GetAddr() + std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct) { Check(); std::vector<CAddress> vAddr; { LOCK(cs); - GetAddr_(vAddr); + GetAddr_(vAddr, max_addresses, max_pct); } Check(); return vAddr; diff --git a/src/base58.cpp b/src/base58.cpp index 9b2946e7a..0dc604414 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -35,7 +35,7 @@ static const int8_t mapBase58[256] = { -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, }; -bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch, int max_ret_len) +NODISCARD static bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch, int max_ret_len) { // Skip leading spaces. while (*psz && IsSpace(*psz)) @@ -84,21 +84,21 @@ bool DecodeBase58(const char* psz, std::vector<unsigned char>& vch, int max_ret_ return true; } -std::string EncodeBase58(const unsigned char* pbegin, const unsigned char* pend) +std::string EncodeBase58(Span<const unsigned char> input) { // Skip & count leading zeroes. int zeroes = 0; int length = 0; - while (pbegin != pend && *pbegin == 0) { - pbegin++; + while (input.size() > 0 && input[0] == 0) { + input = input.subspan(1); zeroes++; } // Allocate enough space in big-endian base58 representation. - int size = (pend - pbegin) * 138 / 100 + 1; // log(256) / log(58), rounded up. + int size = input.size() * 138 / 100 + 1; // log(256) / log(58), rounded up. std::vector<unsigned char> b58(size); // Process the bytes. - while (pbegin != pend) { - int carry = *pbegin; + while (input.size() > 0) { + int carry = input[0]; int i = 0; // Apply "b58 = b58 * 256 + ch". for (std::vector<unsigned char>::reverse_iterator it = b58.rbegin(); (carry != 0 || i < length) && (it != b58.rend()); it++, i++) { @@ -109,7 +109,7 @@ std::string EncodeBase58(const unsigned char* pbegin, const unsigned char* pend) assert(carry == 0); length = i; - pbegin++; + input = input.subspan(1); } // Skip leading zeroes in base58 result. std::vector<unsigned char>::iterator it = b58.begin() + (size - length); @@ -124,11 +124,6 @@ std::string EncodeBase58(const unsigned char* pbegin, const unsigned char* pend) return str; } -std::string EncodeBase58(const std::vector<unsigned char>& vch) -{ - return EncodeBase58(vch.data(), vch.data() + vch.size()); -} - bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret_len) { if (!ValidAsCString(str)) { @@ -137,16 +132,16 @@ bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet, in return DecodeBase58(str.c_str(), vchRet, max_ret_len); } -std::string EncodeBase58Check(const std::vector<unsigned char>& vchIn) +std::string EncodeBase58Check(Span<const unsigned char> input) { // add 4-byte hash check to the end - std::vector<unsigned char> vch(vchIn); + std::vector<unsigned char> vch(input.begin(), input.end()); uint256 hash = Hash(vch); vch.insert(vch.end(), (unsigned char*)&hash, (unsigned char*)&hash + 4); return EncodeBase58(vch); } -bool DecodeBase58Check(const char* psz, std::vector<unsigned char>& vchRet, int max_ret_len) +NODISCARD static bool DecodeBase58Check(const char* psz, std::vector<unsigned char>& vchRet, int max_ret_len) { if (!DecodeBase58(psz, vchRet, max_ret_len > std::numeric_limits<int>::max() - 4 ? std::numeric_limits<int>::max() : max_ret_len + 4) || (vchRet.size() < 4)) { diff --git a/src/base58.h b/src/base58.h index 042ad671d..468c3e258 100644 --- a/src/base58.h +++ b/src/base58.h @@ -15,27 +15,15 @@ #define BITCOIN_BASE58_H #include <attributes.h> +#include <span.h> #include <string> #include <vector> /** - * Encode a byte sequence as a base58-encoded string. - * pbegin and pend cannot be nullptr, unless both are. + * Encode a byte span as a base58-encoded string */ -std::string EncodeBase58(const unsigned char* pbegin, const unsigned char* pend); - -/** - * Encode a byte vector as a base58-encoded string - */ -std::string EncodeBase58(const std::vector<unsigned char>& vch); - -/** - * Decode a base58-encoded string (psz) into a byte vector (vchRet). - * return true if decoding is successful. - * psz cannot be nullptr. - */ -NODISCARD bool DecodeBase58(const char* psz, std::vector<unsigned char>& vchRet, int max_ret_len); +std::string EncodeBase58(Span<const unsigned char> input); /** * Decode a base58-encoded string (str) into a byte vector (vchRet). @@ -44,15 +32,9 @@ NODISCARD bool DecodeBase58(const char* psz, std::vector<unsigned char>& vchRet, NODISCARD bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret_len); /** - * Encode a byte vector into a base58-encoded string, including checksum - */ -std::string EncodeBase58Check(const std::vector<unsigned char>& vchIn); - -/** - * Decode a base58-encoded string (psz) that includes a checksum into a byte - * vector (vchRet), return true if decoding is successful + * Encode a byte span into a base58-encoded string, including checksum */ -NODISCARD bool DecodeBase58Check(const char* psz, std::vector<unsigned char>& vchRet, int max_ret_len); +std::string EncodeBase58Check(Span<const unsigned char> input); /** * Decode a base58-encoded string (str) that includes a checksum into a byte diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index 26d934076..ebdad5a4b 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -98,7 +98,7 @@ static void AddrManGetAddr(benchmark::Bench& bench) FillAddrMan(addrman); bench.run([&] { - const auto& addresses = addrman.GetAddr(); + const auto& addresses = addrman.GetAddr(2500, 23); assert(addresses.size() > 0); }); } diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index 00544cba3..18cb5de19 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -20,7 +20,7 @@ static void Base58Encode(benchmark::Bench& bench) } }; bench.batch(buff.size()).unit("byte").run([&] { - EncodeBase58(buff.data(), buff.data() + buff.size()); + EncodeBase58(buff); }); } @@ -34,10 +34,8 @@ static void Base58CheckEncode(benchmark::Bench& bench) 200, 24 } }; - std::vector<unsigned char> vch; - vch.assign(buff.begin(), buff.end()); bench.batch(buff.size()).unit("byte").run([&] { - EncodeBase58Check(vch); + EncodeBase58Check(buff); }); } diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 01466d0b6..012057e79 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -70,7 +70,10 @@ void benchmark::BenchRunner::RunAll(const Args& args) } std::cout << bench.complexityBigO() << std::endl; } - benchmarkResults.push_back(bench.results().back()); + + if (!bench.results().empty()) { + benchmarkResults.push_back(bench.results().back()); + } } GenerateTemplateResults(benchmarkResults, args.output_csv, "# Benchmark, evals, iterations, total, min, max, median\n" diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 19d7bc0db..ffa772d8c 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -14,8 +14,6 @@ #include <vector> - -static const int MIN_CORES = 2; static const size_t BATCHES = 101; static const size_t BATCH_SIZE = 30; static const int PREVECTOR_SIZE = 28; @@ -26,6 +24,9 @@ static const unsigned int QUEUE_BATCH_SIZE = 128; // and there is a little bit of work done between calls to Add. static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) { + // We shouldn't ever be running with the checkqueue on a single core machine. + if (GetNumCores() <= 1) return; + const ECCVerifyHandle verify_handle; ECC_Start(); @@ -44,7 +45,9 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) }; CCheckQueue<PrevectorJob> queue {QUEUE_BATCH_SIZE}; boost::thread_group tg; - for (auto x = 0; x < std::max(MIN_CORES, GetNumCores()); ++x) { + // The main thread should be counted to prevent thread oversubscription, and + // to decrease the variance of benchmark results. + for (auto x = 0; x < GetNumCores() - 1; ++x) { tg.create_thread([&]{queue.Thread();}); } diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 3a71a6ca0..99aafd8df 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -31,7 +31,7 @@ static void CoinSelection(benchmark::Bench& bench) { NodeContext node; auto chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); wallet.SetupLegacyScriptPubKeyMan(); std::vector<std::unique_ptr<CWalletTx>> wtxs; LOCK(wallet.cs_wallet); @@ -65,7 +65,7 @@ static void CoinSelection(benchmark::Bench& bench) typedef std::set<CInputCoin> CoinSet; static NodeContext testNode; static auto testChain = interfaces::MakeChain(testNode); -static CWallet testWallet(testChain.get(), WalletLocation(), CreateDummyWalletDatabase()); +static CWallet testWallet(testChain.get(), "", CreateDummyWalletDatabase()); std::vector<std::unique_ptr<CWalletTx>> wtxn; // Copied from src/wallet/test/coinselector_tests.cpp diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 36be86bcc..65d16d47d 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -7,6 +7,7 @@ #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> +#include <crypto/sha3.h> #include <crypto/sha512.h> #include <crypto/siphash.h> #include <hash.h> @@ -43,6 +44,15 @@ static void SHA256(benchmark::Bench& bench) }); } +static void SHA3_256_1M(benchmark::Bench& bench) +{ + uint8_t hash[SHA3_256::OUTPUT_SIZE]; + std::vector<uint8_t> in(BUFFER_SIZE,0); + bench.batch(in.size()).unit("byte").run([&] { + SHA3_256().Write(in).Finalize(hash); + }); +} + static void SHA256_32b(benchmark::Bench& bench) { std::vector<uint8_t> in(32,0); @@ -99,6 +109,7 @@ BENCHMARK(RIPEMD160); BENCHMARK(SHA1); BENCHMARK(SHA256); BENCHMARK(SHA512); +BENCHMARK(SHA3_256_1M); BENCHMARK(SHA256_32b); BENCHMARK(SipHash_32b); diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index e16182b48..b3b73284d 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -26,7 +26,7 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b NodeContext node; std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(node); - CWallet wallet{chain.get(), WalletLocation(), CreateMockWalletDatabase()}; + CWallet wallet{chain.get(), "", CreateMockWalletDatabase()}; { wallet.SetupLegacyScriptPubKeyMan(); bool first_run; diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index cf52b710c..198cd5eb5 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -39,6 +39,8 @@ static const char DEFAULT_RPCCONNECT[] = "127.0.0.1"; static const int DEFAULT_HTTP_CLIENT_TIMEOUT=900; static const bool DEFAULT_NAMED=false; static const int CONTINUE_EXECUTION=-1; +static const std::string ONION{".onion"}; +static const size_t ONION_LEN{ONION.size()}; /** Default number of blocks to generate for RPC generatetoaddress. */ static const std::string DEFAULT_NBLOCKS = "1"; @@ -56,6 +58,8 @@ static void SetupCliArgs(ArgsManager& argsman) argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-generate", strprintf("Generate blocks immediately, equivalent to RPC generatenewaddress followed by RPC generatetoaddress. Optional positional integer arguments are number of blocks to generate (default: %s) and maximum iterations to try (default: %s), equivalent to RPC generatetoaddress nblocks and maxtries arguments. Example: bitcoin-cli -generate 4 1000", DEFAULT_NBLOCKS, DEFAULT_MAX_TRIES), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-getinfo", "Get general information from the remote server. Note that unlike server-side RPC calls, the results of -getinfo is the result of multiple non-atomic requests. Some entries in the result may represent results from different states (e.g. wallet balance may be as of a different block from the chain state reported)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-netinfo", "Get network peer connection information from the remote server. An optional integer argument from 0 to 4 can be passed for different peers listings (default: 0).", ArgsManager::ALLOW_INT, OptionsCategory::OPTIONS); + SetupChainParamsBaseOptions(argsman); argsman.AddArg("-named", strprintf("Pass named instead of positional arguments (default: %s)", DEFAULT_NAMED), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-rpcclienttimeout=<n>", strprintf("Timeout in seconds during HTTP requests, or 0 for no timeout. (default: %d)", DEFAULT_HTTP_CLIENT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -271,7 +275,13 @@ public: result.pushKV("headers", batch[ID_BLOCKCHAININFO]["result"]["headers"]); result.pushKV("verificationprogress", batch[ID_BLOCKCHAININFO]["result"]["verificationprogress"]); result.pushKV("timeoffset", batch[ID_NETWORKINFO]["result"]["timeoffset"]); - result.pushKV("connections", batch[ID_NETWORKINFO]["result"]["connections"]); + + UniValue connections(UniValue::VOBJ); + connections.pushKV("in", batch[ID_NETWORKINFO]["result"]["connections_in"]); + connections.pushKV("out", batch[ID_NETWORKINFO]["result"]["connections_out"]); + connections.pushKV("total", batch[ID_NETWORKINFO]["result"]["connections"]); + result.pushKV("connections", connections); + result.pushKV("proxy", batch[ID_NETWORKINFO]["result"]["networks"][0]["proxy"]); result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); result.pushKV("chain", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"])); @@ -291,6 +301,211 @@ public: } }; +/** Process netinfo requests */ +class NetinfoRequestHandler : public BaseRequestHandler +{ +private: + bool IsAddrIPv6(const std::string& addr) const + { + return !addr.empty() && addr.front() == '['; + } + bool IsInboundOnion(const std::string& addr_local, int mapped_as) const + { + return mapped_as == 0 && addr_local.find(ONION) != std::string::npos; + } + bool IsOutboundOnion(const std::string& addr, int mapped_as) const + { + const size_t addr_len{addr.size()}; + const size_t onion_pos{addr.rfind(ONION)}; + return mapped_as == 0 && onion_pos != std::string::npos && addr_len > ONION_LEN && + (onion_pos == addr_len - ONION_LEN || onion_pos == addr.find_last_of(":") - ONION_LEN); + } + uint8_t m_details_level{0}; //!< Optional user-supplied arg to set dashboard details level + bool DetailsRequested() const { return m_details_level > 0 && m_details_level < 5; } + bool IsAddressSelected() const { return m_details_level == 2 || m_details_level == 4; } + bool IsVersionSelected() const { return m_details_level == 3 || m_details_level == 4; } + enum struct NetType { + ipv4, + ipv6, + onion, + }; + struct Peer { + int id; + int mapped_as; + int version; + int64_t conn_time; + int64_t last_blck; + int64_t last_recv; + int64_t last_send; + int64_t last_trxn; + double min_ping; + double ping; + std::string addr; + std::string sub_version; + NetType net_type; + bool is_block_relay; + bool is_outbound; + bool operator<(const Peer& rhs) const { return std::tie(is_outbound, min_ping) < std::tie(rhs.is_outbound, rhs.min_ping); } + }; + std::string NetTypeEnumToString(NetType t) + { + switch (t) { + case NetType::ipv4: return "ipv4"; + case NetType::ipv6: return "ipv6"; + case NetType::onion: return "onion"; + } // no default case, so the compiler can warn about missing cases + assert(false); + } + std::string ChainToString() const + { + if (gArgs.GetChainName() == CBaseChainParams::TESTNET) return " testnet"; + if (gArgs.GetChainName() == CBaseChainParams::REGTEST) return " regtest"; + return ""; + } +public: + const int ID_PEERINFO = 0; + const int ID_NETWORKINFO = 1; + + UniValue PrepareRequest(const std::string& method, const std::vector<std::string>& args) override + { + if (!args.empty()) { + uint8_t n{0}; + if (ParseUInt8(args.at(0), &n)) { + m_details_level = n; + } + } + UniValue result(UniValue::VARR); + result.push_back(JSONRPCRequestObj("getpeerinfo", NullUniValue, ID_PEERINFO)); + result.push_back(JSONRPCRequestObj("getnetworkinfo", NullUniValue, ID_NETWORKINFO)); + return result; + } + + UniValue ProcessReply(const UniValue& batch_in) override + { + const std::vector<UniValue> batch{JSONRPCProcessBatchReply(batch_in)}; + if (!batch[ID_PEERINFO]["error"].isNull()) return batch[ID_PEERINFO]; + if (!batch[ID_NETWORKINFO]["error"].isNull()) return batch[ID_NETWORKINFO]; + + const UniValue& networkinfo{batch[ID_NETWORKINFO]["result"]}; + if (networkinfo["version"].get_int() < 209900) { + throw std::runtime_error("-netinfo requires bitcoind server to be running v0.21.0 and up"); + } + + // Count peer connection totals, and if DetailsRequested(), store peer data in a vector of structs. + const int64_t time_now{GetSystemTimeInSeconds()}; + int ipv4_i{0}, ipv6_i{0}, onion_i{0}, block_relay_i{0}, total_i{0}; // inbound conn counters + int ipv4_o{0}, ipv6_o{0}, onion_o{0}, block_relay_o{0}, total_o{0}; // outbound conn counters + size_t max_peer_id_length{2}, max_addr_length{0}; + bool is_asmap_on{false}; + std::vector<Peer> peers; + const UniValue& getpeerinfo{batch[ID_PEERINFO]["result"]}; + + for (const UniValue& peer : getpeerinfo.getValues()) { + const std::string addr{peer["addr"].get_str()}; + const std::string addr_local{peer["addrlocal"].isNull() ? "" : peer["addrlocal"].get_str()}; + const int mapped_as{peer["mapped_as"].isNull() ? 0 : peer["mapped_as"].get_int()}; + const bool is_block_relay{!peer["relaytxes"].get_bool()}; + const bool is_inbound{peer["inbound"].get_bool()}; + NetType net_type{NetType::ipv4}; + if (is_inbound) { + if (IsAddrIPv6(addr)) { + net_type = NetType::ipv6; + ++ipv6_i; + } else if (IsInboundOnion(addr_local, mapped_as)) { + net_type = NetType::onion; + ++onion_i; + } else { + ++ipv4_i; + } + if (is_block_relay) ++block_relay_i; + } else { + if (IsAddrIPv6(addr)) { + net_type = NetType::ipv6; + ++ipv6_o; + } else if (IsOutboundOnion(addr, mapped_as)) { + net_type = NetType::onion; + ++onion_o; + } else { + ++ipv4_o; + } + if (is_block_relay) ++block_relay_o; + } + if (DetailsRequested()) { + // Push data for this peer to the peers vector. + const int peer_id{peer["id"].get_int()}; + const int version{peer["version"].get_int()}; + const std::string sub_version{peer["subver"].get_str()}; + const int64_t conn_time{peer["conntime"].get_int64()}; + const int64_t last_blck{peer["last_block"].get_int64()}; + const int64_t last_recv{peer["lastrecv"].get_int64()}; + const int64_t last_send{peer["lastsend"].get_int64()}; + const int64_t last_trxn{peer["last_transaction"].get_int64()}; + const double min_ping{peer["minping"].isNull() ? -1 : peer["minping"].get_real()}; + const double ping{peer["pingtime"].isNull() ? -1 : peer["pingtime"].get_real()}; + peers.push_back({peer_id, mapped_as, version, conn_time, last_blck, last_recv, last_send, last_trxn, min_ping, ping, addr, sub_version, net_type, is_block_relay, !is_inbound}); + max_peer_id_length = std::max(ToString(peer_id).length(), max_peer_id_length); + max_addr_length = std::max(addr.length() + 1, max_addr_length); + is_asmap_on |= (mapped_as != 0); + } + } + + // Generate report header. + std::string result{strprintf("%s %s%s - %i%s\n\n", PACKAGE_NAME, FormatFullVersion(), ChainToString(), networkinfo["protocolversion"].get_int(), networkinfo["subversion"].get_str())}; + + // Report detailed peer connections list sorted by direction and minimum ping time. + if (DetailsRequested() && !peers.empty()) { + std::sort(peers.begin(), peers.end()); + result += "Peer connections sorted by direction and min ping\n<-> relay net mping ping send recv txn blk uptime "; + if (is_asmap_on) result += " asmap "; + result += strprintf("%*s %-*s%s\n", max_peer_id_length, "id", IsAddressSelected() ? max_addr_length : 0, IsAddressSelected() ? "address" : "", IsVersionSelected() ? "version" : ""); + for (const Peer& peer : peers) { + std::string version{ToString(peer.version) + peer.sub_version}; + result += strprintf( + "%3s %5s %5s%6s%7s%5s%5s%5s%5s%7s%*i %*s %-*s%s\n", + peer.is_outbound ? "out" : "in", + peer.is_block_relay ? "block" : "full", + NetTypeEnumToString(peer.net_type), + peer.min_ping == -1 ? "" : ToString(round(1000 * peer.min_ping)), + peer.ping == -1 ? "" : ToString(round(1000 * peer.ping)), + peer.last_send == 0 ? "" : ToString(time_now - peer.last_send), + peer.last_recv == 0 ? "" : ToString(time_now - peer.last_recv), + peer.last_trxn == 0 ? "" : ToString((time_now - peer.last_trxn) / 60), + peer.last_blck == 0 ? "" : ToString((time_now - peer.last_blck) / 60), + peer.conn_time == 0 ? "" : ToString((time_now - peer.conn_time) / 60), + is_asmap_on ? 7 : 0, // variable spacing + is_asmap_on && peer.mapped_as != 0 ? ToString(peer.mapped_as) : "", + max_peer_id_length, // variable spacing + peer.id, + IsAddressSelected() ? max_addr_length : 0, // variable spacing + IsAddressSelected() ? peer.addr : "", + IsVersionSelected() && version != "0" ? version : ""); + } + result += " ms ms sec sec min min min\n\n"; + } + + // Report peer connection totals by type. + total_i = ipv4_i + ipv6_i + onion_i; + total_o = ipv4_o + ipv6_o + onion_o; + result += " ipv4 ipv6 onion total block-relay\n"; + result += strprintf("in %5i %5i %5i %5i %5i\n", ipv4_i, ipv6_i, onion_i, total_i, block_relay_i); + result += strprintf("out %5i %5i %5i %5i %5i\n", ipv4_o, ipv6_o, onion_o, total_o, block_relay_o); + result += strprintf("total %5i %5i %5i %5i %5i\n", ipv4_i + ipv4_o, ipv6_i + ipv6_o, onion_i + onion_o, total_i + total_o, block_relay_i + block_relay_o); + + // Report local addresses, ports, and scores. + result += "\nLocal addresses"; + const UniValue& local_addrs{networkinfo["localaddresses"]}; + if (local_addrs.empty()) { + result += ": n/a\n"; + } else { + for (const UniValue& addr : local_addrs.getValues()) { + result += strprintf("\n%-40i port %5i score %6i", addr["address"].get_str(), addr["port"].get_int(), addr["score"].get_int()); + } + } + + return JSONRPCReplyObj(UniValue{result}, NullUniValue, 1); + } +}; + /** Process RPC generatetoaddress request. */ class GenerateToAddressRequestHandler : public BaseRequestHandler { @@ -618,6 +833,8 @@ static int CommandLineRPC(int argc, char *argv[]) std::string method; if (gArgs.IsArgSet("-getinfo")) { rh.reset(new GetinfoRequestHandler()); + } else if (gArgs.GetBoolArg("-netinfo", false)) { + rh.reset(new NetinfoRequestHandler()); } else if (gArgs.GetBoolArg("-generate", false)) { const UniValue getnewaddress{GetNewAddress()}; const UniValue& error{find_value(getnewaddress, "error")}; diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 56afcb6de..a9119d514 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -320,8 +320,8 @@ static void MutateTxAddOutPubKey(CMutableTransaction& tx, const std::string& str if (!pubkey.IsCompressed()) { throw std::runtime_error("Uncompressed pubkeys are not useable for SegWit outputs"); } - // Call GetScriptForWitness() to build a P2WSH scriptPubKey - scriptPubKey = GetScriptForWitness(scriptPubKey); + // Build a P2WPKH script + scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(pubkey)); } if (bScriptHash) { // Get the ID for the script, and then construct a P2SH destination for it. @@ -390,8 +390,8 @@ static void MutateTxAddOutMultiSig(CMutableTransaction& tx, const std::string& s throw std::runtime_error("Uncompressed pubkeys are not useable for SegWit outputs"); } } - // Call GetScriptForWitness() to build a P2WSH scriptPubKey - scriptPubKey = GetScriptForWitness(scriptPubKey); + // Build a P2WSH with the multisig script + scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(scriptPubKey)); } if (bScriptHash) { if (scriptPubKey.size() > MAX_SCRIPT_ELEMENT_SIZE) { @@ -464,7 +464,7 @@ static void MutateTxAddOutScript(CMutableTransaction& tx, const std::string& str } if (bSegWit) { - scriptPubKey = GetScriptForWitness(scriptPubKey); + scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(scriptPubKey)); } if (bScriptHash) { if (scriptPubKey.size() > MAX_SCRIPT_ELEMENT_SIZE) { diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index b04cc1205..02074f820 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -44,34 +44,28 @@ static void WaitForShutdown(NodeContext& node) static bool AppInit(int argc, char* argv[]) { NodeContext node; - node.chain = interfaces::MakeChain(node); bool fRet = false; util::ThreadSetInternalName("init"); - // - // Parameters - // // If Qt is used, parameters/bitcoin.conf are parsed in qt/bitcoin.cpp's main() SetupServerArgs(node); + ArgsManager& args = *Assert(node.args); std::string error; - if (!gArgs.ParseParameters(argc, argv, error)) { + if (!args.ParseParameters(argc, argv, error)) { return InitError(Untranslated(strprintf("Error parsing command line arguments: %s\n", error))); } // Process help and version before taking care about datadir - if (HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { + if (HelpRequested(args) || args.IsArgSet("-version")) { std::string strUsage = PACKAGE_NAME " version " + FormatFullVersion() + "\n"; - if (gArgs.IsArgSet("-version")) - { + if (args.IsArgSet("-version")) { strUsage += FormatParagraph(LicenseInfo()) + "\n"; - } - else - { + } else { strUsage += "\nUsage: bitcoind [options] Start " PACKAGE_NAME "\n"; - strUsage += "\n" + gArgs.GetHelpMessage(); + strUsage += "\n" + args.GetHelpMessage(); } tfm::format(std::cout, "%s", strUsage); @@ -82,14 +76,14 @@ static bool AppInit(int argc, char* argv[]) try { if (!CheckDataDirOption()) { - return InitError(Untranslated(strprintf("Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "")))); + return InitError(Untranslated(strprintf("Specified data directory \"%s\" does not exist.\n", args.GetArg("-datadir", "")))); } - if (!gArgs.ReadConfigFiles(error, true)) { + if (!args.ReadConfigFiles(error, true)) { return InitError(Untranslated(strprintf("Error reading configuration file: %s\n", error))); } // Check for -chain, -testnet or -regtest parameter (Params() calls are only valid after this clause) try { - SelectParams(gArgs.GetChainName()); + SelectParams(args.GetChainName()); } catch (const std::exception& e) { return InitError(Untranslated(strprintf("%s\n", e.what()))); } @@ -101,23 +95,21 @@ static bool AppInit(int argc, char* argv[]) } } - if (!gArgs.InitSettings(error)) { + if (!args.InitSettings(error)) { InitError(Untranslated(error)); return false; } // -server defaults to true for bitcoind but not for the GUI so do this here - gArgs.SoftSetBoolArg("-server", true); + args.SoftSetBoolArg("-server", true); // Set this early so that parameter interactions go to console - InitLogging(); - InitParameterInteraction(); - if (!AppInitBasicSetup()) - { + InitLogging(args); + InitParameterInteraction(args); + if (!AppInitBasicSetup(args)) { // InitError will have been called with detailed error, which ends up on console return false; } - if (!AppInitParameterInteraction()) - { + if (!AppInitParameterInteraction(args)) { // InitError will have been called with detailed error, which ends up on console return false; } @@ -126,8 +118,7 @@ static bool AppInit(int argc, char* argv[]) // InitError will have been called with detailed error, which ends up on console return false; } - if (gArgs.GetBoolArg("-daemon", false)) - { + if (args.GetBoolArg("-daemon", false)) { #if HAVE_DECL_DAEMON #if defined(MAC_OSX) #pragma GCC diagnostic push @@ -152,7 +143,7 @@ static bool AppInit(int argc, char* argv[]) // If locking the data directory failed, exit immediately return false; } - fRet = AppInitMain(context, node); + fRet = AppInitInterfaces(node) && AppInitMain(context, node); } catch (const std::exception& e) { PrintExceptionContinue(&e, "AppInit()"); diff --git a/src/chain.h b/src/chain.h index 802e23f77..43e8a39f3 100644 --- a/src/chain.h +++ b/src/chain.h @@ -398,12 +398,6 @@ public: return vChain[nHeight]; } - /** Compare two chains efficiently. */ - friend bool operator==(const CChain &a, const CChain &b) { - return a.vChain.size() == b.vChain.size() && - a.vChain[a.vChain.size() - 1] == b.vChain[b.vChain.size() - 1]; - } - /** Efficiently check whether a block is present in this chain. */ bool Contains(const CBlockIndex *pindex) const { return (*this)[pindex->nHeight] == pindex; diff --git a/src/chainparams.cpp b/src/chainparams.cpp index a7c9e33f0..ffd2076c9 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -110,7 +110,7 @@ public: // Note that of those which support the service bits prefix, most only support a subset of // possible options. - // This is fine at runtime as we'll fall back to using them as a oneshot if they don't support the + // This is fine at runtime as we'll fall back to using them as an addrfetch if they don't support the // service bits we want, but we should get them updated to support all service bits wanted by any // release ASAP to avoid it where possible. vSeeds.emplace_back("seed.bitcoin.sipa.be"); // Pieter Wuille, only supports x1, x5, x9, and xd diff --git a/src/chainparams.h b/src/chainparams.h index 542ef329d..7aa999fb7 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -23,6 +23,11 @@ typedef std::map<int, uint256> MapCheckpoints; struct CCheckpointData { MapCheckpoints mapCheckpoints; + + int GetHeight() const { + const auto& final_checkpoint = mapCheckpoints.rbegin(); + return final_checkpoint->first /* height */; + } }; /** diff --git a/src/compat.h b/src/compat.h index 68f6eb692..0be02cae0 100644 --- a/src/compat.h +++ b/src/compat.h @@ -11,9 +11,6 @@ #endif #ifdef WIN32 -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN 1 -#endif #ifndef NOMINMAX #define NOMINMAX #endif diff --git a/src/consensus/validation.h b/src/consensus/validation.h index 8de7a8f2d..2a93a090d 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -26,7 +26,8 @@ enum class TxValidationResult { * is uninteresting. */ TX_RECENT_CONSENSUS_CHANGE, - TX_NOT_STANDARD, //!< didn't meet our local policy rules + TX_INPUTS_NOT_STANDARD, //!< inputs (covered by txid) failed policy rules + TX_NOT_STANDARD, //!< otherwise didn't meet our local policy rules TX_MISSING_INPUTS, //!< transaction was missing some of its inputs TX_PREMATURE_SPEND, //!< transaction spends a coinbase too early, or violates locktime/sequence locks /** diff --git a/src/core_write.cpp b/src/core_write.cpp index 34cfeecc6..3980d8cb2 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -48,13 +48,14 @@ std::string FormatScript(const CScript& script) } } if (vch.size() > 0) { - ret += strprintf("0x%x 0x%x ", HexStr(it2, it - vch.size()), HexStr(it - vch.size(), it)); + ret += strprintf("0x%x 0x%x ", HexStr(std::vector<uint8_t>(it2, it - vch.size())), + HexStr(std::vector<uint8_t>(it - vch.size(), it))); } else { - ret += strprintf("0x%x ", HexStr(it2, it)); + ret += strprintf("0x%x ", HexStr(std::vector<uint8_t>(it2, it))); } continue; } - ret += strprintf("0x%x ", HexStr(it2, script.end())); + ret += strprintf("0x%x ", HexStr(std::vector<uint8_t>(it2, script.end()))); break; } return ret.substr(0, ret.size() - 1); @@ -110,8 +111,9 @@ std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDeco // checks in CheckSignatureEncoding. if (CheckSignatureEncoding(vch, SCRIPT_VERIFY_STRICTENC, nullptr)) { const unsigned char chSigHashType = vch.back(); - if (mapSigHashTypes.count(chSigHashType)) { - strSigHashDecode = "[" + mapSigHashTypes.find(chSigHashType)->second + "]"; + const auto it = mapSigHashTypes.find(chSigHashType); + if (it != mapSigHashTypes.end()) { + strSigHashDecode = "[" + it->second + "]"; vch.pop_back(); // remove the sighash type byte. it will be replaced by the decode. } } diff --git a/src/crypto/sha3.cpp b/src/crypto/sha3.cpp new file mode 100644 index 000000000..9c0c42fa7 --- /dev/null +++ b/src/crypto/sha3.cpp @@ -0,0 +1,161 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// Based on https://github.com/mjosaarinen/tiny_sha3/blob/master/sha3.c +// by Markku-Juhani O. Saarinen <[email protected]> + +#include <crypto/sha3.h> +#include <crypto/common.h> +#include <span.h> + +#include <algorithm> +#include <array> // For std::begin and std::end. + +#include <stdint.h> + +// Internal implementation code. +namespace +{ +uint64_t Rotl(uint64_t x, int n) { return (x << n) | (x >> (64 - n)); } +} // namespace + +void KeccakF(uint64_t (&st)[25]) +{ + static constexpr uint64_t RNDC[24] = { + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, 0x8000000080008000, + 0x000000000000808b, 0x0000000080000001, 0x8000000080008081, 0x8000000000008009, + 0x000000000000008a, 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, 0x8000000000008003, + 0x8000000000008002, 0x8000000000000080, 0x000000000000800a, 0x800000008000000a, + 0x8000000080008081, 0x8000000000008080, 0x0000000080000001, 0x8000000080008008 + }; + static constexpr int ROUNDS = 24; + + for (int round = 0; round < ROUNDS; ++round) { + uint64_t bc0, bc1, bc2, bc3, bc4, t; + + // Theta + bc0 = st[0] ^ st[5] ^ st[10] ^ st[15] ^ st[20]; + bc1 = st[1] ^ st[6] ^ st[11] ^ st[16] ^ st[21]; + bc2 = st[2] ^ st[7] ^ st[12] ^ st[17] ^ st[22]; + bc3 = st[3] ^ st[8] ^ st[13] ^ st[18] ^ st[23]; + bc4 = st[4] ^ st[9] ^ st[14] ^ st[19] ^ st[24]; + t = bc4 ^ Rotl(bc1, 1); st[0] ^= t; st[5] ^= t; st[10] ^= t; st[15] ^= t; st[20] ^= t; + t = bc0 ^ Rotl(bc2, 1); st[1] ^= t; st[6] ^= t; st[11] ^= t; st[16] ^= t; st[21] ^= t; + t = bc1 ^ Rotl(bc3, 1); st[2] ^= t; st[7] ^= t; st[12] ^= t; st[17] ^= t; st[22] ^= t; + t = bc2 ^ Rotl(bc4, 1); st[3] ^= t; st[8] ^= t; st[13] ^= t; st[18] ^= t; st[23] ^= t; + t = bc3 ^ Rotl(bc0, 1); st[4] ^= t; st[9] ^= t; st[14] ^= t; st[19] ^= t; st[24] ^= t; + + // Rho Pi + t = st[1]; + bc0 = st[10]; st[10] = Rotl(t, 1); t = bc0; + bc0 = st[7]; st[7] = Rotl(t, 3); t = bc0; + bc0 = st[11]; st[11] = Rotl(t, 6); t = bc0; + bc0 = st[17]; st[17] = Rotl(t, 10); t = bc0; + bc0 = st[18]; st[18] = Rotl(t, 15); t = bc0; + bc0 = st[3]; st[3] = Rotl(t, 21); t = bc0; + bc0 = st[5]; st[5] = Rotl(t, 28); t = bc0; + bc0 = st[16]; st[16] = Rotl(t, 36); t = bc0; + bc0 = st[8]; st[8] = Rotl(t, 45); t = bc0; + bc0 = st[21]; st[21] = Rotl(t, 55); t = bc0; + bc0 = st[24]; st[24] = Rotl(t, 2); t = bc0; + bc0 = st[4]; st[4] = Rotl(t, 14); t = bc0; + bc0 = st[15]; st[15] = Rotl(t, 27); t = bc0; + bc0 = st[23]; st[23] = Rotl(t, 41); t = bc0; + bc0 = st[19]; st[19] = Rotl(t, 56); t = bc0; + bc0 = st[13]; st[13] = Rotl(t, 8); t = bc0; + bc0 = st[12]; st[12] = Rotl(t, 25); t = bc0; + bc0 = st[2]; st[2] = Rotl(t, 43); t = bc0; + bc0 = st[20]; st[20] = Rotl(t, 62); t = bc0; + bc0 = st[14]; st[14] = Rotl(t, 18); t = bc0; + bc0 = st[22]; st[22] = Rotl(t, 39); t = bc0; + bc0 = st[9]; st[9] = Rotl(t, 61); t = bc0; + bc0 = st[6]; st[6] = Rotl(t, 20); t = bc0; + st[1] = Rotl(t, 44); + + // Chi Iota + bc0 = st[0]; bc1 = st[1]; bc2 = st[2]; bc3 = st[3]; bc4 = st[4]; + st[0] = bc0 ^ (~bc1 & bc2) ^ RNDC[round]; + st[1] = bc1 ^ (~bc2 & bc3); + st[2] = bc2 ^ (~bc3 & bc4); + st[3] = bc3 ^ (~bc4 & bc0); + st[4] = bc4 ^ (~bc0 & bc1); + bc0 = st[5]; bc1 = st[6]; bc2 = st[7]; bc3 = st[8]; bc4 = st[9]; + st[5] = bc0 ^ (~bc1 & bc2); + st[6] = bc1 ^ (~bc2 & bc3); + st[7] = bc2 ^ (~bc3 & bc4); + st[8] = bc3 ^ (~bc4 & bc0); + st[9] = bc4 ^ (~bc0 & bc1); + bc0 = st[10]; bc1 = st[11]; bc2 = st[12]; bc3 = st[13]; bc4 = st[14]; + st[10] = bc0 ^ (~bc1 & bc2); + st[11] = bc1 ^ (~bc2 & bc3); + st[12] = bc2 ^ (~bc3 & bc4); + st[13] = bc3 ^ (~bc4 & bc0); + st[14] = bc4 ^ (~bc0 & bc1); + bc0 = st[15]; bc1 = st[16]; bc2 = st[17]; bc3 = st[18]; bc4 = st[19]; + st[15] = bc0 ^ (~bc1 & bc2); + st[16] = bc1 ^ (~bc2 & bc3); + st[17] = bc2 ^ (~bc3 & bc4); + st[18] = bc3 ^ (~bc4 & bc0); + st[19] = bc4 ^ (~bc0 & bc1); + bc0 = st[20]; bc1 = st[21]; bc2 = st[22]; bc3 = st[23]; bc4 = st[24]; + st[20] = bc0 ^ (~bc1 & bc2); + st[21] = bc1 ^ (~bc2 & bc3); + st[22] = bc2 ^ (~bc3 & bc4); + st[23] = bc3 ^ (~bc4 & bc0); + st[24] = bc4 ^ (~bc0 & bc1); + } +} + +SHA3_256& SHA3_256::Write(Span<const unsigned char> data) +{ + if (m_bufsize && m_bufsize + data.size() >= sizeof(m_buffer)) { + // Fill the buffer and process it. + std::copy(data.begin(), data.begin() + sizeof(m_buffer) - m_bufsize, m_buffer + m_bufsize); + data = data.subspan(sizeof(m_buffer) - m_bufsize); + m_state[m_pos++] ^= ReadLE64(m_buffer); + m_bufsize = 0; + if (m_pos == RATE_BUFFERS) { + KeccakF(m_state); + m_pos = 0; + } + } + while (data.size() >= sizeof(m_buffer)) { + // Process chunks directly from the buffer. + m_state[m_pos++] ^= ReadLE64(data.data()); + data = data.subspan(8); + if (m_pos == RATE_BUFFERS) { + KeccakF(m_state); + m_pos = 0; + } + } + if (data.size()) { + // Keep the remainder in the buffer. + std::copy(data.begin(), data.end(), m_buffer + m_bufsize); + m_bufsize += data.size(); + } + return *this; +} + +SHA3_256& SHA3_256::Finalize(Span<unsigned char> output) +{ + assert(output.size() == OUTPUT_SIZE); + std::fill(m_buffer + m_bufsize, m_buffer + sizeof(m_buffer), 0); + m_buffer[m_bufsize] ^= 0x06; + m_state[m_pos] ^= ReadLE64(m_buffer); + m_state[RATE_BUFFERS - 1] ^= 0x8000000000000000; + KeccakF(m_state); + for (unsigned i = 0; i < 4; ++i) { + WriteLE64(output.data() + 8 * i, m_state[i]); + } + return *this; +} + +SHA3_256& SHA3_256::Reset() +{ + m_bufsize = 0; + m_pos = 0; + std::fill(std::begin(m_state), std::end(m_state), 0); + return *this; +} diff --git a/src/crypto/sha3.h b/src/crypto/sha3.h new file mode 100644 index 000000000..88d8c1204 --- /dev/null +++ b/src/crypto/sha3.h @@ -0,0 +1,41 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_SHA3_H +#define BITCOIN_CRYPTO_SHA3_H + +#include <span.h> + +#include <stdint.h> +#include <stdlib.h> + +//! The Keccak-f[1600] transform. +void KeccakF(uint64_t (&st)[25]); + +class SHA3_256 +{ +private: + uint64_t m_state[25] = {0}; + unsigned char m_buffer[8]; + unsigned m_bufsize = 0; + unsigned m_pos = 0; + + //! Sponge rate in bits. + static constexpr unsigned RATE_BITS = 1088; + + //! Sponge rate expressed as a multiple of the buffer size. + static constexpr unsigned RATE_BUFFERS = RATE_BITS / (8 * sizeof(m_buffer)); + + static_assert(RATE_BITS % (8 * sizeof(m_buffer)) == 0, "Rate must be a multiple of 8 bytes"); + +public: + static constexpr size_t OUTPUT_SIZE = 32; + + SHA3_256() {} + SHA3_256& Write(Span<const unsigned char> data); + SHA3_256& Finalize(Span<unsigned char> output); + SHA3_256& Reset(); +}; + +#endif // BITCOIN_CRYPTO_SHA3_H diff --git a/src/crypto/siphash.cpp b/src/crypto/siphash.cpp index e81957111..2e0106b16 100644 --- a/src/crypto/siphash.cpp +++ b/src/crypto/siphash.cpp @@ -49,7 +49,7 @@ CSipHasher& CSipHasher::Write(const unsigned char* data, size_t size) { uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; uint64_t t = tmp; - int c = count; + uint8_t c = count; while (size--) { t |= ((uint64_t)(*(data++))) << (8 * (c % 8)); diff --git a/src/crypto/siphash.h b/src/crypto/siphash.h index b312f913f..6b38950f8 100644 --- a/src/crypto/siphash.h +++ b/src/crypto/siphash.h @@ -15,7 +15,7 @@ class CSipHasher private: uint64_t v[4]; uint64_t tmp; - int count; + uint8_t count; // Only the low 8 bits of the input size matter. public: /** Construct a SipHash calculator initialized with 128-bit key (k0, k1) */ diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 18dc7a69e..8d2dcd027 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -4,11 +4,8 @@ #include <util/system.h> #include <walletinitinterface.h> -#include <support/allocators/secure.h> class CWallet; -enum class WalletCreationStatus; -struct bilingual_str; namespace interfaces { class Chain; @@ -35,6 +32,7 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const "-discardfee=<amt>", "-fallbackfee=<amt>", "-keypool=<n>", + "-maxapsfee=<n>", "-maxtxfee=<amt>", "-mintxfee=<amt>", "-paytxfee=<amt>", @@ -48,7 +46,6 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const "-walletdir=<dir>", "-walletnotify=<cmd>", "-walletrbf", - "-zapwallettxes=<mode>", "-dblogsize=<n>", "-flushwallet", "-privdb", @@ -58,37 +55,6 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const const WalletInitInterface& g_wallet_init_interface = DummyWalletInit(); -fs::path GetWalletDir() -{ - throw std::logic_error("Wallet function called in non-wallet build."); -} - -std::vector<fs::path> ListWalletDir() -{ - throw std::logic_error("Wallet function called in non-wallet build."); -} - -std::vector<std::shared_ptr<CWallet>> GetWallets() -{ - throw std::logic_error("Wallet function called in non-wallet build."); -} - -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) -{ - throw std::logic_error("Wallet function called in non-wallet build."); -} - -WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, std::shared_ptr<CWallet>& result) -{ - throw std::logic_error("Wallet function called in non-wallet build."); -} - -using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wallet)>; -std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet) -{ - throw std::logic_error("Wallet function called in non-wallet build."); -} - namespace interfaces { std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet) diff --git a/src/hash.cpp b/src/hash.cpp index 4c09f5f64..83b90ae06 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -77,3 +77,10 @@ void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char he num[3] = (nChild >> 0) & 0xFF; CHMAC_SHA512(chainCode.begin(), chainCode.size()).Write(&header, 1).Write(data, 32).Write(num, 4).Finalize(output); } + +uint256 SHA256Uint256(const uint256& input) +{ + uint256 result; + CSHA256().Write(input.begin(), 32).Finalize(result.begin()); + return result; +} diff --git a/src/hash.h b/src/hash.h index 71806483f..c16bbb48c 100644 --- a/src/hash.h +++ b/src/hash.h @@ -6,6 +6,7 @@ #ifndef BITCOIN_HASH_H #define BITCOIN_HASH_H +#include <attributes.h> #include <crypto/common.h> #include <crypto/ripemd160.h> #include <crypto/sha256.h> @@ -98,7 +99,7 @@ inline uint160 Hash160(const T1& in1) class CHashWriter { private: - CHash256 ctx; + CSHA256 ctx; const int nType; const int nVersion; @@ -110,13 +111,27 @@ public: int GetVersion() const { return nVersion; } void write(const char *pch, size_t size) { - ctx.Write({(const unsigned char*)pch, size}); + ctx.Write((const unsigned char*)pch, size); } - // invalidates the object + /** Compute the double-SHA256 hash of all data written to this object. + * + * Invalidates this object. + */ uint256 GetHash() { uint256 result; - ctx.Finalize(result); + ctx.Finalize(result.begin()); + ctx.Reset().Write(result.begin(), CSHA256::OUTPUT_SIZE).Finalize(result.begin()); + return result; + } + + /** Compute the SHA256 hash of all data written to this object. + * + * Invalidates this object. + */ + uint256 GetSHA256() { + uint256 result; + ctx.Finalize(result.begin()); return result; } @@ -124,9 +139,8 @@ public: * Returns the first 64 bits from the resulting hash. */ inline uint64_t GetCheapHash() { - unsigned char result[CHash256::OUTPUT_SIZE]; - ctx.Finalize(result); - return ReadLE64(result); + uint256 result = GetHash(); + return ReadLE64(result.begin()); } template<typename T> @@ -181,6 +195,9 @@ uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL return ss.GetHash(); } +/** Single-SHA256 a 32-byte input (represented as uint256). */ +NODISCARD uint256 SHA256Uint256(const uint256& input); + unsigned int MurmurHash3(unsigned int nHashSeed, Span<const unsigned char> vDataToHash); void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]); diff --git a/src/index/base.cpp b/src/index/base.cpp index f587205a2..e67b81376 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -319,3 +319,12 @@ void BaseIndex::Stop() m_thread_sync.join(); } } + +IndexSummary BaseIndex::GetSummary() const +{ + IndexSummary summary{}; + summary.name = GetName(); + summary.synced = m_synced; + summary.best_block_height = m_best_block_index.load()->nHeight; + return summary; +} diff --git a/src/index/base.h b/src/index/base.h index 3fab810bb..8559e3cb6 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -13,6 +13,12 @@ class CBlockIndex; +struct IndexSummary { + std::string name; + bool synced{false}; + int best_block_height{0}; +}; + /** * Base class for indices of blockchain data. This implements * CValidationInterface and ensures blocks are indexed sequentially according @@ -21,6 +27,13 @@ class CBlockIndex; class BaseIndex : public CValidationInterface { protected: + /** + * The database stores a block locator of the chain the database is synced to + * so that the index can efficiently determine the point it last stopped at. + * A locator is used instead of a simple hash of the chain tip because blocks + * and block index entries may not be flushed to disk until after this database + * is updated. + */ class DB : public CDBWrapper { public: @@ -106,6 +119,9 @@ public: /// Stops the instance from staying in sync with blockchain updates. void Stop(); + + /// Get a summary of the index and its state. + IndexSummary GetSummary() const; }; #endif // BITCOIN_INDEX_BASE_H diff --git a/src/index/disktxpos.h b/src/index/disktxpos.h new file mode 100644 index 000000000..69696b0ec --- /dev/null +++ b/src/index/disktxpos.h @@ -0,0 +1,35 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_DISKTXPOS_H +#define BITCOIN_INDEX_DISKTXPOS_H + +#include <flatfile.h> +#include <serialize.h> + +struct CDiskTxPos : public FlatFilePos +{ + unsigned int nTxOffset; // after header + + SERIALIZE_METHODS(CDiskTxPos, obj) + { + READWRITEAS(FlatFilePos, obj); + READWRITE(VARINT(obj.nTxOffset)); + } + + CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { + } + + CDiskTxPos() { + SetNull(); + } + + void SetNull() { + FlatFilePos::SetNull(); + nTxOffset = 0; + } +}; + + +#endif // BITCOIN_INDEX_DISKTXPOS_H diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 64472714c..462ac5962 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <index/disktxpos.h> #include <index/txindex.h> #include <node/ui_interface.h> #include <shutdown.h> @@ -15,38 +16,9 @@ constexpr char DB_TXINDEX_BLOCK = 'T'; std::unique_ptr<TxIndex> g_txindex; -struct CDiskTxPos : public FlatFilePos -{ - unsigned int nTxOffset; // after header - - SERIALIZE_METHODS(CDiskTxPos, obj) - { - READWRITEAS(FlatFilePos, obj); - READWRITE(VARINT(obj.nTxOffset)); - } - - CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { - } - CDiskTxPos() { - SetNull(); - } - void SetNull() { - FlatFilePos::SetNull(); - nTxOffset = 0; - } -}; - -/** - * Access to the txindex database (indexes/txindex/) - * - * The database stores a block locator of the chain the database is synced to - * so that the TxIndex can efficiently determine the point it last stopped at. - * A locator is used instead of a simple hash of the chain tip because blocks - * and block index entries may not be flushed to disk until after this database - * is updated. - */ +/** Access to the txindex database (indexes/txindex/) */ class TxIndex::DB : public BaseIndex::DB { public: diff --git a/src/init.cpp b/src/init.cpp index 6cca21f37..023aa9aba 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -24,6 +24,7 @@ #include <index/blockfilterindex.h> #include <index/txindex.h> #include <interfaces/chain.h> +#include <interfaces/node.h> #include <key.h> #include <miner.h> #include <net.h> @@ -106,14 +107,14 @@ static const char* DEFAULT_ASMAP_FILENAME="ip_asn.map"; */ static const char* BITCOIN_PID_FILENAME = "bitcoind.pid"; -static fs::path GetPidFile() +static fs::path GetPidFile(const ArgsManager& args) { - return AbsPathForConfigVal(fs::path(gArgs.GetArg("-pid", BITCOIN_PID_FILENAME))); + return AbsPathForConfigVal(fs::path(args.GetArg("-pid", BITCOIN_PID_FILENAME))); } -NODISCARD static bool CreatePidFile() +NODISCARD static bool CreatePidFile(const ArgsManager& args) { - fsbridge::ofstream file{GetPidFile()}; + fsbridge::ofstream file{GetPidFile(args)}; if (file) { #ifdef WIN32 tfm::format(file, "%d\n", GetCurrentProcessId()); @@ -122,7 +123,7 @@ NODISCARD static bool CreatePidFile() #endif return true; } else { - return InitError(strprintf(_("Unable to create the PID file '%s': %s"), GetPidFile().string(), std::strerror(errno))); + return InitError(strprintf(_("Unable to create the PID file '%s': %s"), GetPidFile(args).string(), std::strerror(errno))); } } @@ -179,13 +180,14 @@ void Shutdown(NodeContext& node) TRY_LOCK(g_shutdown_mutex, lock_shutdown); if (!lock_shutdown) return; LogPrintf("%s: In progress...\n", __func__); + Assert(node.args); /// Note: Shutdown() must be able to handle cases in which initialization failed part of the way, /// for example if the data directory was found to be locked. /// Be sure that anything that writes files or flushes caches only does this if the respective /// module was initialized. util::ThreadRename("shutoff"); - mempool.AddTransactionsUpdated(1); + if (node.mempool) node.mempool->AddTransactionsUpdated(1); StopHTTPRPC(); StopREST(); @@ -198,7 +200,7 @@ void Shutdown(NodeContext& node) // Because these depend on each-other, we make sure that neither can be // using the other before destroying them. - if (node.peer_logic) UnregisterValidationInterface(node.peer_logic.get()); + if (node.peerman) UnregisterValidationInterface(node.peerman.get()); // Follow the lock order requirements: // * CheckForStaleTipAndEvictPeers locks cs_main before indirectly calling GetExtraOutboundCount // which locks cs_vNodes. @@ -225,12 +227,12 @@ void Shutdown(NodeContext& node) // After the threads that potentially access these pointers have been stopped, // destruct and reset all to nullptr. - node.peer_logic.reset(); + node.peerman.reset(); node.connman.reset(); node.banman.reset(); - if (::mempool.IsLoaded() && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - DumpMempool(::mempool); + if (node.mempool && node.mempool->IsLoaded() && node.args->GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { + DumpMempool(*node.mempool); } if (fFeeEstimatesInitialized) @@ -300,19 +302,19 @@ void Shutdown(NodeContext& node) GetMainSignals().UnregisterBackgroundSignalScheduler(); globalVerifyHandle.reset(); ECC_Stop(); - node.args = nullptr; - node.mempool = nullptr; + node.mempool.reset(); node.chainman = nullptr; node.scheduler.reset(); try { - if (!fs::remove(GetPidFile())) { + if (!fs::remove(GetPidFile(*node.args))) { LogPrintf("%s: Unable to remove PID file: File does not exist\n", __func__); } } catch (const fs::filesystem_error& e) { LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); } + node.args = nullptr; LogPrintf("%s: done\n", __func__); } @@ -371,7 +373,7 @@ void SetupServerArgs(NodeContext& node) node.args = &gArgs; ArgsManager& argsman = *node.args; - SetupHelpOptions(gArgs); + SetupHelpOptions(argsman); argsman.AddArg("-help-debug", "Print help message with debugging options and exit", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); // server-only for now const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); @@ -442,13 +444,13 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-externalip=<ip>", "Specify your own public address", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-forcednsseed", strprintf("Always query for peer addresses via DNS lookup (default: %u)", DEFAULT_FORCEDNSSEED), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listen", "Accept connections from outside (default: 1 if no -proxy or -connect)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-listenonion", strprintf("Automatically create Tor hidden service (default: %d)", DEFAULT_LISTEN_ONION), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-listenonion", strprintf("Automatically create Tor onion service (default: %d)", DEFAULT_LISTEN_ONION), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> connections to peers (default: %u)", DEFAULT_MAX_PEER_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by peers forward or backward by this amount. (default: %u seconds)", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target (in MiB per 24h). Limit does not apply to peers with 'download' permission. 0 = no limit (default: %d)", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor hidden services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-onlynet=<net>", "Make outgoing connections only through network <net> (ipv4, ipv6 or onion). Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple networks.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -505,7 +507,7 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-checklevel=<n>", strprintf("How thorough the block verification of -checkblocks is: %s (0-4, default: %u)", Join(CHECKLEVEL_DOC, ", "), DEFAULT_CHECKLEVEL), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, chainstate, and other validation data structures occasionally. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkmempool=<n>", strprintf("Run checks every <n> transactions (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-checkpoints", strprintf("Enable rejection of any forks from the known historical chain until block 295000 (default: %u)", DEFAULT_CHECKPOINTS_ENABLED), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-checkpoints", strprintf("Enable rejection of any forks from the known historical chain until block %s (default: %u)", defaultChainParams->Checkpoints().GetHeight(), DEFAULT_CHECKPOINTS_ENABLED), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-deprecatedrpc=<method>", "Allows deprecated RPC method(s) to be used", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-dropmessagestest=<n>", "Randomly drop 1 of every <n> network messages", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", DEFAULT_STOPAFTERBLOCKIMPORT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -598,21 +600,6 @@ std::string LicenseInfo() "\n"; } -#if HAVE_SYSTEM -static void BlockNotifyCallback(SynchronizationState sync_state, const CBlockIndex* pBlockIndex) -{ - if (sync_state != SynchronizationState::POST_INIT || !pBlockIndex) - return; - - std::string strCmd = gArgs.GetArg("-blocknotify", ""); - if (!strCmd.empty()) { - boost::replace_all(strCmd, "%s", pBlockIndex->GetBlockHash().GetHex()); - std::thread t(runCommand, strCmd); - t.detach(); // thread runs free - } -} -#endif - static bool fHaveGenesis = false; static Mutex g_genesis_wait_mutex; static std::condition_variable g_genesis_wait_cv; @@ -683,7 +670,7 @@ static void CleanupBlockRevFiles() } } -static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles) +static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args) { const CChainParams& chainparams = Params(); ScheduleBatchPriority(); @@ -745,16 +732,13 @@ static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImp } } - if (gArgs.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { + if (args.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { LogPrintf("Stopping after block import\n"); StartShutdown(); return; } } // End scope of CImportingNow - if (gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - LoadMempool(::mempool); - } - ::mempool.SetIsLoaded(!ShutdownRequested()); + chainman.ActiveChainstate().LoadMempool(args); } /** Sanity checks @@ -779,6 +763,7 @@ static bool InitSanityCheck() static bool AppInitServers(const util::Ref& context, NodeContext& node) { + const ArgsManager& args = *Assert(node.args); RPCServer::OnStarted(&OnRPCStarted); RPCServer::OnStopped(&OnRPCStopped); if (!InitHTTPServer()) @@ -787,71 +772,71 @@ static bool AppInitServers(const util::Ref& context, NodeContext& node) node.rpc_interruption_point = RpcInterruptionPoint; if (!StartHTTPRPC(context)) return false; - if (gArgs.GetBoolArg("-rest", DEFAULT_REST_ENABLE)) StartREST(context); + if (args.GetBoolArg("-rest", DEFAULT_REST_ENABLE)) StartREST(context); StartHTTPServer(); return true; } // Parameter interaction based on rules -void InitParameterInteraction() +void InitParameterInteraction(ArgsManager& args) { // when specifying an explicit binding address, you want to listen on it // even when -connect or -proxy is specified - if (gArgs.IsArgSet("-bind")) { - if (gArgs.SoftSetBoolArg("-listen", true)) + if (args.IsArgSet("-bind")) { + if (args.SoftSetBoolArg("-listen", true)) LogPrintf("%s: parameter interaction: -bind set -> setting -listen=1\n", __func__); } - if (gArgs.IsArgSet("-whitebind")) { - if (gArgs.SoftSetBoolArg("-listen", true)) + if (args.IsArgSet("-whitebind")) { + if (args.SoftSetBoolArg("-listen", true)) LogPrintf("%s: parameter interaction: -whitebind set -> setting -listen=1\n", __func__); } - if (gArgs.IsArgSet("-connect")) { + if (args.IsArgSet("-connect")) { // when only connecting to trusted nodes, do not seed via DNS, or listen by default - if (gArgs.SoftSetBoolArg("-dnsseed", false)) + if (args.SoftSetBoolArg("-dnsseed", false)) LogPrintf("%s: parameter interaction: -connect set -> setting -dnsseed=0\n", __func__); - if (gArgs.SoftSetBoolArg("-listen", false)) + if (args.SoftSetBoolArg("-listen", false)) LogPrintf("%s: parameter interaction: -connect set -> setting -listen=0\n", __func__); } - if (gArgs.IsArgSet("-proxy")) { + if (args.IsArgSet("-proxy")) { // to protect privacy, do not listen by default if a default proxy server is specified - if (gArgs.SoftSetBoolArg("-listen", false)) + if (args.SoftSetBoolArg("-listen", false)) LogPrintf("%s: parameter interaction: -proxy set -> setting -listen=0\n", __func__); // to protect privacy, do not use UPNP when a proxy is set. The user may still specify -listen=1 // to listen locally, so don't rely on this happening through -listen below. - if (gArgs.SoftSetBoolArg("-upnp", false)) + if (args.SoftSetBoolArg("-upnp", false)) LogPrintf("%s: parameter interaction: -proxy set -> setting -upnp=0\n", __func__); // to protect privacy, do not discover addresses by default - if (gArgs.SoftSetBoolArg("-discover", false)) + if (args.SoftSetBoolArg("-discover", false)) LogPrintf("%s: parameter interaction: -proxy set -> setting -discover=0\n", __func__); } - if (!gArgs.GetBoolArg("-listen", DEFAULT_LISTEN)) { + if (!args.GetBoolArg("-listen", DEFAULT_LISTEN)) { // do not map ports or try to retrieve public IP when not listening (pointless) - if (gArgs.SoftSetBoolArg("-upnp", false)) + if (args.SoftSetBoolArg("-upnp", false)) LogPrintf("%s: parameter interaction: -listen=0 -> setting -upnp=0\n", __func__); - if (gArgs.SoftSetBoolArg("-discover", false)) + if (args.SoftSetBoolArg("-discover", false)) LogPrintf("%s: parameter interaction: -listen=0 -> setting -discover=0\n", __func__); - if (gArgs.SoftSetBoolArg("-listenonion", false)) + if (args.SoftSetBoolArg("-listenonion", false)) LogPrintf("%s: parameter interaction: -listen=0 -> setting -listenonion=0\n", __func__); } - if (gArgs.IsArgSet("-externalip")) { + if (args.IsArgSet("-externalip")) { // if an explicit public IP is specified, do not try to find others - if (gArgs.SoftSetBoolArg("-discover", false)) + if (args.SoftSetBoolArg("-discover", false)) LogPrintf("%s: parameter interaction: -externalip set -> setting -discover=0\n", __func__); } // disable whitelistrelay in blocksonly mode - if (gArgs.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY)) { - if (gArgs.SoftSetBoolArg("-whitelistrelay", false)) + if (args.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY)) { + if (args.SoftSetBoolArg("-whitelistrelay", false)) LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -whitelistrelay=0\n", __func__); } // Forcing relay from whitelisted hosts implies we will accept relays from them in the first place. - if (gArgs.GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY)) { - if (gArgs.SoftSetBoolArg("-whitelistrelay", true)) + if (args.GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY)) { + if (args.SoftSetBoolArg("-whitelistrelay", true)) LogPrintf("%s: parameter interaction: -whitelistforcerelay=1 -> setting -whitelistrelay=1\n", __func__); } } @@ -862,18 +847,18 @@ void InitParameterInteraction() * Note that this is called very early in the process lifetime, so you should be * careful about what global state you rely on here. */ -void InitLogging() +void InitLogging(const ArgsManager& args) { - LogInstance().m_print_to_file = !gArgs.IsArgNegated("-debuglogfile"); - LogInstance().m_file_path = AbsPathForConfigVal(gArgs.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE)); - LogInstance().m_print_to_console = gArgs.GetBoolArg("-printtoconsole", !gArgs.GetBoolArg("-daemon", false)); - LogInstance().m_log_timestamps = gArgs.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); - LogInstance().m_log_time_micros = gArgs.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); + LogInstance().m_print_to_file = !args.IsArgNegated("-debuglogfile"); + LogInstance().m_file_path = AbsPathForConfigVal(args.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE)); + LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", !args.GetBoolArg("-daemon", false)); + LogInstance().m_log_timestamps = args.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); + LogInstance().m_log_time_micros = args.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); #ifdef HAVE_THREAD_LOCAL - LogInstance().m_log_threadnames = gArgs.GetBoolArg("-logthreadnames", DEFAULT_LOGTHREADNAMES); + LogInstance().m_log_threadnames = args.GetBoolArg("-logthreadnames", DEFAULT_LOGTHREADNAMES); #endif - fLogIPs = gArgs.GetBoolArg("-logips", DEFAULT_LOGIPS); + fLogIPs = args.GetBoolArg("-logips", DEFAULT_LOGIPS); std::string version_string = FormatFullVersion(); #ifdef DEBUG @@ -908,7 +893,7 @@ std::set<BlockFilterType> g_enabled_filter_types; std::terminate(); }; -bool AppInitBasicSetup() +bool AppInitBasicSetup(ArgsManager& args) { // ********************************************************* Step 1: setup #ifdef _MSC_VER @@ -928,7 +913,7 @@ bool AppInitBasicSetup() } #ifndef WIN32 - if (!gArgs.GetBoolArg("-sysperms", false)) { + if (!args.GetBoolArg("-sysperms", false)) { umask(077); } @@ -950,7 +935,7 @@ bool AppInitBasicSetup() return true; } -bool AppInitParameterInteraction() +bool AppInitParameterInteraction(const ArgsManager& args) { const CChainParams& chainparams = Params(); // ********************************************************* Step 2: parameter interactions @@ -960,9 +945,9 @@ bool AppInitParameterInteraction() // Error if network-specific options (-addnode, -connect, etc) are // specified in default section of config file, but not overridden // on the command line or in this network's section of the config file. - std::string network = gArgs.GetChainName(); + std::string network = args.GetChainName(); bilingual_str errors; - for (const auto& arg : gArgs.GetUnsuitableSectionOnlyArgs()) { + for (const auto& arg : args.GetUnsuitableSectionOnlyArgs()) { errors += strprintf(_("Config setting for %s only applied on %s network when in [%s] section.") + Untranslated("\n"), arg, network, network); } @@ -972,7 +957,7 @@ bool AppInitParameterInteraction() // Warn if unrecognized section name are present in the config file. bilingual_str warnings; - for (const auto& section : gArgs.GetUnrecognizedSections()) { + for (const auto& section : args.GetUnrecognizedSections()) { warnings += strprintf(Untranslated("%s:%i ") + _("Section [%s] is not recognized.") + Untranslated("\n"), section.m_file, section.m_line, section.m_name); } @@ -981,15 +966,15 @@ bool AppInitParameterInteraction() } if (!fs::is_directory(GetBlocksDir())) { - return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist."), gArgs.GetArg("-blocksdir", ""))); + return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist."), args.GetArg("-blocksdir", ""))); } // parse and validate enabled filter types - std::string blockfilterindex_value = gArgs.GetArg("-blockfilterindex", DEFAULT_BLOCKFILTERINDEX); + std::string blockfilterindex_value = args.GetArg("-blockfilterindex", DEFAULT_BLOCKFILTERINDEX); if (blockfilterindex_value == "" || blockfilterindex_value == "1") { g_enabled_filter_types = AllBlockFilterTypes(); } else if (blockfilterindex_value != "0") { - const std::vector<std::string> names = gArgs.GetArgs("-blockfilterindex"); + const std::vector<std::string> names = args.GetArgs("-blockfilterindex"); for (const auto& name : names) { BlockFilterType filter_type; if (!BlockFilterTypeByName(name, filter_type)) { @@ -999,16 +984,18 @@ bool AppInitParameterInteraction() } } - // Basic filters are the only supported filters. The basic filters index must be enabled - // to serve compact filters - if (gArgs.GetBoolArg("-peerblockfilters", DEFAULT_PEERBLOCKFILTERS) && - g_enabled_filter_types.count(BlockFilterType::BASIC) != 1) { - return InitError(_("Cannot set -peerblockfilters without -blockfilterindex.")); + // Signal NODE_COMPACT_FILTERS if peerblockfilters and basic filters index are both enabled. + if (args.GetBoolArg("-peerblockfilters", DEFAULT_PEERBLOCKFILTERS)) { + if (g_enabled_filter_types.count(BlockFilterType::BASIC) != 1) { + return InitError(_("Cannot set -peerblockfilters without -blockfilterindex.")); + } + + nLocalServices = ServiceFlags(nLocalServices | NODE_COMPACT_FILTERS); } // if using block pruning, then disallow txindex - if (gArgs.GetArg("-prune", 0)) { - if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) + if (args.GetArg("-prune", 0)) { + if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) return InitError(_("Prune mode is incompatible with -txindex.")); if (!g_enabled_filter_types.empty()) { return InitError(_("Prune mode is incompatible with -blockfilterindex.")); @@ -1016,14 +1003,14 @@ bool AppInitParameterInteraction() } // -bind and -whitebind can't be set when not listening - size_t nUserBind = gArgs.GetArgs("-bind").size() + gArgs.GetArgs("-whitebind").size(); - if (nUserBind != 0 && !gArgs.GetBoolArg("-listen", DEFAULT_LISTEN)) { + size_t nUserBind = args.GetArgs("-bind").size() + args.GetArgs("-whitebind").size(); + if (nUserBind != 0 && !args.GetBoolArg("-listen", DEFAULT_LISTEN)) { return InitError(Untranslated("Cannot set -bind or -whitebind together with -listen=0")); } // Make sure enough file descriptors are available int nBind = std::max(nUserBind, size_t(1)); - nUserMaxConnections = gArgs.GetArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); + nUserMaxConnections = args.GetArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); nMaxConnections = std::max(nUserMaxConnections, 0); // Trim requested connection counts, to fit into system limitations @@ -1043,9 +1030,9 @@ bool AppInitParameterInteraction() InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); // ********************************************************* Step 3: parameter-to-internal-flags - if (gArgs.IsArgSet("-debug")) { + if (args.IsArgSet("-debug")) { // Special-case: if -debug=0/-nodebug is set, turn off debugging messages - const std::vector<std::string> categories = gArgs.GetArgs("-debug"); + const std::vector<std::string> categories = args.GetArgs("-debug"); if (std::none_of(categories.begin(), categories.end(), [](std::string cat){return cat == "0" || cat == "none";})) { @@ -1058,28 +1045,23 @@ bool AppInitParameterInteraction() } // Now remove the logging categories which were explicitly excluded - for (const std::string& cat : gArgs.GetArgs("-debugexclude")) { + for (const std::string& cat : args.GetArgs("-debugexclude")) { if (!LogInstance().DisableCategory(cat)) { InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); } } - // Checkmempool and checkblockindex default to true in regtest mode - int ratio = std::min<int>(std::max<int>(gArgs.GetArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0), 1000000); - if (ratio != 0) { - mempool.setSanityCheck(1.0 / ratio); - } - fCheckBlockIndex = gArgs.GetBoolArg("-checkblockindex", chainparams.DefaultConsistencyChecks()); - fCheckpointsEnabled = gArgs.GetBoolArg("-checkpoints", DEFAULT_CHECKPOINTS_ENABLED); + fCheckBlockIndex = args.GetBoolArg("-checkblockindex", chainparams.DefaultConsistencyChecks()); + fCheckpointsEnabled = args.GetBoolArg("-checkpoints", DEFAULT_CHECKPOINTS_ENABLED); - hashAssumeValid = uint256S(gArgs.GetArg("-assumevalid", chainparams.GetConsensus().defaultAssumeValid.GetHex())); + hashAssumeValid = uint256S(args.GetArg("-assumevalid", chainparams.GetConsensus().defaultAssumeValid.GetHex())); if (!hashAssumeValid.IsNull()) LogPrintf("Assuming ancestors of block %s have valid signatures.\n", hashAssumeValid.GetHex()); else LogPrintf("Validating signatures for all blocks.\n"); - if (gArgs.IsArgSet("-minimumchainwork")) { - const std::string minChainWorkStr = gArgs.GetArg("-minimumchainwork", ""); + if (args.IsArgSet("-minimumchainwork")) { + const std::string minChainWorkStr = args.GetArg("-minimumchainwork", ""); if (!IsHexNumber(minChainWorkStr)) { return InitError(strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), minChainWorkStr)); } @@ -1093,22 +1075,21 @@ bool AppInitParameterInteraction() } // mempool limits - int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - int64_t nMempoolSizeMin = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000 * 40; + int64_t nMempoolSizeMax = args.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; + int64_t nMempoolSizeMin = args.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000 * 40; if (nMempoolSizeMax < 0 || nMempoolSizeMax < nMempoolSizeMin) return InitError(strprintf(_("-maxmempool must be at least %d MB"), std::ceil(nMempoolSizeMin / 1000000.0))); // incremental relay fee sets the minimum feerate increase necessary for BIP 125 replacement in the mempool // and the amount the mempool min fee increases above the feerate of txs evicted due to mempool limiting. - if (gArgs.IsArgSet("-incrementalrelayfee")) - { + if (args.IsArgSet("-incrementalrelayfee")) { CAmount n = 0; - if (!ParseMoney(gArgs.GetArg("-incrementalrelayfee", ""), n)) - return InitError(AmountErrMsg("incrementalrelayfee", gArgs.GetArg("-incrementalrelayfee", ""))); + if (!ParseMoney(args.GetArg("-incrementalrelayfee", ""), n)) + return InitError(AmountErrMsg("incrementalrelayfee", args.GetArg("-incrementalrelayfee", ""))); incrementalRelayFee = CFeeRate(n); } // block pruning; get the amount of disk space (in MiB) to allot for block & undo files - int64_t nPruneArg = gArgs.GetArg("-prune", 0); + int64_t nPruneArg = args.GetArg("-prune", 0); if (nPruneArg < 0) { return InitError(_("Prune cannot be configured with a negative value.")); } @@ -1125,20 +1106,20 @@ bool AppInitParameterInteraction() fPruneMode = true; } - nConnectTimeout = gArgs.GetArg("-timeout", DEFAULT_CONNECT_TIMEOUT); + nConnectTimeout = args.GetArg("-timeout", DEFAULT_CONNECT_TIMEOUT); if (nConnectTimeout <= 0) { nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; } - peer_connect_timeout = gArgs.GetArg("-peertimeout", DEFAULT_PEER_CONNECT_TIMEOUT); + peer_connect_timeout = args.GetArg("-peertimeout", DEFAULT_PEER_CONNECT_TIMEOUT); if (peer_connect_timeout <= 0) { return InitError(Untranslated("peertimeout cannot be configured with a negative value.")); } - if (gArgs.IsArgSet("-minrelaytxfee")) { + if (args.IsArgSet("-minrelaytxfee")) { CAmount n = 0; - if (!ParseMoney(gArgs.GetArg("-minrelaytxfee", ""), n)) { - return InitError(AmountErrMsg("minrelaytxfee", gArgs.GetArg("-minrelaytxfee", ""))); + if (!ParseMoney(args.GetArg("-minrelaytxfee", ""), n)) { + return InitError(AmountErrMsg("minrelaytxfee", args.GetArg("-minrelaytxfee", ""))); } // High fee check is done afterward in CWallet::CreateWalletFromFile() ::minRelayTxFee = CFeeRate(n); @@ -1150,48 +1131,46 @@ bool AppInitParameterInteraction() // Sanity check argument for min fee for including tx in block // TODO: Harmonize which arguments need sanity checking and where that happens - if (gArgs.IsArgSet("-blockmintxfee")) - { + if (args.IsArgSet("-blockmintxfee")) { CAmount n = 0; - if (!ParseMoney(gArgs.GetArg("-blockmintxfee", ""), n)) - return InitError(AmountErrMsg("blockmintxfee", gArgs.GetArg("-blockmintxfee", ""))); + if (!ParseMoney(args.GetArg("-blockmintxfee", ""), n)) + return InitError(AmountErrMsg("blockmintxfee", args.GetArg("-blockmintxfee", ""))); } // Feerate used to define dust. Shouldn't be changed lightly as old // implementations may inadvertently create non-standard transactions - if (gArgs.IsArgSet("-dustrelayfee")) - { + if (args.IsArgSet("-dustrelayfee")) { CAmount n = 0; - if (!ParseMoney(gArgs.GetArg("-dustrelayfee", ""), n)) - return InitError(AmountErrMsg("dustrelayfee", gArgs.GetArg("-dustrelayfee", ""))); + if (!ParseMoney(args.GetArg("-dustrelayfee", ""), n)) + return InitError(AmountErrMsg("dustrelayfee", args.GetArg("-dustrelayfee", ""))); dustRelayFee = CFeeRate(n); } - fRequireStandard = !gArgs.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); + fRequireStandard = !args.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); if (!chainparams.IsTestChain() && !fRequireStandard) { return InitError(strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.NetworkIDString())); } - nBytesPerSigOp = gArgs.GetArg("-bytespersigop", nBytesPerSigOp); + nBytesPerSigOp = args.GetArg("-bytespersigop", nBytesPerSigOp); if (!g_wallet_init_interface.ParameterInteraction()) return false; - fIsBareMultisigStd = gArgs.GetBoolArg("-permitbaremultisig", DEFAULT_PERMIT_BAREMULTISIG); - fAcceptDatacarrier = gArgs.GetBoolArg("-datacarrier", DEFAULT_ACCEPT_DATACARRIER); - nMaxDatacarrierBytes = gArgs.GetArg("-datacarriersize", nMaxDatacarrierBytes); + fIsBareMultisigStd = args.GetBoolArg("-permitbaremultisig", DEFAULT_PERMIT_BAREMULTISIG); + fAcceptDatacarrier = args.GetBoolArg("-datacarrier", DEFAULT_ACCEPT_DATACARRIER); + nMaxDatacarrierBytes = args.GetArg("-datacarriersize", nMaxDatacarrierBytes); // Option to startup with mocktime set (used for regression testing): - SetMockTime(gArgs.GetArg("-mocktime", 0)); // SetMockTime(0) is a no-op + SetMockTime(args.GetArg("-mocktime", 0)); // SetMockTime(0) is a no-op - if (gArgs.GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) + if (args.GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); - if (gArgs.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) < 0) + if (args.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) < 0) return InitError(Untranslated("rpcserialversion must be non-negative.")); - if (gArgs.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) + if (args.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) return InitError(Untranslated("Unknown rpcserialversion requested.")); - nMaxTipAge = gArgs.GetArg("-maxtipage", DEFAULT_MAX_TIP_AGE); + nMaxTipAge = args.GetArg("-maxtipage", DEFAULT_MAX_TIP_AGE); return true; } @@ -1242,16 +1221,28 @@ bool AppInitLockDataDirectory() return true; } -bool AppInitMain(const util::Ref& context, NodeContext& node) +bool AppInitInterfaces(NodeContext& node) +{ + node.chain = interfaces::MakeChain(node); + // Create client interfaces for wallets that are supposed to be loaded + // according to -wallet and -disablewallet options. This only constructs + // the interfaces, it doesn't load wallet data. Wallets actually get loaded + // when load() and start() interface methods are called below. + g_wallet_init_interface.Construct(node); + return true; +} + +bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) { + const ArgsManager& args = *Assert(node.args); const CChainParams& chainparams = Params(); // ********************************************************* Step 4a: application initialization - if (!CreatePidFile()) { + if (!CreatePidFile(args)) { // Detailed error printed inside CreatePidFile(). return false; } if (LogInstance().m_print_to_file) { - if (gArgs.GetBoolArg("-shrinkdebugfile", LogInstance().DefaultShrinkDebugFile())) { + if (args.GetBoolArg("-shrinkdebugfile", LogInstance().DefaultShrinkDebugFile())) { // Do this first since it both loads a bunch of debug.log into memory, // and because this needs to happen before any other debug.log printing LogInstance().ShrinkDebugFile(); @@ -1268,10 +1259,10 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) LogPrintf("Using data directory %s\n", GetDataDir().string()); // Only log conf file usage message if conf file actually exists. - fs::path config_file_path = GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME)); + fs::path config_file_path = GetConfigFile(args.GetArg("-conf", BITCOIN_CONF_FILENAME)); if (fs::exists(config_file_path)) { LogPrintf("Config file: %s\n", config_file_path.string()); - } else if (gArgs.IsArgSet("-conf")) { + } else if (args.IsArgSet("-conf")) { // Warn if no conf file exists at path provided by user InitWarning(strprintf(_("The specified config file %s does not exist\n"), config_file_path.string())); } else { @@ -1280,23 +1271,23 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) } // Log the config arguments to debug.log - gArgs.LogArgs(); + args.LogArgs(); LogPrintf("Using at most %i automatic connections (%i file descriptors available)\n", nMaxConnections, nFD); // Warn about relative -datadir path. - if (gArgs.IsArgSet("-datadir") && !fs::path(gArgs.GetArg("-datadir", "")).is_absolute()) { + if (args.IsArgSet("-datadir") && !fs::path(args.GetArg("-datadir", "")).is_absolute()) { LogPrintf("Warning: relative datadir option '%s' specified, which will be interpreted relative to the " /* Continued */ "current working directory '%s'. This is fragile, because if bitcoin is started in the future " "from a different location, it will be unable to locate the current data files. There could " "also be data loss if bitcoin is started while in a temporary directory.\n", - gArgs.GetArg("-datadir", ""), fs::current_path().string()); + args.GetArg("-datadir", ""), fs::current_path().string()); } InitSignatureCache(); InitScriptExecutionCache(); - int script_threads = gArgs.GetArg("-par", DEFAULT_SCRIPTCHECK_THREADS); + int script_threads = args.GetArg("-par", DEFAULT_SCRIPTCHECK_THREADS); if (script_threads <= 0) { // -par=0 means autodetect (number of cores - 1 script threads) // -par=-n means "leave n cores free" (number of cores - n - 1 script threads) @@ -1330,12 +1321,6 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) GetMainSignals().RegisterBackgroundSignalScheduler(*node.scheduler); - // Create client interfaces for wallets that are supposed to be loaded - // according to -wallet and -disablewallet options. This only constructs - // the interfaces, it doesn't load wallet data. Wallets actually get loaded - // when load() and start() interface methods are called below. - g_wallet_init_interface.Construct(node); - /* Register RPC commands regardless of -server setting so they will be * available in the GUI RPC console even if external calls are disabled. */ @@ -1352,8 +1337,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) * that the server is there and will be ready later). Warmup mode will * be disabled when initialisation is finished. */ - if (gArgs.GetBoolArg("-server", false)) - { + if (args.GetBoolArg("-server", false)) { uiInterface.InitMessage_connect(SetRPCWarmupStatus); if (!AppInitServers(context, node)) return InitError(_("Unable to start HTTP server. See debug log for details.")); @@ -1373,23 +1357,31 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) // need to reindex later. assert(!node.banman); - node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", &uiInterface, gArgs.GetArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); + node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", &uiInterface, args.GetArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); assert(!node.connman); - node.connman = MakeUnique<CConnman>(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()), gArgs.GetBoolArg("-networkactive", true)); + node.connman = MakeUnique<CConnman>(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()), args.GetBoolArg("-networkactive", true)); + // Make mempool generally available in the node context. For example the connection manager, wallet, or RPC threads, // which are all started after this, may use it from the node context. assert(!node.mempool); - node.mempool = &::mempool; + node.mempool = MakeUnique<CTxMemPool>(&::feeEstimator); + if (node.mempool) { + int ratio = std::min<int>(std::max<int>(args.GetArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0), 1000000); + if (ratio != 0) { + node.mempool->setSanityCheck(1.0 / ratio); + } + } + assert(!node.chainman); node.chainman = &g_chainman; ChainstateManager& chainman = *Assert(node.chainman); - node.peer_logic.reset(new PeerLogicValidation(node.connman.get(), node.banman.get(), *node.scheduler, chainman, *node.mempool)); - RegisterValidationInterface(node.peer_logic.get()); + node.peerman.reset(new PeerManager(chainparams, *node.connman, node.banman.get(), *node.scheduler, chainman, *node.mempool)); + RegisterValidationInterface(node.peerman.get()); // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; - for (const std::string& cmt : gArgs.GetArgs("-uacomment")) { + for (const std::string& cmt : args.GetArgs("-uacomment")) { if (cmt != SanitizeString(cmt, SAFE_CHARS_UA_COMMENT)) return InitError(strprintf(_("User Agent comment (%s) contains unsafe characters."), cmt)); uacomments.push_back(cmt); @@ -1400,9 +1392,9 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) strSubVersion.size(), MAX_SUBVERSION_LENGTH)); } - if (gArgs.IsArgSet("-onlynet")) { + if (args.IsArgSet("-onlynet")) { std::set<enum Network> nets; - for (const std::string& snet : gArgs.GetArgs("-onlynet")) { + for (const std::string& snet : args.GetArgs("-onlynet")) { enum Network net = ParseNetwork(snet); if (net == NET_UNROUTABLE) return InitError(strprintf(_("Unknown network specified in -onlynet: '%s'"), snet)); @@ -1416,12 +1408,12 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) } // Check for host lookup allowed before parsing any network related parameters - fNameLookup = gArgs.GetBoolArg("-dns", DEFAULT_NAME_LOOKUP); + fNameLookup = args.GetBoolArg("-dns", DEFAULT_NAME_LOOKUP); - bool proxyRandomize = gArgs.GetBoolArg("-proxyrandomize", DEFAULT_PROXYRANDOMIZE); + bool proxyRandomize = args.GetBoolArg("-proxyrandomize", DEFAULT_PROXYRANDOMIZE); // -proxy sets a proxy for all outgoing network traffic // -noproxy (or -proxy=0) as well as the empty string can be used to not set a proxy, this is the default - std::string proxyArg = gArgs.GetArg("-proxy", ""); + std::string proxyArg = args.GetArg("-proxy", ""); SetReachable(NET_ONION, false); if (proxyArg != "" && proxyArg != "0") { CService proxyAddr; @@ -1443,7 +1435,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) // -onion can be used to set only a proxy for .onion, or override normal proxy for .onion addresses // -noonion (or -onion=0) disables connecting to .onion entirely // An empty string is used to not override the onion proxy (in which case it defaults to -proxy set above, or none) - std::string onionArg = gArgs.GetArg("-onion", ""); + std::string onionArg = args.GetArg("-onion", ""); if (onionArg != "") { if (onionArg == "0") { // Handle -noonion/-onion=0 SetReachable(NET_ONION, false); @@ -1461,11 +1453,11 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) } // see Step 2: parameter interactions for more information about these - fListen = gArgs.GetBoolArg("-listen", DEFAULT_LISTEN); - fDiscover = gArgs.GetBoolArg("-discover", true); - g_relay_txes = !gArgs.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY); + fListen = args.GetBoolArg("-listen", DEFAULT_LISTEN); + fDiscover = args.GetBoolArg("-discover", true); + g_relay_txes = !args.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY); - for (const std::string& strAddr : gArgs.GetArgs("-externalip")) { + for (const std::string& strAddr : args.GetArgs("-externalip")) { CService addrLocal; if (Lookup(strAddr, addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) AddLocal(addrLocal, LOCAL_MANUAL); @@ -1474,8 +1466,8 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) } // Read asmap file if configured - if (gArgs.IsArgSet("-asmap")) { - fs::path asmap_path = fs::path(gArgs.GetArg("-asmap", "")); + if (args.IsArgSet("-asmap")) { + fs::path asmap_path = fs::path(args.GetArg("-asmap", "")); if (asmap_path.empty()) { asmap_path = DEFAULT_ASMAP_FILENAME; } @@ -1508,22 +1500,22 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) uint64_t nMaxOutboundLimit = 0; //unlimited unless -maxuploadtarget is set uint64_t nMaxOutboundTimeframe = MAX_UPLOAD_TIMEFRAME; - if (gArgs.IsArgSet("-maxuploadtarget")) { - nMaxOutboundLimit = gArgs.GetArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET)*1024*1024; + if (args.IsArgSet("-maxuploadtarget")) { + nMaxOutboundLimit = args.GetArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET) * 1024 * 1024; } // ********************************************************* Step 7: load block chain - fReindex = gArgs.GetBoolArg("-reindex", false); - bool fReindexChainState = gArgs.GetBoolArg("-reindex-chainstate", false); + fReindex = args.GetBoolArg("-reindex", false); + bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); // cache size calculations - int64_t nTotalCache = (gArgs.GetArg("-dbcache", nDefaultDbCache) << 20); + int64_t nTotalCache = (args.GetArg("-dbcache", nDefaultDbCache) << 20); nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greater than nMaxDbcache int64_t nBlockTreeDBCache = std::min(nTotalCache / 8, nMaxBlockDBCache << 20); nTotalCache -= nBlockTreeDBCache; - int64_t nTxIndexCache = std::min(nTotalCache / 8, gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0); + int64_t nTxIndexCache = std::min(nTotalCache / 8, args.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0); nTotalCache -= nTxIndexCache; int64_t filter_index_cache = 0; if (!g_enabled_filter_types.empty()) { @@ -1536,10 +1528,10 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; int64_t nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache - int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; + int64_t nMempoolSizeMax = args.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; LogPrintf("Cache configuration:\n"); LogPrintf("* Using %.1f MiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); - if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { LogPrintf("* Using %.1f MiB for transaction index database\n", nTxIndexCache * (1.0 / 1024 / 1024)); } for (BlockFilterType filter_type : g_enabled_filter_types) { @@ -1563,11 +1555,11 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) const int64_t load_block_index_start_time = GetTimeMillis(); try { LOCK(cs_main); - chainman.InitializeChainstate(); + chainman.InitializeChainstate(*Assert(node.mempool)); chainman.m_total_coinstip_cache = nCoinCacheUsage; chainman.m_total_coinsdb_cache = nCoinDBCache; - UnloadBlockIndex(node.mempool); + UnloadBlockIndex(node.mempool.get()); // new CBlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: @@ -1704,7 +1696,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) for (CChainState* chainstate : chainman.GetAll()) { if (!is_coinsview_empty(chainstate)) { uiInterface.InitMessage(_("Verifying blocks...").translated); - if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { + if (fHavePruned && args.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks\n", MIN_BLOCKS_TO_KEEP); } @@ -1722,10 +1714,10 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) // Only verify the DB of the active chainstate. This is fixed in later // work when we allow VerifyDB to be parameterized by chainstate. if (&::ChainstateActive() == chainstate && - !CVerifyDB().VerifyDB( + !CVerifyDB().VerifyDB( chainparams, &chainstate->CoinsDB(), - gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), - gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { + args.GetArg("-checklevel", DEFAULT_CHECKLEVEL), + args.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { strLoadError = _("Corrupted block database detected"); failed_verification = true; break; @@ -1781,7 +1773,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) fFeeEstimatesInitialized = true; // ********************************************************* Step 8: start indexers - if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { g_txindex = MakeUnique<TxIndex>(nTxIndexCache, false, fReindex); g_txindex->Start(); } @@ -1841,16 +1833,26 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) } #if HAVE_SYSTEM - if (gArgs.IsArgSet("-blocknotify")) - uiInterface.NotifyBlockTip_connect(BlockNotifyCallback); + const std::string block_notify = args.GetArg("-blocknotify", ""); + if (!block_notify.empty()) { + uiInterface.NotifyBlockTip_connect([block_notify](SynchronizationState sync_state, const CBlockIndex* pBlockIndex) { + if (sync_state != SynchronizationState::POST_INIT || !pBlockIndex) return; + std::string command = block_notify; + boost::replace_all(command, "%s", pBlockIndex->GetBlockHash().GetHex()); + std::thread t(runCommand, command); + t.detach(); // thread runs free + }); + } #endif std::vector<fs::path> vImportFiles; - for (const std::string& strFile : gArgs.GetArgs("-loadblock")) { + for (const std::string& strFile : args.GetArgs("-loadblock")) { vImportFiles.push_back(strFile); } - g_load_block = std::thread(&TraceThread<std::function<void()>>, "loadblk", [=, &chainman]{ ThreadImport(chainman, vImportFiles); }); + g_load_block = std::thread(&TraceThread<std::function<void()>>, "loadblk", [=, &chainman, &args] { + ThreadImport(chainman, vImportFiles, args); + }); // Wait for genesis block to be processed { @@ -1877,16 +1879,25 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) LOCK(cs_main); LogPrintf("block tree size = %u\n", chainman.BlockIndex().size()); chain_active_height = chainman.ActiveChain().Height(); + if (tip_info) { + tip_info->block_height = chain_active_height; + tip_info->block_time = chainman.ActiveChain().Tip() ? chainman.ActiveChain().Tip()->GetBlockTime() : Params().GenesisBlock().GetBlockTime(); + tip_info->verification_progress = GuessVerificationProgress(Params().TxData(), chainman.ActiveChain().Tip()); + } + if (tip_info && ::pindexBestHeader) { + tip_info->header_height = ::pindexBestHeader->nHeight; + tip_info->header_time = ::pindexBestHeader->GetBlockTime(); + } } LogPrintf("nBestHeight = %d\n", chain_active_height); - if (gArgs.GetBoolArg("-listenonion", DEFAULT_LISTEN_ONION)) + if (args.GetBoolArg("-listenonion", DEFAULT_LISTEN_ONION)) StartTorControl(); Discover(); // Map ports with UPnP - if (gArgs.GetBoolArg("-upnp", DEFAULT_UPNP)) { + if (args.GetBoolArg("-upnp", DEFAULT_UPNP)) { StartMapPort(); } @@ -1900,42 +1911,42 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) connOptions.nBestHeight = chain_active_height; connOptions.uiInterface = &uiInterface; connOptions.m_banman = node.banman.get(); - connOptions.m_msgproc = node.peer_logic.get(); - connOptions.nSendBufferMaxSize = 1000*gArgs.GetArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); - connOptions.nReceiveFloodSize = 1000*gArgs.GetArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); - connOptions.m_added_nodes = gArgs.GetArgs("-addnode"); + connOptions.m_msgproc = node.peerman.get(); + connOptions.nSendBufferMaxSize = 1000 * args.GetArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); + connOptions.nReceiveFloodSize = 1000 * args.GetArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); + connOptions.m_added_nodes = args.GetArgs("-addnode"); connOptions.nMaxOutboundTimeframe = nMaxOutboundTimeframe; connOptions.nMaxOutboundLimit = nMaxOutboundLimit; connOptions.m_peer_connect_timeout = peer_connect_timeout; - for (const std::string& strBind : gArgs.GetArgs("-bind")) { + for (const std::string& strBind : args.GetArgs("-bind")) { CService addrBind; if (!Lookup(strBind, addrBind, GetListenPort(), false)) { return InitError(ResolveErrMsg("bind", strBind)); } connOptions.vBinds.push_back(addrBind); } - for (const std::string& strBind : gArgs.GetArgs("-whitebind")) { + for (const std::string& strBind : args.GetArgs("-whitebind")) { NetWhitebindPermissions whitebind; bilingual_str error; if (!NetWhitebindPermissions::TryParse(strBind, whitebind, error)) return InitError(error); connOptions.vWhiteBinds.push_back(whitebind); } - for (const auto& net : gArgs.GetArgs("-whitelist")) { + for (const auto& net : args.GetArgs("-whitelist")) { NetWhitelistPermissions subnet; bilingual_str error; if (!NetWhitelistPermissions::TryParse(net, subnet, error)) return InitError(error); connOptions.vWhitelistedRange.push_back(subnet); } - connOptions.vSeedNodes = gArgs.GetArgs("-seednode"); + connOptions.vSeedNodes = args.GetArgs("-seednode"); // Initiate outbound connections unless connect=0 - connOptions.m_use_addrman_outgoing = !gArgs.IsArgSet("-connect"); + connOptions.m_use_addrman_outgoing = !args.IsArgSet("-connect"); if (!connOptions.m_use_addrman_outgoing) { - const auto connect = gArgs.GetArgs("-connect"); + const auto connect = args.GetArgs("-connect"); if (connect.size() != 1 || connect[0] != "0") { connOptions.m_specified_outgoing = connect; } diff --git a/src/init.h b/src/init.h index 33fe96e8e..679e875da 100644 --- a/src/init.h +++ b/src/init.h @@ -8,9 +8,12 @@ #include <memory> #include <string> -#include <util/system.h> +class ArgsManager; struct NodeContext; +namespace interfaces { +struct BlockAndHeaderTipInfo; +} namespace boost { class thread_group; } // namespace boost @@ -22,21 +25,21 @@ class Ref; void Interrupt(NodeContext& node); void Shutdown(NodeContext& node); //!Initialize the logging infrastructure -void InitLogging(); +void InitLogging(const ArgsManager& args); //!Parameter interaction: change current parameters depending on various rules -void InitParameterInteraction(); +void InitParameterInteraction(ArgsManager& args); /** Initialize bitcoin core: Basic context setup. * @note This can be done before daemonization. Do not call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read. */ -bool AppInitBasicSetup(); +bool AppInitBasicSetup(ArgsManager& args); /** * Initialization: parameter interaction. * @note This can be done before daemonization. Do not call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read, AppInitBasicSetup should have been called. */ -bool AppInitParameterInteraction(); +bool AppInitParameterInteraction(const ArgsManager& args); /** * Initialization sanity checks: ecc init, sanity checks, dir lock. * @note This can be done before daemonization. Do not call Shutdown() if this function fails. @@ -50,11 +53,15 @@ bool AppInitSanityChecks(); */ bool AppInitLockDataDirectory(); /** + * Initialize node and wallet interface pointers. Has no prerequisites or side effects besides allocating memory. + */ +bool AppInitInterfaces(NodeContext& node); +/** * Bitcoin core main initialization. * @note This should only be done after daemonization. Call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read, AppInitLockDataDirectory should have been called. */ -bool AppInitMain(const util::Ref& context, NodeContext& node); +bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info = nullptr); /** * Register all arguments with the ArgsManager diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index d49e4454a..13d885a20 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -276,13 +276,15 @@ public: } RBFTransactionState isRBFOptIn(const CTransaction& tx) override { - LOCK(::mempool.cs); - return IsRBFOptIn(tx, ::mempool); + if (!m_node.mempool) return IsRBFOptInEmptyMempool(tx); + LOCK(m_node.mempool->cs); + return IsRBFOptIn(tx, *m_node.mempool); } bool hasDescendantsInMempool(const uint256& txid) override { - LOCK(::mempool.cs); - auto it = ::mempool.GetIter(txid); + if (!m_node.mempool) return false; + LOCK(m_node.mempool->cs); + auto it = m_node.mempool->GetIter(txid); return it && (*it)->GetCountWithDescendants() > 1; } bool broadcastTransaction(const CTransactionRef& tx, @@ -298,7 +300,9 @@ public: } void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) override { - ::mempool.GetTransactionAncestry(txid, ancestors, descendants); + ancestors = descendants = 0; + if (!m_node.mempool) return; + m_node.mempool->GetTransactionAncestry(txid, ancestors, descendants); } void getPackageLimits(unsigned int& limit_ancestor_count, unsigned int& limit_descendant_count) override { @@ -307,6 +311,7 @@ public: } bool checkChainLimits(const CTransactionRef& tx) override { + if (!m_node.mempool) return true; LockPoints lp; CTxMemPoolEntry entry(tx, 0, 0, 0, false, 0, lp); CTxMemPool::setEntries ancestors; @@ -315,8 +320,9 @@ public: auto limit_descendant_count = gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); auto limit_descendant_size = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000; std::string unused_error_string; - LOCK(::mempool.cs); - return ::mempool.CalculateMemPoolAncestors(entry, ancestors, limit_ancestor_count, limit_ancestor_size, + LOCK(m_node.mempool->cs); + return m_node.mempool->CalculateMemPoolAncestors( + entry, ancestors, limit_ancestor_count, limit_ancestor_size, limit_descendant_count, limit_descendant_size, unused_error_string); } CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc) override @@ -329,7 +335,8 @@ public: } CFeeRate mempoolMinFee() override { - return ::mempool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + if (!m_node.mempool) return {}; + return m_node.mempool->GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); } CFeeRate relayMinFee() override { return ::minRelayTxFee; } CFeeRate relayIncrementalFee() override { return ::incrementalRelayFee; } @@ -372,10 +379,32 @@ public: RPCRunLater(name, std::move(fn), seconds); } int rpcSerializationFlags() override { return RPCSerializationFlags(); } + util::SettingsValue getRwSetting(const std::string& name) override + { + util::SettingsValue result; + gArgs.LockSettings([&](const util::Settings& settings) { + if (const util::SettingsValue* value = util::FindKey(settings.rw_settings, name)) { + result = *value; + } + }); + return result; + } + bool updateRwSetting(const std::string& name, const util::SettingsValue& value) override + { + gArgs.LockSettings([&](util::Settings& settings) { + if (value.isNull()) { + settings.rw_settings.erase(name); + } else { + settings.rw_settings[name] = value; + } + }); + return gArgs.WriteSettingsFile(); + } void requestMempoolTransactions(Notifications& notifications) override { - LOCK2(::cs_main, ::mempool.cs); - for (const CTxMemPoolEntry& entry : ::mempool.mapTx) { + if (!m_node.mempool) return; + LOCK2(::cs_main, m_node.mempool->cs); + for (const CTxMemPoolEntry& entry : m_node.mempool->mapTx) { notifications.transactionAddedToMempool(entry.GetSharedTx()); } } diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index bbeb0fa80..6e50ccb27 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -7,6 +7,7 @@ #include <optional.h> // For Optional and nullopt #include <primitives/transaction.h> // For CTransactionRef +#include <util/settings.h> // For util::SettingsValue #include <functional> #include <memory> @@ -269,6 +270,12 @@ public: //! Current RPC serialization flags. virtual int rpcSerializationFlags() = 0; + //! Return <datadir>/settings.json setting value. + virtual util::SettingsValue getRwSetting(const std::string& name) = 0; + + //! Write a setting to <datadir>/settings.json. + virtual bool updateRwSetting(const std::string& name, const util::SettingsValue& value) = 0; + //! Synchronously send transactionAddedToMempool notifications about all //! current mempool transactions to the specified handler and return after //! the last one is sent. These notifications aren't coordinated with async @@ -307,24 +314,11 @@ public: //! Set mock time. virtual void setMockTime(int64_t time) = 0; - - //! Return interfaces for accessing wallets (if any). - virtual std::vector<std::unique_ptr<Wallet>> getWallets() = 0; }; //! Return implementation of Chain interface. std::unique_ptr<Chain> MakeChain(NodeContext& node); -//! Return implementation of ChainClient interface for a wallet client. This -//! function will be undefined in builds where ENABLE_WALLET is false. -//! -//! Currently, wallets are the only chain clients. But in the future, other -//! types of chain clients could be added, such as tools for monitoring, -//! analysis, or fee estimation. These clients need to expose their own -//! MakeXXXClient functions returning their implementations of the ChainClient -//! interface. -std::unique_ptr<ChainClient> MakeWalletClient(Chain& chain, ArgsManager& args, std::vector<std::string> wallet_filenames); - } // namespace interfaces #endif // BITCOIN_INTERFACES_CHAIN_H diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 33f0dac26..2c5f8627e 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -27,6 +27,7 @@ #include <support/allocators/secure.h> #include <sync.h> #include <txmempool.h> +#include <util/check.h> #include <util/ref.h> #include <util/system.h> #include <util/translation.h> @@ -41,53 +42,30 @@ #include <boost/signals2/signal.hpp> -class CWallet; -fs::path GetWalletDir(); -std::vector<fs::path> ListWalletDir(); -std::vector<std::shared_ptr<CWallet>> GetWallets(); -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings); -WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, std::shared_ptr<CWallet>& result); -std::unique_ptr<interfaces::Handler> HandleLoadWallet(interfaces::Node::LoadWalletFn load_wallet); - namespace interfaces { - namespace { class NodeImpl : public Node { public: - void initError(const bilingual_str& message) override { InitError(message); } - bool parseParameters(int argc, const char* const argv[], std::string& error) override - { - return gArgs.ParseParameters(argc, argv, error); - } - bool readConfigFiles(std::string& error) override { return gArgs.ReadConfigFiles(error, true); } - void forceSetArg(const std::string& arg, const std::string& value) override { gArgs.ForceSetArg(arg, value); } - bool softSetArg(const std::string& arg, const std::string& value) override { return gArgs.SoftSetArg(arg, value); } - bool softSetBoolArg(const std::string& arg, bool value) override { return gArgs.SoftSetBoolArg(arg, value); } - void selectParams(const std::string& network) override { SelectParams(network); } - bool initSettings(std::string& error) override { return gArgs.InitSettings(error); } - uint64_t getAssumedBlockchainSize() override { return Params().AssumedBlockchainSize(); } - uint64_t getAssumedChainStateSize() override { return Params().AssumedChainStateSize(); } - std::string getNetwork() override { return Params().NetworkIDString(); } - void initLogging() override { InitLogging(); } - void initParameterInteraction() override { InitParameterInteraction(); } + NodeImpl(NodeContext* context) { setContext(context); } + void initLogging() override { InitLogging(*Assert(m_context->args)); } + void initParameterInteraction() override { InitParameterInteraction(*Assert(m_context->args)); } bilingual_str getWarnings() override { return GetWarnings(true); } uint32_t getLogCategories() override { return LogInstance().GetCategoryMask(); } bool baseInitialize() override { - return AppInitBasicSetup() && AppInitParameterInteraction() && AppInitSanityChecks() && - AppInitLockDataDirectory(); + return AppInitBasicSetup(gArgs) && AppInitParameterInteraction(gArgs) && AppInitSanityChecks() && + AppInitLockDataDirectory() && AppInitInterfaces(*m_context); } - bool appInitMain() override + bool appInitMain(interfaces::BlockAndHeaderTipInfo* tip_info) override { - m_context.chain = MakeChain(m_context); - return AppInitMain(m_context_ref, m_context); + return AppInitMain(m_context_ref, *m_context, tip_info); } void appShutdown() override { - Interrupt(m_context); - Shutdown(m_context); + Interrupt(*m_context); + Shutdown(*m_context); } void startShutdown() override { @@ -108,19 +86,18 @@ public: StopMapPort(); } } - void setupServerArgs() override { return SetupServerArgs(m_context); } bool getProxy(Network net, proxyType& proxy_info) override { return GetProxy(net, proxy_info); } size_t getNodeCount(CConnman::NumConnections flags) override { - return m_context.connman ? m_context.connman->GetNodeCount(flags) : 0; + return m_context->connman ? m_context->connman->GetNodeCount(flags) : 0; } bool getNodesStats(NodesStats& stats) override { stats.clear(); - if (m_context.connman) { + if (m_context->connman) { std::vector<CNodeStats> stats_temp; - m_context.connman->GetNodeStats(stats_temp); + m_context->connman->GetNodeStats(stats_temp); stats.reserve(stats_temp.size()); for (auto& node_stats_temp : stats_temp) { @@ -141,46 +118,46 @@ public: } bool getBanned(banmap_t& banmap) override { - if (m_context.banman) { - m_context.banman->GetBanned(banmap); + if (m_context->banman) { + m_context->banman->GetBanned(banmap); return true; } return false; } bool ban(const CNetAddr& net_addr, int64_t ban_time_offset) override { - if (m_context.banman) { - m_context.banman->Ban(net_addr, ban_time_offset); + if (m_context->banman) { + m_context->banman->Ban(net_addr, ban_time_offset); return true; } return false; } bool unban(const CSubNet& ip) override { - if (m_context.banman) { - m_context.banman->Unban(ip); + if (m_context->banman) { + m_context->banman->Unban(ip); return true; } return false; } bool disconnectByAddress(const CNetAddr& net_addr) override { - if (m_context.connman) { - return m_context.connman->DisconnectNode(net_addr); + if (m_context->connman) { + return m_context->connman->DisconnectNode(net_addr); } return false; } bool disconnectById(NodeId id) override { - if (m_context.connman) { - return m_context.connman->DisconnectNode(id); + if (m_context->connman) { + return m_context->connman->DisconnectNode(id); } return false; } - int64_t getTotalBytesRecv() override { return m_context.connman ? m_context.connman->GetTotalBytesRecv() : 0; } - int64_t getTotalBytesSent() override { return m_context.connman ? m_context.connman->GetTotalBytesSent() : 0; } - size_t getMempoolSize() override { return m_context.mempool ? m_context.mempool->size() : 0; } - size_t getMempoolDynamicUsage() override { return m_context.mempool ? m_context.mempool->DynamicMemoryUsage() : 0; } + int64_t getTotalBytesRecv() override { return m_context->connman ? m_context->connman->GetTotalBytesRecv() : 0; } + int64_t getTotalBytesSent() override { return m_context->connman ? m_context->connman->GetTotalBytesSent() : 0; } + size_t getMempoolSize() override { return m_context->mempool ? m_context->mempool->size() : 0; } + size_t getMempoolDynamicUsage() override { return m_context->mempool ? m_context->mempool->DynamicMemoryUsage() : 0; } bool getHeaderTip(int& height, int64_t& block_time) override { LOCK(::cs_main); @@ -223,11 +200,11 @@ public: bool getImporting() override { return ::fImporting; } void setNetworkActive(bool active) override { - if (m_context.connman) { - m_context.connman->SetNetworkActive(active); + if (m_context->connman) { + m_context->connman->SetNetworkActive(active); } } - bool getNetworkActive() override { return m_context.connman && m_context.connman->GetNetworkActive(); } + bool getNetworkActive() override { return m_context->connman && m_context->connman->GetNetworkActive(); } CFeeRate estimateSmartFee(int num_blocks, bool conservative, int* returned_target = nullptr) override { FeeCalculation fee_calc; @@ -254,36 +231,9 @@ public: LOCK(::cs_main); return ::ChainstateActive().CoinsTip().GetCoin(output, coin); } - std::string getWalletDir() override - { - return GetWalletDir().string(); - } - std::vector<std::string> listWalletDir() override - { - std::vector<std::string> paths; - for (auto& path : ListWalletDir()) { - paths.push_back(path.string()); - } - return paths; - } - std::vector<std::unique_ptr<Wallet>> getWallets() override - { - std::vector<std::unique_ptr<Wallet>> wallets; - for (auto& client : m_context.chain_clients) { - auto client_wallets = client->getWallets(); - std::move(client_wallets.begin(), client_wallets.end(), std::back_inserter(wallets)); - } - return wallets; - } - std::unique_ptr<Wallet> loadWallet(const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) override - { - return MakeWallet(LoadWallet(*m_context.chain, name, error, warnings)); - } - std::unique_ptr<Wallet> createWallet(const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, WalletCreationStatus& status) override + WalletClient& walletClient() override { - std::shared_ptr<CWallet> wallet; - status = CreateWallet(*m_context.chain, passphrase, wallet_creation_flags, name, error, warnings, wallet); - return MakeWallet(wallet); + return *Assert(m_context->wallet_client); } std::unique_ptr<Handler> handleInitMessage(InitMessageFn fn) override { @@ -301,10 +251,6 @@ public: { return MakeHandler(::uiInterface.ShowProgress_connect(fn)); } - std::unique_ptr<Handler> handleLoadWallet(LoadWalletFn fn) override - { - return HandleLoadWallet(std::move(fn)); - } std::unique_ptr<Handler> handleNotifyNumConnectionsChanged(NotifyNumConnectionsChangedFn fn) override { return MakeHandler(::uiInterface.NotifyNumConnectionsChanged_connect(fn)); @@ -336,13 +282,22 @@ public: /* verification progress is unused when a header was received */ 0); })); } - NodeContext* context() override { return &m_context; } - NodeContext m_context; - util::Ref m_context_ref{m_context}; + NodeContext* context() override { return m_context; } + void setContext(NodeContext* context) override + { + m_context = context; + if (context) { + m_context_ref.Set(*context); + } else { + m_context_ref.Clear(); + } + } + NodeContext* m_context{nullptr}; + util::Ref m_context_ref; }; } // namespace -std::unique_ptr<Node> MakeNode() { return MakeUnique<NodeImpl>(); } +std::unique_ptr<Node> MakeNode(NodeContext* context) { return MakeUnique<NodeImpl>(context); } } // namespace interfaces diff --git a/src/interfaces/node.h b/src/interfaces/node.h index a9680c42b..5079be038 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -29,57 +29,31 @@ class RPCTimerInterface; class UniValue; class proxyType; enum class SynchronizationState; -enum class WalletCreationStatus; struct CNodeStateStats; struct NodeContext; struct bilingual_str; namespace interfaces { class Handler; -class Wallet; +class WalletClient; struct BlockTip; +//! Block and header tip information +struct BlockAndHeaderTipInfo +{ + int block_height; + int64_t block_time; + int header_height; + int64_t header_time; + double verification_progress; +}; + //! Top-level interface for a bitcoin node (bitcoind process). class Node { public: virtual ~Node() {} - //! Send init error. - virtual void initError(const bilingual_str& message) = 0; - - //! Set command line arguments. - virtual bool parseParameters(int argc, const char* const argv[], std::string& error) = 0; - - //! Set a command line argument - virtual void forceSetArg(const std::string& arg, const std::string& value) = 0; - - //! Set a command line argument if it doesn't already have a value - virtual bool softSetArg(const std::string& arg, const std::string& value) = 0; - - //! Set a command line boolean argument if it doesn't already have a value - virtual bool softSetBoolArg(const std::string& arg, bool value) = 0; - - //! Load settings from configuration file. - virtual bool readConfigFiles(std::string& error) = 0; - - //! Choose network parameters. - virtual void selectParams(const std::string& network) = 0; - - //! Read and update <datadir>/settings.json file with saved settings. This - //! needs to be called after selectParams() because the settings file - //! location is network-specific. - virtual bool initSettings(std::string& error) = 0; - - //! Get the (assumed) blockchain size. - virtual uint64_t getAssumedBlockchainSize() = 0; - - //! Get the (assumed) chain state size. - virtual uint64_t getAssumedChainStateSize() = 0; - - //! Get network name. - virtual std::string getNetwork() = 0; - //! Init logging. virtual void initLogging() = 0; @@ -96,7 +70,7 @@ public: virtual bool baseInitialize() = 0; //! Start node. - virtual bool appInitMain() = 0; + virtual bool appInitMain(interfaces::BlockAndHeaderTipInfo* tip_info = nullptr) = 0; //! Stop node. virtual void appShutdown() = 0; @@ -107,9 +81,6 @@ public: //! Return whether shutdown was requested. virtual bool shutdownRequested() = 0; - //! Setup arguments - virtual void setupServerArgs() = 0; - //! Map port. virtual void mapPort(bool use_upnp) = 0; @@ -201,22 +172,8 @@ public: //! Get unspent outputs associated with a transaction. virtual bool getUnspentOutput(const COutPoint& output, Coin& coin) = 0; - //! Return default wallet directory. - virtual std::string getWalletDir() = 0; - - //! Return available wallets in wallet directory. - virtual std::vector<std::string> listWalletDir() = 0; - - //! Return interfaces for accessing wallets (if any). - virtual std::vector<std::unique_ptr<Wallet>> getWallets() = 0; - - //! Attempts to load a wallet from file or directory. - //! The loaded wallet is also notified to handlers previously registered - //! with handleLoadWallet. - virtual std::unique_ptr<Wallet> loadWallet(const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) = 0; - - //! Create a wallet from file - virtual std::unique_ptr<Wallet> createWallet(const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, WalletCreationStatus& status) = 0; + //! Get wallet client. + virtual WalletClient& walletClient() = 0; //! Register handler for init messages. using InitMessageFn = std::function<void(const std::string& message)>; @@ -238,10 +195,6 @@ public: using ShowProgressFn = std::function<void(const std::string& title, int progress, bool resume_possible)>; virtual std::unique_ptr<Handler> handleShowProgress(ShowProgressFn fn) = 0; - //! Register handler for load wallet messages. - using LoadWalletFn = std::function<void(std::unique_ptr<Wallet> wallet)>; - virtual std::unique_ptr<Handler> handleLoadWallet(LoadWalletFn fn) = 0; - //! Register handler for number of connections changed messages. using NotifyNumConnectionsChangedFn = std::function<void(int new_num_connections)>; virtual std::unique_ptr<Handler> handleNotifyNumConnectionsChanged(NotifyNumConnectionsChangedFn fn) = 0; @@ -268,12 +221,14 @@ public: std::function<void(SynchronizationState, interfaces::BlockTip tip, double verification_progress)>; virtual std::unique_ptr<Handler> handleNotifyHeaderTip(NotifyHeaderTipFn fn) = 0; - //! Return pointer to internal chain interface, useful for testing. + //! Get and set internal node context. Useful for testing, but not + //! accessible across processes. virtual NodeContext* context() { return nullptr; } + virtual void setContext(NodeContext* context) { } }; //! Return implementation of Node interface. -std::unique_ptr<Node> MakeNode(); +std::unique_ptr<Node> MakeNode(NodeContext* context = nullptr); //! Block tip (could be a header or not, depends on the subscribed signal). struct BlockTip { diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 7fd24425c..94c63da84 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -37,6 +37,7 @@ namespace { //! Construct wallet tx struct. WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) { + LOCK(wallet.cs_wallet); WalletTx result; result.tx = wtx.tx; result.txin_is_mine.reserve(wtx.tx->vin.size()); @@ -132,7 +133,11 @@ public: { return m_wallet->SignMessage(message, pkhash, str_sig); } - bool isSpendable(const CTxDestination& dest) override { return m_wallet->IsMine(dest) & ISMINE_SPENDABLE; } + bool isSpendable(const CTxDestination& dest) override + { + LOCK(m_wallet->cs_wallet); + return m_wallet->IsMine(dest) & ISMINE_SPENDABLE; + } bool haveWatchOnly() override { auto spk_man = m_wallet->GetLegacyScriptPubKeyMan(); @@ -441,7 +446,7 @@ public: CAmount getDefaultMaxTxFee() override { return m_wallet->m_default_max_tx_fee; } void remove() override { - RemoveWallet(m_wallet); + RemoveWallet(m_wallet, false /* load_on_start */); } bool isLegacy() override { return m_wallet->IsLegacy(); } std::unique_ptr<Handler> handleUnload(UnloadFn fn) override @@ -480,15 +485,17 @@ public: std::shared_ptr<CWallet> m_wallet; }; -class WalletClientImpl : public ChainClient +class WalletClientImpl : public WalletClient { public: - WalletClientImpl(Chain& chain, ArgsManager& args, std::vector<std::string> wallet_filenames) - : m_wallet_filenames(std::move(wallet_filenames)) + WalletClientImpl(Chain& chain, ArgsManager& args) { m_context.chain = &chain; m_context.args = &args; } + ~WalletClientImpl() override { UnloadWallets(); } + + //! ChainClient methods void registerRpcs() override { for (const CRPCCommand& command : GetWalletRPCCommands()) { @@ -498,12 +505,43 @@ public: m_rpc_handlers.emplace_back(m_context.chain->handleRpc(m_rpc_commands.back())); } } - bool verify() override { return VerifyWallets(*m_context.chain, m_wallet_filenames); } - bool load() override { return LoadWallets(*m_context.chain, m_wallet_filenames); } + bool verify() override { return VerifyWallets(*m_context.chain); } + bool load() override { return LoadWallets(*m_context.chain); } void start(CScheduler& scheduler) override { return StartWallets(scheduler, *Assert(m_context.args)); } void flush() override { return FlushWallets(); } void stop() override { return StopWallets(); } void setMockTime(int64_t time) override { return SetMockTime(time); } + + //! WalletClient methods + std::unique_ptr<Wallet> createWallet(const std::string& name, const SecureString& passphrase, uint64_t wallet_creation_flags, bilingual_str& error, std::vector<bilingual_str>& warnings) override + { + std::shared_ptr<CWallet> wallet; + DatabaseOptions options; + DatabaseStatus status; + options.require_create = true; + options.create_flags = wallet_creation_flags; + options.create_passphrase = passphrase; + return MakeWallet(CreateWallet(*m_context.chain, name, true /* load_on_start */, options, status, error, warnings)); + } + std::unique_ptr<Wallet> loadWallet(const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) override + { + DatabaseOptions options; + DatabaseStatus status; + options.require_existing = true; + return MakeWallet(LoadWallet(*m_context.chain, name, true /* load_on_start */, options, status, error, warnings)); + } + std::string getWalletDir() override + { + return GetWalletDir().string(); + } + std::vector<std::string> listWalletDir() override + { + std::vector<std::string> paths; + for (auto& path : ListWalletDir()) { + paths.push_back(path.string()); + } + return paths; + } std::vector<std::unique_ptr<Wallet>> getWallets() override { std::vector<std::unique_ptr<Wallet>> wallets; @@ -512,7 +550,10 @@ public: } return wallets; } - ~WalletClientImpl() override { UnloadWallets(); } + std::unique_ptr<Handler> handleLoadWallet(LoadWalletFn fn) override + { + return HandleLoadWallet(std::move(fn)); + } WalletContext m_context; const std::vector<std::string> m_wallet_filenames; @@ -524,9 +565,9 @@ public: std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet) { return wallet ? MakeUnique<WalletImpl>(wallet) : nullptr; } -std::unique_ptr<ChainClient> MakeWalletClient(Chain& chain, ArgsManager& args, std::vector<std::string> wallet_filenames) +std::unique_ptr<WalletClient> MakeWalletClient(Chain& chain, ArgsManager& args) { - return MakeUnique<WalletClientImpl>(chain, args, std::move(wallet_filenames)); + return MakeUnique<WalletClientImpl>(chain, args); } } // namespace interfaces diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 3cdadbc72..6ccfd7fc2 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -6,6 +6,7 @@ #define BITCOIN_INTERFACES_WALLET_H #include <amount.h> // For CAmount +#include <interfaces/chain.h> // For ChainClient #include <pubkey.h> // For CKeyID and CScriptID (definitions needed in CTxDestination instantiation) #include <script/standard.h> // For CTxDestination #include <support/allocators/secure.h> // For SecureString @@ -31,6 +32,7 @@ enum class TransactionError; enum isminetype : unsigned int; struct CRecipient; struct PartiallySignedTransaction; +struct WalletContext; struct bilingual_str; typedef uint8_t isminefilter; @@ -301,6 +303,34 @@ public: virtual CWallet* wallet() { return nullptr; } }; +//! Wallet chain client that in addition to having chain client methods for +//! starting up, shutting down, and registering RPCs, also has additional +//! methods (called by the GUI) to load and create wallets. +class WalletClient : public ChainClient +{ +public: + //! Create new wallet. + virtual std::unique_ptr<Wallet> createWallet(const std::string& name, const SecureString& passphrase, uint64_t wallet_creation_flags, bilingual_str& error, std::vector<bilingual_str>& warnings) = 0; + + //! Load existing wallet. + virtual std::unique_ptr<Wallet> loadWallet(const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) = 0; + + //! Return default wallet directory. + virtual std::string getWalletDir() = 0; + + //! Return available wallets in wallet directory. + virtual std::vector<std::string> listWalletDir() = 0; + + //! Return interfaces for accessing wallets (if any). + virtual std::vector<std::unique_ptr<Wallet>> getWallets() = 0; + + //! Register handler for load wallet messages. This callback is triggered by + //! createWallet and loadWallet above, and also triggered when wallets are + //! loaded at startup or by RPC. + using LoadWalletFn = std::function<void(std::unique_ptr<Wallet> wallet)>; + virtual std::unique_ptr<Handler> handleLoadWallet(LoadWalletFn fn) = 0; +}; + //! Information about one wallet address. struct WalletAddress { @@ -379,6 +409,10 @@ struct WalletTxOut //! dummywallet.cpp and throws if the wallet component is not compiled. std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet); +//! Return implementation of ChainClient interface for a wallet client. This +//! function will be undefined in builds where ENABLE_WALLET is false. +std::unique_ptr<WalletClient> MakeWalletClient(Chain& chain, ArgsManager& args); + } // namespace interfaces #endif // BITCOIN_INTERFACES_WALLET_H diff --git a/src/key.cpp b/src/key.cpp index 4ed74a39b..868a8b9b0 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -31,7 +31,7 @@ static secp256k1_context* secp256k1_context_sign = nullptr; * * out32 must point to an output buffer of length at least 32 bytes. */ -static int ec_seckey_import_der(const secp256k1_context* ctx, unsigned char *out32, const unsigned char *seckey, size_t seckeylen) { +int ec_seckey_import_der(const secp256k1_context* ctx, unsigned char *out32, const unsigned char *seckey, size_t seckeylen) { const unsigned char *end = seckey + seckeylen; memset(out32, 0, 32); /* sequence header */ @@ -88,7 +88,7 @@ static int ec_seckey_import_der(const secp256k1_context* ctx, unsigned char *out * will be set to the number of bytes used in the buffer. * key32 must point to a 32-byte raw private key. */ -static int ec_seckey_export_der(const secp256k1_context *ctx, unsigned char *seckey, size_t *seckeylen, const unsigned char *key32, bool compressed) { +int ec_seckey_export_der(const secp256k1_context *ctx, unsigned char *seckey, size_t *seckeylen, const unsigned char *key32, bool compressed) { assert(*seckeylen >= CKey::SIZE); secp256k1_pubkey pubkey; size_t pubkeylen = 0; diff --git a/src/miner.h b/src/miner.h index 096585dfe..bb7a30b18 100644 --- a/src/miner.h +++ b/src/miner.h @@ -84,7 +84,7 @@ struct CompareTxIterByAncestorCount { { if (a->GetCountWithAncestors() != b->GetCountWithAncestors()) return a->GetCountWithAncestors() < b->GetCountWithAncestors(); - return CTxMemPool::CompareIteratorByHash()(a, b); + return CompareIteratorByHash()(a, b); } }; diff --git a/src/net.cpp b/src/net.cpp index 9c72c62df..e35d05cec 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -105,10 +105,10 @@ std::map<CNetAddr, LocalServiceInfo> mapLocalHost GUARDED_BY(cs_mapLocalHost); static bool vfLimited[NET_MAX] GUARDED_BY(cs_mapLocalHost) = {}; std::string strSubVersion; -void CConnman::AddOneShot(const std::string& strDest) +void CConnman::AddAddrFetch(const std::string& strDest) { - LOCK(cs_vOneShots); - vOneShots.push_back(strDest); + LOCK(m_addr_fetches_mutex); + m_addr_fetches.push_back(strDest); } uint16_t GetListenPort() @@ -346,7 +346,7 @@ bool CConnman::CheckIncomingNonce(uint64_t nonce) { LOCK(cs_vNodes); for (const CNode* pnode : vNodes) { - if (!pnode->fSuccessfullyConnected && !pnode->fInbound && pnode->GetLocalNonce() == nonce) + if (!pnode->fSuccessfullyConnected && !pnode->IsInboundConn() && pnode->GetLocalNonce() == nonce) return false; } return true; @@ -368,8 +368,10 @@ static CAddress GetBindAddress(SOCKET sock) return addr_bind; } -CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure, bool manual_connection, bool block_relay_only) +CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure, ConnectionType conn_type) { + assert(conn_type != ConnectionType::INBOUND); + if (pszDest == nullptr) { if (IsLocal(addrConnect)) return nullptr; @@ -432,7 +434,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo if (hSocket == INVALID_SOCKET) { return nullptr; } - connected = ConnectSocketDirectly(addrConnect, hSocket, nConnectTimeout, manual_connection); + connected = ConnectSocketDirectly(addrConnect, hSocket, nConnectTimeout, conn_type == ConnectionType::MANUAL); } if (!proxyConnectionFailed) { // If a connection to the node was attempted, and failure (if any) is not caused by a problem connecting to @@ -459,7 +461,7 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo NodeId id = GetNewNodeId(); uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); CAddress addr_bind = GetBindAddress(hSocket); - CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", false, block_relay_only); + CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", conn_type); pnode->AddRef(); // We're making a new connection, harvest entropy from the time (and our peer count) @@ -528,6 +530,8 @@ void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) } X(nLastSend); X(nLastRecv); + X(nLastTXTime); + X(nLastBlockTime); X(nTimeConnected); X(nTimeOffset); stats.addrName = GetAddrName(); @@ -536,8 +540,8 @@ void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) LOCK(cs_SubVer); X(cleanSubVer); } - X(fInbound); - X(m_manual_connection); + stats.fInbound = IsInboundConn(); + stats.m_manual_connection = IsManualConn(); X(nStartingHeight); { LOCK(cs_vSend); @@ -722,8 +726,8 @@ CNetMessage V1TransportDeserializer::GetMessage(const CMessageHeader::MessageSta if (!msg.m_valid_checksum) { LogPrint(BCLog::NET, "CHECKSUM ERROR (%s, %u bytes), expected %s was %s\n", SanitizeString(msg.m_command), msg.m_message_size, - HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), - HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); + HexStr(Span<uint8_t>(hash.begin(), hash.begin() + CMessageHeader::CHECKSUM_SIZE)), + HexStr(hdr.pchChecksum)); } // store receive time @@ -812,6 +816,7 @@ struct NodeEvictionCandidate CAddress addr; uint64_t nKeyedNetGroup; bool prefer_evict; + bool m_is_local; }; static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) @@ -824,6 +829,12 @@ static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate &a, cons return a.nTimeConnected > b.nTimeConnected; } +static bool CompareLocalHostTimeConnected(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +{ + if (a.m_is_local != b.m_is_local) return b.m_is_local; + return a.nTimeConnected > b.nTimeConnected; +} + static bool CompareNetGroupKeyed(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { return a.nKeyedNetGroup < b.nKeyedNetGroup; } @@ -845,6 +856,14 @@ static bool CompareNodeTXTime(const NodeEvictionCandidate &a, const NodeEviction return a.nTimeConnected > b.nTimeConnected; } +// Pick out the potential block-relay only peers, and sort them by last block time. +static bool CompareNodeBlockRelayOnlyTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +{ + if (a.fRelayTxes != b.fRelayTxes) return a.fRelayTxes; + if (a.nLastBlockTime != b.nLastBlockTime) return a.nLastBlockTime < b.nLastBlockTime; + if (a.fRelevantServices != b.fRelevantServices) return b.fRelevantServices; + return a.nTimeConnected > b.nTimeConnected; +} //! Sort an array by the specified comparator, then erase the last K elements. template<typename T, typename Comparator> @@ -872,7 +891,7 @@ bool CConnman::AttemptToEvictConnection() for (const CNode* node : vNodes) { if (node->HasPermission(PF_NOBAN)) continue; - if (!node->fInbound) + if (!node->IsInboundConn()) continue; if (node->fDisconnect) continue; @@ -887,7 +906,7 @@ bool CConnman::AttemptToEvictConnection() node->nLastBlockTime, node->nLastTXTime, HasAllDesirableServiceFlags(node->nServices), peer_relay_txes, peer_filter_not_null, node->addr, node->nKeyedNetGroup, - node->m_prefer_evict}; + node->m_prefer_evict, node->addr.IsLocal()}; vEvictionCandidates.push_back(candidate); } } @@ -900,15 +919,34 @@ bool CConnman::AttemptToEvictConnection() // Protect the 8 nodes with the lowest minimum ping time. // An attacker cannot manipulate this metric without physically moving nodes closer to the target. EraseLastKElements(vEvictionCandidates, ReverseCompareNodeMinPingTime, 8); - // Protect 4 nodes that most recently sent us transactions. + // Protect 4 nodes that most recently sent us novel transactions accepted into our mempool. // An attacker cannot manipulate this metric without performing useful work. EraseLastKElements(vEvictionCandidates, CompareNodeTXTime, 4); - // Protect 4 nodes that most recently sent us blocks. + // Protect up to 8 non-tx-relay peers that have sent us novel blocks. + std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNodeBlockRelayOnlyTime); + size_t erase_size = std::min(size_t(8), vEvictionCandidates.size()); + vEvictionCandidates.erase(std::remove_if(vEvictionCandidates.end() - erase_size, vEvictionCandidates.end(), [](NodeEvictionCandidate const &n) { return !n.fRelayTxes && n.fRelevantServices; }), vEvictionCandidates.end()); + + // Protect 4 nodes that most recently sent us novel blocks. // An attacker cannot manipulate this metric without performing useful work. EraseLastKElements(vEvictionCandidates, CompareNodeBlockTime, 4); + // Protect the half of the remaining nodes which have been connected the longest. // This replicates the non-eviction implicit behavior, and precludes attacks that start later. - EraseLastKElements(vEvictionCandidates, ReverseCompareNodeTimeConnected, vEvictionCandidates.size() / 2); + // Reserve half of these protected spots for localhost peers, even if + // they're not longest-uptime overall. This helps protect tor peers, which + // tend to be otherwise disadvantaged under our eviction criteria. + size_t initial_size = vEvictionCandidates.size(); + size_t total_protect_size = initial_size / 2; + + // Pick out up to 1/4 peers that are localhost, sorted by longest uptime. + std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareLocalHostTimeConnected); + size_t local_erase_size = total_protect_size / 2; + vEvictionCandidates.erase(std::remove_if(vEvictionCandidates.end() - local_erase_size, vEvictionCandidates.end(), [](NodeEvictionCandidate const &n) { return n.m_is_local; }), vEvictionCandidates.end()); + // Calculate how many we removed, and update our total number of peers that + // we want to protect based on uptime accordingly. + total_protect_size -= initial_size - vEvictionCandidates.size(); + EraseLastKElements(vEvictionCandidates, ReverseCompareNodeTimeConnected, total_protect_size); if (vEvictionCandidates.empty()) return false; @@ -983,7 +1021,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { { LOCK(cs_vNodes); for (const CNode* pnode : vNodes) { - if (pnode->fInbound) nInbound++; + if (pnode->IsInboundConn()) nInbound++; } } @@ -1048,7 +1086,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { if (NetPermissions::HasFlag(permissionFlags, PF_BLOOMFILTER)) { nodeServices = static_cast<ServiceFlags>(nodeServices | NODE_BLOOM); } - CNode* pnode = new CNode(id, nodeServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), nonce, addr_bind, "", true); + CNode* pnode = new CNode(id, nodeServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), nonce, addr_bind, "", ConnectionType::INBOUND); pnode->AddRef(); pnode->m_permissionFlags = permissionFlags; // If this flag is present, the user probably expect that RPC and QT report it as whitelisted (backward compatibility) @@ -1646,7 +1684,7 @@ void CConnman::ThreadDNSAddressSeed() { LOCK(cs_vNodes); for (const CNode* pnode : vNodes) { - nRelevant += pnode->fSuccessfullyConnected && !pnode->fFeeler && !pnode->fOneShot && !pnode->m_manual_connection && !pnode->fInbound; + if (pnode->fSuccessfullyConnected && pnode->IsOutboundOrBlockRelayConn()) ++nRelevant; } } if (nRelevant >= 2) { @@ -1674,7 +1712,7 @@ void CConnman::ThreadDNSAddressSeed() LogPrintf("Loading addresses from DNS seed %s\n", seed); if (HaveNameProxy()) { - AddOneShot(seed); + AddAddrFetch(seed); } else { std::vector<CNetAddr> vIPs; std::vector<CAddress> vAdd; @@ -1696,8 +1734,8 @@ void CConnman::ThreadDNSAddressSeed() addrman.Add(vAdd, resolveSource); } else { // We now avoid directly using results from DNS Seeds which do not support service bit filtering, - // instead using them as a oneshot to get nodes with our desired service bits. - AddOneShot(seed); + // instead using them as a addrfetch to get nodes with our desired service bits. + AddAddrFetch(seed); } } --seeds_right_now; @@ -1705,17 +1743,6 @@ void CConnman::ThreadDNSAddressSeed() LogPrintf("%d addresses found from DNS seeds\n", found); } - - - - - - - - - - - void CConnman::DumpAddresses() { int64_t nStart = GetTimeMillis(); @@ -1727,20 +1754,20 @@ void CConnman::DumpAddresses() addrman.size(), GetTimeMillis() - nStart); } -void CConnman::ProcessOneShot() +void CConnman::ProcessAddrFetch() { std::string strDest; { - LOCK(cs_vOneShots); - if (vOneShots.empty()) + LOCK(m_addr_fetches_mutex); + if (m_addr_fetches.empty()) return; - strDest = vOneShots.front(); - vOneShots.pop_front(); + strDest = m_addr_fetches.front(); + m_addr_fetches.pop_front(); } CAddress addr; CSemaphoreGrant grant(*semOutbound, true); if (grant) { - OpenNetworkConnection(addr, false, &grant, strDest.c_str(), true); + OpenNetworkConnection(addr, false, &grant, strDest.c_str(), ConnectionType::ADDR_FETCH); } } @@ -1767,7 +1794,7 @@ int CConnman::GetExtraOutboundCount() { LOCK(cs_vNodes); for (const CNode* pnode : vNodes) { - if (!pnode->fInbound && !pnode->m_manual_connection && !pnode->fFeeler && !pnode->fDisconnect && !pnode->fOneShot && pnode->fSuccessfullyConnected) { + if (pnode->fSuccessfullyConnected && !pnode->fDisconnect && pnode->IsOutboundOrBlockRelayConn()) { ++nOutbound; } } @@ -1782,11 +1809,11 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) { for (int64_t nLoop = 0;; nLoop++) { - ProcessOneShot(); + ProcessAddrFetch(); for (const std::string& strAddr : connect) { CAddress addr(CService(), NODE_NONE); - OpenNetworkConnection(addr, false, nullptr, strAddr.c_str(), false, false, true); + OpenNetworkConnection(addr, false, nullptr, strAddr.c_str(), ConnectionType::MANUAL); for (int i = 0; i < 10 && i < nLoop; i++) { if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) @@ -1805,7 +1832,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) int64_t nNextFeeler = PoissonNextSend(nStart*1000*1000, FEELER_INTERVAL); while (!interruptNet) { - ProcessOneShot(); + ProcessAddrFetch(); if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) return; @@ -1838,47 +1865,57 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) int nOutboundFullRelay = 0; int nOutboundBlockRelay = 0; std::set<std::vector<unsigned char> > setConnected; + { LOCK(cs_vNodes); for (const CNode* pnode : vNodes) { - if (!pnode->fInbound && !pnode->m_manual_connection) { - // Netgroups for inbound and addnode peers are not excluded because our goal here - // is to not use multiple of our limited outbound slots on a single netgroup - // but inbound and addnode peers do not use our outbound slots. Inbound peers - // also have the added issue that they're attacker controlled and could be used - // to prevent us from connecting to particular hosts if we used them here. - setConnected.insert(pnode->addr.GetGroup(addrman.m_asmap)); - if (pnode->m_tx_relay == nullptr) { - nOutboundBlockRelay++; - } else if (!pnode->fFeeler) { - nOutboundFullRelay++; - } - } + if (pnode->IsFullOutboundConn()) nOutboundFullRelay++; + if (pnode->IsBlockOnlyConn()) nOutboundBlockRelay++; + + // Netgroups for inbound and manual peers are not excluded because our goal here + // is to not use multiple of our limited outbound slots on a single netgroup + // but inbound and manual peers do not use our outbound slots. Inbound peers + // also have the added issue that they could be attacker controlled and used + // to prevent us from connecting to particular hosts if we used them here. + switch (pnode->m_conn_type) { + case ConnectionType::INBOUND: + case ConnectionType::MANUAL: + break; + case ConnectionType::OUTBOUND_FULL_RELAY: + case ConnectionType::BLOCK_RELAY: + case ConnectionType::ADDR_FETCH: + case ConnectionType::FEELER: + setConnected.insert(pnode->addr.GetGroup(addrman.m_asmap)); + } // no default case, so the compiler can warn about missing cases } } - // Feeler Connections - // - // Design goals: - // * Increase the number of connectable addresses in the tried table. - // - // Method: - // * Choose a random address from new and attempt to connect to it if we can connect - // successfully it is added to tried. - // * Start attempting feeler connections only after node finishes making outbound - // connections. - // * Only make a feeler connection once every few minutes. - // + ConnectionType conn_type = ConnectionType::OUTBOUND_FULL_RELAY; + int64_t nTime = GetTimeMicros(); bool fFeeler = false; - if (nOutboundFullRelay >= m_max_outbound_full_relay && nOutboundBlockRelay >= m_max_outbound_block_relay && !GetTryNewOutboundPeer()) { - int64_t nTime = GetTimeMicros(); // The current time right now (in microseconds). - if (nTime > nNextFeeler) { - nNextFeeler = PoissonNextSend(nTime, FEELER_INTERVAL); - fFeeler = true; - } else { - continue; - } + // Determine what type of connection to open. Opening + // OUTBOUND_FULL_RELAY connections gets the highest priority until we + // meet our full-relay capacity. Then we open BLOCK_RELAY connection + // until we hit our block-relay-only peer limit. + // GetTryNewOutboundPeer() gets set when a stale tip is detected, so we + // try opening an additional OUTBOUND_FULL_RELAY connection. If none of + // these conditions are met, check the nNextFeeler timer to decide if + // we should open a FEELER. + + if (nOutboundFullRelay < m_max_outbound_full_relay) { + // OUTBOUND_FULL_RELAY + } else if (nOutboundBlockRelay < m_max_outbound_block_relay) { + conn_type = ConnectionType::BLOCK_RELAY; + } else if (GetTryNewOutboundPeer()) { + // OUTBOUND_FULL_RELAY + } else if (nTime > nNextFeeler) { + nNextFeeler = PoissonNextSend(nTime, FEELER_INTERVAL); + conn_type = ConnectionType::FEELER; + fFeeler = true; + } else { + // skip to next iteration of while loop + continue; } addrman.ResolveCollisions(); @@ -1945,14 +1982,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) LogPrint(BCLog::NET, "Making feeler connection to %s\n", addrConnect.ToString()); } - // Open this connection as block-relay-only if we're already at our - // full-relay capacity, but not yet at our block-relay peer limit. - // (It should not be possible for fFeeler to be set if we're not - // also at our block-relay peer limit, but check against that as - // well for sanity.) - bool block_relay_only = nOutboundBlockRelay < m_max_outbound_block_relay && !fFeeler && nOutboundFullRelay >= m_max_outbound_full_relay; - - OpenNetworkConnection(addrConnect, (int)setConnected.size() >= std::min(nMaxConnections - 1, 2), &grant, nullptr, false, fFeeler, false, block_relay_only); + OpenNetworkConnection(addrConnect, (int)setConnected.size() >= std::min(nMaxConnections - 1, 2), &grant, nullptr, conn_type); } } } @@ -1976,11 +2006,11 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() LOCK(cs_vNodes); for (const CNode* pnode : vNodes) { if (pnode->addr.IsValid()) { - mapConnected[pnode->addr] = pnode->fInbound; + mapConnected[pnode->addr] = pnode->IsInboundConn(); } std::string addrName = pnode->GetAddrName(); if (!addrName.empty()) { - mapConnectedByName[std::move(addrName)] = std::make_pair(pnode->fInbound, static_cast<const CService&>(pnode->addr)); + mapConnectedByName[std::move(addrName)] = std::make_pair(pnode->IsInboundConn(), static_cast<const CService&>(pnode->addr)); } } } @@ -2027,7 +2057,7 @@ void CConnman::ThreadOpenAddedConnections() } tried = true; CAddress addr(CService(), NODE_NONE); - OpenNetworkConnection(addr, false, &grant, info.strAddedNode.c_str(), false, false, true); + OpenNetworkConnection(addr, false, &grant, info.strAddedNode.c_str(), ConnectionType::MANUAL); if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) return; } @@ -2039,8 +2069,10 @@ void CConnman::ThreadOpenAddedConnections() } // if successful, this moves the passed grant to the constructed node -void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound, const char *pszDest, bool fOneShot, bool fFeeler, bool manual_connection, bool block_relay_only) +void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound, const char *pszDest, ConnectionType conn_type) { + assert(conn_type != ConnectionType::INBOUND); + // // Initiate outbound network connection // @@ -2058,18 +2090,12 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai } else if (FindNode(std::string(pszDest))) return; - CNode* pnode = ConnectNode(addrConnect, pszDest, fCountFailure, manual_connection, block_relay_only); + CNode* pnode = ConnectNode(addrConnect, pszDest, fCountFailure, conn_type); if (!pnode) return; if (grantOutbound) grantOutbound->MoveTo(pnode->grantOutbound); - if (fOneShot) - pnode->fOneShot = true; - if (fFeeler) - pnode->fFeeler = true; - if (manual_connection) - pnode->m_manual_connection = true; m_msgproc->InitializeNode(pnode); { @@ -2127,11 +2153,6 @@ void CConnman::ThreadMessageHandler() } } - - - - - bool CConnman::BindListenPort(const CService& addrBind, bilingual_str& strError, NetPermissionFlags permissions) { int nOne = 1; @@ -2337,7 +2358,7 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) } for (const auto& strDest : connOptions.vSeedNodes) { - AddOneShot(strDest); + AddAddrFetch(strDest); } if (clientInterface) { @@ -2390,7 +2411,7 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) else threadDNSAddressSeed = std::thread(&TraceThread<std::function<void()> >, "dnsseed", std::function<void()>(std::bind(&CConnman::ThreadDNSAddressSeed, this))); - // Initiate outbound connections from -addnode + // Initiate manual connections threadOpenAddedConnections = std::thread(&TraceThread<std::function<void()> >, "addcon", std::function<void()>(std::bind(&CConnman::ThreadOpenAddedConnections, this))); if (connOptions.m_use_addrman_outgoing && !connOptions.m_specified_outgoing.empty()) { @@ -2523,14 +2544,14 @@ void CConnman::MarkAddressGood(const CAddress& addr) addrman.Good(addr); } -void CConnman::AddNewAddresses(const std::vector<CAddress>& vAddr, const CAddress& addrFrom, int64_t nTimePenalty) +bool CConnman::AddNewAddresses(const std::vector<CAddress>& vAddr, const CAddress& addrFrom, int64_t nTimePenalty) { - addrman.Add(vAddr, addrFrom, nTimePenalty); + return addrman.Add(vAddr, addrFrom, nTimePenalty); } -std::vector<CAddress> CConnman::GetAddresses() +std::vector<CAddress> CConnman::GetAddresses(size_t max_addresses, size_t max_pct) { - std::vector<CAddress> addresses = addrman.GetAddr(); + std::vector<CAddress> addresses = addrman.GetAddr(max_addresses, max_pct); if (m_banman) { addresses.erase(std::remove_if(addresses.begin(), addresses.end(), [this](const CAddress& addr){return m_banman->IsDiscouraged(addr) || m_banman->IsBanned(addr);}), @@ -2539,12 +2560,12 @@ std::vector<CAddress> CConnman::GetAddresses() return addresses; } -std::vector<CAddress> CConnman::GetAddresses(Network requestor_network) +std::vector<CAddress> CConnman::GetAddresses(Network requestor_network, size_t max_addresses, size_t max_pct) { const auto current_time = GetTime<std::chrono::microseconds>(); if (m_addr_response_caches.find(requestor_network) == m_addr_response_caches.end() || m_addr_response_caches[requestor_network].m_update_addr_response < current_time) { - m_addr_response_caches[requestor_network].m_addrs_response_cache = GetAddresses(); + m_addr_response_caches[requestor_network].m_addrs_response_cache = GetAddresses(max_addresses, max_pct); m_addr_response_caches[requestor_network].m_update_addr_response = current_time + std::chrono::hours(21) + GetRandMillis(std::chrono::hours(6)); } return m_addr_response_caches[requestor_network].m_addrs_response_cache; @@ -2581,7 +2602,7 @@ size_t CConnman::GetNodeCount(NumConnections flags) int nNum = 0; for (const auto& pnode : vNodes) { - if (flags & (pnode->fInbound ? CONNECTIONS_IN : CONNECTIONS_OUT)) { + if (flags & (pnode->IsInboundConn() ? CONNECTIONS_IN : CONNECTIONS_OUT)) { nNum++; } } @@ -2765,28 +2786,31 @@ int CConnman::GetBestHeight() const unsigned int CConnman::GetReceiveFloodSize() const { return nReceiveFloodSize; } -CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress& addrBindIn, const std::string& addrNameIn, bool fInboundIn, bool block_relay_only) +CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress& addrBindIn, const std::string& addrNameIn, ConnectionType conn_type_in) : nTimeConnected(GetSystemTimeInSeconds()), addr(addrIn), addrBind(addrBindIn), - fInbound(fInboundIn), nKeyedNetGroup(nKeyedNetGroupIn), // Don't relay addr messages to peers that we connect to as block-relay-only // peers (to prevent adversaries from inferring these links from addr // traffic). - m_addr_known{block_relay_only ? nullptr : MakeUnique<CRollingBloomFilter>(5000, 0.001)}, id(idIn), nLocalHostNonce(nLocalHostNonceIn), + m_conn_type(conn_type_in), nLocalServices(nLocalServicesIn), nMyStartingHeight(nMyStartingHeightIn) { hSocket = hSocketIn; addrName = addrNameIn == "" ? addr.ToStringIPPort() : addrNameIn; hashContinue = uint256(); - if (!block_relay_only) { + if (conn_type_in != ConnectionType::BLOCK_RELAY) { m_tx_relay = MakeUnique<TxRelay>(); } + if (RelayAddrsWithConn()) { + m_addr_known = MakeUnique<CRollingBloomFilter>(5000, 0.001); + } + for (const std::string &msg : getAllNetMessageTypes()) mapRecvBytesPerMsgCmd[msg] = 0; mapRecvBytesPerMsgCmd[NET_MESSAGE_COMMAND_OTHER] = 0; @@ -51,11 +51,8 @@ static const bool DEFAULT_WHITELISTFORCERELAY = false; static const int TIMEOUT_INTERVAL = 20 * 60; /** Run the feeler connection loop once every 2 minutes or 120 seconds. **/ static const int FEELER_INTERVAL = 120; -/** The maximum number of new addresses to accumulate before announcing. */ -static const unsigned int MAX_ADDR_TO_SEND = 1000; -// TODO: remove ADDRMAN_GETADDR_MAX and let the caller specify this limit with MAX_ADDR_TO_SEND. -static_assert(MAX_ADDR_TO_SEND == ADDRMAN_GETADDR_MAX, - "Max allowed ADDR message size should be equal to the max number of records returned from AddrMan."); +/** The maximum number of addresses from our addrman to return in response to a getaddr message. */ +static constexpr size_t MAX_ADDR_TO_SEND = 1000; /** Maximum length of incoming protocol messages (no message over 4 MB is currently acceptable). */ static const unsigned int MAX_PROTOCOL_MESSAGE_LENGTH = 4 * 1000 * 1000; /** Maximum length of the user agent string in `version` message */ @@ -117,6 +114,59 @@ struct CSerializedNetMsg std::string m_type; }; +/** Different types of connections to a peer. This enum encapsulates the + * information we have available at the time of opening or accepting the + * connection. Aside from INBOUND, all types are initiated by us. */ +enum class ConnectionType { + /** + * Inbound connections are those initiated by a peer. This is the only + * property we know at the time of connection, until P2P messages are + * exchanged. + */ + INBOUND, + + /** + * These are the default connections that we use to connect with the + * network. There is no restriction on what is relayed- by default we relay + * blocks, addresses & transactions. We automatically attempt to open + * MAX_OUTBOUND_FULL_RELAY_CONNECTIONS using addresses from our AddrMan. + */ + OUTBOUND_FULL_RELAY, + + + /** + * We open manual connections to addresses that users explicitly inputted + * via the addnode RPC, or the -connect command line argument. Even if a + * manual connection is misbehaving, we do not automatically disconnect or + * add it to our discouragement filter. + */ + MANUAL, + + /** + * Feeler connections are short lived connections used to increase the + * number of connectable addresses in our AddrMan. Approximately every + * FEELER_INTERVAL, we attempt to connect to a random address from the new + * table. If successful, we add it to the tried table. + */ + FEELER, + + /** + * We use block-relay-only connections to help prevent against partition + * attacks. By not relaying transactions or addresses, these connections + * are harder to detect by a third party, thus helping obfuscate the + * network topology. We automatically attempt to open + * MAX_BLOCK_RELAY_ONLY_CONNECTIONS using addresses from our AddrMan. + */ + BLOCK_RELAY, + + /** + * AddrFetch connections are short lived connections used to solicit + * addresses from peers. These are initiated to addresses submitted via the + * -seednode command line argument, or under certain conditions when the + * AddrMan is empty. + */ + ADDR_FETCH, +}; class NetEventsInterface; class CConnman @@ -201,7 +251,7 @@ public: bool GetNetworkActive() const { return fNetworkActive; }; bool GetUseAddrmanOutgoing() const { return m_use_addrman_outgoing; }; void SetNetworkActive(bool active); - void OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound = nullptr, const char *strDest = nullptr, bool fOneShot = false, bool fFeeler = false, bool manual_connection = false, bool block_relay_only = false); + void OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant* grantOutbound, const char* strDest, ConnectionType conn_type); bool CheckIncomingNonce(uint64_t nonce); bool ForNode(NodeId id, std::function<bool(CNode* pnode)> func); @@ -253,15 +303,15 @@ public: // Addrman functions void SetServices(const CService &addr, ServiceFlags nServices); void MarkAddressGood(const CAddress& addr); - void AddNewAddresses(const std::vector<CAddress>& vAddr, const CAddress& addrFrom, int64_t nTimePenalty = 0); - std::vector<CAddress> GetAddresses(); + bool AddNewAddresses(const std::vector<CAddress>& vAddr, const CAddress& addrFrom, int64_t nTimePenalty = 0); + std::vector<CAddress> GetAddresses(size_t max_addresses, size_t max_pct); /** * Cache is used to minimize topology leaks, so it should * be used for all non-trusted calls, for example, p2p. * A non-malicious call (from RPC or a peer with addr permission) should * call the function without a parameter to avoid using the cache. */ - std::vector<CAddress> GetAddresses(Network requestor_network); + std::vector<CAddress> GetAddresses(Network requestor_network, size_t max_addresses, size_t max_pct); // This allows temporarily exceeding m_max_outbound_full_relay, with the goal of finding // a peer that is better than all our current peers. @@ -351,8 +401,8 @@ private: bool Bind(const CService& addr, unsigned int flags, NetPermissionFlags permissions); bool InitBinds(const std::vector<CService>& binds, const std::vector<NetWhitebindPermissions>& whiteBinds); void ThreadOpenAddedConnections(); - void AddOneShot(const std::string& strDest); - void ProcessOneShot(); + void AddAddrFetch(const std::string& strDest); + void ProcessAddrFetch(); void ThreadOpenConnections(std::vector<std::string> connect); void ThreadMessageHandler(); void AcceptConnection(const ListenSocket& hListenSocket); @@ -373,7 +423,7 @@ private: CNode* FindNode(const CService& addr); bool AttemptToEvictConnection(); - CNode* ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure, bool manual_connection, bool block_relay_only); + CNode* ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure, ConnectionType conn_type); void AddWhitelistPermissionFlags(NetPermissionFlags& flags, const CNetAddr &addr) const; void DeleteNode(CNode* pnode); @@ -416,8 +466,8 @@ private: std::atomic<bool> fNetworkActive{true}; bool fAddressesInitialized{false}; CAddrMan addrman; - std::deque<std::string> vOneShots GUARDED_BY(cs_vOneShots); - RecursiveMutex cs_vOneShots; + std::deque<std::string> m_addr_fetches GUARDED_BY(m_addr_fetches_mutex); + RecursiveMutex m_addr_fetches_mutex; std::vector<std::string> vAddedNodes GUARDED_BY(cs_vAddedNodes); RecursiveMutex cs_vAddedNodes; std::vector<CNode*> vNodes GUARDED_BY(cs_vNodes); @@ -611,6 +661,8 @@ public: bool fRelayTxes; int64_t nLastSend; int64_t nLastRecv; + int64_t nLastTXTime; + int64_t nLastBlockTime; int64_t nTimeConnected; int64_t nTimeOffset; std::string addrName; @@ -798,12 +850,8 @@ public: } // This boolean is unusued in actual processing, only present for backward compatibility at RPC/QT level bool m_legacyWhitelisted{false}; - bool fFeeler{false}; // If true this node is being used as a short lived feeler. - bool fOneShot{false}; - bool m_manual_connection{false}; bool fClient{false}; // set by version message bool m_limited_node{false}; //after BIP159, set by version message - const bool fInbound; std::atomic_bool fSuccessfullyConnected{false}; // Setting fDisconnect to true will cause the node to be disconnected the // next time DisconnectNodes() runs @@ -816,6 +864,66 @@ public: std::atomic_bool fPauseRecv{false}; std::atomic_bool fPauseSend{false}; + bool IsOutboundOrBlockRelayConn() const { + switch (m_conn_type) { + case ConnectionType::OUTBOUND_FULL_RELAY: + case ConnectionType::BLOCK_RELAY: + return true; + case ConnectionType::INBOUND: + case ConnectionType::MANUAL: + case ConnectionType::ADDR_FETCH: + case ConnectionType::FEELER: + return false; + } // no default case, so the compiler can warn about missing cases + + assert(false); + } + + bool IsFullOutboundConn() const { + return m_conn_type == ConnectionType::OUTBOUND_FULL_RELAY; + } + + bool IsManualConn() const { + return m_conn_type == ConnectionType::MANUAL; + } + + bool IsBlockOnlyConn() const { + return m_conn_type == ConnectionType::BLOCK_RELAY; + } + + bool IsFeelerConn() const { + return m_conn_type == ConnectionType::FEELER; + } + + bool IsAddrFetchConn() const { + return m_conn_type == ConnectionType::ADDR_FETCH; + } + + bool IsInboundConn() const { + return m_conn_type == ConnectionType::INBOUND; + } + + /* Whether we send addr messages over this connection */ + bool RelayAddrsWithConn() const + { + return m_conn_type != ConnectionType::BLOCK_RELAY; + } + + bool ExpectServicesFromConn() const { + switch (m_conn_type) { + case ConnectionType::INBOUND: + case ConnectionType::MANUAL: + case ConnectionType::FEELER: + return false; + case ConnectionType::OUTBOUND_FULL_RELAY: + case ConnectionType::BLOCK_RELAY: + case ConnectionType::ADDR_FETCH: + return true; + } // no default case, so the compiler can warn about missing cases + + assert(false); + } + protected: mapMsgCmdSize mapSendBytesPerMsgCmd; mapMsgCmdSize mapRecvBytesPerMsgCmd GUARDED_BY(cs_vRecv); @@ -826,13 +934,11 @@ public: // flood relay std::vector<CAddress> vAddrToSend; - const std::unique_ptr<CRollingBloomFilter> m_addr_known; + std::unique_ptr<CRollingBloomFilter> m_addr_known{nullptr}; bool fGetAddr{false}; std::chrono::microseconds m_next_addr_send GUARDED_BY(cs_sendProcessing){0}; std::chrono::microseconds m_next_local_addr_send GUARDED_BY(cs_sendProcessing){0}; - bool IsAddrRelayPeer() const { return m_addr_known != nullptr; } - // List of block ids we still have announce. // There is no final sorting before sending, as they are always sent immediately // and in the order requested. @@ -872,8 +978,17 @@ public: // Used for headers announcements - unfiltered blocks to relay std::vector<uint256> vBlockHashesToAnnounce GUARDED_BY(cs_inventory); - // Block and TXN accept times + /** UNIX epoch time of the last block received from this peer that we had + * not yet seen (e.g. not already received from another peer), that passed + * preliminary validity checks and was saved to disk, even if we don't + * connect the block or it eventually fails connection. Used as an inbound + * peer eviction criterium in CConnman::AttemptToEvictConnection. */ std::atomic<int64_t> nLastBlockTime{0}; + + /** UNIX epoch time of the last transaction received from this peer that we + * had not yet seen (e.g. not already received from another peer) and that + * was accepted into our mempool. Used as an inbound peer eviction criterium + * in CConnman::AttemptToEvictConnection. */ std::atomic<int64_t> nLastTXTime{0}; // Ping time measurement: @@ -890,7 +1005,7 @@ public: std::set<uint256> orphan_work_set; - CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress &addrBindIn, const std::string &addrNameIn = "", bool fInboundIn = false, bool block_relay_only = false); + CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress &addrBindIn, const std::string &addrNameIn, ConnectionType conn_type_in); ~CNode(); CNode(const CNode&) = delete; CNode& operator=(const CNode&) = delete; @@ -898,6 +1013,7 @@ public: private: const NodeId id; const uint64_t nLocalHostNonce; + const ConnectionType m_conn_type; //! Services offered to this peer. //! diff --git a/src/net_processing.cpp b/src/net_processing.cpp index c503a97be..166adc05b 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -143,6 +143,8 @@ static constexpr unsigned int MAX_FEEFILTER_CHANGE_DELAY = 5 * 60; static constexpr uint32_t MAX_GETCFILTERS_SIZE = 1000; /** Maximum number of cf hashes that may be requested with one getcfheaders. See BIP 157. */ static constexpr uint32_t MAX_GETCFHEADERS_SIZE = 2000; +/** the maximum percentage of addresses from our addrman to return in response to a getaddr message. */ +static constexpr size_t MAX_PCT_ADDR_TO_SEND = 23; struct COrphanTx { // When modifying, adapt the copy of this definition in tests/DoS_tests. @@ -187,7 +189,7 @@ namespace { * million to make it highly unlikely for users to have issues with this * filter. * - * We only need to add wtxids to this filter. For non-segwit + * We typically only add wtxids to this filter. For non-segwit * transactions, the txid == wtxid, so this only prevents us from * re-downloading non-segwit transactions when communicating with * non-wtxidrelay peers -- which is important for avoiding malleation @@ -196,6 +198,12 @@ namespace { * the reject filter store wtxids is exactly what we want to avoid * redownload of a rejected transaction. * + * In cases where we can tell that a segwit transaction will fail + * validation no matter the witness, we may add the txid of such + * transaction to the filter as well. This can be helpful when + * communicating with txid-relay peers or if we were to otherwise fetch a + * transaction via txid (eg in our orphan handling). + * * Memory used: 1.3 MB */ std::unique_ptr<CRollingBloomFilter> recentRejects GUARDED_BY(cs_main); @@ -270,12 +278,6 @@ struct CNodeState { const CService address; //! Whether we have a fully established connection. bool fCurrentlyConnected; - //! Accumulated misbehaviour score for this peer. - int nMisbehavior; - //! Whether this peer should be disconnected and marked as discouraged (unless it has the noban permission). - bool m_should_discourage; - //! String name of this peer (debugging/logging purposes). - const std::string name; //! The best known block we know this peer has announced. const CBlockIndex *pindexBestKnownBlock; //! The hash of the last unknown block this peer has announced. @@ -424,13 +426,10 @@ struct CNodeState { //! Whether this peer relays txs via wtxid bool m_wtxid_relay{false}; - CNodeState(CAddress addrIn, std::string addrNameIn, bool is_inbound, bool is_manual) : - address(addrIn), name(std::move(addrNameIn)), m_is_inbound(is_inbound), - m_is_manual_connection (is_manual) + CNodeState(CAddress addrIn, bool is_inbound, bool is_manual) + : address(addrIn), m_is_inbound(is_inbound), m_is_manual_connection(is_manual) { fCurrentlyConnected = false; - nMisbehavior = 0; - m_should_discourage = false; pindexBestKnownBlock = nullptr; hashLastUnknownBlock.SetNull(); pindexLastCommonBlock = nullptr; @@ -468,12 +467,56 @@ static CNodeState *State(NodeId pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return &it->second; } +/** + * Data structure for an individual peer. This struct is not protected by + * cs_main since it does not contain validation-critical data. + * + * Memory is owned by shared pointers and this object is destructed when + * the refcount drops to zero. + * + * TODO: move most members from CNodeState to this structure. + * TODO: move remaining application-layer data members from CNode to this structure. + */ +struct Peer { + /** Same id as the CNode object for this peer */ + const NodeId m_id{0}; + + /** Protects misbehavior data members */ + Mutex m_misbehavior_mutex; + /** Accumulated misbehavior score for this peer */ + int m_misbehavior_score GUARDED_BY(m_misbehavior_mutex){0}; + /** Whether this peer should be disconnected and marked as discouraged (unless it has the noban permission). */ + bool m_should_discourage GUARDED_BY(m_misbehavior_mutex){false}; + + Peer(NodeId id) : m_id(id) {} +}; + +using PeerRef = std::shared_ptr<Peer>; + +/** + * Map of all Peer objects, keyed by peer id. This map is protected + * by the global g_peer_mutex. Once a shared pointer reference is + * taken, the lock may be released. Individual fields are protected by + * their own locks. + */ +Mutex g_peer_mutex; +static std::map<NodeId, PeerRef> g_peer_map GUARDED_BY(g_peer_mutex); + +/** Get a shared pointer to the Peer object. + * May return nullptr if the Peer object can't be found. */ +static PeerRef GetPeerRef(NodeId id) +{ + LOCK(g_peer_mutex); + auto it = g_peer_map.find(id); + return it != g_peer_map.end() ? it->second : nullptr; +} + static void UpdatePreferredDownload(const CNode& node, CNodeState* state) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { nPreferredDownload -= state->fPreferredDownload; // Whether this node should be marked as a preferred download node. - state->fPreferredDownload = (!node.fInbound || node.HasPermission(PF_NOBAN)) && !node.fOneShot && !node.fClient; + state->fPreferredDownload = (!node.IsInboundConn() || node.HasPermission(PF_NOBAN)) && !node.IsAddrFetchConn() && !node.fClient; nPreferredDownload += state->fPreferredDownload; } @@ -620,13 +663,12 @@ static void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman& connma } } connman.ForNode(nodeid, [&connman](CNode* pfrom){ - AssertLockHeld(cs_main); + LockAssertion lock(::cs_main); uint64_t nCMPCTBLOCKVersion = (pfrom->GetLocalServices() & NODE_WITNESS) ? 2 : 1; if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { // As per BIP152, we only get 3 of our peers to announce // blocks using compact encodings. connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [&connman, nCMPCTBLOCKVersion](CNode* pnodeStop){ - AssertLockHeld(cs_main); connman.PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/false, nCMPCTBLOCKVersion)); return true; }); @@ -827,36 +869,36 @@ void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds) if (state) state->m_last_block_announcement = time_in_seconds; } -// Returns true for outbound peers, excluding manual connections, feelers, and -// one-shots. -static bool IsOutboundDisconnectionCandidate(const CNode& node) -{ - return !(node.fInbound || node.m_manual_connection || node.fFeeler || node.fOneShot); -} - -void PeerLogicValidation::InitializeNode(CNode *pnode) { +void PeerManager::InitializeNode(CNode *pnode) { CAddress addr = pnode->addr; std::string addrName = pnode->GetAddrName(); NodeId nodeid = pnode->GetId(); { LOCK(cs_main); - mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, std::move(addrName), pnode->fInbound, pnode->m_manual_connection)); + mapNodeState.emplace_hint(mapNodeState.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(addr, pnode->IsInboundConn(), pnode->IsManualConn())); + } + { + PeerRef peer = std::make_shared<Peer>(nodeid); + LOCK(g_peer_mutex); + g_peer_map.emplace_hint(g_peer_map.end(), nodeid, std::move(peer)); + } + if (!pnode->IsInboundConn()) { + PushNodeVersion(*pnode, m_connman, GetTime()); } - if(!pnode->fInbound) - PushNodeVersion(*pnode, *connman, GetTime()); } -void PeerLogicValidation::ReattemptInitialBroadcast(CScheduler& scheduler) const +void PeerManager::ReattemptInitialBroadcast(CScheduler& scheduler) const { - std::map<uint256, uint256> unbroadcast_txids = m_mempool.GetUnbroadcastTxs(); + std::set<uint256> unbroadcast_txids = m_mempool.GetUnbroadcastTxs(); + + for (const auto& txid : unbroadcast_txids) { + CTransactionRef tx = m_mempool.get(txid); - for (const auto& elem : unbroadcast_txids) { - // Sanity check: all unbroadcast txns should exist in the mempool - if (m_mempool.exists(elem.first)) { + if (tx != nullptr) { LOCK(cs_main); - RelayTransaction(elem.first, elem.second, *connman); + RelayTransaction(txid, tx->GetWitnessHash(), m_connman); } else { - m_mempool.RemoveUnbroadcastTx(elem.first, true); + m_mempool.RemoveUnbroadcastTx(txid, true); } } @@ -866,16 +908,24 @@ void PeerLogicValidation::ReattemptInitialBroadcast(CScheduler& scheduler) const scheduler.scheduleFromNow([&] { ReattemptInitialBroadcast(scheduler); }, delta); } -void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { +void PeerManager::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { fUpdateConnectionTime = false; LOCK(cs_main); + int misbehavior{0}; + { + PeerRef peer = GetPeerRef(nodeid); + assert(peer != nullptr); + misbehavior = WITH_LOCK(peer->m_misbehavior_mutex, return peer->m_misbehavior_score); + LOCK(g_peer_mutex); + g_peer_map.erase(nodeid); + } CNodeState *state = State(nodeid); assert(state != nullptr); if (state->fSyncStarted) nSyncStarted--; - if (state->nMisbehavior == 0 && state->fCurrentlyConnected) { + if (misbehavior == 0 && state->fCurrentlyConnected) { fUpdateConnectionTime = true; } @@ -905,17 +955,23 @@ void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTim } bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { - LOCK(cs_main); - CNodeState *state = State(nodeid); - if (state == nullptr) - return false; - stats.nMisbehavior = state->nMisbehavior; - stats.nSyncHeight = state->pindexBestKnownBlock ? state->pindexBestKnownBlock->nHeight : -1; - stats.nCommonHeight = state->pindexLastCommonBlock ? state->pindexLastCommonBlock->nHeight : -1; - for (const QueuedBlock& queue : state->vBlocksInFlight) { - if (queue.pindex) - stats.vHeightInFlight.push_back(queue.pindex->nHeight); + { + LOCK(cs_main); + CNodeState* state = State(nodeid); + if (state == nullptr) + return false; + stats.nSyncHeight = state->pindexBestKnownBlock ? state->pindexBestKnownBlock->nHeight : -1; + stats.nCommonHeight = state->pindexLastCommonBlock ? state->pindexLastCommonBlock->nHeight : -1; + for (const QueuedBlock& queue : state->vBlocksInFlight) { + if (queue.pindex) + stats.vHeightInFlight.push_back(queue.pindex->nHeight); + } } + + PeerRef peer = GetPeerRef(nodeid); + if (peer == nullptr) return false; + stats.m_misbehavior_score = WITH_LOCK(peer->m_misbehavior_mutex, return peer->m_misbehavior_score); + return true; } @@ -1055,39 +1111,27 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) return nEvicted; } -/** - * Increment peer's misbehavior score. If the new value >= DISCOURAGEMENT_THRESHOLD, mark the node - * to be discouraged, meaning the peer might be disconnected and added to the discouragement filter. - */ -void Misbehaving(const NodeId pnode, const int howmuch, const std::string& message) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +void PeerManager::Misbehaving(const NodeId pnode, const int howmuch, const std::string& message) { assert(howmuch > 0); - CNodeState* const state = State(pnode); - if (state == nullptr) return; + PeerRef peer = GetPeerRef(pnode); + if (peer == nullptr) return; - state->nMisbehavior += howmuch; + LOCK(peer->m_misbehavior_mutex); + peer->m_misbehavior_score += howmuch; const std::string message_prefixed = message.empty() ? "" : (": " + message); - if (state->nMisbehavior >= DISCOURAGEMENT_THRESHOLD && state->nMisbehavior - howmuch < DISCOURAGEMENT_THRESHOLD) - { - LogPrint(BCLog::NET, "Misbehaving: peer=%d (%d -> %d) DISCOURAGE THRESHOLD EXCEEDED%s\n", pnode, state->nMisbehavior - howmuch, state->nMisbehavior, message_prefixed); - state->m_should_discourage = true; + if (peer->m_misbehavior_score >= DISCOURAGEMENT_THRESHOLD && peer->m_misbehavior_score - howmuch < DISCOURAGEMENT_THRESHOLD) { + LogPrint(BCLog::NET, "Misbehaving: peer=%d (%d -> %d) DISCOURAGE THRESHOLD EXCEEDED%s\n", pnode, peer->m_misbehavior_score - howmuch, peer->m_misbehavior_score, message_prefixed); + peer->m_should_discourage = true; } else { - LogPrint(BCLog::NET, "Misbehaving: peer=%d (%d -> %d)%s\n", pnode, state->nMisbehavior - howmuch, state->nMisbehavior, message_prefixed); + LogPrint(BCLog::NET, "Misbehaving: peer=%d (%d -> %d)%s\n", pnode, peer->m_misbehavior_score - howmuch, peer->m_misbehavior_score, message_prefixed); } } -/** - * Potentially mark a node discouraged based on the contents of a BlockValidationState object - * - * @param[in] via_compact_block this bool is passed in because net_processing should - * punish peers differently depending on whether the data was provided in a compact - * block message or not. If the compact block had a valid header, but contained invalid - * txs, the peer should not be punished. See BIP 152. - * - * @return Returns true if the peer was punished (probably disconnected) - */ -static bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, bool via_compact_block, const std::string& message = "") { +bool PeerManager::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, + bool via_compact_block, const std::string& message) +{ switch (state.GetResult()) { case BlockValidationResult::BLOCK_RESULT_UNSET: break; @@ -1095,7 +1139,6 @@ static bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& s case BlockValidationResult::BLOCK_CONSENSUS: case BlockValidationResult::BLOCK_MUTATED: if (!via_compact_block) { - LOCK(cs_main); Misbehaving(nodeid, 100, message); return true; } @@ -1119,18 +1162,12 @@ static bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& s case BlockValidationResult::BLOCK_INVALID_HEADER: case BlockValidationResult::BLOCK_CHECKPOINT: case BlockValidationResult::BLOCK_INVALID_PREV: - { - LOCK(cs_main); - Misbehaving(nodeid, 100, message); - } + Misbehaving(nodeid, 100, message); return true; // Conflicting (but not necessarily invalid) data or different policy: case BlockValidationResult::BLOCK_MISSING_PREV: - { - // TODO: Handle this much more gracefully (10 DoS points is super arbitrary) - LOCK(cs_main); - Misbehaving(nodeid, 10, message); - } + // TODO: Handle this much more gracefully (10 DoS points is super arbitrary) + Misbehaving(nodeid, 10, message); return true; case BlockValidationResult::BLOCK_RECENT_CONSENSUS_CHANGE: case BlockValidationResult::BLOCK_TIME_FUTURE: @@ -1142,25 +1179,18 @@ static bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& s return false; } -/** - * Potentially disconnect and discourage a node based on the contents of a TxValidationState object - * - * @return Returns true if the peer was punished (probably disconnected) - */ -static bool MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message = "") +bool PeerManager::MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message) { switch (state.GetResult()) { case TxValidationResult::TX_RESULT_UNSET: break; // The node is providing invalid data: case TxValidationResult::TX_CONSENSUS: - { - LOCK(cs_main); - Misbehaving(nodeid, 100, message); - return true; - } + Misbehaving(nodeid, 100, message); + return true; // Conflicting (but not necessarily invalid) data or different policy: case TxValidationResult::TX_RECENT_CONSENSUS_CHANGE: + case TxValidationResult::TX_INPUTS_NOT_STANDARD: case TxValidationResult::TX_NOT_STANDARD: case TxValidationResult::TX_MISSING_INPUTS: case TxValidationResult::TX_PREMATURE_SPEND: @@ -1195,8 +1225,10 @@ static bool BlockRequestAllowed(const CBlockIndex* pindex, const Consensus::Para (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, consensusParams) < STALE_RELAY_AGE_LIMIT); } -PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool) - : connman(connmanIn), +PeerManager::PeerManager(const CChainParams& chainparams, CConnman& connman, BanMan* banman, + CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool) + : m_chainparams(chainparams), + m_connman(connman), m_banman(banman), m_chainman(chainman), m_mempool(pool), @@ -1216,13 +1248,12 @@ PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CS // same probability that we have in the reject filter). g_recent_confirmed_transactions.reset(new CRollingBloomFilter(48000, 0.000001)); - const Consensus::Params& consensusParams = Params().GetConsensus(); // Stale tip checking and peer eviction are on two different timers, but we // don't want them to get out of sync due to drift in the scheduler, so we // combine them in one function and schedule at the quicker (peer-eviction) // timer. static_assert(EXTRA_PEER_CHECK_INTERVAL < STALE_CHECK_INTERVAL, "peer eviction timer should be less than stale tip check timer"); - scheduler.scheduleEvery([this, consensusParams] { this->CheckForStaleTipAndEvictPeers(consensusParams); }, std::chrono::seconds{EXTRA_PEER_CHECK_INTERVAL}); + scheduler.scheduleEvery([this] { this->CheckForStaleTipAndEvictPeers(); }, std::chrono::seconds{EXTRA_PEER_CHECK_INTERVAL}); // schedule next run for 10-15 minutes in the future const std::chrono::milliseconds delta = std::chrono::minutes{10} + GetRandMillis(std::chrono::minutes{5}); @@ -1233,7 +1264,7 @@ PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CS * Evict orphan txn pool entries (EraseOrphanTx) based on a newly connected * block. Also save the time of the last tip update. */ -void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) +void PeerManager::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) { { LOCK(g_cs_orphans); @@ -1277,7 +1308,7 @@ void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pb } } -void PeerLogicValidation::BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) +void PeerManager::BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex* pindex) { // To avoid relay problems with transactions that were previously // confirmed, clear our filter of recently confirmed transactions whenever @@ -1302,7 +1333,7 @@ static bool fWitnessesPresentInMostRecentCompactBlock GUARDED_BY(cs_most_recent_ * Maintain state about the best-seen block and fast-announce a compact block * to compatible peers. */ -void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) { +void PeerManager::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) { std::shared_ptr<const CBlockHeaderAndShortTxIDs> pcmpctblock = std::make_shared<const CBlockHeaderAndShortTxIDs> (*pblock, true); const CNetMsgMaker msgMaker(PROTOCOL_VERSION); @@ -1313,7 +1344,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: return; nHighestFastAnnounce = pindex->nHeight; - bool fWitnessEnabled = IsWitnessEnabled(pindex->pprev, Params().GetConsensus()); + bool fWitnessEnabled = IsWitnessEnabled(pindex->pprev, m_chainparams.GetConsensus()); uint256 hashBlock(pblock->GetHash()); { @@ -1324,8 +1355,8 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: fWitnessesPresentInMostRecentCompactBlock = fWitnessEnabled; } - connman->ForEachNode([this, &pcmpctblock, pindex, &msgMaker, fWitnessEnabled, &hashBlock](CNode* pnode) { - AssertLockHeld(cs_main); + m_connman.ForEachNode([this, &pcmpctblock, pindex, &msgMaker, fWitnessEnabled, &hashBlock](CNode* pnode) { + LockAssertion lock(::cs_main); // TODO: Avoid the repeated-serialization here if (pnode->nVersion < INVALID_CB_NO_BAN_VERSION || pnode->fDisconnect) @@ -1337,9 +1368,9 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: if (state.fPreferHeaderAndIDs && (!fWitnessEnabled || state.fWantsCmpctWitness) && !PeerHasHeader(&state, pindex) && PeerHasHeader(&state, pindex->pprev)) { - LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", "PeerLogicValidation::NewPoWValidBlock", + LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", "PeerManager::NewPoWValidBlock", hashBlock.ToString(), pnode->GetId()); - connman->PushMessage(pnode, msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock)); + m_connman.PushMessage(pnode, msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock)); state.pindexBestHeaderSent = pindex; } }); @@ -1349,9 +1380,9 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: * Update our best height and announce any block hashes which weren't previously * in ::ChainActive() to our peers. */ -void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { +void PeerManager::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { const int nNewHeight = pindexNew->nHeight; - connman->SetBestHeight(nNewHeight); + m_connman.SetBestHeight(nNewHeight); SetServiceFlagsIBDCache(!fInitialDownload); if (!fInitialDownload) { @@ -1368,7 +1399,7 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB } } // Relay inventory, but don't relay old inventory during initial block download. - connman->ForEachNode([nNewHeight, &vHashes](CNode* pnode) { + m_connman.ForEachNode([nNewHeight, &vHashes](CNode* pnode) { LOCK(pnode->cs_inventory); if (nNewHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : 0)) { for (const uint256& hash : reverse_iterate(vHashes)) { @@ -1376,7 +1407,7 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB } } }); - connman->WakeMessageHandler(); + m_connman.WakeMessageHandler(); } } @@ -1384,7 +1415,7 @@ void PeerLogicValidation::UpdatedBlockTip(const CBlockIndex *pindexNew, const CB * Handle invalid block rejection and consequent peer discouragement, maintain which * peers announce compact blocks. */ -void PeerLogicValidation::BlockChecked(const CBlock& block, const BlockValidationState& state) { +void PeerManager::BlockChecked(const CBlock& block, const BlockValidationState& state) { LOCK(cs_main); const uint256 hash(block.GetHash()); @@ -1407,7 +1438,7 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const BlockValidatio !::ChainstateActive().IsInitialBlockDownload() && mapBlocksInFlight.count(hash) == mapBlocksInFlight.size()) { if (it != mapBlockSource.end()) { - MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, *connman); + MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first, m_connman); } } if (it != mapBlockSource.end()) @@ -1420,56 +1451,51 @@ void PeerLogicValidation::BlockChecked(const CBlock& block, const BlockValidatio // -bool static AlreadyHave(const CInv& inv, const CTxMemPool& mempool) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool static AlreadyHaveTx(const GenTxid& gtxid, const CTxMemPool& mempool) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - switch (inv.type) - { - case MSG_TX: - case MSG_WITNESS_TX: - case MSG_WTX: - { - assert(recentRejects); - if (::ChainActive().Tip()->GetBlockHash() != hashRecentRejectsChainTip) - { - // If the chain tip has changed previously rejected transactions - // might be now valid, e.g. due to a nLockTime'd tx becoming valid, - // or a double-spend. Reset the rejects filter and give those - // txs a second chance. - hashRecentRejectsChainTip = ::ChainActive().Tip()->GetBlockHash(); - recentRejects->reset(); - } - - { - LOCK(g_cs_orphans); - if (!inv.IsMsgWtx() && mapOrphanTransactions.count(inv.hash)) { - return true; - } else if (inv.IsMsgWtx() && g_orphans_by_wtxid.count(inv.hash)) { - return true; - } - } + assert(recentRejects); + if (::ChainActive().Tip()->GetBlockHash() != hashRecentRejectsChainTip) { + // If the chain tip has changed previously rejected transactions + // might be now valid, e.g. due to a nLockTime'd tx becoming valid, + // or a double-spend. Reset the rejects filter and give those + // txs a second chance. + hashRecentRejectsChainTip = ::ChainActive().Tip()->GetBlockHash(); + recentRejects->reset(); + } - { - LOCK(g_cs_recent_confirmed_transactions); - if (g_recent_confirmed_transactions->contains(inv.hash)) return true; - } + const uint256& hash = gtxid.GetHash(); - return recentRejects->contains(inv.hash) || mempool.exists(ToGenTxid(inv)); + { + LOCK(g_cs_orphans); + if (!gtxid.IsWtxid() && mapOrphanTransactions.count(hash)) { + return true; + } else if (gtxid.IsWtxid() && g_orphans_by_wtxid.count(hash)) { + return true; } - case MSG_BLOCK: - case MSG_WITNESS_BLOCK: - return LookupBlockIndex(inv.hash) != nullptr; } - // Don't know what it is, just say we already got one - return true; + + { + LOCK(g_cs_recent_confirmed_transactions); + if (g_recent_confirmed_transactions->contains(hash)) return true; + } + + return recentRejects->contains(hash) || mempool.exists(gtxid); +} + +bool static AlreadyHaveBlock(const uint256& block_hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + return LookupBlockIndex(block_hash) != nullptr; } void RelayTransaction(const uint256& txid, const uint256& wtxid, const CConnman& connman) { connman.ForEachNode([&txid, &wtxid](CNode* pnode) { - AssertLockHeld(cs_main); - CNodeState &state = *State(pnode->GetId()); - if (state.m_wtxid_relay) { + LockAssertion lock(::cs_main); + + CNodeState* state = State(pnode->GetId()); + if (state == nullptr) return; + if (state->m_wtxid_relay) { pnode->PushTxInventory(wtxid); } else { pnode->PushTxInventory(txid); @@ -1479,7 +1505,6 @@ void RelayTransaction(const uint256& txid, const uint256& wtxid, const CConnman& static void RelayAddress(const CAddress& addr, bool fReachable, const CConnman& connman) { - unsigned int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s) // Relay to a limited number of other nodes // Use deterministic randomness to send to the same nodes for 24 hours @@ -1488,11 +1513,14 @@ static void RelayAddress(const CAddress& addr, bool fReachable, const CConnman& const CSipHasher hasher = connman.GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24 * 60 * 60)); FastRandomContext insecure_rand; + // Relay reachable addresses to 2 peers. Unreachable addresses are relayed randomly to 1 or 2 peers. + unsigned int nRelayNodes = (fReachable || (hasher.Finalize() & 1)) ? 2 : 1; + std::array<std::pair<uint64_t, CNode*>,2> best{{{0, nullptr}, {0, nullptr}}}; assert(nRelayNodes <= best.size()); auto sortfunc = [&best, &hasher, nRelayNodes](CNode* pnode) { - if (pnode->IsAddrRelayPeer()) { + if (pnode->RelayAddrsWithConn()) { uint64_t hashKey = CSipHasher(hasher).Write(pnode->GetId()).Finalize(); for (unsigned int i = 0; i < nRelayNodes; i++) { if (hashKey > best[i].first) { @@ -1545,7 +1573,7 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c } // release cs_main before calling ActivateBestChain if (need_activate_chain) { BlockValidationState state; - if (!ActivateBestChain(state, Params(), a_recent_block)) { + if (!ActivateBestChain(state, chainparams, a_recent_block)) { LogPrint(BCLog::NET, "failed to activate chain (%s)\n", state.ToString()); } } @@ -1562,7 +1590,7 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c // disconnect node in case we have reached the outbound limit for serving historical blocks if (send && connman.OutboundTargetReached(true) && - (((pindexBestHeader != nullptr) && (pindexBestHeader->GetBlockTime() - pindex->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.type == MSG_FILTERED_BLOCK) && + (((pindexBestHeader != nullptr) && (pindexBestHeader->GetBlockTime() - pindex->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.IsMsgFilteredBlk()) && !pfrom.HasPermission(PF_DOWNLOAD) // nodes with the download permission may exceed target ) { LogPrint(BCLog::NET, "historical block serving limit reached, disconnect peer=%d\n", pfrom.GetId()); @@ -1588,7 +1616,7 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c std::shared_ptr<const CBlock> pblock; if (a_recent_block && a_recent_block->GetHash() == pindex->GetBlockHash()) { pblock = a_recent_block; - } else if (inv.type == MSG_WITNESS_BLOCK) { + } else if (inv.IsMsgWitnessBlk()) { // Fast-path: in this case it is possible to serve the block directly from disk, // as the network format matches the format on disk std::vector<uint8_t> block_data; @@ -1605,12 +1633,11 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c pblock = pblockRead; } if (pblock) { - if (inv.type == MSG_BLOCK) + if (inv.IsMsgBlk()) { connman.PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); - else if (inv.type == MSG_WITNESS_BLOCK) + } else if (inv.IsMsgWitnessBlk()) { connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); - else if (inv.type == MSG_FILTERED_BLOCK) - { + } else if (inv.IsMsgFilteredBlk()) { bool sendMerkleBlock = false; CMerkleBlock merkleBlock; if (pfrom.m_tx_relay != nullptr) { @@ -1634,9 +1661,7 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c } // else // no response - } - else if (inv.type == MSG_CMPCT_BLOCK) - { + } else if (inv.IsMsgCmpctBlk()) { // If a peer is asking for old blocks, we're almost guaranteed // they won't have a useful mempool to match against a compact block, // and we don't feel like constructing the object for them, so @@ -1671,7 +1696,7 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c } //! Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). -CTransactionRef static FindTxForGetData(const CNode& peer, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) LOCKS_EXCLUDED(cs_main) +static CTransactionRef FindTxForGetData(const CTxMemPool& mempool, const CNode& peer, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) LOCKS_EXCLUDED(cs_main) { auto txinfo = mempool.info(gtxid); if (txinfo.tx) { @@ -1727,23 +1752,34 @@ void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnm continue; } - CTransactionRef tx = FindTxForGetData(pfrom, ToGenTxid(inv), mempool_req, now); + CTransactionRef tx = FindTxForGetData(mempool, pfrom, ToGenTxid(inv), mempool_req, now); if (tx) { // WTX and WITNESS_TX imply we serialize with witness int nSendFlags = (inv.IsMsgTx() ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); mempool.RemoveUnbroadcastTx(tx->GetHash()); // As we're going to send tx, make sure its unconfirmed parents are made requestable. - for (const auto& txin : tx->vin) { - auto txinfo = mempool.info(txin.prevout.hash); - if (txinfo.tx && txinfo.m_time > now - UNCONDITIONAL_RELAY_DELAY) { - // Relaying a transaction with a recent but unconfirmed parent. - if (WITH_LOCK(pfrom.m_tx_relay->cs_tx_inventory, return !pfrom.m_tx_relay->filterInventoryKnown.contains(txin.prevout.hash))) { - LOCK(cs_main); - State(pfrom.GetId())->m_recently_announced_invs.insert(txin.prevout.hash); + std::vector<uint256> parent_ids_to_add; + { + LOCK(mempool.cs); + auto txiter = mempool.GetIter(tx->GetHash()); + if (txiter) { + const CTxMemPoolEntry::Parents& parents = (*txiter)->GetMemPoolParentsConst(); + parent_ids_to_add.reserve(parents.size()); + for (const CTxMemPoolEntry& parent : parents) { + if (parent.GetTime() > now - UNCONDITIONAL_RELAY_DELAY) { + parent_ids_to_add.push_back(parent.GetTx().GetHash()); + } } } } + for (const uint256& parent_txid : parent_ids_to_add) { + // Relaying a transaction with a recent but unconfirmed parent. + if (WITH_LOCK(pfrom.m_tx_relay->cs_tx_inventory, return !pfrom.m_tx_relay->filterInventoryKnown.contains(parent_txid))) { + LOCK(cs_main); + State(pfrom.GetId())->m_recently_announced_invs.insert(parent_txid); + } + } } else { vNotFound.push_back(inv); } @@ -1753,7 +1789,7 @@ void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnm // expensive to process. if (it != pfrom.vRecvGetData.end() && !pfrom.fPauseSend) { const CInv &inv = *it++; - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK || inv.type == MSG_WITNESS_BLOCK) { + if (inv.IsGenBlkMsg()) { ProcessGetBlockData(pfrom, chainparams, inv, connman); } // else: If the first item on the queue is an unknown type, we erase it @@ -1789,11 +1825,10 @@ static uint32_t GetFetchFlags(const CNode& pfrom) EXCLUSIVE_LOCKS_REQUIRED(cs_ma return nFetchFlags; } -inline void static SendBlockTransactions(const CBlock& block, const BlockTransactionsRequest& req, CNode& pfrom, CConnman& connman) { +void PeerManager::SendBlockTransactions(CNode& pfrom, const CBlock& block, const BlockTransactionsRequest& req) { BlockTransactions resp(req); for (size_t i = 0; i < req.indexes.size(); i++) { if (req.indexes[i] >= block.vtx.size()) { - LOCK(cs_main); Misbehaving(pfrom.GetId(), 100, "getblocktxn with out-of-bounds tx indices"); return; } @@ -1802,10 +1837,10 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac LOCK(cs_main); const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); int nSendFlags = State(pfrom.GetId())->fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; - connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); + m_connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); } -static void ProcessHeadersMessage(CNode& pfrom, CConnman& connman, ChainstateManager& chainman, CTxMemPool& mempool, const std::vector<CBlockHeader>& headers, const CChainParams& chainparams, bool via_compact_block) +void PeerManager::ProcessHeadersMessage(CNode& pfrom, const std::vector<CBlockHeader>& headers, bool via_compact_block) { const CNetMsgMaker msgMaker(pfrom.GetSendVersion()); size_t nCount = headers.size(); @@ -1831,7 +1866,7 @@ static void ProcessHeadersMessage(CNode& pfrom, CConnman& connman, ChainstateMan // nUnconnectingHeaders gets reset back to 0. if (!LookupBlockIndex(headers[0].hashPrevBlock) && nCount < MAX_BLOCKS_TO_ANNOUNCE) { nodestate->nUnconnectingHeaders++; - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), @@ -1865,7 +1900,7 @@ static void ProcessHeadersMessage(CNode& pfrom, CConnman& connman, ChainstateMan } BlockValidationState state; - if (!chainman.ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast)) { + if (!m_chainman.ProcessNewBlockHeaders(headers, state, m_chainparams, &pindexLast)) { if (state.IsInvalid()) { MaybePunishNodeForBlock(pfrom.GetId(), state, via_compact_block, "invalid header received"); return; @@ -1896,10 +1931,10 @@ static void ProcessHeadersMessage(CNode& pfrom, CConnman& connman, ChainstateMan // TODO: optimize: if pindexLast is an ancestor of ::ChainActive().Tip or pindexBestHeader, continue // from there instead. LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom.GetId(), pfrom.nStartingHeight); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexLast), uint256())); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexLast), uint256())); } - bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); + bool fCanDirectFetch = CanDirectFetch(m_chainparams.GetConsensus()); // If this set of headers is valid and ends in a block with at least as // much work as our tip, download as much as possible. if (fCanDirectFetch && pindexLast->IsValid(BLOCK_VALID_TREE) && ::ChainActive().Tip()->nChainWork <= pindexLast->nChainWork) { @@ -1909,7 +1944,7 @@ static void ProcessHeadersMessage(CNode& pfrom, CConnman& connman, ChainstateMan while (pindexWalk && !::ChainActive().Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && !mapBlocksInFlight.count(pindexWalk->GetBlockHash()) && - (!IsWitnessEnabled(pindexWalk->pprev, chainparams.GetConsensus()) || State(pfrom.GetId())->fHaveWitness)) { + (!IsWitnessEnabled(pindexWalk->pprev, m_chainparams.GetConsensus()) || State(pfrom.GetId())->fHaveWitness)) { // We don't have this block, and it's not yet in flight. vToFetch.push_back(pindexWalk); } @@ -1933,7 +1968,7 @@ static void ProcessHeadersMessage(CNode& pfrom, CConnman& connman, ChainstateMan } uint32_t nFetchFlags = GetFetchFlags(pfrom); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(mempool, pfrom.GetId(), pindex->GetBlockHash(), pindex); + MarkBlockAsInFlight(m_mempool, pfrom.GetId(), pindex->GetBlockHash(), pindex); LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", pindex->GetBlockHash().ToString(), pfrom.GetId()); } @@ -1946,7 +1981,7 @@ static void ProcessHeadersMessage(CNode& pfrom, CConnman& connman, ChainstateMan // In any case, we want to download using a compact block, not a regular one vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); } - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); } } } @@ -1964,14 +1999,14 @@ static void ProcessHeadersMessage(CNode& pfrom, CConnman& connman, ChainstateMan // until we have a headers chain that has at least // nMinimumChainWork, even if a peer has a chain past our tip, // as an anti-DoS measure. - if (IsOutboundDisconnectionCandidate(pfrom)) { + if (pfrom.IsOutboundOrBlockRelayConn()) { LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom.GetId()); pfrom.fDisconnect = true; } } } - if (!pfrom.fDisconnect && IsOutboundDisconnectionCandidate(pfrom) && nodestate->pindexBestKnownBlock != nullptr && pfrom.m_tx_relay != nullptr) { + if (!pfrom.fDisconnect && pfrom.IsFullOutboundConn() && nodestate->pindexBestKnownBlock != nullptr) { // If this is an outbound full-relay peer, check to see if we should protect // it from the bad/lagging chain logic. // Note that block-relay-only peers are already implicitly protected, so we @@ -1987,7 +2022,7 @@ static void ProcessHeadersMessage(CNode& pfrom, CConnman& connman, ChainstateMan return; } -void static ProcessOrphanTx(CConnman& connman, CTxMemPool& mempool, std::set<uint256>& orphan_work_set, std::list<CTransactionRef>& removed_txn) EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans) +void PeerManager::ProcessOrphanTx(std::set<uint256>& orphan_work_set, std::list<CTransactionRef>& removed_txn) { AssertLockHeld(cs_main); AssertLockHeld(g_cs_orphans); @@ -2009,9 +2044,9 @@ void static ProcessOrphanTx(CConnman& connman, CTxMemPool& mempool, std::set<uin TxValidationState orphan_state; if (setMisbehaving.count(fromPeer)) continue; - if (AcceptToMemoryPool(mempool, orphan_state, porphanTx, &removed_txn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { + if (AcceptToMemoryPool(m_mempool, orphan_state, porphanTx, &removed_txn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); - RelayTransaction(orphanHash, porphanTx->GetWitnessHash(), connman); + RelayTransaction(orphanHash, porphanTx->GetWitnessHash(), m_connman); for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(orphanHash, i)); if (it_by_prev != mapOrphanTransactionsByPrev.end()) { @@ -2052,11 +2087,24 @@ void static ProcessOrphanTx(CConnman& connman, CTxMemPool& mempool, std::set<uin // if we start doing this too early. assert(recentRejects); recentRejects->insert(orphanTx.GetWitnessHash()); + // If the transaction failed for TX_INPUTS_NOT_STANDARD, + // then we know that the witness was irrelevant to the policy + // failure, since this check depends only on the txid + // (the scriptPubKey being spent is covered by the txid). + // Add the txid to the reject filter to prevent repeated + // processing of this transaction in the event that child + // transactions are later received (resulting in + // parent-fetching by txid via the orphan-handling logic). + if (orphan_state.GetResult() == TxValidationResult::TX_INPUTS_NOT_STANDARD && orphanTx.GetWitnessHash() != orphanTx.GetHash()) { + // We only add the txid if it differs from the wtxid, to + // avoid wasting entries in the rolling bloom filter. + recentRejects->insert(orphanTx.GetHash()); + } } EraseOrphanTx(orphanHash); done = true; } - mempool.check(&::ChainstateActive().CoinsTip()); + m_mempool.check(&::ChainstateActive().CoinsTip()); } } @@ -2065,7 +2113,7 @@ void static ProcessOrphanTx(CConnman& connman, CTxMemPool& mempool, std::set<uin * * May disconnect from the peer in the case of a bad request. * - * @param[in] pfrom The peer that we received the request from + * @param[in] peer The peer that we received the request from * @param[in] chain_params Chain parameters * @param[in] filter_type The filter type the request is for. Must be basic filters. * @param[in] start_height The start height for the request @@ -2075,7 +2123,7 @@ void static ProcessOrphanTx(CConnman& connman, CTxMemPool& mempool, std::set<uin * @param[out] filter_index The filter index, if the request can be serviced. * @return True if the request can be serviced. */ -static bool PrepareBlockFilterRequest(CNode& pfrom, const CChainParams& chain_params, +static bool PrepareBlockFilterRequest(CNode& peer, const CChainParams& chain_params, BlockFilterType filter_type, uint32_t start_height, const uint256& stop_hash, uint32_t max_height_diff, const CBlockIndex*& stop_index, @@ -2083,11 +2131,11 @@ static bool PrepareBlockFilterRequest(CNode& pfrom, const CChainParams& chain_pa { const bool supported_filter_type = (filter_type == BlockFilterType::BASIC && - gArgs.GetBoolArg("-peerblockfilters", DEFAULT_PEERBLOCKFILTERS)); + (peer.GetLocalServices() & NODE_COMPACT_FILTERS)); if (!supported_filter_type) { LogPrint(BCLog::NET, "peer %d requested unsupported block filter type: %d\n", - pfrom.GetId(), static_cast<uint8_t>(filter_type)); - pfrom.fDisconnect = true; + peer.GetId(), static_cast<uint8_t>(filter_type)); + peer.fDisconnect = true; return false; } @@ -2098,8 +2146,8 @@ static bool PrepareBlockFilterRequest(CNode& pfrom, const CChainParams& chain_pa // Check that the stop block exists and the peer would be allowed to fetch it. if (!stop_index || !BlockRequestAllowed(stop_index, chain_params.GetConsensus())) { LogPrint(BCLog::NET, "peer %d requested invalid block hash: %s\n", - pfrom.GetId(), stop_hash.ToString()); - pfrom.fDisconnect = true; + peer.GetId(), stop_hash.ToString()); + peer.fDisconnect = true; return false; } } @@ -2108,14 +2156,14 @@ static bool PrepareBlockFilterRequest(CNode& pfrom, const CChainParams& chain_pa if (start_height > stop_height) { LogPrint(BCLog::NET, "peer %d sent invalid getcfilters/getcfheaders with " /* Continued */ "start height %d and stop height %d\n", - pfrom.GetId(), start_height, stop_height); - pfrom.fDisconnect = true; + peer.GetId(), start_height, stop_height); + peer.fDisconnect = true; return false; } if (stop_height - start_height >= max_height_diff) { LogPrint(BCLog::NET, "peer %d requested too many cfilters/cfheaders: %d / %d\n", - pfrom.GetId(), stop_height - start_height + 1, max_height_diff); - pfrom.fDisconnect = true; + peer.GetId(), stop_height - start_height + 1, max_height_diff); + peer.fDisconnect = true; return false; } @@ -2133,12 +2181,12 @@ static bool PrepareBlockFilterRequest(CNode& pfrom, const CChainParams& chain_pa * * May disconnect from the peer in the case of a bad request. * - * @param[in] pfrom The peer that we received the request from + * @param[in] peer The peer that we received the request from * @param[in] vRecv The raw message received * @param[in] chain_params Chain parameters * @param[in] connman Pointer to the connection manager */ -static void ProcessGetCFilters(CNode& pfrom, CDataStream& vRecv, const CChainParams& chain_params, +static void ProcessGetCFilters(CNode& peer, CDataStream& vRecv, const CChainParams& chain_params, CConnman& connman) { uint8_t filter_type_ser; @@ -2151,13 +2199,12 @@ static void ProcessGetCFilters(CNode& pfrom, CDataStream& vRecv, const CChainPar const CBlockIndex* stop_index; BlockFilterIndex* filter_index; - if (!PrepareBlockFilterRequest(pfrom, chain_params, filter_type, start_height, stop_hash, + if (!PrepareBlockFilterRequest(peer, chain_params, filter_type, start_height, stop_hash, MAX_GETCFILTERS_SIZE, stop_index, filter_index)) { return; } std::vector<BlockFilter> filters; - if (!filter_index->LookupFilterRange(start_height, stop_index, filters)) { LogPrint(BCLog::NET, "Failed to find block filter in index: filter_type=%s, start_height=%d, stop_hash=%s\n", BlockFilterTypeName(filter_type), start_height, stop_hash.ToString()); @@ -2165,9 +2212,9 @@ static void ProcessGetCFilters(CNode& pfrom, CDataStream& vRecv, const CChainPar } for (const auto& filter : filters) { - CSerializedNetMsg msg = CNetMsgMaker(pfrom.GetSendVersion()) + CSerializedNetMsg msg = CNetMsgMaker(peer.GetSendVersion()) .Make(NetMsgType::CFILTER, filter); - connman.PushMessage(&pfrom, std::move(msg)); + connman.PushMessage(&peer, std::move(msg)); } } @@ -2176,12 +2223,12 @@ static void ProcessGetCFilters(CNode& pfrom, CDataStream& vRecv, const CChainPar * * May disconnect from the peer in the case of a bad request. * - * @param[in] pfrom The peer that we received the request from + * @param[in] peer The peer that we received the request from * @param[in] vRecv The raw message received * @param[in] chain_params Chain parameters * @param[in] connman Pointer to the connection manager */ -static void ProcessGetCFHeaders(CNode& pfrom, CDataStream& vRecv, const CChainParams& chain_params, +static void ProcessGetCFHeaders(CNode& peer, CDataStream& vRecv, const CChainParams& chain_params, CConnman& connman) { uint8_t filter_type_ser; @@ -2194,7 +2241,7 @@ static void ProcessGetCFHeaders(CNode& pfrom, CDataStream& vRecv, const CChainPa const CBlockIndex* stop_index; BlockFilterIndex* filter_index; - if (!PrepareBlockFilterRequest(pfrom, chain_params, filter_type, start_height, stop_hash, + if (!PrepareBlockFilterRequest(peer, chain_params, filter_type, start_height, stop_hash, MAX_GETCFHEADERS_SIZE, stop_index, filter_index)) { return; } @@ -2217,13 +2264,13 @@ static void ProcessGetCFHeaders(CNode& pfrom, CDataStream& vRecv, const CChainPa return; } - CSerializedNetMsg msg = CNetMsgMaker(pfrom.GetSendVersion()) + CSerializedNetMsg msg = CNetMsgMaker(peer.GetSendVersion()) .Make(NetMsgType::CFHEADERS, filter_type_ser, stop_index->GetBlockHash(), prev_header, filter_hashes); - connman.PushMessage(&pfrom, std::move(msg)); + connman.PushMessage(&peer, std::move(msg)); } /** @@ -2231,12 +2278,12 @@ static void ProcessGetCFHeaders(CNode& pfrom, CDataStream& vRecv, const CChainPa * * May disconnect from the peer in the case of a bad request. * - * @param[in] pfrom The peer that we received the request from + * @param[in] peer The peer that we received the request from * @param[in] vRecv The raw message received * @param[in] chain_params Chain parameters * @param[in] connman Pointer to the connection manager */ -static void ProcessGetCFCheckPt(CNode& pfrom, CDataStream& vRecv, const CChainParams& chain_params, +static void ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv, const CChainParams& chain_params, CConnman& connman) { uint8_t filter_type_ser; @@ -2248,7 +2295,7 @@ static void ProcessGetCFCheckPt(CNode& pfrom, CDataStream& vRecv, const CChainPa const CBlockIndex* stop_index; BlockFilterIndex* filter_index; - if (!PrepareBlockFilterRequest(pfrom, chain_params, filter_type, /*start_height=*/0, stop_hash, + if (!PrepareBlockFilterRequest(peer, chain_params, filter_type, /*start_height=*/0, stop_hash, /*max_height_diff=*/std::numeric_limits<uint32_t>::max(), stop_index, filter_index)) { return; @@ -2269,25 +2316,17 @@ static void ProcessGetCFCheckPt(CNode& pfrom, CDataStream& vRecv, const CChainPa } } - CSerializedNetMsg msg = CNetMsgMaker(pfrom.GetSendVersion()) + CSerializedNetMsg msg = CNetMsgMaker(peer.GetSendVersion()) .Make(NetMsgType::CFCHECKPT, filter_type_ser, stop_index->GetBlockHash(), headers); - connman.PushMessage(&pfrom, std::move(msg)); + connman.PushMessage(&peer, std::move(msg)); } -void ProcessMessage( - CNode& pfrom, - const std::string& msg_type, - CDataStream& vRecv, - const std::chrono::microseconds time_received, - const CChainParams& chainparams, - ChainstateManager& chainman, - CTxMemPool& mempool, - CConnman& connman, - BanMan* banman, - const std::atomic<bool>& interruptMsgProc) +void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, + const std::chrono::microseconds time_received, + const std::atomic<bool>& interruptMsgProc) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(msg_type), vRecv.size(), pfrom.GetId()); if (gArgs.IsArgSet("-dropmessagestest") && GetRand(gArgs.GetArg("-dropmessagestest", 0)) == 0) @@ -2301,7 +2340,6 @@ void ProcessMessage( // Each connection can only send one version message if (pfrom.nVersion != 0) { - LOCK(cs_main); Misbehaving(pfrom.GetId(), 1, "redundant version message"); return; } @@ -2321,11 +2359,11 @@ void ProcessMessage( vRecv >> nVersion >> nServiceInt >> nTime >> addrMe; nSendVersion = std::min(nVersion, PROTOCOL_VERSION); nServices = ServiceFlags(nServiceInt); - if (!pfrom.fInbound) + if (!pfrom.IsInboundConn()) { - connman.SetServices(pfrom.addr, nServices); + m_connman.SetServices(pfrom.addr, nServices); } - if (!pfrom.fInbound && !pfrom.fFeeler && !pfrom.m_manual_connection && !HasAllDesirableServiceFlags(nServices)) + if (pfrom.ExpectServicesFromConn() && !HasAllDesirableServiceFlags(nServices)) { LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom.GetId(), nServices, GetDesirableServiceFlags(nServices)); pfrom.fDisconnect = true; @@ -2352,27 +2390,27 @@ void ProcessMessage( if (!vRecv.empty()) vRecv >> fRelay; // Disconnect if we connected to ourself - if (pfrom.fInbound && !connman.CheckIncomingNonce(nNonce)) + if (pfrom.IsInboundConn() && !m_connman.CheckIncomingNonce(nNonce)) { LogPrintf("connected to self at %s, disconnecting\n", pfrom.addr.ToString()); pfrom.fDisconnect = true; return; } - if (pfrom.fInbound && addrMe.IsRoutable()) + if (pfrom.IsInboundConn() && addrMe.IsRoutable()) { SeenLocal(addrMe); } // Be shy and don't send version until we hear - if (pfrom.fInbound) - PushNodeVersion(pfrom, connman, GetAdjustedTime()); + if (pfrom.IsInboundConn()) + PushNodeVersion(pfrom, m_connman, GetAdjustedTime()); if (nVersion >= WTXID_RELAY_VERSION) { - connman.PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::WTXIDRELAY)); + m_connman.PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::WTXIDRELAY)); } - connman.PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); + m_connman.PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); pfrom.nServices = nServices; pfrom.SetAddrLocal(addrMe); @@ -2409,9 +2447,23 @@ void ProcessMessage( UpdatePreferredDownload(pfrom, State(pfrom.GetId())); } - if (!pfrom.fInbound && pfrom.IsAddrRelayPeer()) - { - // Advertise our address + if (!pfrom.IsInboundConn() && !pfrom.IsBlockOnlyConn()) { + // For outbound peers, we try to relay our address (so that other + // nodes can try to find us more quickly, as we have no guarantee + // that an outbound peer is even aware of how to reach us) and do a + // one-time address fetch (to help populate/update our addrman). If + // we're starting up for the first time, our addrman may be pretty + // empty and no one will know who we are, so these mechanisms are + // important to help us connect to the network. + // + // We also update the addrman to record connection success for + // these peers (which include OUTBOUND_FULL_RELAY and FEELER + // connections) so that addrman will have an up-to-date notion of + // which peers are online and available. + // + // We skip these operations for BLOCK_RELAY peers to avoid + // potentially leaking information about our BLOCK_RELAY + // connections via the addrman or address relay. if (fListen && !::ChainstateActive().IsInitialBlockDownload()) { CAddress addr = GetLocalAddress(&pfrom.addr, pfrom.GetLocalServices()); @@ -2428,9 +2480,12 @@ void ProcessMessage( } // Get recent addresses - connman.PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); + m_connman.PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make(NetMsgType::GETADDR)); pfrom.fGetAddr = true; - connman.MarkAddressGood(pfrom.addr); + + // Moves address from New to Tried table in Addrman, resolves + // tried-table collisions, etc. + m_connman.MarkAddressGood(pfrom.addr); } std::string remoteAddr; @@ -2449,12 +2504,11 @@ void ProcessMessage( // If the peer is old enough to have the old alert system, send it the final alert. if (pfrom.nVersion <= 70012) { CDataStream finalAlert(ParseHex("60010000000000000000000000ffffff7f00000000ffffff7ffeffff7f01ffffff7f00000000ffffff7f00ffffff7f002f555247454e543a20416c657274206b657920636f6d70726f6d697365642c2075706772616465207265717569726564004630440220653febd6410f470f6bae11cad19c48413becb1ac2c17f908fd0fd53bdc3abd5202206d0e9c96fe88d4a0f01ed9dedae2b6f9e00da94cad0fecaae66ecf689bf71b50"), SER_NETWORK, PROTOCOL_VERSION); - connman.PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make("alert", finalAlert)); + m_connman.PushMessage(&pfrom, CNetMsgMaker(nSendVersion).Make("alert", finalAlert)); } // Feeler connections exist only to verify if address is online. - if (pfrom.fFeeler) { - assert(pfrom.fInbound == false); + if (pfrom.IsFeelerConn()) { pfrom.fDisconnect = true; } return; @@ -2462,7 +2516,6 @@ void ProcessMessage( if (pfrom.nVersion == 0) { // Must have a version message before anything else - LOCK(cs_main); Misbehaving(pfrom.GetId(), 1, "non-version message before version handshake"); return; } @@ -2474,7 +2527,7 @@ void ProcessMessage( { pfrom.SetRecvVersion(std::min(pfrom.nVersion.load(), PROTOCOL_VERSION)); - if (!pfrom.fInbound) { + if (!pfrom.IsInboundConn()) { // Mark this node as currently connected, so we update its timestamp later. LOCK(cs_main); State(pfrom.GetId())->fCurrentlyConnected = true; @@ -2489,7 +2542,7 @@ void ProcessMessage( // We send this to non-NODE NETWORK peers as well, because even // non-NODE NETWORK peers can announce blocks (such as pruning // nodes) - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDHEADERS)); } if (pfrom.nVersion >= SHORT_IDS_BLOCKS_VERSION) { // Tell our peer we are willing to provide version 1 or 2 cmpctblocks @@ -2500,9 +2553,9 @@ void ProcessMessage( bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 2; if (pfrom.GetLocalServices() & NODE_WITNESS) - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); nCMPCTBLOCKVersion = 1; - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); } pfrom.fSuccessfullyConnected = true; return; @@ -2529,7 +2582,6 @@ void ProcessMessage( if (!pfrom.fSuccessfullyConnected) { // Must have a verack message before anything else - LOCK(cs_main); Misbehaving(pfrom.GetId(), 1, "non-verack message before version handshake"); return; } @@ -2538,12 +2590,11 @@ void ProcessMessage( std::vector<CAddress> vAddr; vRecv >> vAddr; - if (!pfrom.IsAddrRelayPeer()) { + if (!pfrom.RelayAddrsWithConn()) { return; } if (vAddr.size() > MAX_ADDR_TO_SEND) { - LOCK(cs_main); Misbehaving(pfrom.GetId(), 20, strprintf("addr message size = %u", vAddr.size())); return; } @@ -2566,7 +2617,7 @@ void ProcessMessage( if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) addr.nTime = nNow - 5 * 24 * 60 * 60; pfrom.AddAddressKnown(addr); - if (banman && (banman->IsDiscouraged(addr) || banman->IsBanned(addr))) { + if (m_banman && (m_banman->IsDiscouraged(addr) || m_banman->IsBanned(addr))) { // Do not process banned/discouraged addresses beyond remembering we received them continue; } @@ -2574,16 +2625,16 @@ void ProcessMessage( if (addr.nTime > nSince && !pfrom.fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { // Relay to a limited number of other nodes - RelayAddress(addr, fReachable, connman); + RelayAddress(addr, fReachable, m_connman); } // Do not store addresses outside our network if (fReachable) vAddrOk.push_back(addr); } - connman.AddNewAddresses(vAddrOk, pfrom.addr, 2 * 60 * 60); + m_connman.AddNewAddresses(vAddrOk, pfrom.addr, 2 * 60 * 60); if (vAddr.size() < 1000) pfrom.fGetAddr = false; - if (pfrom.fOneShot) + if (pfrom.IsAddrFetchConn()) pfrom.fDisconnect = true; return; } @@ -2622,7 +2673,6 @@ void ProcessMessage( vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) { - LOCK(cs_main); Misbehaving(pfrom.GetId(), 20, strprintf("inv message size = %u", vInv.size())); return; } @@ -2638,14 +2688,11 @@ void ProcessMessage( LOCK(cs_main); - uint32_t nFetchFlags = GetFetchFlags(pfrom); const auto current_time = GetTime<std::chrono::microseconds>(); uint256* best_block{nullptr}; - for (CInv &inv : vInv) - { - if (interruptMsgProc) - return; + for (CInv& inv : vInv) { + if (interruptMsgProc) return; // Ignore INVs that don't match wtxidrelay setting. // Note that orphan parent fetching always uses MSG_TX GETDATAs regardless of the wtxidrelay setting. @@ -2656,14 +2703,10 @@ void ProcessMessage( if (inv.IsMsgWtx()) continue; } - bool fAlreadyHave = AlreadyHave(inv, mempool); - LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); - - if (inv.IsMsgTx()) { - inv.type |= nFetchFlags; - } + if (inv.IsMsgBlk()) { + const bool fAlreadyHave = AlreadyHaveBlock(inv.hash); + LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); - if (inv.type == MSG_BLOCK) { UpdateBlockAvailability(pfrom.GetId(), inv.hash); if (!fAlreadyHave && !fImporting && !fReindex && !mapBlocksInFlight.count(inv.hash)) { // Headers-first is the primary method of announcement on @@ -2673,20 +2716,26 @@ void ProcessMessage( // then fetch the blocks we need to catch up. best_block = &inv.hash; } - } else { + } else if (inv.IsGenTxMsg()) { + const GenTxid gtxid = ToGenTxid(inv); + const bool fAlreadyHave = AlreadyHaveTx(gtxid, m_mempool); + LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); + pfrom.AddKnownTx(inv.hash); if (fBlocksOnly) { LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); pfrom.fDisconnect = true; return; - } else if (!fAlreadyHave && !chainman.ActiveChainstate().IsInitialBlockDownload()) { - RequestTx(State(pfrom.GetId()), ToGenTxid(inv), current_time); + } else if (!fAlreadyHave && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + RequestTx(State(pfrom.GetId()), gtxid, current_time); } + } else { + LogPrint(BCLog::NET, "Unknown inv type \"%s\" received from peer=%d\n", inv.ToString(), pfrom.GetId()); } } if (best_block != nullptr) { - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), *best_block)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), *best_block)); LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, best_block->ToString(), pfrom.GetId()); } @@ -2698,7 +2747,6 @@ void ProcessMessage( vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) { - LOCK(cs_main); Misbehaving(pfrom.GetId(), 20, strprintf("getdata message size = %u", vInv.size())); return; } @@ -2710,7 +2758,7 @@ void ProcessMessage( } pfrom.vRecvGetData.insert(pfrom.vRecvGetData.end(), vInv.begin(), vInv.end()); - ProcessGetData(pfrom, chainparams, connman, mempool, interruptMsgProc); + ProcessGetData(pfrom, m_chainparams, m_connman, m_mempool, interruptMsgProc); return; } @@ -2739,7 +2787,7 @@ void ProcessMessage( a_recent_block = most_recent_block; } BlockValidationState state; - if (!ActivateBestChain(state, Params(), a_recent_block)) { + if (!ActivateBestChain(state, m_chainparams, a_recent_block)) { LogPrint(BCLog::NET, "failed to activate chain (%s)\n", state.ToString()); } } @@ -2763,7 +2811,7 @@ void ProcessMessage( } // If pruning, don't inv blocks unless we have on disk and are likely to still have // for some reasonable time window (1 hour) that block relay might require. - const int nPrunedBlocksLikelyToHave = MIN_BLOCKS_TO_KEEP - 3600 / chainparams.GetConsensus().nPowTargetSpacing; + const int nPrunedBlocksLikelyToHave = MIN_BLOCKS_TO_KEEP - 3600 / m_chainparams.GetConsensus().nPowTargetSpacing; if (fPruneMode && (!(pindex->nStatus & BLOCK_HAVE_DATA) || pindex->nHeight <= ::ChainActive().Tip()->nHeight - nPrunedBlocksLikelyToHave)) { LogPrint(BCLog::NET, " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); @@ -2794,7 +2842,7 @@ void ProcessMessage( // Unlock cs_most_recent_block to avoid cs_main lock inversion } if (recent_block) { - SendBlockTransactions(*recent_block, req, pfrom, connman); + SendBlockTransactions(pfrom, *recent_block, req); return; } @@ -2824,10 +2872,10 @@ void ProcessMessage( } CBlock block; - bool ret = ReadBlockFromDisk(block, pindex, chainparams.GetConsensus()); + bool ret = ReadBlockFromDisk(block, pindex, m_chainparams.GetConsensus()); assert(ret); - SendBlockTransactions(block, req, pfrom, connman); + SendBlockTransactions(pfrom, block, req); return; } @@ -2858,7 +2906,7 @@ void ProcessMessage( return; } - if (!BlockRequestAllowed(pindex, chainparams.GetConsensus())) { + if (!BlockRequestAllowed(pindex, m_chainparams.GetConsensus())) { LogPrint(BCLog::NET, "%s: ignoring request from peer=%i for old block header that isn't in the main chain\n", __func__, pfrom.GetId()); return; } @@ -2894,7 +2942,7 @@ void ProcessMessage( // will re-announce the new block via headers (or compact blocks again) // in the SendMessages logic. nodestate->pindexBestHeaderSent = pindex ? pindex : ::ChainActive().Tip(); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); return; } @@ -2941,9 +2989,9 @@ void ProcessMessage( std::list<CTransactionRef> lRemovedTxn; - // We do the AlreadyHave() check using wtxid, rather than txid - in the + // We do the AlreadyHaveTx() check using wtxid, rather than txid - in the // absence of witness malleation, this is strictly better, because the - // recent rejects filter may contain the wtxid but will never contain + // recent rejects filter may contain the wtxid but rarely contains // the txid of a segwit transaction that has been rejected. // In the presence of witness malleation, it's possible that by only // doing the check with wtxid, we could overlook a transaction which @@ -2953,10 +3001,10 @@ void ProcessMessage( // already; and an adversary can already relay us old transactions // (older than our recency filter) if trying to DoS us, without any need // for witness malleation. - if (!AlreadyHave(CInv(MSG_WTX, wtxid), mempool) && - AcceptToMemoryPool(mempool, state, ptx, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { - mempool.check(&::ChainstateActive().CoinsTip()); - RelayTransaction(tx.GetHash(), tx.GetWitnessHash(), connman); + if (!AlreadyHaveTx(GenTxid(/* is_wtxid=*/true, wtxid), m_mempool) && + AcceptToMemoryPool(m_mempool, state, ptx, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { + m_mempool.check(&::ChainstateActive().CoinsTip()); + RelayTransaction(tx.GetHash(), tx.GetWitnessHash(), m_connman); for (unsigned int i = 0; i < tx.vout.size(); i++) { auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(txid, i)); if (it_by_prev != mapOrphanTransactionsByPrev.end()) { @@ -2971,33 +3019,43 @@ void ProcessMessage( LogPrint(BCLog::MEMPOOL, "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", pfrom.GetId(), tx.GetHash().ToString(), - mempool.size(), mempool.DynamicMemoryUsage() / 1000); + m_mempool.size(), m_mempool.DynamicMemoryUsage() / 1000); // Recursively process any orphan transactions that depended on this one - ProcessOrphanTx(connman, mempool, pfrom.orphan_work_set, lRemovedTxn); + ProcessOrphanTx(pfrom.orphan_work_set, lRemovedTxn); } else if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { bool fRejectedParents = false; // It may be the case that the orphans parents have all been rejected + + // Deduplicate parent txids, so that we don't have to loop over + // the same parent txid more than once down below. + std::vector<uint256> unique_parents; + unique_parents.reserve(tx.vin.size()); for (const CTxIn& txin : tx.vin) { - if (recentRejects->contains(txin.prevout.hash)) { + // We start with all parents, and then remove duplicates below. + unique_parents.push_back(txin.prevout.hash); + } + std::sort(unique_parents.begin(), unique_parents.end()); + unique_parents.erase(std::unique(unique_parents.begin(), unique_parents.end()), unique_parents.end()); + for (const uint256& parent_txid : unique_parents) { + if (recentRejects->contains(parent_txid)) { fRejectedParents = true; break; } } if (!fRejectedParents) { - uint32_t nFetchFlags = GetFetchFlags(pfrom); const auto current_time = GetTime<std::chrono::microseconds>(); - for (const CTxIn& txin : tx.vin) { + for (const uint256& parent_txid : unique_parents) { // Here, we only have the txid (and not wtxid) of the // inputs, so we only request in txid mode, even for // wtxidrelay peers. // Eventually we should replace this with an improved // protocol for getting all unconfirmed parents. - CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); - pfrom.AddKnownTx(txin.prevout.hash); - if (!AlreadyHave(_inv, mempool)) RequestTx(State(pfrom.GetId()), ToGenTxid(_inv), current_time); + const GenTxid gtxid{/* is_wtxid=*/false, parent_txid}; + pfrom.AddKnownTx(parent_txid); + if (!AlreadyHaveTx(gtxid, m_mempool)) RequestTx(State(pfrom.GetId()), gtxid, current_time); } AddOrphanTx(ptx, pfrom.GetId()); @@ -3035,11 +3093,20 @@ void ProcessMessage( // if we start doing this too early. assert(recentRejects); recentRejects->insert(tx.GetWitnessHash()); + // If the transaction failed for TX_INPUTS_NOT_STANDARD, + // then we know that the witness was irrelevant to the policy + // failure, since this check depends only on the txid + // (the scriptPubKey being spent is covered by the txid). + // Add the txid to the reject filter to prevent repeated + // processing of this transaction in the event that child + // transactions are later received (resulting in + // parent-fetching by txid via the orphan-handling logic). + if (state.GetResult() == TxValidationResult::TX_INPUTS_NOT_STANDARD && tx.GetWitnessHash() != tx.GetHash()) { + recentRejects->insert(tx.GetHash()); + } if (RecursiveDynamicUsage(*ptx) < 100000) { AddToCompactExtraTransactions(ptx); } - } else if (tx.HasWitness() && RecursiveDynamicUsage(*ptx) < 100000) { - AddToCompactExtraTransactions(ptx); } if (pfrom.HasPermission(PF_FORCERELAY)) { @@ -3047,11 +3114,11 @@ void ProcessMessage( // if they were already in the mempool, // allowing the node to function as a gateway for // nodes hidden behind it. - if (!mempool.exists(tx.GetHash())) { + if (!m_mempool.exists(tx.GetHash())) { LogPrintf("Not relaying non-mempool transaction %s from forcerelay peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); } else { LogPrintf("Force relaying tx %s from peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); - RelayTransaction(tx.GetHash(), tx.GetWitnessHash(), connman); + RelayTransaction(tx.GetHash(), tx.GetWitnessHash(), m_connman); } } } @@ -3104,7 +3171,7 @@ void ProcessMessage( if (!LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers if (!::ChainstateActive().IsInitialBlockDownload()) - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); return; } @@ -3115,7 +3182,7 @@ void ProcessMessage( const CBlockIndex *pindex = nullptr; BlockValidationState state; - if (!chainman.ProcessNewBlockHeaders({cmpctblock.header}, state, chainparams, &pindex)) { + if (!m_chainman.ProcessNewBlockHeaders({cmpctblock.header}, state, m_chainparams, &pindex)) { if (state.IsInvalid()) { MaybePunishNodeForBlock(pfrom.GetId(), state, /*via_compact_block*/ true, "invalid header via cmpctblock"); return; @@ -3165,16 +3232,16 @@ void ProcessMessage( // so we just grab the block via normal getdata std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); } return; } // If we're not close to tip yet, give up and let parallel block fetch work its magic - if (!fAlreadyInFlight && !CanDirectFetch(chainparams.GetConsensus())) + if (!fAlreadyInFlight && !CanDirectFetch(m_chainparams.GetConsensus())) return; - if (IsWitnessEnabled(pindex->pprev, chainparams.GetConsensus()) && !nodestate->fSupportsDesiredCmpctVersion) { + if (IsWitnessEnabled(pindex->pprev, m_chainparams.GetConsensus()) && !nodestate->fSupportsDesiredCmpctVersion) { // Don't bother trying to process compact blocks from v1 peers // after segwit activates. return; @@ -3186,9 +3253,9 @@ void ProcessMessage( if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || (fAlreadyInFlight && blockInFlightIt->second.first == pfrom.GetId())) { std::list<QueuedBlock>::iterator* queuedBlockIt = nullptr; - if (!MarkBlockAsInFlight(mempool, pfrom.GetId(), pindex->GetBlockHash(), pindex, &queuedBlockIt)) { + if (!MarkBlockAsInFlight(m_mempool, pfrom.GetId(), pindex->GetBlockHash(), pindex, &queuedBlockIt)) { if (!(*queuedBlockIt)->partialBlock) - (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&mempool)); + (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&m_mempool)); else { // The block was already in flight using compact blocks from the same peer LogPrint(BCLog::NET, "Peer sent us compact block we were already syncing!\n"); @@ -3206,7 +3273,7 @@ void ProcessMessage( // Duplicate txindexes, the block is now in-flight, so just request it std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return; } @@ -3223,7 +3290,7 @@ void ProcessMessage( fProcessBLOCKTXN = true; } else { req.blockhash = pindex->GetBlockHash(); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); } } else { // This block is either already in flight from a different @@ -3231,7 +3298,7 @@ void ProcessMessage( // download from. // Optimistically try to reconstruct anyway since we might be // able to without any round trips. - PartiallyDownloadedBlock tempBlock(&mempool); + PartiallyDownloadedBlock tempBlock(&m_mempool); ReadStatus status = tempBlock.InitData(cmpctblock, vExtraTxnForCompact); if (status != READ_STATUS_OK) { // TODO: don't ignore failures @@ -3249,7 +3316,7 @@ void ProcessMessage( // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return; } else { // If this was an announce-cmpctblock, we want the same treatment as a header message @@ -3258,8 +3325,9 @@ void ProcessMessage( } } // cs_main - if (fProcessBLOCKTXN) - return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, time_received, chainparams, chainman, mempool, connman, banman, interruptMsgProc); + if (fProcessBLOCKTXN) { + return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, time_received, interruptMsgProc); + } if (fRevertToHeaderProcessing) { // Headers received from HB compact block peers are permitted to be @@ -3267,7 +3335,7 @@ void ProcessMessage( // the peer if the header turns out to be for an invalid block. // Note that if a peer tries to build on an invalid chain, that // will be detected and the peer will be disconnected/discouraged. - return ProcessHeadersMessage(pfrom, connman, chainman, mempool, {cmpctblock.header}, chainparams, /*via_compact_block=*/true); + return ProcessHeadersMessage(pfrom, {cmpctblock.header}, /*via_compact_block=*/true); } if (fBlockReconstructed) { @@ -3287,7 +3355,7 @@ void ProcessMessage( // we have a chain with at least nMinimumChainWork), and we ignore // compact blocks with less work than our tip, it is safe to treat // reconstructed compact blocks as having been requested. - chainman.ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); + m_chainman.ProcessNewBlock(m_chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { pfrom.nLastBlockTime = GetTime(); } else { @@ -3339,7 +3407,7 @@ void ProcessMessage( // Might have collided, fall back to getdata now :( std::vector<CInv> invs; invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom), resp.blockhash)); - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); } else { // Block is either okay, or possibly we received // READ_STATUS_CHECKBLOCK_FAILED. @@ -3377,7 +3445,7 @@ void ProcessMessage( // disk-space attacks), but this should be safe due to the // protections in the compact block handler -- see related comment // in compact block optimistic reconstruction handling. - chainman.ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); + m_chainman.ProcessNewBlock(m_chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { pfrom.nLastBlockTime = GetTime(); } else { @@ -3401,7 +3469,6 @@ void ProcessMessage( // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. unsigned int nCount = ReadCompactSize(vRecv); if (nCount > MAX_HEADERS_RESULTS) { - LOCK(cs_main); Misbehaving(pfrom.GetId(), 20, strprintf("headers message size = %u", nCount)); return; } @@ -3411,7 +3478,7 @@ void ProcessMessage( ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } - return ProcessHeadersMessage(pfrom, connman, chainman, mempool, headers, chainparams, /*via_compact_block=*/false); + return ProcessHeadersMessage(pfrom, headers, /*via_compact_block=*/false); } if (msg_type == NetMsgType::BLOCK) @@ -3440,7 +3507,7 @@ void ProcessMessage( mapBlockSource.emplace(hash, std::make_pair(pfrom.GetId(), true)); } bool fNewBlock = false; - chainman.ProcessNewBlock(chainparams, pblock, forceProcessing, &fNewBlock); + m_chainman.ProcessNewBlock(m_chainparams, pblock, forceProcessing, &fNewBlock); if (fNewBlock) { pfrom.nLastBlockTime = GetTime(); } else { @@ -3456,11 +3523,11 @@ void ProcessMessage( // to users' AddrMan and later request them by sending getaddr messages. // Making nodes which are behind NAT and can only make outgoing connections ignore // the getaddr message mitigates the attack. - if (!pfrom.fInbound) { + if (!pfrom.IsInboundConn()) { LogPrint(BCLog::NET, "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom.GetId()); return; } - if (!pfrom.IsAddrRelayPeer()) { + if (!pfrom.RelayAddrsWithConn()) { LogPrint(BCLog::NET, "Ignoring \"getaddr\" from block-relay-only connection. peer=%d\n", pfrom.GetId()); return; } @@ -3476,9 +3543,9 @@ void ProcessMessage( pfrom.vAddrToSend.clear(); std::vector<CAddress> vAddr; if (pfrom.HasPermission(PF_ADDR)) { - vAddr = connman.GetAddresses(); + vAddr = m_connman.GetAddresses(MAX_ADDR_TO_SEND, MAX_PCT_ADDR_TO_SEND); } else { - vAddr = connman.GetAddresses(pfrom.addr.GetNetwork()); + vAddr = m_connman.GetAddresses(pfrom.addr.GetNetwork(), MAX_ADDR_TO_SEND, MAX_PCT_ADDR_TO_SEND); } FastRandomContext insecure_rand; for (const CAddress &addr : vAddr) { @@ -3498,7 +3565,7 @@ void ProcessMessage( return; } - if (connman.OutboundTargetReached(false) && !pfrom.HasPermission(PF_MEMPOOL)) + if (m_connman.OutboundTargetReached(false) && !pfrom.HasPermission(PF_MEMPOOL)) { if (!pfrom.HasPermission(PF_NOBAN)) { @@ -3531,7 +3598,7 @@ void ProcessMessage( // it, if the remote node sends a ping once per second and this node takes 5 // seconds to respond to each, the 5th ping the remote sends would appear to // return very quickly. - connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); } return; } @@ -3603,7 +3670,6 @@ void ProcessMessage( if (!filter.IsWithinSizeConstraints()) { // There is no excuse for sending a too-large filter - LOCK(cs_main); Misbehaving(pfrom.GetId(), 100, "too-large bloom filter"); } else if (pfrom.m_tx_relay != nullptr) @@ -3637,7 +3703,6 @@ void ProcessMessage( } } if (bad) { - LOCK(cs_main); Misbehaving(pfrom.GetId(), 100, "bad filteradd message"); } return; @@ -3671,17 +3736,17 @@ void ProcessMessage( } if (msg_type == NetMsgType::GETCFILTERS) { - ProcessGetCFilters(pfrom, vRecv, chainparams, connman); + ProcessGetCFilters(pfrom, vRecv, m_chainparams, m_connman); return; } if (msg_type == NetMsgType::GETCFHEADERS) { - ProcessGetCFHeaders(pfrom, vRecv, chainparams, connman); + ProcessGetCFHeaders(pfrom, vRecv, m_chainparams, m_connman); return; } if (msg_type == NetMsgType::GETCFCHECKPT) { - ProcessGetCFCheckPt(pfrom, vRecv, chainparams, connman); + ProcessGetCFCheckPt(pfrom, vRecv, m_chainparams, m_connman); return; } @@ -3715,23 +3780,20 @@ void ProcessMessage( return; } -/** Maybe disconnect a peer and discourage future connections from its address. - * - * @param[in] pnode The node to check. - * @return True if the peer was marked for disconnection in this function - */ -bool PeerLogicValidation::MaybeDiscourageAndDisconnect(CNode& pnode) +bool PeerManager::MaybeDiscourageAndDisconnect(CNode& pnode) { const NodeId peer_id{pnode.GetId()}; + PeerRef peer = GetPeerRef(peer_id); + if (peer == nullptr) return false; + { - LOCK(cs_main); - CNodeState& state = *State(peer_id); + LOCK(peer->m_misbehavior_mutex); // There's nothing to do if the m_should_discourage flag isn't set - if (!state.m_should_discourage) return false; + if (!peer->m_should_discourage) return false; - state.m_should_discourage = false; - } // cs_main + peer->m_should_discourage = false; + } // peer.m_misbehavior_mutex if (pnode.HasPermission(PF_NOBAN)) { // We never disconnect or discourage peers for bad behavior if they have the NOBAN permission flag @@ -3739,7 +3801,7 @@ bool PeerLogicValidation::MaybeDiscourageAndDisconnect(CNode& pnode) return false; } - if (pnode.m_manual_connection) { + if (pnode.IsManualConn()) { // We never disconnect or discourage manual peers for bad behavior LogPrintf("Warning: not punishing manually connected peer %d!\n", peer_id); return false; @@ -3756,13 +3818,12 @@ bool PeerLogicValidation::MaybeDiscourageAndDisconnect(CNode& pnode) // Normal case: Disconnect the peer and discourage all nodes sharing the address LogPrintf("Disconnecting and discouraging peer %d!\n", peer_id); if (m_banman) m_banman->Discourage(pnode.addr); - connman->DisconnectNode(pnode.addr); + m_connman.DisconnectNode(pnode.addr); return true; } -bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& interruptMsgProc) +bool PeerManager::ProcessMessages(CNode* pfrom, std::atomic<bool>& interruptMsgProc) { - const CChainParams& chainparams = Params(); // // Message format // (4) message start @@ -3774,12 +3835,12 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter bool fMoreWork = false; if (!pfrom->vRecvGetData.empty()) - ProcessGetData(*pfrom, chainparams, *connman, m_mempool, interruptMsgProc); + ProcessGetData(*pfrom, m_chainparams, m_connman, m_mempool, interruptMsgProc); if (!pfrom->orphan_work_set.empty()) { std::list<CTransactionRef> removed_txn; LOCK2(cs_main, g_cs_orphans); - ProcessOrphanTx(*connman, m_mempool, pfrom->orphan_work_set, removed_txn); + ProcessOrphanTx(pfrom->orphan_work_set, removed_txn); for (const CTransactionRef& removedTx : removed_txn) { AddToCompactExtraTransactions(removedTx); } @@ -3805,7 +3866,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter // Just take one message msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); pfrom->nProcessQueueSize -= msgs.front().m_raw_message_size; - pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman->GetReceiveFloodSize(); + pfrom->fPauseRecv = pfrom->nProcessQueueSize > m_connman.GetReceiveFloodSize(); fMoreWork = !pfrom->vProcessMsg.empty(); } CNetMessage& msg(msgs.front()); @@ -3839,7 +3900,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter } try { - ProcessMessage(*pfrom, msg_type, vRecv, msg.m_time, chainparams, m_chainman, m_mempool, *connman, m_banman, interruptMsgProc); + ProcessMessage(*pfrom, msg_type, vRecv, msg.m_time, interruptMsgProc); if (interruptMsgProc) return false; if (!pfrom->vRecvGetData.empty()) @@ -3853,14 +3914,14 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter return fMoreWork; } -void PeerLogicValidation::ConsiderEviction(CNode& pto, int64_t time_in_seconds) +void PeerManager::ConsiderEviction(CNode& pto, int64_t time_in_seconds) { AssertLockHeld(cs_main); CNodeState &state = *State(pto.GetId()); const CNetMsgMaker msgMaker(pto.GetSendVersion()); - if (!state.m_chain_sync.m_protect && IsOutboundDisconnectionCandidate(pto) && state.fSyncStarted) { + if (!state.m_chain_sync.m_protect && pto.IsOutboundOrBlockRelayConn() && state.fSyncStarted) { // This is an outbound peer subject to disconnection if they don't // announce a block with as much work as the current tip within // CHAIN_SYNC_TIMEOUT + HEADERS_RESPONSE_TIME seconds (note: if @@ -3892,7 +3953,7 @@ void PeerLogicValidation::ConsiderEviction(CNode& pto, int64_t time_in_seconds) } else { assert(state.m_chain_sync.m_work_header); LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto.GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>", state.m_chain_sync.m_work_header->GetBlockHash().ToString()); - connman->PushMessage(&pto, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); + m_connman.PushMessage(&pto, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); state.m_chain_sync.m_sent_getheaders = true; constexpr int64_t HEADERS_RESPONSE_TIME = 120; // 2 minutes // Bump the timeout to allow a response, which could clear the timeout @@ -3906,10 +3967,10 @@ void PeerLogicValidation::ConsiderEviction(CNode& pto, int64_t time_in_seconds) } } -void PeerLogicValidation::EvictExtraOutboundPeers(int64_t time_in_seconds) +void PeerManager::EvictExtraOutboundPeers(int64_t time_in_seconds) { // Check whether we have too many outbound peers - int extra_peers = connman->GetExtraOutboundCount(); + int extra_peers = m_connman.GetExtraOutboundCount(); if (extra_peers > 0) { // If we have more outbound peers than we target, disconnect one. // Pick the outbound peer that least recently announced @@ -3918,11 +3979,11 @@ void PeerLogicValidation::EvictExtraOutboundPeers(int64_t time_in_seconds) NodeId worst_peer = -1; int64_t oldest_block_announcement = std::numeric_limits<int64_t>::max(); - connman->ForEachNode([&](CNode* pnode) { - AssertLockHeld(cs_main); + m_connman.ForEachNode([&](CNode* pnode) { + LockAssertion lock(::cs_main); // Ignore non-outbound peers, or nodes marked for disconnect already - if (!IsOutboundDisconnectionCandidate(*pnode) || pnode->fDisconnect) return; + if (!pnode->IsOutboundOrBlockRelayConn() || pnode->fDisconnect) return; CNodeState *state = State(pnode->GetId()); if (state == nullptr) return; // shouldn't be possible, but just in case // Don't evict our protected peers @@ -3935,8 +3996,8 @@ void PeerLogicValidation::EvictExtraOutboundPeers(int64_t time_in_seconds) } }); if (worst_peer != -1) { - bool disconnected = connman->ForNode(worst_peer, [&](CNode *pnode) { - AssertLockHeld(cs_main); + bool disconnected = m_connman.ForNode(worst_peer, [&](CNode *pnode) { + LockAssertion lock(::cs_main); // Only disconnect a peer that has been connected to us for // some reasonable fraction of our check-frequency, to give @@ -3959,18 +4020,16 @@ void PeerLogicValidation::EvictExtraOutboundPeers(int64_t time_in_seconds) // detected a stale tip. Don't try any more extra peers until // we next detect a stale tip, to limit the load we put on the // network from these extra connections. - connman->SetTryNewOutboundPeer(false); + m_connman.SetTryNewOutboundPeer(false); } } } } -void PeerLogicValidation::CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams) +void PeerManager::CheckForStaleTipAndEvictPeers() { LOCK(cs_main); - if (connman == nullptr) return; - int64_t time_in_seconds = GetTime(); EvictExtraOutboundPeers(time_in_seconds); @@ -3978,11 +4037,11 @@ void PeerLogicValidation::CheckForStaleTipAndEvictPeers(const Consensus::Params if (time_in_seconds > m_stale_tip_check_time) { // Check whether our tip is stale, and if so, allow using an extra // outbound peer - if (!fImporting && !fReindex && connman->GetNetworkActive() && connman->GetUseAddrmanOutgoing() && TipMayBeStale(consensusParams)) { + if (!fImporting && !fReindex && m_connman.GetNetworkActive() && m_connman.GetUseAddrmanOutgoing() && TipMayBeStale(m_chainparams.GetConsensus())) { LogPrintf("Potential stale tip detected, will try using extra outbound peer (last tip update: %d seconds ago)\n", time_in_seconds - g_last_tip_update); - connman->SetTryNewOutboundPeer(true); - } else if (connman->GetTryNewOutboundPeer()) { - connman->SetTryNewOutboundPeer(false); + m_connman.SetTryNewOutboundPeer(true); + } else if (m_connman.GetTryNewOutboundPeer()) { + m_connman.SetTryNewOutboundPeer(false); } m_stale_tip_check_time = time_in_seconds + STALE_CHECK_INTERVAL; } @@ -4009,9 +4068,9 @@ public: }; } -bool PeerLogicValidation::SendMessages(CNode* pto) +bool PeerManager::SendMessages(CNode* pto) { - const Consensus::Params& consensusParams = Params().GetConsensus(); + const Consensus::Params& consensusParams = m_chainparams.GetConsensus(); // We must call MaybeDiscourageAndDisconnect first, to ensure that we'll // disconnect misbehaving peers even before the version handshake is complete. @@ -4045,11 +4104,11 @@ bool PeerLogicValidation::SendMessages(CNode* pto) pto->m_ping_start = GetTime<std::chrono::microseconds>(); if (pto->nVersion > BIP0031_VERSION) { pto->nPingNonceSent = nonce; - connman->PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); } else { // Peer is too old to support ping command with nonce, pong will never arrive. pto->nPingNonceSent = 0; - connman->PushMessage(pto, msgMaker.Make(NetMsgType::PING)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING)); } } @@ -4062,7 +4121,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) int64_t nNow = GetTimeMicros(); auto current_time = GetTime<std::chrono::microseconds>(); - if (pto->IsAddrRelayPeer() && !::ChainstateActive().IsInitialBlockDownload() && pto->m_next_local_addr_send < current_time) { + if (pto->RelayAddrsWithConn() && !::ChainstateActive().IsInitialBlockDownload() && pto->m_next_local_addr_send < current_time) { AdvertiseLocal(pto); pto->m_next_local_addr_send = PoissonNextSend(current_time, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); } @@ -4070,7 +4129,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // // Message: addr // - if (pto->IsAddrRelayPeer() && pto->m_next_addr_send < current_time) { + if (pto->RelayAddrsWithConn() && pto->m_next_addr_send < current_time) { pto->m_next_addr_send = PoissonNextSend(current_time, AVG_ADDRESS_BROADCAST_INTERVAL); std::vector<CAddress> vAddr; vAddr.reserve(pto->vAddrToSend.size()); @@ -4084,14 +4143,14 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // receiver rejects addr messages larger than MAX_ADDR_TO_SEND if (vAddr.size() >= MAX_ADDR_TO_SEND) { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); vAddr.clear(); } } } pto->vAddrToSend.clear(); if (!vAddr.empty()) - connman->PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::ADDR, vAddr)); // we only send the big addr message once if (pto->vAddrToSend.capacity() > 40) pto->vAddrToSend.shrink_to_fit(); @@ -4100,7 +4159,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // Start block sync if (pindexBestHeader == nullptr) pindexBestHeader = ::ChainActive().Tip(); - bool fFetch = state.fPreferredDownload || (nPreferredDownload == 0 && !pto->fClient && !pto->fOneShot); // Download if this is a nice peer, or we have no nice peers and this one might do. + bool fFetch = state.fPreferredDownload || (nPreferredDownload == 0 && !pto->fClient && !pto->IsAddrFetchConn()); // Download if this is a nice peer, or we have no nice peers and this one might do. if (!state.fSyncStarted && !pto->fClient && !fImporting && !fReindex) { // Only actively request headers from a single peer, unless we're close to today. if ((nSyncStarted == 0 && fFetch) || pindexBestHeader->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { @@ -4118,7 +4177,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) if (pindexStart->pprev) pindexStart = pindexStart->pprev; LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), pto->nStartingHeight); - connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexStart), uint256())); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexStart), uint256())); } } @@ -4202,10 +4261,10 @@ bool PeerLogicValidation::SendMessages(CNode* pto) LOCK(cs_most_recent_block); if (most_recent_block_hash == pBestIndex->GetBlockHash()) { if (state.fWantsCmpctWitness || !fWitnessesPresentInMostRecentCompactBlock) - connman->PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); + m_connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); else { CBlockHeaderAndShortTxIDs cmpctblock(*most_recent_block, state.fWantsCmpctWitness); - connman->PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); + m_connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); } fGotBlockFromCache = true; } @@ -4215,7 +4274,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) bool ret = ReadBlockFromDisk(block, pBestIndex, consensusParams); assert(ret); CBlockHeaderAndShortTxIDs cmpctblock(block, state.fWantsCmpctWitness); - connman->PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); + m_connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); } state.pindexBestHeaderSent = pBestIndex; } else if (state.fPreferHeaders) { @@ -4228,7 +4287,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) LogPrint(BCLog::NET, "%s: sending header %s to peer=%d\n", __func__, vHeaders.front().GetHash().ToString(), pto->GetId()); } - connman->PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); state.pindexBestHeaderSent = pBestIndex; } else fRevertToInv = true; @@ -4273,7 +4332,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) for (const uint256& hash : pto->vInventoryBlockToSend) { vInv.push_back(CInv(MSG_BLOCK, hash)); if (vInv.size() == MAX_INV_SZ) { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } @@ -4285,8 +4344,8 @@ bool PeerLogicValidation::SendMessages(CNode* pto) bool fSendTrickle = pto->HasPermission(PF_NOBAN); if (pto->m_tx_relay->nNextInvSend < current_time) { fSendTrickle = true; - if (pto->fInbound) { - pto->m_tx_relay->nNextInvSend = std::chrono::microseconds{connman->PoissonNextSendInbound(nNow, INVENTORY_BROADCAST_INTERVAL)}; + if (pto->IsInboundConn()) { + pto->m_tx_relay->nNextInvSend = std::chrono::microseconds{m_connman.PoissonNextSendInbound(nNow, INVENTORY_BROADCAST_INTERVAL)}; } else { // Use half the delay for outbound peers, as there is less privacy concern for them. pto->m_tx_relay->nNextInvSend = PoissonNextSend(current_time, std::chrono::seconds{INVENTORY_BROADCAST_INTERVAL >> 1}); @@ -4326,7 +4385,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // Responses to MEMPOOL requests bypass the m_recently_announced_invs filter. vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } @@ -4402,7 +4461,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) } } if (vInv.size() == MAX_INV_SZ) { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } pto->m_tx_relay->filterInventoryKnown.insert(hash); @@ -4419,7 +4478,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) } } if (!vInv.empty()) - connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); // Detect whether we're stalling current_time = GetTime<std::chrono::microseconds>(); @@ -4539,7 +4598,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // processing at a later time, see below) tx_process_time.erase(tx_process_time.begin()); CInv inv(gtxid.IsWtxid() ? MSG_WTX : (MSG_TX | GetFetchFlags(*pto)), gtxid.GetHash()); - if (!AlreadyHave(inv, m_mempool)) { + if (!AlreadyHaveTx(ToGenTxid(inv), m_mempool)) { // If this transaction was last requested more than 1 minute ago, // then request. const auto last_request_time = GetTxRequestTime(gtxid); @@ -4547,7 +4606,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) LogPrint(BCLog::NET, "Requesting %s peer=%d\n", inv.ToString(), pto->GetId()); vGetData.push_back(inv); if (vGetData.size() >= MAX_GETDATA_SZ) { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); vGetData.clear(); } UpdateTxRequestTime(gtxid, current_time); @@ -4577,7 +4636,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) if (!vGetData.empty()) - connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); // // Message: feefilter @@ -4605,7 +4664,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // We always have a fee filter of at least minRelayTxFee filterToSend = std::max(filterToSend, ::minRelayTxFee.GetFeePerK()); if (filterToSend != pto->m_tx_relay->lastSentFeeFilter) { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::FEEFILTER, filterToSend)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::FEEFILTER, filterToSend)); pto->m_tx_relay->lastSentFeeFilter = filterToSend; } pto->m_tx_relay->nextSendTimeFeeFilter = PoissonNextSend(timeNow, AVG_FEEFILTER_BROADCAST_INTERVAL); diff --git a/src/net_processing.h b/src/net_processing.h index 2d9871412..3e748c0c5 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -11,8 +11,13 @@ #include <sync.h> #include <validationinterface.h> +class BlockTransactionsRequest; +class BlockValidationState; +class CBlockHeader; +class CChainParams; class CTxMemPool; class ChainstateManager; +class TxValidationState; extern RecursiveMutex cs_main; extern RecursiveMutex g_cs_orphans; @@ -26,18 +31,10 @@ static const bool DEFAULT_PEERBLOCKFILTERS = false; /** Threshold for marking a node to be discouraged, e.g. disconnected and added to the discouragement filter. */ static const int DISCOURAGEMENT_THRESHOLD{100}; -class PeerLogicValidation final : public CValidationInterface, public NetEventsInterface { -private: - CConnman* const connman; - /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ - BanMan* const m_banman; - ChainstateManager& m_chainman; - CTxMemPool& m_mempool; - - bool MaybeDiscourageAndDisconnect(CNode& pnode); - +class PeerManager final : public CValidationInterface, public NetEventsInterface { public: - PeerLogicValidation(CConnman* connman, BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool); + PeerManager(const CChainParams& chainparams, CConnman& connman, BanMan* banman, + CScheduler& scheduler, ChainstateManager& chainman, CTxMemPool& pool); /** * Overridden from CValidationInterface. @@ -79,18 +76,70 @@ public: /** Consider evicting an outbound peer based on the amount of time they've been behind our tip */ void ConsiderEviction(CNode& pto, int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Evict extra outbound peers. If we think our tip may be stale, connect to an extra outbound */ - void CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams); + void CheckForStaleTipAndEvictPeers(); /** If we have extra outbound peers, try to disconnect the one with the oldest block announcement */ void EvictExtraOutboundPeers(int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Retrieve unbroadcast transactions from the mempool and reattempt sending to peers */ void ReattemptInitialBroadcast(CScheduler& scheduler) const; + /** Process a single message from a peer. Public for fuzz testing */ + void ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, + const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc); + + /** + * Increment peer's misbehavior score. If the new value >= DISCOURAGEMENT_THRESHOLD, mark the node + * to be discouraged, meaning the peer might be disconnected and added to the discouragement filter. + * Public for unit testing. + */ + void Misbehaving(const NodeId pnode, const int howmuch, const std::string& message); + private: + /** + * Potentially mark a node discouraged based on the contents of a BlockValidationState object + * + * @param[in] via_compact_block this bool is passed in because net_processing should + * punish peers differently depending on whether the data was provided in a compact + * block message or not. If the compact block had a valid header, but contained invalid + * txs, the peer should not be punished. See BIP 152. + * + * @return Returns true if the peer was punished (probably disconnected) + */ + bool MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, + bool via_compact_block, const std::string& message = ""); + + /** + * Potentially disconnect and discourage a node based on the contents of a TxValidationState object + * + * @return Returns true if the peer was punished (probably disconnected) + */ + bool MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, const std::string& message = ""); + + /** Maybe disconnect a peer and discourage future connections from its address. + * + * @param[in] pnode The node to check. + * @return True if the peer was marked for disconnection in this function + */ + bool MaybeDiscourageAndDisconnect(CNode& pnode); + + void ProcessOrphanTx(std::set<uint256>& orphan_work_set, std::list<CTransactionRef>& removed_txn) + EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans); + /** Process a single headers message from a peer. */ + void ProcessHeadersMessage(CNode& pfrom, const std::vector<CBlockHeader>& headers, bool via_compact_block); + + void SendBlockTransactions(CNode& pfrom, const CBlock& block, const BlockTransactionsRequest& req); + + const CChainParams& m_chainparams; + CConnman& m_connman; + /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ + BanMan* const m_banman; + ChainstateManager& m_chainman; + CTxMemPool& m_mempool; + int64_t m_stale_tip_check_time; //!< Next time to check for stale tip }; struct CNodeStateStats { - int nMisbehavior = 0; + int m_misbehavior_score = 0; int nSyncHeight = -1; int nCommonHeight = -1; std::vector<int> vHeightInFlight; diff --git a/src/netaddress.cpp b/src/netaddress.cpp index d29aed6c8..b50cf7406 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -3,79 +3,91 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <cstdint> #include <netaddress.h> + #include <hash.h> +#include <tinyformat.h> #include <util/strencodings.h> #include <util/asmap.h> -#include <tinyformat.h> -static const unsigned char pchIPv4[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff }; -static const unsigned char pchOnionCat[] = {0xFD,0x87,0xD8,0x7E,0xEB,0x43}; +#include <algorithm> +#include <array> +#include <cstdint> +#include <iterator> +#include <tuple> -// 0xFD + sha256("bitcoin")[0:5] -static const unsigned char g_internal_prefix[] = { 0xFD, 0x6B, 0x88, 0xC0, 0x87, 0x24 }; +constexpr size_t CNetAddr::V1_SERIALIZATION_SIZE; /** * Construct an unspecified IPv6 network address (::/128). * * @note This address is considered invalid by CNetAddr::IsValid() */ -CNetAddr::CNetAddr() -{ - memset(ip, 0, sizeof(ip)); -} +CNetAddr::CNetAddr() {} void CNetAddr::SetIP(const CNetAddr& ipIn) { + // Size check. + switch (ipIn.m_net) { + case NET_IPV4: + assert(ipIn.m_addr.size() == ADDR_IPV4_SIZE); + break; + case NET_IPV6: + assert(ipIn.m_addr.size() == ADDR_IPV6_SIZE); + break; + case NET_ONION: + assert(ipIn.m_addr.size() == ADDR_TORV2_SIZE); + break; + case NET_INTERNAL: + assert(ipIn.m_addr.size() == ADDR_INTERNAL_SIZE); + break; + case NET_UNROUTABLE: + case NET_MAX: + assert(false); + } // no default case, so the compiler can warn about missing cases + m_net = ipIn.m_net; - memcpy(ip, ipIn.ip, sizeof(ip)); + m_addr = ipIn.m_addr; +} + +template <typename T1, size_t PREFIX_LEN> +inline bool HasPrefix(const T1& obj, const std::array<uint8_t, PREFIX_LEN>& prefix) +{ + return obj.size() >= PREFIX_LEN && + std::equal(std::begin(prefix), std::end(prefix), std::begin(obj)); } -void CNetAddr::SetLegacyIPv6(const uint8_t ipv6[16]) +void CNetAddr::SetLegacyIPv6(Span<const uint8_t> ipv6) { - if (memcmp(ipv6, pchIPv4, sizeof(pchIPv4)) == 0) { + assert(ipv6.size() == ADDR_IPV6_SIZE); + + size_t skip{0}; + + if (HasPrefix(ipv6, IPV4_IN_IPV6_PREFIX)) { + // IPv4-in-IPv6 m_net = NET_IPV4; - } else if (memcmp(ipv6, pchOnionCat, sizeof(pchOnionCat)) == 0) { + skip = sizeof(IPV4_IN_IPV6_PREFIX); + } else if (HasPrefix(ipv6, TORV2_IN_IPV6_PREFIX)) { + // TORv2-in-IPv6 m_net = NET_ONION; - } else if (memcmp(ipv6, g_internal_prefix, sizeof(g_internal_prefix)) == 0) { + skip = sizeof(TORV2_IN_IPV6_PREFIX); + } else if (HasPrefix(ipv6, INTERNAL_IN_IPV6_PREFIX)) { + // Internal-in-IPv6 m_net = NET_INTERNAL; + skip = sizeof(INTERNAL_IN_IPV6_PREFIX); } else { + // IPv6 m_net = NET_IPV6; } - memcpy(ip, ipv6, 16); -} -void CNetAddr::SetRaw(Network network, const uint8_t *ip_in) -{ - switch(network) - { - case NET_IPV4: - m_net = NET_IPV4; - memcpy(ip, pchIPv4, 12); - memcpy(ip+12, ip_in, 4); - break; - case NET_IPV6: - SetLegacyIPv6(ip_in); - break; - default: - assert(!"invalid network"); - } + m_addr.assign(ipv6.begin() + skip, ipv6.end()); } /** - * Try to make this a dummy address that maps the specified name into IPv6 like - * so: (0xFD + %sha256("bitcoin")[0:5]) + %sha256(name)[0:10]. Such dummy - * addresses have a prefix of fd6b:88c0:8724::/48 and are guaranteed to not be - * publicly routable as it falls under RFC4193's fc00::/7 subnet allocated to - * unique-local addresses. - * - * CAddrMan uses these fake addresses to keep track of which DNS seeds were - * used. - * + * Create an "internal" address that represents a name or FQDN. CAddrMan uses + * these fake addresses to keep track of which DNS seeds were used. * @returns Whether or not the operation was successful. - * - * @see CNetAddr::IsInternal(), CNetAddr::IsRFC4193() + * @see NET_INTERNAL, INTERNAL_IN_IPV6_PREFIX, CNetAddr::IsInternal(), CNetAddr::IsRFC4193() */ bool CNetAddr::SetInternal(const std::string &name) { @@ -85,31 +97,26 @@ bool CNetAddr::SetInternal(const std::string &name) m_net = NET_INTERNAL; unsigned char hash[32] = {}; CSHA256().Write((const unsigned char*)name.data(), name.size()).Finalize(hash); - memcpy(ip, g_internal_prefix, sizeof(g_internal_prefix)); - memcpy(ip + sizeof(g_internal_prefix), hash, sizeof(ip) - sizeof(g_internal_prefix)); + m_addr.assign(hash, hash + ADDR_INTERNAL_SIZE); return true; } /** - * Try to make this a dummy address that maps the specified onion address into - * IPv6 using OnionCat's range and encoding. Such dummy addresses have a prefix - * of fd87:d87e:eb43::/48 and are guaranteed to not be publicly routable as they - * fall under RFC4193's fc00::/7 subnet allocated to unique-local addresses. + * Parse a TORv2 address and set this object to it. * * @returns Whether or not the operation was successful. * - * @see CNetAddr::IsTor(), CNetAddr::IsRFC4193() + * @see CNetAddr::IsTor() */ bool CNetAddr::SetSpecial(const std::string &strName) { if (strName.size()>6 && strName.substr(strName.size() - 6, 6) == ".onion") { std::vector<unsigned char> vchAddr = DecodeBase32(strName.substr(0, strName.size() - 6).c_str()); - if (vchAddr.size() != 16-sizeof(pchOnionCat)) + if (vchAddr.size() != ADDR_TORV2_SIZE) { return false; + } m_net = NET_ONION; - memcpy(ip, pchOnionCat, sizeof(pchOnionCat)); - for (unsigned int i=0; i<16-sizeof(pchOnionCat); i++) - ip[i + sizeof(pchOnionCat)] = vchAddr[i]; + m_addr.assign(vchAddr.begin(), vchAddr.end()); return true; } return false; @@ -117,28 +124,23 @@ bool CNetAddr::SetSpecial(const std::string &strName) CNetAddr::CNetAddr(const struct in_addr& ipv4Addr) { - SetRaw(NET_IPV4, (const uint8_t*)&ipv4Addr); + m_net = NET_IPV4; + const uint8_t* ptr = reinterpret_cast<const uint8_t*>(&ipv4Addr); + m_addr.assign(ptr, ptr + ADDR_IPV4_SIZE); } CNetAddr::CNetAddr(const struct in6_addr& ipv6Addr, const uint32_t scope) { - SetRaw(NET_IPV6, (const uint8_t*)&ipv6Addr); + SetLegacyIPv6(Span<const uint8_t>(reinterpret_cast<const uint8_t*>(&ipv6Addr), sizeof(ipv6Addr))); scopeId = scope; } -unsigned int CNetAddr::GetByte(int n) const -{ - return ip[15-n]; -} - bool CNetAddr::IsBindAny() const { - const int cmplen = IsIPv4() ? 4 : 16; - for (int i = 0; i < cmplen; ++i) { - if (GetByte(i)) return false; + if (!IsIPv4() && !IsIPv6()) { + return false; } - - return true; + return std::all_of(m_addr.begin(), m_addr.end(), [](uint8_t b) { return b == 0; }); } bool CNetAddr::IsIPv4() const { return m_net == NET_IPV4; } @@ -148,88 +150,88 @@ bool CNetAddr::IsIPv6() const { return m_net == NET_IPV6; } bool CNetAddr::IsRFC1918() const { return IsIPv4() && ( - GetByte(3) == 10 || - (GetByte(3) == 192 && GetByte(2) == 168) || - (GetByte(3) == 172 && (GetByte(2) >= 16 && GetByte(2) <= 31))); + m_addr[0] == 10 || + (m_addr[0] == 192 && m_addr[1] == 168) || + (m_addr[0] == 172 && m_addr[1] >= 16 && m_addr[1] <= 31)); } bool CNetAddr::IsRFC2544() const { - return IsIPv4() && GetByte(3) == 198 && (GetByte(2) == 18 || GetByte(2) == 19); + return IsIPv4() && m_addr[0] == 198 && (m_addr[1] == 18 || m_addr[1] == 19); } bool CNetAddr::IsRFC3927() const { - return IsIPv4() && (GetByte(3) == 169 && GetByte(2) == 254); + return IsIPv4() && HasPrefix(m_addr, std::array<uint8_t, 2>{169, 254}); } bool CNetAddr::IsRFC6598() const { - return IsIPv4() && GetByte(3) == 100 && GetByte(2) >= 64 && GetByte(2) <= 127; + return IsIPv4() && m_addr[0] == 100 && m_addr[1] >= 64 && m_addr[1] <= 127; } bool CNetAddr::IsRFC5737() const { - return IsIPv4() && ((GetByte(3) == 192 && GetByte(2) == 0 && GetByte(1) == 2) || - (GetByte(3) == 198 && GetByte(2) == 51 && GetByte(1) == 100) || - (GetByte(3) == 203 && GetByte(2) == 0 && GetByte(1) == 113)); + return IsIPv4() && (HasPrefix(m_addr, std::array<uint8_t, 3>{192, 0, 2}) || + HasPrefix(m_addr, std::array<uint8_t, 3>{198, 51, 100}) || + HasPrefix(m_addr, std::array<uint8_t, 3>{203, 0, 113})); } bool CNetAddr::IsRFC3849() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && - GetByte(13) == 0x0D && GetByte(12) == 0xB8; + return IsIPv6() && HasPrefix(m_addr, std::array<uint8_t, 4>{0x20, 0x01, 0x0D, 0xB8}); } bool CNetAddr::IsRFC3964() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x02; + return IsIPv6() && HasPrefix(m_addr, std::array<uint8_t, 2>{0x20, 0x02}); } bool CNetAddr::IsRFC6052() const { - static const unsigned char pchRFC6052[] = {0,0x64,0xFF,0x9B,0,0,0,0,0,0,0,0}; - return IsIPv6() && memcmp(ip, pchRFC6052, sizeof(pchRFC6052)) == 0; + return IsIPv6() && + HasPrefix(m_addr, std::array<uint8_t, 12>{0x00, 0x64, 0xFF, 0x9B, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}); } bool CNetAddr::IsRFC4380() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0 && - GetByte(12) == 0; + return IsIPv6() && HasPrefix(m_addr, std::array<uint8_t, 4>{0x20, 0x01, 0x00, 0x00}); } bool CNetAddr::IsRFC4862() const { - static const unsigned char pchRFC4862[] = {0xFE,0x80,0,0,0,0,0,0}; - return IsIPv6() && memcmp(ip, pchRFC4862, sizeof(pchRFC4862)) == 0; + return IsIPv6() && HasPrefix(m_addr, std::array<uint8_t, 8>{0xFE, 0x80, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}); } bool CNetAddr::IsRFC4193() const { - return IsIPv6() && (GetByte(15) & 0xFE) == 0xFC; + return IsIPv6() && (m_addr[0] & 0xFE) == 0xFC; } bool CNetAddr::IsRFC6145() const { - static const unsigned char pchRFC6145[] = {0,0,0,0,0,0,0,0,0xFF,0xFF,0,0}; - return IsIPv6() && memcmp(ip, pchRFC6145, sizeof(pchRFC6145)) == 0; + return IsIPv6() && + HasPrefix(m_addr, std::array<uint8_t, 12>{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00}); } bool CNetAddr::IsRFC4843() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && - GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x10; + return IsIPv6() && HasPrefix(m_addr, std::array<uint8_t, 3>{0x20, 0x01, 0x00}) && + (m_addr[3] & 0xF0) == 0x10; } bool CNetAddr::IsRFC7343() const { - return IsIPv6() && GetByte(15) == 0x20 && GetByte(14) == 0x01 && - GetByte(13) == 0x00 && (GetByte(12) & 0xF0) == 0x20; + return IsIPv6() && HasPrefix(m_addr, std::array<uint8_t, 3>{0x20, 0x01, 0x00}) && + (m_addr[3] & 0xF0) == 0x20; } bool CNetAddr::IsHeNet() const { - return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x04 && GetByte(12) == 0x70); + return IsIPv6() && HasPrefix(m_addr, std::array<uint8_t, 4>{0x20, 0x01, 0x04, 0x70}); } /** @@ -243,13 +245,15 @@ bool CNetAddr::IsTor() const { return m_net == NET_ONION; } bool CNetAddr::IsLocal() const { // IPv4 loopback (127.0.0.0/8 or 0.0.0.0/8) - if (IsIPv4() && (GetByte(3) == 127 || GetByte(3) == 0)) + if (IsIPv4() && (m_addr[0] == 127 || m_addr[0] == 0)) { return true; + } // IPv6 loopback (::1/128) static const unsigned char pchLocal[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1}; - if (IsIPv6() && memcmp(ip, pchLocal, 16) == 0) + if (IsIPv6() && memcmp(m_addr.data(), pchLocal, sizeof(pchLocal)) == 0) { return true; + } return false; } @@ -266,19 +270,11 @@ bool CNetAddr::IsLocal() const */ bool CNetAddr::IsValid() const { - // Cleanup 3-byte shifted addresses caused by garbage in size field - // of addr messages from versions before 0.2.9 checksum. - // Two consecutive addr messages look like this: - // header20 vectorlen3 addr26 addr26 addr26 header20 vectorlen3 addr26 addr26 addr26... - // so if the first length field is garbled, it reads the second batch - // of addr misaligned by 3 bytes. - if (IsIPv6() && memcmp(ip, pchIPv4+3, sizeof(pchIPv4)-3) == 0) - return false; - // unspecified IPv6 address (::/128) unsigned char ipNone6[16] = {}; - if (IsIPv6() && memcmp(ip, ipNone6, 16) == 0) + if (IsIPv6() && memcmp(m_addr.data(), ipNone6, sizeof(ipNone6)) == 0) { return false; + } // documentation IPv6 address if (IsRFC3849()) @@ -287,17 +283,11 @@ bool CNetAddr::IsValid() const if (IsInternal()) return false; - if (IsIPv4()) - { - // INADDR_NONE - uint32_t ipNone = INADDR_NONE; - if (memcmp(ip+12, &ipNone, 4) == 0) - return false; - - // 0 - ipNone = 0; - if (memcmp(ip+12, &ipNone, 4) == 0) + if (IsIPv4()) { + const uint32_t addr = ReadBE32(m_addr.data()); + if (addr == INADDR_ANY || addr == INADDR_NONE) { return false; + } } return true; @@ -318,7 +308,7 @@ bool CNetAddr::IsRoutable() const } /** - * @returns Whether or not this is a dummy address that maps a name into IPv6. + * @returns Whether or not this is a dummy address that represents a name. * * @see CNetAddr::SetInternal(const std::string &) */ @@ -341,9 +331,9 @@ enum Network CNetAddr::GetNetwork() const std::string CNetAddr::ToStringIP() const { if (IsTor()) - return EncodeBase32(&ip[6], 10) + ".onion"; + return EncodeBase32(m_addr) + ".onion"; if (IsInternal()) - return EncodeBase32(ip + sizeof(g_internal_prefix), sizeof(ip) - sizeof(g_internal_prefix)) + ".internal"; + return EncodeBase32(m_addr) + ".internal"; CService serv(*this, 0); struct sockaddr_storage sockaddr; socklen_t socklen = sizeof(sockaddr); @@ -353,13 +343,13 @@ std::string CNetAddr::ToStringIP() const return std::string(name); } if (IsIPv4()) - return strprintf("%u.%u.%u.%u", GetByte(3), GetByte(2), GetByte(1), GetByte(0)); - else - return strprintf("%x:%x:%x:%x:%x:%x:%x:%x", - GetByte(15) << 8 | GetByte(14), GetByte(13) << 8 | GetByte(12), - GetByte(11) << 8 | GetByte(10), GetByte(9) << 8 | GetByte(8), - GetByte(7) << 8 | GetByte(6), GetByte(5) << 8 | GetByte(4), - GetByte(3) << 8 | GetByte(2), GetByte(1) << 8 | GetByte(0)); + return strprintf("%u.%u.%u.%u", m_addr[0], m_addr[1], m_addr[2], m_addr[3]); + assert(IsIPv6()); + return strprintf("%x:%x:%x:%x:%x:%x:%x:%x", + m_addr[0] << 8 | m_addr[1], m_addr[2] << 8 | m_addr[3], + m_addr[4] << 8 | m_addr[5], m_addr[6] << 8 | m_addr[7], + m_addr[8] << 8 | m_addr[9], m_addr[10] << 8 | m_addr[11], + m_addr[12] << 8 | m_addr[13], m_addr[14] << 8 | m_addr[15]); } std::string CNetAddr::ToString() const @@ -369,12 +359,12 @@ std::string CNetAddr::ToString() const bool operator==(const CNetAddr& a, const CNetAddr& b) { - return a.m_net == b.m_net && memcmp(a.ip, b.ip, 16) == 0; + return a.m_net == b.m_net && a.m_addr == b.m_addr; } bool operator<(const CNetAddr& a, const CNetAddr& b) { - return a.m_net < b.m_net || (a.m_net == b.m_net && memcmp(a.ip, b.ip, 16) < 0); + return std::tie(a.m_net, a.m_addr) < std::tie(b.m_net, b.m_addr); } /** @@ -391,7 +381,8 @@ bool CNetAddr::GetInAddr(struct in_addr* pipv4Addr) const { if (!IsIPv4()) return false; - memcpy(pipv4Addr, ip+12, 4); + assert(sizeof(*pipv4Addr) == m_addr.size()); + memcpy(pipv4Addr, m_addr.data(), m_addr.size()); return true; } @@ -410,7 +401,8 @@ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const if (!IsIPv6()) { return false; } - memcpy(pipv6Addr, ip, 16); + assert(sizeof(*pipv6Addr) == m_addr.size()); + memcpy(pipv6Addr, m_addr.data(), m_addr.size()); return true; } @@ -421,15 +413,17 @@ bool CNetAddr::HasLinkedIPv4() const uint32_t CNetAddr::GetLinkedIPv4() const { - if (IsIPv4() || IsRFC6145() || IsRFC6052()) { - // IPv4, mapped IPv4, SIIT translated IPv4: the IPv4 address is the last 4 bytes of the address - return ReadBE32(ip + 12); + if (IsIPv4()) { + return ReadBE32(m_addr.data()); + } else if (IsRFC6052() || IsRFC6145()) { + // mapped IPv4, SIIT translated IPv4: the IPv4 address is the last 4 bytes of the address + return ReadBE32(MakeSpan(m_addr).last(ADDR_IPV4_SIZE).data()); } else if (IsRFC3964()) { // 6to4 tunneled IPv4: the IPv4 address is in bytes 2-6 - return ReadBE32(ip + 2); + return ReadBE32(MakeSpan(m_addr).subspan(2, ADDR_IPV4_SIZE).data()); } else if (IsRFC4380()) { // Teredo tunneled IPv4: the IPv4 address is in the last 4 bytes of the address, but bitflipped - return ~ReadBE32(ip + 12); + return ~ReadBE32(MakeSpan(m_addr).last(ADDR_IPV4_SIZE).data()); } assert(false); } @@ -458,10 +452,10 @@ uint32_t CNetAddr::GetMappedAS(const std::vector<bool> &asmap) const { } std::vector<bool> ip_bits(128); if (HasLinkedIPv4()) { - // For lookup, treat as if it was just an IPv4 address (pchIPv4 prefix + IPv4 bits) + // For lookup, treat as if it was just an IPv4 address (IPV4_IN_IPV6_PREFIX + IPv4 bits) for (int8_t byte_i = 0; byte_i < 12; ++byte_i) { for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { - ip_bits[byte_i * 8 + bit_i] = (pchIPv4[byte_i] >> (7 - bit_i)) & 1; + ip_bits[byte_i * 8 + bit_i] = (IPV4_IN_IPV6_PREFIX[byte_i] >> (7 - bit_i)) & 1; } } uint32_t ipv4 = GetLinkedIPv4(); @@ -470,8 +464,9 @@ uint32_t CNetAddr::GetMappedAS(const std::vector<bool> &asmap) const { } } else { // Use all 128 bits of the IPv6 address otherwise + assert(IsIPv6()); for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { - uint8_t cur_byte = GetByte(15 - byte_i); + uint8_t cur_byte = m_addr[byte_i]; for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; } @@ -507,19 +502,15 @@ std::vector<unsigned char> CNetAddr::GetGroup(const std::vector<bool> &asmap) co } vchRet.push_back(net_class); - int nStartByte = 0; - int nBits = 16; + int nBits{0}; if (IsLocal()) { // all local addresses belong to the same group - nBits = 0; } else if (IsInternal()) { // all internal-usage addresses get their own group - nStartByte = sizeof(g_internal_prefix); - nBits = (sizeof(ip) - sizeof(g_internal_prefix)) * 8; + nBits = ADDR_INTERNAL_SIZE * 8; } else if (!IsRoutable()) { // all other unroutable addresses belong to the same group - nBits = 0; } else if (HasLinkedIPv4()) { // IPv4 addresses (and mapped IPv4 addresses) use /16 groups uint32_t ipv4 = GetLinkedIPv4(); @@ -527,7 +518,6 @@ std::vector<unsigned char> CNetAddr::GetGroup(const std::vector<bool> &asmap) co vchRet.push_back((ipv4 >> 16) & 0xFF); return vchRet; } else if (IsTor()) { - nStartByte = 6; nBits = 4; } else if (IsHeNet()) { // for he.net, use /36 groups @@ -537,23 +527,29 @@ std::vector<unsigned char> CNetAddr::GetGroup(const std::vector<bool> &asmap) co nBits = 32; } - // push our ip onto vchRet byte by byte... - while (nBits >= 8) - { - vchRet.push_back(GetByte(15 - nStartByte)); - nStartByte++; - nBits -= 8; - } + // Push our address onto vchRet. + const size_t num_bytes = nBits / 8; + vchRet.insert(vchRet.end(), m_addr.begin(), m_addr.begin() + num_bytes); + nBits %= 8; // ...for the last byte, push nBits and for the rest of the byte push 1's - if (nBits > 0) - vchRet.push_back(GetByte(15 - nStartByte) | ((1 << (8 - nBits)) - 1)); + if (nBits > 0) { + assert(num_bytes < m_addr.size()); + vchRet.push_back(m_addr[num_bytes] | ((1 << (8 - nBits)) - 1)); + } return vchRet; } +std::vector<unsigned char> CNetAddr::GetAddrBytes() const +{ + uint8_t serialized[V1_SERIALIZATION_SIZE]; + SerializeV1Array(serialized); + return {std::begin(serialized), std::end(serialized)}; +} + uint64_t CNetAddr::GetHash() const { - uint256 hash = Hash(ip); + uint256 hash = Hash(m_addr); uint64_t nRet; memcpy(&nRet, &hash, sizeof(nRet)); return nRet; @@ -764,53 +760,89 @@ CSubNet::CSubNet(): memset(netmask, 0, sizeof(netmask)); } -CSubNet::CSubNet(const CNetAddr &addr, int32_t mask) +CSubNet::CSubNet(const CNetAddr& addr, uint8_t mask) : CSubNet() { - valid = true; + valid = (addr.IsIPv4() && mask <= ADDR_IPV4_SIZE * 8) || + (addr.IsIPv6() && mask <= ADDR_IPV6_SIZE * 8); + if (!valid) { + return; + } + + assert(mask <= sizeof(netmask) * 8); + network = addr; - // Default to /32 (IPv4) or /128 (IPv6), i.e. match single address - memset(netmask, 255, sizeof(netmask)); - - // IPv4 addresses start at offset 12, and first 12 bytes must match, so just offset n - const int astartofs = network.IsIPv4() ? 12 : 0; - - int32_t n = mask; - if(n >= 0 && n <= (128 - astartofs*8)) // Only valid if in range of bits of address - { - n += astartofs*8; - // Clear bits [n..127] - for (; n < 128; ++n) - netmask[n>>3] &= ~(1<<(7-(n&7))); - } else - valid = false; - // Normalize network according to netmask - for(int x=0; x<16; ++x) - network.ip[x] &= netmask[x]; + uint8_t n = mask; + for (size_t i = 0; i < network.m_addr.size(); ++i) { + const uint8_t bits = n < 8 ? n : 8; + netmask[i] = (uint8_t)((uint8_t)0xFF << (8 - bits)); // Set first bits. + network.m_addr[i] &= netmask[i]; // Normalize network according to netmask. + n -= bits; + } +} + +/** + * @returns The number of 1-bits in the prefix of the specified subnet mask. If + * the specified subnet mask is not a valid one, -1. + */ +static inline int NetmaskBits(uint8_t x) +{ + switch(x) { + case 0x00: return 0; + case 0x80: return 1; + case 0xc0: return 2; + case 0xe0: return 3; + case 0xf0: return 4; + case 0xf8: return 5; + case 0xfc: return 6; + case 0xfe: return 7; + case 0xff: return 8; + default: return -1; + } } -CSubNet::CSubNet(const CNetAddr &addr, const CNetAddr &mask) +CSubNet::CSubNet(const CNetAddr& addr, const CNetAddr& mask) : CSubNet() { - valid = true; - network = addr; - // Default to /32 (IPv4) or /128 (IPv6), i.e. match single address - memset(netmask, 255, sizeof(netmask)); + valid = (addr.IsIPv4() || addr.IsIPv6()) && addr.m_net == mask.m_net; + if (!valid) { + return; + } + // Check if `mask` contains 1-bits after 0-bits (which is an invalid netmask). + bool zeros_found = false; + for (auto b : mask.m_addr) { + const int num_bits = NetmaskBits(b); + if (num_bits == -1 || (zeros_found && num_bits != 0)) { + valid = false; + return; + } + if (num_bits < 8) { + zeros_found = true; + } + } + + assert(mask.m_addr.size() <= sizeof(netmask)); - // IPv4 addresses start at offset 12, and first 12 bytes must match, so just offset n - const int astartofs = network.IsIPv4() ? 12 : 0; + memcpy(netmask, mask.m_addr.data(), mask.m_addr.size()); - for(int x=astartofs; x<16; ++x) - netmask[x] = mask.ip[x]; + network = addr; // Normalize network according to netmask - for(int x=0; x<16; ++x) - network.ip[x] &= netmask[x]; + for (size_t x = 0; x < network.m_addr.size(); ++x) { + network.m_addr[x] &= netmask[x]; + } } -CSubNet::CSubNet(const CNetAddr &addr): - valid(addr.IsValid()) +CSubNet::CSubNet(const CNetAddr& addr) : CSubNet() { - memset(netmask, 255, sizeof(netmask)); + valid = addr.IsIPv4() || addr.IsIPv6(); + if (!valid) { + return; + } + + assert(addr.m_addr.size() <= sizeof(netmask)); + + memset(netmask, 0xFF, addr.m_addr.size()); + network = addr; } @@ -822,68 +854,29 @@ bool CSubNet::Match(const CNetAddr &addr) const { if (!valid || !addr.IsValid() || network.m_net != addr.m_net) return false; - for(int x=0; x<16; ++x) - if ((addr.ip[x] & netmask[x]) != network.ip[x]) + assert(network.m_addr.size() == addr.m_addr.size()); + for (size_t x = 0; x < addr.m_addr.size(); ++x) { + if ((addr.m_addr[x] & netmask[x]) != network.m_addr[x]) { return false; - return true; -} - -/** - * @returns The number of 1-bits in the prefix of the specified subnet mask. If - * the specified subnet mask is not a valid one, -1. - */ -static inline int NetmaskBits(uint8_t x) -{ - switch(x) { - case 0x00: return 0; - case 0x80: return 1; - case 0xc0: return 2; - case 0xe0: return 3; - case 0xf0: return 4; - case 0xf8: return 5; - case 0xfc: return 6; - case 0xfe: return 7; - case 0xff: return 8; - default: return -1; + } } + return true; } std::string CSubNet::ToString() const { - /* Parse binary 1{n}0{N-n} to see if mask can be represented as /n */ - int cidr = 0; - bool valid_cidr = true; - int n = network.IsIPv4() ? 12 : 0; - for (; n < 16 && netmask[n] == 0xff; ++n) - cidr += 8; - if (n < 16) { - int bits = NetmaskBits(netmask[n]); - if (bits < 0) - valid_cidr = false; - else - cidr += bits; - ++n; - } - for (; n < 16 && valid_cidr; ++n) - if (netmask[n] != 0x00) - valid_cidr = false; - - /* Format output */ - std::string strNetmask; - if (valid_cidr) { - strNetmask = strprintf("%u", cidr); - } else { - if (network.IsIPv4()) - strNetmask = strprintf("%u.%u.%u.%u", netmask[12], netmask[13], netmask[14], netmask[15]); - else - strNetmask = strprintf("%x:%x:%x:%x:%x:%x:%x:%x", - netmask[0] << 8 | netmask[1], netmask[2] << 8 | netmask[3], - netmask[4] << 8 | netmask[5], netmask[6] << 8 | netmask[7], - netmask[8] << 8 | netmask[9], netmask[10] << 8 | netmask[11], - netmask[12] << 8 | netmask[13], netmask[14] << 8 | netmask[15]); + assert(network.m_addr.size() <= sizeof(netmask)); + + uint8_t cidr = 0; + + for (size_t i = 0; i < network.m_addr.size(); ++i) { + if (netmask[i] == 0x00) { + break; + } + cidr += NetmaskBits(netmask[i]); } - return network.ToString() + "/" + strNetmask; + return network.ToString() + strprintf("/%u", cidr); } bool CSubNet::IsValid() const diff --git a/src/netaddress.h b/src/netaddress.h index 0365907d4..d00f5a6f5 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -9,9 +9,12 @@ #include <config/bitcoin-config.h> #endif +#include <attributes.h> #include <compat.h> +#include <prevector.h> #include <serialize.h> +#include <array> #include <cstdint> #include <string> #include <vector> @@ -39,16 +42,49 @@ enum Network /// TORv2 NET_ONION, - /// A set of dummy addresses that map a name to an IPv6 address. These - /// addresses belong to RFC4193's fc00::/7 subnet (unique-local addresses). - /// We use them to map a string or FQDN to an IPv6 address in CAddrMan to - /// keep track of which DNS seeds were used. + /// A set of addresses that represent the hash of a string or FQDN. We use + /// them in CAddrMan to keep track of which DNS seeds were used. NET_INTERNAL, /// Dummy value to indicate the number of NET_* constants. NET_MAX, }; +/// Prefix of an IPv6 address when it contains an embedded IPv4 address. +/// Used when (un)serializing addresses in ADDRv1 format (pre-BIP155). +static const std::array<uint8_t, 12> IPV4_IN_IPV6_PREFIX{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF +}; + +/// Prefix of an IPv6 address when it contains an embedded TORv2 address. +/// Used when (un)serializing addresses in ADDRv1 format (pre-BIP155). +/// Such dummy IPv6 addresses are guaranteed to not be publicly routable as they +/// fall under RFC4193's fc00::/7 subnet allocated to unique-local addresses. +static const std::array<uint8_t, 6> TORV2_IN_IPV6_PREFIX{ + 0xFD, 0x87, 0xD8, 0x7E, 0xEB, 0x43 +}; + +/// Prefix of an IPv6 address when it contains an embedded "internal" address. +/// Used when (un)serializing addresses in ADDRv1 format (pre-BIP155). +/// The prefix comes from 0xFD + SHA256("bitcoin")[0:5]. +/// Such dummy IPv6 addresses are guaranteed to not be publicly routable as they +/// fall under RFC4193's fc00::/7 subnet allocated to unique-local addresses. +static const std::array<uint8_t, 6> INTERNAL_IN_IPV6_PREFIX{ + 0xFD, 0x6B, 0x88, 0xC0, 0x87, 0x24 // 0xFD + sha256("bitcoin")[0:5]. +}; + +/// Size of IPv4 address (in bytes). +static constexpr size_t ADDR_IPV4_SIZE = 4; + +/// Size of IPv6 address (in bytes). +static constexpr size_t ADDR_IPV6_SIZE = 16; + +/// Size of TORv2 address (in bytes). +static constexpr size_t ADDR_TORV2_SIZE = 10; + +/// Size of "internal" (NET_INTERNAL) address (in bytes). +static constexpr size_t ADDR_INTERNAL_SIZE = 10; + /** * Network address. */ @@ -56,11 +92,16 @@ class CNetAddr { protected: /** + * Raw representation of the network address. + * In network byte order (big endian) for IPv4 and IPv6. + */ + prevector<ADDR_IPV6_SIZE, uint8_t> m_addr{ADDR_IPV6_SIZE, 0x0}; + + /** * Network to which this address belongs. */ Network m_net{NET_IPV6}; - unsigned char ip[16]; // in network byte order uint32_t scopeId{0}; // for scoped/link-local ipv6 addresses public: @@ -74,13 +115,7 @@ class CNetAddr * (e.g. IPv4) disguised as IPv6. This encoding is used in the legacy * `addr` encoding. */ - void SetLegacyIPv6(const uint8_t ipv6[16]); - - /** - * Set raw IPv4 or IPv6 address (in network byte order) - * @note Only NET_IPV4 and NET_IPV6 are allowed for network. - */ - void SetRaw(Network network, const uint8_t *data); + void SetLegacyIPv6(Span<const uint8_t> ipv6); bool SetInternal(const std::string& name); @@ -111,7 +146,6 @@ class CNetAddr enum Network GetNetwork() const; std::string ToString() const; std::string ToStringIP() const; - unsigned int GetByte(int n) const; uint64_t GetHash() const; bool GetInAddr(struct in_addr* pipv4Addr) const; uint32_t GetNetClass() const; @@ -127,7 +161,7 @@ class CNetAddr uint32_t GetMappedAS(const std::vector<bool> &asmap) const; std::vector<unsigned char> GetGroup(const std::vector<bool> &asmap) const; - std::vector<unsigned char> GetAddrBytes() const { return {std::begin(ip), std::end(ip)}; } + std::vector<unsigned char> GetAddrBytes() const; int GetReachabilityFrom(const CNetAddr *paddrPartner = nullptr) const; explicit CNetAddr(const struct in6_addr& pipv6Addr, const uint32_t scope = 0); @@ -143,7 +177,7 @@ class CNetAddr template <typename Stream> void Serialize(Stream& s) const { - s << ip; + SerializeV1Stream(s); } /** @@ -152,14 +186,92 @@ class CNetAddr template <typename Stream> void Unserialize(Stream& s) { - unsigned char ip_temp[sizeof(ip)]; - s >> ip_temp; + UnserializeV1Stream(s); + } + + friend class CSubNet; + + private: + /** + * Size of CNetAddr when serialized as ADDRv1 (pre-BIP155) (in bytes). + */ + static constexpr size_t V1_SERIALIZATION_SIZE = ADDR_IPV6_SIZE; + + /** + * Serialize in pre-ADDRv2/BIP155 format to an array. + * Some addresses (e.g. TORv3) cannot be serialized in pre-BIP155 format. + */ + void SerializeV1Array(uint8_t (&arr)[V1_SERIALIZATION_SIZE]) const + { + size_t prefix_size; + + switch (m_net) { + case NET_IPV6: + assert(m_addr.size() == sizeof(arr)); + memcpy(arr, m_addr.data(), m_addr.size()); + return; + case NET_IPV4: + prefix_size = sizeof(IPV4_IN_IPV6_PREFIX); + assert(prefix_size + m_addr.size() == sizeof(arr)); + memcpy(arr, IPV4_IN_IPV6_PREFIX.data(), prefix_size); + memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); + return; + case NET_ONION: + prefix_size = sizeof(TORV2_IN_IPV6_PREFIX); + assert(prefix_size + m_addr.size() == sizeof(arr)); + memcpy(arr, TORV2_IN_IPV6_PREFIX.data(), prefix_size); + memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); + return; + case NET_INTERNAL: + prefix_size = sizeof(INTERNAL_IN_IPV6_PREFIX); + assert(prefix_size + m_addr.size() == sizeof(arr)); + memcpy(arr, INTERNAL_IN_IPV6_PREFIX.data(), prefix_size); + memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); + return; + case NET_UNROUTABLE: + case NET_MAX: + assert(false); + } // no default case, so the compiler can warn about missing cases + + assert(false); + } + + /** + * Serialize in pre-ADDRv2/BIP155 format to a stream. + * Some addresses (e.g. TORv3) cannot be serialized in pre-BIP155 format. + */ + template <typename Stream> + void SerializeV1Stream(Stream& s) const + { + uint8_t serialized[V1_SERIALIZATION_SIZE]; + + SerializeV1Array(serialized); + + s << serialized; + } + + /** + * Unserialize from a pre-ADDRv2/BIP155 format from an array. + */ + void UnserializeV1Array(uint8_t (&arr)[V1_SERIALIZATION_SIZE]) + { // Use SetLegacyIPv6() so that m_net is set correctly. For example // ::FFFF:0102:0304 should be set as m_net=NET_IPV4 (1.2.3.4). - SetLegacyIPv6(ip_temp); + SetLegacyIPv6(arr); } - friend class CSubNet; + /** + * Unserialize from a pre-ADDRv2/BIP155 format from a stream. + */ + template <typename Stream> + void UnserializeV1Stream(Stream& s) + { + uint8_t serialized[V1_SERIALIZATION_SIZE]; + + s >> serialized; + + UnserializeV1Array(serialized); + } }; class CSubNet @@ -174,11 +286,11 @@ class CSubNet public: CSubNet(); - CSubNet(const CNetAddr &addr, int32_t mask); - CSubNet(const CNetAddr &addr, const CNetAddr &mask); + CSubNet(const CNetAddr& addr, uint8_t mask); + CSubNet(const CNetAddr& addr, const CNetAddr& mask); //constructor for single ip subnet (<ipv4>/32 or <ipv6>/128) - explicit CSubNet(const CNetAddr &addr); + explicit CSubNet(const CNetAddr& addr); bool Match(const CNetAddr &addr) const; diff --git a/src/netbase.cpp b/src/netbase.cpp index 3a3b5f3e6..027383901 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -13,6 +13,7 @@ #include <atomic> #include <cstdint> +#include <limits> #ifndef WIN32 #include <fcntl.h> @@ -838,8 +839,8 @@ bool LookupSubNet(const std::string& strSubnet, CSubNet& ret) if (slash != strSubnet.npos) { std::string strNetmask = strSubnet.substr(slash + 1); - int32_t n; - if (ParseInt32(strNetmask, &n)) { + uint8_t n; + if (ParseUInt8(strNetmask, &n)) { // If valid number, assume CIDR variable-length subnet masking ret = CSubNet(network, n); return ret.IsValid(); diff --git a/src/node/context.cpp b/src/node/context.cpp index 0238aab0d..49d0c3723 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -9,6 +9,7 @@ #include <net.h> #include <net_processing.h> #include <scheduler.h> +#include <txmempool.h> NodeContext::NodeContext() {} NodeContext::~NodeContext() {} diff --git a/src/node/context.h b/src/node/context.h index be568cba3..3228831ed 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -16,10 +16,11 @@ class CConnman; class CScheduler; class CTxMemPool; class ChainstateManager; -class PeerLogicValidation; +class PeerManager; namespace interfaces { class Chain; class ChainClient; +class WalletClient; } // namespace interfaces //! NodeContext struct containing references to chain state and connection @@ -34,13 +35,17 @@ class ChainClient; //! be used without pulling in unwanted dependencies or functionality. struct NodeContext { std::unique_ptr<CConnman> connman; - CTxMemPool* mempool{nullptr}; // Currently a raw pointer because the memory is not managed by this struct - std::unique_ptr<PeerLogicValidation> peer_logic; + std::unique_ptr<CTxMemPool> mempool; + std::unique_ptr<PeerManager> peerman; ChainstateManager* chainman{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<BanMan> banman; ArgsManager* args{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<interfaces::Chain> chain; + //! List of all chain clients (wallet processes or other client) connected to node. std::vector<std::unique_ptr<interfaces::ChainClient>> chain_clients; + //! Reference to chain client that should used to load or create wallets + //! opened by the gui. + interfaces::WalletClient* wallet_client{nullptr}; std::unique_ptr<CScheduler> scheduler; std::function<void()> rpc_interruption_point = [] {}; diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index 5633abe81..9ae470074 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -38,8 +38,8 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t if (!node.mempool->exists(hashTx)) { // Transaction is not already in the mempool. Submit it. TxValidationState state; - if (!AcceptToMemoryPool(*node.mempool, state, std::move(tx), - nullptr /* plTxnReplaced */, false /* bypass_limits */, max_tx_fee)) { + if (!AcceptToMemoryPool(*node.mempool, state, tx, + nullptr /* plTxnReplaced */, false /* bypass_limits */, max_tx_fee)) { err_string = state.ToString(); if (state.IsInvalid()) { if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { @@ -80,7 +80,7 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t if (relay) { // the mempool tracks locally submitted transactions to make a // best-effort of initial broadcast - node.mempool->AddUnbroadcastTx(hashTx, tx->GetWitnessHash()); + node.mempool->AddUnbroadcastTx(hashTx); LOCK(cs_main); RelayTransaction(hashTx, tx->GetWitnessHash(), *node.connman); diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index c56abaf6c..0e9820da1 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -152,6 +152,8 @@ bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, const CFeeR * script can be anything; an attacker could use a very * expensive-to-check-upon-redemption script like: * DUP CHECKSIG DROP ... repeated 100 times... OP_1 + * + * Note that only the non-witness portion of the transaction is checked here. */ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) { @@ -164,7 +166,11 @@ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) std::vector<std::vector<unsigned char> > vSolutions; TxoutType whichType = Solver(prev.scriptPubKey, vSolutions); - if (whichType == TxoutType::NONSTANDARD) { + if (whichType == TxoutType::NONSTANDARD || whichType == TxoutType::WITNESS_UNKNOWN) { + // WITNESS_UNKNOWN failures are typically also caught with a policy + // flag in the script interpreter, but it can be helpful to catch + // this type of NONSTANDARD transaction earlier in transaction + // validation. return false; } else if (whichType == TxoutType::SCRIPTHASH) { std::vector<std::vector<unsigned char> > stack; diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index f8b17d18d..4b5593489 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -36,3 +36,9 @@ RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) } return RBFTransactionState::FINAL; } + +RBFTransactionState IsRBFOptInEmptyMempool(const CTransaction& tx) +{ + // If we don't have a local mempool we can only check the transaction itself. + return SignalsOptInRBF(tx) ? RBFTransactionState::REPLACEABLE_BIP125 : RBFTransactionState::UNKNOWN; +} diff --git a/src/policy/rbf.h b/src/policy/rbf.h index d335fbbb3..f84e6e528 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -7,16 +7,28 @@ #include <txmempool.h> +/** The rbf state of unconfirmed transactions */ enum class RBFTransactionState { + /** Unconfirmed tx that does not signal rbf and is not in the mempool */ UNKNOWN, + /** Either this tx or a mempool ancestor signals rbf */ REPLACEABLE_BIP125, - FINAL + /** Neither this tx nor a mempool ancestor signals rbf */ + FINAL, }; -// Determine whether an in-mempool transaction is signaling opt-in to RBF -// according to BIP 125 -// This involves checking sequence numbers of the transaction, as well -// as the sequence numbers of all in-mempool ancestors. +/** + * Determine whether an unconfirmed transaction is signaling opt-in to RBF + * according to BIP 125 + * This involves checking sequence numbers of the transaction, as well + * as the sequence numbers of all in-mempool ancestors. + * + * @param tx The unconfirmed transaction + * @param pool The mempool, which may contain the tx + * + * @return The rbf state + */ RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(pool.cs); +RBFTransactionState IsRBFOptInEmptyMempool(const CTransaction& tx); #endif // BITCOIN_POLICY_RBF_H diff --git a/src/protocol.cpp b/src/protocol.cpp index 5a91acee0..48ca0c6df 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -8,10 +8,6 @@ #include <util/strencodings.h> #include <util/system.h> -#ifndef WIN32 -# include <arpa/inet.h> -#endif - static std::atomic<bool> g_initial_block_download_completed(false); namespace NetMsgType { @@ -163,7 +159,7 @@ CInv::CInv() hash.SetNull(); } -CInv::CInv(int typeIn, const uint256& hashIn) : type(typeIn), hash(hashIn) {} +CInv::CInv(uint32_t typeIn, const uint256& hashIn) : type(typeIn), hash(hashIn) {} bool operator<(const CInv& a, const CInv& b) { @@ -217,6 +213,7 @@ static std::string serviceFlagToStr(size_t bit) case NODE_GETUTXO: return "GETUTXO"; case NODE_BLOOM: return "BLOOM"; case NODE_WITNESS: return "WITNESS"; + case NODE_COMPACT_FILTERS: return "COMPACT_FILTERS"; case NODE_NETWORK_LIMITED: return "NETWORK_LIMITED"; // Not using default, so we get warned when a case is missing } diff --git a/src/protocol.h b/src/protocol.h index 1d0adaae6..7fb84cddf 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -247,7 +247,7 @@ extern const char* CFCHECKPT; * txid. * @since protocol version 70016 as described by BIP 339. */ -extern const char *WTXIDRELAY; +extern const char* WTXIDRELAY; }; // namespace NetMsgType /* Get a vector of all valid message types (see above) */ @@ -272,6 +272,9 @@ enum ServiceFlags : uint64_t { // NODE_WITNESS indicates that a node can be asked for blocks and transactions including // witness data. NODE_WITNESS = (1 << 3), + // NODE_COMPACT_FILTERS means the node will service basic block filter requests. + // See BIP157 and BIP158 for details on how this is implemented. + NODE_COMPACT_FILTERS = (1 << 6), // NODE_NETWORK_LIMITED means the same as NODE_NETWORK with the limitation of only // serving the last 288 (2 day) blocks // See BIP159 for details on how this is implemented. @@ -371,9 +374,10 @@ public: READWRITEAS(CService, obj); } - ServiceFlags nServices{NODE_NONE}; // disk and network only uint32_t nTime{TIME_INIT}; + + ServiceFlags nServices{NODE_NONE}; }; /** getdata message type flags */ @@ -394,7 +398,9 @@ enum GetDataMsg : uint32_t { MSG_CMPCT_BLOCK = 4, //!< Defined in BIP152 MSG_WITNESS_BLOCK = MSG_BLOCK | MSG_WITNESS_FLAG, //!< Defined in BIP144 MSG_WITNESS_TX = MSG_TX | MSG_WITNESS_FLAG, //!< Defined in BIP144 - MSG_FILTERED_WITNESS_BLOCK = MSG_FILTERED_BLOCK | MSG_WITNESS_FLAG, + // MSG_FILTERED_WITNESS_BLOCK is defined in BIP144 as reserved for future + // use and remains unused. + // MSG_FILTERED_WITNESS_BLOCK = MSG_FILTERED_BLOCK | MSG_WITNESS_FLAG, }; /** inv message data */ @@ -402,7 +408,7 @@ class CInv { public: CInv(); - CInv(int typeIn, const uint256& hashIn); + CInv(uint32_t typeIn, const uint256& hashIn); SERIALIZE_METHODS(CInv, obj) { READWRITE(obj.type, obj.hash); } @@ -412,14 +418,24 @@ public: std::string ToString() const; // Single-message helper methods - bool IsMsgTx() const { return type == MSG_TX; } - bool IsMsgWtx() const { return type == MSG_WTX; } - bool IsMsgWitnessTx() const { return type == MSG_WITNESS_TX; } + bool IsMsgTx() const { return type == MSG_TX; } + bool IsMsgBlk() const { return type == MSG_BLOCK; } + bool IsMsgWtx() const { return type == MSG_WTX; } + bool IsMsgFilteredBlk() const { return type == MSG_FILTERED_BLOCK; } + bool IsMsgCmpctBlk() const { return type == MSG_CMPCT_BLOCK; } + bool IsMsgWitnessBlk() const { return type == MSG_WITNESS_BLOCK; } // Combined-message helper methods - bool IsGenTxMsg() const { return type == MSG_TX || type == MSG_WTX || type == MSG_WITNESS_TX; } + bool IsGenTxMsg() const + { + return type == MSG_TX || type == MSG_WTX || type == MSG_WITNESS_TX; + } + bool IsGenBlkMsg() const + { + return type == MSG_BLOCK || type == MSG_FILTERED_BLOCK || type == MSG_CMPCT_BLOCK || type == MSG_WITNESS_BLOCK; + } - int type; + uint32_t type; uint256 hash; }; diff --git a/src/pubkey.cpp b/src/pubkey.cpp index ef42aa5bc..fc14f41a0 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -24,7 +24,7 @@ secp256k1_context* secp256k1_context_verify = nullptr; * strict DER before being passed to this module, and we know it supports all * violations present in the blockchain before that point. */ -static int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1_ecdsa_signature* sig, const unsigned char *input, size_t inputlen) { +int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1_ecdsa_signature* sig, const unsigned char *input, size_t inputlen) { size_t rpos, rlen, spos, slen; size_t pos = 0; size_t lenbyte; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 523f5c429..7a3fb420c 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -27,8 +27,11 @@ #include <qt/walletmodel.h> #endif // ENABLE_WALLET +#include <init.h> #include <interfaces/handler.h> #include <interfaces/node.h> +#include <node/context.h> +#include <node/ui_interface.h> #include <noui.h> #include <uint256.h> #include <util/system.h> @@ -36,6 +39,7 @@ #include <util/translation.h> #include <validation.h> +#include <boost/signals2/connection.hpp> #include <memory> #include <QApplication> @@ -80,6 +84,7 @@ static void RegisterMetaTypes() qRegisterMetaType<std::function<void()>>("std::function<void()>"); qRegisterMetaType<QMessageBox::Icon>("QMessageBox::Icon"); + qRegisterMetaType<interfaces::BlockAndHeaderTipInfo>("interfaces::BlockAndHeaderTipInfo"); } static QString GetLangTerritory() @@ -161,10 +166,11 @@ void BitcoinCore::initialize() { try { - qDebug() << __func__ << ": Running initialization in thread"; util::ThreadRename("qt-init"); - bool rv = m_node.appInitMain(); - Q_EMIT initializeResult(rv); + qDebug() << __func__ << ": Running initialization in thread"; + interfaces::BlockAndHeaderTipInfo tip_info; + bool rv = m_node.appInitMain(&tip_info); + Q_EMIT initializeResult(rv, tip_info); } catch (const std::exception& e) { handleRunawayException(&e); } catch (...) { @@ -190,10 +196,9 @@ void BitcoinCore::shutdown() static int qt_argc = 1; static const char* qt_argv = "bitcoin-qt"; -BitcoinApplication::BitcoinApplication(interfaces::Node& node): +BitcoinApplication::BitcoinApplication(): QApplication(qt_argc, const_cast<char **>(&qt_argv)), coreThread(nullptr), - m_node(node), optionsModel(nullptr), clientModel(nullptr), window(nullptr), @@ -201,6 +206,7 @@ BitcoinApplication::BitcoinApplication(interfaces::Node& node): returnValue(0), platformStyle(nullptr) { + // Qt runs setlocale(LC_ALL, "") on initialization. RegisterMetaTypes(); setQuitOnLastWindowClosed(false); } @@ -243,12 +249,12 @@ void BitcoinApplication::createPaymentServer() void BitcoinApplication::createOptionsModel(bool resetSettings) { - optionsModel = new OptionsModel(m_node, this, resetSettings); + optionsModel = new OptionsModel(this, resetSettings); } void BitcoinApplication::createWindow(const NetworkStyle *networkStyle) { - window = new BitcoinGUI(m_node, platformStyle, networkStyle, nullptr); + window = new BitcoinGUI(node(), platformStyle, networkStyle, nullptr); pollShutdownTimer = new QTimer(window); connect(pollShutdownTimer, &QTimer::timeout, window, &BitcoinGUI::detectShutdown); @@ -256,17 +262,27 @@ void BitcoinApplication::createWindow(const NetworkStyle *networkStyle) void BitcoinApplication::createSplashScreen(const NetworkStyle *networkStyle) { - SplashScreen *splash = new SplashScreen(m_node, nullptr, networkStyle); + assert(!m_splash); + m_splash = new SplashScreen(nullptr, networkStyle); // We don't hold a direct pointer to the splash screen after creation, but the splash // screen will take care of deleting itself when finish() happens. - splash->show(); - connect(this, &BitcoinApplication::splashFinished, splash, &SplashScreen::finish); - connect(this, &BitcoinApplication::requestedShutdown, splash, &QWidget::close); + m_splash->show(); + connect(this, &BitcoinApplication::requestedInitialize, m_splash, &SplashScreen::handleLoadWallet); + connect(this, &BitcoinApplication::splashFinished, m_splash, &SplashScreen::finish); + connect(this, &BitcoinApplication::requestedShutdown, m_splash, &QWidget::close); +} + +void BitcoinApplication::setNode(interfaces::Node& node) +{ + assert(!m_node); + m_node = &node; + if (optionsModel) optionsModel->setNode(*m_node); + if (m_splash) m_splash->setNode(*m_node); } bool BitcoinApplication::baseInitialize() { - return m_node.baseInitialize(); + return node().baseInitialize(); } void BitcoinApplication::startThread() @@ -274,7 +290,7 @@ void BitcoinApplication::startThread() if(coreThread) return; coreThread = new QThread(this); - BitcoinCore *executor = new BitcoinCore(m_node); + BitcoinCore *executor = new BitcoinCore(node()); executor->moveToThread(coreThread); /* communication to and from thread */ @@ -295,8 +311,8 @@ void BitcoinApplication::parameterSetup() // print to the console unnecessarily. gArgs.SoftSetBoolArg("-printtoconsole", false); - m_node.initLogging(); - m_node.initParameterInteraction(); + InitLogging(gArgs); + InitParameterInteraction(gArgs); } void BitcoinApplication::InitializePruneSetting(bool prune) @@ -328,7 +344,7 @@ void BitcoinApplication::requestShutdown() window->unsubscribeFromCoreSignals(); // Request node shutdown, which can interrupt long operations, like // rescanning a wallet. - m_node.startShutdown(); + node().startShutdown(); // Unsetting the client model can cause the current thread to wait for node // to complete an operation, like wait for a RPC execution to complete. window->setClientModel(nullptr); @@ -341,7 +357,7 @@ void BitcoinApplication::requestShutdown() Q_EMIT requestedShutdown(); } -void BitcoinApplication::initializeResult(bool success) +void BitcoinApplication::initializeResult(bool success, interfaces::BlockAndHeaderTipInfo tip_info) { qDebug() << __func__ << ": Initialization result: " << success; // Set exit result. @@ -350,8 +366,8 @@ void BitcoinApplication::initializeResult(bool success) { // Log this only after AppInitMain finishes, as then logging setup is guaranteed complete qInfo() << "Platform customization:" << platformStyle->getName(); - clientModel = new ClientModel(m_node, optionsModel); - window->setClientModel(clientModel); + clientModel = new ClientModel(node(), optionsModel); + window->setClientModel(clientModel, &tip_info); #ifdef ENABLE_WALLET if (WalletModel::isWalletEnabled()) { m_wallet_controller = new WalletController(*clientModel, platformStyle, this); @@ -430,12 +446,13 @@ int GuiMain(int argc, char* argv[]) SetupEnvironment(); util::ThreadSetInternalName("main"); - std::unique_ptr<interfaces::Node> node = interfaces::MakeNode(); + NodeContext node_context; + std::unique_ptr<interfaces::Node> node = interfaces::MakeNode(&node_context); // Subscribe to global signals from core - std::unique_ptr<interfaces::Handler> handler_message_box = node->handleMessageBox(noui_ThreadSafeMessageBox); - std::unique_ptr<interfaces::Handler> handler_question = node->handleQuestion(noui_ThreadSafeQuestion); - std::unique_ptr<interfaces::Handler> handler_init_message = node->handleInitMessage(noui_InitMessage); + boost::signals2::scoped_connection handler_message_box = ::uiInterface.ThreadSafeMessageBox_connect(noui_ThreadSafeMessageBox); + boost::signals2::scoped_connection handler_question = ::uiInterface.ThreadSafeQuestion_connect(noui_ThreadSafeQuestion); + boost::signals2::scoped_connection handler_init_message = ::uiInterface.InitMessage_connect(noui_InitMessage); // Do not refer to data directory yet, this can be overridden by Intro::pickDataDirectory @@ -449,15 +466,15 @@ int GuiMain(int argc, char* argv[]) QCoreApplication::setAttribute(Qt::AA_EnableHighDpiScaling); #endif - BitcoinApplication app(*node); + BitcoinApplication app; /// 2. Parse command-line options. We do this after qt in order to show an error if there are problems parsing these // Command-line options take precedence: - node->setupServerArgs(); + SetupServerArgs(node_context); SetupUIArgs(gArgs); std::string error; - if (!node->parseParameters(argc, argv, error)) { - node->initError(strprintf(Untranslated("Error parsing command line arguments: %s\n"), error)); + if (!gArgs.ParseParameters(argc, argv, error)) { + InitError(strprintf(Untranslated("Error parsing command line arguments: %s\n"), error)); // Create a message box, because the gui has neither been created nor has subscribed to core signals QMessageBox::critical(nullptr, PACKAGE_NAME, // message can not be translated because translations have not been initialized @@ -483,7 +500,7 @@ int GuiMain(int argc, char* argv[]) // Show help message immediately after parsing command-line options (for "-lang") and setting locale, // but before showing splash screen. if (HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { - HelpMessageDialog help(*node, nullptr, gArgs.IsArgSet("-version")); + HelpMessageDialog help(nullptr, gArgs.IsArgSet("-version")); help.showOrPrint(); return EXIT_SUCCESS; } @@ -493,18 +510,18 @@ int GuiMain(int argc, char* argv[]) bool did_show_intro = false; bool prune = false; // Intro dialog prune check box // Gracefully exit if the user cancels - if (!Intro::showIfNeeded(*node, did_show_intro, prune)) return EXIT_SUCCESS; + if (!Intro::showIfNeeded(did_show_intro, prune)) return EXIT_SUCCESS; /// 6. Determine availability of data directory and parse bitcoin.conf /// - Do not call GetDataDir(true) before this step finishes if (!CheckDataDirOption()) { - node->initError(strprintf(Untranslated("Specified data directory \"%s\" does not exist.\n"), gArgs.GetArg("-datadir", ""))); + InitError(strprintf(Untranslated("Specified data directory \"%s\" does not exist.\n"), gArgs.GetArg("-datadir", ""))); QMessageBox::critical(nullptr, PACKAGE_NAME, QObject::tr("Error: Specified data directory \"%1\" does not exist.").arg(QString::fromStdString(gArgs.GetArg("-datadir", "")))); return EXIT_FAILURE; } - if (!node->readConfigFiles(error)) { - node->initError(strprintf(Untranslated("Error reading configuration file: %s\n"), error)); + if (!gArgs.ReadConfigFiles(error, true)) { + InitError(strprintf(Untranslated("Error reading configuration file: %s\n"), error)); QMessageBox::critical(nullptr, PACKAGE_NAME, QObject::tr("Error: Cannot parse configuration file: %1.").arg(QString::fromStdString(error))); return EXIT_FAILURE; @@ -518,18 +535,18 @@ int GuiMain(int argc, char* argv[]) // Check for -chain, -testnet or -regtest parameter (Params() calls are only valid after this clause) try { - node->selectParams(gArgs.GetChainName()); + SelectParams(gArgs.GetChainName()); } catch(std::exception &e) { - node->initError(Untranslated(strprintf("%s\n", e.what()))); + InitError(Untranslated(strprintf("%s\n", e.what()))); QMessageBox::critical(nullptr, PACKAGE_NAME, QObject::tr("Error: %1").arg(e.what())); return EXIT_FAILURE; } #ifdef ENABLE_WALLET // Parse URIs on command line -- this can affect Params() - PaymentServer::ipcParseCommandLine(*node, argc, argv); + PaymentServer::ipcParseCommandLine(argc, argv); #endif - if (!node->initSettings(error)) { - node->initError(Untranslated(error)); + if (!gArgs.InitSettings(error)) { + InitError(Untranslated(error)); QMessageBox::critical(nullptr, PACKAGE_NAME, QObject::tr("Error initializing settings: %1").arg(QString::fromStdString(error))); return EXIT_FAILURE; } @@ -583,6 +600,8 @@ int GuiMain(int argc, char* argv[]) if (gArgs.GetBoolArg("-splash", DEFAULT_SPLASHSCREEN) && !gArgs.GetBoolArg("-min", false)) app.createSplashScreen(networkStyle.data()); + app.setNode(*node); + int rv = EXIT_SUCCESS; try { @@ -605,10 +624,10 @@ int GuiMain(int argc, char* argv[]) } } catch (const std::exception& e) { PrintExceptionContinue(&e, "Runaway exception"); - app.handleRunawayException(QString::fromStdString(node->getWarnings().translated)); + app.handleRunawayException(QString::fromStdString(app.node().getWarnings().translated)); } catch (...) { PrintExceptionContinue(nullptr, "Runaway exception"); - app.handleRunawayException(QString::fromStdString(node->getWarnings().translated)); + app.handleRunawayException(QString::fromStdString(app.node().getWarnings().translated)); } return rv; } diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index 077a37fde..69e0a5921 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -10,21 +10,21 @@ #endif #include <QApplication> +#include <assert.h> #include <memory> +#include <interfaces/node.h> + class BitcoinGUI; class ClientModel; class NetworkStyle; class OptionsModel; class PaymentServer; class PlatformStyle; +class SplashScreen; class WalletController; class WalletModel; -namespace interfaces { -class Handler; -class Node; -} // namespace interfaces /** Class encapsulating Bitcoin Core startup and shutdown. * Allows running startup and shutdown in a different thread from the UI thread. @@ -40,7 +40,7 @@ public Q_SLOTS: void shutdown(); Q_SIGNALS: - void initializeResult(bool success); + void initializeResult(bool success, interfaces::BlockAndHeaderTipInfo tip_info); void shutdownResult(); void runawayException(const QString &message); @@ -56,7 +56,7 @@ class BitcoinApplication: public QApplication { Q_OBJECT public: - explicit BitcoinApplication(interfaces::Node& node); + explicit BitcoinApplication(); ~BitcoinApplication(); #ifdef ENABLE_WALLET @@ -90,8 +90,11 @@ public: /// Setup platform style void setupPlatformStyle(); + interfaces::Node& node() const { assert(m_node); return *m_node; } + void setNode(interfaces::Node& node); + public Q_SLOTS: - void initializeResult(bool success); + void initializeResult(bool success, interfaces::BlockAndHeaderTipInfo tip_info); void shutdownResult(); /// Handle runaway exceptions. Shows a message box with the problem and quits the program. void handleRunawayException(const QString &message); @@ -104,7 +107,6 @@ Q_SIGNALS: private: QThread *coreThread; - interfaces::Node& m_node; OptionsModel *optionsModel; ClientModel *clientModel; BitcoinGUI *window; @@ -116,6 +118,8 @@ private: int returnValue; const PlatformStyle *platformStyle; std::unique_ptr<QWidget> shutdownWindow; + SplashScreen* m_splash = nullptr; + interfaces::Node* m_node = nullptr; void startThread(); }; diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index ebcc04a5e..0c2dcc358 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -95,7 +95,7 @@ BitcoinGUI::BitcoinGUI(interfaces::Node& node, const PlatformStyle *_platformSty updateWindowTitle(); rpcConsole = new RPCConsole(node, _platformStyle, nullptr); - helpMessageDialog = new HelpMessageDialog(node, this, false); + helpMessageDialog = new HelpMessageDialog(this, false); #ifdef ENABLE_WALLET if(enableWallet) { @@ -574,7 +574,7 @@ void BitcoinGUI::createToolBars() } } -void BitcoinGUI::setClientModel(ClientModel *_clientModel) +void BitcoinGUI::setClientModel(ClientModel *_clientModel, interfaces::BlockAndHeaderTipInfo* tip_info) { this->clientModel = _clientModel; if(_clientModel) @@ -588,8 +588,8 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) connect(_clientModel, &ClientModel::numConnectionsChanged, this, &BitcoinGUI::setNumConnections); connect(_clientModel, &ClientModel::networkActiveChanged, this, &BitcoinGUI::setNetworkActive); - modalOverlay->setKnownBestHeight(_clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(_clientModel->getHeaderTipTime())); - setNumBlocks(m_node.getNumBlocks(), QDateTime::fromTime_t(m_node.getLastBlockTime()), m_node.getVerificationProgress(), false, SynchronizationState::INIT_DOWNLOAD); + modalOverlay->setKnownBestHeight(tip_info->header_height, QDateTime::fromTime_t(tip_info->header_time)); + setNumBlocks(tip_info->block_height, QDateTime::fromTime_t(tip_info->block_time), tip_info->verification_progress, false, SynchronizationState::INIT_DOWNLOAD); connect(_clientModel, &ClientModel::numBlocksChanged, this, &BitcoinGUI::setNumBlocks); // Receive and report messages from client model @@ -600,7 +600,7 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) // Show progress dialog connect(_clientModel, &ClientModel::showProgress, this, &BitcoinGUI::showProgress); - rpcConsole->setClientModel(_clientModel); + rpcConsole->setClientModel(_clientModel, tip_info->block_height, tip_info->block_time, tip_info->verification_progress); updateProxyIcon(); @@ -660,6 +660,11 @@ void BitcoinGUI::setWalletController(WalletController* wallet_controller) } } +WalletController* BitcoinGUI::getWalletController() +{ + return m_wallet_controller; +} + void BitcoinGUI::addWallet(WalletModel* walletModel) { if (!walletFrame) return; @@ -821,7 +826,7 @@ void BitcoinGUI::aboutClicked() if(!clientModel) return; - HelpMessageDialog dlg(m_node, this, true); + HelpMessageDialog dlg(this, true); dlg.exec(); } @@ -1189,7 +1194,7 @@ void BitcoinGUI::incomingTransaction(const QString& date, int unit, const CAmoun // On new transaction, make an info balloon QString msg = tr("Date: %1\n").arg(date) + tr("Amount: %1\n").arg(BitcoinUnits::formatWithUnit(unit, amount, true)); - if (m_node.getWallets().size() > 1 && !walletName.isEmpty()) { + if (m_node.walletClient().getWallets().size() > 1 && !walletName.isEmpty()) { msg += tr("Wallet: %1\n").arg(walletName); } msg += tr("Type: %1\n").arg(type); diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 697e83e77..912297a74 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -43,6 +43,7 @@ enum class SynchronizationState; namespace interfaces { class Handler; class Node; +struct BlockAndHeaderTipInfo; } QT_BEGIN_NAMESPACE @@ -75,9 +76,10 @@ public: /** Set the client model. The client model represents the part of the core that communicates with the P2P network, and is wallet-agnostic. */ - void setClientModel(ClientModel *clientModel); + void setClientModel(ClientModel *clientModel = nullptr, interfaces::BlockAndHeaderTipInfo* tip_info = nullptr); #ifdef ENABLE_WALLET void setWalletController(WalletController* wallet_controller); + WalletController* getWalletController(); #endif #ifdef ENABLE_WALLET diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 7822d4c5f..a2f46c339 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -15,6 +15,7 @@ #include <net.h> #include <netbase.h> #include <util/system.h> +#include <util/threadnames.h> #include <validation.h> #include <stdint.h> @@ -52,6 +53,9 @@ ClientModel::ClientModel(interfaces::Node& node, OptionsModel *_optionsModel, QO // move timer to thread so that polling doesn't disturb main event loop timer->moveToThread(m_thread); m_thread->start(); + QTimer::singleShot(0, timer, []() { + util::ThreadRename("qt-clientmodl"); + }); subscribeToCoreSignals(); } diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index 93840b416..d210faec0 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -290,7 +290,7 @@ <item row="11" column="0"> <widget class="QLabel" name="label_3"> <property name="text"> - <string>Current number of blocks</string> + <string>Current block height</string> </property> </widget> </item> diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index fea759dee..0016fb973 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -459,10 +459,10 @@ <item> <widget class="QCheckBox" name="connectSocksTor"> <property name="toolTip"> - <string>Connect to the Bitcoin network through a separate SOCKS5 proxy for Tor hidden services.</string> + <string>Connect to the Bitcoin network through a separate SOCKS5 proxy for Tor onion services.</string> </property> <property name="text"> - <string>Use separate SOCKS&5 proxy to reach peers via Tor hidden services:</string> + <string>Use separate SOCKS&5 proxy to reach peers via Tor onion services:</string> </property> </widget> </item> diff --git a/src/qt/forms/receivecoinsdialog.ui b/src/qt/forms/receivecoinsdialog.ui index 7dbee6d68..06d39426c 100644 --- a/src/qt/forms/receivecoinsdialog.ui +++ b/src/qt/forms/receivecoinsdialog.ui @@ -114,6 +114,12 @@ <iconset resource="../bitcoin.qrc"> <normaloff>:/icons/receiving_addresses</normaloff>:/icons/receiving_addresses</iconset> </property> + <property name="autoDefault"> + <bool>false</bool> + </property> + <property name="default"> + <bool>true</bool> + </property> </widget> </item> <item> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 7f439fa45..bab17562a 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -21,11 +21,6 @@ #include <util/system.h> #ifdef WIN32 -#ifdef _WIN32_IE -#undef _WIN32_IE -#endif -#define _WIN32_IE 0x0501 -#define WIN32_LEAN_AND_MEAN 1 #ifndef NOMINMAX #define NOMINMAX #endif @@ -94,7 +89,7 @@ static std::string DummyAddress(const CChainParams ¶ms) std::vector<unsigned char> sourcedata = params.Base58Prefix(CChainParams::PUBKEY_ADDRESS); sourcedata.insert(sourcedata.end(), dummydata, dummydata + sizeof(dummydata)); for(int i=0; i<256; ++i) { // Try every trailing byte - std::string s = EncodeBase58(sourcedata.data(), sourcedata.data() + sourcedata.size()); + std::string s = EncodeBase58(sourcedata); if (!IsValidDestinationString(s)) { return s; } diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index ad21dfc3e..235722d09 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -6,6 +6,7 @@ #include <config/bitcoin-config.h> #endif +#include <chainparams.h> #include <fs.h> #include <qt/intro.h> #include <qt/forms/ui_intro.h> @@ -181,7 +182,7 @@ void Intro::setDataDirectory(const QString &dataDir) } } -bool Intro::showIfNeeded(interfaces::Node& node, bool& did_show_intro, bool& prune) +bool Intro::showIfNeeded(bool& did_show_intro, bool& prune) { did_show_intro = false; @@ -199,13 +200,13 @@ bool Intro::showIfNeeded(interfaces::Node& node, bool& did_show_intro, bool& pru { /* Use selectParams here to guarantee Params() can be used by node interface */ try { - node.selectParams(gArgs.GetChainName()); + SelectParams(gArgs.GetChainName()); } catch (const std::exception&) { return false; } /* If current default data directory does not exist, let the user choose one */ - Intro intro(0, node.getAssumedBlockchainSize(), node.getAssumedChainStateSize()); + Intro intro(0, Params().AssumedBlockchainSize(), Params().AssumedChainStateSize()); intro.setDataDirectory(dataDir); intro.setWindowIcon(QIcon(":icons/bitcoin")); did_show_intro = true; @@ -242,7 +243,7 @@ bool Intro::showIfNeeded(interfaces::Node& node, bool& did_show_intro, bool& pru * (to be consistent with bitcoind behavior) */ if(dataDir != GUIUtil::getDefaultDataDirectory()) { - node.softSetArg("-datadir", GUIUtil::qstringToBoostPath(dataDir).string()); // use OS locale for path setting + gArgs.SoftSetArg("-datadir", GUIUtil::qstringToBoostPath(dataDir).string()); // use OS locale for path setting } return true; } diff --git a/src/qt/intro.h b/src/qt/intro.h index 732393246..51f42de7a 100644 --- a/src/qt/intro.h +++ b/src/qt/intro.h @@ -47,7 +47,7 @@ public: * @note do NOT call global GetDataDir() before calling this function, this * will cause the wrong path to be cached. */ - static bool showIfNeeded(interfaces::Node& node, bool& did_show_intro, bool& prune); + static bool showIfNeeded(bool& did_show_intro, bool& prune); Q_SIGNALS: void requestCheck(); diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index 58a7591c9..7e089b4f9 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -27,8 +27,8 @@ const char *DEFAULT_GUI_PROXY_HOST = "127.0.0.1"; static const QString GetDefaultProxyAddress(); -OptionsModel::OptionsModel(interfaces::Node& node, QObject *parent, bool resetSettings) : - QAbstractListModel(parent), m_node(node) +OptionsModel::OptionsModel(QObject *parent, bool resetSettings) : + QAbstractListModel(parent) { Init(resetSettings); } @@ -97,12 +97,12 @@ void OptionsModel::Init(bool resetSettings) if (!settings.contains("nDatabaseCache")) settings.setValue("nDatabaseCache", (qint64)nDefaultDbCache); - if (!m_node.softSetArg("-dbcache", settings.value("nDatabaseCache").toString().toStdString())) + if (!gArgs.SoftSetArg("-dbcache", settings.value("nDatabaseCache").toString().toStdString())) addOverriddenOption("-dbcache"); if (!settings.contains("nThreadsScriptVerif")) settings.setValue("nThreadsScriptVerif", DEFAULT_SCRIPTCHECK_THREADS); - if (!m_node.softSetArg("-par", settings.value("nThreadsScriptVerif").toString().toStdString())) + if (!gArgs.SoftSetArg("-par", settings.value("nThreadsScriptVerif").toString().toStdString())) addOverriddenOption("-par"); if (!settings.contains("strDataDir")) @@ -112,19 +112,19 @@ void OptionsModel::Init(bool resetSettings) #ifdef ENABLE_WALLET if (!settings.contains("bSpendZeroConfChange")) settings.setValue("bSpendZeroConfChange", true); - if (!m_node.softSetBoolArg("-spendzeroconfchange", settings.value("bSpendZeroConfChange").toBool())) + if (!gArgs.SoftSetBoolArg("-spendzeroconfchange", settings.value("bSpendZeroConfChange").toBool())) addOverriddenOption("-spendzeroconfchange"); #endif // Network if (!settings.contains("fUseUPnP")) settings.setValue("fUseUPnP", DEFAULT_UPNP); - if (!m_node.softSetBoolArg("-upnp", settings.value("fUseUPnP").toBool())) + if (!gArgs.SoftSetBoolArg("-upnp", settings.value("fUseUPnP").toBool())) addOverriddenOption("-upnp"); if (!settings.contains("fListen")) settings.setValue("fListen", DEFAULT_LISTEN); - if (!m_node.softSetBoolArg("-listen", settings.value("fListen").toBool())) + if (!gArgs.SoftSetBoolArg("-listen", settings.value("fListen").toBool())) addOverriddenOption("-listen"); if (!settings.contains("fUseProxy")) @@ -132,7 +132,7 @@ void OptionsModel::Init(bool resetSettings) if (!settings.contains("addrProxy")) settings.setValue("addrProxy", GetDefaultProxyAddress()); // Only try to set -proxy, if user has enabled fUseProxy - if (settings.value("fUseProxy").toBool() && !m_node.softSetArg("-proxy", settings.value("addrProxy").toString().toStdString())) + if ((settings.value("fUseProxy").toBool() && !gArgs.SoftSetArg("-proxy", settings.value("addrProxy").toString().toStdString()))) addOverriddenOption("-proxy"); else if(!settings.value("fUseProxy").toBool() && !gArgs.GetArg("-proxy", "").empty()) addOverriddenOption("-proxy"); @@ -142,7 +142,7 @@ void OptionsModel::Init(bool resetSettings) if (!settings.contains("addrSeparateProxyTor")) settings.setValue("addrSeparateProxyTor", GetDefaultProxyAddress()); // Only try to set -onion, if user has enabled fUseSeparateProxyTor - if (settings.value("fUseSeparateProxyTor").toBool() && !m_node.softSetArg("-onion", settings.value("addrSeparateProxyTor").toString().toStdString())) + if ((settings.value("fUseSeparateProxyTor").toBool() && !gArgs.SoftSetArg("-onion", settings.value("addrSeparateProxyTor").toString().toStdString()))) addOverriddenOption("-onion"); else if(!settings.value("fUseSeparateProxyTor").toBool() && !gArgs.GetArg("-onion", "").empty()) addOverriddenOption("-onion"); @@ -150,7 +150,7 @@ void OptionsModel::Init(bool resetSettings) // Display if (!settings.contains("language")) settings.setValue("language", ""); - if (!m_node.softSetArg("-lang", settings.value("language").toString().toStdString())) + if (!gArgs.SoftSetArg("-lang", settings.value("language").toString().toStdString())) addOverriddenOption("-lang"); language = settings.value("language").toString(); @@ -244,10 +244,10 @@ void OptionsModel::SetPruneEnabled(bool prune, bool force) const int64_t prune_target_mib = PruneGBtoMiB(settings.value("nPruneSize").toInt()); std::string prune_val = prune ? ToString(prune_target_mib) : "0"; if (force) { - m_node.forceSetArg("-prune", prune_val); + gArgs.ForceSetArg("-prune", prune_val); return; } - if (!m_node.softSetArg("-prune", prune_val)) { + if (!gArgs.SoftSetArg("-prune", prune_val)) { addOverriddenOption("-prune"); } } @@ -353,7 +353,7 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in break; case MapPortUPnP: // core option - can be changed on-the-fly settings.setValue("fUseUPnP", value.toBool()); - m_node.mapPort(value.toBool()); + node().mapPort(value.toBool()); break; case MinimizeOnClose: fMinimizeOnClose = value.toBool(); diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 14fdf9046..3d9e7bbb8 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -11,6 +11,8 @@ #include <QAbstractListModel> +#include <assert.h> + namespace interfaces { class Node; } @@ -39,7 +41,7 @@ class OptionsModel : public QAbstractListModel Q_OBJECT public: - explicit OptionsModel(interfaces::Node& node, QObject *parent = nullptr, bool resetSettings = false); + explicit OptionsModel(QObject *parent = nullptr, bool resetSettings = false); enum OptionID { StartAtStartup, // bool @@ -92,10 +94,11 @@ public: void setRestartRequired(bool fRequired); bool isRestartRequired() const; - interfaces::Node& node() const { return m_node; } + interfaces::Node& node() const { assert(m_node); return *m_node; } + void setNode(interfaces::Node& node) { assert(!m_node); m_node = &node; } private: - interfaces::Node& m_node; + interfaces::Node* m_node = nullptr; /* Qt-only settings */ bool fHideTrayIcon; bool fMinimizeToTray; diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index a1da85bda..8679ced68 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -74,7 +74,7 @@ static QSet<QString> savedPaymentRequests; // Warning: ipcSendCommandLine() is called early in init, // so don't use "Q_EMIT message()", but "QMessageBox::"! // -void PaymentServer::ipcParseCommandLine(interfaces::Node& node, int argc, char* argv[]) +void PaymentServer::ipcParseCommandLine(int argc, char* argv[]) { for (int i = 1; i < argc; i++) { @@ -97,11 +97,11 @@ void PaymentServer::ipcParseCommandLine(interfaces::Node& node, int argc, char* auto tempChainParams = CreateChainParams(CBaseChainParams::MAIN); if (IsValidDestinationString(r.address.toStdString(), *tempChainParams)) { - node.selectParams(CBaseChainParams::MAIN); + SelectParams(CBaseChainParams::MAIN); } else { tempChainParams = CreateChainParams(CBaseChainParams::TESTNET); if (IsValidDestinationString(r.address.toStdString(), *tempChainParams)) { - node.selectParams(CBaseChainParams::TESTNET); + SelectParams(CBaseChainParams::TESTNET); } } } diff --git a/src/qt/paymentserver.h b/src/qt/paymentserver.h index 154f4a7ea..eaf2bafe5 100644 --- a/src/qt/paymentserver.h +++ b/src/qt/paymentserver.h @@ -61,7 +61,7 @@ class PaymentServer : public QObject public: // Parse URIs on command line // Returns false on error - static void ipcParseCommandLine(interfaces::Node& node, int argc, char *argv[]); + static void ipcParseCommandLine(int argc, char *argv[]); // Returns true if there were URIs on the command line // which were successfully sent to an already-running diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index 5debded4e..d374d610e 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -242,22 +242,6 @@ void ReceiveCoinsDialog::resizeEvent(QResizeEvent *event) columnResizingFixer->stretchColumnWidth(RecentRequestsTableModel::Message); } -void ReceiveCoinsDialog::keyPressEvent(QKeyEvent *event) -{ - if (event->key() == Qt::Key_Return) - { - // press return -> submit form - if (ui->reqLabel->hasFocus() || ui->reqAmount->hasFocus() || ui->reqMessage->hasFocus()) - { - event->ignore(); - on_receiveButton_clicked(); - return; - } - } - - this->QDialog::keyPressEvent(event); -} - QModelIndex ReceiveCoinsDialog::selectedRow() { if(!model || !model->getRecentRequestsTableModel() || !ui->recentRequestsView->selectionModel()) diff --git a/src/qt/receivecoinsdialog.h b/src/qt/receivecoinsdialog.h index 2f48cd58f..27455e290 100644 --- a/src/qt/receivecoinsdialog.h +++ b/src/qt/receivecoinsdialog.h @@ -49,9 +49,6 @@ public Q_SLOTS: void reject() override; void accept() override; -protected: - virtual void keyPressEvent(QKeyEvent *event) override; - private: Ui::ReceiveCoinsDialog *ui; GUIUtil::TableViewLastColumnResizingFixer *columnResizingFixer; diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 821a337a6..4c5601242 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -20,6 +20,7 @@ #include <rpc/client.h> #include <util/strencodings.h> #include <util/system.h> +#include <util/threadnames.h> #include <univalue.h> @@ -556,7 +557,7 @@ bool RPCConsole::eventFilter(QObject* obj, QEvent *event) return QWidget::eventFilter(obj, event); } -void RPCConsole::setClientModel(ClientModel *model) +void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_t bestblock_date, double verification_progress) { clientModel = model; @@ -576,13 +577,13 @@ void RPCConsole::setClientModel(ClientModel *model) setNumConnections(model->getNumConnections()); connect(model, &ClientModel::numConnectionsChanged, this, &RPCConsole::setNumConnections); - interfaces::Node& node = clientModel->node(); - setNumBlocks(node.getNumBlocks(), QDateTime::fromTime_t(node.getLastBlockTime()), node.getVerificationProgress(), false); + setNumBlocks(bestblock_height, QDateTime::fromTime_t(bestblock_date), verification_progress, false); connect(model, &ClientModel::numBlocksChanged, this, &RPCConsole::setNumBlocks); updateNetworkState(); connect(model, &ClientModel::networkActiveChanged, this, &RPCConsole::setNetworkActive); + interfaces::Node& node = clientModel->node(); updateTrafficStats(node.getTotalBytesRecv(), node.getTotalBytesSent()); connect(model, &ClientModel::bytesChanged, this, &RPCConsole::updateTrafficStats); @@ -978,6 +979,9 @@ void RPCConsole::startExecutor() // Default implementation of QThread::run() simply spins up an event loop in the thread, // which is what we want. thread.start(); + QTimer::singleShot(0, executor, []() { + util::ThreadRename("qt-rpcconsole"); + }); } void RPCConsole::on_tabWidget_currentChanged(int index) diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index de8e37cca..280c5bd71 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -46,7 +46,7 @@ public: return RPCParseCommandLine(&node, strResult, strCommand, true, pstrFilteredOut, wallet_model); } - void setClientModel(ClientModel *model); + void setClientModel(ClientModel *model = nullptr, int bestblock_height = 0, int64_t bestblock_date = 0, double verification_progress = 0.0); void addWallet(WalletModel * const walletModel); void removeWallet(WalletModel* const walletModel); diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 6e6b2b846..8e381861a 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -24,8 +24,8 @@ #include <QScreen> -SplashScreen::SplashScreen(interfaces::Node& node, Qt::WindowFlags f, const NetworkStyle *networkStyle) : - QWidget(nullptr, f), curAlignment(0), m_node(node) +SplashScreen::SplashScreen(Qt::WindowFlags f, const NetworkStyle *networkStyle) : + QWidget(nullptr, f), curAlignment(0) { // set reference point, paddings int paddingRight = 50; @@ -124,7 +124,6 @@ SplashScreen::SplashScreen(interfaces::Node& node, Qt::WindowFlags f, const Netw setFixedSize(r.size()); move(QGuiApplication::primaryScreen()->geometry().center() - r.center()); - subscribeToCoreSignals(); installEventFilter(this); GUIUtil::handleCloseWindowShortcut(this); @@ -132,14 +131,28 @@ SplashScreen::SplashScreen(interfaces::Node& node, Qt::WindowFlags f, const Netw SplashScreen::~SplashScreen() { - unsubscribeFromCoreSignals(); + if (m_node) unsubscribeFromCoreSignals(); +} + +void SplashScreen::setNode(interfaces::Node& node) +{ + assert(!m_node); + m_node = &node; + subscribeToCoreSignals(); + if (m_shutdown) m_node->startShutdown(); +} + +void SplashScreen::shutdown() +{ + m_shutdown = true; + if (m_node) m_node->startShutdown(); } bool SplashScreen::eventFilter(QObject * obj, QEvent * ev) { if (ev->type() == QEvent::KeyPress) { QKeyEvent *keyEvent = static_cast<QKeyEvent *>(ev); if (keyEvent->key() == Qt::Key_Q) { - m_node.startShutdown(); + shutdown(); } } return QObject::eventFilter(obj, ev); @@ -172,21 +185,21 @@ static void ShowProgress(SplashScreen *splash, const std::string &title, int nPr : _("press q to shutdown").translated) + strprintf("\n%d", nProgress) + "%"); } -#ifdef ENABLE_WALLET -void SplashScreen::ConnectWallet(std::unique_ptr<interfaces::Wallet> wallet) -{ - m_connected_wallet_handlers.emplace_back(wallet->handleShowProgress(std::bind(ShowProgress, this, std::placeholders::_1, std::placeholders::_2, false))); - m_connected_wallets.emplace_back(std::move(wallet)); -} -#endif void SplashScreen::subscribeToCoreSignals() { // Connect signals to client - m_handler_init_message = m_node.handleInitMessage(std::bind(InitMessage, this, std::placeholders::_1)); - m_handler_show_progress = m_node.handleShowProgress(std::bind(ShowProgress, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); + m_handler_init_message = m_node->handleInitMessage(std::bind(InitMessage, this, std::placeholders::_1)); + m_handler_show_progress = m_node->handleShowProgress(std::bind(ShowProgress, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); +} + +void SplashScreen::handleLoadWallet() +{ #ifdef ENABLE_WALLET - m_handler_load_wallet = m_node.handleLoadWallet([this](std::unique_ptr<interfaces::Wallet> wallet) { ConnectWallet(std::move(wallet)); }); + m_handler_load_wallet = m_node->walletClient().handleLoadWallet([this](std::unique_ptr<interfaces::Wallet> wallet) { + m_connected_wallet_handlers.emplace_back(wallet->handleShowProgress(std::bind(ShowProgress, this, std::placeholders::_1, std::placeholders::_2, false))); + m_connected_wallets.emplace_back(std::move(wallet)); + }); #endif } @@ -221,6 +234,6 @@ void SplashScreen::paintEvent(QPaintEvent *event) void SplashScreen::closeEvent(QCloseEvent *event) { - m_node.startShutdown(); // allows an "emergency" shutdown during startup + shutdown(); // allows an "emergency" shutdown during startup event->ignore(); } diff --git a/src/qt/splashscreen.h b/src/qt/splashscreen.h index 315852411..a0cd677d3 100644 --- a/src/qt/splashscreen.h +++ b/src/qt/splashscreen.h @@ -28,8 +28,9 @@ class SplashScreen : public QWidget Q_OBJECT public: - explicit SplashScreen(interfaces::Node& node, Qt::WindowFlags f, const NetworkStyle *networkStyle); + explicit SplashScreen(Qt::WindowFlags f, const NetworkStyle *networkStyle); ~SplashScreen(); + void setNode(interfaces::Node& node); protected: void paintEvent(QPaintEvent *event) override; @@ -42,6 +43,9 @@ public Q_SLOTS: /** Show message and progress */ void showMessage(const QString &message, int alignment, const QColor &color); + /** Handle wallet load notifications. */ + void handleLoadWallet(); + protected: bool eventFilter(QObject * obj, QEvent * ev) override; @@ -50,15 +54,16 @@ private: void subscribeToCoreSignals(); /** Disconnect core signals to splash screen */ void unsubscribeFromCoreSignals(); - /** Connect wallet signals to splash screen */ - void ConnectWallet(std::unique_ptr<interfaces::Wallet> wallet); + /** Initiate shutdown */ + void shutdown(); QPixmap pixmap; QString curMessage; QColor curColor; int curAlignment; - interfaces::Node& m_node; + interfaces::Node* m_node = nullptr; + bool m_shutdown = false; std::unique_ptr<interfaces::Handler> m_handler_init_message; std::unique_ptr<interfaces::Handler> m_handler_show_progress; std::unique_ptr<interfaces::Handler> m_handler_load_wallet; diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 9347ff9e4..35fcb2b0c 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -18,6 +18,7 @@ #include <key.h> #include <key_io.h> #include <wallet/wallet.h> +#include <walletinitinterface.h> #include <QApplication> #include <QTimer> @@ -59,7 +60,8 @@ void EditAddressAndSubmit( void TestAddAddressesToSendBook(interfaces::Node& node) { TestChain100Setup test; - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), WalletLocation(), CreateMockWalletDatabase()); + node.setContext(&test.m_node); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); bool firstRun; wallet->LoadWallet(firstRun); @@ -106,11 +108,11 @@ void TestAddAddressesToSendBook(interfaces::Node& node) // Initialize relevant QT models. std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); - OptionsModel optionsModel(node); + OptionsModel optionsModel; ClientModel clientModel(node, &optionsModel); AddWallet(wallet); WalletModel walletModel(interfaces::MakeWallet(wallet), clientModel, platformStyle.get()); - RemoveWallet(wallet); + RemoveWallet(wallet, nullopt); EditAddressDialog editAddressDialog(EditAddressDialog::NewSendingAddress); editAddressDialog.setModel(walletModel.getAddressTableModel()); diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index b880a99ba..0b5c34154 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -67,6 +67,7 @@ void AppTests::appTests() return GetDataDir() / "blocks"; }()); + qRegisterMetaType<interfaces::BlockAndHeaderTipInfo>("interfaces::BlockAndHeaderTipInfo"); m_app.parameterSetup(); m_app.createOptionsModel(true /* reset settings */); QScopedPointer<const NetworkStyle> style(NetworkStyle::instantiate(Params().NetworkIDString())); diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 12efca250..86356b43c 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -52,7 +52,8 @@ int main(int argc, char* argv[]) BasicTestingSetup dummy{CBaseChainParams::REGTEST}; } - std::unique_ptr<interfaces::Node> node = interfaces::MakeNode(); + NodeContext node_context; + std::unique_ptr<interfaces::Node> node = interfaces::MakeNode(&node_context); bool fInvalid = false; @@ -67,11 +68,11 @@ int main(int argc, char* argv[]) // Don't remove this, it's needed to access // QApplication:: and QCoreApplication:: in the tests - BitcoinApplication app(*node); + BitcoinApplication app; + app.setNode(*node); app.setApplicationName("Bitcoin-Qt-test"); - node->setupServerArgs(); // Make gArgs available in the NodeContext - node->context()->args->ClearArgs(); // Clear added args again + app.node().context()->args = &gArgs; // Make gArgs available in the NodeContext AppTests app_tests(app); if (QTest::qExec(&app_tests) != 0) { fInvalid = true; @@ -80,7 +81,7 @@ int main(int argc, char* argv[]) if (QTest::qExec(&test1) != 0) { fInvalid = true; } - RPCNestedTests test3(*node); + RPCNestedTests test3(app.node()); if (QTest::qExec(&test3) != 0) { fInvalid = true; } @@ -89,11 +90,11 @@ int main(int argc, char* argv[]) fInvalid = true; } #ifdef ENABLE_WALLET - WalletTests test5(*node); + WalletTests test5(app.node()); if (QTest::qExec(&test5) != 0) { fInvalid = true; } - AddressBookTests test6(*node); + AddressBookTests test6(app.node()); if (QTest::qExec(&test6) != 0) { fInvalid = true; } diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 6648029ba..d6d2d0e3d 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -138,9 +138,8 @@ void TestGUI(interfaces::Node& node) for (int i = 0; i < 5; ++i) { test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); } - node.context()->connman = std::move(test.m_node.connman); - node.context()->mempool = std::move(test.m_node.mempool); - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), WalletLocation(), CreateMockWalletDatabase()); + node.setContext(&test.m_node); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); bool firstRun; wallet->LoadWallet(firstRun); { @@ -164,11 +163,11 @@ void TestGUI(interfaces::Node& node) std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); SendCoinsDialog sendCoinsDialog(platformStyle.get()); TransactionView transactionView(platformStyle.get()); - OptionsModel optionsModel(node); + OptionsModel optionsModel; ClientModel clientModel(node, &optionsModel); AddWallet(wallet); WalletModel walletModel(interfaces::MakeWallet(wallet), clientModel, platformStyle.get()); - RemoveWallet(wallet); + RemoveWallet(wallet, nullopt); sendCoinsDialog.setModel(&walletModel); transactionView.setModel(&walletModel); diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index 01922cf99..b7f85446f 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -28,7 +28,7 @@ #include <QVBoxLayout> /** "Help message" or "About" dialog box */ -HelpMessageDialog::HelpMessageDialog(interfaces::Node& node, QWidget *parent, bool about) : +HelpMessageDialog::HelpMessageDialog(QWidget *parent, bool about) : QDialog(parent), ui(new Ui::HelpMessageDialog) { diff --git a/src/qt/utilitydialog.h b/src/qt/utilitydialog.h index 425b468f4..d2a5d5f67 100644 --- a/src/qt/utilitydialog.h +++ b/src/qt/utilitydialog.h @@ -12,10 +12,6 @@ QT_BEGIN_NAMESPACE class QMainWindow; QT_END_NAMESPACE -namespace interfaces { - class Node; -} - namespace Ui { class HelpMessageDialog; } @@ -26,7 +22,7 @@ class HelpMessageDialog : public QDialog Q_OBJECT public: - explicit HelpMessageDialog(interfaces::Node& node, QWidget *parent, bool about); + explicit HelpMessageDialog(QWidget *parent, bool about); ~HelpMessageDialog(); void printToConsole(); diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index 3aed98e0e..d9e0274d0 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -14,6 +14,7 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <util/string.h> +#include <util/threadnames.h> #include <util/translation.h> #include <wallet/wallet.h> @@ -35,16 +36,19 @@ WalletController::WalletController(ClientModel& client_model, const PlatformStyl , m_platform_style(platform_style) , m_options_model(client_model.getOptionsModel()) { - m_handler_load_wallet = m_node.handleLoadWallet([this](std::unique_ptr<interfaces::Wallet> wallet) { + m_handler_load_wallet = m_node.walletClient().handleLoadWallet([this](std::unique_ptr<interfaces::Wallet> wallet) { getOrCreateWallet(std::move(wallet)); }); - for (std::unique_ptr<interfaces::Wallet>& wallet : m_node.getWallets()) { + for (std::unique_ptr<interfaces::Wallet>& wallet : m_node.walletClient().getWallets()) { getOrCreateWallet(std::move(wallet)); } m_activity_worker->moveToThread(m_activity_thread); m_activity_thread->start(); + QTimer::singleShot(0, m_activity_worker, []() { + util::ThreadRename("qt-walletctrl"); + }); } // Not using the default destructor because not all member types definitions are @@ -66,7 +70,7 @@ std::map<std::string, bool> WalletController::listWalletDir() const { QMutexLocker locker(&m_mutex); std::map<std::string, bool> wallets; - for (const std::string& name : m_node.listWalletDir()) { + for (const std::string& name : m_node.walletClient().listWalletDir()) { wallets[name] = false; } for (WalletModel* wallet_model : m_wallets) { @@ -249,10 +253,9 @@ void CreateWalletActivity::createWallet() } QTimer::singleShot(500, worker(), [this, name, flags] { - WalletCreationStatus status; - std::unique_ptr<interfaces::Wallet> wallet = node().createWallet(m_passphrase, flags, name, m_error_message, m_warning_message, status); + std::unique_ptr<interfaces::Wallet> wallet = node().walletClient().createWallet(name, m_passphrase, flags, m_error_message, m_warning_message); - if (status == WalletCreationStatus::SUCCESS) m_wallet_model = m_wallet_controller->getOrCreateWallet(std::move(wallet)); + if (wallet) m_wallet_model = m_wallet_controller->getOrCreateWallet(std::move(wallet)); QTimer::singleShot(500, this, &CreateWalletActivity::finish); }); @@ -321,7 +324,7 @@ void OpenWalletActivity::open(const std::string& path) showProgressDialog(tr("Opening Wallet <b>%1</b>...").arg(name.toHtmlEscaped())); QTimer::singleShot(0, worker(), [this, path] { - std::unique_ptr<interfaces::Wallet> wallet = node().loadWallet(path, m_error_message, m_warning_message); + std::unique_ptr<interfaces::Wallet> wallet = node().walletClient().loadWallet(path, m_error_message, m_warning_message); if (wallet) m_wallet_model = m_wallet_controller->getOrCreateWallet(std::move(wallet)); diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index ec56f2755..f16761d6b 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -2,6 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <qt/createwalletdialog.h> +#include <qt/walletcontroller.h> #include <qt/walletframe.h> #include <qt/walletmodel.h> @@ -10,8 +12,11 @@ #include <cassert> +#include <QGroupBox> #include <QHBoxLayout> #include <QLabel> +#include <QPushButton> +#include <QVBoxLayout> WalletFrame::WalletFrame(const PlatformStyle *_platformStyle, BitcoinGUI *_gui) : QFrame(_gui), @@ -25,9 +30,25 @@ WalletFrame::WalletFrame(const PlatformStyle *_platformStyle, BitcoinGUI *_gui) walletFrameLayout->setContentsMargins(0,0,0,0); walletFrameLayout->addWidget(walletStack); - QLabel *noWallet = new QLabel(tr("No wallet has been loaded.")); + // hbox for no wallet + QGroupBox* no_wallet_group = new QGroupBox(walletStack); + QVBoxLayout* no_wallet_layout = new QVBoxLayout(no_wallet_group); + + QLabel *noWallet = new QLabel(tr("No wallet has been loaded.\nGo to File > Open Wallet to load a wallet.\n- OR -")); noWallet->setAlignment(Qt::AlignCenter); - walletStack->addWidget(noWallet); + no_wallet_layout->addWidget(noWallet, 0, Qt::AlignHCenter | Qt::AlignBottom); + + // A button for create wallet dialog + QPushButton* create_wallet_button = new QPushButton(tr("Create a new wallet"), walletStack); + connect(create_wallet_button, &QPushButton::clicked, [this] { + auto activity = new CreateWalletActivity(gui->getWalletController(), this); + connect(activity, &CreateWalletActivity::finished, activity, &QObject::deleteLater); + activity->create(); + }); + no_wallet_layout->addWidget(create_wallet_button, 0, Qt::AlignHCenter | Qt::AlignTop); + no_wallet_group->setLayout(no_wallet_layout); + + walletStack->addWidget(no_wallet_group); } WalletFrame::~WalletFrame() diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index e374dd191..6a3f90320 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -413,7 +413,7 @@ void WalletModel::subscribeToCoreSignals() m_handler_transaction_changed = m_wallet->handleTransactionChanged(std::bind(NotifyTransactionChanged, this, std::placeholders::_1, std::placeholders::_2)); m_handler_show_progress = m_wallet->handleShowProgress(std::bind(ShowProgress, this, std::placeholders::_1, std::placeholders::_2)); m_handler_watch_only_changed = m_wallet->handleWatchOnlyChanged(std::bind(NotifyWatchonlyChanged, this, std::placeholders::_1)); - m_handler_can_get_addrs_changed = m_wallet->handleCanGetAddressesChanged(boost::bind(NotifyCanGetAddressesChanged, this)); + m_handler_can_get_addrs_changed = m_wallet->handleCanGetAddressesChanged(std::bind(NotifyCanGetAddressesChanged, this)); } void WalletModel::unsubscribeFromCoreSignals() @@ -581,7 +581,7 @@ QString WalletModel::getDisplayName() const bool WalletModel::isMultiwallet() { - return m_node.getWallets().size() > 1; + return m_node.walletClient().getWallets().size() > 1; } void WalletModel::refresh(bool pk_hash_only) diff --git a/src/random.cpp b/src/random.cpp index 9c9a35709..af9504e0c 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -315,12 +315,16 @@ void GetOSRand(unsigned char *ent32) if (getentropy(ent32, NUM_OS_RANDOM_BYTES) != 0) { RandFailure(); } + // Silence a compiler warning about unused function. + (void)GetDevURandom; #elif defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX) /* getentropy() is available on macOS 10.12 and later. */ if (getentropy(ent32, NUM_OS_RANDOM_BYTES) != 0) { RandFailure(); } + // Silence a compiler warning about unused function. + (void)GetDevURandom; #elif defined(HAVE_SYSCTL_ARND) /* FreeBSD, NetBSD and similar. It is possible for the call to return less * bytes than requested, so need to read in a loop. @@ -334,6 +338,8 @@ void GetOSRand(unsigned char *ent32) } have += len; } while (have < NUM_OS_RANDOM_BYTES); + // Silence a compiler warning about unused function. + (void)GetDevURandom; #else /* Fall back to /dev/urandom if there is no specific method implemented to * get system entropy for this OS. diff --git a/src/rest.cpp b/src/rest.cpp index 7130625d5..f0bcbe55f 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -102,7 +102,7 @@ static CTxMemPool* GetMemPool(const util::Ref& context, HTTPRequest* req) RESTERR(req, HTTP_NOT_FOUND, "Mempool disabled or instance not found"); return nullptr; } - return node->mempool; + return node->mempool.get(); } static RetFormat ParseDataFormat(std::string& param, const std::string& strReq) @@ -393,7 +393,7 @@ static bool rest_tx(const util::Ref& context, HTTPRequest* req, const std::strin const NodeContext* const node = GetNodeContext(context, req); if (!node) return false; uint256 hashBlock = uint256(); - const CTransactionRef tx = GetTransaction(/* block_index */ nullptr, node->mempool, hash, Params().GetConsensus(), hashBlock); + const CTransactionRef tx = GetTransaction(/* block_index */ nullptr, node->mempool.get(), hash, Params().GetConsensus(), hashBlock); if (!tx) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f27373b57..04b9c6b1c 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -463,9 +463,9 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool UniValue spent(UniValue::VARR); const CTxMemPool::txiter& it = pool.mapTx.find(tx.GetHash()); - const CTxMemPool::setEntries& setChildren = pool.GetMemPoolChildren(it); - for (CTxMemPool::txiter childiter : setChildren) { - spent.push_back(childiter->GetTx().GetHash().ToString()); + const CTxMemPoolEntry::Children& children = it->GetMemPoolChildrenConst(); + for (const CTxMemPoolEntry& child : children) { + spent.push_back(child.GetTx().GetHash().ToString()); } info.pushKV("spentby", spent); @@ -556,7 +556,10 @@ static UniValue getmempoolancestors(const JSONRPCRequest& request) RPCResult::Type::ARR, "", "", {{RPCResult::Type::STR_HEX, "", "The transaction id of an in-mempool ancestor transaction"}}}, RPCResult{"for verbose = true", - RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, + RPCResult::Type::OBJ_DYN, "", "", + { + {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()}, + }}, }, RPCExamples{ HelpExampleCli("getmempoolancestors", "\"mytxid\"") @@ -588,7 +591,6 @@ static UniValue getmempoolancestors(const JSONRPCRequest& request) for (CTxMemPool::txiter ancestorIt : setAncestors) { o.push_back(ancestorIt->GetTx().GetHash().ToString()); } - return o; } else { UniValue o(UniValue::VOBJ); @@ -1739,7 +1741,7 @@ static UniValue getblockstats(const JSONRPCRequest& request) {RPCResult::Type::NUM, "total_size", "Total size of all non-coinbase transactions"}, {RPCResult::Type::NUM, "total_weight", "Total weight of all non-coinbase transactions divided by segwit scale factor (4)"}, {RPCResult::Type::NUM, "totalfee", "The fee total"}, - {RPCResult::Type::NUM, "txs", "The number of transactions (excluding coinbase)"}, + {RPCResult::Type::NUM, "txs", "The number of transactions (including coinbase)"}, {RPCResult::Type::NUM, "utxo_increase", "The increase/decrease in the number of unspent outputs"}, {RPCResult::Type::NUM, "utxo_size_inc", "The increase/decrease in size for the utxo index (not discounting op_return and similar)"}, }}, @@ -2407,7 +2409,7 @@ static const CRPCCommand commands[] = { "hidden", "dumptxoutset", &dumptxoutset, {"path"} }, }; // clang-format on - - for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) - t.appendCommand(commands[vcidx].name, &commands[vcidx]); + for (const auto& c : commands) { + t.appendCommand(c.name, &c); + } } diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 66ace7263..6ef329413 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -125,6 +125,9 @@ static const CRPCConvertParam vRPCConvertParams[] = { "gettxoutproof", 0, "txids" }, { "lockunspent", 0, "unlock" }, { "lockunspent", 1, "transactions" }, + { "send", 0, "outputs" }, + { "send", 1, "conf_target" }, + { "send", 3, "options" }, { "importprivkey", 2, "rescan" }, { "importaddress", 2, "rescan" }, { "importaddress", 3, "p2sh" }, @@ -151,6 +154,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getmempoolancestors", 1, "verbose" }, { "getmempooldescendants", 1, "verbose" }, { "bumpfee", 1, "options" }, + { "psbtbumpfee", 1, "options" }, { "logging", 0, "include" }, { "logging", 1, "exclude" }, { "disconnectnode", 1, "nodeid" }, @@ -172,7 +176,11 @@ static const CRPCConvertParam vRPCConvertParams[] = { "createwallet", 2, "blank"}, { "createwallet", 4, "avoid_reuse"}, { "createwallet", 5, "descriptors"}, + { "createwallet", 6, "load_on_startup"}, + { "loadwallet", 1, "load_on_startup"}, + { "unloadwallet", 1, "load_on_startup"}, { "getnodeaddresses", 0, "count"}, + { "addpeeraddress", 1, "port"}, { "stop", 0, "wait" }, }; // clang-format on diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index fee6a893e..b04e106b2 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -81,9 +81,9 @@ static UniValue GetNetworkHashPS(int lookup, int height) { return workDiff.getdouble() / timeDiff; } -static UniValue getnetworkhashps(const JSONRPCRequest& request) +static RPCHelpMan getnetworkhashps() { - RPCHelpMan{"getnetworkhashps", + return RPCHelpMan{"getnetworkhashps", "\nReturns the estimated network hashes per second based on the last n blocks.\n" "Pass in [blocks] to override # of blocks, -1 specifies since last difficulty change.\n" "Pass in [height] to estimate the network speed at the time when a certain block was found.\n", @@ -97,10 +97,12 @@ static UniValue getnetworkhashps(const JSONRPCRequest& request) HelpExampleCli("getnetworkhashps", "") + HelpExampleRpc("getnetworkhashps", "") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ LOCK(cs_main); return GetNetworkHashPS(!request.params[0].isNull() ? request.params[0].get_int() : 120, !request.params[1].isNull() ? request.params[1].get_int() : -1); +}, + }; } static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& max_tries, unsigned int& extra_nonce, uint256& block_hash) @@ -200,9 +202,9 @@ static bool getScriptFromDescriptor(const std::string& descriptor, CScript& scri } } -static UniValue generatetodescriptor(const JSONRPCRequest& request) +static RPCHelpMan generatetodescriptor() { - RPCHelpMan{ + return RPCHelpMan{ "generatetodescriptor", "\nMine blocks immediately to a specified descriptor (before the RPC call returns)\n", { @@ -218,9 +220,8 @@ static UniValue generatetodescriptor(const JSONRPCRequest& request) }, RPCExamples{ "\nGenerate 11 blocks to mydesc\n" + HelpExampleCli("generatetodescriptor", "11 \"mydesc\"")}, - } - .Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ const int num_blocks{request.params[0].get_int()}; const uint64_t max_tries{request.params[2].isNull() ? DEFAULT_MAX_TRIES : request.params[2].get_int()}; @@ -234,11 +235,25 @@ static UniValue generatetodescriptor(const JSONRPCRequest& request) ChainstateManager& chainman = EnsureChainman(request.context); return generateBlocks(chainman, mempool, coinbase_script, num_blocks, max_tries); +}, + }; +} + +static RPCHelpMan generate() +{ + return RPCHelpMan{"generate", "has been replaced by the -generate cli option. Refer to -help for more information.", {}, {}, RPCExamples{""}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + + if (request.fHelp) { + throw std::runtime_error(self.ToString()); + } else { + throw JSONRPCError(RPC_METHOD_NOT_FOUND, self.ToString()); + } + }}; } -static UniValue generatetoaddress(const JSONRPCRequest& request) +static RPCHelpMan generatetoaddress() { - RPCHelpMan{"generatetoaddress", + return RPCHelpMan{"generatetoaddress", "\nMine blocks immediately to a specified address (before the RPC call returns)\n", { {"nblocks", RPCArg::Type::NUM, RPCArg::Optional::NO, "How many blocks are generated immediately."}, @@ -256,8 +271,8 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) + "If you are using the " PACKAGE_NAME " wallet, you can get a new address to send the newly generated bitcoin to with:\n" + HelpExampleCli("getnewaddress", "") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ const int num_blocks{request.params[0].get_int()}; const uint64_t max_tries{request.params[2].isNull() ? DEFAULT_MAX_TRIES : request.params[2].get_int()}; @@ -272,11 +287,13 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) CScript coinbase_script = GetScriptForDestination(destination); return generateBlocks(chainman, mempool, coinbase_script, num_blocks, max_tries); +}, + }; } -static UniValue generateblock(const JSONRPCRequest& request) +static RPCHelpMan generateblock() { - RPCHelpMan{"generateblock", + return RPCHelpMan{"generateblock", "\nMine a block with a set of ordered transactions immediately to a specified address or descriptor (before the RPC call returns)\n", { {"output", RPCArg::Type::STR, RPCArg::Optional::NO, "The address or descriptor to send the newly generated bitcoin to."}, @@ -298,8 +315,8 @@ static UniValue generateblock(const JSONRPCRequest& request) "\nGenerate a block to myaddress, with txs rawtx and mempool_txid\n" + HelpExampleCli("generateblock", R"("myaddress" '["rawtx", "mempool_txid"]')") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ const auto address_or_descriptor = request.params[0].get_str(); CScript coinbase_script; std::string error; @@ -379,11 +396,13 @@ static UniValue generateblock(const JSONRPCRequest& request) UniValue obj(UniValue::VOBJ); obj.pushKV("hash", block_hash.GetHex()); return obj; +}, + }; } -static UniValue getmininginfo(const JSONRPCRequest& request) +static RPCHelpMan getmininginfo() { - RPCHelpMan{"getmininginfo", + return RPCHelpMan{"getmininginfo", "\nReturns a json object containing mining-related information.", {}, RPCResult{ @@ -402,8 +421,8 @@ static UniValue getmininginfo(const JSONRPCRequest& request) HelpExampleCli("getmininginfo", "") + HelpExampleRpc("getmininginfo", "") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ LOCK(cs_main); const CTxMemPool& mempool = EnsureMemPool(request.context); @@ -412,18 +431,20 @@ static UniValue getmininginfo(const JSONRPCRequest& request) if (BlockAssembler::m_last_block_weight) obj.pushKV("currentblockweight", *BlockAssembler::m_last_block_weight); if (BlockAssembler::m_last_block_num_txs) obj.pushKV("currentblocktx", *BlockAssembler::m_last_block_num_txs); obj.pushKV("difficulty", (double)GetDifficulty(::ChainActive().Tip())); - obj.pushKV("networkhashps", getnetworkhashps(request)); + obj.pushKV("networkhashps", getnetworkhashps().HandleRequest(request)); obj.pushKV("pooledtx", (uint64_t)mempool.size()); obj.pushKV("chain", Params().NetworkIDString()); obj.pushKV("warnings", GetWarnings(false).original); return obj; +}, + }; } // NOTE: Unlike wallet RPC (which use BTC values), mining RPCs follow GBT (BIP 22) in using satoshi amounts -static UniValue prioritisetransaction(const JSONRPCRequest& request) +static RPCHelpMan prioritisetransaction() { - RPCHelpMan{"prioritisetransaction", + return RPCHelpMan{"prioritisetransaction", "Accepts the transaction into mined blocks at a higher (or lower) priority\n", { {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id."}, @@ -440,8 +461,8 @@ static UniValue prioritisetransaction(const JSONRPCRequest& request) HelpExampleCli("prioritisetransaction", "\"txid\" 0.0 10000") + HelpExampleRpc("prioritisetransaction", "\"txid\", 0.0, 10000") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ LOCK(cs_main); uint256 hash(ParseHashV(request.params[0], "txid")); @@ -453,6 +474,8 @@ static UniValue prioritisetransaction(const JSONRPCRequest& request) EnsureMemPool(request.context).PrioritiseTransaction(hash, nAmount); return true; +}, + }; } @@ -484,9 +507,9 @@ static std::string gbt_vb_name(const Consensus::DeploymentPos pos) { return s; } -static UniValue getblocktemplate(const JSONRPCRequest& request) +static RPCHelpMan getblocktemplate() { - RPCHelpMan{"getblocktemplate", + return RPCHelpMan{"getblocktemplate", "\nIf the request parameters include a 'mode' key, that is used to explicitly select between the default 'template' request or a 'proposal'.\n" "It returns data needed to construct a block to work on.\n" "For full specification, see BIPs 22, 23, 9, and 145:\n" @@ -500,12 +523,13 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) {"mode", RPCArg::Type::STR, /* treat as named arg */ RPCArg::Optional::OMITTED_NAMED_ARG, "This must be set to \"template\", \"proposal\" (see BIP 23), or omitted"}, {"capabilities", RPCArg::Type::ARR, /* treat as named arg */ RPCArg::Optional::OMITTED_NAMED_ARG, "A list of strings", { - {"support", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "client side supported feature, 'longpoll', 'coinbasetxn', 'coinbasevalue', 'proposal', 'serverlist', 'workid'"}, + {"str", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "client side supported feature, 'longpoll', 'coinbasevalue', 'proposal', 'serverlist', 'workid'"}, }, }, {"rules", RPCArg::Type::ARR, RPCArg::Optional::NO, "A list of strings", { - {"support", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "client side supported softfork deployment"}, + {"segwit", RPCArg::Type::STR, RPCArg::Optional::NO, "(literal) indicates client side segwit support"}, + {"str", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "other client side supported softfork deployment"}, }, }, }, @@ -517,7 +541,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) {RPCResult::Type::NUM, "version", "The preferred block version"}, {RPCResult::Type::ARR, "rules", "specific block rules that are to be enforced", { - {RPCResult::Type::STR, "", "rulename"}, + {RPCResult::Type::STR, "", "name of a rule the client must understand to some extent; see BIP 9 for format"}, }}, {RPCResult::Type::OBJ_DYN, "vbavailable", "set of pending, supported versionbit (BIP 9) softfork deployments", { @@ -525,7 +549,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) }}, {RPCResult::Type::NUM, "vbrequired", "bit mask of versionbits the server requires set in submissions"}, {RPCResult::Type::STR, "previousblockhash", "The hash of current highest block"}, - {RPCResult::Type::ARR, "", "contents of non-coinbase transactions that should be included in the next block", + {RPCResult::Type::ARR, "transactions", "contents of non-coinbase transactions that should be included in the next block", { {RPCResult::Type::OBJ, "", "", { @@ -541,15 +565,12 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) {RPCResult::Type::NUM, "weight", "total transaction weight, as counted for purposes of block limits"}, }}, }}, - {RPCResult::Type::OBJ, "coinbaseaux", "data that should be included in the coinbase's scriptSig content", + {RPCResult::Type::OBJ_DYN, "coinbaseaux", "data that should be included in the coinbase's scriptSig content", { - {RPCResult::Type::ELISION, "", ""}, + {RPCResult::Type::STR_HEX, "key", "values must be in the coinbase (keys may be ignored)"}, }}, {RPCResult::Type::NUM, "coinbasevalue", "maximum allowable input to coinbase transaction, including the generation award and transaction fees (in satoshis)"}, - {RPCResult::Type::OBJ, "coinbasetxn", "information for coinbase transaction", - { - {RPCResult::Type::ELISION, "", ""}, - }}, + {RPCResult::Type::STR, "longpollid", "an id to include with a request to longpoll on an update to this template"}, {RPCResult::Type::STR, "target", "The hash target"}, {RPCResult::Type::NUM_TIME, "mintime", "The minimum timestamp appropriate for the next block time, expressed in " + UNIX_EPOCH_TIME}, {RPCResult::Type::ARR, "mutable", "list of ways the block template may be changed", @@ -563,13 +584,14 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) {RPCResult::Type::NUM_TIME, "curtime", "current timestamp in " + UNIX_EPOCH_TIME}, {RPCResult::Type::STR, "bits", "compressed target of next block"}, {RPCResult::Type::NUM, "height", "The height of the next block"}, + {RPCResult::Type::STR, "default_witness_commitment", /* optional */ true, "a valid witness commitment for the unmodified block template"} }}, RPCExamples{ HelpExampleCli("getblocktemplate", "'{\"rules\": [\"segwit\"]}'") + HelpExampleRpc("getblocktemplate", "{\"rules\": [\"segwit\"]}") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ LOCK(cs_main); std::string strMode = "template"; @@ -877,6 +899,8 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) } return result; +}, + }; } class submitblock_StateCatcher final : public CValidationInterface @@ -897,10 +921,10 @@ protected: } }; -static UniValue submitblock(const JSONRPCRequest& request) +static RPCHelpMan submitblock() { // We allow 2 arguments for compliance with BIP22. Argument 2 is ignored. - RPCHelpMan{"submitblock", + return RPCHelpMan{"submitblock", "\nAttempts to submit new block to network.\n" "See https://en.bitcoin.it/wiki/BIP_0022 for full specification.\n", { @@ -912,8 +936,8 @@ static UniValue submitblock(const JSONRPCRequest& request) HelpExampleCli("submitblock", "\"mydata\"") + HelpExampleRpc("submitblock", "\"mydata\"") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ std::shared_ptr<CBlock> blockptr = std::make_shared<CBlock>(); CBlock& block = *blockptr; if (!DecodeHexBlk(block, request.params[0].get_str())) { @@ -958,11 +982,13 @@ static UniValue submitblock(const JSONRPCRequest& request) return "inconclusive"; } return BIP22ValidationResult(sc->state); +}, + }; } -static UniValue submitheader(const JSONRPCRequest& request) +static RPCHelpMan submitheader() { - RPCHelpMan{"submitheader", + return RPCHelpMan{"submitheader", "\nDecode the given hexdata as a header and submit it as a candidate chain tip if valid." "\nThrows when the header is invalid.\n", { @@ -974,8 +1000,8 @@ static UniValue submitheader(const JSONRPCRequest& request) HelpExampleCli("submitheader", "\"aabbcc\"") + HelpExampleRpc("submitheader", "\"aabbcc\"") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ CBlockHeader h; if (!DecodeHexBlockHeader(h, request.params[0].get_str())) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block header decode failed"); @@ -994,11 +1020,13 @@ static UniValue submitheader(const JSONRPCRequest& request) throw JSONRPCError(RPC_VERIFY_ERROR, state.ToString()); } throw JSONRPCError(RPC_VERIFY_ERROR, state.GetRejectReason()); +}, + }; } -static UniValue estimatesmartfee(const JSONRPCRequest& request) +static RPCHelpMan estimatesmartfee() { - RPCHelpMan{"estimatesmartfee", + return RPCHelpMan{"estimatesmartfee", "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" "confirmation within conf_target blocks if possible and return the number of blocks\n" "for which the estimate is valid. Uses virtual transaction size as defined\n" @@ -1019,7 +1047,7 @@ static UniValue estimatesmartfee(const JSONRPCRequest& request) RPCResult::Type::OBJ, "", "", { {RPCResult::Type::NUM, "feerate", /* optional */ true, "estimate fee rate in " + CURRENCY_UNIT + "/kB (only present if no errors were encountered)"}, - {RPCResult::Type::ARR, "errors", "Errors encountered during processing", + {RPCResult::Type::ARR, "errors", /* optional */ true, "Errors encountered during processing (if there are any)", { {RPCResult::Type::STR, "", "error"}, }}, @@ -1032,8 +1060,8 @@ static UniValue estimatesmartfee(const JSONRPCRequest& request) RPCExamples{ HelpExampleCli("estimatesmartfee", "6") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VSTR}); RPCTypeCheckArgument(request.params[0], UniValue::VNUM); unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); @@ -1059,11 +1087,13 @@ static UniValue estimatesmartfee(const JSONRPCRequest& request) } result.pushKV("blocks", feeCalc.returnedTarget); return result; +}, + }; } -static UniValue estimaterawfee(const JSONRPCRequest& request) +static RPCHelpMan estimaterawfee() { - RPCHelpMan{"estimaterawfee", + return RPCHelpMan{"estimaterawfee", "\nWARNING: This interface is unstable and may disappear or change!\n" "\nWARNING: This is an advanced API call that is tightly coupled to the specific\n" " implementation of fee estimation. The parameters it can be called with\n" @@ -1098,7 +1128,7 @@ static UniValue estimaterawfee(const JSONRPCRequest& request) { {RPCResult::Type::ELISION, "", ""}, }}, - {RPCResult::Type::ARR, "errors", /* optional */ true, "Errors encountered during processing", + {RPCResult::Type::ARR, "errors", /* optional */ true, "Errors encountered during processing (if there are any)", { {RPCResult::Type::STR, "error", ""}, }}, @@ -1115,8 +1145,8 @@ static UniValue estimaterawfee(const JSONRPCRequest& request) RPCExamples{ HelpExampleCli("estimaterawfee", "6 0.9") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VNUM}, true); RPCTypeCheckArgument(request.params[0], UniValue::VNUM); unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); @@ -1175,6 +1205,8 @@ static UniValue estimaterawfee(const JSONRPCRequest& request) result.pushKV(StringForFeeEstimateHorizon(horizon), horizon_result); } return result; +}, + }; } void RegisterMiningRPCCommands(CRPCTable &t) @@ -1198,9 +1230,10 @@ static const CRPCCommand commands[] = { "util", "estimatesmartfee", &estimatesmartfee, {"conf_target", "estimate_mode"} }, { "hidden", "estimaterawfee", &estimaterawfee, {"conf_target", "threshold"} }, + { "hidden", "generate", &generate, {} }, }; // clang-format on - - for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) - t.appendCommand(commands[vcidx].name, &commands[vcidx]); + for (const auto& c : commands) { + t.appendCommand(c.name, &c); + } } diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 53d38f4e1..0c982317f 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -4,6 +4,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <httpserver.h> +#include <index/blockfilterindex.h> +#include <index/txindex.h> #include <interfaces/chain.h> #include <key_io.h> #include <node/context.h> @@ -27,9 +29,9 @@ #include <univalue.h> -static UniValue validateaddress(const JSONRPCRequest& request) +static RPCHelpMan validateaddress() { - RPCHelpMan{"validateaddress", + return RPCHelpMan{"validateaddress", "\nReturn information about the given bitcoin address.\n", { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to validate"}, @@ -50,8 +52,8 @@ static UniValue validateaddress(const JSONRPCRequest& request) HelpExampleCli("validateaddress", "\"" + EXAMPLE_ADDRESS[0] + "\"") + HelpExampleRpc("validateaddress", "\"" + EXAMPLE_ADDRESS[0] + "\"") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ CTxDestination dest = DecodeDestination(request.params[0].get_str()); bool isValid = IsValidDestination(dest); @@ -69,11 +71,13 @@ static UniValue validateaddress(const JSONRPCRequest& request) ret.pushKVs(detail); } return ret; +}, + }; } -static UniValue createmultisig(const JSONRPCRequest& request) +static RPCHelpMan createmultisig() { - RPCHelpMan{"createmultisig", + return RPCHelpMan{"createmultisig", "\nCreates a multi-signature address with n signature of m keys required.\n" "It returns a json object with the address and redeemScript.\n", { @@ -98,8 +102,8 @@ static UniValue createmultisig(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("createmultisig", "2, \"[\\\"03789ed0bb717d88f7d321a368d905e7430207ebbd82bd342cf11ae157a7ace5fd\\\",\\\"03dbc6764b8884a92e871274b87583e6d5c2a58819473e17e107ef3f6aa5a61626\\\"]\"") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ int required = request.params[0].get_int(); // Get the public keys @@ -135,11 +139,13 @@ static UniValue createmultisig(const JSONRPCRequest& request) result.pushKV("descriptor", descriptor->ToString()); return result; +}, + }; } -UniValue getdescriptorinfo(const JSONRPCRequest& request) +static RPCHelpMan getdescriptorinfo() { - RPCHelpMan{"getdescriptorinfo", + return RPCHelpMan{"getdescriptorinfo", {"\nAnalyses a descriptor.\n"}, { {"descriptor", RPCArg::Type::STR, RPCArg::Optional::NO, "The descriptor."}, @@ -157,8 +163,9 @@ UniValue getdescriptorinfo(const JSONRPCRequest& request) RPCExamples{ "Analyse a descriptor\n" + HelpExampleCli("getdescriptorinfo", "\"wpkh([d34db33f/84h/0h/0h]0279be667ef9dcbbac55a06295Ce870b07029Bfcdb2dce28d959f2815b16f81798)\"") - }}.Check(request); - + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ RPCTypeCheck(request.params, {UniValue::VSTR}); FlatSigningProvider provider; @@ -175,11 +182,13 @@ UniValue getdescriptorinfo(const JSONRPCRequest& request) result.pushKV("issolvable", desc->IsSolvable()); result.pushKV("hasprivatekeys", provider.keys.size() > 0); return result; +}, + }; } -UniValue deriveaddresses(const JSONRPCRequest& request) +static RPCHelpMan deriveaddresses() { - RPCHelpMan{"deriveaddresses", + return RPCHelpMan{"deriveaddresses", {"\nDerives one or more addresses corresponding to an output descriptor.\n" "Examples of output descriptors are:\n" " pkh(<pubkey>) P2PKH outputs for the given pubkey\n" @@ -202,8 +211,9 @@ UniValue deriveaddresses(const JSONRPCRequest& request) RPCExamples{ "First three native segwit receive addresses\n" + HelpExampleCli("deriveaddresses", "\"wpkh([d34db33f/84h/0h/0h]xpub6DJ2dNUysrn5Vt36jH2KLBT2i1auw1tTSSomg8PhqNiUtx8QX2SvC9nrHu81fT41fvDUnhMjEzQgXnQjKEu3oaqMSzhSrHMxyyoEAmUHQbY/0/*)#cjjspncu\" \"[0,2]\"") - }}.Check(request); - + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ RPCTypeCheck(request.params, {UniValue::VSTR, UniValueType()}); // Range argument is checked later const std::string desc_str = request.params[0].get_str(); @@ -254,11 +264,13 @@ UniValue deriveaddresses(const JSONRPCRequest& request) } return addresses; +}, + }; } -static UniValue verifymessage(const JSONRPCRequest& request) +static RPCHelpMan verifymessage() { - RPCHelpMan{"verifymessage", + return RPCHelpMan{"verifymessage", "\nVerify a signed message\n", { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to use for the signature."}, @@ -278,8 +290,8 @@ static UniValue verifymessage(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("verifymessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\", \"signature\", \"my message\"") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ LOCK(cs_main); std::string strAddress = request.params[0].get_str(); @@ -301,11 +313,13 @@ static UniValue verifymessage(const JSONRPCRequest& request) } return false; +}, + }; } -static UniValue signmessagewithprivkey(const JSONRPCRequest& request) +static RPCHelpMan signmessagewithprivkey() { - RPCHelpMan{"signmessagewithprivkey", + return RPCHelpMan{"signmessagewithprivkey", "\nSign a message with the private key of an address\n", { {"privkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The private key to sign the message with."}, @@ -322,8 +336,8 @@ static UniValue signmessagewithprivkey(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("signmessagewithprivkey", "\"privkey\", \"my message\"") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ std::string strPrivkey = request.params[0].get_str(); std::string strMessage = request.params[1].get_str(); @@ -339,11 +353,13 @@ static UniValue signmessagewithprivkey(const JSONRPCRequest& request) } return signature; +}, + }; } -static UniValue setmocktime(const JSONRPCRequest& request) +static RPCHelpMan setmocktime() { - RPCHelpMan{"setmocktime", + return RPCHelpMan{"setmocktime", "\nSet the local time to given timestamp (-regtest only)\n", { {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, UNIX_EPOCH_TIME + "\n" @@ -351,8 +367,8 @@ static UniValue setmocktime(const JSONRPCRequest& request) }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{""}, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ if (!Params().IsMockableChain()) { throw std::runtime_error("setmocktime is for regression testing (-regtest mode) only"); } @@ -374,19 +390,21 @@ static UniValue setmocktime(const JSONRPCRequest& request) } return NullUniValue; +}, + }; } -static UniValue mockscheduler(const JSONRPCRequest& request) +static RPCHelpMan mockscheduler() { - RPCHelpMan{"mockscheduler", + return RPCHelpMan{"mockscheduler", "\nBump the scheduler into the future (-regtest only)\n", { {"delta_time", RPCArg::Type::NUM, RPCArg::Optional::NO, "Number of seconds to forward the scheduler into the future." }, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{""}, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ if (!Params().IsMockableChain()) { throw std::runtime_error("mockscheduler is for regression testing (-regtest mode) only"); } @@ -405,6 +423,8 @@ static UniValue mockscheduler(const JSONRPCRequest& request) node.scheduler->MockForward(std::chrono::seconds(delta_seconds)); return NullUniValue; +}, + }; } static UniValue RPCLockedMemoryInfo() @@ -439,12 +459,12 @@ static std::string RPCMallocInfo() } #endif -static UniValue getmemoryinfo(const JSONRPCRequest& request) +static RPCHelpMan getmemoryinfo() { /* Please, avoid using the word "pool" here in the RPC interface or help, * as users will undoubtedly confuse it with the other "memory pool" */ - RPCHelpMan{"getmemoryinfo", + return RPCHelpMan{"getmemoryinfo", "Returns an object containing information about memory usage.\n", { {"mode", RPCArg::Type::STR, /* default */ "\"stats\"", "determines what kind of information is returned.\n" @@ -474,8 +494,8 @@ static UniValue getmemoryinfo(const JSONRPCRequest& request) HelpExampleCli("getmemoryinfo", "") + HelpExampleRpc("getmemoryinfo", "") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ std::string mode = request.params[0].isNull() ? "stats" : request.params[0].get_str(); if (mode == "stats") { UniValue obj(UniValue::VOBJ); @@ -490,6 +510,8 @@ static UniValue getmemoryinfo(const JSONRPCRequest& request) } else { throw JSONRPCError(RPC_INVALID_PARAMETER, "unknown mode " + mode); } +}, + }; } static void EnableOrDisableLogCategories(UniValue cats, bool enable) { @@ -510,9 +532,9 @@ static void EnableOrDisableLogCategories(UniValue cats, bool enable) { } } -UniValue logging(const JSONRPCRequest& request) +static RPCHelpMan logging() { - RPCHelpMan{"logging", + return RPCHelpMan{"logging", "Gets and sets the logging configuration.\n" "When called without an argument, returns the list of categories with status that are currently being debug logged or not.\n" "When called with arguments, adds or removes categories from debug logging and return the lists above.\n" @@ -543,8 +565,8 @@ UniValue logging(const JSONRPCRequest& request) HelpExampleCli("logging", "\"[\\\"all\\\"]\" \"[\\\"http\\\"]\"") + HelpExampleRpc("logging", "[\"all\"], [\"libevent\"]") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ uint32_t original_log_categories = LogInstance().GetCategoryMask(); if (request.params[0].isArray()) { EnableOrDisableLogCategories(request.params[0], true); @@ -575,26 +597,99 @@ UniValue logging(const JSONRPCRequest& request) } return result; +}, + }; } -static UniValue echo(const JSONRPCRequest& request) +static RPCHelpMan echo(const std::string& name) { - if (request.fHelp) - throw std::runtime_error( - RPCHelpMan{"echo|echojson ...", + return RPCHelpMan{name, "\nSimply echo back the input arguments. This command is for testing.\n" - "\nIt will return an internal bug report when exactly 100 arguments are passed.\n" + "\nIt will return an internal bug report when arg9='trigger_internal_bug' is passed.\n" "\nThe difference between echo and echojson is that echojson has argument conversion enabled in the client-side table in " "bitcoin-cli and the GUI. There is no server-side difference.", - {}, + { + {"arg0", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg1", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg2", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg3", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg4", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg5", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg6", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg7", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg8", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + {"arg9", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, ""}, + }, RPCResult{RPCResult::Type::NONE, "", "Returns whatever was passed in"}, RPCExamples{""}, - }.ToString() - ); + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + if (request.fHelp) throw std::runtime_error(self.ToString()); - CHECK_NONFATAL(request.params.size() != 100); + if (request.params[9].isStr()) { + CHECK_NONFATAL(request.params[9].get_str() != "trigger_internal_bug"); + } return request.params; +}, + }; +} + +static RPCHelpMan echo() { return echo("echo"); } +static RPCHelpMan echojson() { return echo("echojson"); } + +static UniValue SummaryToJSON(const IndexSummary&& summary, std::string index_name) +{ + UniValue ret_summary(UniValue::VOBJ); + if (!index_name.empty() && index_name != summary.name) return ret_summary; + + UniValue entry(UniValue::VOBJ); + entry.pushKV("synced", summary.synced); + entry.pushKV("best_block_height", summary.best_block_height); + ret_summary.pushKV(summary.name, entry); + return ret_summary; +} + +static RPCHelpMan getindexinfo() +{ + return RPCHelpMan{"getindexinfo", + "\nReturns the status of one or all available indices currently running in the node.\n", + { + {"index_name", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Filter results for an index with a specific name."}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", { + { + RPCResult::Type::OBJ, "name", "The name of the index", + { + {RPCResult::Type::BOOL, "synced", "Whether the index is synced or not"}, + {RPCResult::Type::NUM, "best_block_height", "The block height to which the index is synced"}, + } + }, + }, + }, + RPCExamples{ + HelpExampleCli("getindexinfo", "") + + HelpExampleRpc("getindexinfo", "") + + HelpExampleCli("getindexinfo", "txindex") + + HelpExampleRpc("getindexinfo", "txindex") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + UniValue result(UniValue::VOBJ); + const std::string index_name = request.params[0].isNull() ? "" : request.params[0].get_str(); + + if (g_txindex) { + result.pushKVs(SummaryToJSON(g_txindex->GetSummary(), index_name)); + } + + ForEachBlockFilterIndex([&result, &index_name](const BlockFilterIndex& index) { + result.pushKVs(SummaryToJSON(index.GetSummary(), index_name)); + }); + + return result; +}, + }; } void RegisterMiscRPCCommands(CRPCTable &t) @@ -611,15 +706,16 @@ static const CRPCCommand commands[] = { "util", "getdescriptorinfo", &getdescriptorinfo, {"descriptor"} }, { "util", "verifymessage", &verifymessage, {"address","signature","message"} }, { "util", "signmessagewithprivkey", &signmessagewithprivkey, {"privkey","message"} }, + { "util", "getindexinfo", &getindexinfo, {"index_name"} }, /* Not shown in help */ { "hidden", "setmocktime", &setmocktime, {"timestamp"}}, { "hidden", "mockscheduler", &mockscheduler, {"delta_time"}}, { "hidden", "echo", &echo, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, - { "hidden", "echojson", &echo, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, + { "hidden", "echojson", &echojson, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, }; // clang-format on - - for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) - t.appendCommand(commands[vcidx].name, &commands[vcidx]); + for (const auto& c : commands) { + t.appendCommand(c.name, &c); + } } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 9981ea35d..5af438985 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -100,6 +100,8 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) {RPCResult::Type::BOOL, "relaytxes", "Whether peer has asked us to relay transactions to it"}, {RPCResult::Type::NUM_TIME, "lastsend", "The " + UNIX_EPOCH_TIME + " of the last send"}, {RPCResult::Type::NUM_TIME, "lastrecv", "The " + UNIX_EPOCH_TIME + " of the last receive"}, + {RPCResult::Type::NUM_TIME, "last_transaction", "The " + UNIX_EPOCH_TIME + " of the last valid transaction received from this peer"}, + {RPCResult::Type::NUM_TIME, "last_block", "The " + UNIX_EPOCH_TIME + " of the last block received from this peer"}, {RPCResult::Type::NUM, "bytessent", "The total bytes sent"}, {RPCResult::Type::NUM, "bytesrecv", "The total bytes received"}, {RPCResult::Type::NUM_TIME, "conntime", "The " + UNIX_EPOCH_TIME + " of the connection"}, @@ -169,6 +171,8 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) obj.pushKV("relaytxes", stats.fRelayTxes); obj.pushKV("lastsend", stats.nLastSend); obj.pushKV("lastrecv", stats.nLastRecv); + obj.pushKV("last_transaction", stats.nLastTXTime); + obj.pushKV("last_block", stats.nLastBlockTime); obj.pushKV("bytessent", stats.nSendBytes); obj.pushKV("bytesrecv", stats.nRecvBytes); obj.pushKV("conntime", stats.nTimeConnected); @@ -193,7 +197,7 @@ static UniValue getpeerinfo(const JSONRPCRequest& request) if (fStateStats) { if (IsDeprecatedRPCEnabled("banscore")) { // banscore is deprecated in v0.21 for removal in v0.22 - obj.pushKV("banscore", statestats.nMisbehavior); + obj.pushKV("banscore", statestats.m_misbehavior_score); } obj.pushKV("synced_headers", statestats.nSyncHeight); obj.pushKV("synced_blocks", statestats.nCommonHeight); @@ -264,7 +268,7 @@ static UniValue addnode(const JSONRPCRequest& request) if (strCommand == "onetry") { CAddress addr; - node.connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), false, false, true); + node.connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), ConnectionType::MANUAL); return NullUniValue; } @@ -276,7 +280,7 @@ static UniValue addnode(const JSONRPCRequest& request) else if(strCommand == "remove") { if(!node.connman->RemoveAddedNode(strNode)) - throw JSONRPCError(RPC_CLIENT_NODE_NOT_ADDED, "Error: Node has not been added."); + throw JSONRPCError(RPC_CLIENT_NODE_NOT_ADDED, "Error: Node could not be removed. It has not been added previously."); } return NullUniValue; @@ -410,7 +414,7 @@ static UniValue getnettotals(const JSONRPCRequest& request) { {RPCResult::Type::NUM, "totalbytesrecv", "Total bytes received"}, {RPCResult::Type::NUM, "totalbytessent", "Total bytes sent"}, - {RPCResult::Type::NUM_TIME, "timemillis", "Current UNIX time in milliseconds"}, + {RPCResult::Type::NUM_TIME, "timemillis", "Current " + UNIX_EPOCH_TIME + " in milliseconds"}, {RPCResult::Type::OBJ, "uploadtarget", "", { {RPCResult::Type::NUM, "timeframe", "Length of the measuring timeframe in seconds"}, @@ -486,7 +490,9 @@ static UniValue getnetworkinfo(const JSONRPCRequest& request) }}, {RPCResult::Type::BOOL, "localrelay", "true if transaction relay is requested from peers"}, {RPCResult::Type::NUM, "timeoffset", "the time offset"}, - {RPCResult::Type::NUM, "connections", "the number of connections"}, + {RPCResult::Type::NUM, "connections", "the total number of connections"}, + {RPCResult::Type::NUM, "connections_in", "the number of inbound connections"}, + {RPCResult::Type::NUM, "connections_out", "the number of outbound connections"}, {RPCResult::Type::BOOL, "networkactive", "whether p2p networking is enabled"}, {RPCResult::Type::ARR, "networks", "information per network", { @@ -534,7 +540,9 @@ static UniValue getnetworkinfo(const JSONRPCRequest& request) obj.pushKV("timeoffset", GetTimeOffset()); if (node.connman) { obj.pushKV("networkactive", node.connman->GetNetworkActive()); - obj.pushKV("connections", (int)node.connman->GetNodeCount(CConnman::CONNECTIONS_ALL)); + obj.pushKV("connections", (int)node.connman->GetNodeCount(CConnman::CONNECTIONS_ALL)); + obj.pushKV("connections_in", (int)node.connman->GetNodeCount(CConnman::CONNECTIONS_IN)); + obj.pushKV("connections_out", (int)node.connman->GetNodeCount(CConnman::CONNECTIONS_OUT)); } obj.pushKV("networks", GetNetworksInfo()); obj.pushKV("relayfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); @@ -727,7 +735,7 @@ static UniValue getnodeaddresses(const JSONRPCRequest& request) RPCHelpMan{"getnodeaddresses", "\nReturn known addresses which can potentially be used to find new nodes in the network\n", { - {"count", RPCArg::Type::NUM, /* default */ "1", "How many addresses to return. Limited to the smaller of " + ToString(ADDRMAN_GETADDR_MAX) + " or " + ToString(ADDRMAN_GETADDR_MAX_PCT) + "% of all known addresses."}, + {"count", RPCArg::Type::NUM, /* default */ "1", "The maximum number of addresses to return. Specify 0 to return all known addresses."}, }, RPCResult{ RPCResult::Type::ARR, "", "", @@ -754,18 +762,16 @@ static UniValue getnodeaddresses(const JSONRPCRequest& request) int count = 1; if (!request.params[0].isNull()) { count = request.params[0].get_int(); - if (count <= 0) { + if (count < 0) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Address count out of range"); } } // returns a shuffled list of CAddress - std::vector<CAddress> vAddr = node.connman->GetAddresses(); + std::vector<CAddress> vAddr = node.connman->GetAddresses(count, /* max_pct */ 0); UniValue ret(UniValue::VARR); - int address_return_count = std::min<int>(count, vAddr.size()); - for (int i = 0; i < address_return_count; ++i) { + for (const CAddress& addr : vAddr) { UniValue obj(UniValue::VOBJ); - const CAddress& addr = vAddr[i]; obj.pushKV("time", (int)addr.nTime); obj.pushKV("services", (uint64_t)addr.nServices); obj.pushKV("address", addr.ToStringIP()); @@ -775,6 +781,54 @@ static UniValue getnodeaddresses(const JSONRPCRequest& request) return ret; } +static UniValue addpeeraddress(const JSONRPCRequest& request) +{ + RPCHelpMan{"addpeeraddress", + "\nAdd the address of a potential peer to the address manager. This RPC is for testing only.\n", + { + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The IP address of the peer"}, + {"port", RPCArg::Type::NUM, RPCArg::Optional::NO, "The port of the peer"}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::BOOL, "success", "whether the peer address was successfully added to the address manager"}, + }, + }, + RPCExamples{ + HelpExampleCli("addpeeraddress", "\"1.2.3.4\" 8333") + + HelpExampleRpc("addpeeraddress", "\"1.2.3.4\", 8333") + }, + }.Check(request); + + NodeContext& node = EnsureNodeContext(request.context); + if (!node.connman) { + throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); + } + + UniValue obj(UniValue::VOBJ); + + std::string addr_string = request.params[0].get_str(); + uint16_t port = request.params[1].get_int(); + + CNetAddr net_addr; + if (!LookupHost(addr_string, net_addr, false)) { + obj.pushKV("success", false); + return obj; + } + CAddress address = CAddress({net_addr, port}, ServiceFlags(NODE_NETWORK|NODE_WITNESS)); + address.nTime = GetAdjustedTime(); + // The source address is set equal to the address. This is equivalent to the peer + // announcing itself. + if (!node.connman->AddNewAddresses({address}, address)) { + obj.pushKV("success", false); + return obj; + } + + obj.pushKV("success", true); + return obj; +} + void RegisterNetRPCCommands(CRPCTable &t) { // clang-format off @@ -794,9 +848,10 @@ static const CRPCCommand commands[] = { "network", "clearbanned", &clearbanned, {} }, { "network", "setnetworkactive", &setnetworkactive, {"state"} }, { "network", "getnodeaddresses", &getnodeaddresses, {"count"} }, + { "hidden", "addpeeraddress", &addpeeraddress, {"address", "port"} }, }; // clang-format on - - for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) - t.appendCommand(commands[vcidx].name, &commands[vcidx]); + for (const auto& c : commands) { + t.appendCommand(c.name, &c); + } } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index cf856af6e..1a43ffcc5 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -191,7 +191,7 @@ static UniValue getrawtransaction(const JSONRPCRequest& request) } uint256 hash_block; - const CTransactionRef tx = GetTransaction(blockindex, node.mempool, hash, Params().GetConsensus(), hash_block); + const CTransactionRef tx = GetTransaction(blockindex, node.mempool.get(), hash, Params().GetConsensus(), hash_block); if (!tx) { std::string errmsg; if (blockindex) { @@ -627,7 +627,7 @@ static UniValue combinerawtransaction(const JSONRPCRequest& request) { {"txs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The hex strings of partially signed transactions", { - {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A transaction hash"}, + {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A hex-encoded raw transaction"}, }, }, }, @@ -744,7 +744,7 @@ static UniValue signrawtransactionwithkey(const JSONRPCRequest& request) { {RPCResult::Type::STR_HEX, "hex", "The hex-encoded raw transaction with signature(s)"}, {RPCResult::Type::BOOL, "complete", "If the transaction has a complete set of signatures"}, - {RPCResult::Type::ARR, "errors", "Script verification errors (if there are any)", + {RPCResult::Type::ARR, "errors", /* optional */ true, "Script verification errors (if there are any)", { {RPCResult::Type::OBJ, "", "", { @@ -878,6 +878,11 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) { {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, {RPCResult::Type::BOOL, "allowed", "If the mempool allows this tx to be inserted"}, + {RPCResult::Type::NUM, "vsize", "Virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted (only present when 'allowed' is true)"}, + {RPCResult::Type::OBJ, "fees", "Transaction fees (only present if 'allowed' is true)", + { + {RPCResult::Type::STR_AMOUNT, "base", "transaction fee in " + CURRENCY_UNIT}, + }}, {RPCResult::Type::STR, "reject-reason", "Rejection string (only present when 'allowed' is false)"}, }}, } @@ -924,13 +929,22 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) TxValidationState state; bool test_accept_res; + CAmount fee; { LOCK(cs_main); test_accept_res = AcceptToMemoryPool(mempool, state, std::move(tx), - nullptr /* plTxnReplaced */, false /* bypass_limits */, max_raw_tx_fee, /* test_accept */ true); + nullptr /* plTxnReplaced */, false /* bypass_limits */, max_raw_tx_fee, /* test_accept */ true, &fee); } result_0.pushKV("allowed", test_accept_res); - if (!test_accept_res) { + + // Only return the fee and vsize if the transaction would pass ATMP. + // These can be used to calculate the feerate. + if (test_accept_res) { + result_0.pushKV("vsize", virtual_size); + UniValue fees(UniValue::VOBJ); + fees.pushKV("base", ValueFromAmount(fee)); + result_0.pushKV("fees", fees); + } else { if (state.IsInvalid()) { if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { result_0.pushKV("reject-reason", "missing-inputs"); @@ -1300,7 +1314,7 @@ UniValue combinepsbt(const JSONRPCRequest& request) CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << merged_psbt; - return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); + return EncodeBase64(MakeUCharSpan(ssTx)); } UniValue finalizepsbt(const JSONRPCRequest& request) @@ -1348,7 +1362,7 @@ UniValue finalizepsbt(const JSONRPCRequest& request) if (complete && extract) { ssTx << mtx; - result_str = HexStr(ssTx.str()); + result_str = HexStr(ssTx); result.pushKV("hex", result_str); } else { ssTx << psbtx; @@ -1435,7 +1449,7 @@ UniValue createpsbt(const JSONRPCRequest& request) CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); + return EncodeBase64(MakeUCharSpan(ssTx)); } UniValue converttopsbt(const JSONRPCRequest& request) @@ -1502,7 +1516,7 @@ UniValue converttopsbt(const JSONRPCRequest& request) CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); + return EncodeBase64(MakeUCharSpan(ssTx)); } UniValue utxoupdatepsbt(const JSONRPCRequest& request) @@ -1590,7 +1604,7 @@ UniValue utxoupdatepsbt(const JSONRPCRequest& request) CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); + return EncodeBase64(MakeUCharSpan(ssTx)); } UniValue joinpsbts(const JSONRPCRequest& request) @@ -1683,7 +1697,7 @@ UniValue joinpsbts(const JSONRPCRequest& request) CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << shuffled_psbt; - return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); + return EncodeBase64(MakeUCharSpan(ssTx)); } UniValue analyzepsbt(const JSONRPCRequest& request) @@ -1722,7 +1736,7 @@ UniValue analyzepsbt(const JSONRPCRequest& request) {RPCResult::Type::STR_AMOUNT, "estimated_feerate", /* optional */ true, "Estimated feerate of the final signed transaction in " + CURRENCY_UNIT + "/kB. Shown only if all UTXO slots in the PSBT have been filled"}, {RPCResult::Type::STR_AMOUNT, "fee", /* optional */ true, "The transaction fee paid. Shown only if all UTXO slots in the PSBT have been filled"}, {RPCResult::Type::STR, "next", "Role of the next person that this psbt needs to go to"}, - {RPCResult::Type::STR, "error", "Error message if there is one"}, + {RPCResult::Type::STR, "error", /* optional */ true, "Error message (if there is one)"}, } }, RPCExamples { @@ -1821,7 +1835,7 @@ static const CRPCCommand commands[] = { "blockchain", "verifytxoutproof", &verifytxoutproof, {"proof"} }, }; // clang-format on - - for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++) - t.appendCommand(commands[vcidx].name, &commands[vcidx]); + for (const auto& c : commands) { + t.appendCommand(c.name, &c); + } } diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index 1031716b4..cfe457509 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -21,10 +21,15 @@ CMutableTransaction ConstructTransaction(const UniValue& inputs_in, const UniValue& outputs_in, const UniValue& locktime, bool rbf) { - if (inputs_in.isNull() || outputs_in.isNull()) - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, arguments 1 and 2 must be non-null"); + if (outputs_in.isNull()) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, output argument must be non-null"); + + UniValue inputs; + if (inputs_in.isNull()) + inputs = UniValue::VARR; + else + inputs = inputs_in.get_array(); - UniValue inputs = inputs_in.get_array(); const bool outputs_is_obj = outputs_in.isObject(); UniValue outputs = outputs_is_obj ? outputs_in.get_obj() : outputs_in.get_array(); diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index 7fef45f50..d9ad70fa3 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -78,7 +78,7 @@ bool GenerateAuthCookie(std::string *cookie_out) const size_t COOKIE_SIZE = 32; unsigned char rand_pwd[COOKIE_SIZE]; GetRandBytes(rand_pwd, COOKIE_SIZE); - std::string cookie = COOKIEAUTH_USER + ":" + HexStr(rand_pwd, rand_pwd+COOKIE_SIZE); + std::string cookie = COOKIEAUTH_USER + ":" + HexStr(rand_pwd); /** the umask determines what permissions are used to create this file - * these are set to 077 in init.cpp unless overridden with -sysperms. diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index e5f6b1b9f..f32d9abac 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -256,23 +256,16 @@ static const CRPCCommand vRPCCommands[] = CRPCTable::CRPCTable() { - unsigned int vcidx; - for (vcidx = 0; vcidx < (sizeof(vRPCCommands) / sizeof(vRPCCommands[0])); vcidx++) - { - const CRPCCommand *pcmd; - - pcmd = &vRPCCommands[vcidx]; - mapCommands[pcmd->name].push_back(pcmd); + for (const auto& c : vRPCCommands) { + appendCommand(c.name, &c); } } -bool CRPCTable::appendCommand(const std::string& name, const CRPCCommand* pcmd) +void CRPCTable::appendCommand(const std::string& name, const CRPCCommand* pcmd) { - if (IsRPCRunning()) - return false; + CHECK_NONFATAL(!IsRPCRunning()); // Only add commands before rpc is running mapCommands[name].push_back(pcmd); - return true; } bool CRPCTable::removeCommand(const std::string& name, const CRPCCommand* pcmd) diff --git a/src/rpc/server.h b/src/rpc/server.h index 6da3e94ea..b2358ac5b 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -160,7 +160,7 @@ public: /** * Appends a CRPCCommand to the dispatch table. * - * Returns false if RPC server is already running (dump concurrency protection). + * Precondition: RPC server is not running * * Commands with different method names but the same unique_id will * be considered aliases, and only the first registered method name will @@ -169,7 +169,7 @@ public: * between calls based on method name, and aliased commands can also * register different names, types, and numbers of parameters. */ - bool appendCommand(const std::string& name, const CRPCCommand* pcmd); + void appendCommand(const std::string& name, const CRPCCommand* pcmd); bool removeCommand(const std::string& name, const CRPCCommand* pcmd); }; diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 9f4c7bee9..40dfdb587 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -260,7 +260,7 @@ public: UniValue obj(UniValue::VOBJ); obj.pushKV("iswitness", true); obj.pushKV("witness_version", (int)id.version); - obj.pushKV("witness_program", HexStr(id.program, id.program + id.length)); + obj.pushKV("witness_program", HexStr(Span<const unsigned char>(id.program, id.length))); return obj; } }; @@ -504,7 +504,7 @@ std::string RPCHelpMan::ToString() const ret += m_name; bool was_optional{false}; for (const auto& arg : m_args) { - if (arg.m_hidden) continue; + if (arg.m_hidden) break; // Any arg that follows is also hidden const bool optional = arg.IsOptional(); ret += " "; if (optional) { @@ -526,7 +526,7 @@ std::string RPCHelpMan::ToString() const Sections sections; for (size_t i{0}; i < m_args.size(); ++i) { const auto& arg = m_args.at(i); - if (arg.m_hidden) continue; + if (arg.m_hidden) break; // Any arg that follows is also hidden if (i == 0) ret += "\nArguments:\n"; diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 9978d084d..6c0a98cca 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -190,7 +190,7 @@ class OriginPubkeyProvider final : public PubkeyProvider std::string OriginString() const { - return HexStr(std::begin(m_origin.fingerprint), std::end(m_origin.fingerprint)) + FormatHDKeypath(m_origin.path); + return HexStr(m_origin.fingerprint) + FormatHDKeypath(m_origin.path); } public: diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 39feb4ccc..7b2457a5e 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1258,34 +1258,37 @@ public: } }; +/** Compute the (single) SHA256 of the concatenation of all prevouts of a tx. */ template <class T> -uint256 GetPrevoutHash(const T& txTo) +uint256 GetPrevoutsSHA256(const T& txTo) { CHashWriter ss(SER_GETHASH, 0); for (const auto& txin : txTo.vin) { ss << txin.prevout; } - return ss.GetHash(); + return ss.GetSHA256(); } +/** Compute the (single) SHA256 of the concatenation of all nSequences of a tx. */ template <class T> -uint256 GetSequenceHash(const T& txTo) +uint256 GetSequencesSHA256(const T& txTo) { CHashWriter ss(SER_GETHASH, 0); for (const auto& txin : txTo.vin) { ss << txin.nSequence; } - return ss.GetHash(); + return ss.GetSHA256(); } +/** Compute the (single) SHA256 of the concatenation of all txouts of a tx. */ template <class T> -uint256 GetOutputsHash(const T& txTo) +uint256 GetOutputsSHA256(const T& txTo) { CHashWriter ss(SER_GETHASH, 0); for (const auto& txout : txTo.vout) { ss << txout; } - return ss.GetHash(); + return ss.GetSHA256(); } } // namespace @@ -1297,9 +1300,9 @@ void PrecomputedTransactionData::Init(const T& txTo) // Cache is calculated only for transactions with witness if (txTo.HasWitness()) { - hashPrevouts = GetPrevoutHash(txTo); - hashSequence = GetSequenceHash(txTo); - hashOutputs = GetOutputsHash(txTo); + hashPrevouts = SHA256Uint256(GetPrevoutsSHA256(txTo)); + hashSequence = SHA256Uint256(GetSequencesSHA256(txTo)); + hashOutputs = SHA256Uint256(GetOutputsSHA256(txTo)); } m_ready = true; @@ -1329,16 +1332,16 @@ uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn const bool cacheready = cache && cache->m_ready; if (!(nHashType & SIGHASH_ANYONECANPAY)) { - hashPrevouts = cacheready ? cache->hashPrevouts : GetPrevoutHash(txTo); + hashPrevouts = cacheready ? cache->hashPrevouts : SHA256Uint256(GetPrevoutsSHA256(txTo)); } if (!(nHashType & SIGHASH_ANYONECANPAY) && (nHashType & 0x1f) != SIGHASH_SINGLE && (nHashType & 0x1f) != SIGHASH_NONE) { - hashSequence = cacheready ? cache->hashSequence : GetSequenceHash(txTo); + hashSequence = cacheready ? cache->hashSequence : SHA256Uint256(GetSequencesSHA256(txTo)); } if ((nHashType & 0x1f) != SIGHASH_SINGLE && (nHashType & 0x1f) != SIGHASH_NONE) { - hashOutputs = cacheready ? cache->hashOutputs : GetOutputsHash(txTo); + hashOutputs = cacheready ? cache->hashOutputs : SHA256Uint256(GetOutputsSHA256(txTo)); } else if ((nHashType & 0x1f) == SIGHASH_SINGLE && nIn < txTo.vout.size()) { CHashWriter ss(SER_GETHASH, 0); ss << txTo.vout[nIn]; diff --git a/src/script/sign.cpp b/src/script/sign.cpp index f42521554..9b3f94f14 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -186,6 +186,8 @@ static CScript PushAll(const std::vector<valtype>& values) result << OP_0; } else if (v.size() == 1 && v[0] >= 1 && v[0] <= 16) { result << CScript::EncodeOP_N(v[0]); + } else if (v.size() == 1 && v[0] == 0x81) { + result << OP_1NEGATE; } else { result << v; } diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 3a4882f28..96a3d311a 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -313,18 +313,6 @@ CScript GetScriptForMultisig(int nRequired, const std::vector<CPubKey>& keys) return script; } -CScript GetScriptForWitness(const CScript& redeemscript) -{ - std::vector<std::vector<unsigned char> > vSolutions; - TxoutType typ = Solver(redeemscript, vSolutions); - if (typ == TxoutType::PUBKEY) { - return GetScriptForDestination(WitnessV0KeyHash(Hash160(vSolutions[0]))); - } else if (typ == TxoutType::PUBKEYHASH) { - return GetScriptForDestination(WitnessV0KeyHash(uint160{vSolutions[0]})); - } - return GetScriptForDestination(WitnessV0ScriptHash(redeemscript)); -} - bool IsValidDestination(const CTxDestination& dest) { return dest.which() != 0; } diff --git a/src/script/standard.h b/src/script/standard.h index 992e37675..6dbcd0496 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -263,14 +263,4 @@ CScript GetScriptForRawPubKey(const CPubKey& pubkey); /** Generate a multisig script. */ CScript GetScriptForMultisig(int nRequired, const std::vector<CPubKey>& keys); -/** - * Generate a pay-to-witness script for the given redeem script. If the redeem - * script is P2PK or P2PKH, this returns a P2WPKH script, otherwise it returns a - * P2WSH script. - * - * TODO: replace calls to GetScriptForWitness with GetScriptForDestination using - * the various witness-specific CTxDestination subtypes. - */ -CScript GetScriptForWitness(const CScript& redeemscript); - #endif // BITCOIN_SCRIPT_STANDARD_H diff --git a/src/secp256k1/.gitignore b/src/secp256k1/.gitignore index cb4331aa9..ccdef02b2 100644 --- a/src/secp256k1/.gitignore +++ b/src/secp256k1/.gitignore @@ -1,9 +1,9 @@ bench_inv bench_ecdh bench_ecmult +bench_schnorrsig bench_sign bench_verify -bench_schnorr_verify bench_recover bench_internal tests @@ -31,6 +31,8 @@ libtool *.lo *.o *~ +*.log +*.trs src/libsecp256k1-config.h src/libsecp256k1-config.h.in src/ecmult_static_context.h diff --git a/src/secp256k1/.travis.yml b/src/secp256k1/.travis.yml index a6ad6fb27..e1a88c405 100644 --- a/src/secp256k1/.travis.yml +++ b/src/secp256k1/.travis.yml @@ -17,19 +17,19 @@ compiler: - gcc env: global: - - FIELD=auto BIGNUM=auto SCALAR=auto ENDOMORPHISM=no STATICPRECOMPUTATION=yes ECMULTGENPRECISION=auto ASM=no BUILD=check EXTRAFLAGS= HOST= ECDH=no RECOVERY=no EXPERIMENTAL=no CTIMETEST=yes BENCH=yes ITERS=2 + - WIDEMUL=auto BIGNUM=auto ENDOMORPHISM=no STATICPRECOMPUTATION=yes ECMULTGENPRECISION=auto ASM=no BUILD=check EXTRAFLAGS= HOST= ECDH=no RECOVERY=no SCHNORRSIG=no EXPERIMENTAL=no CTIMETEST=yes BENCH=yes ITERS=2 matrix: - - SCALAR=32bit RECOVERY=yes - - SCALAR=32bit FIELD=32bit ECDH=yes EXPERIMENTAL=yes - - SCALAR=64bit - - FIELD=64bit RECOVERY=yes - - FIELD=64bit ENDOMORPHISM=yes - - FIELD=64bit ENDOMORPHISM=yes ECDH=yes EXPERIMENTAL=yes - - FIELD=64bit ASM=x86_64 - - FIELD=64bit ENDOMORPHISM=yes ASM=x86_64 - - FIELD=32bit ENDOMORPHISM=yes + - WIDEMUL=int64 RECOVERY=yes + - WIDEMUL=int64 ECDH=yes EXPERIMENTAL=yes SCHNORRSIG=yes + - WIDEMUL=int64 ENDOMORPHISM=yes + - WIDEMUL=int128 + - WIDEMUL=int128 RECOVERY=yes EXPERIMENTAL=yes SCHNORRSIG=yes + - WIDEMUL=int128 ENDOMORPHISM=yes + - WIDEMUL=int128 ENDOMORPHISM=yes ECDH=yes EXPERIMENTAL=yes SCHNORRSIG=yes + - WIDEMUL=int128 ASM=x86_64 + - WIDEMUL=int128 ENDOMORPHISM=yes ASM=x86_64 - BIGNUM=no - - BIGNUM=no ENDOMORPHISM=yes RECOVERY=yes EXPERIMENTAL=yes + - BIGNUM=no ENDOMORPHISM=yes RECOVERY=yes EXPERIMENTAL=yes SCHNORRSIG=yes - BIGNUM=no STATICPRECOMPUTATION=no - BUILD=distcheck CTIMETEST= BENCH= - CPPFLAGS=-DDETERMINISTIC @@ -83,6 +83,10 @@ matrix: - valgrind - libtool-bin - libc6-dbg:i386 + # S390x build (big endian system) + - compiler: gcc + env: HOST=s390x-unknown-linux-gnu ECDH=yes RECOVERY=yes EXPERIMENTAL=yes CTIMETEST= + arch: s390x # We use this to install macOS dependencies instead of the built in `homebrew` plugin, # because in xcode earlier than 11 they have a bug requiring updating the system which overall takes ~8 minutes. diff --git a/src/secp256k1/Makefile.am b/src/secp256k1/Makefile.am index d8c1c79e8..023fa6067 100644 --- a/src/secp256k1/Makefile.am +++ b/src/secp256k1/Makefile.am @@ -34,9 +34,11 @@ noinst_HEADERS += src/field_5x52.h noinst_HEADERS += src/field_5x52_impl.h noinst_HEADERS += src/field_5x52_int128_impl.h noinst_HEADERS += src/field_5x52_asm_impl.h +noinst_HEADERS += src/assumptions.h noinst_HEADERS += src/util.h noinst_HEADERS += src/scratch.h noinst_HEADERS += src/scratch_impl.h +noinst_HEADERS += src/selftest.h noinst_HEADERS += src/testrand.h noinst_HEADERS += src/testrand_impl.h noinst_HEADERS += src/hash.h @@ -99,7 +101,7 @@ if VALGRIND_ENABLED tests_CPPFLAGS += -DVALGRIND noinst_PROGRAMS += valgrind_ctime_test valgrind_ctime_test_SOURCES = src/valgrind_ctime_test.c -valgrind_ctime_test_LDADD = libsecp256k1.la $(SECP_LIBS) $(SECP_TEST_LIBS) $(COMMON_LIB) +valgrind_ctime_test_LDADD = libsecp256k1.la $(SECP_LIBS) $(SECP_LIBS) $(COMMON_LIB) endif if !ENABLE_COVERAGE tests_CPPFLAGS += -DVERIFY @@ -152,3 +154,11 @@ endif if ENABLE_MODULE_RECOVERY include src/modules/recovery/Makefile.am.include endif + +if ENABLE_MODULE_EXTRAKEYS +include src/modules/extrakeys/Makefile.am.include +endif + +if ENABLE_MODULE_SCHNORRSIG +include src/modules/schnorrsig/Makefile.am.include +endif diff --git a/src/secp256k1/TODO b/src/secp256k1/TODO deleted file mode 100644 index a300e1c5e..000000000 --- a/src/secp256k1/TODO +++ /dev/null @@ -1,3 +0,0 @@ -* Unit tests for fieldelem/groupelem, including ones intended to - trigger fieldelem's boundary cases. -* Complete constant-time operations for signing/keygen diff --git a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 index 1b2b71e6a..57595f449 100644 --- a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 +++ b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 @@ -1,8 +1,3 @@ -dnl libsecp25k1 helper checks -AC_DEFUN([SECP_INT128_CHECK],[ -has_int128=$ac_cv_type___int128 -]) - dnl escape "$0x" below using the m4 quadrigaph @S|@, and escape it again with a \ for the shell. AC_DEFUN([SECP_64BIT_ASM_CHECK],[ AC_MSG_CHECKING(for x86_64 assembly availability) diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index 6021b760b..6fe8984f4 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -136,20 +136,28 @@ AC_ARG_ENABLE(module_recovery, [enable_module_recovery=$enableval], [enable_module_recovery=no]) +AC_ARG_ENABLE(module_extrakeys, + AS_HELP_STRING([--enable-module-extrakeys],[enable extrakeys module (experimental)]), + [enable_module_extrakeys=$enableval], + [enable_module_extrakeys=no]) + +AC_ARG_ENABLE(module_schnorrsig, + AS_HELP_STRING([--enable-module-schnorrsig],[enable schnorrsig module (experimental)]), + [enable_module_schnorrsig=$enableval], + [enable_module_schnorrsig=no]) + AC_ARG_ENABLE(external_default_callbacks, AS_HELP_STRING([--enable-external-default-callbacks],[enable external default callback functions [default=no]]), [use_external_default_callbacks=$enableval], [use_external_default_callbacks=no]) -AC_ARG_WITH([field], [AS_HELP_STRING([--with-field=64bit|32bit|auto], -[finite field implementation to use [default=auto]])],[req_field=$withval], [req_field=auto]) +dnl Test-only override of the (autodetected by the C code) "widemul" setting. +dnl Legal values are int64 (for [u]int64_t), int128 (for [unsigned] __int128), and auto (the default). +AC_ARG_WITH([test-override-wide-multiply], [] ,[set_widemul=$withval], [set_widemul=auto]) AC_ARG_WITH([bignum], [AS_HELP_STRING([--with-bignum=gmp|no|auto], [bignum implementation to use [default=auto]])],[req_bignum=$withval], [req_bignum=auto]) -AC_ARG_WITH([scalar], [AS_HELP_STRING([--with-scalar=64bit|32bit|auto], -[scalar implementation to use [default=auto]])],[req_scalar=$withval], [req_scalar=auto]) - AC_ARG_WITH([asm], [AS_HELP_STRING([--with-asm=x86_64|arm|no|auto], [assembly optimizations to use (experimental: arm) [default=auto]])],[req_asm=$withval], [req_asm=auto]) @@ -170,8 +178,6 @@ AC_ARG_WITH([ecmult-gen-precision], [AS_HELP_STRING([--with-ecmult-gen-precision )], [req_ecmult_gen_precision=$withval], [req_ecmult_gen_precision=auto]) -AC_CHECK_TYPES([__int128]) - AC_CHECK_HEADER([valgrind/memcheck.h], [enable_valgrind=yes], [enable_valgrind=no], []) AM_CONDITIONAL([VALGRIND_ENABLED],[test "$enable_valgrind" = "yes"]) @@ -265,63 +271,6 @@ else esac fi -if test x"$req_field" = x"auto"; then - if test x"set_asm" = x"x86_64"; then - set_field=64bit - fi - if test x"$set_field" = x; then - SECP_INT128_CHECK - if test x"$has_int128" = x"yes"; then - set_field=64bit - fi - fi - if test x"$set_field" = x; then - set_field=32bit - fi -else - set_field=$req_field - case $set_field in - 64bit) - if test x"$set_asm" != x"x86_64"; then - SECP_INT128_CHECK - if test x"$has_int128" != x"yes"; then - AC_MSG_ERROR([64bit field explicitly requested but neither __int128 support or x86_64 assembly available]) - fi - fi - ;; - 32bit) - ;; - *) - AC_MSG_ERROR([invalid field implementation selection]) - ;; - esac -fi - -if test x"$req_scalar" = x"auto"; then - SECP_INT128_CHECK - if test x"$has_int128" = x"yes"; then - set_scalar=64bit - fi - if test x"$set_scalar" = x; then - set_scalar=32bit - fi -else - set_scalar=$req_scalar - case $set_scalar in - 64bit) - SECP_INT128_CHECK - if test x"$has_int128" != x"yes"; then - AC_MSG_ERROR([64bit scalar explicitly requested but __int128 support not available]) - fi - ;; - 32bit) - ;; - *) - AC_MSG_ERROR([invalid scalar implementation selected]) - ;; - esac -fi - if test x"$req_bignum" = x"auto"; then SECP_GMP_CHECK if test x"$has_gmp" = x"yes"; then @@ -365,16 +314,18 @@ no) ;; esac -# select field implementation -case $set_field in -64bit) - AC_DEFINE(USE_FIELD_5X52, 1, [Define this symbol to use the FIELD_5X52 implementation]) +# select wide multiplication implementation +case $set_widemul in +int128) + AC_DEFINE(USE_FORCE_WIDEMUL_INT128, 1, [Define this symbol to force the use of the (unsigned) __int128 based wide multiplication implementation]) + ;; +int64) + AC_DEFINE(USE_FORCE_WIDEMUL_INT64, 1, [Define this symbol to force the use of the (u)int64_t based wide multiplication implementation]) ;; -32bit) - AC_DEFINE(USE_FIELD_10X26, 1, [Define this symbol to use the FIELD_10X26 implementation]) +auto) ;; *) - AC_MSG_ERROR([invalid field implementation]) + AC_MSG_ERROR([invalid wide multiplication implementation]) ;; esac @@ -396,19 +347,6 @@ no) ;; esac -#select scalar implementation -case $set_scalar in -64bit) - AC_DEFINE(USE_SCALAR_4X64, 1, [Define this symbol to use the 4x64 scalar implementation]) - ;; -32bit) - AC_DEFINE(USE_SCALAR_8X32, 1, [Define this symbol to use the 8x32 scalar implementation]) - ;; -*) - AC_MSG_ERROR([invalid scalar implementation]) - ;; -esac - #set ecmult window size if test x"$req_ecmult_window" = x"auto"; then set_ecmult_window=15 @@ -493,7 +431,16 @@ if test x"$enable_module_recovery" = x"yes"; then AC_DEFINE(ENABLE_MODULE_RECOVERY, 1, [Define this symbol to enable the ECDSA pubkey recovery module]) fi -AC_C_BIGENDIAN() +if test x"$enable_module_schnorrsig" = x"yes"; then + AC_DEFINE(ENABLE_MODULE_SCHNORRSIG, 1, [Define this symbol to enable the schnorrsig module]) + enable_module_extrakeys=yes +fi + +# Test if extrakeys is set after the schnorrsig module to allow the schnorrsig +# module to set enable_module_extrakeys=yes +if test x"$enable_module_extrakeys" = x"yes"; then + AC_DEFINE(ENABLE_MODULE_EXTRAKEYS, 1, [Define this symbol to enable the extrakeys module]) +fi if test x"$use_external_asm" = x"yes"; then AC_DEFINE(USE_EXTERNAL_ASM, 1, [Define this symbol if an external (non-inline) assembly implementation is used]) @@ -508,11 +455,19 @@ if test x"$enable_experimental" = x"yes"; then AC_MSG_NOTICE([WARNING: experimental build]) AC_MSG_NOTICE([Experimental features do not have stable APIs or properties, and may not be safe for production use.]) AC_MSG_NOTICE([Building ECDH module: $enable_module_ecdh]) + AC_MSG_NOTICE([Building extrakeys module: $enable_module_extrakeys]) + AC_MSG_NOTICE([Building schnorrsig module: $enable_module_schnorrsig]) AC_MSG_NOTICE([******]) else if test x"$enable_module_ecdh" = x"yes"; then AC_MSG_ERROR([ECDH module is experimental. Use --enable-experimental to allow.]) fi + if test x"$enable_module_extrakeys" = x"yes"; then + AC_MSG_ERROR([extrakeys module is experimental. Use --enable-experimental to allow.]) + fi + if test x"$enable_module_schnorrsig" = x"yes"; then + AC_MSG_ERROR([schnorrsig module is experimental. Use --enable-experimental to allow.]) + fi if test x"$set_asm" = x"arm"; then AC_MSG_ERROR([ARM assembly optimization is experimental. Use --enable-experimental to allow.]) fi @@ -531,6 +486,8 @@ AM_CONDITIONAL([USE_BENCHMARK], [test x"$use_benchmark" = x"yes"]) AM_CONDITIONAL([USE_ECMULT_STATIC_PRECOMPUTATION], [test x"$set_precomp" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_ECDH], [test x"$enable_module_ecdh" = x"yes"]) AM_CONDITIONAL([ENABLE_MODULE_RECOVERY], [test x"$enable_module_recovery" = x"yes"]) +AM_CONDITIONAL([ENABLE_MODULE_EXTRAKEYS], [test x"$enable_module_extrakeys" = x"yes"]) +AM_CONDITIONAL([ENABLE_MODULE_SCHNORRSIG], [test x"$enable_module_schnorrsig" = x"yes"]) AM_CONDITIONAL([USE_EXTERNAL_ASM], [test x"$use_external_asm" = x"yes"]) AM_CONDITIONAL([USE_ASM_ARM], [test x"$set_asm" = x"arm"]) @@ -550,13 +507,17 @@ echo " with benchmarks = $use_benchmark" echo " with coverage = $enable_coverage" echo " module ecdh = $enable_module_ecdh" echo " module recovery = $enable_module_recovery" +echo " module extrakeys = $enable_module_extrakeys" +echo " module schnorrsig = $enable_module_schnorrsig" echo echo " asm = $set_asm" echo " bignum = $set_bignum" -echo " field = $set_field" -echo " scalar = $set_scalar" echo " ecmult window size = $set_ecmult_window" echo " ecmult gen prec. bits = $set_ecmult_gen_precision" +dnl Hide test-only options unless they're used. +if test x"$set_widemul" != xauto; then +echo " wide multiplication = $set_widemul" +fi echo echo " valgrind = $enable_valgrind" echo " CC = $CC" diff --git a/src/secp256k1/contrib/lax_der_parsing.c b/src/secp256k1/contrib/lax_der_parsing.c index e177a0562..f71db4b53 100644 --- a/src/secp256k1/contrib/lax_der_parsing.c +++ b/src/secp256k1/contrib/lax_der_parsing.c @@ -112,7 +112,6 @@ int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1_ecdsa_ return 0; } spos = pos; - pos += slen; /* Ignore leading zeroes in R */ while (rlen > 0 && input[rpos] == 0) { diff --git a/src/secp256k1/contrib/travis.sh b/src/secp256k1/contrib/travis.sh index 3909d16a2..b0b55b44b 100755 --- a/src/secp256k1/contrib/travis.sh +++ b/src/secp256k1/contrib/travis.sh @@ -3,10 +3,6 @@ set -e set -x -if [ -n "$HOST" ] -then - export USE_HOST="--host=$HOST" -fi if [ "$HOST" = "i686-linux-gnu" ] then export CC="$CC -m32" @@ -18,9 +14,11 @@ fi ./configure \ --enable-experimental="$EXPERIMENTAL" --enable-endomorphism="$ENDOMORPHISM" \ - --with-field="$FIELD" --with-bignum="$BIGNUM" --with-asm="$ASM" --with-scalar="$SCALAR" \ + --with-test-override-wide-multiply="$WIDEMUL" --with-bignum="$BIGNUM" --with-asm="$ASM" \ --enable-ecmult-static-precomputation="$STATICPRECOMPUTATION" --with-ecmult-gen-precision="$ECMULTGENPRECISION" \ - --enable-module-ecdh="$ECDH" --enable-module-recovery="$RECOVERY" "$EXTRAFLAGS" "$USE_HOST" + --enable-module-ecdh="$ECDH" --enable-module-recovery="$RECOVERY" \ + --enable-module-schnorrsig="$SCHNORRSIG" \ + --host="$HOST" $EXTRAFLAGS if [ -n "$BUILD" ] then diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index 2ba2dca38..2178c8e2d 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -134,7 +134,7 @@ typedef int (*secp256k1_nonce_function)( # else # define SECP256K1_API # endif -# elif defined(__GNUC__) && defined(SECP256K1_BUILD) +# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(SECP256K1_BUILD) # define SECP256K1_API __attribute__ ((visibility ("default"))) # else # define SECP256K1_API diff --git a/src/secp256k1/include/secp256k1_extrakeys.h b/src/secp256k1/include/secp256k1_extrakeys.h new file mode 100644 index 000000000..0c5dff2c9 --- /dev/null +++ b/src/secp256k1/include/secp256k1_extrakeys.h @@ -0,0 +1,236 @@ +#ifndef SECP256K1_EXTRAKEYS_H +#define SECP256K1_EXTRAKEYS_H + +#include "secp256k1.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** Opaque data structure that holds a parsed and valid "x-only" public key. + * An x-only pubkey encodes a point whose Y coordinate is even. It is + * serialized using only its X coordinate (32 bytes). See BIP-340 for more + * information about x-only pubkeys. + * + * The exact representation of data inside is implementation defined and not + * guaranteed to be portable between different platforms or versions. It is + * however guaranteed to be 64 bytes in size, and can be safely copied/moved. + * If you need to convert to a format suitable for storage, transmission, or + * comparison, use secp256k1_xonly_pubkey_serialize and + * secp256k1_xonly_pubkey_parse. + */ +typedef struct { + unsigned char data[64]; +} secp256k1_xonly_pubkey; + +/** Opaque data structure that holds a keypair consisting of a secret and a + * public key. + * + * The exact representation of data inside is implementation defined and not + * guaranteed to be portable between different platforms or versions. It is + * however guaranteed to be 96 bytes in size, and can be safely copied/moved. + */ +typedef struct { + unsigned char data[96]; +} secp256k1_keypair; + +/** Parse a 32-byte sequence into a xonly_pubkey object. + * + * Returns: 1 if the public key was fully valid. + * 0 if the public key could not be parsed or is invalid. + * + * Args: ctx: a secp256k1 context object (cannot be NULL). + * Out: pubkey: pointer to a pubkey object. If 1 is returned, it is set to a + * parsed version of input. If not, it's set to an invalid value. + * (cannot be NULL). + * In: input32: pointer to a serialized xonly_pubkey (cannot be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_parse( + const secp256k1_context* ctx, + secp256k1_xonly_pubkey* pubkey, + const unsigned char *input32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize an xonly_pubkey object into a 32-byte sequence. + * + * Returns: 1 always. + * + * Args: ctx: a secp256k1 context object (cannot be NULL). + * Out: output32: a pointer to a 32-byte array to place the serialized key in + * (cannot be NULL). + * In: pubkey: a pointer to a secp256k1_xonly_pubkey containing an + * initialized public key (cannot be NULL). + */ +SECP256K1_API int secp256k1_xonly_pubkey_serialize( + const secp256k1_context* ctx, + unsigned char *output32, + const secp256k1_xonly_pubkey* pubkey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Converts a secp256k1_pubkey into a secp256k1_xonly_pubkey. + * + * Returns: 1 if the public key was successfully converted + * 0 otherwise + * + * Args: ctx: pointer to a context object (cannot be NULL) + * Out: xonly_pubkey: pointer to an x-only public key object for placing the + * converted public key (cannot be NULL) + * pk_parity: pointer to an integer that will be set to 1 if the point + * encoded by xonly_pubkey is the negation of the pubkey and + * set to 0 otherwise. (can be NULL) + * In: pubkey: pointer to a public key that is converted (cannot be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_from_pubkey( + const secp256k1_context* ctx, + secp256k1_xonly_pubkey *xonly_pubkey, + int *pk_parity, + const secp256k1_pubkey *pubkey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(4); + +/** Tweak an x-only public key by adding the generator multiplied with tweak32 + * to it. + * + * Note that the resulting point can not in general be represented by an x-only + * pubkey because it may have an odd Y coordinate. Instead, the output_pubkey + * is a normal secp256k1_pubkey. + * + * Returns: 0 if the arguments are invalid or the resulting public key would be + * invalid (only when the tweak is the negation of the corresponding + * secret key). 1 otherwise. + * + * Args: ctx: pointer to a context object initialized for verification + * (cannot be NULL) + * Out: output_pubkey: pointer to a public key to store the result. Will be set + * to an invalid value if this function returns 0 (cannot + * be NULL) + * In: internal_pubkey: pointer to an x-only pubkey to apply the tweak to. + * (cannot be NULL). + * tweak32: pointer to a 32-byte tweak. If the tweak is invalid + * according to secp256k1_ec_seckey_verify, this function + * returns 0. For uniformly random 32-byte arrays the + * chance of being invalid is negligible (around 1 in + * 2^128) (cannot be NULL). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_tweak_add( + const secp256k1_context* ctx, + secp256k1_pubkey *output_pubkey, + const secp256k1_xonly_pubkey *internal_pubkey, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Checks that a tweaked pubkey is the result of calling + * secp256k1_xonly_pubkey_tweak_add with internal_pubkey and tweak32. + * + * The tweaked pubkey is represented by its 32-byte x-only serialization and + * its pk_parity, which can both be obtained by converting the result of + * tweak_add to a secp256k1_xonly_pubkey. + * + * Note that this alone does _not_ verify that the tweaked pubkey is a + * commitment. If the tweak is not chosen in a specific way, the tweaked pubkey + * can easily be the result of a different internal_pubkey and tweak. + * + * Returns: 0 if the arguments are invalid or the tweaked pubkey is not the + * result of tweaking the internal_pubkey with tweak32. 1 otherwise. + * Args: ctx: pointer to a context object initialized for verification + * (cannot be NULL) + * In: tweaked_pubkey32: pointer to a serialized xonly_pubkey (cannot be NULL) + * tweaked_pk_parity: the parity of the tweaked pubkey (whose serialization + * is passed in as tweaked_pubkey32). This must match the + * pk_parity value that is returned when calling + * secp256k1_xonly_pubkey with the tweaked pubkey, or + * this function will fail. + * internal_pubkey: pointer to an x-only public key object to apply the + * tweak to (cannot be NULL) + * tweak32: pointer to a 32-byte tweak (cannot be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_tweak_add_check( + const secp256k1_context* ctx, + const unsigned char *tweaked_pubkey32, + int tweaked_pk_parity, + const secp256k1_xonly_pubkey *internal_pubkey, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5); + +/** Compute the keypair for a secret key. + * + * Returns: 1: secret was valid, keypair is ready to use + * 0: secret was invalid, try again with a different secret + * Args: ctx: pointer to a context object, initialized for signing (cannot be NULL) + * Out: keypair: pointer to the created keypair (cannot be NULL) + * In: seckey: pointer to a 32-byte secret key (cannot be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_create( + const secp256k1_context* ctx, + secp256k1_keypair *keypair, + const unsigned char *seckey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Get the public key from a keypair. + * + * Returns: 0 if the arguments are invalid. 1 otherwise. + * Args: ctx: pointer to a context object (cannot be NULL) + * Out: pubkey: pointer to a pubkey object. If 1 is returned, it is set to + * the keypair public key. If not, it's set to an invalid value. + * (cannot be NULL) + * In: keypair: pointer to a keypair (cannot be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_pub( + const secp256k1_context* ctx, + secp256k1_pubkey *pubkey, + const secp256k1_keypair *keypair +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Get the x-only public key from a keypair. + * + * This is the same as calling secp256k1_keypair_pub and then + * secp256k1_xonly_pubkey_from_pubkey. + * + * Returns: 0 if the arguments are invalid. 1 otherwise. + * Args: ctx: pointer to a context object (cannot be NULL) + * Out: pubkey: pointer to an xonly_pubkey object. If 1 is returned, it is set + * to the keypair public key after converting it to an + * xonly_pubkey. If not, it's set to an invalid value (cannot be + * NULL). + * pk_parity: pointer to an integer that will be set to the pk_parity + * argument of secp256k1_xonly_pubkey_from_pubkey (can be NULL). + * In: keypair: pointer to a keypair (cannot be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_xonly_pub( + const secp256k1_context* ctx, + secp256k1_xonly_pubkey *pubkey, + int *pk_parity, + const secp256k1_keypair *keypair +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(4); + +/** Tweak a keypair by adding tweak32 to the secret key and updating the public + * key accordingly. + * + * Calling this function and then secp256k1_keypair_pub results in the same + * public key as calling secp256k1_keypair_xonly_pub and then + * secp256k1_xonly_pubkey_tweak_add. + * + * Returns: 0 if the arguments are invalid or the resulting keypair would be + * invalid (only when the tweak is the negation of the keypair's + * secret key). 1 otherwise. + * + * Args: ctx: pointer to a context object initialized for verification + * (cannot be NULL) + * In/Out: keypair: pointer to a keypair to apply the tweak to. Will be set to + * an invalid value if this function returns 0 (cannot be + * NULL). + * In: tweak32: pointer to a 32-byte tweak. If the tweak is invalid according + * to secp256k1_ec_seckey_verify, this function returns 0. For + * uniformly random 32-byte arrays the chance of being invalid + * is negligible (around 1 in 2^128) (cannot be NULL). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_xonly_tweak_add( + const secp256k1_context* ctx, + secp256k1_keypair *keypair, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_EXTRAKEYS_H */ diff --git a/src/secp256k1/include/secp256k1_schnorrsig.h b/src/secp256k1/include/secp256k1_schnorrsig.h new file mode 100644 index 000000000..0150cd339 --- /dev/null +++ b/src/secp256k1/include/secp256k1_schnorrsig.h @@ -0,0 +1,111 @@ +#ifndef SECP256K1_SCHNORRSIG_H +#define SECP256K1_SCHNORRSIG_H + +#include "secp256k1.h" +#include "secp256k1_extrakeys.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** This module implements a variant of Schnorr signatures compliant with + * Bitcoin Improvement Proposal 340 "Schnorr Signatures for secp256k1" + * (https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki). + */ + +/** A pointer to a function to deterministically generate a nonce. + * + * Same as secp256k1_nonce function with the exception of accepting an + * additional pubkey argument and not requiring an attempt argument. The pubkey + * argument can protect signature schemes with key-prefixed challenge hash + * inputs against reusing the nonce when signing with the wrong precomputed + * pubkey. + * + * Returns: 1 if a nonce was successfully generated. 0 will cause signing to + * return an error. + * Out: nonce32: pointer to a 32-byte array to be filled by the function. + * In: msg32: the 32-byte message hash being verified (will not be NULL) + * key32: pointer to a 32-byte secret key (will not be NULL) + * xonly_pk32: the 32-byte serialized xonly pubkey corresponding to key32 + * (will not be NULL) + * algo16: pointer to a 16-byte array describing the signature + * algorithm (will not be NULL). + * data: Arbitrary data pointer that is passed through. + * + * Except for test cases, this function should compute some cryptographic hash of + * the message, the key, the pubkey, the algorithm description, and data. + */ +typedef int (*secp256k1_nonce_function_hardened)( + unsigned char *nonce32, + const unsigned char *msg32, + const unsigned char *key32, + const unsigned char *xonly_pk32, + const unsigned char *algo16, + void *data +); + +/** An implementation of the nonce generation function as defined in Bitcoin + * Improvement Proposal 340 "Schnorr Signatures for secp256k1" + * (https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki). + * + * If a data pointer is passed, it is assumed to be a pointer to 32 bytes of + * auxiliary random data as defined in BIP-340. If the data pointer is NULL, + * schnorrsig_sign does not produce BIP-340 compliant signatures. The algo16 + * argument must be non-NULL, otherwise the function will fail and return 0. + * The hash will be tagged with algo16 after removing all terminating null + * bytes. Therefore, to create BIP-340 compliant signatures, algo16 must be set + * to "BIP0340/nonce\0\0\0" + */ +SECP256K1_API extern const secp256k1_nonce_function_hardened secp256k1_nonce_function_bip340; + +/** Create a Schnorr signature. + * + * Does _not_ strictly follow BIP-340 because it does not verify the resulting + * signature. Instead, you can manually use secp256k1_schnorrsig_verify and + * abort if it fails. + * + * Otherwise BIP-340 compliant if the noncefp argument is NULL or + * secp256k1_nonce_function_bip340 and the ndata argument is 32-byte auxiliary + * randomness. + * + * Returns 1 on success, 0 on failure. + * Args: ctx: pointer to a context object, initialized for signing (cannot be NULL) + * Out: sig64: pointer to a 64-byte array to store the serialized signature (cannot be NULL) + * In: msg32: the 32-byte message being signed (cannot be NULL) + * keypair: pointer to an initialized keypair (cannot be NULL) + * noncefp: pointer to a nonce generation function. If NULL, secp256k1_nonce_function_bip340 is used + * ndata: pointer to arbitrary data used by the nonce generation + * function (can be NULL). If it is non-NULL and + * secp256k1_nonce_function_bip340 is used, then ndata must be a + * pointer to 32-byte auxiliary randomness as per BIP-340. + */ +SECP256K1_API int secp256k1_schnorrsig_sign( + const secp256k1_context* ctx, + unsigned char *sig64, + const unsigned char *msg32, + const secp256k1_keypair *keypair, + secp256k1_nonce_function_hardened noncefp, + void *ndata +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Verify a Schnorr signature. + * + * Returns: 1: correct signature + * 0: incorrect signature + * Args: ctx: a secp256k1 context object, initialized for verification. + * In: sig64: pointer to the 64-byte signature to verify (cannot be NULL) + * msg32: the 32-byte message being verified (cannot be NULL) + * pubkey: pointer to an x-only public key to verify with (cannot be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_schnorrsig_verify( + const secp256k1_context* ctx, + const unsigned char *sig64, + const unsigned char *msg32, + const secp256k1_xonly_pubkey *pubkey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_SCHNORRSIG_H */ diff --git a/src/secp256k1/src/assumptions.h b/src/secp256k1/src/assumptions.h new file mode 100644 index 000000000..f9d4e8e79 --- /dev/null +++ b/src/secp256k1/src/assumptions.h @@ -0,0 +1,74 @@ +/********************************************************************** + * Copyright (c) 2020 Pieter Wuille * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef SECP256K1_ASSUMPTIONS_H +#define SECP256K1_ASSUMPTIONS_H + +#include "util.h" + +/* This library, like most software, relies on a number of compiler implementation defined (but not undefined) + behaviours. Although the behaviours we require are essentially universal we test them specifically here to + reduce the odds of experiencing an unwelcome surprise. +*/ + +struct secp256k1_assumption_checker { + /* This uses a trick to implement a static assertion in C89: a type with an array of negative size is not + allowed. */ + int dummy_array[( + /* Bytes are 8 bits. */ + CHAR_BIT == 8 && + + /* Conversions from unsigned to signed outside of the bounds of the signed type are + implementation-defined. Verify that they function as reinterpreting the lower + bits of the input in two's complement notation. Do this for conversions: + - from uint(N)_t to int(N)_t with negative result + - from uint(2N)_t to int(N)_t with negative result + - from int(2N)_t to int(N)_t with negative result + - from int(2N)_t to int(N)_t with positive result */ + + /* To int8_t. */ + ((int8_t)(uint8_t)0xAB == (int8_t)-(int8_t)0x55) && + ((int8_t)(uint16_t)0xABCD == (int8_t)-(int8_t)0x33) && + ((int8_t)(int16_t)(uint16_t)0xCDEF == (int8_t)(uint8_t)0xEF) && + ((int8_t)(int16_t)(uint16_t)0x9234 == (int8_t)(uint8_t)0x34) && + + /* To int16_t. */ + ((int16_t)(uint16_t)0xBCDE == (int16_t)-(int16_t)0x4322) && + ((int16_t)(uint32_t)0xA1B2C3D4 == (int16_t)-(int16_t)0x3C2C) && + ((int16_t)(int32_t)(uint32_t)0xC1D2E3F4 == (int16_t)(uint16_t)0xE3F4) && + ((int16_t)(int32_t)(uint32_t)0x92345678 == (int16_t)(uint16_t)0x5678) && + + /* To int32_t. */ + ((int32_t)(uint32_t)0xB2C3D4E5 == (int32_t)-(int32_t)0x4D3C2B1B) && + ((int32_t)(uint64_t)0xA123B456C789D012ULL == (int32_t)-(int32_t)0x38762FEE) && + ((int32_t)(int64_t)(uint64_t)0xC1D2E3F4A5B6C7D8ULL == (int32_t)(uint32_t)0xA5B6C7D8) && + ((int32_t)(int64_t)(uint64_t)0xABCDEF0123456789ULL == (int32_t)(uint32_t)0x23456789) && + + /* To int64_t. */ + ((int64_t)(uint64_t)0xB123C456D789E012ULL == (int64_t)-(int64_t)0x4EDC3BA928761FEEULL) && +#if defined(SECP256K1_WIDEMUL_INT128) + ((int64_t)(((uint128_t)0xA1234567B8901234ULL << 64) + 0xC5678901D2345678ULL) == (int64_t)-(int64_t)0x3A9876FE2DCBA988ULL) && + (((int64_t)(int128_t)(((uint128_t)0xB1C2D3E4F5A6B7C8ULL << 64) + 0xD9E0F1A2B3C4D5E6ULL)) == (int64_t)(uint64_t)0xD9E0F1A2B3C4D5E6ULL) && + (((int64_t)(int128_t)(((uint128_t)0xABCDEF0123456789ULL << 64) + 0x0123456789ABCDEFULL)) == (int64_t)(uint64_t)0x0123456789ABCDEFULL) && + + /* To int128_t. */ + ((int128_t)(((uint128_t)0xB1234567C8901234ULL << 64) + 0xD5678901E2345678ULL) == (int128_t)(-(int128_t)0x8E1648B3F50E80DCULL * 0x8E1648B3F50E80DDULL + 0x5EA688D5482F9464ULL)) && +#endif + + /* Right shift on negative signed values is implementation defined. Verify that it + acts as a right shift in two's complement with sign extension (i.e duplicating + the top bit into newly added bits). */ + ((((int8_t)0xE8) >> 2) == (int8_t)(uint8_t)0xFA) && + ((((int16_t)0xE9AC) >> 4) == (int16_t)(uint16_t)0xFE9A) && + ((((int32_t)0x937C918A) >> 9) == (int32_t)(uint32_t)0xFFC9BE48) && + ((((int64_t)0xA8B72231DF9CF4B9ULL) >> 19) == (int64_t)(uint64_t)0xFFFFF516E4463BF3ULL) && +#if defined(SECP256K1_WIDEMUL_INT128) + ((((int128_t)(((uint128_t)0xCD833A65684A0DBCULL << 64) + 0xB349312F71EA7637ULL)) >> 39) == (int128_t)(((uint128_t)0xFFFFFFFFFF9B0674ULL << 64) + 0xCAD0941B79669262ULL)) && +#endif + 1) * 2 - 1]; +}; + +#endif /* SECP256K1_ASSUMPTIONS_H */ diff --git a/src/secp256k1/src/basic-config.h b/src/secp256k1/src/basic-config.h index e9be39d4c..83dbe6f25 100644 --- a/src/secp256k1/src/basic-config.h +++ b/src/secp256k1/src/basic-config.h @@ -14,23 +14,20 @@ #undef USE_ENDOMORPHISM #undef USE_EXTERNAL_ASM #undef USE_EXTERNAL_DEFAULT_CALLBACKS -#undef USE_FIELD_10X26 -#undef USE_FIELD_5X52 #undef USE_FIELD_INV_BUILTIN #undef USE_FIELD_INV_NUM #undef USE_NUM_GMP #undef USE_NUM_NONE -#undef USE_SCALAR_4X64 -#undef USE_SCALAR_8X32 #undef USE_SCALAR_INV_BUILTIN #undef USE_SCALAR_INV_NUM +#undef USE_FORCE_WIDEMUL_INT64 +#undef USE_FORCE_WIDEMUL_INT128 #undef ECMULT_WINDOW_SIZE #define USE_NUM_NONE 1 #define USE_FIELD_INV_BUILTIN 1 #define USE_SCALAR_INV_BUILTIN 1 -#define USE_FIELD_10X26 1 -#define USE_SCALAR_8X32 1 +#define USE_WIDEMUL_64 1 #define ECMULT_WINDOW_SIZE 15 #endif /* USE_BASIC_CONFIG */ diff --git a/src/secp256k1/src/bench_internal.c b/src/secp256k1/src/bench_internal.c index 20759127d..9687fe448 100644 --- a/src/secp256k1/src/bench_internal.c +++ b/src/secp256k1/src/bench_internal.c @@ -7,6 +7,7 @@ #include "include/secp256k1.h" +#include "assumptions.h" #include "util.h" #include "hash_impl.h" #include "num_impl.h" @@ -19,10 +20,10 @@ #include "secp256k1.c" typedef struct { - secp256k1_scalar scalar_x, scalar_y; - secp256k1_fe fe_x, fe_y; - secp256k1_ge ge_x, ge_y; - secp256k1_gej gej_x, gej_y; + secp256k1_scalar scalar[2]; + secp256k1_fe fe[4]; + secp256k1_ge ge[2]; + secp256k1_gej gej[2]; unsigned char data[64]; int wnaf[256]; } bench_inv; @@ -30,30 +31,53 @@ typedef struct { void bench_setup(void* arg) { bench_inv *data = (bench_inv*)arg; - static const unsigned char init_x[32] = { - 0x02, 0x03, 0x05, 0x07, 0x0b, 0x0d, 0x11, 0x13, - 0x17, 0x1d, 0x1f, 0x25, 0x29, 0x2b, 0x2f, 0x35, - 0x3b, 0x3d, 0x43, 0x47, 0x49, 0x4f, 0x53, 0x59, - 0x61, 0x65, 0x67, 0x6b, 0x6d, 0x71, 0x7f, 0x83 + static const unsigned char init[4][32] = { + /* Initializer for scalar[0], fe[0], first half of data, the X coordinate of ge[0], + and the (implied affine) X coordinate of gej[0]. */ + { + 0x02, 0x03, 0x05, 0x07, 0x0b, 0x0d, 0x11, 0x13, + 0x17, 0x1d, 0x1f, 0x25, 0x29, 0x2b, 0x2f, 0x35, + 0x3b, 0x3d, 0x43, 0x47, 0x49, 0x4f, 0x53, 0x59, + 0x61, 0x65, 0x67, 0x6b, 0x6d, 0x71, 0x7f, 0x83 + }, + /* Initializer for scalar[1], fe[1], first half of data, the X coordinate of ge[1], + and the (implied affine) X coordinate of gej[1]. */ + { + 0x82, 0x83, 0x85, 0x87, 0x8b, 0x8d, 0x81, 0x83, + 0x97, 0xad, 0xaf, 0xb5, 0xb9, 0xbb, 0xbf, 0xc5, + 0xdb, 0xdd, 0xe3, 0xe7, 0xe9, 0xef, 0xf3, 0xf9, + 0x11, 0x15, 0x17, 0x1b, 0x1d, 0xb1, 0xbf, 0xd3 + }, + /* Initializer for fe[2] and the Z coordinate of gej[0]. */ + { + 0x3d, 0x2d, 0xef, 0xf4, 0x25, 0x98, 0x4f, 0x5d, + 0xe2, 0xca, 0x5f, 0x41, 0x3f, 0x3f, 0xce, 0x44, + 0xaa, 0x2c, 0x53, 0x8a, 0xc6, 0x59, 0x1f, 0x38, + 0x38, 0x23, 0xe4, 0x11, 0x27, 0xc6, 0xa0, 0xe7 + }, + /* Initializer for fe[3] and the Z coordinate of gej[1]. */ + { + 0xbd, 0x21, 0xa5, 0xe1, 0x13, 0x50, 0x73, 0x2e, + 0x52, 0x98, 0xc8, 0x9e, 0xab, 0x00, 0xa2, 0x68, + 0x43, 0xf5, 0xd7, 0x49, 0x80, 0x72, 0xa7, 0xf3, + 0xd7, 0x60, 0xe6, 0xab, 0x90, 0x92, 0xdf, 0xc5 + } }; - static const unsigned char init_y[32] = { - 0x82, 0x83, 0x85, 0x87, 0x8b, 0x8d, 0x81, 0x83, - 0x97, 0xad, 0xaf, 0xb5, 0xb9, 0xbb, 0xbf, 0xc5, - 0xdb, 0xdd, 0xe3, 0xe7, 0xe9, 0xef, 0xf3, 0xf9, - 0x11, 0x15, 0x17, 0x1b, 0x1d, 0xb1, 0xbf, 0xd3 - }; - - secp256k1_scalar_set_b32(&data->scalar_x, init_x, NULL); - secp256k1_scalar_set_b32(&data->scalar_y, init_y, NULL); - secp256k1_fe_set_b32(&data->fe_x, init_x); - secp256k1_fe_set_b32(&data->fe_y, init_y); - CHECK(secp256k1_ge_set_xo_var(&data->ge_x, &data->fe_x, 0)); - CHECK(secp256k1_ge_set_xo_var(&data->ge_y, &data->fe_y, 1)); - secp256k1_gej_set_ge(&data->gej_x, &data->ge_x); - secp256k1_gej_set_ge(&data->gej_y, &data->ge_y); - memcpy(data->data, init_x, 32); - memcpy(data->data + 32, init_y, 32); + secp256k1_scalar_set_b32(&data->scalar[0], init[0], NULL); + secp256k1_scalar_set_b32(&data->scalar[1], init[1], NULL); + secp256k1_fe_set_b32(&data->fe[0], init[0]); + secp256k1_fe_set_b32(&data->fe[1], init[1]); + secp256k1_fe_set_b32(&data->fe[2], init[2]); + secp256k1_fe_set_b32(&data->fe[3], init[3]); + CHECK(secp256k1_ge_set_xo_var(&data->ge[0], &data->fe[0], 0)); + CHECK(secp256k1_ge_set_xo_var(&data->ge[1], &data->fe[1], 1)); + secp256k1_gej_set_ge(&data->gej[0], &data->ge[0]); + secp256k1_gej_rescale(&data->gej[0], &data->fe[2]); + secp256k1_gej_set_ge(&data->gej[1], &data->ge[1]); + secp256k1_gej_rescale(&data->gej[1], &data->fe[3]); + memcpy(data->data, init[0], 32); + memcpy(data->data + 32, init[1], 32); } void bench_scalar_add(void* arg, int iters) { @@ -61,7 +85,7 @@ void bench_scalar_add(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - j += secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); + j += secp256k1_scalar_add(&data->scalar[0], &data->scalar[0], &data->scalar[1]); } CHECK(j <= iters); } @@ -71,7 +95,7 @@ void bench_scalar_negate(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_scalar_negate(&data->scalar_x, &data->scalar_x); + secp256k1_scalar_negate(&data->scalar[0], &data->scalar[0]); } } @@ -80,7 +104,7 @@ void bench_scalar_sqr(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_scalar_sqr(&data->scalar_x, &data->scalar_x); + secp256k1_scalar_sqr(&data->scalar[0], &data->scalar[0]); } } @@ -89,7 +113,7 @@ void bench_scalar_mul(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_scalar_mul(&data->scalar_x, &data->scalar_x, &data->scalar_y); + secp256k1_scalar_mul(&data->scalar[0], &data->scalar[0], &data->scalar[1]); } } @@ -99,8 +123,8 @@ void bench_scalar_split(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_scalar_split_lambda(&data->scalar_x, &data->scalar_y, &data->scalar_x); - j += secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); + secp256k1_scalar_split_lambda(&data->scalar[0], &data->scalar[1], &data->scalar[0]); + j += secp256k1_scalar_add(&data->scalar[0], &data->scalar[0], &data->scalar[1]); } CHECK(j <= iters); } @@ -111,8 +135,8 @@ void bench_scalar_inverse(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_scalar_inverse(&data->scalar_x, &data->scalar_x); - j += secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); + secp256k1_scalar_inverse(&data->scalar[0], &data->scalar[0]); + j += secp256k1_scalar_add(&data->scalar[0], &data->scalar[0], &data->scalar[1]); } CHECK(j <= iters); } @@ -122,8 +146,8 @@ void bench_scalar_inverse_var(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_scalar_inverse_var(&data->scalar_x, &data->scalar_x); - j += secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); + secp256k1_scalar_inverse_var(&data->scalar[0], &data->scalar[0]); + j += secp256k1_scalar_add(&data->scalar[0], &data->scalar[0], &data->scalar[1]); } CHECK(j <= iters); } @@ -133,7 +157,7 @@ void bench_field_normalize(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_fe_normalize(&data->fe_x); + secp256k1_fe_normalize(&data->fe[0]); } } @@ -142,7 +166,7 @@ void bench_field_normalize_weak(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_fe_normalize_weak(&data->fe_x); + secp256k1_fe_normalize_weak(&data->fe[0]); } } @@ -151,7 +175,7 @@ void bench_field_mul(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_fe_mul(&data->fe_x, &data->fe_x, &data->fe_y); + secp256k1_fe_mul(&data->fe[0], &data->fe[0], &data->fe[1]); } } @@ -160,7 +184,7 @@ void bench_field_sqr(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_fe_sqr(&data->fe_x, &data->fe_x); + secp256k1_fe_sqr(&data->fe[0], &data->fe[0]); } } @@ -169,8 +193,8 @@ void bench_field_inverse(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_fe_inv(&data->fe_x, &data->fe_x); - secp256k1_fe_add(&data->fe_x, &data->fe_y); + secp256k1_fe_inv(&data->fe[0], &data->fe[0]); + secp256k1_fe_add(&data->fe[0], &data->fe[1]); } } @@ -179,8 +203,8 @@ void bench_field_inverse_var(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_fe_inv_var(&data->fe_x, &data->fe_x); - secp256k1_fe_add(&data->fe_x, &data->fe_y); + secp256k1_fe_inv_var(&data->fe[0], &data->fe[0]); + secp256k1_fe_add(&data->fe[0], &data->fe[1]); } } @@ -190,9 +214,9 @@ void bench_field_sqrt(void* arg, int iters) { secp256k1_fe t; for (i = 0; i < iters; i++) { - t = data->fe_x; - j += secp256k1_fe_sqrt(&data->fe_x, &t); - secp256k1_fe_add(&data->fe_x, &data->fe_y); + t = data->fe[0]; + j += secp256k1_fe_sqrt(&data->fe[0], &t); + secp256k1_fe_add(&data->fe[0], &data->fe[1]); } CHECK(j <= iters); } @@ -202,7 +226,7 @@ void bench_group_double_var(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_gej_double_var(&data->gej_x, &data->gej_x, NULL); + secp256k1_gej_double_var(&data->gej[0], &data->gej[0], NULL); } } @@ -211,7 +235,7 @@ void bench_group_add_var(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_gej_add_var(&data->gej_x, &data->gej_x, &data->gej_y, NULL); + secp256k1_gej_add_var(&data->gej[0], &data->gej[0], &data->gej[1], NULL); } } @@ -220,7 +244,7 @@ void bench_group_add_affine(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_gej_add_ge(&data->gej_x, &data->gej_x, &data->ge_y); + secp256k1_gej_add_ge(&data->gej[0], &data->gej[0], &data->ge[1]); } } @@ -229,7 +253,7 @@ void bench_group_add_affine_var(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - secp256k1_gej_add_ge_var(&data->gej_x, &data->gej_x, &data->ge_y, NULL); + secp256k1_gej_add_ge_var(&data->gej[0], &data->gej[0], &data->ge[1], NULL); } } @@ -238,9 +262,37 @@ void bench_group_jacobi_var(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - j += secp256k1_gej_has_quad_y_var(&data->gej_x); + j += secp256k1_gej_has_quad_y_var(&data->gej[0]); + /* Vary the Y and Z coordinates of the input (the X coordinate doesn't matter to + secp256k1_gej_has_quad_y_var). Note that the resulting coordinates will + generally not correspond to a point on the curve, but this is not a problem + for the code being benchmarked here. Adding and normalizing have less + overhead than EC operations (which could guarantee the point remains on the + curve). */ + secp256k1_fe_add(&data->gej[0].y, &data->fe[1]); + secp256k1_fe_add(&data->gej[0].z, &data->fe[2]); + secp256k1_fe_normalize_var(&data->gej[0].y); + secp256k1_fe_normalize_var(&data->gej[0].z); + } + CHECK(j <= iters); +} + +void bench_group_to_affine_var(void* arg, int iters) { + int i; + bench_inv *data = (bench_inv*)arg; + + for (i = 0; i < iters; ++i) { + secp256k1_ge_set_gej_var(&data->ge[1], &data->gej[0]); + /* Use the output affine X/Y coordinates to vary the input X/Y/Z coordinates. + Similar to bench_group_jacobi_var, this approach does not result in + coordinates of points on the curve. */ + secp256k1_fe_add(&data->gej[0].x, &data->ge[1].y); + secp256k1_fe_add(&data->gej[0].y, &data->fe[2]); + secp256k1_fe_add(&data->gej[0].z, &data->ge[1].x); + secp256k1_fe_normalize_var(&data->gej[0].x); + secp256k1_fe_normalize_var(&data->gej[0].y); + secp256k1_fe_normalize_var(&data->gej[0].z); } - CHECK(j == iters); } void bench_ecmult_wnaf(void* arg, int iters) { @@ -248,8 +300,8 @@ void bench_ecmult_wnaf(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - bits += secp256k1_ecmult_wnaf(data->wnaf, 256, &data->scalar_x, WINDOW_A); - overflow += secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); + bits += secp256k1_ecmult_wnaf(data->wnaf, 256, &data->scalar[0], WINDOW_A); + overflow += secp256k1_scalar_add(&data->scalar[0], &data->scalar[0], &data->scalar[1]); } CHECK(overflow >= 0); CHECK(bits <= 256*iters); @@ -260,8 +312,8 @@ void bench_wnaf_const(void* arg, int iters) { bench_inv *data = (bench_inv*)arg; for (i = 0; i < iters; i++) { - bits += secp256k1_wnaf_const(data->wnaf, &data->scalar_x, WINDOW_A, 256); - overflow += secp256k1_scalar_add(&data->scalar_x, &data->scalar_x, &data->scalar_y); + bits += secp256k1_wnaf_const(data->wnaf, &data->scalar[0], WINDOW_A, 256); + overflow += secp256k1_scalar_add(&data->scalar[0], &data->scalar[0], &data->scalar[1]); } CHECK(overflow >= 0); CHECK(bits <= 256*iters); @@ -323,14 +375,15 @@ void bench_context_sign(void* arg, int iters) { void bench_num_jacobi(void* arg, int iters) { int i, j = 0; bench_inv *data = (bench_inv*)arg; - secp256k1_num nx, norder; + secp256k1_num nx, na, norder; - secp256k1_scalar_get_num(&nx, &data->scalar_x); + secp256k1_scalar_get_num(&nx, &data->scalar[0]); secp256k1_scalar_order_get_num(&norder); - secp256k1_scalar_get_num(&norder, &data->scalar_y); + secp256k1_scalar_get_num(&na, &data->scalar[1]); for (i = 0; i < iters; i++) { j += secp256k1_num_jacobi(&nx, &norder); + secp256k1_num_add(&nx, &nx, &na); } CHECK(j <= iters); } @@ -363,6 +416,7 @@ int main(int argc, char **argv) { if (have_flag(argc, argv, "group") || have_flag(argc, argv, "add")) run_benchmark("group_add_affine", bench_group_add_affine, bench_setup, NULL, &data, 10, iters*10); if (have_flag(argc, argv, "group") || have_flag(argc, argv, "add")) run_benchmark("group_add_affine_var", bench_group_add_affine_var, bench_setup, NULL, &data, 10, iters*10); if (have_flag(argc, argv, "group") || have_flag(argc, argv, "jacobi")) run_benchmark("group_jacobi_var", bench_group_jacobi_var, bench_setup, NULL, &data, 10, iters); + if (have_flag(argc, argv, "group") || have_flag(argc, argv, "to_affine")) run_benchmark("group_to_affine_var", bench_group_to_affine_var, bench_setup, NULL, &data, 10, iters); if (have_flag(argc, argv, "ecmult") || have_flag(argc, argv, "wnaf")) run_benchmark("wnaf_const", bench_wnaf_const, bench_setup, NULL, &data, 10, iters); if (have_flag(argc, argv, "ecmult") || have_flag(argc, argv, "wnaf")) run_benchmark("ecmult_wnaf", bench_ecmult_wnaf, bench_setup, NULL, &data, 10, iters); diff --git a/src/secp256k1/src/bench_schnorrsig.c b/src/secp256k1/src/bench_schnorrsig.c new file mode 100644 index 000000000..315f5af28 --- /dev/null +++ b/src/secp256k1/src/bench_schnorrsig.c @@ -0,0 +1,102 @@ +/********************************************************************** + * Copyright (c) 2018-2020 Andrew Poelstra, Jonas Nick * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#include <string.h> +#include <stdlib.h> + + +#include "include/secp256k1.h" +#include "include/secp256k1_schnorrsig.h" +#include "util.h" +#include "bench.h" + +typedef struct { + secp256k1_context *ctx; + int n; + + const secp256k1_keypair **keypairs; + const unsigned char **pk; + const unsigned char **sigs; + const unsigned char **msgs; +} bench_schnorrsig_data; + +void bench_schnorrsig_sign(void* arg, int iters) { + bench_schnorrsig_data *data = (bench_schnorrsig_data *)arg; + int i; + unsigned char msg[32] = "benchmarkexamplemessagetemplate"; + unsigned char sig[64]; + + for (i = 0; i < iters; i++) { + msg[0] = i; + msg[1] = i >> 8; + CHECK(secp256k1_schnorrsig_sign(data->ctx, sig, msg, data->keypairs[i], NULL, NULL)); + } +} + +void bench_schnorrsig_verify(void* arg, int iters) { + bench_schnorrsig_data *data = (bench_schnorrsig_data *)arg; + int i; + + for (i = 0; i < iters; i++) { + secp256k1_xonly_pubkey pk; + CHECK(secp256k1_xonly_pubkey_parse(data->ctx, &pk, data->pk[i]) == 1); + CHECK(secp256k1_schnorrsig_verify(data->ctx, data->sigs[i], data->msgs[i], &pk)); + } +} + +int main(void) { + int i; + bench_schnorrsig_data data; + int iters = get_iters(10000); + + data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY | SECP256K1_CONTEXT_SIGN); + data.keypairs = (const secp256k1_keypair **)malloc(iters * sizeof(secp256k1_keypair *)); + data.pk = (const unsigned char **)malloc(iters * sizeof(unsigned char *)); + data.msgs = (const unsigned char **)malloc(iters * sizeof(unsigned char *)); + data.sigs = (const unsigned char **)malloc(iters * sizeof(unsigned char *)); + + for (i = 0; i < iters; i++) { + unsigned char sk[32]; + unsigned char *msg = (unsigned char *)malloc(32); + unsigned char *sig = (unsigned char *)malloc(64); + secp256k1_keypair *keypair = (secp256k1_keypair *)malloc(sizeof(*keypair)); + unsigned char *pk_char = (unsigned char *)malloc(32); + secp256k1_xonly_pubkey pk; + msg[0] = sk[0] = i; + msg[1] = sk[1] = i >> 8; + msg[2] = sk[2] = i >> 16; + msg[3] = sk[3] = i >> 24; + memset(&msg[4], 'm', 28); + memset(&sk[4], 's', 28); + + data.keypairs[i] = keypair; + data.pk[i] = pk_char; + data.msgs[i] = msg; + data.sigs[i] = sig; + + CHECK(secp256k1_keypair_create(data.ctx, keypair, sk)); + CHECK(secp256k1_schnorrsig_sign(data.ctx, sig, msg, keypair, NULL, NULL)); + CHECK(secp256k1_keypair_xonly_pub(data.ctx, &pk, NULL, keypair)); + CHECK(secp256k1_xonly_pubkey_serialize(data.ctx, pk_char, &pk) == 1); + } + + run_benchmark("schnorrsig_sign", bench_schnorrsig_sign, NULL, NULL, (void *) &data, 10, iters); + run_benchmark("schnorrsig_verify", bench_schnorrsig_verify, NULL, NULL, (void *) &data, 10, iters); + + for (i = 0; i < iters; i++) { + free((void *)data.keypairs[i]); + free((void *)data.pk[i]); + free((void *)data.msgs[i]); + free((void *)data.sigs[i]); + } + free(data.keypairs); + free(data.pk); + free(data.msgs); + free(data.sigs); + + secp256k1_context_destroy(data.ctx); + return 0; +} diff --git a/src/secp256k1/src/ecmult_const_impl.h b/src/secp256k1/src/ecmult_const_impl.h index 6d6d354aa..55b61e493 100644 --- a/src/secp256k1/src/ecmult_const_impl.h +++ b/src/secp256k1/src/ecmult_const_impl.h @@ -105,16 +105,22 @@ static int secp256k1_wnaf_const(int *wnaf, const secp256k1_scalar *scalar, int w /* 4 */ u_last = secp256k1_scalar_shr_int(&s, w); do { - int sign; int even; /* 4.1 4.4 */ u = secp256k1_scalar_shr_int(&s, w); /* 4.2 */ even = ((u & 1) == 0); - sign = 2 * (u_last > 0) - 1; - u += sign * even; - u_last -= sign * even * (1 << w); + /* In contrast to the original algorithm, u_last is always > 0 and + * therefore we do not need to check its sign. In particular, it's easy + * to see that u_last is never < 0 because u is never < 0. Moreover, + * u_last is never = 0 because u is never even after a loop + * iteration. The same holds analogously for the initial value of + * u_last (in the first loop iteration). */ + VERIFY_CHECK(u_last > 0); + VERIFY_CHECK((u_last & 1) == 1); + u += even; + u_last -= even * (1 << w); /* 4.3, adapted for global sign change */ wnaf[word++] = u_last * global_sign; @@ -202,7 +208,7 @@ static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, cons int n; int j; for (j = 0; j < WINDOW_A - 1; ++j) { - secp256k1_gej_double_nonzero(r, r); + secp256k1_gej_double(r, r); } n = wnaf_1[i]; diff --git a/src/secp256k1/src/field.h b/src/secp256k1/src/field.h index 7993a1f11..aca1fb72c 100644 --- a/src/secp256k1/src/field.h +++ b/src/secp256k1/src/field.h @@ -22,16 +22,16 @@ #include "libsecp256k1-config.h" #endif -#if defined(USE_FIELD_10X26) -#include "field_10x26.h" -#elif defined(USE_FIELD_5X52) +#include "util.h" + +#if defined(SECP256K1_WIDEMUL_INT128) #include "field_5x52.h" +#elif defined(SECP256K1_WIDEMUL_INT64) +#include "field_10x26.h" #else -#error "Please select field implementation" +#error "Please select wide multiplication implementation" #endif -#include "util.h" - /** Normalize a field element. This brings the field element to a canonical representation, reduces * its magnitude to 1, and reduces it modulo field size `p`. */ diff --git a/src/secp256k1/src/field_5x52.h b/src/secp256k1/src/field_5x52.h index fc5bfe357..6a068484c 100644 --- a/src/secp256k1/src/field_5x52.h +++ b/src/secp256k1/src/field_5x52.h @@ -46,4 +46,10 @@ typedef struct { (d6) | (((uint64_t)(d7)) << 32) \ }} +#define SECP256K1_FE_STORAGE_CONST_GET(d) \ + (uint32_t)(d.n[3] >> 32), (uint32_t)d.n[3], \ + (uint32_t)(d.n[2] >> 32), (uint32_t)d.n[2], \ + (uint32_t)(d.n[1] >> 32), (uint32_t)d.n[1], \ + (uint32_t)(d.n[0] >> 32), (uint32_t)d.n[0] + #endif /* SECP256K1_FIELD_REPR_H */ diff --git a/src/secp256k1/src/field_impl.h b/src/secp256k1/src/field_impl.h index 485921a60..18e4d2f30 100644 --- a/src/secp256k1/src/field_impl.h +++ b/src/secp256k1/src/field_impl.h @@ -14,12 +14,12 @@ #include "util.h" #include "num.h" -#if defined(USE_FIELD_10X26) -#include "field_10x26_impl.h" -#elif defined(USE_FIELD_5X52) +#if defined(SECP256K1_WIDEMUL_INT128) #include "field_5x52_impl.h" +#elif defined(SECP256K1_WIDEMUL_INT64) +#include "field_10x26_impl.h" #else -#error "Please select field implementation" +#error "Please select wide multiplication implementation" #endif SECP256K1_INLINE static int secp256k1_fe_equal(const secp256k1_fe *a, const secp256k1_fe *b) { diff --git a/src/secp256k1/src/gen_context.c b/src/secp256k1/src/gen_context.c index 539f574bf..8b7729aee 100644 --- a/src/secp256k1/src/gen_context.c +++ b/src/secp256k1/src/gen_context.c @@ -13,6 +13,7 @@ #include "basic-config.h" #include "include/secp256k1.h" +#include "assumptions.h" #include "util.h" #include "field_impl.h" #include "scalar_impl.h" diff --git a/src/secp256k1/src/group.h b/src/secp256k1/src/group.h index 863644f0f..6185be052 100644 --- a/src/secp256k1/src/group.h +++ b/src/secp256k1/src/group.h @@ -95,8 +95,8 @@ static int secp256k1_gej_is_infinity(const secp256k1_gej *a); /** Check whether a group element's y coordinate is a quadratic residue. */ static int secp256k1_gej_has_quad_y_var(const secp256k1_gej *a); -/** Set r equal to the double of a, a cannot be infinity. Constant time. */ -static void secp256k1_gej_double_nonzero(secp256k1_gej *r, const secp256k1_gej *a); +/** Set r equal to the double of a. Constant time. */ +static void secp256k1_gej_double(secp256k1_gej *r, const secp256k1_gej *a); /** Set r equal to the double of a. If rzr is not-NULL this sets *rzr such that r->z == a->z * *rzr (where infinity means an implicit z = 0). */ static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, secp256k1_fe *rzr); diff --git a/src/secp256k1/src/group_impl.h b/src/secp256k1/src/group_impl.h index 43b039bec..ccd93d348 100644 --- a/src/secp256k1/src/group_impl.h +++ b/src/secp256k1/src/group_impl.h @@ -303,7 +303,7 @@ static int secp256k1_ge_is_valid_var(const secp256k1_ge *a) { return secp256k1_fe_equal_var(&y2, &x3); } -static SECP256K1_INLINE void secp256k1_gej_double_nonzero(secp256k1_gej *r, const secp256k1_gej *a) { +static SECP256K1_INLINE void secp256k1_gej_double(secp256k1_gej *r, const secp256k1_gej *a) { /* Operations: 3 mul, 4 sqr, 0 normalize, 12 mul_int/add/negate. * * Note that there is an implementation described at @@ -313,8 +313,7 @@ static SECP256K1_INLINE void secp256k1_gej_double_nonzero(secp256k1_gej *r, cons */ secp256k1_fe t1,t2,t3,t4; - VERIFY_CHECK(!secp256k1_gej_is_infinity(a)); - r->infinity = 0; + r->infinity = a->infinity; secp256k1_fe_mul(&r->z, &a->z, &a->y); secp256k1_fe_mul_int(&r->z, 2); /* Z' = 2*Y*Z (2) */ @@ -363,7 +362,7 @@ static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, s secp256k1_fe_mul_int(rzr, 2); } - secp256k1_gej_double_nonzero(r, a); + secp256k1_gej_double(r, a); } static void secp256k1_gej_add_var(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_gej *b, secp256k1_fe *rzr) { @@ -400,7 +399,7 @@ static void secp256k1_gej_add_var(secp256k1_gej *r, const secp256k1_gej *a, cons if (rzr != NULL) { secp256k1_fe_set_int(rzr, 0); } - r->infinity = 1; + secp256k1_gej_set_infinity(r); } return; } @@ -450,7 +449,7 @@ static void secp256k1_gej_add_ge_var(secp256k1_gej *r, const secp256k1_gej *a, c if (rzr != NULL) { secp256k1_fe_set_int(rzr, 0); } - r->infinity = 1; + secp256k1_gej_set_infinity(r); } return; } @@ -509,7 +508,7 @@ static void secp256k1_gej_add_zinv_var(secp256k1_gej *r, const secp256k1_gej *a, if (secp256k1_fe_normalizes_to_zero_var(&i)) { secp256k1_gej_double_var(r, a, NULL); } else { - r->infinity = 1; + secp256k1_gej_set_infinity(r); } return; } diff --git a/src/secp256k1/src/hash_impl.h b/src/secp256k1/src/hash_impl.h index 782f97216..409772587 100644 --- a/src/secp256k1/src/hash_impl.h +++ b/src/secp256k1/src/hash_impl.h @@ -8,6 +8,7 @@ #define SECP256K1_HASH_IMPL_H #include "hash.h" +#include "util.h" #include <stdlib.h> #include <stdint.h> @@ -27,9 +28,9 @@ (h) = t1 + t2; \ } while(0) -#ifdef WORDS_BIGENDIAN +#if defined(SECP256K1_BIG_ENDIAN) #define BE32(x) (x) -#else +#elif defined(SECP256K1_LITTLE_ENDIAN) #define BE32(p) ((((p) & 0xFF) << 24) | (((p) & 0xFF00) << 8) | (((p) & 0xFF0000) >> 8) | (((p) & 0xFF000000) >> 24)) #endif @@ -163,6 +164,19 @@ static void secp256k1_sha256_finalize(secp256k1_sha256 *hash, unsigned char *out memcpy(out32, (const unsigned char*)out, 32); } +/* Initializes a sha256 struct and writes the 64 byte string + * SHA256(tag)||SHA256(tag) into it. */ +static void secp256k1_sha256_initialize_tagged(secp256k1_sha256 *hash, const unsigned char *tag, size_t taglen) { + unsigned char buf[32]; + secp256k1_sha256_initialize(hash); + secp256k1_sha256_write(hash, tag, taglen); + secp256k1_sha256_finalize(hash, buf); + + secp256k1_sha256_initialize(hash); + secp256k1_sha256_write(hash, buf, 32); + secp256k1_sha256_write(hash, buf, 32); +} + static void secp256k1_hmac_sha256_initialize(secp256k1_hmac_sha256 *hash, const unsigned char *key, size_t keylen) { size_t n; unsigned char rkey[64]; diff --git a/src/secp256k1/src/modules/extrakeys/Makefile.am.include b/src/secp256k1/src/modules/extrakeys/Makefile.am.include new file mode 100644 index 000000000..8515f92e7 --- /dev/null +++ b/src/secp256k1/src/modules/extrakeys/Makefile.am.include @@ -0,0 +1,3 @@ +include_HEADERS += include/secp256k1_extrakeys.h +noinst_HEADERS += src/modules/extrakeys/tests_impl.h +noinst_HEADERS += src/modules/extrakeys/main_impl.h diff --git a/src/secp256k1/src/modules/extrakeys/main_impl.h b/src/secp256k1/src/modules/extrakeys/main_impl.h new file mode 100644 index 000000000..d31921535 --- /dev/null +++ b/src/secp256k1/src/modules/extrakeys/main_impl.h @@ -0,0 +1,248 @@ +/********************************************************************** + * Copyright (c) 2020 Jonas Nick * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _SECP256K1_MODULE_EXTRAKEYS_MAIN_ +#define _SECP256K1_MODULE_EXTRAKEYS_MAIN_ + +#include "include/secp256k1.h" +#include "include/secp256k1_extrakeys.h" + +static SECP256K1_INLINE int secp256k1_xonly_pubkey_load(const secp256k1_context* ctx, secp256k1_ge *ge, const secp256k1_xonly_pubkey *pubkey) { + return secp256k1_pubkey_load(ctx, ge, (const secp256k1_pubkey *) pubkey); +} + +static SECP256K1_INLINE void secp256k1_xonly_pubkey_save(secp256k1_xonly_pubkey *pubkey, secp256k1_ge *ge) { + secp256k1_pubkey_save((secp256k1_pubkey *) pubkey, ge); +} + +int secp256k1_xonly_pubkey_parse(const secp256k1_context* ctx, secp256k1_xonly_pubkey *pubkey, const unsigned char *input32) { + secp256k1_ge pk; + secp256k1_fe x; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(pubkey != NULL); + memset(pubkey, 0, sizeof(*pubkey)); + ARG_CHECK(input32 != NULL); + + if (!secp256k1_fe_set_b32(&x, input32)) { + return 0; + } + if (!secp256k1_ge_set_xo_var(&pk, &x, 0)) { + return 0; + } + secp256k1_xonly_pubkey_save(pubkey, &pk); + return 1; +} + +int secp256k1_xonly_pubkey_serialize(const secp256k1_context* ctx, unsigned char *output32, const secp256k1_xonly_pubkey *pubkey) { + secp256k1_ge pk; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(output32 != NULL); + memset(output32, 0, 32); + ARG_CHECK(pubkey != NULL); + + if (!secp256k1_xonly_pubkey_load(ctx, &pk, pubkey)) { + return 0; + } + secp256k1_fe_get_b32(output32, &pk.x); + return 1; +} + +/** Keeps a group element as is if it has an even Y and otherwise negates it. + * y_parity is set to 0 in the former case and to 1 in the latter case. + * Requires that the coordinates of r are normalized. */ +static int secp256k1_extrakeys_ge_even_y(secp256k1_ge *r) { + int y_parity = 0; + VERIFY_CHECK(!secp256k1_ge_is_infinity(r)); + + if (secp256k1_fe_is_odd(&r->y)) { + secp256k1_fe_negate(&r->y, &r->y, 1); + y_parity = 1; + } + return y_parity; +} + +int secp256k1_xonly_pubkey_from_pubkey(const secp256k1_context* ctx, secp256k1_xonly_pubkey *xonly_pubkey, int *pk_parity, const secp256k1_pubkey *pubkey) { + secp256k1_ge pk; + int tmp; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(xonly_pubkey != NULL); + ARG_CHECK(pubkey != NULL); + + if (!secp256k1_pubkey_load(ctx, &pk, pubkey)) { + return 0; + } + tmp = secp256k1_extrakeys_ge_even_y(&pk); + if (pk_parity != NULL) { + *pk_parity = tmp; + } + secp256k1_xonly_pubkey_save(xonly_pubkey, &pk); + return 1; +} + +int secp256k1_xonly_pubkey_tweak_add(const secp256k1_context* ctx, secp256k1_pubkey *output_pubkey, const secp256k1_xonly_pubkey *internal_pubkey, const unsigned char *tweak32) { + secp256k1_ge pk; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(output_pubkey != NULL); + memset(output_pubkey, 0, sizeof(*output_pubkey)); + ARG_CHECK(secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)); + ARG_CHECK(internal_pubkey != NULL); + ARG_CHECK(tweak32 != NULL); + + if (!secp256k1_xonly_pubkey_load(ctx, &pk, internal_pubkey) + || !secp256k1_ec_pubkey_tweak_add_helper(&ctx->ecmult_ctx, &pk, tweak32)) { + return 0; + } + secp256k1_pubkey_save(output_pubkey, &pk); + return 1; +} + +int secp256k1_xonly_pubkey_tweak_add_check(const secp256k1_context* ctx, const unsigned char *tweaked_pubkey32, int tweaked_pk_parity, const secp256k1_xonly_pubkey *internal_pubkey, const unsigned char *tweak32) { + secp256k1_ge pk; + unsigned char pk_expected32[32]; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)); + ARG_CHECK(internal_pubkey != NULL); + ARG_CHECK(tweaked_pubkey32 != NULL); + ARG_CHECK(tweak32 != NULL); + + if (!secp256k1_xonly_pubkey_load(ctx, &pk, internal_pubkey) + || !secp256k1_ec_pubkey_tweak_add_helper(&ctx->ecmult_ctx, &pk, tweak32)) { + return 0; + } + secp256k1_fe_normalize_var(&pk.x); + secp256k1_fe_normalize_var(&pk.y); + secp256k1_fe_get_b32(pk_expected32, &pk.x); + + return memcmp(&pk_expected32, tweaked_pubkey32, 32) == 0 + && secp256k1_fe_is_odd(&pk.y) == tweaked_pk_parity; +} + +static void secp256k1_keypair_save(secp256k1_keypair *keypair, const secp256k1_scalar *sk, secp256k1_ge *pk) { + secp256k1_scalar_get_b32(&keypair->data[0], sk); + secp256k1_pubkey_save((secp256k1_pubkey *)&keypair->data[32], pk); +} + + +static int secp256k1_keypair_seckey_load(const secp256k1_context* ctx, secp256k1_scalar *sk, const secp256k1_keypair *keypair) { + int ret; + + ret = secp256k1_scalar_set_b32_seckey(sk, &keypair->data[0]); + /* We can declassify ret here because sk is only zero if a keypair function + * failed (which zeroes the keypair) and its return value is ignored. */ + secp256k1_declassify(ctx, &ret, sizeof(ret)); + ARG_CHECK(ret); + return ret; +} + +/* Load a keypair into pk and sk (if non-NULL). This function declassifies pk + * and ARG_CHECKs that the keypair is not invalid. It always initializes sk and + * pk with dummy values. */ +static int secp256k1_keypair_load(const secp256k1_context* ctx, secp256k1_scalar *sk, secp256k1_ge *pk, const secp256k1_keypair *keypair) { + int ret; + const secp256k1_pubkey *pubkey = (const secp256k1_pubkey *)&keypair->data[32]; + + /* Need to declassify the pubkey because pubkey_load ARG_CHECKs if it's + * invalid. */ + secp256k1_declassify(ctx, pubkey, sizeof(*pubkey)); + ret = secp256k1_pubkey_load(ctx, pk, pubkey); + if (sk != NULL) { + ret = ret && secp256k1_keypair_seckey_load(ctx, sk, keypair); + } + if (!ret) { + *pk = secp256k1_ge_const_g; + if (sk != NULL) { + *sk = secp256k1_scalar_one; + } + } + return ret; +} + +int secp256k1_keypair_create(const secp256k1_context* ctx, secp256k1_keypair *keypair, const unsigned char *seckey32) { + secp256k1_scalar sk; + secp256k1_ge pk; + int ret = 0; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(keypair != NULL); + memset(keypair, 0, sizeof(*keypair)); + ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); + ARG_CHECK(seckey32 != NULL); + + ret = secp256k1_ec_pubkey_create_helper(&ctx->ecmult_gen_ctx, &sk, &pk, seckey32); + secp256k1_keypair_save(keypair, &sk, &pk); + memczero(keypair, sizeof(*keypair), !ret); + + secp256k1_scalar_clear(&sk); + return ret; +} + +int secp256k1_keypair_pub(const secp256k1_context* ctx, secp256k1_pubkey *pubkey, const secp256k1_keypair *keypair) { + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(pubkey != NULL); + memset(pubkey, 0, sizeof(*pubkey)); + ARG_CHECK(keypair != NULL); + + memcpy(pubkey->data, &keypair->data[32], sizeof(*pubkey)); + return 1; +} + +int secp256k1_keypair_xonly_pub(const secp256k1_context* ctx, secp256k1_xonly_pubkey *pubkey, int *pk_parity, const secp256k1_keypair *keypair) { + secp256k1_ge pk; + int tmp; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(pubkey != NULL); + memset(pubkey, 0, sizeof(*pubkey)); + ARG_CHECK(keypair != NULL); + + if (!secp256k1_keypair_load(ctx, NULL, &pk, keypair)) { + return 0; + } + tmp = secp256k1_extrakeys_ge_even_y(&pk); + if (pk_parity != NULL) { + *pk_parity = tmp; + } + secp256k1_xonly_pubkey_save(pubkey, &pk); + + return 1; +} + +int secp256k1_keypair_xonly_tweak_add(const secp256k1_context* ctx, secp256k1_keypair *keypair, const unsigned char *tweak32) { + secp256k1_ge pk; + secp256k1_scalar sk; + int y_parity; + int ret; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)); + ARG_CHECK(keypair != NULL); + ARG_CHECK(tweak32 != NULL); + + ret = secp256k1_keypair_load(ctx, &sk, &pk, keypair); + memset(keypair, 0, sizeof(*keypair)); + + y_parity = secp256k1_extrakeys_ge_even_y(&pk); + if (y_parity == 1) { + secp256k1_scalar_negate(&sk, &sk); + } + + ret &= secp256k1_ec_seckey_tweak_add_helper(&sk, tweak32); + ret &= secp256k1_ec_pubkey_tweak_add_helper(&ctx->ecmult_ctx, &pk, tweak32); + + secp256k1_declassify(ctx, &ret, sizeof(ret)); + if (ret) { + secp256k1_keypair_save(keypair, &sk, &pk); + } + + secp256k1_scalar_clear(&sk); + return ret; +} + +#endif diff --git a/src/secp256k1/src/modules/extrakeys/tests_impl.h b/src/secp256k1/src/modules/extrakeys/tests_impl.h new file mode 100644 index 000000000..fc9d40eda --- /dev/null +++ b/src/secp256k1/src/modules/extrakeys/tests_impl.h @@ -0,0 +1,524 @@ +/********************************************************************** + * Copyright (c) 2020 Jonas Nick * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _SECP256K1_MODULE_EXTRAKEYS_TESTS_ +#define _SECP256K1_MODULE_EXTRAKEYS_TESTS_ + +#include "secp256k1_extrakeys.h" + +static secp256k1_context* api_test_context(int flags, int *ecount) { + secp256k1_context *ctx0 = secp256k1_context_create(flags); + secp256k1_context_set_error_callback(ctx0, counting_illegal_callback_fn, ecount); + secp256k1_context_set_illegal_callback(ctx0, counting_illegal_callback_fn, ecount); + return ctx0; +} + +void test_xonly_pubkey(void) { + secp256k1_pubkey pk; + secp256k1_xonly_pubkey xonly_pk, xonly_pk_tmp; + secp256k1_ge pk1; + secp256k1_ge pk2; + secp256k1_fe y; + unsigned char sk[32]; + unsigned char xy_sk[32]; + unsigned char buf32[32]; + unsigned char ones32[32]; + unsigned char zeros64[64] = { 0 }; + int pk_parity; + int i; + + int ecount; + secp256k1_context *none = api_test_context(SECP256K1_CONTEXT_NONE, &ecount); + secp256k1_context *sign = api_test_context(SECP256K1_CONTEXT_SIGN, &ecount); + secp256k1_context *verify = api_test_context(SECP256K1_CONTEXT_VERIFY, &ecount); + + secp256k1_rand256(sk); + memset(ones32, 0xFF, 32); + secp256k1_rand256(xy_sk); + CHECK(secp256k1_ec_pubkey_create(sign, &pk, sk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &xonly_pk, &pk_parity, &pk) == 1); + + /* Test xonly_pubkey_from_pubkey */ + ecount = 0; + CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &xonly_pk, &pk_parity, &pk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(sign, &xonly_pk, &pk_parity, &pk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(verify, &xonly_pk, &pk_parity, &pk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(none, NULL, &pk_parity, &pk) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &xonly_pk, NULL, &pk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &xonly_pk, &pk_parity, NULL) == 0); + CHECK(ecount == 2); + memset(&pk, 0, sizeof(pk)); + CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &xonly_pk, &pk_parity, &pk) == 0); + CHECK(ecount == 3); + + /* Choose a secret key such that the resulting pubkey and xonly_pubkey match. */ + memset(sk, 0, sizeof(sk)); + sk[0] = 1; + CHECK(secp256k1_ec_pubkey_create(ctx, &pk, sk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &xonly_pk, &pk_parity, &pk) == 1); + CHECK(memcmp(&pk, &xonly_pk, sizeof(pk)) == 0); + CHECK(pk_parity == 0); + + /* Choose a secret key such that pubkey and xonly_pubkey are each others + * negation. */ + sk[0] = 2; + CHECK(secp256k1_ec_pubkey_create(ctx, &pk, sk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &xonly_pk, &pk_parity, &pk) == 1); + CHECK(memcmp(&xonly_pk, &pk, sizeof(xonly_pk)) != 0); + CHECK(pk_parity == 1); + secp256k1_pubkey_load(ctx, &pk1, &pk); + secp256k1_pubkey_load(ctx, &pk2, (secp256k1_pubkey *) &xonly_pk); + CHECK(secp256k1_fe_equal(&pk1.x, &pk2.x) == 1); + secp256k1_fe_negate(&y, &pk2.y, 1); + CHECK(secp256k1_fe_equal(&pk1.y, &y) == 1); + + /* Test xonly_pubkey_serialize and xonly_pubkey_parse */ + ecount = 0; + CHECK(secp256k1_xonly_pubkey_serialize(none, NULL, &xonly_pk) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_xonly_pubkey_serialize(none, buf32, NULL) == 0); + CHECK(memcmp(buf32, zeros64, 32) == 0); + CHECK(ecount == 2); + { + /* A pubkey filled with 0s will fail to serialize due to pubkey_load + * special casing. */ + secp256k1_xonly_pubkey pk_tmp; + memset(&pk_tmp, 0, sizeof(pk_tmp)); + CHECK(secp256k1_xonly_pubkey_serialize(none, buf32, &pk_tmp) == 0); + } + /* pubkey_load called illegal callback */ + CHECK(ecount == 3); + + CHECK(secp256k1_xonly_pubkey_serialize(none, buf32, &xonly_pk) == 1); + ecount = 0; + CHECK(secp256k1_xonly_pubkey_parse(none, NULL, buf32) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_xonly_pubkey_parse(none, &xonly_pk, NULL) == 0); + CHECK(ecount == 2); + + /* Serialization and parse roundtrip */ + CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &xonly_pk, NULL, &pk) == 1); + CHECK(secp256k1_xonly_pubkey_serialize(ctx, buf32, &xonly_pk) == 1); + CHECK(secp256k1_xonly_pubkey_parse(ctx, &xonly_pk_tmp, buf32) == 1); + CHECK(memcmp(&xonly_pk, &xonly_pk_tmp, sizeof(xonly_pk)) == 0); + + /* Test parsing invalid field elements */ + memset(&xonly_pk, 1, sizeof(xonly_pk)); + /* Overflowing field element */ + CHECK(secp256k1_xonly_pubkey_parse(none, &xonly_pk, ones32) == 0); + CHECK(memcmp(&xonly_pk, zeros64, sizeof(xonly_pk)) == 0); + memset(&xonly_pk, 1, sizeof(xonly_pk)); + /* There's no point with x-coordinate 0 on secp256k1 */ + CHECK(secp256k1_xonly_pubkey_parse(none, &xonly_pk, zeros64) == 0); + CHECK(memcmp(&xonly_pk, zeros64, sizeof(xonly_pk)) == 0); + /* If a random 32-byte string can not be parsed with ec_pubkey_parse + * (because interpreted as X coordinate it does not correspond to a point on + * the curve) then xonly_pubkey_parse should fail as well. */ + for (i = 0; i < count; i++) { + unsigned char rand33[33]; + secp256k1_rand256(&rand33[1]); + rand33[0] = SECP256K1_TAG_PUBKEY_EVEN; + if (!secp256k1_ec_pubkey_parse(ctx, &pk, rand33, 33)) { + memset(&xonly_pk, 1, sizeof(xonly_pk)); + CHECK(secp256k1_xonly_pubkey_parse(ctx, &xonly_pk, &rand33[1]) == 0); + CHECK(memcmp(&xonly_pk, zeros64, sizeof(xonly_pk)) == 0); + } else { + CHECK(secp256k1_xonly_pubkey_parse(ctx, &xonly_pk, &rand33[1]) == 1); + } + } + CHECK(ecount == 2); + + secp256k1_context_destroy(none); + secp256k1_context_destroy(sign); + secp256k1_context_destroy(verify); +} + +void test_xonly_pubkey_tweak(void) { + unsigned char zeros64[64] = { 0 }; + unsigned char overflows[32]; + unsigned char sk[32]; + secp256k1_pubkey internal_pk; + secp256k1_xonly_pubkey internal_xonly_pk; + secp256k1_pubkey output_pk; + int pk_parity; + unsigned char tweak[32]; + int i; + + int ecount; + secp256k1_context *none = api_test_context(SECP256K1_CONTEXT_NONE, &ecount); + secp256k1_context *sign = api_test_context(SECP256K1_CONTEXT_SIGN, &ecount); + secp256k1_context *verify = api_test_context(SECP256K1_CONTEXT_VERIFY, &ecount); + + memset(overflows, 0xff, sizeof(overflows)); + secp256k1_rand256(tweak); + secp256k1_rand256(sk); + CHECK(secp256k1_ec_pubkey_create(ctx, &internal_pk, sk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &internal_xonly_pk, &pk_parity, &internal_pk) == 1); + + ecount = 0; + CHECK(secp256k1_xonly_pubkey_tweak_add(none, &output_pk, &internal_xonly_pk, tweak) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add(sign, &output_pk, &internal_xonly_pk, tweak) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, tweak) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add(verify, NULL, &internal_xonly_pk, tweak) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, NULL, tweak) == 0); + CHECK(ecount == 4); + /* NULL internal_xonly_pk zeroes the output_pk */ + CHECK(memcmp(&output_pk, zeros64, sizeof(output_pk)) == 0); + CHECK(secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, NULL) == 0); + CHECK(ecount == 5); + /* NULL tweak zeroes the output_pk */ + CHECK(memcmp(&output_pk, zeros64, sizeof(output_pk)) == 0); + + /* Invalid tweak zeroes the output_pk */ + CHECK(secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, overflows) == 0); + CHECK(memcmp(&output_pk, zeros64, sizeof(output_pk)) == 0); + + /* A zero tweak is fine */ + CHECK(secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, zeros64) == 1); + + /* Fails if the resulting key was infinity */ + for (i = 0; i < count; i++) { + secp256k1_scalar scalar_tweak; + /* Because sk may be negated before adding, we need to try with tweak = + * sk as well as tweak = -sk. */ + secp256k1_scalar_set_b32(&scalar_tweak, sk, NULL); + secp256k1_scalar_negate(&scalar_tweak, &scalar_tweak); + secp256k1_scalar_get_b32(tweak, &scalar_tweak); + CHECK((secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, sk) == 0) + || (secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, tweak) == 0)); + CHECK(memcmp(&output_pk, zeros64, sizeof(output_pk)) == 0); + } + + /* Invalid pk with a valid tweak */ + memset(&internal_xonly_pk, 0, sizeof(internal_xonly_pk)); + secp256k1_rand256(tweak); + ecount = 0; + CHECK(secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, tweak) == 0); + CHECK(ecount == 1); + CHECK(memcmp(&output_pk, zeros64, sizeof(output_pk)) == 0); + + secp256k1_context_destroy(none); + secp256k1_context_destroy(sign); + secp256k1_context_destroy(verify); +} + +void test_xonly_pubkey_tweak_check(void) { + unsigned char zeros64[64] = { 0 }; + unsigned char overflows[32]; + unsigned char sk[32]; + secp256k1_pubkey internal_pk; + secp256k1_xonly_pubkey internal_xonly_pk; + secp256k1_pubkey output_pk; + secp256k1_xonly_pubkey output_xonly_pk; + unsigned char output_pk32[32]; + unsigned char buf32[32]; + int pk_parity; + unsigned char tweak[32]; + + int ecount; + secp256k1_context *none = api_test_context(SECP256K1_CONTEXT_NONE, &ecount); + secp256k1_context *sign = api_test_context(SECP256K1_CONTEXT_SIGN, &ecount); + secp256k1_context *verify = api_test_context(SECP256K1_CONTEXT_VERIFY, &ecount); + + memset(overflows, 0xff, sizeof(overflows)); + secp256k1_rand256(tweak); + secp256k1_rand256(sk); + CHECK(secp256k1_ec_pubkey_create(ctx, &internal_pk, sk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &internal_xonly_pk, &pk_parity, &internal_pk) == 1); + + ecount = 0; + CHECK(secp256k1_xonly_pubkey_tweak_add(verify, &output_pk, &internal_xonly_pk, tweak) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(verify, &output_xonly_pk, &pk_parity, &output_pk) == 1); + CHECK(secp256k1_xonly_pubkey_serialize(ctx, buf32, &output_xonly_pk) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(none, buf32, pk_parity, &internal_xonly_pk, tweak) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(sign, buf32, pk_parity, &internal_xonly_pk, tweak) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(verify, buf32, pk_parity, &internal_xonly_pk, tweak) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(verify, NULL, pk_parity, &internal_xonly_pk, tweak) == 0); + CHECK(ecount == 3); + /* invalid pk_parity value */ + CHECK(secp256k1_xonly_pubkey_tweak_add_check(verify, buf32, 2, &internal_xonly_pk, tweak) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(verify, buf32, pk_parity, NULL, tweak) == 0); + CHECK(ecount == 4); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(verify, buf32, pk_parity, &internal_xonly_pk, NULL) == 0); + CHECK(ecount == 5); + + memset(tweak, 1, sizeof(tweak)); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &internal_xonly_pk, NULL, &internal_pk) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk, &internal_xonly_pk, tweak) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &output_xonly_pk, &pk_parity, &output_pk) == 1); + CHECK(secp256k1_xonly_pubkey_serialize(ctx, output_pk32, &output_xonly_pk) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(ctx, output_pk32, pk_parity, &internal_xonly_pk, tweak) == 1); + + /* Wrong pk_parity */ + CHECK(secp256k1_xonly_pubkey_tweak_add_check(ctx, output_pk32, !pk_parity, &internal_xonly_pk, tweak) == 0); + /* Wrong public key */ + CHECK(secp256k1_xonly_pubkey_serialize(ctx, buf32, &internal_xonly_pk) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(ctx, buf32, pk_parity, &internal_xonly_pk, tweak) == 0); + + /* Overflowing tweak not allowed */ + CHECK(secp256k1_xonly_pubkey_tweak_add_check(ctx, output_pk32, pk_parity, &internal_xonly_pk, overflows) == 0); + CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk, &internal_xonly_pk, overflows) == 0); + CHECK(memcmp(&output_pk, zeros64, sizeof(output_pk)) == 0); + CHECK(ecount == 5); + + secp256k1_context_destroy(none); + secp256k1_context_destroy(sign); + secp256k1_context_destroy(verify); +} + +/* Starts with an initial pubkey and recursively creates N_PUBKEYS - 1 + * additional pubkeys by calling tweak_add. Then verifies every tweak starting + * from the last pubkey. */ +#define N_PUBKEYS 32 +void test_xonly_pubkey_tweak_recursive(void) { + unsigned char sk[32]; + secp256k1_pubkey pk[N_PUBKEYS]; + unsigned char pk_serialized[32]; + unsigned char tweak[N_PUBKEYS - 1][32]; + int i; + + secp256k1_rand256(sk); + CHECK(secp256k1_ec_pubkey_create(ctx, &pk[0], sk) == 1); + /* Add tweaks */ + for (i = 0; i < N_PUBKEYS - 1; i++) { + secp256k1_xonly_pubkey xonly_pk; + memset(tweak[i], i + 1, sizeof(tweak[i])); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &xonly_pk, NULL, &pk[i]) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, &pk[i + 1], &xonly_pk, tweak[i]) == 1); + } + + /* Verify tweaks */ + for (i = N_PUBKEYS - 1; i > 0; i--) { + secp256k1_xonly_pubkey xonly_pk; + int pk_parity; + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &xonly_pk, &pk_parity, &pk[i]) == 1); + CHECK(secp256k1_xonly_pubkey_serialize(ctx, pk_serialized, &xonly_pk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(ctx, &xonly_pk, NULL, &pk[i - 1]) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(ctx, pk_serialized, pk_parity, &xonly_pk, tweak[i - 1]) == 1); + } +} +#undef N_PUBKEYS + +void test_keypair(void) { + unsigned char sk[32]; + unsigned char zeros96[96] = { 0 }; + unsigned char overflows[32]; + secp256k1_keypair keypair; + secp256k1_pubkey pk, pk_tmp; + secp256k1_xonly_pubkey xonly_pk, xonly_pk_tmp; + int pk_parity, pk_parity_tmp; + int ecount; + secp256k1_context *none = api_test_context(SECP256K1_CONTEXT_NONE, &ecount); + secp256k1_context *sign = api_test_context(SECP256K1_CONTEXT_SIGN, &ecount); + secp256k1_context *verify = api_test_context(SECP256K1_CONTEXT_VERIFY, &ecount); + + CHECK(sizeof(zeros96) == sizeof(keypair)); + memset(overflows, 0xFF, sizeof(overflows)); + + /* Test keypair_create */ + ecount = 0; + secp256k1_rand256(sk); + CHECK(secp256k1_keypair_create(none, &keypair, sk) == 0); + CHECK(memcmp(zeros96, &keypair, sizeof(keypair)) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_keypair_create(verify, &keypair, sk) == 0); + CHECK(memcmp(zeros96, &keypair, sizeof(keypair)) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_keypair_create(sign, &keypair, sk) == 1); + CHECK(secp256k1_keypair_create(sign, NULL, sk) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_keypair_create(sign, &keypair, NULL) == 0); + CHECK(memcmp(zeros96, &keypair, sizeof(keypair)) == 0); + CHECK(ecount == 4); + + /* Invalid secret key */ + CHECK(secp256k1_keypair_create(sign, &keypair, zeros96) == 0); + CHECK(memcmp(zeros96, &keypair, sizeof(keypair)) == 0); + CHECK(secp256k1_keypair_create(sign, &keypair, overflows) == 0); + CHECK(memcmp(zeros96, &keypair, sizeof(keypair)) == 0); + + /* Test keypair_pub */ + ecount = 0; + secp256k1_rand256(sk); + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); + CHECK(secp256k1_keypair_pub(none, &pk, &keypair) == 1); + CHECK(secp256k1_keypair_pub(none, NULL, &keypair) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_keypair_pub(none, &pk, NULL) == 0); + CHECK(ecount == 2); + CHECK(memcmp(zeros96, &pk, sizeof(pk)) == 0); + + /* Using an invalid keypair is fine for keypair_pub */ + memset(&keypair, 0, sizeof(keypair)); + CHECK(secp256k1_keypair_pub(none, &pk, &keypair) == 1); + CHECK(memcmp(zeros96, &pk, sizeof(pk)) == 0); + + /* keypair holds the same pubkey as pubkey_create */ + CHECK(secp256k1_ec_pubkey_create(sign, &pk, sk) == 1); + CHECK(secp256k1_keypair_create(sign, &keypair, sk) == 1); + CHECK(secp256k1_keypair_pub(none, &pk_tmp, &keypair) == 1); + CHECK(memcmp(&pk, &pk_tmp, sizeof(pk)) == 0); + + /** Test keypair_xonly_pub **/ + ecount = 0; + secp256k1_rand256(sk); + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); + CHECK(secp256k1_keypair_xonly_pub(none, &xonly_pk, &pk_parity, &keypair) == 1); + CHECK(secp256k1_keypair_xonly_pub(none, NULL, &pk_parity, &keypair) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_keypair_xonly_pub(none, &xonly_pk, NULL, &keypair) == 1); + CHECK(secp256k1_keypair_xonly_pub(none, &xonly_pk, &pk_parity, NULL) == 0); + CHECK(ecount == 2); + CHECK(memcmp(zeros96, &xonly_pk, sizeof(xonly_pk)) == 0); + /* Using an invalid keypair will set the xonly_pk to 0 (first reset + * xonly_pk). */ + CHECK(secp256k1_keypair_xonly_pub(none, &xonly_pk, &pk_parity, &keypair) == 1); + memset(&keypair, 0, sizeof(keypair)); + CHECK(secp256k1_keypair_xonly_pub(none, &xonly_pk, &pk_parity, &keypair) == 0); + CHECK(memcmp(zeros96, &xonly_pk, sizeof(xonly_pk)) == 0); + CHECK(ecount == 3); + + /** keypair holds the same xonly pubkey as pubkey_create **/ + CHECK(secp256k1_ec_pubkey_create(sign, &pk, sk) == 1); + CHECK(secp256k1_xonly_pubkey_from_pubkey(none, &xonly_pk, &pk_parity, &pk) == 1); + CHECK(secp256k1_keypair_create(sign, &keypair, sk) == 1); + CHECK(secp256k1_keypair_xonly_pub(none, &xonly_pk_tmp, &pk_parity_tmp, &keypair) == 1); + CHECK(memcmp(&xonly_pk, &xonly_pk_tmp, sizeof(pk)) == 0); + CHECK(pk_parity == pk_parity_tmp); + + secp256k1_context_destroy(none); + secp256k1_context_destroy(sign); + secp256k1_context_destroy(verify); +} + +void test_keypair_add(void) { + unsigned char sk[32]; + secp256k1_keypair keypair; + unsigned char overflows[32]; + unsigned char zeros96[96] = { 0 }; + unsigned char tweak[32]; + int i; + int ecount = 0; + secp256k1_context *none = api_test_context(SECP256K1_CONTEXT_NONE, &ecount); + secp256k1_context *sign = api_test_context(SECP256K1_CONTEXT_SIGN, &ecount); + secp256k1_context *verify = api_test_context(SECP256K1_CONTEXT_VERIFY, &ecount); + + CHECK(sizeof(zeros96) == sizeof(keypair)); + secp256k1_rand256(sk); + secp256k1_rand256(tweak); + memset(overflows, 0xFF, 32); + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); + + CHECK(secp256k1_keypair_xonly_tweak_add(none, &keypair, tweak) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_keypair_xonly_tweak_add(sign, &keypair, tweak) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_keypair_xonly_tweak_add(verify, &keypair, tweak) == 1); + CHECK(secp256k1_keypair_xonly_tweak_add(verify, NULL, tweak) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_keypair_xonly_tweak_add(verify, &keypair, NULL) == 0); + CHECK(ecount == 4); + /* This does not set the keypair to zeroes */ + CHECK(memcmp(&keypair, zeros96, sizeof(keypair)) != 0); + + /* Invalid tweak zeroes the keypair */ + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); + CHECK(secp256k1_keypair_xonly_tweak_add(ctx, &keypair, overflows) == 0); + CHECK(memcmp(&keypair, zeros96, sizeof(keypair)) == 0); + + /* A zero tweak is fine */ + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); + CHECK(secp256k1_keypair_xonly_tweak_add(ctx, &keypair, zeros96) == 1); + + /* Fails if the resulting keypair was (sk=0, pk=infinity) */ + for (i = 0; i < count; i++) { + secp256k1_scalar scalar_tweak; + secp256k1_keypair keypair_tmp; + secp256k1_rand256(sk); + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); + memcpy(&keypair_tmp, &keypair, sizeof(keypair)); + /* Because sk may be negated before adding, we need to try with tweak = + * sk as well as tweak = -sk. */ + secp256k1_scalar_set_b32(&scalar_tweak, sk, NULL); + secp256k1_scalar_negate(&scalar_tweak, &scalar_tweak); + secp256k1_scalar_get_b32(tweak, &scalar_tweak); + CHECK((secp256k1_keypair_xonly_tweak_add(ctx, &keypair, sk) == 0) + || (secp256k1_keypair_xonly_tweak_add(ctx, &keypair_tmp, tweak) == 0)); + CHECK(memcmp(&keypair, zeros96, sizeof(keypair)) == 0 + || memcmp(&keypair_tmp, zeros96, sizeof(keypair_tmp)) == 0); + } + + /* Invalid keypair with a valid tweak */ + memset(&keypair, 0, sizeof(keypair)); + secp256k1_rand256(tweak); + ecount = 0; + CHECK(secp256k1_keypair_xonly_tweak_add(verify, &keypair, tweak) == 0); + CHECK(ecount == 1); + CHECK(memcmp(&keypair, zeros96, sizeof(keypair)) == 0); + /* Only seckey part of keypair invalid */ + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); + memset(&keypair, 0, 32); + CHECK(secp256k1_keypair_xonly_tweak_add(verify, &keypair, tweak) == 0); + CHECK(ecount == 2); + /* Only pubkey part of keypair invalid */ + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); + memset(&keypair.data[32], 0, 64); + CHECK(secp256k1_keypair_xonly_tweak_add(verify, &keypair, tweak) == 0); + CHECK(ecount == 3); + + /* Check that the keypair_tweak_add implementation is correct */ + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); + for (i = 0; i < count; i++) { + secp256k1_xonly_pubkey internal_pk; + secp256k1_xonly_pubkey output_pk; + secp256k1_pubkey output_pk_xy; + secp256k1_pubkey output_pk_expected; + unsigned char pk32[32]; + int pk_parity; + + secp256k1_rand256(tweak); + CHECK(secp256k1_keypair_xonly_pub(ctx, &internal_pk, NULL, &keypair) == 1); + CHECK(secp256k1_keypair_xonly_tweak_add(ctx, &keypair, tweak) == 1); + CHECK(secp256k1_keypair_xonly_pub(ctx, &output_pk, &pk_parity, &keypair) == 1); + + /* Check that it passes xonly_pubkey_tweak_add_check */ + CHECK(secp256k1_xonly_pubkey_serialize(ctx, pk32, &output_pk) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(ctx, pk32, pk_parity, &internal_pk, tweak) == 1); + + /* Check that the resulting pubkey matches xonly_pubkey_tweak_add */ + CHECK(secp256k1_keypair_pub(ctx, &output_pk_xy, &keypair) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add(ctx, &output_pk_expected, &internal_pk, tweak) == 1); + CHECK(memcmp(&output_pk_xy, &output_pk_expected, sizeof(output_pk_xy)) == 0); + + /* Check that the secret key in the keypair is tweaked correctly */ + CHECK(secp256k1_ec_pubkey_create(ctx, &output_pk_expected, &keypair.data[0]) == 1); + CHECK(memcmp(&output_pk_xy, &output_pk_expected, sizeof(output_pk_xy)) == 0); + } + secp256k1_context_destroy(none); + secp256k1_context_destroy(sign); + secp256k1_context_destroy(verify); +} + +void run_extrakeys_tests(void) { + /* xonly key test cases */ + test_xonly_pubkey(); + test_xonly_pubkey_tweak(); + test_xonly_pubkey_tweak_check(); + test_xonly_pubkey_tweak_recursive(); + + /* keypair tests */ + test_keypair(); + test_keypair_add(); +} + +#endif diff --git a/src/secp256k1/src/modules/schnorrsig/Makefile.am.include b/src/secp256k1/src/modules/schnorrsig/Makefile.am.include new file mode 100644 index 000000000..a82bafe43 --- /dev/null +++ b/src/secp256k1/src/modules/schnorrsig/Makefile.am.include @@ -0,0 +1,8 @@ +include_HEADERS += include/secp256k1_schnorrsig.h +noinst_HEADERS += src/modules/schnorrsig/main_impl.h +noinst_HEADERS += src/modules/schnorrsig/tests_impl.h +if USE_BENCHMARK +noinst_PROGRAMS += bench_schnorrsig +bench_schnorrsig_SOURCES = src/bench_schnorrsig.c +bench_schnorrsig_LDADD = libsecp256k1.la $(SECP_LIBS) $(COMMON_LIB) +endif diff --git a/src/secp256k1/src/modules/schnorrsig/main_impl.h b/src/secp256k1/src/modules/schnorrsig/main_impl.h new file mode 100644 index 000000000..a0218f881 --- /dev/null +++ b/src/secp256k1/src/modules/schnorrsig/main_impl.h @@ -0,0 +1,238 @@ +/********************************************************************** + * Copyright (c) 2018-2020 Andrew Poelstra, Jonas Nick * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _SECP256K1_MODULE_SCHNORRSIG_MAIN_ +#define _SECP256K1_MODULE_SCHNORRSIG_MAIN_ + +#include "include/secp256k1.h" +#include "include/secp256k1_schnorrsig.h" +#include "hash.h" + +/* Initializes SHA256 with fixed midstate. This midstate was computed by applying + * SHA256 to SHA256("BIP0340/nonce")||SHA256("BIP0340/nonce"). */ +static void secp256k1_nonce_function_bip340_sha256_tagged(secp256k1_sha256 *sha) { + secp256k1_sha256_initialize(sha); + sha->s[0] = 0x46615b35ul; + sha->s[1] = 0xf4bfbff7ul; + sha->s[2] = 0x9f8dc671ul; + sha->s[3] = 0x83627ab3ul; + sha->s[4] = 0x60217180ul; + sha->s[5] = 0x57358661ul; + sha->s[6] = 0x21a29e54ul; + sha->s[7] = 0x68b07b4cul; + + sha->bytes = 64; +} + +/* Initializes SHA256 with fixed midstate. This midstate was computed by applying + * SHA256 to SHA256("BIP0340/aux")||SHA256("BIP0340/aux"). */ +static void secp256k1_nonce_function_bip340_sha256_tagged_aux(secp256k1_sha256 *sha) { + secp256k1_sha256_initialize(sha); + sha->s[0] = 0x24dd3219ul; + sha->s[1] = 0x4eba7e70ul; + sha->s[2] = 0xca0fabb9ul; + sha->s[3] = 0x0fa3166dul; + sha->s[4] = 0x3afbe4b1ul; + sha->s[5] = 0x4c44df97ul; + sha->s[6] = 0x4aac2739ul; + sha->s[7] = 0x249e850aul; + + sha->bytes = 64; +} + +/* algo16 argument for nonce_function_bip340 to derive the nonce exactly as stated in BIP-340 + * by using the correct tagged hash function. */ +static const unsigned char bip340_algo16[16] = "BIP0340/nonce\0\0\0"; + +static int nonce_function_bip340(unsigned char *nonce32, const unsigned char *msg32, const unsigned char *key32, const unsigned char *xonly_pk32, const unsigned char *algo16, void *data) { + secp256k1_sha256 sha; + unsigned char masked_key[32]; + int i; + + if (algo16 == NULL) { + return 0; + } + + if (data != NULL) { + secp256k1_nonce_function_bip340_sha256_tagged_aux(&sha); + secp256k1_sha256_write(&sha, data, 32); + secp256k1_sha256_finalize(&sha, masked_key); + for (i = 0; i < 32; i++) { + masked_key[i] ^= key32[i]; + } + } + + /* Tag the hash with algo16 which is important to avoid nonce reuse across + * algorithms. If this nonce function is used in BIP-340 signing as defined + * in the spec, an optimized tagging implementation is used. */ + if (memcmp(algo16, bip340_algo16, 16) == 0) { + secp256k1_nonce_function_bip340_sha256_tagged(&sha); + } else { + int algo16_len = 16; + /* Remove terminating null bytes */ + while (algo16_len > 0 && !algo16[algo16_len - 1]) { + algo16_len--; + } + secp256k1_sha256_initialize_tagged(&sha, algo16, algo16_len); + } + + /* Hash (masked-)key||pk||msg using the tagged hash as per the spec */ + if (data != NULL) { + secp256k1_sha256_write(&sha, masked_key, 32); + } else { + secp256k1_sha256_write(&sha, key32, 32); + } + secp256k1_sha256_write(&sha, xonly_pk32, 32); + secp256k1_sha256_write(&sha, msg32, 32); + secp256k1_sha256_finalize(&sha, nonce32); + return 1; +} + +const secp256k1_nonce_function_hardened secp256k1_nonce_function_bip340 = nonce_function_bip340; + +/* Initializes SHA256 with fixed midstate. This midstate was computed by applying + * SHA256 to SHA256("BIP0340/challenge")||SHA256("BIP0340/challenge"). */ +static void secp256k1_schnorrsig_sha256_tagged(secp256k1_sha256 *sha) { + secp256k1_sha256_initialize(sha); + sha->s[0] = 0x9cecba11ul; + sha->s[1] = 0x23925381ul; + sha->s[2] = 0x11679112ul; + sha->s[3] = 0xd1627e0ful; + sha->s[4] = 0x97c87550ul; + sha->s[5] = 0x003cc765ul; + sha->s[6] = 0x90f61164ul; + sha->s[7] = 0x33e9b66aul; + sha->bytes = 64; +} + +int secp256k1_schnorrsig_sign(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg32, const secp256k1_keypair *keypair, secp256k1_nonce_function_hardened noncefp, void *ndata) { + secp256k1_scalar sk; + secp256k1_scalar e; + secp256k1_scalar k; + secp256k1_gej rj; + secp256k1_ge pk; + secp256k1_ge r; + secp256k1_sha256 sha; + unsigned char buf[32] = { 0 }; + unsigned char pk_buf[32]; + unsigned char seckey[32]; + int ret = 1; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); + ARG_CHECK(sig64 != NULL); + ARG_CHECK(msg32 != NULL); + ARG_CHECK(keypair != NULL); + + if (noncefp == NULL) { + noncefp = secp256k1_nonce_function_bip340; + } + + ret &= secp256k1_keypair_load(ctx, &sk, &pk, keypair); + /* Because we are signing for a x-only pubkey, the secret key is negated + * before signing if the point corresponding to the secret key does not + * have an even Y. */ + if (secp256k1_fe_is_odd(&pk.y)) { + secp256k1_scalar_negate(&sk, &sk); + } + + secp256k1_scalar_get_b32(seckey, &sk); + secp256k1_fe_get_b32(pk_buf, &pk.x); + ret &= !!noncefp(buf, msg32, seckey, pk_buf, bip340_algo16, ndata); + secp256k1_scalar_set_b32(&k, buf, NULL); + ret &= !secp256k1_scalar_is_zero(&k); + secp256k1_scalar_cmov(&k, &secp256k1_scalar_one, !ret); + + secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &rj, &k); + secp256k1_ge_set_gej(&r, &rj); + + /* We declassify r to allow using it as a branch point. This is fine + * because r is not a secret. */ + secp256k1_declassify(ctx, &r, sizeof(r)); + secp256k1_fe_normalize_var(&r.y); + if (secp256k1_fe_is_odd(&r.y)) { + secp256k1_scalar_negate(&k, &k); + } + secp256k1_fe_normalize_var(&r.x); + secp256k1_fe_get_b32(&sig64[0], &r.x); + + /* tagged hash(r.x, pk.x, msg32) */ + secp256k1_schnorrsig_sha256_tagged(&sha); + secp256k1_sha256_write(&sha, &sig64[0], 32); + secp256k1_sha256_write(&sha, pk_buf, sizeof(pk_buf)); + secp256k1_sha256_write(&sha, msg32, 32); + secp256k1_sha256_finalize(&sha, buf); + + /* Set scalar e to the challenge hash modulo the curve order as per + * BIP340. */ + secp256k1_scalar_set_b32(&e, buf, NULL); + secp256k1_scalar_mul(&e, &e, &sk); + secp256k1_scalar_add(&e, &e, &k); + secp256k1_scalar_get_b32(&sig64[32], &e); + + memczero(sig64, 64, !ret); + secp256k1_scalar_clear(&k); + secp256k1_scalar_clear(&sk); + memset(seckey, 0, sizeof(seckey)); + + return ret; +} + +int secp256k1_schnorrsig_verify(const secp256k1_context* ctx, const unsigned char *sig64, const unsigned char *msg32, const secp256k1_xonly_pubkey *pubkey) { + secp256k1_scalar s; + secp256k1_scalar e; + secp256k1_gej rj; + secp256k1_ge pk; + secp256k1_gej pkj; + secp256k1_fe rx; + secp256k1_ge r; + secp256k1_sha256 sha; + unsigned char buf[32]; + int overflow; + + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)); + ARG_CHECK(sig64 != NULL); + ARG_CHECK(msg32 != NULL); + ARG_CHECK(pubkey != NULL); + + if (!secp256k1_fe_set_b32(&rx, &sig64[0])) { + return 0; + } + + secp256k1_scalar_set_b32(&s, &sig64[32], &overflow); + if (overflow) { + return 0; + } + + if (!secp256k1_xonly_pubkey_load(ctx, &pk, pubkey)) { + return 0; + } + + secp256k1_schnorrsig_sha256_tagged(&sha); + secp256k1_sha256_write(&sha, &sig64[0], 32); + secp256k1_fe_get_b32(buf, &pk.x); + secp256k1_sha256_write(&sha, buf, sizeof(buf)); + secp256k1_sha256_write(&sha, msg32, 32); + secp256k1_sha256_finalize(&sha, buf); + secp256k1_scalar_set_b32(&e, buf, NULL); + + /* Compute rj = s*G + (-e)*pkj */ + secp256k1_scalar_negate(&e, &e); + secp256k1_gej_set_ge(&pkj, &pk); + secp256k1_ecmult(&ctx->ecmult_ctx, &rj, &pkj, &e, &s); + + secp256k1_ge_set_gej_var(&r, &rj); + if (secp256k1_ge_is_infinity(&r)) { + return 0; + } + + secp256k1_fe_normalize_var(&r.y); + return !secp256k1_fe_is_odd(&r.y) && + secp256k1_fe_equal_var(&rx, &r.x); +} + +#endif diff --git a/src/secp256k1/src/modules/schnorrsig/tests_impl.h b/src/secp256k1/src/modules/schnorrsig/tests_impl.h new file mode 100644 index 000000000..88d8f5640 --- /dev/null +++ b/src/secp256k1/src/modules/schnorrsig/tests_impl.h @@ -0,0 +1,806 @@ +/********************************************************************** + * Copyright (c) 2018-2020 Andrew Poelstra, Jonas Nick * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _SECP256K1_MODULE_SCHNORRSIG_TESTS_ +#define _SECP256K1_MODULE_SCHNORRSIG_TESTS_ + +#include "secp256k1_schnorrsig.h" + +/* Checks that a bit flip in the n_flip-th argument (that has n_bytes many + * bytes) changes the hash function + */ +void nonce_function_bip340_bitflip(unsigned char **args, size_t n_flip, size_t n_bytes) { + unsigned char nonces[2][32]; + CHECK(nonce_function_bip340(nonces[0], args[0], args[1], args[2], args[3], args[4]) == 1); + secp256k1_rand_flip(args[n_flip], n_bytes); + CHECK(nonce_function_bip340(nonces[1], args[0], args[1], args[2], args[3], args[4]) == 1); + CHECK(memcmp(nonces[0], nonces[1], 32) != 0); +} + +/* Tests for the equality of two sha256 structs. This function only produces a + * correct result if an integer multiple of 64 many bytes have been written + * into the hash functions. */ +void test_sha256_eq(const secp256k1_sha256 *sha1, const secp256k1_sha256 *sha2) { + /* Is buffer fully consumed? */ + CHECK((sha1->bytes & 0x3F) == 0); + + CHECK(sha1->bytes == sha2->bytes); + CHECK(memcmp(sha1->s, sha2->s, sizeof(sha1->s)) == 0); +} + +void run_nonce_function_bip340_tests(void) { + unsigned char tag[13] = "BIP0340/nonce"; + unsigned char aux_tag[11] = "BIP0340/aux"; + unsigned char algo16[16] = "BIP0340/nonce\0\0\0"; + secp256k1_sha256 sha; + secp256k1_sha256 sha_optimized; + unsigned char nonce[32]; + unsigned char msg[32]; + unsigned char key[32]; + unsigned char pk[32]; + unsigned char aux_rand[32]; + unsigned char *args[5]; + int i; + + /* Check that hash initialized by + * secp256k1_nonce_function_bip340_sha256_tagged has the expected + * state. */ + secp256k1_sha256_initialize_tagged(&sha, tag, sizeof(tag)); + secp256k1_nonce_function_bip340_sha256_tagged(&sha_optimized); + test_sha256_eq(&sha, &sha_optimized); + + /* Check that hash initialized by + * secp256k1_nonce_function_bip340_sha256_tagged_aux has the expected + * state. */ + secp256k1_sha256_initialize_tagged(&sha, aux_tag, sizeof(aux_tag)); + secp256k1_nonce_function_bip340_sha256_tagged_aux(&sha_optimized); + test_sha256_eq(&sha, &sha_optimized); + + secp256k1_rand256(msg); + secp256k1_rand256(key); + secp256k1_rand256(pk); + secp256k1_rand256(aux_rand); + + /* Check that a bitflip in an argument results in different nonces. */ + args[0] = msg; + args[1] = key; + args[2] = pk; + args[3] = algo16; + args[4] = aux_rand; + for (i = 0; i < count; i++) { + nonce_function_bip340_bitflip(args, 0, 32); + nonce_function_bip340_bitflip(args, 1, 32); + nonce_function_bip340_bitflip(args, 2, 32); + /* Flip algo16 special case "BIP0340/nonce" */ + nonce_function_bip340_bitflip(args, 3, 16); + /* Flip algo16 again */ + nonce_function_bip340_bitflip(args, 3, 16); + nonce_function_bip340_bitflip(args, 4, 32); + } + + /* NULL algo16 is disallowed */ + CHECK(nonce_function_bip340(nonce, msg, key, pk, NULL, NULL) == 0); + /* Empty algo16 is fine */ + memset(algo16, 0x00, 16); + CHECK(nonce_function_bip340(nonce, msg, key, pk, algo16, NULL) == 1); + /* algo16 with terminating null bytes is fine */ + algo16[1] = 65; + CHECK(nonce_function_bip340(nonce, msg, key, pk, algo16, NULL) == 1); + /* Other algo16 is fine */ + memset(algo16, 0xFF, 16); + CHECK(nonce_function_bip340(nonce, msg, key, pk, algo16, NULL) == 1); + + /* NULL aux_rand argument is allowed. */ + CHECK(nonce_function_bip340(nonce, msg, key, pk, algo16, NULL) == 1); +} + +void test_schnorrsig_api(void) { + unsigned char sk1[32]; + unsigned char sk2[32]; + unsigned char sk3[32]; + unsigned char msg[32]; + secp256k1_keypair keypairs[3]; + secp256k1_keypair invalid_keypair = { 0 }; + secp256k1_xonly_pubkey pk[3]; + secp256k1_xonly_pubkey zero_pk; + unsigned char sig[64]; + + /** setup **/ + secp256k1_context *none = secp256k1_context_create(SECP256K1_CONTEXT_NONE); + secp256k1_context *sign = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + secp256k1_context *vrfy = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); + secp256k1_context *both = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + int ecount; + + secp256k1_context_set_error_callback(none, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(sign, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(vrfy, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(both, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(none, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(sign, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(vrfy, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(both, counting_illegal_callback_fn, &ecount); + + secp256k1_rand256(sk1); + secp256k1_rand256(sk2); + secp256k1_rand256(sk3); + secp256k1_rand256(msg); + CHECK(secp256k1_keypair_create(ctx, &keypairs[0], sk1) == 1); + CHECK(secp256k1_keypair_create(ctx, &keypairs[1], sk2) == 1); + CHECK(secp256k1_keypair_create(ctx, &keypairs[2], sk3) == 1); + CHECK(secp256k1_keypair_xonly_pub(ctx, &pk[0], NULL, &keypairs[0]) == 1); + CHECK(secp256k1_keypair_xonly_pub(ctx, &pk[1], NULL, &keypairs[1]) == 1); + CHECK(secp256k1_keypair_xonly_pub(ctx, &pk[2], NULL, &keypairs[2]) == 1); + memset(&zero_pk, 0, sizeof(zero_pk)); + + /** main test body **/ + ecount = 0; + CHECK(secp256k1_schnorrsig_sign(none, sig, msg, &keypairs[0], NULL, NULL) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_schnorrsig_sign(vrfy, sig, msg, &keypairs[0], NULL, NULL) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, &keypairs[0], NULL, NULL) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_schnorrsig_sign(sign, NULL, msg, &keypairs[0], NULL, NULL) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_schnorrsig_sign(sign, sig, NULL, &keypairs[0], NULL, NULL) == 0); + CHECK(ecount == 4); + CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, NULL, NULL, NULL) == 0); + CHECK(ecount == 5); + CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, &invalid_keypair, NULL, NULL) == 0); + CHECK(ecount == 6); + + ecount = 0; + CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, &keypairs[0], NULL, NULL) == 1); + CHECK(secp256k1_schnorrsig_verify(none, sig, msg, &pk[0]) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_schnorrsig_verify(sign, sig, msg, &pk[0]) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_schnorrsig_verify(vrfy, sig, msg, &pk[0]) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_schnorrsig_verify(vrfy, NULL, msg, &pk[0]) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_schnorrsig_verify(vrfy, sig, NULL, &pk[0]) == 0); + CHECK(ecount == 4); + CHECK(secp256k1_schnorrsig_verify(vrfy, sig, msg, NULL) == 0); + CHECK(ecount == 5); + CHECK(secp256k1_schnorrsig_verify(vrfy, sig, msg, &zero_pk) == 0); + CHECK(ecount == 6); + + secp256k1_context_destroy(none); + secp256k1_context_destroy(sign); + secp256k1_context_destroy(vrfy); + secp256k1_context_destroy(both); +} + +/* Checks that hash initialized by secp256k1_schnorrsig_sha256_tagged has the + * expected state. */ +void test_schnorrsig_sha256_tagged(void) { + char tag[17] = "BIP0340/challenge"; + secp256k1_sha256 sha; + secp256k1_sha256 sha_optimized; + + secp256k1_sha256_initialize_tagged(&sha, (unsigned char *) tag, sizeof(tag)); + secp256k1_schnorrsig_sha256_tagged(&sha_optimized); + test_sha256_eq(&sha, &sha_optimized); +} + +/* Helper function for schnorrsig_bip_vectors + * Signs the message and checks that it's the same as expected_sig. */ +void test_schnorrsig_bip_vectors_check_signing(const unsigned char *sk, const unsigned char *pk_serialized, unsigned char *aux_rand, const unsigned char *msg, const unsigned char *expected_sig) { + unsigned char sig[64]; + secp256k1_keypair keypair; + secp256k1_xonly_pubkey pk, pk_expected; + + CHECK(secp256k1_keypair_create(ctx, &keypair, sk)); + CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, NULL, aux_rand)); + CHECK(memcmp(sig, expected_sig, 64) == 0); + + CHECK(secp256k1_xonly_pubkey_parse(ctx, &pk_expected, pk_serialized)); + CHECK(secp256k1_keypair_xonly_pub(ctx, &pk, NULL, &keypair)); + CHECK(memcmp(&pk, &pk_expected, sizeof(pk)) == 0); + CHECK(secp256k1_schnorrsig_verify(ctx, sig, msg, &pk)); +} + +/* Helper function for schnorrsig_bip_vectors + * Checks that both verify and verify_batch (TODO) return the same value as expected. */ +void test_schnorrsig_bip_vectors_check_verify(const unsigned char *pk_serialized, const unsigned char *msg32, const unsigned char *sig, int expected) { + secp256k1_xonly_pubkey pk; + + CHECK(secp256k1_xonly_pubkey_parse(ctx, &pk, pk_serialized)); + CHECK(expected == secp256k1_schnorrsig_verify(ctx, sig, msg32, &pk)); +} + +/* Test vectors according to BIP-340 ("Schnorr Signatures for secp256k1"). See + * https://github.com/bitcoin/bips/blob/master/bip-0340/test-vectors.csv. */ +void test_schnorrsig_bip_vectors(void) { + { + /* Test vector 0 */ + const unsigned char sk[32] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 + }; + const unsigned char pk[32] = { + 0xF9, 0x30, 0x8A, 0x01, 0x92, 0x58, 0xC3, 0x10, + 0x49, 0x34, 0x4F, 0x85, 0xF8, 0x9D, 0x52, 0x29, + 0xB5, 0x31, 0xC8, 0x45, 0x83, 0x6F, 0x99, 0xB0, + 0x86, 0x01, 0xF1, 0x13, 0xBC, 0xE0, 0x36, 0xF9 + }; + unsigned char aux_rand[32] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }; + const unsigned char msg[32] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }; + const unsigned char sig[64] = { + 0xE9, 0x07, 0x83, 0x1F, 0x80, 0x84, 0x8D, 0x10, + 0x69, 0xA5, 0x37, 0x1B, 0x40, 0x24, 0x10, 0x36, + 0x4B, 0xDF, 0x1C, 0x5F, 0x83, 0x07, 0xB0, 0x08, + 0x4C, 0x55, 0xF1, 0xCE, 0x2D, 0xCA, 0x82, 0x15, + 0x25, 0xF6, 0x6A, 0x4A, 0x85, 0xEA, 0x8B, 0x71, + 0xE4, 0x82, 0xA7, 0x4F, 0x38, 0x2D, 0x2C, 0xE5, + 0xEB, 0xEE, 0xE8, 0xFD, 0xB2, 0x17, 0x2F, 0x47, + 0x7D, 0xF4, 0x90, 0x0D, 0x31, 0x05, 0x36, 0xC0 + }; + test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sig); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 1); + } + { + /* Test vector 1 */ + const unsigned char sk[32] = { + 0xB7, 0xE1, 0x51, 0x62, 0x8A, 0xED, 0x2A, 0x6A, + 0xBF, 0x71, 0x58, 0x80, 0x9C, 0xF4, 0xF3, 0xC7, + 0x62, 0xE7, 0x16, 0x0F, 0x38, 0xB4, 0xDA, 0x56, + 0xA7, 0x84, 0xD9, 0x04, 0x51, 0x90, 0xCF, 0xEF + }; + const unsigned char pk[32] = { + 0xDF, 0xF1, 0xD7, 0x7F, 0x2A, 0x67, 0x1C, 0x5F, + 0x36, 0x18, 0x37, 0x26, 0xDB, 0x23, 0x41, 0xBE, + 0x58, 0xFE, 0xAE, 0x1D, 0xA2, 0xDE, 0xCE, 0xD8, + 0x43, 0x24, 0x0F, 0x7B, 0x50, 0x2B, 0xA6, 0x59 + }; + unsigned char aux_rand[32] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 + }; + const unsigned char msg[32] = { + 0x24, 0x3F, 0x6A, 0x88, 0x85, 0xA3, 0x08, 0xD3, + 0x13, 0x19, 0x8A, 0x2E, 0x03, 0x70, 0x73, 0x44, + 0xA4, 0x09, 0x38, 0x22, 0x29, 0x9F, 0x31, 0xD0, + 0x08, 0x2E, 0xFA, 0x98, 0xEC, 0x4E, 0x6C, 0x89 + }; + const unsigned char sig[64] = { + 0x68, 0x96, 0xBD, 0x60, 0xEE, 0xAE, 0x29, 0x6D, + 0xB4, 0x8A, 0x22, 0x9F, 0xF7, 0x1D, 0xFE, 0x07, + 0x1B, 0xDE, 0x41, 0x3E, 0x6D, 0x43, 0xF9, 0x17, + 0xDC, 0x8D, 0xCF, 0x8C, 0x78, 0xDE, 0x33, 0x41, + 0x89, 0x06, 0xD1, 0x1A, 0xC9, 0x76, 0xAB, 0xCC, + 0xB2, 0x0B, 0x09, 0x12, 0x92, 0xBF, 0xF4, 0xEA, + 0x89, 0x7E, 0xFC, 0xB6, 0x39, 0xEA, 0x87, 0x1C, + 0xFA, 0x95, 0xF6, 0xDE, 0x33, 0x9E, 0x4B, 0x0A + }; + test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sig); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 1); + } + { + /* Test vector 2 */ + const unsigned char sk[32] = { + 0xC9, 0x0F, 0xDA, 0xA2, 0x21, 0x68, 0xC2, 0x34, + 0xC4, 0xC6, 0x62, 0x8B, 0x80, 0xDC, 0x1C, 0xD1, + 0x29, 0x02, 0x4E, 0x08, 0x8A, 0x67, 0xCC, 0x74, + 0x02, 0x0B, 0xBE, 0xA6, 0x3B, 0x14, 0xE5, 0xC9 + }; + const unsigned char pk[32] = { + 0xDD, 0x30, 0x8A, 0xFE, 0xC5, 0x77, 0x7E, 0x13, + 0x12, 0x1F, 0xA7, 0x2B, 0x9C, 0xC1, 0xB7, 0xCC, + 0x01, 0x39, 0x71, 0x53, 0x09, 0xB0, 0x86, 0xC9, + 0x60, 0xE1, 0x8F, 0xD9, 0x69, 0x77, 0x4E, 0xB8 + }; + unsigned char aux_rand[32] = { + 0xC8, 0x7A, 0xA5, 0x38, 0x24, 0xB4, 0xD7, 0xAE, + 0x2E, 0xB0, 0x35, 0xA2, 0xB5, 0xBB, 0xBC, 0xCC, + 0x08, 0x0E, 0x76, 0xCD, 0xC6, 0xD1, 0x69, 0x2C, + 0x4B, 0x0B, 0x62, 0xD7, 0x98, 0xE6, 0xD9, 0x06 + }; + const unsigned char msg[32] = { + 0x7E, 0x2D, 0x58, 0xD8, 0xB3, 0xBC, 0xDF, 0x1A, + 0xBA, 0xDE, 0xC7, 0x82, 0x90, 0x54, 0xF9, 0x0D, + 0xDA, 0x98, 0x05, 0xAA, 0xB5, 0x6C, 0x77, 0x33, + 0x30, 0x24, 0xB9, 0xD0, 0xA5, 0x08, 0xB7, 0x5C + }; + const unsigned char sig[64] = { + 0x58, 0x31, 0xAA, 0xEE, 0xD7, 0xB4, 0x4B, 0xB7, + 0x4E, 0x5E, 0xAB, 0x94, 0xBA, 0x9D, 0x42, 0x94, + 0xC4, 0x9B, 0xCF, 0x2A, 0x60, 0x72, 0x8D, 0x8B, + 0x4C, 0x20, 0x0F, 0x50, 0xDD, 0x31, 0x3C, 0x1B, + 0xAB, 0x74, 0x58, 0x79, 0xA5, 0xAD, 0x95, 0x4A, + 0x72, 0xC4, 0x5A, 0x91, 0xC3, 0xA5, 0x1D, 0x3C, + 0x7A, 0xDE, 0xA9, 0x8D, 0x82, 0xF8, 0x48, 0x1E, + 0x0E, 0x1E, 0x03, 0x67, 0x4A, 0x6F, 0x3F, 0xB7 + }; + test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sig); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 1); + } + { + /* Test vector 3 */ + const unsigned char sk[32] = { + 0x0B, 0x43, 0x2B, 0x26, 0x77, 0x93, 0x73, 0x81, + 0xAE, 0xF0, 0x5B, 0xB0, 0x2A, 0x66, 0xEC, 0xD0, + 0x12, 0x77, 0x30, 0x62, 0xCF, 0x3F, 0xA2, 0x54, + 0x9E, 0x44, 0xF5, 0x8E, 0xD2, 0x40, 0x17, 0x10 + }; + const unsigned char pk[32] = { + 0x25, 0xD1, 0xDF, 0xF9, 0x51, 0x05, 0xF5, 0x25, + 0x3C, 0x40, 0x22, 0xF6, 0x28, 0xA9, 0x96, 0xAD, + 0x3A, 0x0D, 0x95, 0xFB, 0xF2, 0x1D, 0x46, 0x8A, + 0x1B, 0x33, 0xF8, 0xC1, 0x60, 0xD8, 0xF5, 0x17 + }; + unsigned char aux_rand[32] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF + }; + const unsigned char msg[32] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF + }; + const unsigned char sig[64] = { + 0x7E, 0xB0, 0x50, 0x97, 0x57, 0xE2, 0x46, 0xF1, + 0x94, 0x49, 0x88, 0x56, 0x51, 0x61, 0x1C, 0xB9, + 0x65, 0xEC, 0xC1, 0xA1, 0x87, 0xDD, 0x51, 0xB6, + 0x4F, 0xDA, 0x1E, 0xDC, 0x96, 0x37, 0xD5, 0xEC, + 0x97, 0x58, 0x2B, 0x9C, 0xB1, 0x3D, 0xB3, 0x93, + 0x37, 0x05, 0xB3, 0x2B, 0xA9, 0x82, 0xAF, 0x5A, + 0xF2, 0x5F, 0xD7, 0x88, 0x81, 0xEB, 0xB3, 0x27, + 0x71, 0xFC, 0x59, 0x22, 0xEF, 0xC6, 0x6E, 0xA3 + }; + test_schnorrsig_bip_vectors_check_signing(sk, pk, aux_rand, msg, sig); + test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 1); + } + { + /* Test vector 4 */ + const unsigned char pk[32] = { + 0xD6, 0x9C, 0x35, 0x09, 0xBB, 0x99, 0xE4, 0x12, + 0xE6, 0x8B, 0x0F, 0xE8, 0x54, 0x4E, 0x72, 0x83, + 0x7D, 0xFA, 0x30, 0x74, 0x6D, 0x8B, 0xE2, 0xAA, + 0x65, 0x97, 0x5F, 0x29, 0xD2, 0x2D, 0xC7, 0xB9 + }; + const unsigned char msg[32] = { + 0x4D, 0xF3, 0xC3, 0xF6, 0x8F, 0xCC, 0x83, 0xB2, + 0x7E, 0x9D, 0x42, 0xC9, 0x04, 0x31, 0xA7, 0x24, + 0x99, 0xF1, 0x78, 0x75, 0xC8, 0x1A, 0x59, 0x9B, + 0x56, 0x6C, 0x98, 0x89, 0xB9, 0x69, 0x67, 0x03 + }; + const unsigned char sig[64] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x3B, 0x78, 0xCE, 0x56, 0x3F, + 0x89, 0xA0, 0xED, 0x94, 0x14, 0xF5, 0xAA, 0x28, + 0xAD, 0x0D, 0x96, 0xD6, 0x79, 0x5F, 0x9C, 0x63, + 0x76, 0xAF, 0xB1, 0x54, 0x8A, 0xF6, 0x03, 0xB3, + 0xEB, 0x45, 0xC9, 0xF8, 0x20, 0x7D, 0xEE, 0x10, + 0x60, 0xCB, 0x71, 0xC0, 0x4E, 0x80, 0xF5, 0x93, + 0x06, 0x0B, 0x07, 0xD2, 0x83, 0x08, 0xD7, 0xF4 + }; + test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 1); + } + { + /* Test vector 5 */ + const unsigned char pk[32] = { + 0xEE, 0xFD, 0xEA, 0x4C, 0xDB, 0x67, 0x77, 0x50, + 0xA4, 0x20, 0xFE, 0xE8, 0x07, 0xEA, 0xCF, 0x21, + 0xEB, 0x98, 0x98, 0xAE, 0x79, 0xB9, 0x76, 0x87, + 0x66, 0xE4, 0xFA, 0xA0, 0x4A, 0x2D, 0x4A, 0x34 + }; + secp256k1_xonly_pubkey pk_parsed; + /* No need to check the signature of the test vector as parsing the pubkey already fails */ + CHECK(!secp256k1_xonly_pubkey_parse(ctx, &pk_parsed, pk)); + } + { + /* Test vector 6 */ + const unsigned char pk[32] = { + 0xDF, 0xF1, 0xD7, 0x7F, 0x2A, 0x67, 0x1C, 0x5F, + 0x36, 0x18, 0x37, 0x26, 0xDB, 0x23, 0x41, 0xBE, + 0x58, 0xFE, 0xAE, 0x1D, 0xA2, 0xDE, 0xCE, 0xD8, + 0x43, 0x24, 0x0F, 0x7B, 0x50, 0x2B, 0xA6, 0x59 + }; + const unsigned char msg[32] = { + 0x24, 0x3F, 0x6A, 0x88, 0x85, 0xA3, 0x08, 0xD3, + 0x13, 0x19, 0x8A, 0x2E, 0x03, 0x70, 0x73, 0x44, + 0xA4, 0x09, 0x38, 0x22, 0x29, 0x9F, 0x31, 0xD0, + 0x08, 0x2E, 0xFA, 0x98, 0xEC, 0x4E, 0x6C, 0x89 + }; + const unsigned char sig[64] = { + 0xFF, 0xF9, 0x7B, 0xD5, 0x75, 0x5E, 0xEE, 0xA4, + 0x20, 0x45, 0x3A, 0x14, 0x35, 0x52, 0x35, 0xD3, + 0x82, 0xF6, 0x47, 0x2F, 0x85, 0x68, 0xA1, 0x8B, + 0x2F, 0x05, 0x7A, 0x14, 0x60, 0x29, 0x75, 0x56, + 0x3C, 0xC2, 0x79, 0x44, 0x64, 0x0A, 0xC6, 0x07, + 0xCD, 0x10, 0x7A, 0xE1, 0x09, 0x23, 0xD9, 0xEF, + 0x7A, 0x73, 0xC6, 0x43, 0xE1, 0x66, 0xBE, 0x5E, + 0xBE, 0xAF, 0xA3, 0x4B, 0x1A, 0xC5, 0x53, 0xE2 + }; + test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + } + { + /* Test vector 7 */ + const unsigned char pk[32] = { + 0xDF, 0xF1, 0xD7, 0x7F, 0x2A, 0x67, 0x1C, 0x5F, + 0x36, 0x18, 0x37, 0x26, 0xDB, 0x23, 0x41, 0xBE, + 0x58, 0xFE, 0xAE, 0x1D, 0xA2, 0xDE, 0xCE, 0xD8, + 0x43, 0x24, 0x0F, 0x7B, 0x50, 0x2B, 0xA6, 0x59 + }; + const unsigned char msg[32] = { + 0x24, 0x3F, 0x6A, 0x88, 0x85, 0xA3, 0x08, 0xD3, + 0x13, 0x19, 0x8A, 0x2E, 0x03, 0x70, 0x73, 0x44, + 0xA4, 0x09, 0x38, 0x22, 0x29, 0x9F, 0x31, 0xD0, + 0x08, 0x2E, 0xFA, 0x98, 0xEC, 0x4E, 0x6C, 0x89 + }; + const unsigned char sig[64] = { + 0x1F, 0xA6, 0x2E, 0x33, 0x1E, 0xDB, 0xC2, 0x1C, + 0x39, 0x47, 0x92, 0xD2, 0xAB, 0x11, 0x00, 0xA7, + 0xB4, 0x32, 0xB0, 0x13, 0xDF, 0x3F, 0x6F, 0xF4, + 0xF9, 0x9F, 0xCB, 0x33, 0xE0, 0xE1, 0x51, 0x5F, + 0x28, 0x89, 0x0B, 0x3E, 0xDB, 0x6E, 0x71, 0x89, + 0xB6, 0x30, 0x44, 0x8B, 0x51, 0x5C, 0xE4, 0xF8, + 0x62, 0x2A, 0x95, 0x4C, 0xFE, 0x54, 0x57, 0x35, + 0xAA, 0xEA, 0x51, 0x34, 0xFC, 0xCD, 0xB2, 0xBD + }; + test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + } + { + /* Test vector 8 */ + const unsigned char pk[32] = { + 0xDF, 0xF1, 0xD7, 0x7F, 0x2A, 0x67, 0x1C, 0x5F, + 0x36, 0x18, 0x37, 0x26, 0xDB, 0x23, 0x41, 0xBE, + 0x58, 0xFE, 0xAE, 0x1D, 0xA2, 0xDE, 0xCE, 0xD8, + 0x43, 0x24, 0x0F, 0x7B, 0x50, 0x2B, 0xA6, 0x59 + }; + const unsigned char msg[32] = { + 0x24, 0x3F, 0x6A, 0x88, 0x85, 0xA3, 0x08, 0xD3, + 0x13, 0x19, 0x8A, 0x2E, 0x03, 0x70, 0x73, 0x44, + 0xA4, 0x09, 0x38, 0x22, 0x29, 0x9F, 0x31, 0xD0, + 0x08, 0x2E, 0xFA, 0x98, 0xEC, 0x4E, 0x6C, 0x89 + }; + const unsigned char sig[64] = { + 0x6C, 0xFF, 0x5C, 0x3B, 0xA8, 0x6C, 0x69, 0xEA, + 0x4B, 0x73, 0x76, 0xF3, 0x1A, 0x9B, 0xCB, 0x4F, + 0x74, 0xC1, 0x97, 0x60, 0x89, 0xB2, 0xD9, 0x96, + 0x3D, 0xA2, 0xE5, 0x54, 0x3E, 0x17, 0x77, 0x69, + 0x96, 0x17, 0x64, 0xB3, 0xAA, 0x9B, 0x2F, 0xFC, + 0xB6, 0xEF, 0x94, 0x7B, 0x68, 0x87, 0xA2, 0x26, + 0xE8, 0xD7, 0xC9, 0x3E, 0x00, 0xC5, 0xED, 0x0C, + 0x18, 0x34, 0xFF, 0x0D, 0x0C, 0x2E, 0x6D, 0xA6 + }; + test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + } + { + /* Test vector 9 */ + const unsigned char pk[32] = { + 0xDF, 0xF1, 0xD7, 0x7F, 0x2A, 0x67, 0x1C, 0x5F, + 0x36, 0x18, 0x37, 0x26, 0xDB, 0x23, 0x41, 0xBE, + 0x58, 0xFE, 0xAE, 0x1D, 0xA2, 0xDE, 0xCE, 0xD8, + 0x43, 0x24, 0x0F, 0x7B, 0x50, 0x2B, 0xA6, 0x59 + }; + const unsigned char msg[32] = { + 0x24, 0x3F, 0x6A, 0x88, 0x85, 0xA3, 0x08, 0xD3, + 0x13, 0x19, 0x8A, 0x2E, 0x03, 0x70, 0x73, 0x44, + 0xA4, 0x09, 0x38, 0x22, 0x29, 0x9F, 0x31, 0xD0, + 0x08, 0x2E, 0xFA, 0x98, 0xEC, 0x4E, 0x6C, 0x89 + }; + const unsigned char sig[64] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x12, 0x3D, 0xDA, 0x83, 0x28, 0xAF, 0x9C, 0x23, + 0xA9, 0x4C, 0x1F, 0xEE, 0xCF, 0xD1, 0x23, 0xBA, + 0x4F, 0xB7, 0x34, 0x76, 0xF0, 0xD5, 0x94, 0xDC, + 0xB6, 0x5C, 0x64, 0x25, 0xBD, 0x18, 0x60, 0x51 + }; + test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + } + { + /* Test vector 10 */ + const unsigned char pk[32] = { + 0xDF, 0xF1, 0xD7, 0x7F, 0x2A, 0x67, 0x1C, 0x5F, + 0x36, 0x18, 0x37, 0x26, 0xDB, 0x23, 0x41, 0xBE, + 0x58, 0xFE, 0xAE, 0x1D, 0xA2, 0xDE, 0xCE, 0xD8, + 0x43, 0x24, 0x0F, 0x7B, 0x50, 0x2B, 0xA6, 0x59 + }; + const unsigned char msg[32] = { + 0x24, 0x3F, 0x6A, 0x88, 0x85, 0xA3, 0x08, 0xD3, + 0x13, 0x19, 0x8A, 0x2E, 0x03, 0x70, 0x73, 0x44, + 0xA4, 0x09, 0x38, 0x22, 0x29, 0x9F, 0x31, 0xD0, + 0x08, 0x2E, 0xFA, 0x98, 0xEC, 0x4E, 0x6C, 0x89 + }; + const unsigned char sig[64] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x76, 0x15, 0xFB, 0xAF, 0x5A, 0xE2, 0x88, 0x64, + 0x01, 0x3C, 0x09, 0x97, 0x42, 0xDE, 0xAD, 0xB4, + 0xDB, 0xA8, 0x7F, 0x11, 0xAC, 0x67, 0x54, 0xF9, + 0x37, 0x80, 0xD5, 0xA1, 0x83, 0x7C, 0xF1, 0x97 + }; + test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + } + { + /* Test vector 11 */ + const unsigned char pk[32] = { + 0xDF, 0xF1, 0xD7, 0x7F, 0x2A, 0x67, 0x1C, 0x5F, + 0x36, 0x18, 0x37, 0x26, 0xDB, 0x23, 0x41, 0xBE, + 0x58, 0xFE, 0xAE, 0x1D, 0xA2, 0xDE, 0xCE, 0xD8, + 0x43, 0x24, 0x0F, 0x7B, 0x50, 0x2B, 0xA6, 0x59 + }; + const unsigned char msg[32] = { + 0x24, 0x3F, 0x6A, 0x88, 0x85, 0xA3, 0x08, 0xD3, + 0x13, 0x19, 0x8A, 0x2E, 0x03, 0x70, 0x73, 0x44, + 0xA4, 0x09, 0x38, 0x22, 0x29, 0x9F, 0x31, 0xD0, + 0x08, 0x2E, 0xFA, 0x98, 0xEC, 0x4E, 0x6C, 0x89 + }; + const unsigned char sig[64] = { + 0x4A, 0x29, 0x8D, 0xAC, 0xAE, 0x57, 0x39, 0x5A, + 0x15, 0xD0, 0x79, 0x5D, 0xDB, 0xFD, 0x1D, 0xCB, + 0x56, 0x4D, 0xA8, 0x2B, 0x0F, 0x26, 0x9B, 0xC7, + 0x0A, 0x74, 0xF8, 0x22, 0x04, 0x29, 0xBA, 0x1D, + 0x69, 0xE8, 0x9B, 0x4C, 0x55, 0x64, 0xD0, 0x03, + 0x49, 0x10, 0x6B, 0x84, 0x97, 0x78, 0x5D, 0xD7, + 0xD1, 0xD7, 0x13, 0xA8, 0xAE, 0x82, 0xB3, 0x2F, + 0xA7, 0x9D, 0x5F, 0x7F, 0xC4, 0x07, 0xD3, 0x9B + }; + test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + } + { + /* Test vector 12 */ + const unsigned char pk[32] = { + 0xDF, 0xF1, 0xD7, 0x7F, 0x2A, 0x67, 0x1C, 0x5F, + 0x36, 0x18, 0x37, 0x26, 0xDB, 0x23, 0x41, 0xBE, + 0x58, 0xFE, 0xAE, 0x1D, 0xA2, 0xDE, 0xCE, 0xD8, + 0x43, 0x24, 0x0F, 0x7B, 0x50, 0x2B, 0xA6, 0x59 + }; + const unsigned char msg[32] = { + 0x24, 0x3F, 0x6A, 0x88, 0x85, 0xA3, 0x08, 0xD3, + 0x13, 0x19, 0x8A, 0x2E, 0x03, 0x70, 0x73, 0x44, + 0xA4, 0x09, 0x38, 0x22, 0x29, 0x9F, 0x31, 0xD0, + 0x08, 0x2E, 0xFA, 0x98, 0xEC, 0x4E, 0x6C, 0x89 + }; + const unsigned char sig[64] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFC, 0x2F, + 0x69, 0xE8, 0x9B, 0x4C, 0x55, 0x64, 0xD0, 0x03, + 0x49, 0x10, 0x6B, 0x84, 0x97, 0x78, 0x5D, 0xD7, + 0xD1, 0xD7, 0x13, 0xA8, 0xAE, 0x82, 0xB3, 0x2F, + 0xA7, 0x9D, 0x5F, 0x7F, 0xC4, 0x07, 0xD3, 0x9B + }; + test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + } + { + /* Test vector 13 */ + const unsigned char pk[32] = { + 0xDF, 0xF1, 0xD7, 0x7F, 0x2A, 0x67, 0x1C, 0x5F, + 0x36, 0x18, 0x37, 0x26, 0xDB, 0x23, 0x41, 0xBE, + 0x58, 0xFE, 0xAE, 0x1D, 0xA2, 0xDE, 0xCE, 0xD8, + 0x43, 0x24, 0x0F, 0x7B, 0x50, 0x2B, 0xA6, 0x59 + }; + const unsigned char msg[32] = { + 0x24, 0x3F, 0x6A, 0x88, 0x85, 0xA3, 0x08, 0xD3, + 0x13, 0x19, 0x8A, 0x2E, 0x03, 0x70, 0x73, 0x44, + 0xA4, 0x09, 0x38, 0x22, 0x29, 0x9F, 0x31, 0xD0, + 0x08, 0x2E, 0xFA, 0x98, 0xEC, 0x4E, 0x6C, 0x89 + }; + const unsigned char sig[64] = { + 0x6C, 0xFF, 0x5C, 0x3B, 0xA8, 0x6C, 0x69, 0xEA, + 0x4B, 0x73, 0x76, 0xF3, 0x1A, 0x9B, 0xCB, 0x4F, + 0x74, 0xC1, 0x97, 0x60, 0x89, 0xB2, 0xD9, 0x96, + 0x3D, 0xA2, 0xE5, 0x54, 0x3E, 0x17, 0x77, 0x69, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, + 0xBA, 0xAE, 0xDC, 0xE6, 0xAF, 0x48, 0xA0, 0x3B, + 0xBF, 0xD2, 0x5E, 0x8C, 0xD0, 0x36, 0x41, 0x41 + }; + test_schnorrsig_bip_vectors_check_verify(pk, msg, sig, 0); + } + { + /* Test vector 14 */ + const unsigned char pk[32] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFC, 0x30 + }; + secp256k1_xonly_pubkey pk_parsed; + /* No need to check the signature of the test vector as parsing the pubkey already fails */ + CHECK(!secp256k1_xonly_pubkey_parse(ctx, &pk_parsed, pk)); + } +} + +/* Nonce function that returns constant 0 */ +static int nonce_function_failing(unsigned char *nonce32, const unsigned char *msg32, const unsigned char *key32, const unsigned char *xonly_pk32, const unsigned char *algo16, void *data) { + (void) msg32; + (void) key32; + (void) xonly_pk32; + (void) algo16; + (void) data; + (void) nonce32; + return 0; +} + +/* Nonce function that sets nonce to 0 */ +static int nonce_function_0(unsigned char *nonce32, const unsigned char *msg32, const unsigned char *key32, const unsigned char *xonly_pk32, const unsigned char *algo16, void *data) { + (void) msg32; + (void) key32; + (void) xonly_pk32; + (void) algo16; + (void) data; + + memset(nonce32, 0, 32); + return 1; +} + +/* Nonce function that sets nonce to 0xFF...0xFF */ +static int nonce_function_overflowing(unsigned char *nonce32, const unsigned char *msg32, const unsigned char *key32, const unsigned char *xonly_pk32, const unsigned char *algo16, void *data) { + (void) msg32; + (void) key32; + (void) xonly_pk32; + (void) algo16; + (void) data; + + memset(nonce32, 0xFF, 32); + return 1; +} + +void test_schnorrsig_sign(void) { + unsigned char sk[32]; + secp256k1_keypair keypair; + const unsigned char msg[32] = "this is a msg for a schnorrsig.."; + unsigned char sig[64]; + unsigned char zeros64[64] = { 0 }; + + secp256k1_rand256(sk); + CHECK(secp256k1_keypair_create(ctx, &keypair, sk)); + CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, NULL, NULL) == 1); + + /* Test different nonce functions */ + memset(sig, 1, sizeof(sig)); + CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, nonce_function_failing, NULL) == 0); + CHECK(memcmp(sig, zeros64, sizeof(sig)) == 0); + memset(&sig, 1, sizeof(sig)); + CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, nonce_function_0, NULL) == 0); + CHECK(memcmp(sig, zeros64, sizeof(sig)) == 0); + CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, nonce_function_overflowing, NULL) == 1); + CHECK(memcmp(sig, zeros64, sizeof(sig)) != 0); +} + +#define N_SIGS 3 +/* Creates N_SIGS valid signatures and verifies them with verify and + * verify_batch (TODO). Then flips some bits and checks that verification now + * fails. */ +void test_schnorrsig_sign_verify(void) { + unsigned char sk[32]; + unsigned char msg[N_SIGS][32]; + unsigned char sig[N_SIGS][64]; + size_t i; + secp256k1_keypair keypair; + secp256k1_xonly_pubkey pk; + secp256k1_scalar s; + + secp256k1_rand256(sk); + CHECK(secp256k1_keypair_create(ctx, &keypair, sk)); + CHECK(secp256k1_keypair_xonly_pub(ctx, &pk, NULL, &keypair)); + + for (i = 0; i < N_SIGS; i++) { + secp256k1_rand256(msg[i]); + CHECK(secp256k1_schnorrsig_sign(ctx, sig[i], msg[i], &keypair, NULL, NULL)); + CHECK(secp256k1_schnorrsig_verify(ctx, sig[i], msg[i], &pk)); + } + + { + /* Flip a few bits in the signature and in the message and check that + * verify and verify_batch (TODO) fail */ + size_t sig_idx = secp256k1_rand_int(N_SIGS); + size_t byte_idx = secp256k1_rand_int(32); + unsigned char xorbyte = secp256k1_rand_int(254)+1; + sig[sig_idx][byte_idx] ^= xorbyte; + CHECK(!secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], &pk)); + sig[sig_idx][byte_idx] ^= xorbyte; + + byte_idx = secp256k1_rand_int(32); + sig[sig_idx][32+byte_idx] ^= xorbyte; + CHECK(!secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], &pk)); + sig[sig_idx][32+byte_idx] ^= xorbyte; + + byte_idx = secp256k1_rand_int(32); + msg[sig_idx][byte_idx] ^= xorbyte; + CHECK(!secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], &pk)); + msg[sig_idx][byte_idx] ^= xorbyte; + + /* Check that above bitflips have been reversed correctly */ + CHECK(secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], &pk)); + } + + /* Test overflowing s */ + CHECK(secp256k1_schnorrsig_sign(ctx, sig[0], msg[0], &keypair, NULL, NULL)); + CHECK(secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], &pk)); + memset(&sig[0][32], 0xFF, 32); + CHECK(!secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], &pk)); + + /* Test negative s */ + CHECK(secp256k1_schnorrsig_sign(ctx, sig[0], msg[0], &keypair, NULL, NULL)); + CHECK(secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], &pk)); + secp256k1_scalar_set_b32(&s, &sig[0][32], NULL); + secp256k1_scalar_negate(&s, &s); + secp256k1_scalar_get_b32(&sig[0][32], &s); + CHECK(!secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], &pk)); +} +#undef N_SIGS + +void test_schnorrsig_taproot(void) { + unsigned char sk[32]; + secp256k1_keypair keypair; + secp256k1_xonly_pubkey internal_pk; + unsigned char internal_pk_bytes[32]; + secp256k1_xonly_pubkey output_pk; + unsigned char output_pk_bytes[32]; + unsigned char tweak[32]; + int pk_parity; + unsigned char msg[32]; + unsigned char sig[64]; + + /* Create output key */ + secp256k1_rand256(sk); + CHECK(secp256k1_keypair_create(ctx, &keypair, sk) == 1); + CHECK(secp256k1_keypair_xonly_pub(ctx, &internal_pk, NULL, &keypair) == 1); + /* In actual taproot the tweak would be hash of internal_pk */ + CHECK(secp256k1_xonly_pubkey_serialize(ctx, tweak, &internal_pk) == 1); + CHECK(secp256k1_keypair_xonly_tweak_add(ctx, &keypair, tweak) == 1); + CHECK(secp256k1_keypair_xonly_pub(ctx, &output_pk, &pk_parity, &keypair) == 1); + CHECK(secp256k1_xonly_pubkey_serialize(ctx, output_pk_bytes, &output_pk) == 1); + + /* Key spend */ + secp256k1_rand256(msg); + CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, NULL, NULL) == 1); + /* Verify key spend */ + CHECK(secp256k1_xonly_pubkey_parse(ctx, &output_pk, output_pk_bytes) == 1); + CHECK(secp256k1_schnorrsig_verify(ctx, sig, msg, &output_pk) == 1); + + /* Script spend */ + CHECK(secp256k1_xonly_pubkey_serialize(ctx, internal_pk_bytes, &internal_pk) == 1); + /* Verify script spend */ + CHECK(secp256k1_xonly_pubkey_parse(ctx, &internal_pk, internal_pk_bytes) == 1); + CHECK(secp256k1_xonly_pubkey_tweak_add_check(ctx, output_pk_bytes, pk_parity, &internal_pk, tweak) == 1); +} + +void run_schnorrsig_tests(void) { + int i; + run_nonce_function_bip340_tests(); + + test_schnorrsig_api(); + test_schnorrsig_sha256_tagged(); + test_schnorrsig_bip_vectors(); + for (i = 0; i < count; i++) { + test_schnorrsig_sign(); + test_schnorrsig_sign_verify(); + } + test_schnorrsig_taproot(); +} + +#endif diff --git a/src/secp256k1/src/scalar.h b/src/secp256k1/src/scalar.h index 2a7470352..95d3e326c 100644 --- a/src/secp256k1/src/scalar.h +++ b/src/secp256k1/src/scalar.h @@ -8,6 +8,7 @@ #define SECP256K1_SCALAR_H #include "num.h" +#include "util.h" #if defined HAVE_CONFIG_H #include "libsecp256k1-config.h" @@ -15,12 +16,12 @@ #if defined(EXHAUSTIVE_TEST_ORDER) #include "scalar_low.h" -#elif defined(USE_SCALAR_4X64) +#elif defined(SECP256K1_WIDEMUL_INT128) #include "scalar_4x64.h" -#elif defined(USE_SCALAR_8X32) +#elif defined(SECP256K1_WIDEMUL_INT64) #include "scalar_8x32.h" #else -#error "Please select scalar implementation" +#error "Please select wide multiplication implementation" #endif /** Clear a scalar to prevent the leak of sensitive data. */ diff --git a/src/secp256k1/src/scalar_4x64_impl.h b/src/secp256k1/src/scalar_4x64_impl.h index 8f539c4bc..7f3992786 100644 --- a/src/secp256k1/src/scalar_4x64_impl.h +++ b/src/secp256k1/src/scalar_4x64_impl.h @@ -192,9 +192,9 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { tl = t; \ } \ c0 += tl; /* overflow is handled on the next line */ \ - th += (c0 < tl) ? 1 : 0; /* at most 0xFFFFFFFFFFFFFFFF */ \ + th += (c0 < tl); /* at most 0xFFFFFFFFFFFFFFFF */ \ c1 += th; /* overflow is handled on the next line */ \ - c2 += (c1 < th) ? 1 : 0; /* never overflows by contract (verified in the next line) */ \ + c2 += (c1 < th); /* never overflows by contract (verified in the next line) */ \ VERIFY_CHECK((c1 >= th) || (c2 != 0)); \ } @@ -207,7 +207,7 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { tl = t; \ } \ c0 += tl; /* overflow is handled on the next line */ \ - th += (c0 < tl) ? 1 : 0; /* at most 0xFFFFFFFFFFFFFFFF */ \ + th += (c0 < tl); /* at most 0xFFFFFFFFFFFFFFFF */ \ c1 += th; /* never overflows by contract (verified in the next line) */ \ VERIFY_CHECK(c1 >= th); \ } @@ -221,16 +221,16 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { tl = t; \ } \ th2 = th + th; /* at most 0xFFFFFFFFFFFFFFFE (in case th was 0x7FFFFFFFFFFFFFFF) */ \ - c2 += (th2 < th) ? 1 : 0; /* never overflows by contract (verified the next line) */ \ + c2 += (th2 < th); /* never overflows by contract (verified the next line) */ \ VERIFY_CHECK((th2 >= th) || (c2 != 0)); \ tl2 = tl + tl; /* at most 0xFFFFFFFFFFFFFFFE (in case the lowest 63 bits of tl were 0x7FFFFFFFFFFFFFFF) */ \ - th2 += (tl2 < tl) ? 1 : 0; /* at most 0xFFFFFFFFFFFFFFFF */ \ + th2 += (tl2 < tl); /* at most 0xFFFFFFFFFFFFFFFF */ \ c0 += tl2; /* overflow is handled on the next line */ \ - th2 += (c0 < tl2) ? 1 : 0; /* second overflow is handled on the next line */ \ + th2 += (c0 < tl2); /* second overflow is handled on the next line */ \ c2 += (c0 < tl2) & (th2 == 0); /* never overflows by contract (verified the next line) */ \ VERIFY_CHECK((c0 >= tl2) || (th2 != 0) || (c2 != 0)); \ c1 += th2; /* overflow is handled on the next line */ \ - c2 += (c1 < th2) ? 1 : 0; /* never overflows by contract (verified the next line) */ \ + c2 += (c1 < th2); /* never overflows by contract (verified the next line) */ \ VERIFY_CHECK((c1 >= th2) || (c2 != 0)); \ } @@ -238,15 +238,15 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { #define sumadd(a) { \ unsigned int over; \ c0 += (a); /* overflow is handled on the next line */ \ - over = (c0 < (a)) ? 1 : 0; \ + over = (c0 < (a)); \ c1 += over; /* overflow is handled on the next line */ \ - c2 += (c1 < over) ? 1 : 0; /* never overflows by contract */ \ + c2 += (c1 < over); /* never overflows by contract */ \ } /** Add a to the number defined by (c0,c1). c1 must never overflow, c2 must be zero. */ #define sumadd_fast(a) { \ c0 += (a); /* overflow is handled on the next line */ \ - c1 += (c0 < (a)) ? 1 : 0; /* never overflows by contract (verified the next line) */ \ + c1 += (c0 < (a)); /* never overflows by contract (verified the next line) */ \ VERIFY_CHECK((c1 != 0) | (c0 >= (a))); \ VERIFY_CHECK(c2 == 0); \ } diff --git a/src/secp256k1/src/scalar_8x32_impl.h b/src/secp256k1/src/scalar_8x32_impl.h index 3c372f34f..f8c7fa7ef 100644 --- a/src/secp256k1/src/scalar_8x32_impl.h +++ b/src/secp256k1/src/scalar_8x32_impl.h @@ -271,9 +271,9 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { tl = t; \ } \ c0 += tl; /* overflow is handled on the next line */ \ - th += (c0 < tl) ? 1 : 0; /* at most 0xFFFFFFFF */ \ + th += (c0 < tl); /* at most 0xFFFFFFFF */ \ c1 += th; /* overflow is handled on the next line */ \ - c2 += (c1 < th) ? 1 : 0; /* never overflows by contract (verified in the next line) */ \ + c2 += (c1 < th); /* never overflows by contract (verified in the next line) */ \ VERIFY_CHECK((c1 >= th) || (c2 != 0)); \ } @@ -286,7 +286,7 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { tl = t; \ } \ c0 += tl; /* overflow is handled on the next line */ \ - th += (c0 < tl) ? 1 : 0; /* at most 0xFFFFFFFF */ \ + th += (c0 < tl); /* at most 0xFFFFFFFF */ \ c1 += th; /* never overflows by contract (verified in the next line) */ \ VERIFY_CHECK(c1 >= th); \ } @@ -300,16 +300,16 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { tl = t; \ } \ th2 = th + th; /* at most 0xFFFFFFFE (in case th was 0x7FFFFFFF) */ \ - c2 += (th2 < th) ? 1 : 0; /* never overflows by contract (verified the next line) */ \ + c2 += (th2 < th); /* never overflows by contract (verified the next line) */ \ VERIFY_CHECK((th2 >= th) || (c2 != 0)); \ tl2 = tl + tl; /* at most 0xFFFFFFFE (in case the lowest 63 bits of tl were 0x7FFFFFFF) */ \ - th2 += (tl2 < tl) ? 1 : 0; /* at most 0xFFFFFFFF */ \ + th2 += (tl2 < tl); /* at most 0xFFFFFFFF */ \ c0 += tl2; /* overflow is handled on the next line */ \ - th2 += (c0 < tl2) ? 1 : 0; /* second overflow is handled on the next line */ \ + th2 += (c0 < tl2); /* second overflow is handled on the next line */ \ c2 += (c0 < tl2) & (th2 == 0); /* never overflows by contract (verified the next line) */ \ VERIFY_CHECK((c0 >= tl2) || (th2 != 0) || (c2 != 0)); \ c1 += th2; /* overflow is handled on the next line */ \ - c2 += (c1 < th2) ? 1 : 0; /* never overflows by contract (verified the next line) */ \ + c2 += (c1 < th2); /* never overflows by contract (verified the next line) */ \ VERIFY_CHECK((c1 >= th2) || (c2 != 0)); \ } @@ -317,15 +317,15 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { #define sumadd(a) { \ unsigned int over; \ c0 += (a); /* overflow is handled on the next line */ \ - over = (c0 < (a)) ? 1 : 0; \ + over = (c0 < (a)); \ c1 += over; /* overflow is handled on the next line */ \ - c2 += (c1 < over) ? 1 : 0; /* never overflows by contract */ \ + c2 += (c1 < over); /* never overflows by contract */ \ } /** Add a to the number defined by (c0,c1). c1 must never overflow, c2 must be zero. */ #define sumadd_fast(a) { \ c0 += (a); /* overflow is handled on the next line */ \ - c1 += (c0 < (a)) ? 1 : 0; /* never overflows by contract (verified the next line) */ \ + c1 += (c0 < (a)); /* never overflows by contract (verified the next line) */ \ VERIFY_CHECK((c1 != 0) | (c0 >= (a))); \ VERIFY_CHECK(c2 == 0); \ } diff --git a/src/secp256k1/src/scalar_impl.h b/src/secp256k1/src/scalar_impl.h index 70cd73db0..2ec04b1ae 100644 --- a/src/secp256k1/src/scalar_impl.h +++ b/src/secp256k1/src/scalar_impl.h @@ -16,12 +16,12 @@ #if defined(EXHAUSTIVE_TEST_ORDER) #include "scalar_low_impl.h" -#elif defined(USE_SCALAR_4X64) +#elif defined(SECP256K1_WIDEMUL_INT128) #include "scalar_4x64_impl.h" -#elif defined(USE_SCALAR_8X32) +#elif defined(SECP256K1_WIDEMUL_INT64) #include "scalar_8x32_impl.h" #else -#error "Please select scalar implementation" +#error "Please select wide multiplication implementation" #endif static const secp256k1_scalar secp256k1_scalar_one = SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 1); diff --git a/src/secp256k1/src/scratch_impl.h b/src/secp256k1/src/scratch_impl.h index 4cee70000..b20562022 100644 --- a/src/secp256k1/src/scratch_impl.h +++ b/src/secp256k1/src/scratch_impl.h @@ -11,7 +11,7 @@ #include "scratch.h" static secp256k1_scratch* secp256k1_scratch_create(const secp256k1_callback* error_callback, size_t size) { - const size_t base_alloc = ((sizeof(secp256k1_scratch) + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT; + const size_t base_alloc = ROUND_TO_ALIGN(sizeof(secp256k1_scratch)); void *alloc = checked_malloc(error_callback, base_alloc + size); secp256k1_scratch* ret = (secp256k1_scratch *)alloc; if (ret != NULL) { @@ -60,6 +60,10 @@ static size_t secp256k1_scratch_max_allocation(const secp256k1_callback* error_c secp256k1_callback_call(error_callback, "invalid scratch space"); return 0; } + /* Ensure that multiplication will not wrap around */ + if (ALIGNMENT > 1 && objects > SIZE_MAX/(ALIGNMENT - 1)) { + return 0; + } if (scratch->max_size - scratch->alloc_size <= objects * (ALIGNMENT - 1)) { return 0; } @@ -68,7 +72,14 @@ static size_t secp256k1_scratch_max_allocation(const secp256k1_callback* error_c static void *secp256k1_scratch_alloc(const secp256k1_callback* error_callback, secp256k1_scratch* scratch, size_t size) { void *ret; - size = ROUND_TO_ALIGN(size); + size_t rounded_size; + + rounded_size = ROUND_TO_ALIGN(size); + /* Check that rounding did not wrap around */ + if (rounded_size < size) { + return NULL; + } + size = rounded_size; if (memcmp(scratch->magic, "scratch", 8) != 0) { secp256k1_callback_call(error_callback, "invalid scratch space"); diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index b03a6e634..eaafb3a21 100644 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -7,6 +7,7 @@ #include "include/secp256k1.h" #include "include/secp256k1_preallocated.h" +#include "assumptions.h" #include "util.h" #include "num_impl.h" #include "field_impl.h" @@ -19,6 +20,7 @@ #include "eckey_impl.h" #include "hash_impl.h" #include "scratch_impl.h" +#include "selftest.h" #if defined(VALGRIND) # include <valgrind/memcheck.h> @@ -117,6 +119,9 @@ secp256k1_context* secp256k1_context_preallocated_create(void* prealloc, unsigne size_t prealloc_size; secp256k1_context* ret; + if (!secp256k1_selftest()) { + secp256k1_callback_call(&default_error_callback, "self test failed"); + } VERIFY_CHECK(prealloc != NULL); prealloc_size = secp256k1_context_preallocated_size(flags); ret = (secp256k1_context*)manual_alloc(&prealloc, sizeof(secp256k1_context), base, prealloc_size); @@ -226,7 +231,7 @@ void secp256k1_scratch_space_destroy(const secp256k1_context *ctx, secp256k1_scr * of the software. This is setup for use with valgrind but could be substituted with * the appropriate instrumentation for other analysis tools. */ -static SECP256K1_INLINE void secp256k1_declassify(const secp256k1_context* ctx, void *p, size_t len) { +static SECP256K1_INLINE void secp256k1_declassify(const secp256k1_context* ctx, const void *p, size_t len) { #if defined(VALGRIND) if (EXPECT(ctx->declassify,0)) VALGRIND_MAKE_MEM_DEFINED(p, len); #else @@ -291,7 +296,7 @@ int secp256k1_ec_pubkey_serialize(const secp256k1_context* ctx, unsigned char *o VERIFY_CHECK(ctx != NULL); ARG_CHECK(outputlen != NULL); - ARG_CHECK(*outputlen >= ((flags & SECP256K1_FLAGS_BIT_COMPRESSION) ? 33 : 65)); + ARG_CHECK(*outputlen >= ((flags & SECP256K1_FLAGS_BIT_COMPRESSION) ? 33u : 65u)); len = *outputlen; *outputlen = 0; ARG_CHECK(output != NULL); @@ -548,10 +553,21 @@ int secp256k1_ec_seckey_verify(const secp256k1_context* ctx, const unsigned char return ret; } -int secp256k1_ec_pubkey_create(const secp256k1_context* ctx, secp256k1_pubkey *pubkey, const unsigned char *seckey) { +static int secp256k1_ec_pubkey_create_helper(const secp256k1_ecmult_gen_context *ecmult_gen_ctx, secp256k1_scalar *seckey_scalar, secp256k1_ge *p, const unsigned char *seckey) { secp256k1_gej pj; + int ret; + + ret = secp256k1_scalar_set_b32_seckey(seckey_scalar, seckey); + secp256k1_scalar_cmov(seckey_scalar, &secp256k1_scalar_one, !ret); + + secp256k1_ecmult_gen(ecmult_gen_ctx, &pj, seckey_scalar); + secp256k1_ge_set_gej(p, &pj); + return ret; +} + +int secp256k1_ec_pubkey_create(const secp256k1_context* ctx, secp256k1_pubkey *pubkey, const unsigned char *seckey) { secp256k1_ge p; - secp256k1_scalar sec; + secp256k1_scalar seckey_scalar; int ret = 0; VERIFY_CHECK(ctx != NULL); ARG_CHECK(pubkey != NULL); @@ -559,15 +575,11 @@ int secp256k1_ec_pubkey_create(const secp256k1_context* ctx, secp256k1_pubkey *p ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); ARG_CHECK(seckey != NULL); - ret = secp256k1_scalar_set_b32_seckey(&sec, seckey); - secp256k1_scalar_cmov(&sec, &secp256k1_scalar_one, !ret); - - secp256k1_ecmult_gen(&ctx->ecmult_gen_ctx, &pj, &sec); - secp256k1_ge_set_gej(&p, &pj); + ret = secp256k1_ec_pubkey_create_helper(&ctx->ecmult_gen_ctx, &seckey_scalar, &p, seckey); secp256k1_pubkey_save(pubkey, &p); memczero(pubkey, sizeof(*pubkey), !ret); - secp256k1_scalar_clear(&sec); + secp256k1_scalar_clear(&seckey_scalar); return ret; } @@ -605,24 +617,31 @@ int secp256k1_ec_pubkey_negate(const secp256k1_context* ctx, secp256k1_pubkey *p return ret; } -int secp256k1_ec_seckey_tweak_add(const secp256k1_context* ctx, unsigned char *seckey, const unsigned char *tweak) { + +static int secp256k1_ec_seckey_tweak_add_helper(secp256k1_scalar *sec, const unsigned char *tweak) { secp256k1_scalar term; + int overflow = 0; + int ret = 0; + + secp256k1_scalar_set_b32(&term, tweak, &overflow); + ret = (!overflow) & secp256k1_eckey_privkey_tweak_add(sec, &term); + secp256k1_scalar_clear(&term); + return ret; +} + +int secp256k1_ec_seckey_tweak_add(const secp256k1_context* ctx, unsigned char *seckey, const unsigned char *tweak) { secp256k1_scalar sec; int ret = 0; - int overflow = 0; VERIFY_CHECK(ctx != NULL); ARG_CHECK(seckey != NULL); ARG_CHECK(tweak != NULL); - secp256k1_scalar_set_b32(&term, tweak, &overflow); ret = secp256k1_scalar_set_b32_seckey(&sec, seckey); - - ret &= (!overflow) & secp256k1_eckey_privkey_tweak_add(&sec, &term); + ret &= secp256k1_ec_seckey_tweak_add_helper(&sec, tweak); secp256k1_scalar_cmov(&sec, &secp256k1_scalar_zero, !ret); secp256k1_scalar_get_b32(seckey, &sec); secp256k1_scalar_clear(&sec); - secp256k1_scalar_clear(&term); return ret; } @@ -630,25 +649,26 @@ int secp256k1_ec_privkey_tweak_add(const secp256k1_context* ctx, unsigned char * return secp256k1_ec_seckey_tweak_add(ctx, seckey, tweak); } +static int secp256k1_ec_pubkey_tweak_add_helper(const secp256k1_ecmult_context* ecmult_ctx, secp256k1_ge *p, const unsigned char *tweak) { + secp256k1_scalar term; + int overflow = 0; + secp256k1_scalar_set_b32(&term, tweak, &overflow); + return !overflow && secp256k1_eckey_pubkey_tweak_add(ecmult_ctx, p, &term); +} + int secp256k1_ec_pubkey_tweak_add(const secp256k1_context* ctx, secp256k1_pubkey *pubkey, const unsigned char *tweak) { secp256k1_ge p; - secp256k1_scalar term; int ret = 0; - int overflow = 0; VERIFY_CHECK(ctx != NULL); ARG_CHECK(secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)); ARG_CHECK(pubkey != NULL); ARG_CHECK(tweak != NULL); - secp256k1_scalar_set_b32(&term, tweak, &overflow); - ret = !overflow && secp256k1_pubkey_load(ctx, &p, pubkey); + ret = secp256k1_pubkey_load(ctx, &p, pubkey); memset(pubkey, 0, sizeof(*pubkey)); + ret = ret && secp256k1_ec_pubkey_tweak_add_helper(&ctx->ecmult_ctx, &p, tweak); if (ret) { - if (secp256k1_eckey_pubkey_tweak_add(&ctx->ecmult_ctx, &p, &term)) { - secp256k1_pubkey_save(pubkey, &p); - } else { - ret = 0; - } + secp256k1_pubkey_save(pubkey, &p); } return ret; @@ -741,3 +761,11 @@ int secp256k1_ec_pubkey_combine(const secp256k1_context* ctx, secp256k1_pubkey * #ifdef ENABLE_MODULE_RECOVERY # include "modules/recovery/main_impl.h" #endif + +#ifdef ENABLE_MODULE_EXTRAKEYS +# include "modules/extrakeys/main_impl.h" +#endif + +#ifdef ENABLE_MODULE_SCHNORRSIG +# include "modules/schnorrsig/main_impl.h" +#endif diff --git a/src/secp256k1/src/selftest.h b/src/secp256k1/src/selftest.h new file mode 100644 index 000000000..885983aa2 --- /dev/null +++ b/src/secp256k1/src/selftest.h @@ -0,0 +1,32 @@ +/********************************************************************** + * Copyright (c) 2020 Pieter Wuille * + * Distributed under the MIT software license, see the accompanying * + * file COPYING or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef SECP256K1_SELFTEST_H +#define SECP256K1_SELFTEST_H + +#include "hash.h" + +#include <string.h> + +static int secp256k1_selftest_sha256(void) { + static const char *input63 = "For this sample, this 63-byte string will be used as input data"; + static const unsigned char output32[32] = { + 0xf0, 0x8a, 0x78, 0xcb, 0xba, 0xee, 0x08, 0x2b, 0x05, 0x2a, 0xe0, 0x70, 0x8f, 0x32, 0xfa, 0x1e, + 0x50, 0xc5, 0xc4, 0x21, 0xaa, 0x77, 0x2b, 0xa5, 0xdb, 0xb4, 0x06, 0xa2, 0xea, 0x6b, 0xe3, 0x42, + }; + unsigned char out[32]; + secp256k1_sha256 hasher; + secp256k1_sha256_initialize(&hasher); + secp256k1_sha256_write(&hasher, (const unsigned char*)input63, 63); + secp256k1_sha256_finalize(&hasher, out); + return memcmp(out, output32, 32) == 0; +} + +static int secp256k1_selftest(void) { + return secp256k1_selftest_sha256(); +} + +#endif /* SECP256K1_SELFTEST_H */ diff --git a/src/secp256k1/src/testrand.h b/src/secp256k1/src/testrand.h index f1f9be077..bcbe15a6f 100644 --- a/src/secp256k1/src/testrand.h +++ b/src/secp256k1/src/testrand.h @@ -35,4 +35,7 @@ static void secp256k1_rand256_test(unsigned char *b32); /** Generate pseudorandom bytes with long sequences of zero and one bits. */ static void secp256k1_rand_bytes_test(unsigned char *bytes, size_t len); +/** Flip a single random bit in a byte array */ +static void secp256k1_rand_flip(unsigned char *b, size_t len); + #endif /* SECP256K1_TESTRAND_H */ diff --git a/src/secp256k1/src/testrand_impl.h b/src/secp256k1/src/testrand_impl.h index 30a91e529..dfb658d9c 100644 --- a/src/secp256k1/src/testrand_impl.h +++ b/src/secp256k1/src/testrand_impl.h @@ -107,4 +107,8 @@ static void secp256k1_rand256_test(unsigned char *b32) { secp256k1_rand_bytes_test(b32, 32); } +static void secp256k1_rand_flip(unsigned char *b, size_t len) { + b[secp256k1_rand_int(len)] ^= (1 << secp256k1_rand_int(8)); +} + #endif /* SECP256K1_TESTRAND_IMPL_H */ diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index 374ed7dc1..4780e9319 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -182,8 +182,10 @@ void run_context_tests(int use_prealloc) { ecount2 = 10; secp256k1_context_set_illegal_callback(vrfy, counting_illegal_callback_fn, &ecount); secp256k1_context_set_illegal_callback(sign, counting_illegal_callback_fn, &ecount2); - secp256k1_context_set_error_callback(sign, counting_illegal_callback_fn, NULL); - CHECK(vrfy->error_callback.fn != sign->error_callback.fn); + /* set error callback (to a function that still aborts in case malloc() fails in secp256k1_context_clone() below) */ + secp256k1_context_set_error_callback(sign, secp256k1_default_illegal_callback_fn, NULL); + CHECK(sign->error_callback.fn != vrfy->error_callback.fn); + CHECK(sign->error_callback.fn == secp256k1_default_illegal_callback_fn); /* check if sizes for cloning are consistent */ CHECK(secp256k1_context_preallocated_clone_size(none) == secp256k1_context_preallocated_size(SECP256K1_CONTEXT_NONE)); @@ -239,7 +241,8 @@ void run_context_tests(int use_prealloc) { } /* Verify that the error callback makes it across the clone. */ - CHECK(vrfy->error_callback.fn != sign->error_callback.fn); + CHECK(sign->error_callback.fn != vrfy->error_callback.fn); + CHECK(sign->error_callback.fn == secp256k1_default_illegal_callback_fn); /* And that it resets back to default. */ secp256k1_context_set_error_callback(sign, NULL, NULL); CHECK(vrfy->error_callback.fn == sign->error_callback.fn); @@ -361,8 +364,8 @@ void run_scratch_tests(void) { CHECK(scratch->alloc_size != 0); CHECK(scratch->alloc_size % ALIGNMENT == 0); - /* Allocating another 500 bytes fails */ - CHECK(secp256k1_scratch_alloc(&none->error_callback, scratch, 500) == NULL); + /* Allocating another 501 bytes fails */ + CHECK(secp256k1_scratch_alloc(&none->error_callback, scratch, 501) == NULL); CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 0) == 1000 - adj_alloc); CHECK(secp256k1_scratch_max_allocation(&none->error_callback, scratch, 1) == 1000 - adj_alloc - (ALIGNMENT - 1)); CHECK(scratch->alloc_size != 0); @@ -395,6 +398,18 @@ void run_scratch_tests(void) { secp256k1_scratch_space_destroy(none, scratch); CHECK(ecount == 5); + /* Test that large integers do not wrap around in a bad way */ + scratch = secp256k1_scratch_space_create(none, 1000); + /* Try max allocation with a large number of objects. Only makes sense if + * ALIGNMENT is greater than 1 because otherwise the objects take no extra + * space. */ + CHECK(ALIGNMENT <= 1 || !secp256k1_scratch_max_allocation(&none->error_callback, scratch, (SIZE_MAX / (ALIGNMENT - 1)) + 1)); + /* Try allocating SIZE_MAX to test wrap around which only happens if + * ALIGNMENT > 1, otherwise it returns NULL anyway because the scratch + * space is too small. */ + CHECK(secp256k1_scratch_alloc(&none->error_callback, scratch, SIZE_MAX) == NULL); + secp256k1_scratch_space_destroy(none, scratch); + /* cleanup */ secp256k1_scratch_space_destroy(none, NULL); /* no-op */ secp256k1_context_destroy(none); @@ -2215,6 +2230,9 @@ void test_ge(void) { /* Normal doubling. */ secp256k1_gej_double_var(&resj, &gej[i2], NULL); ge_equals_gej(&ref, &resj); + /* Constant-time doubling. */ + secp256k1_gej_double(&resj, &gej[i2]); + ge_equals_gej(&ref, &resj); } /* Test adding opposites. */ @@ -2300,6 +2318,39 @@ void test_ge(void) { free(zinv); } + +void test_intialized_inf(void) { + secp256k1_ge p; + secp256k1_gej pj, npj, infj1, infj2, infj3; + secp256k1_fe zinv; + + /* Test that adding P+(-P) results in a fully initalized infinity*/ + random_group_element_test(&p); + secp256k1_gej_set_ge(&pj, &p); + secp256k1_gej_neg(&npj, &pj); + + secp256k1_gej_add_var(&infj1, &pj, &npj, NULL); + CHECK(secp256k1_gej_is_infinity(&infj1)); + CHECK(secp256k1_fe_is_zero(&infj1.x)); + CHECK(secp256k1_fe_is_zero(&infj1.y)); + CHECK(secp256k1_fe_is_zero(&infj1.z)); + + secp256k1_gej_add_ge_var(&infj2, &npj, &p, NULL); + CHECK(secp256k1_gej_is_infinity(&infj2)); + CHECK(secp256k1_fe_is_zero(&infj2.x)); + CHECK(secp256k1_fe_is_zero(&infj2.y)); + CHECK(secp256k1_fe_is_zero(&infj2.z)); + + secp256k1_fe_set_int(&zinv, 1); + secp256k1_gej_add_zinv_var(&infj3, &npj, &p, &zinv); + CHECK(secp256k1_gej_is_infinity(&infj3)); + CHECK(secp256k1_fe_is_zero(&infj3.x)); + CHECK(secp256k1_fe_is_zero(&infj3.y)); + CHECK(secp256k1_fe_is_zero(&infj3.z)); + + +} + void test_add_neg_y_diff_x(void) { /* The point of this test is to check that we can add two points * whose y-coordinates are negatives of each other but whose x @@ -2373,6 +2424,7 @@ void run_ge(void) { test_ge(); } test_add_neg_y_diff_x(); + test_intialized_inf(); } void test_ec_combine(void) { @@ -2967,14 +3019,16 @@ void test_ecmult_multi(secp256k1_scratch *scratch, secp256k1_ecmult_multi_func e void test_ecmult_multi_batch_single(secp256k1_ecmult_multi_func ecmult_multi) { secp256k1_scalar szero; - secp256k1_scalar sc[32]; - secp256k1_ge pt[32]; + secp256k1_scalar sc; + secp256k1_ge pt; secp256k1_gej r; ecmult_multi_data data; secp256k1_scratch *scratch_empty; - data.sc = sc; - data.pt = pt; + random_group_element_test(&pt); + random_scalar_order(&sc); + data.sc = ≻ + data.pt = &pt; secp256k1_scalar_set_int(&szero, 0); /* Try to multiply 1 point, but scratch space is empty.*/ @@ -3232,6 +3286,7 @@ void test_constant_wnaf(const secp256k1_scalar *number, int w) { int skew; int bits = 256; secp256k1_scalar num = *number; + secp256k1_scalar scalar_skew; secp256k1_scalar_set_int(&x, 0); secp256k1_scalar_set_int(&shift, 1 << w); @@ -3262,7 +3317,8 @@ void test_constant_wnaf(const secp256k1_scalar *number, int w) { secp256k1_scalar_add(&x, &x, &t); } /* Skew num because when encoding numbers as odd we use an offset */ - secp256k1_scalar_cadd_bit(&num, skew == 2, 1); + secp256k1_scalar_set_int(&scalar_skew, 1 << (skew == 2)); + secp256k1_scalar_add(&num, &num, &scalar_skew); CHECK(secp256k1_scalar_eq(&x, &num)); } @@ -3374,13 +3430,32 @@ void run_wnaf(void) { int i; secp256k1_scalar n = {{0}}; + test_constant_wnaf(&n, 4); /* Sanity check: 1 and 2 are the smallest odd and even numbers and should * have easier-to-diagnose failure modes */ n.d[0] = 1; test_constant_wnaf(&n, 4); n.d[0] = 2; test_constant_wnaf(&n, 4); - /* Test 0 */ + /* Test -1, because it's a special case in wnaf_const */ + n = secp256k1_scalar_one; + secp256k1_scalar_negate(&n, &n); + test_constant_wnaf(&n, 4); + + /* Test -2, which may not lead to overflows in wnaf_const */ + secp256k1_scalar_add(&n, &secp256k1_scalar_one, &secp256k1_scalar_one); + secp256k1_scalar_negate(&n, &n); + test_constant_wnaf(&n, 4); + + /* Test (1/2) - 1 = 1/-2 and 1/2 = (1/-2) + 1 + as corner cases of negation handling in wnaf_const */ + secp256k1_scalar_inverse(&n, &n); + test_constant_wnaf(&n, 4); + + secp256k1_scalar_add(&n, &n, &secp256k1_scalar_one); + test_constant_wnaf(&n, 4); + + /* Test 0 for fixed wnaf */ test_fixed_wnaf_small(); /* Random tests */ for (i = 0; i < count; i++) { @@ -5277,6 +5352,14 @@ void run_ecdsa_openssl(void) { # include "modules/recovery/tests_impl.h" #endif +#ifdef ENABLE_MODULE_EXTRAKEYS +# include "modules/extrakeys/tests_impl.h" +#endif + +#ifdef ENABLE_MODULE_SCHNORRSIG +# include "modules/schnorrsig/tests_impl.h" +#endif + void run_memczero_test(void) { unsigned char buf1[6] = {1, 2, 3, 4, 5, 6}; unsigned char buf2[sizeof(buf1)]; @@ -5583,6 +5666,14 @@ int main(int argc, char **argv) { run_recovery_tests(); #endif +#ifdef ENABLE_MODULE_EXTRAKEYS + run_extrakeys_tests(); +#endif + +#ifdef ENABLE_MODULE_SCHNORRSIG + run_schnorrsig_tests(); +#endif + /* util tests */ run_memczero_test(); diff --git a/src/secp256k1/src/tests_exhaustive.c b/src/secp256k1/src/tests_exhaustive.c index 8cca1cef2..681ed80bd 100644 --- a/src/secp256k1/src/tests_exhaustive.c +++ b/src/secp256k1/src/tests_exhaustive.c @@ -22,6 +22,7 @@ #endif #include "include/secp256k1.h" +#include "assumptions.h" #include "group.h" #include "secp256k1.c" #include "testrand_impl.h" @@ -141,10 +142,8 @@ void test_exhaustive_addition(const secp256k1_ge *group, const secp256k1_gej *gr /* Check doubling */ for (i = 0; i < order; i++) { secp256k1_gej tmp; - if (i > 0) { - secp256k1_gej_double_nonzero(&tmp, &groupj[i]); - ge_equals_gej(&group[(2 * i) % order], &tmp); - } + secp256k1_gej_double(&tmp, &groupj[i]); + ge_equals_gej(&group[(2 * i) % order], &tmp); secp256k1_gej_double_var(&tmp, &groupj[i], NULL); ge_equals_gej(&group[(2 * i) % order], &tmp); } diff --git a/src/secp256k1/src/util.h b/src/secp256k1/src/util.h index 8289e23e0..a5cbe03ef 100644 --- a/src/secp256k1/src/util.h +++ b/src/secp256k1/src/util.h @@ -170,13 +170,35 @@ static SECP256K1_INLINE void *manual_alloc(void** prealloc_ptr, size_t alloc_siz # define I64uFORMAT "llu" #endif -#if defined(HAVE___INT128) -# if defined(__GNUC__) -# define SECP256K1_GNUC_EXT __extension__ -# else -# define SECP256K1_GNUC_EXT +#if defined(__GNUC__) +# define SECP256K1_GNUC_EXT __extension__ +#else +# define SECP256K1_GNUC_EXT +#endif + +/* If SECP256K1_{LITTLE,BIG}_ENDIAN is not explicitly provided, infer from various other system macros. */ +#if !defined(SECP256K1_LITTLE_ENDIAN) && !defined(SECP256K1_BIG_ENDIAN) +/* Inspired by https://github.com/rofl0r/endianness.h/blob/9853923246b065a3b52d2c43835f3819a62c7199/endianness.h#L52L73 */ +# if (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) || \ + defined(_X86_) || defined(__x86_64__) || defined(__i386__) || \ + defined(__i486__) || defined(__i586__) || defined(__i686__) || \ + defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) || \ + defined(__ARMEL__) || defined(__AARCH64EL__) || \ + (defined(__LITTLE_ENDIAN__) && __LITTLE_ENDIAN__ == 1) || \ + (defined(_LITTLE_ENDIAN) && _LITTLE_ENDIAN == 1) || \ + defined(_M_IX86) || defined(_M_AMD64) || defined(_M_ARM) /* MSVC */ +# define SECP256K1_LITTLE_ENDIAN # endif -SECP256K1_GNUC_EXT typedef unsigned __int128 uint128_t; +# if (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) || \ + defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) || \ + defined(__MICROBLAZEEB__) || defined(__ARMEB__) || defined(__AARCH64EB__) || \ + (defined(__BIG_ENDIAN__) && __BIG_ENDIAN__ == 1) || \ + (defined(_BIG_ENDIAN) && _BIG_ENDIAN == 1) +# define SECP256K1_BIG_ENDIAN +# endif +#endif +#if defined(SECP256K1_LITTLE_ENDIAN) == defined(SECP256K1_BIG_ENDIAN) +# error Please make sure that either SECP256K1_LITTLE_ENDIAN or SECP256K1_BIG_ENDIAN is set, see src/util.h. #endif /* Zero memory if flag == 1. Flag must be 0 or 1. Constant time. */ @@ -197,10 +219,15 @@ static SECP256K1_INLINE void memczero(void *s, size_t len, int flag) { /** If flag is true, set *r equal to *a; otherwise leave it. Constant-time. Both *r and *a must be initialized and non-negative.*/ static SECP256K1_INLINE void secp256k1_int_cmov(int *r, const int *a, int flag) { unsigned int mask0, mask1, r_masked, a_masked; + /* Access flag with a volatile-qualified lvalue. + This prevents clang from figuring out (after inlining) that flag can + take only be 0 or 1, which leads to variable time code. */ + volatile int vflag = flag; + /* Casting a negative int to unsigned and back to int is implementation defined behavior */ VERIFY_CHECK(*r >= 0 && *a >= 0); - mask0 = (unsigned int)flag + ~0u; + mask0 = (unsigned int)vflag + ~0u; mask1 = ~mask0; r_masked = ((unsigned int)*r & mask0); a_masked = ((unsigned int)*a & mask1); @@ -208,4 +235,21 @@ static SECP256K1_INLINE void secp256k1_int_cmov(int *r, const int *a, int flag) *r = (int)(r_masked | a_masked); } +/* If USE_FORCE_WIDEMUL_{INT128,INT64} is set, use that wide multiplication implementation. + * Otherwise use the presence of __SIZEOF_INT128__ to decide. + */ +#if defined(USE_FORCE_WIDEMUL_INT128) +# define SECP256K1_WIDEMUL_INT128 1 +#elif defined(USE_FORCE_WIDEMUL_INT64) +# define SECP256K1_WIDEMUL_INT64 1 +#elif defined(__SIZEOF_INT128__) +# define SECP256K1_WIDEMUL_INT128 1 +#else +# define SECP256K1_WIDEMUL_INT64 1 +#endif +#if defined(SECP256K1_WIDEMUL_INT128) +SECP256K1_GNUC_EXT typedef unsigned __int128 uint128_t; +SECP256K1_GNUC_EXT typedef __int128 int128_t; +#endif + #endif /* SECP256K1_UTIL_H */ diff --git a/src/secp256k1/src/valgrind_ctime_test.c b/src/secp256k1/src/valgrind_ctime_test.c index 60a82d599..e676a8326 100644 --- a/src/secp256k1/src/valgrind_ctime_test.c +++ b/src/secp256k1/src/valgrind_ctime_test.c @@ -6,6 +6,7 @@ #include <valgrind/memcheck.h> #include "include/secp256k1.h" +#include "assumptions.h" #include "util.h" #if ENABLE_MODULE_ECDH @@ -16,6 +17,14 @@ # include "include/secp256k1_recovery.h" #endif +#if ENABLE_MODULE_EXTRAKEYS +# include "include/secp256k1_extrakeys.h" +#endif + +#if ENABLE_MODULE_SCHNORRSIG +#include "include/secp256k1_schnorrsig.h" +#endif + int main(void) { secp256k1_context* ctx; secp256k1_ecdsa_signature signature; @@ -32,6 +41,9 @@ int main(void) { secp256k1_ecdsa_recoverable_signature recoverable_signature; int recid; #endif +#if ENABLE_MODULE_EXTRAKEYS + secp256k1_keypair keypair; +#endif if (!RUNNING_ON_VALGRIND) { fprintf(stderr, "This test can only usefully be run inside valgrind.\n"); @@ -49,7 +61,9 @@ int main(void) { msg[i] = i + 1; } - ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_DECLASSIFY); + ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN + | SECP256K1_CONTEXT_VERIFY + | SECP256K1_CONTEXT_DECLASSIFY); /* Test keygen. */ VALGRIND_MAKE_MEM_UNDEFINED(key, 32); @@ -114,6 +128,30 @@ int main(void) { VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); CHECK(ret); + /* Test keypair_create and keypair_xonly_tweak_add. */ +#if ENABLE_MODULE_EXTRAKEYS + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + ret = secp256k1_keypair_create(ctx, &keypair, key); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret == 1); + + /* The tweak is not treated as a secret in keypair_tweak_add */ + VALGRIND_MAKE_MEM_DEFINED(msg, 32); + ret = secp256k1_keypair_xonly_tweak_add(ctx, &keypair, msg); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret == 1); +#endif + +#if ENABLE_MODULE_SCHNORRSIG + VALGRIND_MAKE_MEM_UNDEFINED(key, 32); + ret = secp256k1_keypair_create(ctx, &keypair, key); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret == 1); + ret = secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, NULL, NULL); + VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); + CHECK(ret == 1); +#endif + secp256k1_context_destroy(ctx); return 0; } diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index b4f392116..26de780f2 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -10,7 +10,6 @@ #endif #ifdef WIN32 -#define WIN32_LEAN_AND_MEAN 1 #ifndef NOMINMAX #define NOMINMAX #endif diff --git a/src/sync.cpp b/src/sync.cpp index 10f048318..322198a85 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -149,12 +149,17 @@ static void push_lock(void* c, const CLockLocation& locklocation) const LockPair p1 = std::make_pair(i.first, c); if (lockdata.lockorders.count(p1)) continue; - lockdata.lockorders.emplace(p1, lock_stack); const LockPair p2 = std::make_pair(c, i.first); + if (lockdata.lockorders.count(p2)) { + auto lock_stack_copy = lock_stack; + lock_stack.pop_back(); + potential_deadlock_detected(p1, lockdata.lockorders[p2], lock_stack_copy); + // potential_deadlock_detected() does not return. + } + + lockdata.lockorders.emplace(p1, lock_stack); lockdata.invlockorders.insert(p2); - if (lockdata.lockorders.count(p2)) - potential_deadlock_detected(p1, lockdata.lockorders[p2], lockdata.lockorders[p1]); } } @@ -233,12 +238,15 @@ void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, template void AssertLockHeldInternal(const char*, const char*, int, Mutex*); template void AssertLockHeldInternal(const char*, const char*, int, RecursiveMutex*); -void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) +template <typename MutexType> +void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) { if (!LockHeld(cs)) return; tfm::format(std::cerr, "Assertion failed: lock %s held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld()); abort(); } +template void AssertLockNotHeldInternal(const char*, const char*, int, Mutex*); +template void AssertLockNotHeldInternal(const char*, const char*, int, RecursiveMutex*); void DeleteLock(void* cs) { @@ -259,6 +267,17 @@ void DeleteLock(void* cs) } } +bool LockStackEmpty() +{ + LockData& lockdata = GetLockData(); + std::lock_guard<std::mutex> lock(lockdata.dd_mutex); + const auto it = lockdata.m_lock_stacks.find(std::this_thread::get_id()); + if (it == lockdata.m_lock_stacks.end()) { + return true; + } + return it->second.empty(); +} + bool g_debug_lockorder_abort = true; #endif /* DEBUG_LOCKORDER */ diff --git a/src/sync.h b/src/sync.h index 77327d8bf..7b397a800 100644 --- a/src/sync.h +++ b/src/sync.h @@ -53,9 +53,11 @@ void LeaveCritical(); void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line); std::string LocksHeld(); template <typename MutexType> -void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) ASSERT_EXCLUSIVE_LOCK(cs); -void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs); +void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(cs); +template <typename MutexType> +void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(!cs); void DeleteLock(void* cs); +bool LockStackEmpty(); /** * Call abort() if a potential lock order deadlock bug is detected, instead of @@ -64,13 +66,15 @@ void DeleteLock(void* cs); */ extern bool g_debug_lockorder_abort; #else -void static inline EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry = false) {} -void static inline LeaveCritical() {} -void static inline CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line) {} +inline void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry = false) {} +inline void LeaveCritical() {} +inline void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line) {} +template <typename MutexType> +inline void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(cs) {} template <typename MutexType> -void static inline AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) ASSERT_EXCLUSIVE_LOCK(cs) {} -void static inline AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) {} -void static inline DeleteLock(void* cs) {} +void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(!cs) {} +inline void DeleteLock(void* cs) {} +inline bool LockStackEmpty() { return true; } #endif #define AssertLockHeld(cs) AssertLockHeldInternal(#cs, __FILE__, __LINE__, &cs) #define AssertLockNotHeld(cs) AssertLockNotHeldInternal(#cs, __FILE__, __LINE__, &cs) diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index bc6b38c68..25fdd6456 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -392,7 +392,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) // Test: Sanity check, GetAddr should never return anything if addrman // is empty. BOOST_CHECK_EQUAL(addrman.size(), 0U); - std::vector<CAddress> vAddr1 = addrman.GetAddr(); + std::vector<CAddress> vAddr1 = addrman.GetAddr(/* max_addresses */ 0, /* max_pct */0); BOOST_CHECK_EQUAL(vAddr1.size(), 0U); CAddress addr1 = CAddress(ResolveService("250.250.2.1", 8333), NODE_NONE); @@ -415,13 +415,15 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) BOOST_CHECK(addrman.Add(addr4, source2)); BOOST_CHECK(addrman.Add(addr5, source1)); - // GetAddr returns 23% of addresses, 23% of 5 is 1 rounded down. - BOOST_CHECK_EQUAL(addrman.GetAddr().size(), 1U); + BOOST_CHECK_EQUAL(addrman.GetAddr(/* max_addresses */ 0, /* max_pct */ 0).size(), 5U); + // Net processing asks for 23% of addresses. 23% of 5 is 1 rounded down. + BOOST_CHECK_EQUAL(addrman.GetAddr(/* max_addresses */ 2500, /* max_pct */ 23).size(), 1U); // Test: Ensure GetAddr works with new and tried addresses. addrman.Good(CAddress(addr1, NODE_NONE)); addrman.Good(CAddress(addr2, NODE_NONE)); - BOOST_CHECK_EQUAL(addrman.GetAddr().size(), 1U); + BOOST_CHECK_EQUAL(addrman.GetAddr(/* max_addresses */ 0, /* max_pct */ 0).size(), 5U); + BOOST_CHECK_EQUAL(addrman.GetAddr(/* max_addresses */ 2500, /* max_pct */ 23).size(), 1U); // Test: Ensure GetAddr still returns 23% when addrman has many addrs. for (unsigned int i = 1; i < (8 * 256); i++) { @@ -436,7 +438,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) if (i % 8 == 0) addrman.Good(addr); } - std::vector<CAddress> vAddr = addrman.GetAddr(); + std::vector<CAddress> vAddr = addrman.GetAddr(/* max_addresses */ 2500, /* max_pct */ 23); size_t percent23 = (addrman.size() * 23) / 100; BOOST_CHECK_EQUAL(vAddr.size(), percent23); diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index 57559fa68..6a636f257 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -33,7 +33,7 @@ BOOST_AUTO_TEST_CASE(base58_EncodeBase58) std::vector<unsigned char> sourcedata = ParseHex(test[0].get_str()); std::string base58string = test[1].get_str(); BOOST_CHECK_MESSAGE( - EncodeBase58(sourcedata.data(), sourcedata.data() + sourcedata.size()) == base58string, + EncodeBase58(sourcedata) == base58string, strTest); } } diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index bf5c58777..0ad506660 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -12,6 +12,7 @@ #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> +#include <crypto/sha3.h> #include <crypto/sha512.h> #include <random.h> #include <test/util/setup_common.h> @@ -183,7 +184,7 @@ static void TestHKDF_SHA256_32(const std::string &ikm_hex, const std::string &sa CHKDF_HMAC_SHA256_L32 hkdf32(initial_key_material.data(), initial_key_material.size(), salt_stringified); unsigned char out[32]; hkdf32.Expand32(info_stringified, out); - BOOST_CHECK(HexStr(out, out + 32) == okm_check_hex); + BOOST_CHECK(HexStr(out) == okm_check_hex); } static std::string LongTestString() @@ -750,4 +751,110 @@ BOOST_AUTO_TEST_CASE(sha256d64) } } +static void TestSHA3_256(const std::string& input, const std::string& output) +{ + const auto in_bytes = ParseHex(input); + const auto out_bytes = ParseHex(output); + + SHA3_256 sha; + // Hash the whole thing. + unsigned char out[SHA3_256::OUTPUT_SIZE]; + sha.Write(in_bytes).Finalize(out); + assert(out_bytes.size() == sizeof(out)); + BOOST_CHECK(std::equal(std::begin(out_bytes), std::end(out_bytes), out)); + + // Reset and split randomly in 3 + sha.Reset(); + int s1 = InsecureRandRange(in_bytes.size() + 1); + int s2 = InsecureRandRange(in_bytes.size() + 1 - s1); + int s3 = in_bytes.size() - s1 - s2; + sha.Write(MakeSpan(in_bytes).first(s1)).Write(MakeSpan(in_bytes).subspan(s1, s2)); + sha.Write(MakeSpan(in_bytes).last(s3)).Finalize(out); + BOOST_CHECK(std::equal(std::begin(out_bytes), std::end(out_bytes), out)); +} + +BOOST_AUTO_TEST_CASE(keccak_tests) +{ + // Start with the zero state. + uint64_t state[25] = {0}; + CSHA256 tester; + for (int i = 0; i < 262144; ++i) { + KeccakF(state); + for (int j = 0; j < 25; ++j) { + unsigned char buf[8]; + WriteLE64(buf, state[j]); + tester.Write(buf, 8); + } + } + uint256 out; + tester.Finalize(out.begin()); + // Expected hash of the concatenated serialized states after 1...262144 iterations of KeccakF. + // Verified against an independent implementation. + BOOST_CHECK_EQUAL(out.ToString(), "5f4a7f2eca7d57740ef9f1a077b4fc67328092ec62620447fe27ad8ed5f7e34f"); +} + +BOOST_AUTO_TEST_CASE(sha3_256_tests) +{ + // Test vectors from https://csrc.nist.gov/CSRC/media/Projects/Cryptographic-Algorithm-Validation-Program/documents/sha3/sha-3bytetestvectors.zip + + // SHA3-256 Short test vectors (SHA3_256ShortMsg.rsp) + TestSHA3_256("", "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a"); + TestSHA3_256("e9", "f0d04dd1e6cfc29a4460d521796852f25d9ef8d28b44ee91ff5b759d72c1e6d6"); + TestSHA3_256("d477", "94279e8f5ccdf6e17f292b59698ab4e614dfe696a46c46da78305fc6a3146ab7"); + TestSHA3_256("b053fa", "9d0ff086cd0ec06a682c51c094dc73abdc492004292344bd41b82a60498ccfdb"); + TestSHA3_256("e7372105", "3a42b68ab079f28c4ca3c752296f279006c4fe78b1eb79d989777f051e4046ae"); + TestSHA3_256("0296f2c40a", "53a018937221081d09ed0497377e32a1fa724025dfdc1871fa503d545df4b40d"); + TestSHA3_256("e6fd42037f80", "2294f8d3834f24aa9037c431f8c233a66a57b23fa3de10530bbb6911f6e1850f"); + TestSHA3_256("37b442385e0538", "cfa55031e716bbd7a83f2157513099e229a88891bb899d9ccd317191819998f8"); + TestSHA3_256("8bca931c8a132d2f", "dbb8be5dec1d715bd117b24566dc3f24f2cc0c799795d0638d9537481ef1e03e"); + TestSHA3_256("fb8dfa3a132f9813ac", "fd09b3501888445ffc8c3bb95d106440ceee469415fce1474743273094306e2e"); + TestSHA3_256("71fbacdbf8541779c24a", "cc4e5a216b01f987f24ab9cad5eb196e89d32ed4aac85acb727e18e40ceef00e"); + TestSHA3_256("7e8f1fd1882e4a7c49e674", "79bef78c78aa71e11a3375394c2562037cd0f82a033b48a6cc932cc43358fd9e"); + TestSHA3_256("5c56a6b18c39e66e1b7a993a", "b697556cb30d6df448ee38b973cb6942559de4c2567b1556240188c55ec0841c"); + TestSHA3_256("9c76ca5b6f8d1212d8e6896ad8", "69dfc3a25865f3535f18b4a7bd9c0c69d78455f1fc1f4bf4e29fc82bf32818ec"); + TestSHA3_256("687ff7485b7eb51fe208f6ff9a1b", "fe7e68ae3e1a91944e4d1d2146d9360e5333c099a256f3711edc372bc6eeb226"); + TestSHA3_256("4149f41be1d265e668c536b85dde41", "229a7702448c640f55dafed08a52aa0b1139657ba9fc4c5eb8587e174ecd9b92"); + TestSHA3_256("d83c721ee51b060c5a41438a8221e040", "b87d9e4722edd3918729ded9a6d03af8256998ee088a1ae662ef4bcaff142a96"); + TestSHA3_256("266e8cbd3e73d80df2a49cfdaf0dc39cd1", "6c2de3c95900a1bcec6bd4ca780056af4acf3aa36ee640474b6e870187f59361"); + TestSHA3_256("a1d7ce5104eb25d6131bb8f66e1fb13f3523", "ee9062f39720b821b88be5e64621d7e0ca026a9fe7248d78150b14bdbaa40bed"); + TestSHA3_256("d751ccd2cd65f27db539176920a70057a08a6b", "7aaca80dbeb8dc3677d18b84795985463650d72f2543e0ec709c9e70b8cd7b79"); + TestSHA3_256("b32dec58865ab74614ea982efb93c08d9acb1bb0", "6a12e535dbfddab6d374058d92338e760b1a211451a6c09be9b61ee22f3bb467"); + TestSHA3_256("4e0cc4f5c6dcf0e2efca1f9f129372e2dcbca57ea6", "d2b7717864e9438dd02a4f8bb0203b77e2d3cd8f8ffcf9dc684e63de5ef39f0d"); + TestSHA3_256("d16d978dfbaecf2c8a04090f6eebdb421a5a711137a6", "7f497913318defdc60c924b3704b65ada7ca3ba203f23fb918c6fb03d4b0c0da"); + TestSHA3_256("47249c7cb85d8f0242ab240efd164b9c8b0bd3104bba3b", "435e276f06ae73aa5d5d6018f58e0f009be351eada47b677c2f7c06455f384e7"); + TestSHA3_256("cf549a383c0ac31eae870c40867eeb94fa1b6f3cac4473f2", "cdfd1afa793e48fd0ee5b34dfc53fbcee43e9d2ac21515e4746475453ab3831f"); + TestSHA3_256("9b3fdf8d448680840d6284f2997d3af55ffd85f6f4b33d7f8d", "25005d10e84ff97c74a589013be42fb37f68db64bdfc7626efc0dd628077493a"); + TestSHA3_256("6b22fe94be2d0b2528d9847e127eb6c7d6967e7ec8b9660e77cc", "157a52b0477639b3bc179667b35c1cdfbb3eef845e4486f0f84a526e940b518c"); + TestSHA3_256("d8decafdad377904a2789551135e782e302aed8450a42cfb89600c", "3ddecf5bba51643cd77ebde2141c8545f862067b209990d4cb65bfa65f4fa0c0"); + TestSHA3_256("938fe6afdbf14d1229e03576e532f078898769e20620ae2164f5abfa", "9511abd13c756772b852114578ef9b96f9dc7d0f2b8dcde6ea7d1bd14c518890"); + TestSHA3_256("66eb5e7396f5b451a02f39699da4dbc50538fb10678ec39a5e28baa3c0", "540acf81810a199996a612e885781308802fe460e9c638cc022e17076be8597a"); + TestSHA3_256("de98968c8bd9408bd562ac6efbca2b10f5769aacaa01365763e1b2ce8048", "6b2f2547781449d4fa158180a178ef68d7056121bf8a2f2f49891afc24978521"); + TestSHA3_256("94464e8fafd82f630e6aab9aa339d981db0a372dc5c1efb177305995ae2dc0", "ea7952ad759653cd47a18004ac2dbb9cf4a1e7bba8a530cf070570c711a634ea"); + TestSHA3_256("c178ce0f720a6d73c6cf1caa905ee724d5ba941c2e2628136e3aad7d853733ba", "64537b87892835ff0963ef9ad5145ab4cfce5d303a0cb0415b3b03f9d16e7d6b"); + TestSHA3_256("14365d3301150d7c5ba6bb8c1fc26e9dab218fc5d01c9ed528b72482aadee9c27bef667907797d55514468f68791f053daa2df598d7db7d54beea493bdcbb0c75c7b36ad84b9996dca96354190bd96d9d7fbe8ff54ffaf77c55eb92985da50825ee3b4179f5ec88b6fa60bb361d0caf9493494fe4d28ef843f0f498a2a9331b82a", "9b690531dee948a9c559a2e0efab2ec824151a9175f2730a030b748d07cbaa7f"); + TestSHA3_256("4a757db93f6d4c6529211d70d5f8491799c0f73ae7f24bbd2138db2eaf2c63a85063b9f7adaa03fc348f275323248334e3ffdf9798859f9cf6693d29566ff7d50976c505ecb58e543c459b39acdf4ce4b5e80a682eaa7c1f1ce5fe4acb864ff91eb6892b23165735ea49626898b40ceeb78161f5d0ea4a103cb404d937f9d1dc362b", "1ac7cc7e2e8ea14fb1b90096f41265100712c5dd41519d78b2786cfb6355af72"); + TestSHA3_256("da11c39c77250f6264dda4b096341ff9c4cc2c900633b20ea1664bf32193f790a923112488f882450cf334819bbaca46ffb88eff0265aa803bc79ca42739e4347c6bff0bb9aa99780261ffe42be0d3b5135d03723338fb2776841a0b4bc26360f9ef769b34c2bec5ed2feb216e2fa30fa5c37430c0360ecbfba3af6fb6b8dedacbb95c", "c163cd43de224ac5c262ae39db746cfcad66074ebaec4a6da23d86b310520f21"); + TestSHA3_256("3341ca020d4835838b0d6c8f93aaaebb7af60730d208c85283f6369f1ee27fd96d38f2674f316ef9c29c1b6b42dd59ec5236f65f5845a401adceaa4cf5bbd91cac61c21102052634e99faedd6cdddcd4426b42b6a372f29a5a5f35f51ce580bb1845a3c7cfcd447d269e8caeb9b320bb731f53fe5c969a65b12f40603a685afed86bfe53", "6c3e93f2b49f493344cc3eb1e9454f79363032beee2f7ea65b3d994b5cae438f"); + TestSHA3_256("989fc49594afc73405bacee4dbbe7135804f800368de39e2ea3bbec04e59c6c52752927ee3aa233ba0d8aab5410240f4c109d770c8c570777c928fce9a0bec9bc5156c821e204f0f14a9ab547e0319d3e758ae9e28eb2dbc3d9f7acf51bd52f41bf23aeb6d97b5780a35ba08b94965989744edd3b1d6d67ad26c68099af85f98d0f0e4fff9", "b10adeb6a9395a48788931d45a7b4e4f69300a76d8b716c40c614c3113a0f051"); + TestSHA3_256("e5022f4c7dfe2dbd207105e2f27aaedd5a765c27c0bc60de958b49609440501848ccf398cf66dfe8dd7d131e04f1432f32827a057b8904d218e68ba3b0398038d755bd13d5f168cfa8a11ab34c0540873940c2a62eace3552dcd6953c683fdb29983d4e417078f1988c560c9521e6f8c78997c32618fc510db282a985f868f2d973f82351d11", "3293a4b9aeb8a65e1014d3847500ffc8241594e9c4564cbd7ce978bfa50767fe"); + TestSHA3_256("b1f6076509938432145bb15dbe1a7b2e007934be5f753908b50fd24333455970a7429f2ffbd28bd6fe1804c4688311f318fe3fcd9f6744410243e115bcb00d7e039a4fee4c326c2d119c42abd2e8f4155a44472643704cc0bc72403b8a8ab0fd4d68e04a059d6e5ed45033b906326abb4eb4147052779bad6a03b55ca5bd8b140e131bed2dfada", "f82d9602b231d332d902cb6436b15aef89acc591cb8626233ced20c0a6e80d7a"); + TestSHA3_256("56ea14d7fcb0db748ff649aaa5d0afdc2357528a9aad6076d73b2805b53d89e73681abfad26bee6c0f3d20215295f354f538ae80990d2281be6de0f6919aa9eb048c26b524f4d91ca87b54c0c54aa9b54ad02171e8bf31e8d158a9f586e92ffce994ecce9a5185cc80364d50a6f7b94849a914242fcb73f33a86ecc83c3403630d20650ddb8cd9c4", "4beae3515ba35ec8cbd1d94567e22b0d7809c466abfbafe9610349597ba15b45"); + + // SHA3-256 Long test vectors (SHA3_256LongMsg.rsp) + TestSHA3_256("b1caa396771a09a1db9bc20543e988e359d47c2a616417bbca1b62cb02796a888fc6eeff5c0b5c3d5062fcb4256f6ae1782f492c1cf03610b4a1fb7b814c057878e1190b9835425c7a4a0e182ad1f91535ed2a35033a5d8c670e21c575ff43c194a58a82d4a1a44881dd61f9f8161fc6b998860cbe4975780be93b6f87980bad0a99aa2cb7556b478ca35d1f3746c33e2bb7c47af426641cc7bbb3425e2144820345e1d0ea5b7da2c3236a52906acdc3b4d34e474dd714c0c40bf006a3a1d889a632983814bbc4a14fe5f159aa89249e7c738b3b73666bac2a615a83fd21ae0a1ce7352ade7b278b587158fd2fabb217aa1fe31d0bda53272045598015a8ae4d8cec226fefa58daa05500906c4d85e7567", "cb5648a1d61c6c5bdacd96f81c9591debc3950dcf658145b8d996570ba881a05"); + TestSHA3_256("712b03d9ebe78d3a032a612939c518a6166ca9a161183a7596aa35b294d19d1f962da3ff64b57494cb5656e24adcf3b50e16f4e52135d2d9de76e94aa801cf49db10e384035329c54c9455bb3a9725fd9a44f44cb9078d18d3783d46ce372c31281aecef2f8b53d5702b863d71bc5786a33dd15d9256103b5ff7572f703d5cde6695e6c84f239acd1d6512ef581330590f4ab2a114ea064a693d5f8df5d908587bc7f998cde4a8b43d8821595566597dc8b3bf9ea78b154bd8907ee6c5d4d8a851f94be510962292b7ddda04d17b79fab4c022deb400e5489639dbc448f573d5cf72073a8001b36f73ac6677351b39d9bdb900e9a1121f488a7fa0aee60682e7dc7c531c85ec0154593ded3ae70e4121cae58445d8896b549cacf22d07cdace7625d57158721b44851d796d6511c38dac28dd37cbf2d7073b407fbc813149adc485e3dacee66755443c389d2d90dc70d8ff91816c0c5d7adbad7e30772a1f3ce76c72a6a2284ec7f174aefb6e9a895c118717999421b470a9665d2728c3c60c6d3e048d58b43c0d1b5b2f00be8b64bfe453d1e8fadf5699331f9", "095dcd0bc55206d2e1e715fb7173fc16a81979f278495dfc69a6d8f3174eba5a"); + TestSHA3_256("2a459282195123ebc6cf5782ab611a11b9487706f7795e236df3a476404f4b8c1e9904e2dc5ef29c5e06b179b8649707928c3913d1e53164747f1fa9bba6eeaf8fb759d71e32adc8c611d061345882f1cdeee3ab4cab3554adb2e43f4b01c37b4546994b25f4dcd6c497bc206865643930157cb5b2f4f25be235fa223688535907efcc253bcd083021407ea09cb1c34684aa0c1849e7efe2d9af6938c46525af9e5afb4da6e5b83da4b61dc718672a8090549cbe5aadb44f5bc93a6b3fbdc2e6d32e2eaaae637465179ea17f23ad1e4f1ebc328e2c6dc90c302b74a1edbbb0676c136b269d70c41040a313af06ab291bf489d9700950b77f207c1fc41884799931b3bca8b93331a6e96b7a3f0a8bd24cdb64964c377e0512f36444bb0643a4e3ecb328194cd5428fd89ede167472a14a9bf5730aff1e3b2c708de96eff1ebaaf63beb75f9c7d8034d6e5471e8f8a1f7efce37793a958e134619c19c54d3d42645f7a7263f25471fbaae8be3ea2fbd34ec6d7aacd7d5680948c3cd9a837c9c469a88f600d95829f4d1e4e4a5ef4ed4623c07815a1c33d9fb3b91333ff04eac92806a68a46cf2e9293f8bff466ce87fe66b46fbff7c238c7f9b2c92eb2fdc7d8084167f6f4e680d03301e5c33f78f1857d6863b1b8c36c7fce3e07d2a96a8979712079ae0023a1e3970165bfcf3a5463d2a4fdf1ca0e044f9a247528cd935734cb6d85ba53ceb95325c0eaf0ff5cd81ecb32e58917eb26bfc52dba3704bf5a927fee3220", "cb1c691c87244c0caf733aacd427f83412cd48820b358c1b15dd9fadee54e5af"); + TestSHA3_256("32659902674c94473a283be00835eb86339d394a189a87da41dad500db27da6b6a4753b2bb219c961a227d88c6df466ba2fc1e9a2d4c982db4398778c76714d5e9940da48bc3808f3c9989131a07683b8c29d6af336e9aee1dfa57d83c48a86f17146edec07869bb06550689ebf4788159ed0a921048b4a6e3e3ec272413bec15d8e1f6a40897fa0e11d9df223ef9fc270106249ae220fdc6ebdef6d6611805421ccc850f53ee9c836baf657a94005883b5a85def344d218264f07b2ea8714afcc941096c6ded0bb6bf5b8bf652fd15a21931c58c9f526e27363ddff98c0a25bc7af9f469ab35bffea948b333f042cc18a82cec0177f33c3bdbf185b580353de79e51e675b03b31e195f19ba1f063d44def0441dc52820426c2c61cf12974ec249fd3502f017ffa06220075ced7e2d6b86a52677ba3916e8e8726062aec5bc8ea1c18b1e4137680b2c9d002191b423bee8691bd7e0f93c3b9959bc1c14d5c5cbe8f7c9c336aa16e9de9faa12f3f048c66d04cb441eb2bbc5e8a91e052c0f9000856896f9b7ba30c1e2eead36fc7ac30a7d3ddfc65caaba0e3b292d26dfba46b5e2dc9bc9acadde1c9f52b2969299bd1281ddff65822b629cfba2928613200e73661b803afdcc4a817d9361389e975e67dfadd22a797bdaf991ddf42db18711c079ecec55925f9978e478612609bacd900172011c27e24bad639ffc24a23877278318872153aef6893ccb5b68b94b33154df7334375aadd3edbb35272cc7b672dec68faa62900873ded52f6049891b77f2d0311a84b19b73660e09d1f1998095c1da1edecfa9f741b5fd6db048dd68255085d43529279021d59ed853470d6863b7c8e07fcb0d1e6acfb1eb16f7f60bb1f46ce70493010e57930a3b4b8b87e065272f6f1dd31df057627f4214e58798b664e1e40960f2789d44ccacfb3dbd8b02a68a053976711f8034c1ed3a8", "5ac9275e02543410359a3f364b2ae3b85763321fd6d374d13fe54314e5561b01"); + TestSHA3_256("a65da8277a3b3738432bca9822d43b3d810cdad3b0ed2468d02bd269f1a416cd77392190c2dde8630eeb28a297bda786017abe9cf82f14751422ac9fff6322d5d9a33173db49792d3bc37fff501af667f7ca3dd335d028551e04039ef5a9d42a9443e1b80ea872fd945ad8999514ae4a29a35f60b0f7e971b67ae04d1ba1b53470c03847a3225c3ddf593a57aed3599661ae2d2bb1cddd2fa62c4a94b8704c5c35c33e08e2debe54e567ae21e27e7eb36593ae1c807a8ef8b5c1495b15412108aaf3fce4130520aa6e2d3bdf7b3ea609fdf9ea1c64258435aae2e58a7b3abda198f979c17dbe0aa74253e979bf3a5800f388ea11a7f7454c4e36270a3083a790c77cbe89693205b32880c0d8f79b1c000ee9b5e58f175ba7696616c17c45673cff25d1221f899836e95cc9e26a887a7115c4537e65ad4eacc319ba98a9a8860c089cbc76e7ea4c984d900b80622afbbbd1c0cdc670e3a4c523f81c77fed38b6aa988876b097da8411cc48e9b25a826460a862aa3fadfe75952aa4347c2effebdac9138ebcc6c34991e9f5b19fc2b847a87be72ff49c99ecf19d837ee3e23686cd760d9dd7adc78091bca79e42fdb9bc0120faec1a6ca52913e2a0156ba9850e1f39d712859f7fdf7daedf0e206dff67e7121e5d1590a8a068947a8657d753e83c7f009b6b2e54acc24afc9fdc9601a1d6d9d1f17aab0ce96c4d83405d1e3baba1dffa86ecccee7f1c1b80b1bbf859106ce2b647ae1e4a6a9b584ae1dfc0a4deebb755638f1d95dcc79b1be263177e2a05c72bde545d09ba726f41d9547117e876af81bfc672e33c71442eb05675d9552df1b313d1f9934f9ddd08955fa21d6edf23000a277f6f149591299a0a96032861ecdc96bb76afa05a2bffb445d61dc891bc70c13695920b911cad0df3fa842a3e2318c57556974343f69794cb8fa18c1ad624835857e4781041198aa705c4d11f3ef82e941be2aee7a770e54521312fe6facbaf1138eee08fa90fae986a5d93719aeb30ac292a49c1d91bf4574d553a92a4a6c305ab09db6bbeffd84c7aa707f1c1628a0220d6ba4ee5e960566686228a6e766d8a30dddf30ed5aa637c949950c3d0e894a7560670b6879a7d70f3c7e5ab29aed236cc3527bdea076fec8add12d784fbcf9a", "68f62c418a6b97026cc70f6abf8419b671ee373709fa13074e37bd39f0a50fcb"); + TestSHA3_256("460f8c7aac921fa9a55800b1d04cf981717c78217cd43f98f02c5c0e66865c2eea90bcce0971a0d22bc1c74d24d9bfea054e558b38b8502fccb85f190d394f2f58f581a02d3b9cc986f07f5a67d57ab4b707bd964ecc10f94f8cc538b81eeb743746c537407b7b575ced0e1ec4c691a72eb0978be798e8be22b278b390be99c730896fdc69b6a44456be5ee261366e8b1351cbb22aa53e45ec325ed2bca0bfeeebc867d7d07681581b6d56ed66ac78280df04053407a7b57561261dd644cc7b20f0a95709e42795b5402dd89fcb11746c597e0b650a008bc085c681bb24b17db4458e1effba3f414a883ddfc4bccb3ace24d9223839d4b3ca9185ad5cc24193134b9339b0e205a4cc0fa3d8f7a85b4230d1b3ee101fbae9ee14c2153da5f337c853573bd004114cb436ee58ab1648373ee07cc39f14198ac5a02a4dd0585cf83dfd4899df88e8859dae8bc351af286642c1c25737bf8712cb941cbbb741d540feb9f5d831f901fbe2d6facd7dab626bd705f2fd7c9a7a0e7a9127e3451af2ae8509dd7b79dce41c1e30b9dba1c38cb4861dad3ac00d68fa5d07ba591c1c3b9d6b7d6e08099d0572ca4c475240601decba894fa3c4b0ea52ed687281beee268a1c8535e283b1fc7c51aa31d5ec098c50fec958acdd0d54a49643bef170093a1102a1b3bf5ad42fb55ebaf7db07385eadcd6e66da8b7b6e6c022a1e3d01f5fccec86365d3014c159a3bff17d614751b3fa0e8e89152936e159b7c0ea8d71cd4ffd83adae209b254b793f6f06bb63838a303b95c85b4edfa4ddcca0ed952165930bca87140f67f5389d1233fe04f0a3d647050410c44d389513084ad53155af00de02cc7943a3b988d8e1454f85153aff0816e24b964ec91dc514c588a93634ff3dd485c40575faa2f254abdf86fbcf6d381337601a7b1ba5b99719f045eb7bf6f2e8b9dd9d053ef0b3126f984fc9ea87a2a70b3798fab593b83a4ff44d9c0c4ec3e570ac537c10d9e3c4996027a813b70d7867b858f31f508aa56e7b087370707974b2186f02f5c549112f2158c0d365402e52cba18fe245f77f7e6fbf952ec2dc3c880b38be771caea23bc22838b1f70472d558bdf585d9c77088b7ba2dceaeb3e6f96df7d91d47da1ec42be03936d621ecf747f24f9073c122923b4161d99bc8190e24f57b6fac952ed344c7eae86a5f43c08089c28c7daf3aa7e39c59d6f1e17ece1977caf6b4a77a6ff52774521b861f38ebc978005e5763cc97123e4d17c7bc4134c8f139c7d7a9a02646fef9525d2a6871fc99747e81430b3fec38c677427c6f5e2f16c14eee646ebf6eb16775ad0957f8684c7045f7826bc3736eca", "7d495ddf961cbff060f80b509f2b9e20bed95319eef61c7adb5edeec18e64713"); + TestSHA3_256("c8a2a26587d0126abe9ba8031f37d8a7d18219c41fe639bc7281f32d7c83c376b7d8f9770e080d98d95b320c0f402d57b7ef680da04e42dd5211aacf4426ecca5050ca596312cfae79cee0e8c92e14913cc3c66b24ece86c2bfa99078991faad7b513e94f0b601b7853ddb1eb3c9345f47445a651389d070e482ea5db48d962820257daf1cbe4bb8e5f04a3637d836c8c1bc4d83d6eda5f165f2c2592be268412712ae324ef054bb812f56b8bc25c1d59071c64dd3e00df896924c84575817027861faa5f016c5c74142272daa767e8c9dacee4c732ab08b5fa9ad65a0b74c73fb5a889169f645e50d70e41d689415f7d0b4ec071e9238b5a88110856fc6ae9b9944817e21597d1ccd03b60e60472d1e11d3e9063de24a7b59609b6a2a4ee68238690cf2800614746941c48af9566e07494f0dd236e091e75a8f769e3b179b30c10f5277eec7b3f5c97337189b8b82bc5e717ff27355b2009356caa908e976ae1d7f7a94d36202a8d5e03641aeac0e453a8168ee5a0858ceecfcbf11fb8c1f033201add297a0a89476d2ea8b9a82bda8c3c7ef4f55c3295a4ecb7c607ac73d37eadc13b7a2494ec1928f7a80c8d534efe38a3d9ccb4ccdab9f092a1def6478532c5ad3cd5c259b3812600fa89e6d1e228114795d246cedc9c9fff0d1c1297a5ddfc1169c2efb3800df8dd18a8511214785abcc1bc7eb31bdb2f5f70358dfe860ed5a03ab7e95cc21df5ee7aee68be568d6985e5c1e91408e4432663b1c4e6d613d6dc382b5b900a4fc1b7a9c27a1138c5e2356ab9026c34465006602753daf6ab7427da93c307c901d0bb1ddb21c53bc0493dd8d857161e8ffa51fdecb75568243205aa979c2e7ed2a77b5f8edc34cffb0321a8c653bc381f96ab85a86bf0bb2c9518208d636eac40aa7ad754260a75d4a46362f994c90173b975afb0ee17601311b1c51ba562c1ca7e3c2dd18b90bdebb1858fe876c71b3ad742c4bcba33e7763c750098de856fde8731cb6d698218be9f0a98298630e5b374957d126cf0b1c489c48bab6b50f6fb59ee28be6c3916bbd16514234f80e1ac15d0215852b87f9c6e429eb9f85007bf6ae3de1af0202861fd177c7c4f51af533f956a051815815c6e51e25af20d02893e95442991f1de5f86a4397ae20d9f675657bf9f397267831e94cef4e4d287f759850350ce0898f2e29de3c5c41f4246fe998a8d1359a2bed36ded1e4d6b08682025843700fee8cab56703e342212870acdd53655255b35e414fa53d9810f47a37195f22d72f6e555392023a08adc282c585b2ae62e129efccdc9fe9617eecac12b2ecdabd247a1161a17750740f90ebed3520ceb17676f1fa87259815ff415c2794c5953f689c8d5407dbbd10d1241a986e265cea901af34ec1ded0323ca3290a317208ba865637af4797e65b9cfcad3b931bbf6ac896623e2f4408529172911f1b6a9bcae8279ec7e33452d0cd7b026b46a99cbe8a69cd4d21cdc6d3a84002fab527c4fd18a121526d49890ced3fb89beb384b524015a2e03c049241eb9", "b8d4b29b086ef6d6f73802b9e7a4f2001e384c8258e7046e6779662fd958517e"); + TestSHA3_256("3a86a182b54704a3af811e3e660abcfbaef2fb8f39bab09115c1068976ff694bb6f5a3839ae44590d73e4996d45af5ceb26b03218ab3fef6f5f4ef48d22839fb4371c270f9535357b22142c4ffb54e854b64cab41932fe888d41ca702e908c63eae244715bfbf69f481250f16f848dc881c6996e6f9d76f0e491de2c129f2a2ab22e72b04644f610a2fabc45aa2d7b3e5d77b87a135d2fd502ca74a207bddaf9a43e945245961a53c7bfcfe73a1ae090e6606ffe8ddbf1e0f0d6d4fa94526578c6faf282dd592b10bf4bce00a7b1846625690623667e83b9b59b465d42c6944e224ad36698f5f2ee938404b7775c2e66207bc41025adaf07590312f398812d24c0178126fdd334964a54b8353482a83be17cf2ee52d23b72e5f57fe31eebf8a1a64742eb9459bcb0eca231a1658ab88b7056d8e47554f0a46058d6565c6cbf6edec45fdde6f051e38255b82493de27ffd3efbe1b179b9642d2166073db6d4832707420237a00bad7125795e645e5bc3e1431ecbabf0ff5f74416626322545c966241cce6d8f2c035a78f100e030741f13b02a9eaf618d468bc40274db98bc342be12ad4d892c2ba546e571c556ac7cbf4e4c3fd3431efd40457cf65a297845dd8cce09811418c3cef941ff32c43c375157f6f49c2e893625e4b216b1f985aa0fd25f29a9011d4f59c78b037ed71f384e5de8116e3fc148c0a3cad07cb119b9829aac55eed9a299edb9abc5d017be485f690add70ff2efbb889ac6ce0da9b3bdbeb9dd47823116733d58a8d510b7f2e2c8244a2cbf53816b59e413207fb75f9c5ce1af06e67d182d3250ea3283bcbb45cb07ea6a6aa486361eb6f69199c0eb8e6490beff82e4ab274b1204e7f2f0ba097fba0332aa4c4a861771f5b3d45ce43e667581a40fee4bebe7fa9d87b70a5bb876c928f7e6d16ae604b3a4e9c7f1d616e2deab96b6207705b9a8f87468503cdd20a3c02cc8da43d046da68b5ed163d926a5a714a4df1b8ef007bca408f68b9e20de86d6398ad81df5e74d5aaac40874b5d6787211ff88e128cf1676e84ca7f51aee5951efee1915dcc11502a8df74fac4c8451dda49b631a8fb87470f0ebe9b67449bbd1640ceee6101e8cd82aa1033fa84f75b28450e461b93f65da5c43759b0e83660d50961702bb1ad015dad42e600117475237cf6e7279d4a02d1f67cf59de0108355d03963e3d84ce7647173dd7d77a6b3f275d7de74236d7bbb2df437d536136dbe1dbe8f307facc7bc7d0cde1abf745cbeb81af1ab2c46138cf007e901f22668377958bcbbadb7e9905973b27ff0c5baaece25e974c1bd116cc81dd1c81a30bae86a6fb12c6a5494068e122153128313eb3e628d76e9babc823c9eb9d3b81bacfa7a6b372abe6b1246a350f23e2e95b09c9037a75aac255ef7d4f267cad3ce869531b4165db2e5a9792094efea4ae3d9ea4d0efdc712e63df21882a353743190e016b2166e4da8a2c78e48defc7155d5fdfc4e596624e6a19c91b43719a22c1204b1cefe05989d455773d3881fa8d3eefc255f81dfe90bd41dc6f1e9c265a753298a6e98c999acd9525a9db5f9f9456a0f51a93dd9693e1d9c3fa283f7c58a9c752afcaa635abea8dfc80e2c326b939260069457fdad68c341852dcb5fcbbd351318defd7ae3b9f827478eb77306a5ae14cf8895f2bc6f0f361ffc8aa37e286629dc7e59b73a8712525e851c64d363065631edc1609f3d49a09575876a", "b71ec00c0fcc4f8663312711540df1cd236eb52f237409415b749ff9436dc331"); + TestSHA3_256("c041e23b6d55998681802114abc73d2776967cab715572698d3d497ec66a790b0531d32f45b3c432f5b2d8039ea47de5c6060a6514f3ff8fb5f58e61fd1b5b80524c812a46dad56c035a6e95ecb465ea8176d99b836e36f65977b7dbb3932a706d3af415b6f2549b7120ecb0db1e7d9e6f8df23607eda006436bccd32ef96d431fa434d9de22ca2608ab593eb50b4d6a57f45c1ce698c3283a77d330b876ad6030324a5c0693be7790a4bd26c0a25eb403531f37689829c20546d6dc97327131688b3d88766db8f5d1b22050450c37e53951446dd7155a3e6d7edcbe1354411d8f58154475d74008937e8ba48b706066c296d1a87936dd023ac8eebe7605a58c6c40da774cf9df189db0050adcf7629e66cbd1cf9824397834cb13c4066c26e6c8ec950b44fc1c8db8ef976a7ec8c4f4ec9849ca7a07f906223053b80db24b946b034ee7a30880d0ace348acba0d0ed21ea443816706a216ce9eb682d1fe9dfc1d2e0bf3b1449247413520b8d8ebc99fc298c6dca949be0ffebe450b9b79a387a615d617b8d9da5b3e8d2776208c7cc2a11bdbc387f9d4597b380739b24ae59dcd5fb63bfeefe0746d9266cfda18afa583d6891e483e6d5c0db305f5609beba75bb5b447ccac2dfb94ede4a94db6eaaf3070d8d5353f107f7bd74528eb913e0b19bed6236a3b48567c46a9eec28fb6486f92d0d09625452d8f4dd1b89c566533cc2326b820c2b9efed43be8481cb9ad809e47af7b31795cb0fbdb18fbb12e8853f8bacec366a092daf8f2a55d2911fc7c70ddd33d33e86c2c4ceeb9390ec506b399f6fa8f35abf7789d0f547fd09cb7e6fb6016a3fc2a27a762989ae620d234c810777d5a1bb633744af2844495d2963c986ef8540ca715bed7692c77b9dec90e06acc5986b47dd4a8d3ca3300b2bedf9f26ae6d1c7e7acef05c0fc521c3309e1e70771eea6e96b67de5e3fb6833145bb73d46081b074539498307929da779e003c27f0a171035458b8c7c86c905b23dda74c040878d5a05be94821537724ebd5608ec0754c3e3e99a719bbb6d5320eed07323fca637429b18378936364c389de1e9c6fce8af270a713b4b829b43e7d761e17724c22e84611e1322dde45cbee86a0e16d01cfb8910d99391c39afd8e5f5567c59f219aa8c19ad158f287cb6807ba1fe46d38d091639a217766b3aa9ded73ac14570ba236225218305d68c0be6099c336ba8455c86b7c8c04542e729ceb84596c33ca6eb7ec1091b406cf64495ccfa2169f47b3b590477d4073537c14c05015d51ba527b3869ae4ebd603df906323658b04cb11e13bc29b34ac69f18dd49f8958f7e3f5b05ab8b8ddb34e581bde5eb49dd15698d2d2b68fe7e8baf88d8f395cfcafcdff38cf34b59386f6f77333483655ee316f12bfeb00610d8cba9e59e637ca2cab6ed24dd584143844e61fcca994ba44a4c029682997ab04285f479a6dc2c854c569073c62cd68af804aa70f4976d5b9f6b09d3738fcccb6d60e11ba97a4001062195d05a43798d5f24e9466f082ac367169f892dfd6cc0adeef82212c867a49cba65e0e636bab91e2176d3865634aa45b13c1e3e7cdb4e7872b2437f40f3de5493792c06611a9ca97d0baed71bfb4e9fdd58191198a8b371aea7f65b6e851ce22f4808377d09b6a5a9f04eddc3ff4ef9fd8bf043bb559e1df5319113cb8beea9e06b0c05c50885873acd19f6e8a109c894403a415f627cd1e8f7ca54c288c230795aaddde3a787c2a20ac6dee4913da0240d6d971f3fce31fc53087afe0c45fa3c8f744c53673bec6231abe2623029053f4be0b1557e00b291ebb212d876e88bcc81e5bd9eb820691dca5fbdcc1e7a6c58945a2cac8db2d86c2a7d98dc5908598bda78ce202ac3cd174d48ad9cac9039e27f30658eef6317cd87c199944343e7fce1b3ea7", "ad635385a289163fbaf04b5850285bfe3759774aee7fd0211d770f63985e1b44"); + TestSHA3_256("01ec0bfc6cc56e4964808e2f1e516416717dad133061e30cb6b66b1dc213103b86b3b017fa7935457631c79e801941e3e3a0e1a3016d435e69a390eaac64f3166d944c8eb8df29fe95fdf27adc34631e4a1f3ff1d5af430f3d6f5908e40c0f83df1447274dfe30bbe76b758bd9abb40ed18331c7552dcc6959a1303e11134ec904bd0aab62de33c39703b99920851afd9d531eeb28f1c4b2e6c17c55db8296320316fbe19e881b5fcb4d266c58ca7f31d9176e26f70315330b58a516ec60d10404a78393aa03ced7acd225cb2a83caf3ab5888406a69a534f1ed1346e9b5e68831f90b872d57367361191c803eb7e38b3b9cd601282d5efdbf082db07d89bd06b093f986d08d3a7b12aa74513b6eb241b26ebf31da5726d59e315d1b4ee53ec6a9fdb6583bacc136e90e9607cab01e5d3853ab9727ede706b6f10b4e04d0510f45c0abc515bcb5ed0bcce86a92861126f4d502fcb8f988d62ecf9d124853de2bab633f9506c6fde8a36cd4413cf773e50f7b2d283482f18e2f547c2fc275cd60056ed98fb8d0816fd777c1566f0c2ae3b1cd92e344910a75e006106d193e06f7786ae37dd0e529cacf74176fd4cc1f6500549af5902dbbd56a70c194f5b671372edec425f90add40b4eb3d55123f3ab62797ad25bf5eecf4f417f86b00e6f76a4f52e44fd949851aae649dd0d26d641d4c1f343c7a2c851ca7851bbbdfd57ed6024eabc518a909a1e4689ea7bc5f83e19872950368a06e93ab41944c3d8befc5705b814e5f33511a7f7ea8a4771c804b321a3a3f32c18fa127d3c9e6c011337dc100ceb156ed45d0a62f238dacac44a3429f89bb7f98d09043c42451106e30471cc6fab7a4e1ce0a8202772b0218b631f287ec3ef82b1aa6299a0b54d6aad06aa9346d28f117d20f3b7f0d462267bd3c685cca8f4584532dfee0e8b9bacefa3092d28fcce7953a28f82e4ba6b3a1430ecca58b770dab656bed1b224663e196dffc28c96a2c65ef9de1989a125ecf2fed47eb96bef8a636a91bd521c47aeb8bc011bf81cc688fd8b620446353cbf7692201b5552cb07fb02eb3954dfaa6f5c31bf91e20b84419dcbbdaba0c31a124d8f4218b2f88da3eba44dbe40eb290052538dccd0ff7670de5f33a83ff74895b66adcff58c9c21e93b31bb49ccb2e026995ee155b5517b72daa76526a2e42aa6fa94357cd42e2a8a1d3e7d4cefc33d5d07d6303d798d2551a21f862b5f492d0c7cf078a77007a02847b34675dfad4fb457e9f20dc5750fb127a3c31b9d6a3996d50ac3ffc6ef29cca1d8414d0438bf3271dc4f4e00cfe19a507b447dc310f74aeb2a3c0b3fae6d7d13f4935bc72c35df3efa6e879164421505ee32d93b030e32a7970b53430b1643855167278e5058c4a48a7840e2fcdb282e45b5b86c0b2756f19b595f3bcfc926df35e33ac26dd1e88cd394015a5f54deb4c9f4a0bef0eabcb27c4eb88dc2302f09e92f1bcc4b4754df1eeb536154543c7dbf181c9979fe6ed08311e5a3acf365ebb5745212b2630e83b3a5bd5fa4834c727248b165700c7435f8cb6ee455bad16ee0da68fe6acd2062dae9c8bc178b157b29ade98a9bbbd4c723a3dcb7852c7978b488e4f73a2c9163dbdffae175119f812b6f4b70c2b498704bc2b58603f167f277a74e64ec296a6dfdb0de3486c0f36ac1b55f80af9fc817ba4f84b898b2a3c5725e2faf466bb26a8a84f91e123d182033a7ae2029236aa4b673ceb50c1733d7edd60e3f119b7141c882d508e0331689c96fbfb9f7e888fe88561de427c721123036737c1460b0da00d3f958b948f68fcb321ab4e297290f781ff8afb06b755d82a7e6ce1963761d799eed786524bf19801b4877b2d856becdf7e87d71aa359f2d51f09de64bcbf27d0c3aceac70790e314fd06c2f5216f3d10574b7302d6bc2775b185145c1b741524567c456d42c5826f93afa20ae7196ca7224c3b69b1eada9eee752fb6d43f24170fcc02af7e1dea73f0f884f936f900165800acb9d57480a31e409d3f676ed92b6812cf182a088fc49d68082aa19c7be0711f436db1d7be44d97dc9405591a8d3e7f6f731c6f3e6c401749829b7624497f5eeac1fc782e7d6988340541f2617a317e", "2a6283b1c02c6aaf74c4155091ff54a904bb700077f96a9c4bd84e8e51b54d01"); + TestSHA3_256("9271fd111dcf260c04cf4b748f269ac80f7485c41f7724352a7ed40b2e2125b0bf30f3984ee9d21aab6eb07ec976b557c2426e131ad32bd0485aa57172f0e4f1798760f8352067ac023fbeca7b9c8bf5851c724e90ffff44195b44ae73c9c317c85e8e585bddac6d0f2abf812d02e44b62eadb9d0765683aa56af8e9b91588c7b49dc3e146866a02dc18f9ca680f88006094ef29096c2d5af5700b4aca3dfcab462c48bb8085691671efb5ceb22b3ebd8702f71a1d7c184b1053c3fa30a7e76b85f3650d9140714fd4993bb496becf2ae01d3a98ccfdefb6fefd692173bd11af7adb61ffff214a550ffcd3a5993004ee72cb02ca9c577b42c85444e619e6411e2bca86bb548ebbd12a02c5c945eaa3b246f595d817f3849875429e72ac894160a2a91a6617f18e6b2b9258472152741d62843cebc537d25f0daebdedb410d71ee761662bd1b189ca1c93d648b5d141d8d05e3f2b2d8c2c40997fea7eb7e2cc0000d8b2300936759704ef85f38ad0d08a986de6bfd75b5be3209f6d4d3f67e7adf7f8469d47e81979ec8dae7127b5eadcc09779cf4b0a28efaaf58e83d307f2ced4a8699b142f3f19db5598e914e9577652c63f851203580d40699548fc2ab30a9dcf6452f673ad1ed92f8d84dad5dfff55e18107b3acb6e4e8e3c9c34038f40a5c577fe9771c2c31ef03d36a00e04a20d2d0877db66f091dac4b741d2a997b75182702881f9284fa23b9b3c20e715f80d07b9910a4b3185f9489dc7d3fb510f4da273559753d7d207f3975b48df2e7c857caffe703dfac53a786490c09f57d2fa93f60810186df4c0b6b616a04caab9f70a5002c5e5d8da0ed2805f20fbf89cd8d57ca2b4bd37125ce38bf09fb6170ae21f4e6043a9483ef6e585756d97cfb778e57bc7ddc8dfc54d086d6bcfa1f019c749ff79921ec56e833ff8660f0959cd4b5277e9f3b1d4880193fefa98a6c2512718e7c139acdcd324303db3adb70348d09b058baf0e91d52b24952f832b0a3b81fa9bc9a2e9fb276a64e9e0922778b4992d892f6845b4372a28e47d27b53443586d9015463cacb5b65c617f84e1168b15988737a7eda8187f1f4165fecbdd032ae04916cc4b6e18a87558d2ce6a5946c65a9446f66cda139a76506c60d560f56a013b508d6ccbbaa14e24ad0729dd823bf214efcc59e6932cdc860306687c84a63efb551237223641554940a7a60fa7e6ddad64a21b4a2176b046dc480b6c5b5ff7ed96e3211df609195b4028756c22479ba278105771493870372abe24dcc407daa69878b12b845908cf2e220e7fabeeaab88c8f64f864c2bacba0c14b2a693e45aacc6b7db76bc1a2195cfce7b68f3c99440477ea4c1ea5ee78c109f4f1b553c76eb513dd6e16c383ce7f3187ad66c1d5c982724de8e16299c2fde0a8af22e8de56e50a56ac0fef1c52e76864c0ad1eeedd8907065b37892b3eca0ddcdf5c8e0917dec78fedd194ea4b380a059ccc9452e48a9eba2f8b7a4150b7ba17feac83c61604c3cfcfe6655c2be37ef0ae6fc29072f9b1cfb277b64a8d499dd079ad9aa3d5e9a7ccbec8c100596c6fac51e13a260d78d8cd9066edc558e2219cfcda1310dc1fbbdd36f348756855349f33eb6b82186a8c1a55f361305833edd3e4ac8d9b9cf99897c4e06c19ed10765fd0c8c7433851445c5f87b119ef913b2bcdbf7aa2ad19c672e53a9c6c3c309d549513edd7c1cf8a0a399e6df0939cc1fb146d6ad460e2ce05144c69eafa3822141d473fbe5927c58a50c1e842f8b8fad85540ce9f6d06f7b4dea045248b999d24c5fd4d75631caf73518cc08f73684e2a1cd4266235d90c08a0d0ce8784c776fd1b80978b83f0705ba8498744884d5496b791f2db3ffb5377175856b25a643803aa8b9e7f1055e089c1929cf0cbba7674c204c4590fb076968e918e0390d268eeef78c2aebcbf58a429f28212a2425c6ad8970b6a09cadddd8336d519bca4820556d2c4b8cd9f41216de3c728a0774edf47d3489cd29cf1b2a192bc53325d0bed7d23e51be7684297f9d0ecb14acbf648bc440c5fde997acc464fb45e965e6f0dced6d4568ebcd55e5a64633b05a2cb4d8263b721a252b1710dc84d8a5d4b43fcc875e2e7281f621b0bf8bb3465be364456bcd990b26b3e474486f864fb85f320f68bc14c37d271249b18552bef50dfc385a9f41b831589c5a716357cf5a12520d582d00452a8ab21643dd180071d2041bbc5972099141c6292009540d02f3252f1f59f8dfcf4488803f3b0df41759055559a334e68c98ea491b0984f2f82a35db84ea0779b3801cf06b463a832e", "4e75bf3c580474575c96ec7faa03feb732379f95660b77149974133644f5d2a0"); + TestSHA3_256("075997f09ab1980a3179d4da78c2e914a1ff48f34e5d3c2ab157281ef1841052d0b45a228c3cd6b5028efd2d190d76205e1fdf4cec83c9868fe504f429af1e7c5423267c48a7b5bc005f30a1980147a3fae5c100b95c7cb23d43af9f21d87311d9cc826598993e077015f59ebc476383bb7a78787d915c97039ab188a2a618f7a8d7f64542ba787e9dd7d48c4c87d2aaea068c1b00c9711b2812901673c11418096d0a850fb36b0acece56d311689dfeceb0835009adc427f6d2d6b05ed26f5a43b6478bc72c1f914a2202dbd393cb69b1a1e78162e55ca4b3030ac0298131a7a0d934c032cc9dfc5afa600c59b064d2d9013f15d1184278a8ccb5ad9d7563e666fe5a8c173cec34467ef9cf6d6671208ff714741fee7c8d1d565edf82570dffde4f3f584024142056d8548ad55df83d1babed06141114c95ac88dbea0ce35d950f16d8a732a1ea7d22dfaa75a3e0410c546523277261116a64bcbb2be83e55e040f6c8c79f911b301a8718cc4b19a81d5f0cb6312d87c5b4b079e23a61d247541cfc2c41a37f52b2c6e43a3db5dc47892d0e1feabcc5c808f2391791e45fb065159f99c1d8dd2f69baaf75267eb89dd460f1b6c0badb96cbbc8291cefa370fa7ad6997a4ca2b1fe968216032f02f29837d40215fa219c09161df074e1de8e37056e28c86d1f992a651e271dfc4b0592ad481c613fd00c3eea4b6deabb9f5aa63a4830ed49ab93624fa7b208966eccb1f293f4b9a46411f37d7928e4478dde2f608d3851a8efa68e9d45402bc5124fde4ddc0f83ef82b31019d0aacb4b5121bbc064c95c5292da97981f58f051df9502054bf728e9d4fb7e04787a0890922b30a3f66a760e3d3763855e82be017fa603630a33115a02f02386982001def905784f6ba307a598c6dbaf2946fe9e978acbaf3e4ba50ab49ae8e9582520fc2eb6790deafc77e04a8ee75da92d16f0d249403112c74bc09102b573e110ccb4d8461d249bfe2e85fc9770d606be6fbfd5ec4c30ac306d46412f736e5b696ccc9fbe4adea730955c55ea5c63678271d34b7bd6f6340e72626d290820eeb96a0d2d25ea81361a122ffe8e954cf4ff84f4dafcc5c9d3e7c2ddbdf95ed2c0862d3f2783e4566f450ec49e8b01d9d7bf11e92a7903f2b045c57ed8a65ccbfc5b1d2a38e020a57b38f2e4deea8a52354a7e7be4f977b8f5afe30f6738e955c8bda295064586b6827b245766b217fe39263572b0850965c7ae845611b8efb64c36244a39b9fed0ab970ee5ddeb8f2608dd9c963524a14050c9101d7f2d5537b24d0b0f7a45703c1e131656ec9edc12cdf71dae1cde2790b888ef2a589f03201f8bbfad71f0c4430477a6713ad2e50aaefa1f840cbb839e277389454517e0b9bd76a8ecc5c2e22b854c25ff708f9256d3700adeaec49eb2c4134638ee9bd649b4982f931ec3b23cc819fbc835ddcb3d65e04585aa005e13b7ef8fcafa36cc1a2c79ba6c26fc1dc0f6668f9432c578088cd33a41a778ac0b298fcac212edab724c9fb33d827409fd36bc4b2b0e4e81006fd050d94d3271e0427c61e9ddca599a3c9480cfdd33603cb1a196557281ce6a375fef17463893db293dba0704d4bfda25e08beadd4208c58ea0d8d9066448910b087fc13792fc44075a3fe42e13c5792f093a552aa8ebe0f63e7a807102d5bc145468a0cb469263035c5647049054c18199f7da6d6defd51105e2125c605e327aca137ca85e3f7f46ca69f92d5252f84418293f4e9afeeb067c79576e88cc3c64f3e61d76e1e9e2f72cdfc35261a9679f0c374d7436ff6cfe2ba71650810522fa554a4aded87ad23f0b206b1bc63f56bbff8bcc8849d99e209bd519a953f32c667aa8cd874ad99846ed94b92f88fe0dbf788c8431dc76ca9553692622077da2cdea666c1b3fee7c335da37737afccd3d400a23d18f5bd3784dbcd0663a38acb5a2beef03fc0a1c52ee0b56bda4493f2221e35bee59f962f16bc6781133204f032c7a6209dd3dabd6100325ec14e3ab0d05aadd03fdfe9f8737da15edab9d2598046f8c6dd8381aaf244821994d5a956073c733bcebf9edbc2a6e2676242dc4e6a2e4ba8a7d57ed509340d61fae2c82bee4dedc73b469e202cc0916250d40a1718090690a1d3b986cf593b019b7b7f79ae14843b2e7ccf0fd85218184f7844fbb35e934476841b056b3a75bf20abb6866e19a0614e6a1af0eee4de510535724363b6598cccf08a99066021653177559c57e5aaff4417670a98fe4bd41a137c384f98c0324c20ef8bc851a9b975e9440191ff08deb78c9fa6fc29c76b371a4a1fa08c30fc9d1b3323d897738495086bfd43ef24c650cfa80c42ecbadc0453c4437d1a11b467e93ca95fbae98d38dcb2da953e657fb7ea6c8493d08cf028c5d3eb0fcbcb205493f4658440719e076e02deb07332d093e4d256175ca56f4c785d5e7e26c6090a20429f70b3757daac54153bc16f5828dc6c1c9f5186e2117754be5f1b46b3631980d9e4a9a5c", "2e07737d271b9a0162eb2f4be1be54887118c462317eb6bd9f9baf1e24111848"); + TestSHA3_256("119a356f8c0790bbd5e9f3b4c5c4a70e97f462364c88cad04d5435645342b35484e94e12df61908fd95546f74859849b817ee92fbd242435c210b7b9bfbffb3f77f965faa1a9073e8feb5a380f673add8fde32208402fa680c8b3e41d187a15131f1028f9d86feaf3fd4b6e0e094d2ba0839c67267c9535173ec51645343ad74fcfaae389aa17cca3137e2588488531c36ba2b8e2f2238d8415c798a0b9a258f1e3cef605fa18977ad3d6707c3ecc5ea5f86ebdaa4b4b0e5bc023d1bc335138ae0de506cb52f2d9efa0ecc546468310cccc88ec08d28c3602e07257f41bb7e4d8a0956c564f3712761d199a931a39e69c5a69aa7b3257931dd92b91e4ed56fbf64e48bd334945cfa2aaf576df04614eb914899f7df54db4012cc8261b12bedcab69876feedbbf7009dcf8d076af89b797ad71217d75cf07514dc07ae34640055c74c9faf560f491f015ac3e167623cfbc67b8e7163e7c1b92debd06e9d28b049e0298f4c38395a40a0778162af2cfe5abe5b946c4d9a54f2a321660ab521068c4957cd3f5be0324cc04f50f209fdea7caaa0ac705c1fb30abfa550e844f509074afde1ee87adda29aa09b7f93e7d064ad2715ee5571ee6e7c9a01672124cc2a22b4354c3844759c1a6ce3fdf17555cac7df73334073ef3730939410fe6cf37463352ad241958b7fafbc66e0d592df48bf55ab2c33428e494c6995826892572d9ab52747b1085fcbef318cfe9cfacd4cd80c164fba584c1344ae7e321c4f77b44db6b322f2f7831f4d4ede7dc407b065c6754cf4424f9903adb4c6b675ded58700bc36da83fd95e84e76c404f7342921ef23d7d07772f5b8ec2f077601cae13448385b04e074f895574be61a831a87efd68a1f6aa67cf291847fa8f74cbe3b4a78ad780895183bc51c30ad2514255d4e013abc097bc8103c0b1933b0b303341836ae167c1e31dfe5f1b791cb06ef29cae398065343eecf06e4ae2048d1547c4bf69ccec5e86c45867c633c62f7d27dc51234b6debb5b9f80a5810716240c64443d0c098c80220d0520a5f5834369b9eb019325e23e88f237c24440bf27959caf7e7e4f1671fda710630255a2962f7e9b3625dc243a0177aacf6a758a68aa85dc3f56181a4a59a406c7fae5575c9e2c64248f520b5a5f904821661e2e43a5a058f445fd0e55b07476c4122d18033053b45112201e0bfdcc9e7cb9931155018ca431a0564930aca8defbca954b2680753a4060bec2cb668d2c15e77cba29589b5c7c07bc7177a8b1adb3a6968732f9213476fd96901514626fa17243af1d156cd037eea81d773f1f71a018d942b524b851794b300c7591ecd783ec8066ccb261bdf9b7a183dbda42b92593b614297dcb0fabcc23ae69797d0251b8ab57a4da2a544615216b01f4dbe2d8c9b5520c7ed9cd9312e9ec6d05a36e7f693d1821d727518169b03976394b9d1e1d7fa2daa25529d391eb5d0cf0f07a8160be2ee043d9345037c655c4f2023689f14d8d2072dd92c1dba056a5b5d4c4fc4196e25caab05b1701ec666ac9a04d90f7d7575a7ac3970252c18fd3bec0cc448e5ff8f3765d546a4a8ad1d41a9640c79375b80534b7b50989976f238654fefea981c9413130beae943a3e9d8f64ce9256d1259d1b2a6b3c02ca5af1a701db8f25a4e9c255dad8785172f323728c3585a45206ae988c283e30a2f9ea9b47f07a7521b0f36e9c504c14bd96027e8d24161e70f196576d8a74a5e9c26acda7cc452a90e550e625a49e50829db70de808c827c67d00c23ee073d4e72aeed891dd73b86acd6756e753e3975a80cdab1d521052caef6a5380f8b03023ba0326a6928aa127ffb33b51dcb05bbdd592d0ad9e8321e6ef2f95c401be6a37e634425689fe7750e2a0fe05ad89001502b309095ca517b2e2ed0388b9f2c59c45feb61222539d6e1ccd397344d23708aebacec10ada96a7711f173f7ff1e4b94fceec6a0a0ea5d814a4581b412063012ff6ac5527b8314d00326b68c2304a276a217fde9fa4034750a2e47e10f816870d12fc4641a27a1c16c35a953f32685f2b92cae0519848045765591c42ddc402dc7c6914d74dd38d2b5e7f35358cb1d91a9f681fde7fd6c7af5840663525ee1d04bf6d3156fed018c44043d95383d92dada3d1cd84af51d9bee814ec8675073e1c48632c5f59e257682542e1f7dc20b56b5c92a9e2cb2be30cb1512fb55fa1de99a3f5864ed3acc19d79e6ffb0da3b08ba0615157747d75c1f308fa0202a4086f34e9eafa3e071dfbacaca731d228aa0304cf390c0a9e6ad7ce22ade758965cfbfc4c9390d24e41a667447fc7b29821464ad98bc5d65dc7f9c42bd4b23e174015592ff92c905660a2722f9fc7973d3cdad848ef88bf02b1b03dea16699b71dc46b35bc4d96069a0753335ae38685d244918e30c5fb0d45283a1281c1659ea591573999d9c2acd2ca9141d55230d41011b70748b518e1cd2fa58ad8dc05fcbdf0bffaf2c7fd6cb2ac67bb13b8f6d31fad64ac113664223599dca411270955c95aec06518894dabc352d2b70984727437040d944da7b42e0ef560ac532de3e4a4891e8509c275b51ed780f8660b0354e12c21b3e11bcc88198980b5f7ff31ad342182d5a933373164dced3cfb2a081720d7eee676cb7378a3e19326a7ee67fd6c00521f9de37c66bcea814b6feb6a061b8cdcf7b4bd8f45d48602c5", "c26d0b064e409df64819cd7c1a3b8076f19815b9823adac4e3ce0b4d3a29de18"); + TestSHA3_256("72c57c359e10684d0517e46653a02d18d29eff803eb009e4d5eb9e95add9ad1a4ac1f38a70296f3a369a16985ca3c957de2084cdc9bdd8994eb59b8815e0debad4ec1f001feac089820db8becdaf896aaf95721e8674e5d476b43bd2b873a7d135cd685f545b438210f9319e4dcd55986c85303c1ddf18dc746fe63a409df0a998ed376eb683e16c09e6e9018504152b3e7628ef350659fb716e058a5263a18823d2f2f6ee6a8091945a48ae1c5cb1694cf2c1fe76ef9177953afe8899cfa2b7fe0603bfa3180937dadfb66fbbdd119bbf8063338aa4a699075a3bfdbae8db7e5211d0917e9665a702fc9b0a0a901d08bea97654162d82a9f05622b060b634244779c33427eb7a29353a5f48b07cbefa72f3622ac5900bef77b71d6b314296f304c8426f451f32049b1f6af156a9dab702e8907d3cd72bb2c50493f4d593e731b285b70c803b74825b3524cda3205a8897106615260ac93c01c5ec14f5b11127783989d1824527e99e04f6a340e827b559f24db9292fcdd354838f9339a5fa1d7f6b2087f04835828b13463dd40927866f16ae33ed501ec0e6c4e63948768c5aeea3e4f6754985954bea7d61088c44430204ef491b74a64bde1358cecb2cad28ee6a3de5b752ff6a051104d88478653339457ac45ba44cbb65f54d1969d047cda746931d5e6a8b48e211416aefd5729f3d60b56b54e7f85aa2f42de3cb69419240c24e67139a11790a709edef2ac52cf35dd0a08af45926ebe9761f498ff83bfe263d6897ee97943a4b982fe3404ef0b4a45e06113c60340e0664f14799bf59cb4b3934b465fabefd87155905ee5309ba41e9e402973311831ea600b16437f71df39ee77130490c4d0227e5d1757fdc66af3ae6b9953053ed9aafca0160209858a7d4dd38fe10e0cb153672d08633ed6c54977aa0a6e67f9ff2f8c9d22dd7b21de08192960fd0e0da68d77c8d810db11dcaa61c725cd4092cbff76c8e1debd8d0361bb3f2e607911d45716f53067bdc0d89dd4889177765166a424e9fc0cb711201099dda213355e6639ac7eb86eca2ae0ab38b7f674f37ef8a6fcca1a6f52f55d9e1dcd631d2c3c82bba129172feb991d5af51afecd9d61a88b6832e4107480e392aed61a8644f551665ebff6b20953b635737a4f895e429fddcfe801f606fbda74b3bf6f5767d0fac14907fcfd0aa1d4c11b9e91b01d68052399b51a29f1ae6acd965109977c14a555cbcbd21ad8cb9f8853506d4bc21c01e62d61d7b21be1b923be54914e6b0a7ca84dd11f1159193e1184568a6134a6bbadf5b4df986edcf2019390ae841cfaa44435e28ce877d3dae4177992fa5d4e5c005876dbe3d1e63bec7dcc0942762b48b1ecc6c1a918409a8a72812a1e245c0c67be6e729c2b49bc6ee4d24a8f63e78e75db45655c26a9a78aff36fcd67117f26b8f654dca664b9f0e30681874cb749e1a692720078856286c2560b0292cc837933423147569350955c9571bf8941ba128fd339cb4268f46b94bc6ee203eb7026813706ea51c4f24c91866fc23a724bf2501327e6ae89c29f8db315dc28d2c7c719514036367e018f4835f63fdecd71f9bdced7132b6c4f8b13c69a517026fcd3622d67cb632320d5e7308f78f4b7cea11f6291b137851dc6cd6366f2785c71c3f237f81a7658b2a8d512b61e0ad5a4710b7b124151689fcb2116063fbff7e9115fed7b93de834970b838e49f8f8ba5f1f874c354078b5810a55ae289a56da563f1da6cd80a3757d6073fa55e016e45ac6cec1f69d871c92fd0ae9670c74249045e6b464787f9504128736309fed205f8df4d90e332908581298d9c75a3fa36ab0c3c9272e62de53ab290c803d67b696fd615c260a47bffad16746f18ba1a10a061bacbea9369693b3c042eec36bed289d7d12e52bca8aa1c2dff88ca7816498d25626d0f1e106ebb0b4a12138e00f3df5b1c2f49d98b1756e69b641b7c6353d99dbff050f4d76842c6cf1c2a4b062fc8e6336fa689b7c9d5c6b4ab8c15a5c20e514ff070a602d85ae52fa7810c22f8eeffd34a095b93342144f7a98d024216b3d68ed7bea047517bfcd83ec83febd1ba0e5858e2bdc1d8b1f7b0f89e90ccc432a3f930cb8209462e64556c5054c56ca2a85f16b32eb83a10459d13516faa4d23302b7607b9bd38dab2239ac9e9440c314433fdfb3ceadab4b4f87415ed6f240e017221f3b5f7ac196cdf54957bec42fe6893994b46de3d27dc7fb58ca88feb5b9e79cf20053d12530ac524337b22a3629bea52f40b06d3e2128f32060f9105847daed81d35f20e2002817434659baff64494c5b5c7f9216bfda38412a0f70511159dc73bb6bae1f8eaa0ef08d99bcb31f94f6be12c29c83df45926430b366c99fca3270c15fc4056398fdf3135b7779e3066a006961d1ac0ad1c83179ce39e87a96b722ec23aabc065badf3e188347a360772ca6a447abac7e6a44f0d4632d52926332e44a0a86bff5ce699fd063bdda3ffd4c41b53ded49fecec67f40599b934e16e3fd1bc063ad7026f8d71bfd4cbaf56599586774723194b692036f1b6bb242e2ffb9c600b5215b412764599476ce475c9e5b396fbcebd6be323dcf4d0048077400aac7500db41dc95fc7f7edbe7c9c2ec5ea89943fe13b42217eef530bbd023671509e12dfce4e1c1c82955d965e6a68aa66f6967dba48feda572db1f099d9a6dc4bc8edade852b5e824a06890dc48a6a6510ecaf8cf7620d757290e3166d431abecc624fa9ac2234d2eb783308ead45544910c633a94964b2ef5fbc409cb8835ac4147d384e12e0a5e13951f7de0ee13eafcb0ca0c04946d7804040c0a3cd088352424b097adb7aad1ca4495952f3e6c0158c02d2bcec33bfda69301434a84d9027ce02c0b9725dad118", "d894b86261436362e64241e61f6b3e6589daf64dc641f60570c4c0bf3b1f2ca3"); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index b1a635d9d..e4ee08db6 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -47,7 +47,6 @@ struct CConnmanTest : public CConnman { extern bool AddOrphanTx(const CTransactionRef& tx, NodeId peer); extern void EraseOrphansFor(NodeId peer); extern unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans); -extern void Misbehaving(NodeId nodeid, int howmuch, const std::string& message=""); struct COrphanTx { CTransactionRef tx; @@ -79,12 +78,13 @@ BOOST_FIXTURE_TEST_SUITE(denialofservice_tests, TestingSetup) // work. BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) { + const CChainParams& chainparams = Params(); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), nullptr, *m_node.scheduler, *m_node.chainman, *m_node.mempool); + auto peerLogic = MakeUnique<PeerManager>(chainparams, *connman, nullptr, *m_node.scheduler, *m_node.chainman, *m_node.mempool); // Mock an outbound peer CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, ServiceFlags(NODE_NETWORK|NODE_WITNESS), 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", /*fInboundIn=*/ false); + CNode dummyNode1(id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", ConnectionType::OUTBOUND_FULL_RELAY); dummyNode1.SetSendVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode1); @@ -133,10 +133,10 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); } -static void AddRandomOutboundPeer(std::vector<CNode *> &vNodes, PeerLogicValidation &peerLogic, CConnmanTest* connman) +static void AddRandomOutboundPeer(std::vector<CNode *> &vNodes, PeerManager &peerLogic, CConnmanTest* connman) { CAddress addr(ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE); - vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK|NODE_WITNESS), 0, INVALID_SOCKET, addr, 0, 0, CAddress(), "", /*fInboundIn=*/ false)); + vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), 0, INVALID_SOCKET, addr, 0, 0, CAddress(), "", ConnectionType::OUTBOUND_FULL_RELAY)); CNode &node = *vNodes.back(); node.SetSendVersion(PROTOCOL_VERSION); @@ -149,10 +149,10 @@ static void AddRandomOutboundPeer(std::vector<CNode *> &vNodes, PeerLogicValidat BOOST_AUTO_TEST_CASE(stale_tip_peer_management) { + const CChainParams& chainparams = Params(); auto connman = MakeUnique<CConnmanTest>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), nullptr, *m_node.scheduler, *m_node.chainman, *m_node.mempool); + auto peerLogic = MakeUnique<PeerManager>(chainparams, *connman, nullptr, *m_node.scheduler, *m_node.chainman, *m_node.mempool); - const Consensus::Params& consensusParams = Params().GetConsensus(); constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; CConnman::Options options; options.nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; @@ -167,18 +167,18 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) AddRandomOutboundPeer(vNodes, *peerLogic, connman.get()); } - peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + peerLogic->CheckForStaleTipAndEvictPeers(); // No nodes should be marked for disconnection while we have no extra peers for (const CNode *node : vNodes) { BOOST_CHECK(node->fDisconnect == false); } - SetMockTime(GetTime() + 3*consensusParams.nPowTargetSpacing + 1); + SetMockTime(GetTime() + 3 * chainparams.GetConsensus().nPowTargetSpacing + 1); // Now tip should definitely be stale, and we should look for an extra // outbound peer - peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + peerLogic->CheckForStaleTipAndEvictPeers(); BOOST_CHECK(connman->GetTryNewOutboundPeer()); // Still no peers should be marked for disconnection @@ -191,8 +191,8 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) // required time connected check should be satisfied). AddRandomOutboundPeer(vNodes, *peerLogic, connman.get()); - peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); - for (int i=0; i<max_outbound_full_relay; ++i) { + peerLogic->CheckForStaleTipAndEvictPeers(); + for (int i = 0; i < max_outbound_full_relay; ++i) { BOOST_CHECK(vNodes[i]->fDisconnect == false); } // Last added node should get marked for eviction @@ -204,8 +204,8 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) // peer, and check that the next newest node gets evicted. UpdateLastBlockAnnounceTime(vNodes.back()->GetId(), GetTime()); - peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); - for (int i=0; i<max_outbound_full_relay-1; ++i) { + peerLogic->CheckForStaleTipAndEvictPeers(); + for (int i = 0; i < max_outbound_full_relay - 1; ++i) { BOOST_CHECK(vNodes[i]->fDisconnect == false); } BOOST_CHECK(vNodes[max_outbound_full_relay-1]->fDisconnect == true); @@ -221,21 +221,19 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) BOOST_AUTO_TEST_CASE(peer_discouragement) { + const CChainParams& chainparams = Params(); auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); + auto peerLogic = MakeUnique<PeerManager>(chainparams, *connman, banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); banman->ClearBanned(); CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", true); + CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", ConnectionType::INBOUND); dummyNode1.SetSendVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode1); dummyNode1.nVersion = 1; dummyNode1.fSuccessfullyConnected = true; - { - LOCK(cs_main); - Misbehaving(dummyNode1.GetId(), DISCOURAGEMENT_THRESHOLD); // Should be discouraged - } + peerLogic->Misbehaving(dummyNode1.GetId(), DISCOURAGEMENT_THRESHOLD, /* message */ ""); // Should be discouraged { LOCK(dummyNode1.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); @@ -244,25 +242,19 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) BOOST_CHECK(!banman->IsDiscouraged(ip(0xa0b0c001|0x0000ff00))); // Different IP, not discouraged CAddress addr2(ip(0xa0b0c002), NODE_NONE); - CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, CAddress(), "", true); + CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, CAddress(), "", ConnectionType::INBOUND); dummyNode2.SetSendVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode2); dummyNode2.nVersion = 1; dummyNode2.fSuccessfullyConnected = true; - { - LOCK(cs_main); - Misbehaving(dummyNode2.GetId(), DISCOURAGEMENT_THRESHOLD - 1); - } + peerLogic->Misbehaving(dummyNode2.GetId(), DISCOURAGEMENT_THRESHOLD - 1, /* message */ ""); { LOCK(dummyNode2.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode2)); } BOOST_CHECK(!banman->IsDiscouraged(addr2)); // 2 not discouraged yet... BOOST_CHECK(banman->IsDiscouraged(addr1)); // ... but 1 still should be - { - LOCK(cs_main); - Misbehaving(dummyNode2.GetId(), 1); // 2 reaches discouragement threshold - } + peerLogic->Misbehaving(dummyNode2.GetId(), 1, /* message */ ""); // 2 reaches discouragement threshold { LOCK(dummyNode2.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode2)); @@ -277,25 +269,23 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) BOOST_AUTO_TEST_CASE(DoS_bantime) { + const CChainParams& chainparams = Params(); auto banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = MakeUnique<CConnman>(0x1337, 0x1337); - auto peerLogic = MakeUnique<PeerLogicValidation>(connman.get(), banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); + auto peerLogic = MakeUnique<PeerManager>(chainparams, *connman, banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); banman->ClearBanned(); int64_t nStartTime = GetTime(); SetMockTime(nStartTime); // Overrides future calls to GetTime() CAddress addr(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, 4, CAddress(), "", true); + CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, 4, CAddress(), "", ConnectionType::INBOUND); dummyNode.SetSendVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode); dummyNode.nVersion = 1; dummyNode.fSuccessfullyConnected = true; - { - LOCK(cs_main); - Misbehaving(dummyNode.GetId(), DISCOURAGEMENT_THRESHOLD); - } + peerLogic->Misbehaving(dummyNode.GetId(), DISCOURAGEMENT_THRESHOLD, /* message */ ""); { LOCK(dummyNode.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode)); diff --git a/src/test/fuzz/asmap.cpp b/src/test/fuzz/asmap.cpp index 40ca01bd9..e3aefa18a 100644 --- a/src/test/fuzz/asmap.cpp +++ b/src/test/fuzz/asmap.cpp @@ -33,7 +33,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) if (buffer.size() < 1 + 3 + 4) return; int asmap_size = 3 + (buffer[0] & 127); bool ipv6 = buffer[0] & 128; - int addr_size = ipv6 ? 16 : 4; + const size_t addr_size = ipv6 ? ADDR_IPV6_SIZE : ADDR_IPV4_SIZE; if (buffer.size() < size_t(1 + asmap_size + addr_size)) return; std::vector<bool> asmap = ipv6 ? IPV6_PREFIX_ASMAP : IPV4_PREFIX_ASMAP; asmap.reserve(asmap.size() + 8 * asmap_size); @@ -43,7 +43,17 @@ void test_one_input(const std::vector<uint8_t>& buffer) } } if (!SanityCheckASMap(asmap)) return; + + const uint8_t* addr_data = buffer.data() + 1 + asmap_size; CNetAddr net_addr; - net_addr.SetRaw(ipv6 ? NET_IPV6 : NET_IPV4, buffer.data() + 1 + asmap_size); + if (ipv6) { + assert(addr_size == ADDR_IPV6_SIZE); + net_addr.SetLegacyIPv6(Span<const uint8_t>(addr_data, addr_size)); + } else { + assert(addr_size == ADDR_IPV4_SIZE); + in_addr ipv4; + memcpy(&ipv4, addr_data, addr_size); + net_addr.SetIP(CNetAddr{ipv4}); + } (void)net_addr.GetMappedAS(asmap); } diff --git a/src/test/fuzz/crypto.cpp b/src/test/fuzz/crypto.cpp index 3edcf9649..664e65acc 100644 --- a/src/test/fuzz/crypto.cpp +++ b/src/test/fuzz/crypto.cpp @@ -7,6 +7,7 @@ #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> +#include <crypto/sha3.h> #include <crypto/sha512.h> #include <hash.h> #include <test/fuzz/FuzzedDataProvider.h> @@ -32,6 +33,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) CSHA1 sha1; CSHA256 sha256; CSHA512 sha512; + SHA3_256 sha3; CSipHasher sip_hasher{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>()}; while (fuzzed_data_provider.ConsumeBool()) { @@ -51,6 +53,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)ripemd160.Write(data.data(), data.size()); (void)sha1.Write(data.data(), data.size()); (void)sha256.Write(data.data(), data.size()); + (void)sha3.Write(data); (void)sha512.Write(data.data(), data.size()); (void)sip_hasher.Write(data.data(), data.size()); @@ -65,11 +68,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)ripemd160.Reset(); (void)sha1.Reset(); (void)sha256.Reset(); + (void)sha3.Reset(); (void)sha512.Reset(); break; } case 2: { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 8)) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 9)) { case 0: { data.resize(CHash160::OUTPUT_SIZE); hash160.Finalize(data); @@ -115,9 +119,21 @@ void test_one_input(const std::vector<uint8_t>& buffer) data[0] = sip_hasher.Finalize() % 256; break; } + case 9: { + data.resize(SHA3_256::OUTPUT_SIZE); + sha3.Finalize(data); + break; + } } break; } } } + if (fuzzed_data_provider.ConsumeBool()) { + uint64_t state[25]; + for (size_t i = 0; i < 25; ++i) { + state[i] = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); + } + KeccakF(state); + } } diff --git a/src/test/fuzz/locale.cpp b/src/test/fuzz/locale.cpp index 3597f51e5..2b181c6da 100644 --- a/src/test/fuzz/locale.cpp +++ b/src/test/fuzz/locale.cpp @@ -52,7 +52,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) const bool parseint64_without_locale = ParseInt64(random_string, &parseint64_out_without_locale); const int64_t atoi64_without_locale = atoi64(random_string); const int atoi_without_locale = atoi(random_string); - const int64_t atoi64c_without_locale = atoi64(random_string.c_str()); const int64_t random_int64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); const std::string tostring_without_locale = ToString(random_int64); // The variable `random_int32` is no longer used, but the harness still needs to @@ -80,8 +79,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) } const int64_t atoi64_with_locale = atoi64(random_string); assert(atoi64_without_locale == atoi64_with_locale); - const int64_t atoi64c_with_locale = atoi64(random_string.c_str()); - assert(atoi64c_without_locale == atoi64c_with_locale); const int atoi_with_locale = atoi(random_string); assert(atoi_without_locale == atoi_with_locale); const std::string tostring_with_locale = ToString(random_int64); diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp new file mode 100644 index 000000000..cd0c93b8d --- /dev/null +++ b/src/test/fuzz/net.cpp @@ -0,0 +1,156 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <chainparamsbase.h> +#include <net.h> +#include <net_permissions.h> +#include <netaddress.h> +#include <optional.h> +#include <protocol.h> +#include <random.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> + +#include <cstdint> +#include <string> +#include <vector> + +void initialize() +{ + static const BasicTestingSetup basic_testing_setup; +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + const std::optional<CAddress> address = ConsumeDeserializable<CAddress>(fuzzed_data_provider); + if (!address) { + return; + } + const std::optional<CAddress> address_bind = ConsumeDeserializable<CAddress>(fuzzed_data_provider); + if (!address_bind) { + return; + } + + CNode node{fuzzed_data_provider.ConsumeIntegral<NodeId>(), + static_cast<ServiceFlags>(fuzzed_data_provider.ConsumeIntegral<uint64_t>()), + fuzzed_data_provider.ConsumeIntegral<int>(), + INVALID_SOCKET, + *address, + fuzzed_data_provider.ConsumeIntegral<uint64_t>(), + fuzzed_data_provider.ConsumeIntegral<uint64_t>(), + *address_bind, + fuzzed_data_provider.ConsumeRandomLengthString(32), + fuzzed_data_provider.PickValueInArray({ConnectionType::INBOUND, ConnectionType::OUTBOUND_FULL_RELAY, ConnectionType::MANUAL, ConnectionType::FEELER, ConnectionType::BLOCK_RELAY, ConnectionType::ADDR_FETCH})}; + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 12)) { + case 0: { + node.CloseSocketDisconnect(); + break; + } + case 1: { + node.MaybeSetAddrName(fuzzed_data_provider.ConsumeRandomLengthString(32)); + break; + } + case 2: { + node.SetSendVersion(fuzzed_data_provider.ConsumeIntegral<int>()); + break; + } + case 3: { + const std::vector<bool> asmap = ConsumeRandomLengthIntegralVector<bool>(fuzzed_data_provider, 128); + if (!SanityCheckASMap(asmap)) { + break; + } + CNodeStats stats; + node.copyStats(stats, asmap); + break; + } + case 4: { + node.SetRecvVersion(fuzzed_data_provider.ConsumeIntegral<int>()); + break; + } + case 5: { + const CNode* add_ref_node = node.AddRef(); + assert(add_ref_node == &node); + break; + } + case 6: { + if (node.GetRefCount() > 0) { + node.Release(); + } + break; + } + case 7: { + if (node.m_addr_known == nullptr) { + break; + } + const std::optional<CAddress> addr_opt = ConsumeDeserializable<CAddress>(fuzzed_data_provider); + if (!addr_opt) { + break; + } + node.AddAddressKnown(*addr_opt); + break; + } + case 8: { + if (node.m_addr_known == nullptr) { + break; + } + const std::optional<CAddress> addr_opt = ConsumeDeserializable<CAddress>(fuzzed_data_provider); + if (!addr_opt) { + break; + } + FastRandomContext fast_random_context{ConsumeUInt256(fuzzed_data_provider)}; + node.PushAddress(*addr_opt, fast_random_context); + break; + } + case 9: { + const std::optional<CInv> inv_opt = ConsumeDeserializable<CInv>(fuzzed_data_provider); + if (!inv_opt) { + break; + } + node.AddKnownTx(inv_opt->hash); + break; + } + case 10: { + node.PushTxInventory(ConsumeUInt256(fuzzed_data_provider)); + break; + } + case 11: { + const std::optional<CService> service_opt = ConsumeDeserializable<CService>(fuzzed_data_provider); + if (!service_opt) { + break; + } + node.SetAddrLocal(*service_opt); + break; + } + case 12: { + const std::vector<uint8_t> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); + bool complete; + node.ReceiveMsgBytes((const char*)b.data(), b.size(), complete); + break; + } + } + } + + (void)node.GetAddrLocal(); + (void)node.GetAddrName(); + (void)node.GetId(); + (void)node.GetLocalNonce(); + (void)node.GetLocalServices(); + (void)node.GetMyStartingHeight(); + (void)node.GetRecvVersion(); + const int ref_count = node.GetRefCount(); + assert(ref_count >= 0); + (void)node.GetSendVersion(); + (void)node.RelayAddrsWithConn(); + + const NetPermissionFlags net_permission_flags = fuzzed_data_provider.ConsumeBool() ? + fuzzed_data_provider.PickValueInArray<NetPermissionFlags>({NetPermissionFlags::PF_NONE, NetPermissionFlags::PF_BLOOMFILTER, NetPermissionFlags::PF_RELAY, NetPermissionFlags::PF_FORCERELAY, NetPermissionFlags::PF_NOBAN, NetPermissionFlags::PF_MEMPOOL, NetPermissionFlags::PF_ISIMPLICIT, NetPermissionFlags::PF_ALL}) : + static_cast<NetPermissionFlags>(fuzzed_data_provider.ConsumeIntegral<uint32_t>()); + (void)node.HasPermission(net_permission_flags); +} diff --git a/src/test/fuzz/netaddress.cpp b/src/test/fuzz/netaddress.cpp index 2901c704f..8252f3872 100644 --- a/src/test/fuzz/netaddress.cpp +++ b/src/test/fuzz/netaddress.cpp @@ -17,9 +17,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const CNetAddr net_addr = ConsumeNetAddr(fuzzed_data_provider); - for (int i = 0; i < 15; ++i) { - (void)net_addr.GetByte(i); - } (void)net_addr.GetHash(); (void)net_addr.GetNetClass(); if (net_addr.GetNetwork() == Network::NET_IPV4) { @@ -78,7 +75,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)net_addr.ToString(); (void)net_addr.ToStringIP(); - const CSubNet sub_net{net_addr, fuzzed_data_provider.ConsumeIntegral<int32_t>()}; + const CSubNet sub_net{net_addr, fuzzed_data_provider.ConsumeIntegral<uint8_t>()}; (void)sub_net.IsValid(); (void)sub_net.ToString(); diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 9e40d5cd5..3d6947ca9 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -30,18 +30,6 @@ #include <string> #include <vector> -void ProcessMessage( - CNode& pfrom, - const std::string& msg_type, - CDataStream& vRecv, - const std::chrono::microseconds time_received, - const CChainParams& chainparams, - ChainstateManager& chainman, - CTxMemPool& mempool, - CConnman& connman, - BanMan* banman, - const std::atomic<bool>& interruptMsgProc); - namespace { #ifdef MESSAGE_TYPE @@ -80,17 +68,15 @@ void test_one_input(const std::vector<uint8_t>& buffer) return; } CDataStream random_bytes_data_stream{fuzzed_data_provider.ConsumeRemainingBytes<unsigned char>(), SER_NETWORK, PROTOCOL_VERSION}; - CNode& p2p_node = *MakeUnique<CNode>(0, ServiceFlags(NODE_NETWORK | NODE_WITNESS | NODE_BLOOM), 0, INVALID_SOCKET, CAddress{CService{in_addr{0x0100007f}, 7777}, NODE_NETWORK}, 0, 0, CAddress{}, std::string{}, false).release(); + CNode& p2p_node = *MakeUnique<CNode>(0, ServiceFlags(NODE_NETWORK | NODE_WITNESS | NODE_BLOOM), 0, INVALID_SOCKET, CAddress{CService{in_addr{0x0100007f}, 7777}, NODE_NETWORK}, 0, 0, CAddress{}, std::string{}, ConnectionType::OUTBOUND_FULL_RELAY).release(); p2p_node.fSuccessfullyConnected = true; p2p_node.nVersion = PROTOCOL_VERSION; p2p_node.SetSendVersion(PROTOCOL_VERSION); connman.AddTestNode(p2p_node); - g_setup->m_node.peer_logic->InitializeNode(&p2p_node); + g_setup->m_node.peerman->InitializeNode(&p2p_node); try { - ProcessMessage(p2p_node, random_message_type, random_bytes_data_stream, GetTime<std::chrono::microseconds>(), - Params(), *g_setup->m_node.chainman, *g_setup->m_node.mempool, - *g_setup->m_node.connman, g_setup->m_node.banman.get(), - std::atomic<bool>{false}); + g_setup->m_node.peerman->ProcessMessage(p2p_node, random_message_type, random_bytes_data_stream, + GetTime<std::chrono::microseconds>(), std::atomic<bool>{false}); } catch (const std::ios_base::failure&) { } SyncWithValidationInterfaceQueue(); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 91ebf9fb1..c9433d325 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -44,16 +44,15 @@ void test_one_input(const std::vector<uint8_t>& buffer) const auto num_peers_to_add = fuzzed_data_provider.ConsumeIntegralInRange(1, 3); for (int i = 0; i < num_peers_to_add; ++i) { const ServiceFlags service_flags = ServiceFlags(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); - const bool inbound{fuzzed_data_provider.ConsumeBool()}; - const bool block_relay_only{fuzzed_data_provider.ConsumeBool()}; - peers.push_back(MakeUnique<CNode>(i, service_flags, 0, INVALID_SOCKET, CAddress{CService{in_addr{0x0100007f}, 7777}, NODE_NETWORK}, 0, 0, CAddress{}, std::string{}, inbound, block_relay_only).release()); + const ConnectionType conn_type = fuzzed_data_provider.PickValueInArray({ConnectionType::INBOUND, ConnectionType::OUTBOUND_FULL_RELAY, ConnectionType::MANUAL, ConnectionType::FEELER, ConnectionType::BLOCK_RELAY, ConnectionType::ADDR_FETCH}); + peers.push_back(MakeUnique<CNode>(i, service_flags, 0, INVALID_SOCKET, CAddress{CService{in_addr{0x0100007f}, 7777}, NODE_NETWORK}, 0, 0, CAddress{}, std::string{}, conn_type).release()); CNode& p2p_node = *peers.back(); p2p_node.fSuccessfullyConnected = true; p2p_node.fPauseSend = false; p2p_node.nVersion = PROTOCOL_VERSION; p2p_node.SetSendVersion(PROTOCOL_VERSION); - g_setup->m_node.peer_logic->InitializeNode(&p2p_node); + g_setup->m_node.peerman->InitializeNode(&p2p_node); connman.AddTestNode(p2p_node); } diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index 85aac6ac7..4274fa435 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -63,8 +63,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) int required_ret; (void)ExtractDestinations(script, type_ret, addresses, required_ret); - (void)GetScriptForWitness(script); - const FlatSigningProvider signing_provider; (void)InferDescriptor(script, signing_provider); diff --git a/src/test/fuzz/secp256k1_ec_seckey_import_export_der.cpp b/src/test/fuzz/secp256k1_ec_seckey_import_export_der.cpp new file mode 100644 index 000000000..d4f302a8d --- /dev/null +++ b/src/test/fuzz/secp256k1_ec_seckey_import_export_der.cpp @@ -0,0 +1,38 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <key.h> +#include <secp256k1.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <vector> + +int ec_seckey_import_der(const secp256k1_context* ctx, unsigned char* out32, const unsigned char* seckey, size_t seckeylen); +int ec_seckey_export_der(const secp256k1_context* ctx, unsigned char* seckey, size_t* seckeylen, const unsigned char* key32, bool compressed); + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + secp256k1_context* secp256k1_context_sign = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + { + std::vector<uint8_t> out32(32); + (void)ec_seckey_import_der(secp256k1_context_sign, out32.data(), ConsumeFixedLengthByteVector(fuzzed_data_provider, CKey::SIZE).data(), CKey::SIZE); + } + { + std::vector<uint8_t> seckey(CKey::SIZE); + const std::vector<uint8_t> key32 = ConsumeFixedLengthByteVector(fuzzed_data_provider, 32); + size_t seckeylen = CKey::SIZE; + const bool compressed = fuzzed_data_provider.ConsumeBool(); + const bool exported = ec_seckey_export_der(secp256k1_context_sign, seckey.data(), &seckeylen, key32.data(), compressed); + if (exported) { + std::vector<uint8_t> out32(32); + const bool imported = ec_seckey_import_der(secp256k1_context_sign, out32.data(), seckey.data(), seckey.size()) == 1; + assert(imported && key32 == out32); + } + } + secp256k1_context_destroy(secp256k1_context_sign); +} diff --git a/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp b/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp new file mode 100644 index 000000000..ed8c7aba8 --- /dev/null +++ b/src/test/fuzz/secp256k1_ecdsa_signature_parse_der_lax.cpp @@ -0,0 +1,33 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <key.h> +#include <secp256k1.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <vector> + +bool SigHasLowR(const secp256k1_ecdsa_signature* sig); +int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1_ecdsa_signature* sig, const unsigned char* input, size_t inputlen); + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + const std::vector<uint8_t> signature_bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider); + if (signature_bytes.data() == nullptr) { + return; + } + secp256k1_context* secp256k1_context_verify = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); + secp256k1_ecdsa_signature sig_der_lax; + const bool parsed_der_lax = ecdsa_signature_parse_der_lax(secp256k1_context_verify, &sig_der_lax, signature_bytes.data(), signature_bytes.size()) == 1; + if (parsed_der_lax) { + ECC_Start(); + (void)SigHasLowR(&sig_der_lax); + ECC_Stop(); + } + secp256k1_context_destroy(secp256k1_context_verify); +} diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 9f9552edb..ed6093a8a 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -257,7 +257,7 @@ CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept CSubNet ConsumeSubNet(FuzzedDataProvider& fuzzed_data_provider) noexcept { - return {ConsumeNetAddr(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<int32_t>()}; + return {ConsumeNetAddr(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<uint8_t>()}; } void InitializeFuzzingContext(const std::string& chain_name = CBaseChainParams::REGTEST) diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index ab42be21b..85ebc8967 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -13,8 +13,10 @@ #include <streams.h> #include <test/util/setup_common.h> #include <util/memory.h> +#include <util/strencodings.h> #include <util/string.h> #include <util/system.h> +#include <version.h> #include <boost/test/unit_test.hpp> @@ -180,17 +182,94 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) CAddress addr = CAddress(CService(ipv4Addr, 7777), NODE_NETWORK); std::string pszDest; - bool fInboundIn = false; - // Test that fFeeler is false by default. - std::unique_ptr<CNode> pnode1 = MakeUnique<CNode>(id++, NODE_NETWORK, height, hSocket, addr, 0, 0, CAddress(), pszDest, fInboundIn); - BOOST_CHECK(pnode1->fInbound == false); - BOOST_CHECK(pnode1->fFeeler == false); + std::unique_ptr<CNode> pnode1 = MakeUnique<CNode>(id++, NODE_NETWORK, height, hSocket, addr, 0, 0, CAddress(), pszDest, ConnectionType::OUTBOUND_FULL_RELAY); + BOOST_CHECK(pnode1->IsFullOutboundConn() == true); + BOOST_CHECK(pnode1->IsManualConn() == false); + BOOST_CHECK(pnode1->IsBlockOnlyConn() == false); + BOOST_CHECK(pnode1->IsFeelerConn() == false); + BOOST_CHECK(pnode1->IsAddrFetchConn() == false); + BOOST_CHECK(pnode1->IsInboundConn() == false); + + std::unique_ptr<CNode> pnode2 = MakeUnique<CNode>(id++, NODE_NETWORK, height, hSocket, addr, 1, 1, CAddress(), pszDest, ConnectionType::INBOUND); + BOOST_CHECK(pnode2->IsFullOutboundConn() == false); + BOOST_CHECK(pnode2->IsManualConn() == false); + BOOST_CHECK(pnode2->IsBlockOnlyConn() == false); + BOOST_CHECK(pnode2->IsFeelerConn() == false); + BOOST_CHECK(pnode2->IsAddrFetchConn() == false); + BOOST_CHECK(pnode2->IsInboundConn() == true); +} + +BOOST_AUTO_TEST_CASE(cnetaddr_basic) +{ + CNetAddr addr; + + // IPv4, INADDR_ANY + BOOST_REQUIRE(LookupHost("0.0.0.0", addr, false)); + BOOST_REQUIRE(!addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv4()); + + BOOST_CHECK(addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "0.0.0.0"); + + // IPv4, INADDR_NONE + BOOST_REQUIRE(LookupHost("255.255.255.255", addr, false)); + BOOST_REQUIRE(!addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv4()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "255.255.255.255"); + + // IPv4, casual + BOOST_REQUIRE(LookupHost("12.34.56.78", addr, false)); + BOOST_REQUIRE(addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv4()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "12.34.56.78"); + + // IPv6, in6addr_any + BOOST_REQUIRE(LookupHost("::", addr, false)); + BOOST_REQUIRE(!addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv6()); + + BOOST_CHECK(addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "::"); + + // IPv6, casual + BOOST_REQUIRE(LookupHost("1122:3344:5566:7788:9900:aabb:ccdd:eeff", addr, false)); + BOOST_REQUIRE(addr.IsValid()); + BOOST_REQUIRE(addr.IsIPv6()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "1122:3344:5566:7788:9900:aabb:ccdd:eeff"); + + // TORv2 + addr.SetSpecial("6hzph5hv6337r6p2.onion"); + BOOST_REQUIRE(addr.IsValid()); + BOOST_REQUIRE(addr.IsTor()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "6hzph5hv6337r6p2.onion"); + + // Internal + addr.SetInternal("esffpp"); + BOOST_REQUIRE(!addr.IsValid()); // "internal" is considered invalid + BOOST_REQUIRE(addr.IsInternal()); + + BOOST_CHECK(!addr.IsBindAny()); + BOOST_CHECK_EQUAL(addr.ToString(), "esffpvrt3wpeaygy.internal"); +} + +BOOST_AUTO_TEST_CASE(cnetaddr_serialize) +{ + CNetAddr addr; + CDataStream s(SER_NETWORK, PROTOCOL_VERSION); - fInboundIn = true; - std::unique_ptr<CNode> pnode2 = MakeUnique<CNode>(id++, NODE_NETWORK, height, hSocket, addr, 1, 1, CAddress(), pszDest, fInboundIn); - BOOST_CHECK(pnode2->fInbound == true); - BOOST_CHECK(pnode2->fFeeler == false); + addr.SetInternal("a"); + s << addr; + BOOST_CHECK_EQUAL(HexStr(s), "fd6b88c08724ca978112ca1bbdcafac2"); + s.clear(); } // prior to PR #14728, this test triggers an undefined behavior @@ -214,7 +293,7 @@ BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) in_addr ipv4AddrPeer; ipv4AddrPeer.s_addr = 0xa0b0c001; CAddress addr = CAddress(CService(ipv4AddrPeer, 7777), NODE_NETWORK); - std::unique_ptr<CNode> pnode = MakeUnique<CNode>(0, NODE_NETWORK, 0, INVALID_SOCKET, addr, 0, 0, CAddress{}, std::string{}, false); + std::unique_ptr<CNode> pnode = MakeUnique<CNode>(0, NODE_NETWORK, 0, INVALID_SOCKET, addr, 0, 0, CAddress{}, std::string{}, ConnectionType::OUTBOUND_FULL_RELAY); pnode->fSuccessfullyConnected.store(true); // the peer claims to be reaching us via IPv6 diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 49073ea65..6681c92bb 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -185,6 +185,7 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_CHECK(!ResolveSubNet("1.2.3.0/-1").IsValid()); BOOST_CHECK(ResolveSubNet("1.2.3.0/32").IsValid()); BOOST_CHECK(!ResolveSubNet("1.2.3.0/33").IsValid()); + BOOST_CHECK(!ResolveSubNet("1.2.3.0/300").IsValid()); BOOST_CHECK(ResolveSubNet("1:2:3:4:5:6:7:8/0").IsValid()); BOOST_CHECK(ResolveSubNet("1:2:3:4:5:6:7:8/33").IsValid()); BOOST_CHECK(!ResolveSubNet("1:2:3:4:5:6:7:8/-1").IsValid()); @@ -216,6 +217,11 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_CHECK(CSubNet(ResolveIP("1:2:3:4:5:6:7:8")).Match(ResolveIP("1:2:3:4:5:6:7:8"))); BOOST_CHECK(!CSubNet(ResolveIP("1:2:3:4:5:6:7:8")).Match(ResolveIP("1:2:3:4:5:6:7:9"))); BOOST_CHECK(CSubNet(ResolveIP("1:2:3:4:5:6:7:8")).ToString() == "1:2:3:4:5:6:7:8/128"); + // IPv4 address with IPv6 netmask or the other way around. + BOOST_CHECK(!CSubNet(ResolveIP("1.1.1.1"), ResolveIP("ffff::")).IsValid()); + BOOST_CHECK(!CSubNet(ResolveIP("::1"), ResolveIP("255.0.0.0")).IsValid()); + // Can't subnet TOR (or any other non-IPv4 and non-IPv6 network). + BOOST_CHECK(!CSubNet(ResolveIP("5wyqrzbvrdsumnok.onion"), ResolveIP("255.0.0.0")).IsValid()); subnet = ResolveSubNet("1.2.3.4/255.255.255.255"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.4/32"); @@ -290,11 +296,13 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_CHECK_EQUAL(subnet.ToString(), "1::/16"); subnet = ResolveSubNet("1:2:3:4:5:6:7:8/0000:0000:0000:0000:0000:0000:0000:0000"); BOOST_CHECK_EQUAL(subnet.ToString(), "::/0"); + // Invalid netmasks (with 1-bits after 0-bits) subnet = ResolveSubNet("1.2.3.4/255.255.232.0"); - BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/255.255.232.0"); + BOOST_CHECK(!subnet.IsValid()); + subnet = ResolveSubNet("1.2.3.4/255.0.255.255"); + BOOST_CHECK(!subnet.IsValid()); subnet = ResolveSubNet("1:2:3:4:5:6:7:8/ffff:ffff:ffff:fffe:ffff:ffff:ffff:ff0f"); - BOOST_CHECK_EQUAL(subnet.ToString(), "1:2:3:4:5:6:7:8/ffff:ffff:ffff:fffe:ffff:ffff:ffff:ff0f"); - + BOOST_CHECK(!subnet.IsValid()); } BOOST_AUTO_TEST_CASE(netbase_getgroup) @@ -428,7 +436,8 @@ BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0", 11), ret)); BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0example.com", 22), ret)); BOOST_CHECK(!LookupSubNet(std::string("1.2.3.0/24\0example.com\0", 23), ret)); - BOOST_CHECK(LookupSubNet(std::string("5wyqrzbvrdsumnok.onion", 22), ret)); + // We only do subnetting for IPv4 and IPv6 + BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion", 22), ret)); BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0", 23), ret)); BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com", 34), ret)); BOOST_CHECK(!LookupSubNet(std::string("5wyqrzbvrdsumnok.onion\0example.com\0", 35), ret)); diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index 87678af4d..1d6bcadf6 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -349,21 +349,16 @@ BOOST_AUTO_TEST_CASE(script_standard_GetScriptFor_) result = GetScriptForMultisig(2, std::vector<CPubKey>(pubkeys, pubkeys + 3)); BOOST_CHECK(result == expected); - // GetScriptForWitness - CScript witnessScript; - - witnessScript << ToByteVector(pubkeys[0]) << OP_CHECKSIG; + // WitnessV0KeyHash expected.clear(); expected << OP_0 << ToByteVector(pubkeys[0].GetID()); - result = GetScriptForWitness(witnessScript); + result = GetScriptForDestination(WitnessV0KeyHash(Hash160(ToByteVector(pubkeys[0])))); BOOST_CHECK(result == expected); - - witnessScript.clear(); - witnessScript << OP_DUP << OP_HASH160 << ToByteVector(pubkeys[0].GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; - result = GetScriptForWitness(witnessScript); + result = GetScriptForDestination(WitnessV0KeyHash(pubkeys[0].GetID())); BOOST_CHECK(result == expected); - witnessScript.clear(); + // WitnessV0ScriptHash (multisig) + CScript witnessScript; witnessScript << OP_1 << ToByteVector(pubkeys[0]) << OP_1 << OP_CHECKMULTISIG; uint256 scriptHash; @@ -372,7 +367,7 @@ BOOST_AUTO_TEST_CASE(script_standard_GetScriptFor_) expected.clear(); expected << OP_0 << ToByteVector(scriptHash); - result = GetScriptForWitness(witnessScript); + result = GetScriptForDestination(WitnessV0ScriptHash(witnessScript)); BOOST_CHECK(result == expected); } diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp index 91e039416..548fd020a 100644 --- a/src/test/settings_tests.cpp +++ b/src/test/settings_tests.cpp @@ -241,7 +241,7 @@ BOOST_FIXTURE_TEST_CASE(Merge, MergeTestingSetup) unsigned char out_sha_bytes[CSHA256::OUTPUT_SIZE]; out_sha.Finalize(out_sha_bytes); - std::string out_sha_hex = HexStr(std::begin(out_sha_bytes), std::end(out_sha_bytes)); + std::string out_sha_hex = HexStr(out_sha_bytes); // If check below fails, should manually dump the results with: // diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 6e36bce7a..7e5274450 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -154,8 +154,7 @@ BOOST_AUTO_TEST_CASE(GetTxSigOpCost) // P2WPKH witness program { - CScript p2pk = CScript() << ToByteVector(pubkey) << OP_CHECKSIG; - CScript scriptPubKey = GetScriptForWitness(p2pk); + CScript scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(pubkey)); CScript scriptSig = CScript(); CScriptWitness scriptWitness; scriptWitness.stack.push_back(std::vector<unsigned char>(0)); @@ -183,8 +182,7 @@ BOOST_AUTO_TEST_CASE(GetTxSigOpCost) // P2WPKH nested in P2SH { - CScript p2pk = CScript() << ToByteVector(pubkey) << OP_CHECKSIG; - CScript scriptSig = GetScriptForWitness(p2pk); + CScript scriptSig = GetScriptForDestination(WitnessV0KeyHash(pubkey)); CScript scriptPubKey = GetScriptForDestination(ScriptHash(scriptSig)); scriptSig = CScript() << ToByteVector(scriptSig); CScriptWitness scriptWitness; @@ -199,7 +197,7 @@ BOOST_AUTO_TEST_CASE(GetTxSigOpCost) // P2WSH witness program { CScript witnessScript = CScript() << 1 << ToByteVector(pubkey) << ToByteVector(pubkey) << 2 << OP_CHECKMULTISIGVERIFY; - CScript scriptPubKey = GetScriptForWitness(witnessScript); + CScript scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(witnessScript)); CScript scriptSig = CScript(); CScriptWitness scriptWitness; scriptWitness.stack.push_back(std::vector<unsigned char>(0)); @@ -215,7 +213,7 @@ BOOST_AUTO_TEST_CASE(GetTxSigOpCost) // P2WSH nested in P2SH { CScript witnessScript = CScript() << 1 << ToByteVector(pubkey) << ToByteVector(pubkey) << 2 << OP_CHECKMULTISIGVERIFY; - CScript redeemScript = GetScriptForWitness(witnessScript); + CScript redeemScript = GetScriptForDestination(WitnessV0ScriptHash(witnessScript)); CScript scriptPubKey = GetScriptForDestination(ScriptHash(redeemScript)); CScript scriptSig = CScript() << ToByteVector(redeemScript); CScriptWitness scriptWitness; diff --git a/src/test/sync_tests.cpp b/src/test/sync_tests.cpp index 3ea8714f3..19029ebd3 100644 --- a/src/test/sync_tests.cpp +++ b/src/test/sync_tests.cpp @@ -14,6 +14,7 @@ void TestPotentialDeadLockDetected(MutexType& mutex1, MutexType& mutex2) { LOCK2(mutex1, mutex2); } + BOOST_CHECK(LockStackEmpty()); bool error_thrown = false; try { LOCK2(mutex2, mutex1); @@ -21,6 +22,7 @@ void TestPotentialDeadLockDetected(MutexType& mutex1, MutexType& mutex2) BOOST_CHECK_EQUAL(e.what(), "potential deadlock detected: mutex1 -> mutex2 -> mutex1"); error_thrown = true; } + BOOST_CHECK(LockStackEmpty()); #ifdef DEBUG_LOCKORDER BOOST_CHECK(error_thrown); #else @@ -40,9 +42,13 @@ BOOST_AUTO_TEST_CASE(potential_deadlock_detected) RecursiveMutex rmutex1, rmutex2; TestPotentialDeadLockDetected(rmutex1, rmutex2); + // The second test ensures that lock tracking data have not been broken by exception. + TestPotentialDeadLockDetected(rmutex1, rmutex2); Mutex mutex1, mutex2; TestPotentialDeadLockDetected(mutex1, mutex2); + // The second test ensures that lock tracking data have not been broken by exception. + TestPotentialDeadLockDetected(mutex1, mutex2); #ifdef DEBUG_LOCKORDER g_debug_lockorder_abort = prev; diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp new file mode 100644 index 000000000..a55145c73 --- /dev/null +++ b/src/test/system_tests.cpp @@ -0,0 +1,95 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include <test/util/setup_common.h> +#include <util/system.h> +#include <univalue.h> + +#ifdef HAVE_BOOST_PROCESS +#include <boost/process.hpp> +#endif // HAVE_BOOST_PROCESS + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(system_tests, BasicTestingSetup) + +// At least one test is required (in case HAVE_BOOST_PROCESS is not defined). +// Workaround for https://github.com/bitcoin/bitcoin/issues/19128 +BOOST_AUTO_TEST_CASE(dummy) +{ + BOOST_CHECK(true); +} + +#ifdef HAVE_BOOST_PROCESS + +bool checkMessage(const std::runtime_error& ex) +{ + // On Linux & Mac: "No such file or directory" + // On Windows: "The system cannot find the file specified." + const std::string what(ex.what()); + BOOST_CHECK(what.find("file") != std::string::npos); + return true; +} + +bool checkMessageFalse(const std::runtime_error& ex) +{ + BOOST_CHECK_EQUAL(ex.what(), std::string("RunCommandParseJSON error: process(false) returned 1: \n")); + return true; +} + +bool checkMessageStdErr(const std::runtime_error& ex) +{ + const std::string what(ex.what()); + BOOST_CHECK(what.find("RunCommandParseJSON error:") != std::string::npos); + return checkMessage(ex); +} + +BOOST_AUTO_TEST_CASE(run_command) +{ + { + const UniValue result = RunCommandParseJSON(""); + BOOST_CHECK(result.isNull()); + } + { +#ifdef WIN32 + // Windows requires single quotes to prevent escaping double quotes from the JSON... + const UniValue result = RunCommandParseJSON("echo '{\"success\": true}'"); +#else + // ... but Linux and macOS echo a single quote if it's used + const UniValue result = RunCommandParseJSON("echo \"{\"success\": true}\""); +#endif + BOOST_CHECK(result.isObject()); + const UniValue& success = find_value(result, "success"); + BOOST_CHECK(!success.isNull()); + BOOST_CHECK_EQUAL(success.getBool(), true); + } + { + // An invalid command is handled by Boost + BOOST_CHECK_EXCEPTION(RunCommandParseJSON("invalid_command"), boost::process::process_error, checkMessage); // Command failed + } + { + // Return non-zero exit code, no output to stderr + BOOST_CHECK_EXCEPTION(RunCommandParseJSON("false"), std::runtime_error, checkMessageFalse); + } + { + // Return non-zero exit code, with error message for stderr + BOOST_CHECK_EXCEPTION(RunCommandParseJSON("ls nosuchfile"), std::runtime_error, checkMessageStdErr); + } + { + BOOST_REQUIRE_THROW(RunCommandParseJSON("echo \"{\""), std::runtime_error); // Unable to parse JSON + } + // Test std::in, except for Windows +#ifndef WIN32 + { + const UniValue result = RunCommandParseJSON("cat", "{\"success\": true}"); + BOOST_CHECK(result.isObject()); + const UniValue& success = find_value(result, "success"); + BOOST_CHECK(!success.isNull()); + BOOST_CHECK_EQUAL(success.getBool(), true); + } +#endif +} +#endif // HAVE_BOOST_PROCESS + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 4bf6e734c..94b5dba91 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -361,6 +361,8 @@ static CScript PushAll(const std::vector<valtype>& values) result << OP_0; } else if (v.size() == 1 && v[0] >= 1 && v[0] <= 16) { result << CScript::EncodeOP_N(v[0]); + } else if (v.size() == 1 && v[0] == 0x81) { + result << OP_1NEGATE; } else { result << v; } @@ -499,13 +501,19 @@ BOOST_AUTO_TEST_CASE(test_witness) BOOST_CHECK(keystore.AddCScript(scriptPubkey1L)); BOOST_CHECK(keystore.AddCScript(scriptPubkey2L)); BOOST_CHECK(keystore.AddCScript(scriptMulti)); - BOOST_CHECK(keystore.AddCScript(GetScriptForWitness(scriptPubkey1))); - BOOST_CHECK(keystore.AddCScript(GetScriptForWitness(scriptPubkey2))); - BOOST_CHECK(keystore.AddCScript(GetScriptForWitness(scriptPubkey1L))); - BOOST_CHECK(keystore.AddCScript(GetScriptForWitness(scriptPubkey2L))); - BOOST_CHECK(keystore.AddCScript(GetScriptForWitness(scriptMulti))); + CScript destination_script_1, destination_script_2, destination_script_1L, destination_script_2L, destination_script_multi; + destination_script_1 = GetScriptForDestination(WitnessV0KeyHash(pubkey1)); + destination_script_2 = GetScriptForDestination(WitnessV0KeyHash(pubkey2)); + destination_script_1L = GetScriptForDestination(WitnessV0KeyHash(pubkey1L)); + destination_script_2L = GetScriptForDestination(WitnessV0KeyHash(pubkey2L)); + destination_script_multi = GetScriptForDestination(WitnessV0ScriptHash(scriptMulti)); + BOOST_CHECK(keystore.AddCScript(destination_script_1)); + BOOST_CHECK(keystore.AddCScript(destination_script_2)); + BOOST_CHECK(keystore.AddCScript(destination_script_1L)); + BOOST_CHECK(keystore.AddCScript(destination_script_2L)); + BOOST_CHECK(keystore.AddCScript(destination_script_multi)); BOOST_CHECK(keystore2.AddCScript(scriptMulti)); - BOOST_CHECK(keystore2.AddCScript(GetScriptForWitness(scriptMulti))); + BOOST_CHECK(keystore2.AddCScript(destination_script_multi)); BOOST_CHECK(keystore2.AddKeyPubKey(key3, pubkey3)); CTransactionRef output1, output2; @@ -537,8 +545,8 @@ BOOST_AUTO_TEST_CASE(test_witness) CheckWithFlag(output1, input2, STANDARD_SCRIPT_VERIFY_FLAGS, false); // Witness pay-to-compressed-pubkey (v0). - CreateCreditAndSpend(keystore, GetScriptForWitness(scriptPubkey1), output1, input1); - CreateCreditAndSpend(keystore, GetScriptForWitness(scriptPubkey2), output2, input2); + CreateCreditAndSpend(keystore, destination_script_1, output1, input1); + CreateCreditAndSpend(keystore, destination_script_2, output2, input2); CheckWithFlag(output1, input1, 0, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH, true); @@ -549,9 +557,9 @@ BOOST_AUTO_TEST_CASE(test_witness) CheckWithFlag(output1, input2, STANDARD_SCRIPT_VERIFY_FLAGS, false); // P2SH witness pay-to-compressed-pubkey (v0). - CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(GetScriptForWitness(scriptPubkey1))), output1, input1); - CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(GetScriptForWitness(scriptPubkey2))), output2, input2); - ReplaceRedeemScript(input2.vin[0].scriptSig, GetScriptForWitness(scriptPubkey1)); + CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(destination_script_1)), output1, input1); + CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(destination_script_2)), output2, input2); + ReplaceRedeemScript(input2.vin[0].scriptSig, destination_script_1); CheckWithFlag(output1, input1, 0, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH, true); @@ -587,12 +595,12 @@ BOOST_AUTO_TEST_CASE(test_witness) CheckWithFlag(output1, input2, STANDARD_SCRIPT_VERIFY_FLAGS, false); // Signing disabled for witness pay-to-uncompressed-pubkey (v1). - CreateCreditAndSpend(keystore, GetScriptForWitness(scriptPubkey1L), output1, input1, false); - CreateCreditAndSpend(keystore, GetScriptForWitness(scriptPubkey2L), output2, input2, false); + CreateCreditAndSpend(keystore, destination_script_1L, output1, input1, false); + CreateCreditAndSpend(keystore, destination_script_2L, output2, input2, false); // Signing disabled for P2SH witness pay-to-uncompressed-pubkey (v1). - CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(GetScriptForWitness(scriptPubkey1L))), output1, input1, false); - CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(GetScriptForWitness(scriptPubkey2L))), output2, input2, false); + CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(destination_script_1L)), output1, input1, false); + CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(destination_script_2L)), output2, input2, false); // Normal 2-of-2 multisig CreateCreditAndSpend(keystore, scriptMulti, output1, input1, false); @@ -616,10 +624,10 @@ BOOST_AUTO_TEST_CASE(test_witness) CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); // Witness 2-of-2 multisig - CreateCreditAndSpend(keystore, GetScriptForWitness(scriptMulti), output1, input1, false); + CreateCreditAndSpend(keystore, destination_script_multi, output1, input1, false); CheckWithFlag(output1, input1, 0, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false); - CreateCreditAndSpend(keystore2, GetScriptForWitness(scriptMulti), output2, input2, false); + CreateCreditAndSpend(keystore2, destination_script_multi, output2, input2, false); CheckWithFlag(output2, input2, 0, true); CheckWithFlag(output2, input2, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false); BOOST_CHECK(*output1 == *output2); @@ -628,10 +636,10 @@ BOOST_AUTO_TEST_CASE(test_witness) CheckWithFlag(output1, input1, STANDARD_SCRIPT_VERIFY_FLAGS, true); // P2SH witness 2-of-2 multisig - CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(GetScriptForWitness(scriptMulti))), output1, input1, false); + CreateCreditAndSpend(keystore, GetScriptForDestination(ScriptHash(destination_script_multi)), output1, input1, false); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output1, input1, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false); - CreateCreditAndSpend(keystore2, GetScriptForDestination(ScriptHash(GetScriptForWitness(scriptMulti))), output2, input2, false); + CreateCreditAndSpend(keystore2, GetScriptForDestination(ScriptHash(destination_script_multi)), output2, input2, false); CheckWithFlag(output2, input2, SCRIPT_VERIFY_P2SH, true); CheckWithFlag(output2, input2, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false); BOOST_CHECK(*output1 == *output2); diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index cdef7dcc3..034577aa2 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -157,7 +157,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) CScript p2pk_scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; CScript p2sh_scriptPubKey = GetScriptForDestination(ScriptHash(p2pk_scriptPubKey)); CScript p2pkh_scriptPubKey = GetScriptForDestination(PKHash(coinbaseKey.GetPubKey())); - CScript p2wpkh_scriptPubKey = GetScriptForWitness(p2pkh_scriptPubKey); + CScript p2wpkh_scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(coinbaseKey.GetPubKey())); FillableSigningProvider keystore; BOOST_CHECK(keystore.AddKey(coinbaseKey)); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 14f65dcb7..08aff0744 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -11,6 +11,7 @@ #include <consensus/validation.h> #include <crypto/sha256.h> #include <init.h> +#include <interfaces/chain.h> #include <miner.h> #include <net.h> #include <net_processing.h> @@ -32,6 +33,7 @@ #include <util/vector.h> #include <validation.h> #include <validationinterface.h> +#include <walletinitinterface.h> #include <functional> @@ -95,8 +97,8 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve SelectParams(chainName); SeedInsecureRand(); if (G_TEST_LOG_FUN) LogInstance().PushBackCallback(G_TEST_LOG_FUN); - InitLogging(); - AppInitParameterInteraction(); + InitLogging(*m_node.args); + AppInitParameterInteraction(*m_node.args); LogInstance().StartLogging(); SHA256AutoDetect(); ECC_Start(); @@ -104,6 +106,8 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve SetupNetworking(); InitSignatureCache(); InitScriptExecutionCache(); + m_node.chain = interfaces::MakeChain(m_node); + g_wallet_init_interface.Construct(m_node); fCheckBlockIndex = true; static bool noui_connected = false; if (!noui_connected) { @@ -137,8 +141,11 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const pblocktree.reset(new CBlockTreeDB(1 << 20, true)); + m_node.mempool = MakeUnique<CTxMemPool>(&::feeEstimator); + m_node.mempool->setSanityCheck(1.0); + m_node.chainman = &::g_chainman; - m_node.chainman->InitializeChainstate(); + m_node.chainman->InitializeChainstate(*m_node.mempool); ::ChainstateActive().InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); assert(!::ChainstateActive().CanFlushToDisk()); @@ -160,14 +167,12 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const } g_parallel_script_checks = true; - m_node.mempool = &::mempool; - m_node.mempool->setSanityCheck(1.0); m_node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); m_node.connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. - m_node.peer_logic = MakeUnique<PeerLogicValidation>(m_node.connman.get(), m_node.banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); + m_node.peerman = MakeUnique<PeerManager>(chainparams, *m_node.connman, m_node.banman.get(), *m_node.scheduler, *m_node.chainman, *m_node.mempool); { CConnman::Options options; - options.m_msgproc = m_node.peer_logic.get(); + options.m_msgproc = m_node.peerman.get(); m_node.connman->Init(options); } } @@ -182,8 +187,8 @@ TestingSetup::~TestingSetup() m_node.connman.reset(); m_node.banman.reset(); m_node.args = nullptr; - UnloadBlockIndex(m_node.mempool); - m_node.mempool = nullptr; + UnloadBlockIndex(m_node.mempool.get()); + m_node.mempool.reset(); m_node.scheduler.reset(); m_node.chainman->Reset(); m_node.chainman = nullptr; @@ -192,49 +197,34 @@ TestingSetup::~TestingSetup() TestChain100Setup::TestChain100Setup() { - // CreateAndProcessBlock() does not support building SegWit blocks, so don't activate in these tests. - // TODO: fix the code to support SegWit blocks. - gArgs.ForceSetArg("-segwitheight", "432"); - // Need to recreate chainparams - SelectParams(CBaseChainParams::REGTEST); - // Generate a 100-block chain: coinbaseKey.MakeNewKey(true); - CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; - for (int i = 0; i < COINBASE_MATURITY; i++) - { + CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; + for (int i = 0; i < COINBASE_MATURITY; i++) { std::vector<CMutableTransaction> noTxns; CBlock b = CreateAndProcessBlock(noTxns, scriptPubKey); m_coinbase_txns.push_back(b.vtx[0]); } } -// Create a new block with just given transactions, coinbase paying to -// scriptPubKey, and try to add it to the current chain. CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey) { const CChainParams& chainparams = Params(); - std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(*m_node.mempool, chainparams).CreateNewBlock(scriptPubKey); - CBlock& block = pblocktemplate->block; + CTxMemPool empty_pool; + CBlock block = BlockAssembler(empty_pool, chainparams).CreateNewBlock(scriptPubKey)->block; - // Replace mempool-selected txns with just coinbase plus passed-in txns: - block.vtx.resize(1); - for (const CMutableTransaction& tx : txns) + Assert(block.vtx.size() == 1); + for (const CMutableTransaction& tx : txns) { block.vtx.push_back(MakeTransactionRef(tx)); - // IncrementExtraNonce creates a valid coinbase and merkleRoot - { - LOCK(cs_main); - unsigned int extraNonce = 0; - IncrementExtraNonce(&block, ::ChainActive().Tip(), extraNonce); } + RegenerateCommitments(block); while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr); - CBlock result = block; - return result; + return block; } TestChain100Setup::~TestChain100Setup() @@ -242,8 +232,8 @@ TestChain100Setup::~TestChain100Setup() gArgs.ForceSetArg("-segwitheight", "0"); } - -CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction &tx) { +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) +{ return FromTx(MakeTransactionRef(tx)); } diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 78b279e42..22f5d6d93 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -102,15 +102,16 @@ class CBlock; struct CMutableTransaction; class CScript; -// -// Testing fixture that pre-creates a -// 100-block REGTEST-mode block chain -// +/** + * Testing fixture that pre-creates a 100-block REGTEST-mode block chain + */ struct TestChain100Setup : public RegTestingSetup { TestChain100Setup(); - // Create a new block with just given transactions, coinbase paying to - // scriptPubKey, and try to add it to the current chain. + /** + * Create a new block with just given transactions, coinbase paying to + * scriptPubKey, and try to add it to the current chain. + */ CBlock CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey); diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 15a2c1e30..bf7c6c3e3 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -105,47 +105,24 @@ BOOST_AUTO_TEST_CASE(util_ParseHex) BOOST_AUTO_TEST_CASE(util_HexStr) { BOOST_CHECK_EQUAL( - HexStr(ParseHex_expected, ParseHex_expected + sizeof(ParseHex_expected)), + HexStr(ParseHex_expected), "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f"); BOOST_CHECK_EQUAL( - HexStr(ParseHex_expected + sizeof(ParseHex_expected), - ParseHex_expected + sizeof(ParseHex_expected)), + HexStr(Span<const unsigned char>( + ParseHex_expected + sizeof(ParseHex_expected), + ParseHex_expected + sizeof(ParseHex_expected))), ""); BOOST_CHECK_EQUAL( - HexStr(ParseHex_expected, ParseHex_expected), + HexStr(Span<const unsigned char>(ParseHex_expected, ParseHex_expected)), ""); std::vector<unsigned char> ParseHex_vec(ParseHex_expected, ParseHex_expected + 5); BOOST_CHECK_EQUAL( - HexStr(ParseHex_vec.rbegin(), ParseHex_vec.rend()), - "b0fd8a6704" - ); - - BOOST_CHECK_EQUAL( - HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected), - std::reverse_iterator<const uint8_t *>(ParseHex_expected)), - "" - ); - - BOOST_CHECK_EQUAL( - HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected + 1), - std::reverse_iterator<const uint8_t *>(ParseHex_expected)), - "04" - ); - - BOOST_CHECK_EQUAL( - HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected + 5), - std::reverse_iterator<const uint8_t *>(ParseHex_expected)), - "b0fd8a6704" - ); - - BOOST_CHECK_EQUAL( - HexStr(std::reverse_iterator<const uint8_t *>(ParseHex_expected + 65), - std::reverse_iterator<const uint8_t *>(ParseHex_expected)), - "5f1df16b2b704c8a578d0bbaf74d385cde12c11ee50455f3c438ef4c3fbcf649b6de611feae06279a60939e028a8d65c10b73071a6f16719274855feb0fd8a6704" + HexStr(ParseHex_vec), + "04678afdb0" ); } @@ -573,57 +550,52 @@ BOOST_AUTO_TEST_CASE(util_ReadConfigStream) BOOST_CHECK(test_args.m_settings.ro_config["sec1"].size() == 3); BOOST_CHECK(test_args.m_settings.ro_config["sec2"].size() == 2); - BOOST_CHECK(test_args.m_settings.ro_config[""].count("a") - && test_args.m_settings.ro_config[""].count("b") - && test_args.m_settings.ro_config[""].count("ccc") - && test_args.m_settings.ro_config[""].count("d") - && test_args.m_settings.ro_config[""].count("fff") - && test_args.m_settings.ro_config[""].count("ggg") - && test_args.m_settings.ro_config[""].count("h") - && test_args.m_settings.ro_config[""].count("i") - ); - BOOST_CHECK(test_args.m_settings.ro_config["sec1"].count("ccc") - && test_args.m_settings.ro_config["sec1"].count("h") - && test_args.m_settings.ro_config["sec2"].count("ccc") - && test_args.m_settings.ro_config["sec2"].count("iii") - ); - - BOOST_CHECK(test_args.IsArgSet("-a") - && test_args.IsArgSet("-b") - && test_args.IsArgSet("-ccc") - && test_args.IsArgSet("-d") - && test_args.IsArgSet("-fff") - && test_args.IsArgSet("-ggg") - && test_args.IsArgSet("-h") - && test_args.IsArgSet("-i") - && !test_args.IsArgSet("-zzz") - && !test_args.IsArgSet("-iii") - ); - - BOOST_CHECK(test_args.GetArg("-a", "xxx") == "" - && test_args.GetArg("-b", "xxx") == "1" - && test_args.GetArg("-ccc", "xxx") == "argument" - && test_args.GetArg("-d", "xxx") == "e" - && test_args.GetArg("-fff", "xxx") == "0" - && test_args.GetArg("-ggg", "xxx") == "1" - && test_args.GetArg("-h", "xxx") == "0" - && test_args.GetArg("-i", "xxx") == "1" - && test_args.GetArg("-zzz", "xxx") == "xxx" - && test_args.GetArg("-iii", "xxx") == "xxx" - ); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("a")); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("b")); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("ccc")); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("d")); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("fff")); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("ggg")); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("h")); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("i")); + BOOST_CHECK(test_args.m_settings.ro_config["sec1"].count("ccc")); + BOOST_CHECK(test_args.m_settings.ro_config["sec1"].count("h")); + BOOST_CHECK(test_args.m_settings.ro_config["sec2"].count("ccc")); + BOOST_CHECK(test_args.m_settings.ro_config["sec2"].count("iii")); + + BOOST_CHECK(test_args.IsArgSet("-a")); + BOOST_CHECK(test_args.IsArgSet("-b")); + BOOST_CHECK(test_args.IsArgSet("-ccc")); + BOOST_CHECK(test_args.IsArgSet("-d")); + BOOST_CHECK(test_args.IsArgSet("-fff")); + BOOST_CHECK(test_args.IsArgSet("-ggg")); + BOOST_CHECK(test_args.IsArgSet("-h")); + BOOST_CHECK(test_args.IsArgSet("-i")); + BOOST_CHECK(!test_args.IsArgSet("-zzz")); + BOOST_CHECK(!test_args.IsArgSet("-iii")); + + BOOST_CHECK_EQUAL(test_args.GetArg("-a", "xxx"), ""); + BOOST_CHECK_EQUAL(test_args.GetArg("-b", "xxx"), "1"); + BOOST_CHECK_EQUAL(test_args.GetArg("-ccc", "xxx"), "argument"); + BOOST_CHECK_EQUAL(test_args.GetArg("-d", "xxx"), "e"); + BOOST_CHECK_EQUAL(test_args.GetArg("-fff", "xxx"), "0"); + BOOST_CHECK_EQUAL(test_args.GetArg("-ggg", "xxx"), "1"); + BOOST_CHECK_EQUAL(test_args.GetArg("-h", "xxx"), "0"); + BOOST_CHECK_EQUAL(test_args.GetArg("-i", "xxx"), "1"); + BOOST_CHECK_EQUAL(test_args.GetArg("-zzz", "xxx"), "xxx"); + BOOST_CHECK_EQUAL(test_args.GetArg("-iii", "xxx"), "xxx"); for (const bool def : {false, true}) { - BOOST_CHECK(test_args.GetBoolArg("-a", def) - && test_args.GetBoolArg("-b", def) - && !test_args.GetBoolArg("-ccc", def) - && !test_args.GetBoolArg("-d", def) - && !test_args.GetBoolArg("-fff", def) - && test_args.GetBoolArg("-ggg", def) - && !test_args.GetBoolArg("-h", def) - && test_args.GetBoolArg("-i", def) - && test_args.GetBoolArg("-zzz", def) == def - && test_args.GetBoolArg("-iii", def) == def - ); + BOOST_CHECK(test_args.GetBoolArg("-a", def)); + BOOST_CHECK(test_args.GetBoolArg("-b", def)); + BOOST_CHECK(!test_args.GetBoolArg("-ccc", def)); + BOOST_CHECK(!test_args.GetBoolArg("-d", def)); + BOOST_CHECK(!test_args.GetBoolArg("-fff", def)); + BOOST_CHECK(test_args.GetBoolArg("-ggg", def)); + BOOST_CHECK(!test_args.GetBoolArg("-h", def)); + BOOST_CHECK(test_args.GetBoolArg("-i", def)); + BOOST_CHECK(test_args.GetBoolArg("-zzz", def) == def); + BOOST_CHECK(test_args.GetBoolArg("-iii", def) == def); } BOOST_CHECK(test_args.GetArgs("-a").size() == 1 @@ -659,13 +631,12 @@ BOOST_AUTO_TEST_CASE(util_ReadConfigStream) test_args.SelectConfigNetwork("sec1"); // same as original - BOOST_CHECK(test_args.GetArg("-a", "xxx") == "" - && test_args.GetArg("-b", "xxx") == "1" - && test_args.GetArg("-fff", "xxx") == "0" - && test_args.GetArg("-ggg", "xxx") == "1" - && test_args.GetArg("-zzz", "xxx") == "xxx" - && test_args.GetArg("-iii", "xxx") == "xxx" - ); + BOOST_CHECK_EQUAL(test_args.GetArg("-a", "xxx"), ""); + BOOST_CHECK_EQUAL(test_args.GetArg("-b", "xxx"), "1"); + BOOST_CHECK_EQUAL(test_args.GetArg("-fff", "xxx"), "0"); + BOOST_CHECK_EQUAL(test_args.GetArg("-ggg", "xxx"), "1"); + BOOST_CHECK_EQUAL(test_args.GetArg("-zzz", "xxx"), "xxx"); + BOOST_CHECK_EQUAL(test_args.GetArg("-iii", "xxx"), "xxx"); // d is overridden BOOST_CHECK(test_args.GetArg("-d", "xxx") == "eee"); // section-specific setting @@ -680,14 +651,13 @@ BOOST_AUTO_TEST_CASE(util_ReadConfigStream) test_args.SelectConfigNetwork("sec2"); // same as original - BOOST_CHECK(test_args.GetArg("-a", "xxx") == "" - && test_args.GetArg("-b", "xxx") == "1" - && test_args.GetArg("-d", "xxx") == "e" - && test_args.GetArg("-fff", "xxx") == "0" - && test_args.GetArg("-ggg", "xxx") == "1" - && test_args.GetArg("-zzz", "xxx") == "xxx" - && test_args.GetArg("-h", "xxx") == "0" - ); + BOOST_CHECK(test_args.GetArg("-a", "xxx") == ""); + BOOST_CHECK(test_args.GetArg("-b", "xxx") == "1"); + BOOST_CHECK(test_args.GetArg("-d", "xxx") == "e"); + BOOST_CHECK(test_args.GetArg("-fff", "xxx") == "0"); + BOOST_CHECK(test_args.GetArg("-ggg", "xxx") == "1"); + BOOST_CHECK(test_args.GetArg("-zzz", "xxx") == "xxx"); + BOOST_CHECK(test_args.GetArg("-h", "xxx") == "0"); // section-specific setting BOOST_CHECK(test_args.GetArg("-iii", "xxx") == "2"); // section takes priority for multiple values @@ -1022,7 +992,7 @@ BOOST_FIXTURE_TEST_CASE(util_ArgsMerge, ArgsMergeTestingSetup) unsigned char out_sha_bytes[CSHA256::OUTPUT_SIZE]; out_sha.Finalize(out_sha_bytes); - std::string out_sha_hex = HexStr(std::begin(out_sha_bytes), std::end(out_sha_bytes)); + std::string out_sha_hex = HexStr(out_sha_bytes); // If check below fails, should manually dump the results with: // @@ -1125,7 +1095,7 @@ BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup) unsigned char out_sha_bytes[CSHA256::OUTPUT_SIZE]; out_sha.Finalize(out_sha_bytes); - std::string out_sha_hex = HexStr(std::begin(out_sha_bytes), std::end(out_sha_bytes)); + std::string out_sha_hex = HexStr(out_sha_bytes); // If check below fails, should manually dump the results with: // diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 2076a1096..c8a375275 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -20,6 +20,7 @@ BOOST_FIXTURE_TEST_SUITE(validation_chainstate_tests, TestingSetup) BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) { ChainstateManager manager; + CTxMemPool mempool; //! Create and add a Coin with DynamicMemoryUsage of 80 bytes to the given view. auto add_coin = [](CCoinsViewCache& coins_view) -> COutPoint { @@ -34,7 +35,7 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) return outp; }; - CChainState& c1 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate()); + CChainState& c1 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate(mempool)); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 887a48124..36badafc4 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -23,12 +23,13 @@ BOOST_FIXTURE_TEST_SUITE(validation_chainstatemanager_tests, TestingSetup) BOOST_AUTO_TEST_CASE(chainstatemanager) { ChainstateManager manager; + CTxMemPool mempool; std::vector<CChainState*> chainstates; const CChainParams& chainparams = Params(); // Create a legacy (IBD) chainstate. // - CChainState& c1 = *WITH_LOCK(::cs_main, return &manager.InitializeChainstate()); + CChainState& c1 = *WITH_LOCK(::cs_main, return &manager.InitializeChainstate(mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -54,7 +55,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Create a snapshot-based chainstate. // - CChainState& c2 = *WITH_LOCK(::cs_main, return &manager.InitializeChainstate(GetRandHash())); + CChainState& c2 = *WITH_LOCK(::cs_main, return &manager.InitializeChainstate(mempool, GetRandHash())); chainstates.push_back(&c2); c2.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -104,6 +105,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) { ChainstateManager manager; + CTxMemPool mempool; size_t max_cache = 10000; manager.m_total_coinsdb_cache = max_cache; manager.m_total_coinstip_cache = max_cache; @@ -112,7 +114,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a legacy (IBD) chainstate. // - CChainState& c1 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate()); + CChainState& c1 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate(mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -129,7 +131,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a snapshot-based chainstate. // - CChainState& c2 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate(GetRandHash())); + CChainState& c2 = *WITH_LOCK(cs_main, return &manager.InitializeChainstate(mempool, GetRandHash())); chainstates.push_back(&c2); c2.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -147,7 +149,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) BOOST_CHECK_CLOSE(c1.m_coinsdb_cache_size_bytes, max_cache * 0.05, 1); BOOST_CHECK_CLOSE(c2.m_coinstip_cache_size_bytes, max_cache * 0.95, 1); BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1); - } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index 8bac914f0..a3b344d2c 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -18,8 +18,9 @@ BOOST_FIXTURE_TEST_SUITE(validation_flush_tests, BasicTestingSetup) //! BOOST_AUTO_TEST_CASE(getcoinscachesizestate) { + CTxMemPool mempool; BlockManager blockman{}; - CChainState chainstate{blockman}; + CChainState chainstate{mempool, blockman}; chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, /*should_wipe*/ false); WITH_LOCK(::cs_main, chainstate.InitCoinsCache(1 << 10)); CTxMemPool tx_pool{}; diff --git a/src/threadsafety.h b/src/threadsafety.h index 5f2c40bac..52bf83b67 100644 --- a/src/threadsafety.h +++ b/src/threadsafety.h @@ -18,9 +18,7 @@ #define LOCKABLE __attribute__((lockable)) #define SCOPED_LOCKABLE __attribute__((scoped_lockable)) #define GUARDED_BY(x) __attribute__((guarded_by(x))) -#define GUARDED_VAR __attribute__((guarded_var)) #define PT_GUARDED_BY(x) __attribute__((pt_guarded_by(x))) -#define PT_GUARDED_VAR __attribute__((pt_guarded_var)) #define ACQUIRED_AFTER(...) __attribute__((acquired_after(__VA_ARGS__))) #define ACQUIRED_BEFORE(...) __attribute__((acquired_before(__VA_ARGS__))) #define EXCLUSIVE_LOCK_FUNCTION(...) __attribute__((exclusive_lock_function(__VA_ARGS__))) @@ -33,14 +31,12 @@ #define EXCLUSIVE_LOCKS_REQUIRED(...) __attribute__((exclusive_locks_required(__VA_ARGS__))) #define SHARED_LOCKS_REQUIRED(...) __attribute__((shared_locks_required(__VA_ARGS__))) #define NO_THREAD_SAFETY_ANALYSIS __attribute__((no_thread_safety_analysis)) -#define ASSERT_EXCLUSIVE_LOCK(...) __attribute((assert_exclusive_lock(__VA_ARGS__))) +#define ASSERT_EXCLUSIVE_LOCK(...) __attribute__((assert_exclusive_lock(__VA_ARGS__))) #else #define LOCKABLE #define SCOPED_LOCKABLE #define GUARDED_BY(x) -#define GUARDED_VAR #define PT_GUARDED_BY(x) -#define PT_GUARDED_VAR #define ACQUIRED_AFTER(...) #define ACQUIRED_BEFORE(...) #define EXCLUSIVE_LOCK_FUNCTION(...) diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 84118b36e..5d56d1ff8 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -405,7 +405,7 @@ static bool WriteBinaryFile(const fs::path &filename, const std::string &data) /****** Bitcoin specific TorController implementation ********/ /** Controller that connects to Tor control socket, authenticate, then create - * and maintain an ephemeral hidden service. + * and maintain an ephemeral onion service. */ class TorController { @@ -534,7 +534,7 @@ void TorController::auth_cb(TorControlConnection& _conn, const TorControlReply& // Finally - now create the service if (private_key.empty()) // No private key, generate one private_key = "NEW:RSA1024"; // Explicitly request RSA1024 - see issue #9214 - // Request hidden service, redirect port. + // Request onion service, redirect port. // Note that the 'virtual' port is always the default port to avoid decloaking nodes using other ports. _conn.Command(strprintf("ADD_ONION %s Port=%i,127.0.0.1:%i", private_key, Params().GetDefaultPort(), GetListenPort()), std::bind(&TorController::add_onion_cb, this, std::placeholders::_1, std::placeholders::_2)); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index de1a3ec68..6a525c97d 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -55,45 +55,45 @@ size_t CTxMemPoolEntry::GetTxSize() const } // Update the given tx for any in-mempool descendants. -// Assumes that setMemPoolChildren is correct for the given tx and all +// Assumes that CTxMemPool::m_children is correct for the given tx and all // descendants. void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap &cachedDescendants, const std::set<uint256> &setExclude) { - setEntries stageEntries, setAllDescendants; - stageEntries = GetMemPoolChildren(updateIt); + CTxMemPoolEntry::Children stageEntries, descendants; + stageEntries = updateIt->GetMemPoolChildrenConst(); while (!stageEntries.empty()) { - const txiter cit = *stageEntries.begin(); - setAllDescendants.insert(cit); - stageEntries.erase(cit); - const setEntries &setChildren = GetMemPoolChildren(cit); - for (txiter childEntry : setChildren) { - cacheMap::iterator cacheIt = cachedDescendants.find(childEntry); + const CTxMemPoolEntry& descendant = *stageEntries.begin(); + descendants.insert(descendant); + stageEntries.erase(descendant); + const CTxMemPoolEntry::Children& children = descendant.GetMemPoolChildrenConst(); + for (const CTxMemPoolEntry& childEntry : children) { + cacheMap::iterator cacheIt = cachedDescendants.find(mapTx.iterator_to(childEntry)); if (cacheIt != cachedDescendants.end()) { // We've already calculated this one, just add the entries for this set // but don't traverse again. for (txiter cacheEntry : cacheIt->second) { - setAllDescendants.insert(cacheEntry); + descendants.insert(*cacheEntry); } - } else if (!setAllDescendants.count(childEntry)) { + } else if (!descendants.count(childEntry)) { // Schedule for later processing stageEntries.insert(childEntry); } } } - // setAllDescendants now contains all in-mempool descendants of updateIt. + // descendants now contains all in-mempool descendants of updateIt. // Update and add to cached descendant map int64_t modifySize = 0; CAmount modifyFee = 0; int64_t modifyCount = 0; - for (txiter cit : setAllDescendants) { - if (!setExclude.count(cit->GetTx().GetHash())) { - modifySize += cit->GetTxSize(); - modifyFee += cit->GetModifiedFee(); + for (const CTxMemPoolEntry& descendant : descendants) { + if (!setExclude.count(descendant.GetTx().GetHash())) { + modifySize += descendant.GetTxSize(); + modifyFee += descendant.GetModifiedFee(); modifyCount++; - cachedDescendants[updateIt].insert(cit); + cachedDescendants[updateIt].insert(mapTx.iterator_to(descendant)); // Update ancestor state for each descendant - mapTx.modify(cit, update_ancestor_state(updateIt->GetTxSize(), updateIt->GetModifiedFee(), 1, updateIt->GetSigOpCost())); + mapTx.modify(mapTx.iterator_to(descendant), update_ancestor_state(updateIt->GetTxSize(), updateIt->GetModifiedFee(), 1, updateIt->GetSigOpCost())); } } mapTx.modify(updateIt, update_descendant_state(modifySize, modifyFee, modifyCount)); @@ -119,7 +119,7 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashes // Iterate in reverse, so that whenever we are looking at a transaction // we are sure that all in-mempool descendants have already been processed. // This maximizes the benefit of the descendant cache and guarantees that - // setMemPoolChildren will be updated, an assumption made in + // CTxMemPool::m_children will be updated, an assumption made in // UpdateForDescendants. for (const uint256 &hash : reverse_iterate(vHashesToUpdate)) { // calculate children from mapNextTx @@ -128,8 +128,8 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashes continue; } auto iter = mapNextTx.lower_bound(COutPoint(hash, 0)); - // First calculate the children, and update setMemPoolChildren to - // include them, and update their setMemPoolParents to include this tx. + // First calculate the children, and update CTxMemPool::m_children to + // include them, and update their CTxMemPoolEntry::m_parents to include this tx. // we cache the in-mempool children to avoid duplicate updates { const auto epoch = GetFreshEpoch(); @@ -151,7 +151,7 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashes bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents /* = true */) const { - setEntries parentHashes; + CTxMemPoolEntry::Parents staged_ancestors; const CTransaction &tx = entry.GetTx(); if (fSearchForParents) { @@ -161,8 +161,8 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntr for (unsigned int i = 0; i < tx.vin.size(); i++) { Optional<txiter> piter = GetIter(tx.vin[i].prevout.hash); if (piter) { - parentHashes.insert(*piter); - if (parentHashes.size() + 1 > limitAncestorCount) { + staged_ancestors.insert(**piter); + if (staged_ancestors.size() + 1 > limitAncestorCount) { errString = strprintf("too many unconfirmed parents [limit: %u]", limitAncestorCount); return false; } @@ -172,16 +172,17 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntr // If we're not searching for parents, we require this to be an // entry in the mempool already. txiter it = mapTx.iterator_to(entry); - parentHashes = GetMemPoolParents(it); + staged_ancestors = it->GetMemPoolParentsConst(); } size_t totalSizeWithAncestors = entry.GetTxSize(); - while (!parentHashes.empty()) { - txiter stageit = *parentHashes.begin(); + while (!staged_ancestors.empty()) { + const CTxMemPoolEntry& stage = staged_ancestors.begin()->get(); + txiter stageit = mapTx.iterator_to(stage); setAncestors.insert(stageit); - parentHashes.erase(stageit); + staged_ancestors.erase(stage); totalSizeWithAncestors += stageit->GetTxSize(); if (stageit->GetSizeWithDescendants() + entry.GetTxSize() > limitDescendantSize) { @@ -195,13 +196,15 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntr return false; } - const setEntries & setMemPoolParents = GetMemPoolParents(stageit); - for (txiter phash : setMemPoolParents) { + const CTxMemPoolEntry::Parents& parents = stageit->GetMemPoolParentsConst(); + for (const CTxMemPoolEntry& parent : parents) { + txiter parent_it = mapTx.iterator_to(parent); + // If this is a new ancestor, add it. - if (setAncestors.count(phash) == 0) { - parentHashes.insert(phash); + if (setAncestors.count(parent_it) == 0) { + staged_ancestors.insert(parent); } - if (parentHashes.size() + setAncestors.size() + 1 > limitAncestorCount) { + if (staged_ancestors.size() + setAncestors.size() + 1 > limitAncestorCount) { errString = strprintf("too many unconfirmed ancestors [limit: %u]", limitAncestorCount); return false; } @@ -213,10 +216,10 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntr void CTxMemPool::UpdateAncestorsOf(bool add, txiter it, setEntries &setAncestors) { - setEntries parentIters = GetMemPoolParents(it); + CTxMemPoolEntry::Parents parents = it->GetMemPoolParents(); // add or remove this tx as a child of each parent - for (txiter piter : parentIters) { - UpdateChild(piter, it, add); + for (const CTxMemPoolEntry& parent : parents) { + UpdateChild(mapTx.iterator_to(parent), it, add); } const int64_t updateCount = (add ? 1 : -1); const int64_t updateSize = updateCount * it->GetTxSize(); @@ -242,9 +245,9 @@ void CTxMemPool::UpdateEntryForAncestors(txiter it, const setEntries &setAncesto void CTxMemPool::UpdateChildrenForRemoval(txiter it) { - const setEntries &setMemPoolChildren = GetMemPoolChildren(it); - for (txiter updateIt : setMemPoolChildren) { - UpdateParent(updateIt, it, false); + const CTxMemPoolEntry::Children& children = it->GetMemPoolChildrenConst(); + for (const CTxMemPoolEntry& updateIt : children) { + UpdateParent(mapTx.iterator_to(updateIt), it, false); } } @@ -257,9 +260,9 @@ void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove, b // updateDescendants should be true whenever we're not recursively // removing a tx and all its descendants, eg when a transaction is // confirmed in a block. - // Here we only update statistics and not data in mapLinks (which - // we need to preserve until we're finished with all operations that - // need to traverse the mempool). + // Here we only update statistics and not data in CTxMemPool::Parents + // and CTxMemPoolEntry::Children (which we need to preserve until we're + // finished with all operations that need to traverse the mempool). for (txiter removeIt : entriesToRemove) { setEntries setDescendants; CalculateDescendants(removeIt, setDescendants); @@ -282,24 +285,26 @@ void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove, b // should be a bit faster. // However, if we happen to be in the middle of processing a reorg, then // the mempool can be in an inconsistent state. In this case, the set - // of ancestors reachable via mapLinks will be the same as the set of - // ancestors whose packages include this transaction, because when we - // add a new transaction to the mempool in addUnchecked(), we assume it - // has no children, and in the case of a reorg where that assumption is - // false, the in-mempool children aren't linked to the in-block tx's - // until UpdateTransactionsFromBlock() is called. + // of ancestors reachable via GetMemPoolParents()/GetMemPoolChildren() + // will be the same as the set of ancestors whose packages include this + // transaction, because when we add a new transaction to the mempool in + // addUnchecked(), we assume it has no children, and in the case of a + // reorg where that assumption is false, the in-mempool children aren't + // linked to the in-block tx's until UpdateTransactionsFromBlock() is + // called. // So if we're being called during a reorg, ie before - // UpdateTransactionsFromBlock() has been called, then mapLinks[] will - // differ from the set of mempool parents we'd calculate by searching, - // and it's important that we use the mapLinks[] notion of ancestor - // transactions as the set of things to update for removal. + // UpdateTransactionsFromBlock() has been called, then + // GetMemPoolParents()/GetMemPoolChildren() will differ from the set of + // mempool parents we'd calculate by searching, and it's important that + // we use the cached notion of ancestor transactions as the set of + // things to update for removal. CalculateMemPoolAncestors(entry, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false); // Note that UpdateAncestorsOf severs the child links that point to // removeIt in the entries for the parents of removeIt. UpdateAncestorsOf(false, removeIt, setAncestors); } // After updating all the ancestor sizes, we can now sever the link between each - // transaction being removed and any mempool children (ie, update setMemPoolParents + // transaction being removed and any mempool children (ie, update CTxMemPoolEntry::m_parents // for each direct child of a transaction being removed). for (txiter removeIt : entriesToRemove) { UpdateChildrenForRemoval(removeIt); @@ -359,7 +364,6 @@ void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAnces // Used by AcceptToMemoryPool(), which DOES do // all the appropriate checks. indexed_transaction_set::iterator newit = mapTx.insert(entry).first; - mapLinks.insert(make_pair(newit, TxLinks())); // Update transaction for any feeDelta created by PrioritiseTransaction // TODO: refactor so that the fee delta is calculated before inserting @@ -430,15 +434,14 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) totalTxSize -= it->GetTxSize(); cachedInnerUsage -= it->DynamicMemoryUsage(); - cachedInnerUsage -= memusage::DynamicUsage(mapLinks[it].parents) + memusage::DynamicUsage(mapLinks[it].children); - mapLinks.erase(it); + cachedInnerUsage -= memusage::DynamicUsage(it->GetMemPoolParentsConst()) + memusage::DynamicUsage(it->GetMemPoolChildrenConst()); mapTx.erase(it); nTransactionsUpdated++; if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash, false);} } // Calculates descendants of entry that are not already in setDescendants, and adds to -// setDescendants. Assumes entryit is already a tx in the mempool and setMemPoolChildren +// setDescendants. Assumes entryit is already a tx in the mempool and CTxMemPoolEntry::m_children // is correct for tx and all descendants. // Also assumes that if an entry is in setDescendants already, then all // in-mempool descendants of it are already in setDescendants as well, so that we @@ -457,8 +460,9 @@ void CTxMemPool::CalculateDescendants(txiter entryit, setEntries& setDescendants setDescendants.insert(it); stage.erase(it); - const setEntries &setChildren = GetMemPoolChildren(it); - for (txiter childiter : setChildren) { + const CTxMemPoolEntry::Children& children = it->GetMemPoolChildrenConst(); + for (const CTxMemPoolEntry& child : children) { + txiter childiter = mapTx.iterator_to(child); if (!setDescendants.count(childiter)) { stage.insert(childiter); } @@ -584,7 +588,6 @@ void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigne void CTxMemPool::_clear() { - mapLinks.clear(); mapTx.clear(); mapNextTx.clear(); totalTxSize = 0; @@ -633,12 +636,9 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const checkTotal += it->GetTxSize(); innerUsage += it->DynamicMemoryUsage(); const CTransaction& tx = it->GetTx(); - txlinksMap::const_iterator linksiter = mapLinks.find(it); - assert(linksiter != mapLinks.end()); - const TxLinks &links = linksiter->second; - innerUsage += memusage::DynamicUsage(links.parents) + memusage::DynamicUsage(links.children); + innerUsage += memusage::DynamicUsage(it->GetMemPoolParentsConst()) + memusage::DynamicUsage(it->GetMemPoolChildrenConst()); bool fDependsWait = false; - setEntries setParentCheck; + CTxMemPoolEntry::Parents setParentCheck; for (const CTxIn &txin : tx.vin) { // Check that every mempool transaction's inputs refer to available coins, or other mempool tx's. indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); @@ -646,7 +646,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const const CTransaction& tx2 = it2->GetTx(); assert(tx2.vout.size() > txin.prevout.n && !tx2.vout[txin.prevout.n].IsNull()); fDependsWait = true; - setParentCheck.insert(it2); + setParentCheck.insert(*it2); } else { assert(pcoins->HaveCoin(txin.prevout)); } @@ -657,7 +657,11 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const assert(it3->second == &tx); i++; } - assert(setParentCheck == GetMemPoolParents(it)); + auto comp = [](const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) -> bool { + return a.GetTx().GetHash() == b.GetTx().GetHash(); + }; + assert(setParentCheck.size() == it->GetMemPoolParentsConst().size()); + assert(std::equal(setParentCheck.begin(), setParentCheck.end(), it->GetMemPoolParentsConst().begin(), comp)); // Verify ancestor state is correct. setEntries setAncestors; uint64_t nNoLimit = std::numeric_limits<uint64_t>::max(); @@ -680,17 +684,18 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const assert(it->GetModFeesWithAncestors() == nFeesCheck); // Check children against mapNextTx - CTxMemPool::setEntries setChildrenCheck; + CTxMemPoolEntry::Children setChildrenCheck; auto iter = mapNextTx.lower_bound(COutPoint(it->GetTx().GetHash(), 0)); uint64_t child_sizes = 0; for (; iter != mapNextTx.end() && iter->first->hash == it->GetTx().GetHash(); ++iter) { txiter childit = mapTx.find(iter->second->GetHash()); assert(childit != mapTx.end()); // mapNextTx points to in-mempool transactions - if (setChildrenCheck.insert(childit).second) { + if (setChildrenCheck.insert(*childit).second) { child_sizes += childit->GetTxSize(); } } - assert(setChildrenCheck == GetMemPoolChildren(it)); + assert(setChildrenCheck.size() == it->GetMemPoolChildrenConst().size()); + assert(std::equal(setChildrenCheck.begin(), setChildrenCheck.end(), it->GetMemPoolChildrenConst().begin(), comp)); // Also check to make sure size is greater than sum with immediate children. // just a sanity check, not definitive that this calc is correct... assert(it->GetSizeWithDescendants() >= child_sizes + it->GetTxSize()); @@ -852,9 +857,9 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeD LogPrintf("PrioritiseTransaction: %s feerate += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); } -void CTxMemPool::ApplyDelta(const uint256 hash, CAmount &nFeeDelta) const +void CTxMemPool::ApplyDelta(const uint256& hash, CAmount &nFeeDelta) const { - LOCK(cs); + AssertLockHeld(cs); std::map<uint256, CAmount>::const_iterator pos = mapDeltas.find(hash); if (pos == mapDeltas.end()) return; @@ -862,9 +867,9 @@ void CTxMemPool::ApplyDelta(const uint256 hash, CAmount &nFeeDelta) const nFeeDelta += delta; } -void CTxMemPool::ClearPrioritisation(const uint256 hash) +void CTxMemPool::ClearPrioritisation(const uint256& hash) { - LOCK(cs); + AssertLockHeld(cs); mapDeltas.erase(hash); } @@ -920,7 +925,7 @@ bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); // Estimate the overhead of mapTx to be 15 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. - return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; + return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; } void CTxMemPool::RemoveUnbroadcastTx(const uint256& txid, const bool unchecked) { @@ -968,40 +973,26 @@ void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, bool validFeeEstimat void CTxMemPool::UpdateChild(txiter entry, txiter child, bool add) { - setEntries s; - if (add && mapLinks[entry].children.insert(child).second) { + AssertLockHeld(cs); + CTxMemPoolEntry::Children s; + if (add && entry->GetMemPoolChildren().insert(*child).second) { cachedInnerUsage += memusage::IncrementalDynamicUsage(s); - } else if (!add && mapLinks[entry].children.erase(child)) { + } else if (!add && entry->GetMemPoolChildren().erase(*child)) { cachedInnerUsage -= memusage::IncrementalDynamicUsage(s); } } void CTxMemPool::UpdateParent(txiter entry, txiter parent, bool add) { - setEntries s; - if (add && mapLinks[entry].parents.insert(parent).second) { + AssertLockHeld(cs); + CTxMemPoolEntry::Parents s; + if (add && entry->GetMemPoolParents().insert(*parent).second) { cachedInnerUsage += memusage::IncrementalDynamicUsage(s); - } else if (!add && mapLinks[entry].parents.erase(parent)) { + } else if (!add && entry->GetMemPoolParents().erase(*parent)) { cachedInnerUsage -= memusage::IncrementalDynamicUsage(s); } } -const CTxMemPool::setEntries & CTxMemPool::GetMemPoolParents(txiter entry) const -{ - assert (entry != mapTx.end()); - txlinksMap::const_iterator it = mapLinks.find(entry); - assert(it != mapLinks.end()); - return it->second.parents; -} - -const CTxMemPool::setEntries & CTxMemPool::GetMemPoolChildren(txiter entry) const -{ - assert (entry != mapTx.end()); - txlinksMap::const_iterator it = mapLinks.find(entry); - assert(it != mapLinks.end()); - return it->second.children; -} - CFeeRate CTxMemPool::GetMinFee(size_t sizelimit) const { LOCK(cs); if (!blockSinceLastRollingFeeBump || rollingMinimumFeeRate == 0) @@ -1087,12 +1078,12 @@ uint64_t CTxMemPool::CalculateDescendantMaximum(txiter entry) const { txiter candidate = candidates.back(); candidates.pop_back(); if (!counted.insert(candidate).second) continue; - const setEntries& parents = GetMemPoolParents(candidate); + const CTxMemPoolEntry::Parents& parents = candidate->GetMemPoolParentsConst(); if (parents.size() == 0) { maximum = std::max(maximum, candidate->GetCountWithDescendants()); } else { - for (txiter i : parents) { - candidates.push_back(i); + for (const CTxMemPoolEntry& i : parents) { + candidates.push_back(mapTx.iterator_to(i)); } } } diff --git a/src/txmempool.h b/src/txmempool.h index 4743e1b63..664fb5986 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -49,6 +49,20 @@ struct LockPoints LockPoints() : height(0), time(0), maxInputBlock(nullptr) { } }; +struct CompareIteratorByHash { + // SFINAE for T where T is either a pointer type (e.g., a txiter) or a reference_wrapper<T> + // (e.g. a wrapped CTxMemPoolEntry&) + template <typename T> + bool operator()(const std::reference_wrapper<T>& a, const std::reference_wrapper<T>& b) const + { + return a.get().GetTx().GetHash() < b.get().GetTx().GetHash(); + } + template <typename T> + bool operator()(const T& a, const T& b) const + { + return a->GetTx().GetHash() < b->GetTx().GetHash(); + } +}; /** \class CTxMemPoolEntry * * CTxMemPoolEntry stores data about the corresponding transaction, as well @@ -63,8 +77,16 @@ struct LockPoints class CTxMemPoolEntry { +public: + typedef std::reference_wrapper<const CTxMemPoolEntry> CTxMemPoolEntryRef; + // two aliases, should the types ever diverge + typedef std::set<CTxMemPoolEntryRef, CompareIteratorByHash> Parents; + typedef std::set<CTxMemPoolEntryRef, CompareIteratorByHash> Children; + private: const CTransactionRef tx; + mutable Parents m_parents; + mutable Children m_children; const CAmount nFee; //!< Cached to avoid expensive parent-transaction lookups const size_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) const size_t nUsageSize; //!< ... and total memory usage @@ -127,6 +149,11 @@ public: CAmount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } int64_t GetSigOpCostWithAncestors() const { return nSigOpCostWithAncestors; } + const Parents& GetMemPoolParentsConst() const { return m_parents; } + const Children& GetMemPoolChildrenConst() const { return m_children; } + Parents& GetMemPoolParents() const { return m_parents; } + Children& GetMemPoolChildren() const { return m_children; } + mutable size_t vTxHashesIdx; //!< Index in mempool's vTxHashes mutable uint64_t m_epoch; //!< epoch when last touched, useful for graph algorithms }; @@ -547,37 +574,22 @@ public: using txiter = indexed_transaction_set::nth_index<0>::type::const_iterator; std::vector<std::pair<uint256, txiter>> vTxHashes GUARDED_BY(cs); //!< All tx witness hashes/entries in mapTx, in random order - struct CompareIteratorByHash { - bool operator()(const txiter &a, const txiter &b) const { - return a->GetTx().GetHash() < b->GetTx().GetHash(); - } - }; typedef std::set<txiter, CompareIteratorByHash> setEntries; - const setEntries & GetMemPoolParents(txiter entry) const EXCLUSIVE_LOCKS_REQUIRED(cs); - const setEntries & GetMemPoolChildren(txiter entry) const EXCLUSIVE_LOCKS_REQUIRED(cs); uint64_t CalculateDescendantMaximum(txiter entry) const EXCLUSIVE_LOCKS_REQUIRED(cs); private: typedef std::map<txiter, setEntries, CompareIteratorByHash> cacheMap; - struct TxLinks { - setEntries parents; - setEntries children; - }; - - typedef std::map<txiter, TxLinks, CompareIteratorByHash> txlinksMap; - txlinksMap mapLinks; - void UpdateParent(txiter entry, txiter parent, bool add); - void UpdateChild(txiter entry, txiter child, bool add); + void UpdateParent(txiter entry, txiter parent, bool add) EXCLUSIVE_LOCKS_REQUIRED(cs); + void UpdateChild(txiter entry, txiter child, bool add) EXCLUSIVE_LOCKS_REQUIRED(cs); std::vector<indexed_transaction_set::const_iterator> GetSortedDepthAndScore() const EXCLUSIVE_LOCKS_REQUIRED(cs); /** - * track locally submitted transactions to periodically retry initial broadcast - * map of txid -> wtxid + * Track locally submitted transactions to periodically retry initial broadcast. */ - std::map<uint256, uint256> m_unbroadcast_txids GUARDED_BY(cs); + std::set<uint256> m_unbroadcast_txids GUARDED_BY(cs); public: indirectmap<COutPoint, const CTransaction*> mapNextTx GUARDED_BY(cs); @@ -626,8 +638,8 @@ public: /** Affect CreateNewBlock prioritisation of transactions */ void PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta); - void ApplyDelta(const uint256 hash, CAmount &nFeeDelta) const; - void ClearPrioritisation(const uint256 hash); + void ApplyDelta(const uint256& hash, CAmount &nFeeDelta) const EXCLUSIVE_LOCKS_REQUIRED(cs); + void ClearPrioritisation(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs); /** Get the transaction in the pool that spends the same prevout */ const CTransaction* GetConflictTx(const COutPoint& prevout) const EXCLUSIVE_LOCKS_REQUIRED(cs); @@ -710,9 +722,9 @@ public: return mapTx.size(); } - uint64_t GetTotalTxSize() const + uint64_t GetTotalTxSize() const EXCLUSIVE_LOCKS_REQUIRED(cs) { - LOCK(cs); + AssertLockHeld(cs); return totalTxSize; } @@ -739,27 +751,29 @@ public: size_t DynamicMemoryUsage() const; /** Adds a transaction to the unbroadcast set */ - void AddUnbroadcastTx(const uint256& txid, const uint256& wtxid) { + void AddUnbroadcastTx(const uint256& txid) + { LOCK(cs); - // Sanity Check: the transaction should also be in the mempool - if (exists(txid)) { - m_unbroadcast_txids[txid] = wtxid; - } - } + // Sanity check the transaction is in the mempool & insert into + // unbroadcast set. + if (exists(txid)) m_unbroadcast_txids.insert(txid); + }; /** Removes a transaction from the unbroadcast set */ void RemoveUnbroadcastTx(const uint256& txid, const bool unchecked = false); /** Returns transactions in unbroadcast set */ - std::map<uint256, uint256> GetUnbroadcastTxs() const { + std::set<uint256> GetUnbroadcastTxs() const + { LOCK(cs); return m_unbroadcast_txids; } /** Returns whether a txid is in the unbroadcast set */ - bool IsUnbroadcastTx(const uint256& txid) const { - LOCK(cs); - return (m_unbroadcast_txids.count(txid) != 0); + bool IsUnbroadcastTx(const uint256& txid) const EXCLUSIVE_LOCKS_REQUIRED(cs) + { + AssertLockHeld(cs); + return m_unbroadcast_txids.count(txid) != 0; } private: diff --git a/src/uint256.cpp b/src/uint256.cpp index a5dfba41e..ee1b34ead 100644 --- a/src/uint256.cpp +++ b/src/uint256.cpp @@ -19,7 +19,11 @@ base_blob<BITS>::base_blob(const std::vector<unsigned char>& vch) template <unsigned int BITS> std::string base_blob<BITS>::GetHex() const { - return HexStr(std::reverse_iterator<const uint8_t*>(m_data + sizeof(m_data)), std::reverse_iterator<const uint8_t*>(m_data)); + uint8_t m_data_rev[WIDTH]; + for (int i = 0; i < WIDTH; ++i) { + m_data_rev[i] = m_data[WIDTH - 1 - i]; + } + return HexStr(m_data_rev); } template <unsigned int BITS> diff --git a/src/util/message.cpp b/src/util/message.cpp index 1e7128d22..e1d5cff48 100644 --- a/src/util/message.cpp +++ b/src/util/message.cpp @@ -64,7 +64,7 @@ bool MessageSign( return false; } - signature = EncodeBase64(signature_bytes.data(), signature_bytes.size()); + signature = EncodeBase64(signature_bytes); return true; } diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 3a903b689..079a4529a 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -126,20 +126,20 @@ void SplitHostPort(std::string in, int &portOut, std::string &hostOut) { hostOut = in; } -std::string EncodeBase64(const unsigned char* pch, size_t len) +std::string EncodeBase64(Span<const unsigned char> input) { static const char *pbase64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; std::string str; - str.reserve(((len + 2) / 3) * 4); - ConvertBits<8, 6, true>([&](int v) { str += pbase64[v]; }, pch, pch + len); + str.reserve(((input.size() + 2) / 3) * 4); + ConvertBits<8, 6, true>([&](int v) { str += pbase64[v]; }, input.begin(), input.end()); while (str.size() % 4) str += '='; return str; } std::string EncodeBase64(const std::string& str) { - return EncodeBase64((const unsigned char*)str.data(), str.size()); + return EncodeBase64(MakeUCharSpan(str)); } std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid) @@ -201,20 +201,20 @@ std::string DecodeBase64(const std::string& str, bool* pf_invalid) return std::string((const char*)vchRet.data(), vchRet.size()); } -std::string EncodeBase32(const unsigned char* pch, size_t len) +std::string EncodeBase32(Span<const unsigned char> input) { static const char *pbase32 = "abcdefghijklmnopqrstuvwxyz234567"; std::string str; - str.reserve(((len + 4) / 5) * 8); - ConvertBits<8, 5, true>([&](int v) { str += pbase32[v]; }, pch, pch + len); + str.reserve(((input.size() + 4) / 5) * 8); + ConvertBits<8, 5, true>([&](int v) { str += pbase32[v]; }, input.begin(), input.end()); while (str.size() % 8) str += '='; return str; } std::string EncodeBase32(const std::string& str) { - return EncodeBase32((const unsigned char*)str.data(), str.size()); + return EncodeBase32(MakeUCharSpan(str)); } std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid) @@ -318,6 +318,18 @@ bool ParseInt64(const std::string& str, int64_t *out) n <= std::numeric_limits<int64_t>::max(); } +bool ParseUInt8(const std::string& str, uint8_t *out) +{ + uint32_t u32; + if (!ParseUInt32(str, &u32) || u32 > std::numeric_limits<uint8_t>::max()) { + return false; + } + if (out != nullptr) { + *out = static_cast<uint8_t>(u32); + } + return true; +} + bool ParseUInt32(const std::string& str, uint32_t *out) { if (!ParsePrechecks(str)) @@ -407,15 +419,6 @@ std::string FormatParagraph(const std::string& in, size_t width, size_t indent) return out.str(); } -int64_t atoi64(const char* psz) -{ -#ifdef _MSC_VER - return _atoi64(psz); -#else - return strtoll(psz, nullptr, 10); -#endif -} - int64_t atoi64(const std::string& str) { #ifdef _MSC_VER @@ -569,3 +572,16 @@ std::string Capitalize(std::string str) str[0] = ToUpper(str.front()); return str; } + +std::string HexStr(const Span<const uint8_t> s) +{ + std::string rv; + static constexpr char hexmap[16] = { '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; + rv.reserve(s.size() * 2); + for (uint8_t v: s) { + rv.push_back(hexmap[v >> 4]); + rv.push_back(hexmap[v & 15]); + } + return rv; +} diff --git a/src/util/strencodings.h b/src/util/strencodings.h index bd988f141..151921414 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -10,6 +10,7 @@ #define BITCOIN_UTIL_STRENCODINGS_H #include <attributes.h> +#include <span.h> #include <cstdint> #include <iterator> @@ -47,15 +48,14 @@ bool IsHex(const std::string& str); bool IsHexNumber(const std::string& str); std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid = nullptr); std::string DecodeBase64(const std::string& str, bool* pf_invalid = nullptr); -std::string EncodeBase64(const unsigned char* pch, size_t len); +std::string EncodeBase64(Span<const unsigned char> input); std::string EncodeBase64(const std::string& str); std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid = nullptr); std::string DecodeBase32(const std::string& str, bool* pf_invalid = nullptr); -std::string EncodeBase32(const unsigned char* pch, size_t len); +std::string EncodeBase32(Span<const unsigned char> input); std::string EncodeBase32(const std::string& str); void SplitHostPort(std::string in, int& portOut, std::string& hostOut); -int64_t atoi64(const char* psz); int64_t atoi64(const std::string& str); int atoi(const std::string& str); @@ -99,6 +99,13 @@ NODISCARD bool ParseInt32(const std::string& str, int32_t *out); NODISCARD bool ParseInt64(const std::string& str, int64_t *out); /** + * Convert decimal string to unsigned 8-bit integer with strict parse error feedback. + * @returns true if the entire string could be parsed as valid integer, + * false if not the entire string could be parsed or when overflow or underflow occurred. + */ +NODISCARD bool ParseUInt8(const std::string& str, uint8_t *out); + +/** * Convert decimal string to unsigned 32-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. @@ -119,27 +126,11 @@ NODISCARD bool ParseUInt64(const std::string& str, uint64_t *out); */ NODISCARD bool ParseDouble(const std::string& str, double *out); -template<typename T> -std::string HexStr(const T itbegin, const T itend) -{ - std::string rv; - static const char hexmap[16] = { '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; - rv.reserve(std::distance(itbegin, itend) * 2); - for(T it = itbegin; it < itend; ++it) - { - unsigned char val = (unsigned char)(*it); - rv.push_back(hexmap[val>>4]); - rv.push_back(hexmap[val&15]); - } - return rv; -} - -template<typename T> -inline std::string HexStr(const T& vch) -{ - return HexStr(vch.begin(), vch.end()); -} +/** + * Convert a span of bytes to a lower-case hexadecimal string. + */ +std::string HexStr(const Span<const uint8_t> s); +inline std::string HexStr(const Span<const char> s) { return HexStr(MakeUCharSpan(s)); } /** * Format a paragraph of text to a fixed width, adding spaces for diff --git a/src/util/system.cpp b/src/util/system.cpp index 8164e884b..999937d90 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -6,6 +6,10 @@ #include <sync.h> #include <util/system.h> +#ifdef HAVE_BOOST_PROCESS +#include <boost/process.hpp> +#endif // HAVE_BOOST_PROCESS + #include <chainparamsbase.h> #include <util/strencodings.h> #include <util/string.h> @@ -44,12 +48,6 @@ #pragma warning(disable:4717) #endif -#ifdef _WIN32_IE -#undef _WIN32_IE -#endif -#define _WIN32_IE 0x0501 - -#define WIN32_LEAN_AND_MEAN 1 #ifndef NOMINMAX #define NOMINMAX #endif @@ -1021,7 +1019,7 @@ bool FileCommit(FILE *file) return false; } #else - #if defined(__linux__) || defined(__NetBSD__) + #if HAVE_FDATASYNC if (fdatasync(fileno(file)) != 0 && errno != EINVAL) { // Ignore EINVAL for filesystems that don't support sync LogPrintf("%s: fdatasync failed: %d\n", __func__, errno); return false; @@ -1161,6 +1159,43 @@ void runCommand(const std::string& strCommand) } #endif +#ifdef HAVE_BOOST_PROCESS +UniValue RunCommandParseJSON(const std::string& str_command, const std::string& str_std_in) +{ + namespace bp = boost::process; + + UniValue result_json; + bp::opstream stdin_stream; + bp::ipstream stdout_stream; + bp::ipstream stderr_stream; + + if (str_command.empty()) return UniValue::VNULL; + + bp::child c( + str_command, + bp::std_out > stdout_stream, + bp::std_err > stderr_stream, + bp::std_in < stdin_stream + ); + if (!str_std_in.empty()) { + stdin_stream << str_std_in << std::endl; + } + stdin_stream.pipe().close(); + + std::string result; + std::string error; + std::getline(stdout_stream, result); + std::getline(stderr_stream, error); + + c.wait(); + const int n_error = c.exit_code(); + if (n_error) throw std::runtime_error(strprintf("RunCommandParseJSON error: process(%s) returned %d: %s\n", str_command, n_error, error)); + if (!result_json.read(result)) throw std::runtime_error("Unable to parse JSON: " + result); + + return result_json; +} +#endif // HAVE_BOOST_PROCESS + void SetupEnvironment() { #ifdef HAVE_MALLOPT_ARENA_MAX diff --git a/src/util/system.h b/src/util/system.h index 0bd14cc9e..1df194ca8 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -37,6 +37,8 @@ #include <boost/thread/condition_variable.hpp> // for boost::thread_interrupted +class UniValue; + // Application startup time (used for uptime calculation) int64_t GetStartupTime(); @@ -96,6 +98,16 @@ std::string ShellEscape(const std::string& arg); #if HAVE_SYSTEM void runCommand(const std::string& strCommand); #endif +#ifdef HAVE_BOOST_PROCESS +/** + * Execute a command which returns JSON, and parse the result. + * + * @param str_command The command to execute, including any arguments + * @param str_std_in string to pass to stdin + * @return parsed JSON + */ +UniValue RunCommandParseJSON(const std::string& str_command, const std::string& str_std_in=""); +#endif // HAVE_BOOST_PROCESS /** * Most paths passed as configuration arguments are treated as relative to diff --git a/src/validation.cpp b/src/validation.cpp index 81af97277..0823c6b12 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -148,7 +148,6 @@ arith_uint256 nMinimumChainWork; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CBlockPolicyEstimator feeEstimator; -CTxMemPool mempool(&feeEstimator); // Internal stuff namespace { @@ -370,9 +369,10 @@ static bool IsCurrentForFeeEstimation() EXCLUSIVE_LOCKS_REQUIRED(cs_main) * and instead just erase from the mempool as needed. */ -static void UpdateMempoolForReorg(DisconnectedBlockTransactions& disconnectpool, bool fAddToMempool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs) +static void UpdateMempoolForReorg(CTxMemPool& mempool, DisconnectedBlockTransactions& disconnectpool, bool fAddToMempool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, mempool.cs) { AssertLockHeld(cs_main); + AssertLockHeld(mempool.cs); std::vector<uint256> vHashUpdate; // disconnectpool's insertion_order index sorts the entries from // oldest to newest, but the oldest entry will be the last tx from the @@ -475,6 +475,7 @@ public: */ std::vector<COutPoint>& m_coins_to_uncache; const bool m_test_accept; + CAmount* m_fee_out; }; // Single transaction acceptance @@ -687,9 +688,15 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) return false; // state filled in by CheckTxInputs } + // If fee_out is passed, return the fee to the caller + if (args.m_fee_out) { + *args.m_fee_out = nFees; + } + // Check for non-standard pay-to-script-hash in inputs - if (fRequireStandard && !AreInputsStandard(tx, m_view)) - return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "bad-txns-nonstandard-inputs"); + if (fRequireStandard && !AreInputsStandard(tx, m_view)) { + return state.Invalid(TxValidationResult::TX_INPUTS_NOT_STANDARD, "bad-txns-nonstandard-inputs"); + } // Check for non-standard witness in P2WSH if (tx.HasWitness() && fRequireStandard && !IsWitnessStandard(tx, m_view)) @@ -1060,10 +1067,10 @@ bool MemPoolAccept::AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs /** (try to) add transaction to memory pool with a specified acceptance time **/ static bool AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, - bool bypass_limits, const CAmount nAbsurdFee, bool test_accept) EXCLUSIVE_LOCKS_REQUIRED(cs_main) + bool bypass_limits, const CAmount nAbsurdFee, bool test_accept, CAmount* fee_out=nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { std::vector<COutPoint> coins_to_uncache; - MemPoolAccept::ATMPArgs args { chainparams, state, nAcceptTime, plTxnReplaced, bypass_limits, nAbsurdFee, coins_to_uncache, test_accept }; + MemPoolAccept::ATMPArgs args { chainparams, state, nAcceptTime, plTxnReplaced, bypass_limits, nAbsurdFee, coins_to_uncache, test_accept, fee_out }; bool res = MemPoolAccept(pool).AcceptSingleTransaction(tx, args); if (!res) { // Remove coins that were not present in the coins cache before calling ATMPW; @@ -1082,10 +1089,10 @@ static bool AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPo bool AcceptToMemoryPool(CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, std::list<CTransactionRef>* plTxnReplaced, - bool bypass_limits, const CAmount nAbsurdFee, bool test_accept) + bool bypass_limits, const CAmount nAbsurdFee, bool test_accept, CAmount* fee_out) { const CChainParams& chainparams = Params(); - return AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, GetTime(), plTxnReplaced, bypass_limits, nAbsurdFee, test_accept); + return AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, GetTime(), plTxnReplaced, bypass_limits, nAbsurdFee, test_accept, fee_out); } CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock) @@ -1198,8 +1205,8 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, c if (memcmp(blk_start, message_start, CMessageHeader::MESSAGE_START_SIZE)) { return error("%s: Block magic mismatch for %s: %s versus expected %s", __func__, pos.ToString(), - HexStr(blk_start, blk_start + CMessageHeader::MESSAGE_START_SIZE), - HexStr(message_start, message_start + CMessageHeader::MESSAGE_START_SIZE)); + HexStr(blk_start), + HexStr(message_start)); } if (blk_size > MAX_SIZE) { @@ -1253,8 +1260,9 @@ void CoinsViews::InitCache() m_cacheview = MakeUnique<CCoinsViewCache>(&m_catcherview); } -CChainState::CChainState(BlockManager& blockman, uint256 from_snapshot_blockhash) +CChainState::CChainState(CTxMemPool& mempool, BlockManager& blockman, uint256 from_snapshot_blockhash) : m_blockman(blockman), + m_mempool(mempool), m_from_snapshot_blockhash(from_snapshot_blockhash) {} void CChainState::InitCoinsDB( @@ -2279,7 +2287,7 @@ bool CChainState::FlushStateToDisk( { bool fFlushForPrune = false; bool fDoFullFlush = false; - CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(&::mempool); + CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(&m_mempool); LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { if (nManualPruneHeight > 0) { @@ -2425,7 +2433,7 @@ static void AppendWarning(bilingual_str& res, const bilingual_str& warn) } /** Check warning conditions and do some notifications on new chain tip set. */ -void static UpdateTip(const CBlockIndex* pindexNew, const CChainParams& chainParams) +static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const CChainParams& chainParams) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { // New best block @@ -2471,7 +2479,6 @@ void static UpdateTip(const CBlockIndex* pindexNew, const CChainParams& chainPar FormatISO8601DateTime(pindexNew->GetBlockTime()), GuessVerificationProgress(chainParams.TxData(), pindexNew), ::ChainstateActive().CoinsTip().DynamicMemoryUsage() * (1.0 / (1<<20)), ::ChainstateActive().CoinsTip().GetCacheSize(), !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages.original) : ""); - } /** Disconnect m_chain's tip. @@ -2484,8 +2491,11 @@ void static UpdateTip(const CBlockIndex* pindexNew, const CChainParams& chainPar * disconnectpool (note that the caller is responsible for mempool consistency * in any case). */ -bool CChainState::DisconnectTip(BlockValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool) +bool CChainState::DisconnectTip(BlockValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions* disconnectpool) { + AssertLockHeld(cs_main); + AssertLockHeld(m_mempool.cs); + CBlockIndex *pindexDelete = m_chain.Tip(); assert(pindexDelete); // Read block from disk. @@ -2516,14 +2526,14 @@ bool CChainState::DisconnectTip(BlockValidationState& state, const CChainParams& while (disconnectpool->DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE * 1000) { // Drop the earliest entry, and remove its children from the mempool. auto it = disconnectpool->queuedTx.get<insertion_order>().begin(); - mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + m_mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); disconnectpool->removeEntry(it); } } m_chain.SetTip(pindexDelete->pprev); - UpdateTip(pindexDelete->pprev, chainparams); + UpdateTip(m_mempool, pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: GetMainSignals().BlockDisconnected(pblock, pindexDelete); @@ -2584,6 +2594,9 @@ public: */ bool CChainState::ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) { + AssertLockHeld(cs_main); + AssertLockHeld(m_mempool.cs); + assert(pindexNew->pprev == m_chain.Tip()); // Read block from disk. int64_t nTime1 = GetTimeMicros(); @@ -2624,11 +2637,11 @@ bool CChainState::ConnectTip(BlockValidationState& state, const CChainParams& ch int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime5 - nTime4) * MILLI, nTimeChainState * MICRO, nTimeChainState * MILLI / nBlocksTotal); // Remove conflicting transactions from the mempool.; - mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); + m_mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); disconnectpool.removeForBlock(blockConnecting.vtx); // Update m_chain & related variables. m_chain.SetTip(pindexNew); - UpdateTip(pindexNew, chainparams); + UpdateTip(m_mempool, pindexNew, chainparams); int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; LogPrint(BCLog::BENCH, " - Connect postprocess: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime6 - nTime5) * MILLI, nTimePostConnect * MICRO, nTimePostConnect * MILLI / nBlocksTotal); @@ -2718,6 +2731,7 @@ void CChainState::PruneBlockIndexCandidates() { bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) { AssertLockHeld(cs_main); + AssertLockHeld(m_mempool.cs); const CBlockIndex *pindexOldTip = m_chain.Tip(); const CBlockIndex *pindexFork = m_chain.FindFork(pindexMostWork); @@ -2729,7 +2743,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai if (!DisconnectTip(state, chainparams, &disconnectpool)) { // This is likely a fatal error, but keep the mempool consistent, // just in case. Only remove from the mempool in this case. - UpdateMempoolForReorg(disconnectpool, false); + UpdateMempoolForReorg(m_mempool, disconnectpool, false); // If we're unable to disconnect a block during normal operation, // then that is a failure of our local system -- we should abort @@ -2773,7 +2787,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai // A system error occurred (disk space, database error, ...). // Make the mempool consistent with the current tip, just in case // any observers try to use it before shutdown. - UpdateMempoolForReorg(disconnectpool, false); + UpdateMempoolForReorg(m_mempool, disconnectpool, false); return false; } } else { @@ -2790,9 +2804,9 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai if (fBlocksDisconnected) { // If any blocks were disconnected, disconnectpool may be non empty. Add // any disconnected transactions back to the mempool. - UpdateMempoolForReorg(disconnectpool, true); + UpdateMempoolForReorg(m_mempool, disconnectpool, true); } - mempool.check(&CoinsTip()); + m_mempool.check(&CoinsTip()); // Callbacks/notifications for a new best chain. if (fInvalidFound) @@ -2866,7 +2880,8 @@ bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainPar LimitValidationInterfaceQueue(); { - LOCK2(cs_main, ::mempool.cs); // Lock transaction pool for at least as long as it takes for connectTrace to be consumed + LOCK(cs_main); + LOCK(m_mempool.cs); // Lock transaction pool for at least as long as it takes for connectTrace to be consumed CBlockIndex* starting_tip = m_chain.Tip(); bool blocks_connected = false; do { @@ -3019,7 +3034,7 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, const CChainParam LimitValidationInterfaceQueue(); LOCK(cs_main); - LOCK(::mempool.cs); // Lock for as long as disconnectpool is in scope to make sure UpdateMempoolForReorg is called after DisconnectTip without unlocking in between + LOCK(m_mempool.cs); // Lock for as long as disconnectpool is in scope to make sure UpdateMempoolForReorg is called after DisconnectTip without unlocking in between if (!m_chain.Contains(pindex)) break; pindex_was_in_chain = true; CBlockIndex *invalid_walk_tip = m_chain.Tip(); @@ -3033,7 +3048,7 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, const CChainParam // transactions back to the mempool if disconnecting was successful, // and we're not doing a very deep invalidation (in which case // keeping the mempool up to date is probably futile anyway). - UpdateMempoolForReorg(disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); + UpdateMempoolForReorg(m_mempool, disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); if (!ret) return false; assert(invalid_walk_tip->pprev == m_chain.Tip()); @@ -4217,6 +4232,14 @@ bool static LoadBlockIndexDB(ChainstateManager& chainman, const CChainParams& ch return true; } +void CChainState::LoadMempool(const ArgsManager& args) +{ + if (args.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { + ::LoadMempool(m_mempool); + } + m_mempool.SetIsLoaded(!ShutdownRequested()); +} + bool CChainState::LoadChainTip(const CChainParams& chainparams) { AssertLockHeld(cs_main); @@ -4516,7 +4539,8 @@ bool CChainState::RewindBlockIndex(const CChainParams& params) // Loop until the tip is below nHeight, or we reach a pruned block. while (!ShutdownRequested()) { { - LOCK2(cs_main, ::mempool.cs); + LOCK(cs_main); + LOCK(m_mempool.cs); // Make sure nothing changed from under us (this won't happen because RewindBlockIndex runs before importing/network are active) assert(tip == m_chain.Tip()); if (tip == nullptr || tip->nHeight < nHeight) break; @@ -5105,7 +5129,7 @@ bool LoadMempool(CTxMemPool& pool) } // TODO: remove this try except in v0.22 - std::map<uint256, uint256> unbroadcast_txids; + std::set<uint256> unbroadcast_txids; try { file >> unbroadcast_txids; unbroadcast = unbroadcast_txids.size(); @@ -5113,13 +5137,10 @@ bool LoadMempool(CTxMemPool& pool) // mempool.dat files created prior to v0.21 will not have an // unbroadcast set. No need to log a failure if parsing fails here. } - for (const auto& elem : unbroadcast_txids) { - // Don't add unbroadcast transactions that didn't get back into the - // mempool. - const CTransactionRef& added_tx = pool.get(elem.first); - if (added_tx != nullptr) { - pool.AddUnbroadcastTx(elem.first, added_tx->GetWitnessHash()); - } + for (const auto& txid : unbroadcast_txids) { + // Ensure transactions were accepted to mempool then add to + // unbroadcast set. + if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); } } catch (const std::exception& e) { LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); @@ -5136,7 +5157,7 @@ bool DumpMempool(const CTxMemPool& pool) std::map<uint256, CAmount> mapDeltas; std::vector<TxMempoolInfo> vinfo; - std::map<uint256, uint256> unbroadcast_txids; + std::set<uint256> unbroadcast_txids; static Mutex dump_mutex; LOCK(dump_mutex); @@ -5245,7 +5266,7 @@ std::vector<CChainState*> ChainstateManager::GetAll() return out; } -CChainState& ChainstateManager::InitializeChainstate(const uint256& snapshot_blockhash) +CChainState& ChainstateManager::InitializeChainstate(CTxMemPool& mempool, const uint256& snapshot_blockhash) { bool is_snapshot = !snapshot_blockhash.IsNull(); std::unique_ptr<CChainState>& to_modify = @@ -5254,8 +5275,7 @@ CChainState& ChainstateManager::InitializeChainstate(const uint256& snapshot_blo if (to_modify) { throw std::logic_error("should not be overwriting a chainstate"); } - - to_modify.reset(new CChainState(m_blockman, snapshot_blockhash)); + to_modify.reset(new CChainState(mempool, m_blockman, snapshot_blockhash)); // Snapshot chainstates and initial IBD chaintates always become active. if (is_snapshot || (!is_snapshot && !m_active_chainstate)) { diff --git a/src/validation.h b/src/validation.h index 534162d64..0bc80e1ce 100644 --- a/src/validation.h +++ b/src/validation.h @@ -113,7 +113,6 @@ enum class SynchronizationState { extern RecursiveMutex cs_main; extern CBlockPolicyEstimator feeEstimator; -extern CTxMemPool mempool; typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern Mutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; @@ -200,10 +199,11 @@ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune); void PruneBlockFilesManual(int nManualPruneHeight); /** (try to) add transaction to memory pool - * plTxnReplaced will be appended to with all transactions replaced from mempool **/ + * plTxnReplaced will be appended to with all transactions replaced from mempool + * @param[out] fee_out optional argument to return tx fee to the caller **/ bool AcceptToMemoryPool(CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, std::list<CTransactionRef>* plTxnReplaced, - bool bypass_limits, const CAmount nAbsurdFee, bool test_accept=false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool bypass_limits, const CAmount nAbsurdFee, bool test_accept=false, CAmount* fee_out=nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Get the BIP9 state for a given deployment at the current tip. */ ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::DeploymentPos pos); @@ -245,7 +245,7 @@ bool TestLockPointValidity(const LockPoints* lp) EXCLUSIVE_LOCKS_REQUIRED(cs_mai * * See consensus/consensus.h for flag definitions. */ -bool CheckSequenceLocks(const CTxMemPool& pool, const CTransaction& tx, int flags, LockPoints* lp = nullptr, bool useExistingLockPoints = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +bool CheckSequenceLocks(const CTxMemPool& pool, const CTransaction& tx, int flags, LockPoints* lp = nullptr, bool useExistingLockPoints = false) EXCLUSIVE_LOCKS_REQUIRED(::cs_main, pool.cs); /** * Closure representing one script verification @@ -511,11 +511,14 @@ private: //! easily as opposed to referencing a global. BlockManager& m_blockman; + //! mempool that is kept in sync with the chain + CTxMemPool& m_mempool; + //! Manages the UTXO set, which is a reflection of the contents of `m_chain`. std::unique_ptr<CoinsViews> m_coins_views; public: - explicit CChainState(BlockManager& blockman, uint256 from_snapshot_blockhash = uint256()); + explicit CChainState(CTxMemPool& mempool, BlockManager& blockman, uint256 from_snapshot_blockhash = uint256()); /** * Initialize the CoinsViews UTXO set database management data structures. The in-memory @@ -642,7 +645,7 @@ public: CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Apply the effects of a block disconnection on the UTXO set. - bool DisconnectTip(BlockValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); + bool DisconnectTip(BlockValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool.cs); // Manual block validity manipulation: bool PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); @@ -668,6 +671,9 @@ public: */ void CheckBlockIndex(const Consensus::Params& consensusParams); + /** Load the persisted mempool from disk */ + void LoadMempool(const ArgsManager& args); + /** Update the chain tip based on database information, i.e. CoinsTip()'s best block. */ bool LoadChainTip(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -685,8 +691,8 @@ public: std::string ToString() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); private: - bool ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); - bool ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); + bool ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool.cs); + bool ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool.cs); void InvalidBlockFound(CBlockIndex *pindex, const BlockValidationState &state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -818,9 +824,11 @@ public: //! Instantiate a new chainstate and assign it based upon whether it is //! from a snapshot. //! + //! @param[in] mempool The mempool to pass to the chainstate + // constructor //! @param[in] snapshot_blockhash If given, signify that this chainstate //! is based on a snapshot. - CChainState& InitializeChainstate(const uint256& snapshot_blockhash = uint256()) + CChainState& InitializeChainstate(CTxMemPool& mempool, const uint256& snapshot_blockhash = uint256()) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); //! Get all chainstates currently being used. diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index a04311fdf..fbb3d2cac 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -38,7 +38,7 @@ void CheckUniqueFileid(const BerkeleyEnvironment& env, const std::string& filena for (const auto& item : env.m_fileids) { if (fileid == item.second && &fileid != &item.second) { throw std::runtime_error(strprintf("BerkeleyDatabase: Can't open database %s (duplicates fileid %s from %s)", filename, - HexStr(std::begin(item.second.value), std::end(item.second.value)), item.first)); + HexStr(item.second.value), item.first)); } } } @@ -52,18 +52,6 @@ bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const return memcmp(value, &rhs.value, sizeof(value)) == 0; } -bool IsBDBWalletLoaded(const fs::path& wallet_path) -{ - fs::path env_directory; - std::string database_filename; - SplitWalletPath(wallet_path, env_directory, database_filename); - LOCK(cs_db); - auto env = g_dbenvs.find(env_directory.string()); - if (env == g_dbenvs.end()) return false; - auto database = env->second.lock(); - return database && database->IsDatabaseLoaded(database_filename); -} - /** * @param[in] wallet_path Path to wallet directory. Or (for backwards compatibility only) a path to a berkeley btree data file inside a wallet directory. * @param[out] database_filename Filename of berkeley btree data file inside the wallet directory. @@ -371,7 +359,6 @@ void BerkeleyDatabase::Open(const char* pszMode) if (ret != 0) { throw std::runtime_error(strprintf("BerkeleyDatabase: Error %d, can't open database %s", ret, strFile)); } - m_file_path = (env->Directory() / strFile).string(); // Call CheckUniqueFileid on the containing BDB environment to // avoid BDB data consistency bugs that happen when different data @@ -824,3 +811,35 @@ std::unique_ptr<DatabaseBatch> BerkeleyDatabase::MakeBatch(const char* mode, boo { return MakeUnique<BerkeleyBatch>(*this, mode, flush_on_close); } + +bool ExistsBerkeleyDatabase(const fs::path& path) +{ + fs::path env_directory; + std::string data_filename; + SplitWalletPath(path, env_directory, data_filename); + return IsBerkeleyBtree(env_directory / data_filename); +} + +std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error) +{ + std::unique_ptr<BerkeleyDatabase> db; + { + LOCK(cs_db); // Lock env.m_databases until insert in BerkeleyDatabase constructor + std::string data_filename; + std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(path, data_filename); + if (env->m_databases.count(data_filename)) { + error = Untranslated(strprintf("Refusing to load database. Data file '%s' is already loaded.", (env->Directory() / data_filename).string())); + status = DatabaseStatus::FAILED_ALREADY_LOADED; + return nullptr; + } + db = MakeUnique<BerkeleyDatabase>(std::move(env), std::move(data_filename)); + } + + if (options.verify && !db->Verify(error)) { + status = DatabaseStatus::FAILED_VERIFY; + return nullptr; + } + + status = DatabaseStatus::SUCCESS; + return db; +} diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index 75546924e..fd5a49acc 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -63,7 +63,6 @@ public: bool IsMock() const { return fMockDb; } bool IsInitialized() const { return fDbEnvInit; } - bool IsDatabaseLoaded(const std::string& db_filename) const { return m_databases.find(db_filename) != m_databases.end(); } fs::path Directory() const { return strPath; } bool Open(bilingual_str& error); @@ -87,8 +86,8 @@ public: /** Get BerkeleyEnvironment and database filename given a wallet path. */ std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename); -/** Return whether a BDB wallet database is currently loaded. */ -bool IsBDBWalletLoaded(const fs::path& wallet_path); +/** Check format of database file */ +bool IsBerkeleyBtree(const fs::path& path); class BerkeleyBatch; @@ -143,7 +142,10 @@ public: void ReloadDbEnv() override; /** Verifies the environment and database file */ - bool Verify(bilingual_str& error) override; + bool Verify(bilingual_str& error); + + /** Return path to main database filename */ + std::string Filename() override { return (env->Directory() / strFile).string(); } /** * Pointer to shared database environment. @@ -224,4 +226,10 @@ public: std::string BerkeleyDatabaseVersion(); +//! Check if Berkeley database exists at specified path. +bool ExistsBerkeleyDatabase(const fs::path& path); + +//! Return object giving access to Berkeley database at specified path. +std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); + #endif // BITCOIN_WALLET_BDB_H diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 079a5d3d5..1a45a2b31 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -5,6 +5,7 @@ #include <wallet/coinselection.h> #include <optional.h> +#include <policy/feerate.h> #include <util/system.h> #include <util/moneystr.h> @@ -302,7 +303,7 @@ bool KnapsackSolver(const CAmount& nTargetValue, std::vector<OutputGroup>& group void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants) { m_outputs.push_back(output); m_from_me &= from_me; - m_value += output.effective_value; + m_value += output.txout.nValue; m_depth = std::min(m_depth, depth); // ancestors here express the number of ancestors the new coin will end up having, which is // the sum, rather than the max; this will overestimate in the cases where multiple inputs @@ -311,15 +312,19 @@ void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size // descendants is the count as seen from the top ancestor, not the descendants as seen from the // coin itself; thus, this value is counted as the max, not the sum m_descendants = std::max(m_descendants, descendants); - effective_value = m_value; + effective_value += output.effective_value; + fee += output.m_fee; + long_term_fee += output.m_long_term_fee; } std::vector<CInputCoin>::iterator OutputGroup::Discard(const CInputCoin& output) { auto it = m_outputs.begin(); while (it != m_outputs.end() && it->outpoint != output.outpoint) ++it; if (it == m_outputs.end()) return it; - m_value -= output.effective_value; + m_value -= output.txout.nValue; effective_value -= output.effective_value; + fee -= output.m_fee; + long_term_fee -= output.m_long_term_fee; return m_outputs.erase(it); } @@ -329,3 +334,35 @@ bool OutputGroup::EligibleForSpending(const CoinEligibilityFilter& eligibility_f && m_ancestors <= eligibility_filter.max_ancestors && m_descendants <= eligibility_filter.max_descendants; } + +void OutputGroup::SetFees(const CFeeRate effective_feerate, const CFeeRate long_term_feerate) +{ + fee = 0; + long_term_fee = 0; + effective_value = 0; + for (CInputCoin& coin : m_outputs) { + coin.m_fee = coin.m_input_bytes < 0 ? 0 : effective_feerate.GetFee(coin.m_input_bytes); + fee += coin.m_fee; + + coin.m_long_term_fee = coin.m_input_bytes < 0 ? 0 : long_term_feerate.GetFee(coin.m_input_bytes); + long_term_fee += coin.m_long_term_fee; + + coin.effective_value = coin.txout.nValue - coin.m_fee; + effective_value += coin.effective_value; + } +} + +OutputGroup OutputGroup::GetPositiveOnlyGroup() +{ + OutputGroup group(*this); + for (auto it = group.m_outputs.begin(); it != group.m_outputs.end(); ) { + const CInputCoin& coin = *it; + // Only include outputs that are positive effective value (i.e. not dust) + if (coin.effective_value <= 0) { + it = group.Discard(coin); + } else { + ++it; + } + } + return group; +} diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index 5348401f4..49c1134ec 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -9,6 +9,8 @@ #include <primitives/transaction.h> #include <random.h> +class CFeeRate; + //! target minimum change amount static constexpr CAmount MIN_CHANGE{COIN / 100}; //! final minimum change amount after paying for fees @@ -36,6 +38,8 @@ public: COutPoint outpoint; CTxOut txout; CAmount effective_value; + CAmount m_fee{0}; + CAmount m_long_term_fee{0}; /** Pre-computed estimated size of this output as a fully-signed input in a transaction. Can be -1 if it could not be calculated */ int m_input_bytes{-1}; @@ -91,6 +95,10 @@ struct OutputGroup void Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants); std::vector<CInputCoin>::iterator Discard(const CInputCoin& output); bool EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const; + + //! Update the OutputGroup's fee, long_term_fee, and effective_value based on the given feerates + void SetFees(const CFeeRate effective_feerate, const CFeeRate long_term_feerate); + OutputGroup GetPositiveOnlyGroup(); }; bool SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& target_value, const CAmount& cost_of_change, std::set<CInputCoin>& out_set, CAmount& value_ret, CAmount not_input_fees); diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 1eb82a03c..bd1d11473 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -23,11 +23,3 @@ void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std:: database_filename = "wallet.dat"; } } - -fs::path WalletDataFilePath(const fs::path& wallet_path) -{ - fs::path env_directory; - std::string database_filename; - SplitWalletPath(wallet_path, env_directory, database_filename); - return env_directory / database_filename; -} diff --git a/src/wallet/db.h b/src/wallet/db.h index 0afaba5fd..617ed4614 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -9,6 +9,7 @@ #include <clientversion.h> #include <fs.h> #include <streams.h> +#include <support/allocators/secure.h> #include <util/memory.h> #include <atomic> @@ -17,8 +18,6 @@ struct bilingual_str; -/** Given a wallet directory path or legacy file path, return path to main data file in the wallet database. */ -fs::path WalletDataFilePath(const fs::path& wallet_path); void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std::string& database_filename); /** RAII class that provides access to a WalletDatabase */ @@ -141,16 +140,14 @@ public: virtual void ReloadDbEnv() = 0; + /** Return path to main database file for logs and error messages. */ + virtual std::string Filename() = 0; + std::atomic<unsigned int> nUpdateCounter; unsigned int nLastSeen; unsigned int nLastFlushed; int64_t nLastWalletUpdate; - /** Verifies the environment and database file */ - virtual bool Verify(bilingual_str& error) = 0; - - std::string m_file_path; - /** Make a DatabaseBatch connected to this database */ virtual std::unique_ptr<DatabaseBatch> MakeBatch(const char* mode = "r+", bool flush_on_close = true) = 0; }; @@ -191,8 +188,35 @@ public: bool PeriodicFlush() override { return true; } void IncrementUpdateCounter() override { ++nUpdateCounter; } void ReloadDbEnv() override {} - bool Verify(bilingual_str& errorStr) override { return true; } + std::string Filename() override { return "dummy"; } std::unique_ptr<DatabaseBatch> MakeBatch(const char* mode = "r+", bool flush_on_close = true) override { return MakeUnique<DummyBatch>(); } }; +enum class DatabaseFormat { + BERKELEY, +}; + +struct DatabaseOptions { + bool require_existing = false; + bool require_create = false; + uint64_t create_flags = 0; + SecureString create_passphrase; + bool verify = true; +}; + +enum class DatabaseStatus { + SUCCESS, + FAILED_BAD_PATH, + FAILED_BAD_FORMAT, + FAILED_ALREADY_LOADED, + FAILED_ALREADY_EXISTS, + FAILED_NOT_FOUND, + FAILED_CREATE, + FAILED_LOAD, + FAILED_VERIFY, + FAILED_ENCRYPT, +}; + +std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); + #endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 52162ab52..5d8c4fba2 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -5,10 +5,12 @@ #include <init.h> #include <interfaces/chain.h> +#include <interfaces/wallet.h> #include <net.h> #include <node/context.h> #include <node/ui_interface.h> #include <outputtype.h> +#include <univalue.h> #include <util/check.h> #include <util/moneystr.h> #include <util/system.h> @@ -48,6 +50,7 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const argsman.AddArg("-fallbackfee=<amt>", strprintf("A fee rate (in %s/kB) that will be used when fee estimation has insufficient data. 0 to entirely disable the fallbackfee feature. (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_FALLBACK_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-keypool=<n>", strprintf("Set key pool size to <n> (default: %u). Warning: Smaller sizes may increase the risk of losing funds when restoring from an old backup, if none of the addresses in the original keypool have been used.", DEFAULT_KEYPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); + argsman.AddArg("-maxapsfee=<n>", strprintf("Spend up to this amount in additional (absolute) fees (in %s) if it allows the use of partial spend avoidance (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_MAX_AVOIDPARTIALSPEND_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction; setting this too low may abort large transactions (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-mintxfee=<amt>", strprintf("Fees (in %s/kB) smaller than this are considered zero fee for transaction creation (default: %s)", @@ -64,13 +67,13 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const argsman.AddArg("-walletnotify=<cmd>", "Execute command when a wallet transaction changes. %s in cmd is replaced by TxID and %w is replaced by wallet name. %w is not currently implemented on windows. On systems where %w is supported, it should NOT be quoted because this would break shell escaping used to invoke the command.", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); #endif argsman.AddArg("-walletrbf", strprintf("Send transactions with full-RBF opt-in enabled (RPC only, default: %u)", DEFAULT_WALLET_RBF), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - argsman.AddArg("-zapwallettxes=<mode>", "Delete all wallet transactions and only recover those parts of the blockchain through -rescan on startup" - " (1 = keep tx meta data e.g. payment request information, 2 = drop tx meta data)", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-dblogsize=<n>", strprintf("Flush wallet database activity from memory to disk log every <n> megabytes (default: %u)", DEFAULT_WALLET_DBLOGSIZE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", DEFAULT_FLUSHWALLET), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-privdb", strprintf("Sets the DB_PRIVATE flag in the wallet db environment (default: %u)", DEFAULT_WALLET_PRIVDB), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-walletrejectlongchains", strprintf("Wallet will not create transactions that violate mempool chain limits (default: %u)", DEFAULT_WALLET_REJECT_LONG_CHAINS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); + + argsman.AddHiddenArgs({"-zapwallettxes"}); } bool WalletInit::ParameterInteraction() const @@ -83,26 +86,12 @@ bool WalletInit::ParameterInteraction() const return true; } - const bool is_multiwallet = gArgs.GetArgs("-wallet").size() > 1; - if (gArgs.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY) && gArgs.SoftSetBoolArg("-walletbroadcast", false)) { LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -walletbroadcast=0\n", __func__); } - bool zapwallettxes = gArgs.GetBoolArg("-zapwallettxes", false); - // -zapwallettxes implies dropping the mempool on startup - if (zapwallettxes && gArgs.SoftSetBoolArg("-persistmempool", false)) { - LogPrintf("%s: parameter interaction: -zapwallettxes enabled -> setting -persistmempool=0\n", __func__); - } - - // -zapwallettxes implies a rescan - if (zapwallettxes) { - if (is_multiwallet) { - return InitError(strprintf(Untranslated("%s is only allowed with a single wallet file"), "-zapwallettxes")); - } - if (gArgs.SoftSetBoolArg("-rescan", true)) { - LogPrintf("%s: parameter interaction: -zapwallettxes enabled -> setting -rescan=1\n", __func__); - } + if (gArgs.IsArgSet("-zapwallettxes")) { + return InitError(Untranslated("-zapwallettxes has been removed. If you are attempting to remove a stuck transaction from your wallet, please use abandontransaction instead.")); } if (gArgs.GetBoolArg("-sysperms", false)) @@ -118,6 +107,7 @@ void WalletInit::Construct(NodeContext& node) const LogPrintf("Wallet disabled!\n"); return; } - args.SoftSetArg("-wallet", ""); - node.chain_clients.emplace_back(interfaces::MakeWalletClient(*node.chain, args, args.GetArgs("-wallet"))); + auto wallet_client = interfaces::MakeWalletClient(*node.chain, args); + node.wallet_client = wallet_client.get(); + node.chain_clients.emplace_back(std::move(wallet_client)); } diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 2a81d3013..1b057000d 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -5,6 +5,7 @@ #include <wallet/load.h> +#include <fs.h> #include <interfaces/chain.h> #include <scheduler.h> #include <util/string.h> @@ -13,7 +14,9 @@ #include <wallet/wallet.h> #include <wallet/walletdb.h> -bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files) +#include <univalue.h> + +bool VerifyWallets(interfaces::Chain& chain) { if (gArgs.IsArgSet("-walletdir")) { fs::path wallet_dir = gArgs.GetArg("-walletdir", ""); @@ -38,22 +41,39 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal chain.initMessage(_("Verifying wallet(s)...").translated); + // For backwards compatibility if an unnamed top level wallet exists in the + // wallets directory, include it in the default list of wallets to load. + if (!gArgs.IsArgSet("wallet")) { + DatabaseOptions options; + DatabaseStatus status; + bilingual_str error_string; + options.require_existing = true; + options.verify = false; + if (MakeWalletDatabase("", options, status, error_string)) { + gArgs.LockSettings([&](util::Settings& settings) { + util::SettingsValue wallets(util::SettingsValue::VARR); + wallets.push_back(""); // Default wallet name is "" + settings.rw_settings["wallet"] = wallets; + }); + } + } + // Keep track of each wallet absolute path to detect duplicates. std::set<fs::path> wallet_paths; - for (const auto& wallet_file : wallet_files) { - WalletLocation location(wallet_file); + for (const auto& wallet_file : gArgs.GetArgs("-wallet")) { + const fs::path path = fs::absolute(wallet_file, GetWalletDir()); - if (!wallet_paths.insert(location.GetPath()).second) { + if (!wallet_paths.insert(path).second) { chain.initError(strprintf(_("Error loading wallet %s. Duplicate -wallet filename specified."), wallet_file)); return false; } + DatabaseOptions options; + DatabaseStatus status; + options.verify = true; bilingual_str error_string; - std::vector<bilingual_str> warnings; - bool verify_success = CWallet::Verify(chain, location, error_string, warnings); - if (!warnings.empty()) chain.initWarning(Join(warnings, Untranslated("\n"))); - if (!verify_success) { + if (!MakeWalletDatabase(wallet_file, options, status, error_string)) { chain.initError(error_string); return false; } @@ -62,13 +82,17 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal return true; } -bool LoadWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files) +bool LoadWallets(interfaces::Chain& chain) { try { - for (const std::string& walletFile : wallet_files) { + for (const std::string& name : gArgs.GetArgs("-wallet")) { + DatabaseOptions options; + DatabaseStatus status; + options.verify = false; // No need to verify, assuming verified earlier in VerifyWallets() bilingual_str error; std::vector<bilingual_str> warnings; - std::shared_ptr<CWallet> pwallet = CWallet::CreateWalletFromFile(chain, WalletLocation(walletFile), error, warnings); + std::unique_ptr<WalletDatabase> database = MakeWalletDatabase(name, options, status, error); + std::shared_ptr<CWallet> pwallet = database ? CWallet::Create(chain, name, std::move(database), options.create_flags, error, warnings) : nullptr; if (!warnings.empty()) chain.initWarning(Join(warnings, Untranslated("\n"))); if (!pwallet) { chain.initError(error); @@ -116,7 +140,8 @@ void UnloadWallets() while (!wallets.empty()) { auto wallet = wallets.back(); wallets.pop_back(); - RemoveWallet(wallet); + std::vector<bilingual_str> warnings; + RemoveWallet(wallet, nullopt, warnings); UnloadWallet(std::move(wallet)); } } diff --git a/src/wallet/load.h b/src/wallet/load.h index ff4f5b4b2..e12343de2 100644 --- a/src/wallet/load.h +++ b/src/wallet/load.h @@ -17,10 +17,10 @@ class Chain; } // namespace interfaces //! Responsible for reading and validating the -wallet arguments and verifying the wallet database. -bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files); +bool VerifyWallets(interfaces::Chain& chain); //! Load wallet databases. -bool LoadWallets(interfaces::Chain& chain, const std::vector<std::string>& wallet_files); +bool LoadWallets(interfaces::Chain& chain); //! Complete startup of wallets. void StartWallets(CScheduler& scheduler, const ArgsManager& args); diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 3b752ca93..9e36a0978 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -34,7 +34,7 @@ std::string static EncodeDumpString(const std::string &str) { std::stringstream ret; for (const unsigned char c : str) { if (c <= 32 || c >= 128 || c == '%') { - ret << '%' << HexStr(&c, &c + 1); + ret << '%' << HexStr(Span<const unsigned char>(&c, 1)); } else { ret << c; } @@ -90,9 +90,9 @@ static void RescanWallet(CWallet& wallet, const WalletRescanReserver& reserver, } } -UniValue importprivkey(const JSONRPCRequest& request) +RPCHelpMan importprivkey() { - RPCHelpMan{"importprivkey", + return RPCHelpMan{"importprivkey", "\nAdds a private key (as returned by dumpprivkey) to your wallet. Requires a new wallet backup.\n" "Hint: use importmulti to import more than one private key.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" @@ -116,8 +116,8 @@ UniValue importprivkey(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("importprivkey", "\"mykey\", \"testing\", false") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); if (!wallet) return NullUniValue; CWallet* const pwallet = wallet.get(); @@ -189,11 +189,13 @@ UniValue importprivkey(const JSONRPCRequest& request) } return NullUniValue; +}, + }; } -UniValue abortrescan(const JSONRPCRequest& request) +RPCHelpMan abortrescan() { - RPCHelpMan{"abortrescan", + return RPCHelpMan{"abortrescan", "\nStops current wallet rescan triggered by an RPC call, e.g. by an importprivkey call.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", {}, @@ -206,8 +208,8 @@ UniValue abortrescan(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("abortrescan", "") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); if (!wallet) return NullUniValue; CWallet* const pwallet = wallet.get(); @@ -215,11 +217,13 @@ UniValue abortrescan(const JSONRPCRequest& request) if (!pwallet->IsScanning() || pwallet->IsAbortingRescan()) return false; pwallet->AbortRescan(); return true; +}, + }; } -UniValue importaddress(const JSONRPCRequest& request) +RPCHelpMan importaddress() { - RPCHelpMan{"importaddress", + return RPCHelpMan{"importaddress", "\nAdds an address or script (in hex) that can be watched as if it were in your wallet but cannot be used to spend. Requires a new wallet backup.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported address exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" @@ -243,8 +247,8 @@ UniValue importaddress(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("importaddress", "\"myaddress\", \"testing\", false") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); if (!wallet) return NullUniValue; CWallet* const pwallet = wallet.get(); @@ -315,11 +319,13 @@ UniValue importaddress(const JSONRPCRequest& request) } return NullUniValue; +}, + }; } -UniValue importprunedfunds(const JSONRPCRequest& request) +RPCHelpMan importprunedfunds() { - RPCHelpMan{"importprunedfunds", + return RPCHelpMan{"importprunedfunds", "\nImports funds without rescan. Corresponding address or script must previously be included in wallet. Aimed towards pruned wallets. The end-user is responsible to import additional transactions that subsequently spend the imported outputs or rescan after the point in the blockchain the transaction is included.\n", { {"rawtransaction", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "A raw transaction in hex funding an already-existing address in wallet"}, @@ -327,8 +333,8 @@ UniValue importprunedfunds(const JSONRPCRequest& request) }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{""}, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); if (!wallet) return NullUniValue; CWallet* const pwallet = wallet.get(); @@ -371,11 +377,13 @@ UniValue importprunedfunds(const JSONRPCRequest& request) } throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No addresses in wallet correspond to included transaction"); +}, + }; } -UniValue removeprunedfunds(const JSONRPCRequest& request) +RPCHelpMan removeprunedfunds() { - RPCHelpMan{"removeprunedfunds", + return RPCHelpMan{"removeprunedfunds", "\nDeletes the specified transaction from the wallet. Meant for use with pruned wallets and as a companion to importprunedfunds. This will affect wallet balances.\n", { {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex-encoded id of the transaction you are deleting"}, @@ -386,8 +394,8 @@ UniValue removeprunedfunds(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("removeprunedfunds", "\"a8d0c0184dde994a09ec054286f1ce581bebf46446a512166eae7628734ea0a5\"") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); if (!wallet) return NullUniValue; CWallet* const pwallet = wallet.get(); @@ -408,11 +416,13 @@ UniValue removeprunedfunds(const JSONRPCRequest& request) } return NullUniValue; +}, + }; } -UniValue importpubkey(const JSONRPCRequest& request) +RPCHelpMan importpubkey() { - RPCHelpMan{"importpubkey", + return RPCHelpMan{"importpubkey", "\nAdds a public key (in hex) that can be watched as if it were in your wallet but cannot be used to spend. Requires a new wallet backup.\n" "Hint: use importmulti to import more than one public key.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" @@ -432,8 +442,8 @@ UniValue importpubkey(const JSONRPCRequest& request) "\nAs a JSON-RPC call\n" + HelpExampleRpc("importpubkey", "\"mypubkey\", \"testing\", false") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); if (!wallet) return NullUniValue; CWallet* const pwallet = wallet.get(); @@ -492,12 +502,14 @@ UniValue importpubkey(const JSONRPCRequest& request) } return NullUniValue; +}, + }; } -UniValue importwallet(const JSONRPCRequest& request) +RPCHelpMan importwallet() { - RPCHelpMan{"importwallet", + return RPCHelpMan{"importwallet", "\nImports keys from a wallet dump file (see dumpwallet). Requires a new wallet backup to include imported keys.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { @@ -512,8 +524,8 @@ UniValue importwallet(const JSONRPCRequest& request) "\nImport using the json rpc call\n" + HelpExampleRpc("importwallet", "\"test\"") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); if (!wallet) return NullUniValue; CWallet* const pwallet = wallet.get(); @@ -649,11 +661,13 @@ UniValue importwallet(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Error adding some keys/scripts to wallet"); return NullUniValue; +}, + }; } -UniValue dumpprivkey(const JSONRPCRequest& request) +RPCHelpMan dumpprivkey() { - RPCHelpMan{"dumpprivkey", + return RPCHelpMan{"dumpprivkey", "\nReveals the private key corresponding to 'address'.\n" "Then the importprivkey can be used with this output\n", { @@ -667,8 +681,8 @@ UniValue dumpprivkey(const JSONRPCRequest& request) + HelpExampleCli("importprivkey", "\"mykey\"") + HelpExampleRpc("dumpprivkey", "\"myaddress\"") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); if (!wallet) return NullUniValue; const CWallet* const pwallet = wallet.get(); @@ -693,12 +707,14 @@ UniValue dumpprivkey(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Private key for address " + strAddress + " is not known"); } return EncodeSecret(vchSecret); +}, + }; } -UniValue dumpwallet(const JSONRPCRequest& request) +RPCHelpMan dumpwallet() { - RPCHelpMan{"dumpwallet", + return RPCHelpMan{"dumpwallet", "\nDumps all wallet keys in a human-readable format to a server-side file. This does not allow overwriting existing files.\n" "Imported scripts are included in the dumpfile, but corresponding BIP173 addresses, etc. may not be added automatically by importwallet.\n" "Note that if your wallet contains keys which are not derived from your HD seed (e.g. imported keys), these are not covered by\n" @@ -716,8 +732,8 @@ UniValue dumpwallet(const JSONRPCRequest& request) HelpExampleCli("dumpwallet", "\"test\"") + HelpExampleRpc("dumpwallet", "\"test\"") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; @@ -829,6 +845,8 @@ UniValue dumpwallet(const JSONRPCRequest& request) reply.pushKV("filename", filepath.string()); return reply; +}, + }; } struct ImportData @@ -1239,9 +1257,9 @@ static int64_t GetImportTimestamp(const UniValue& data, int64_t now) throw JSONRPCError(RPC_TYPE_ERROR, "Missing required timestamp field for key"); } -UniValue importmulti(const JSONRPCRequest& mainRequest) +RPCHelpMan importmulti() { - RPCHelpMan{"importmulti", + return RPCHelpMan{"importmulti", "\nImport addresses/scripts (with private or public keys, redeem script (P2SH)), optionally rescanning the blockchain from the earliest creation time of the imported scripts. Requires a new wallet backup.\n" "If an address/script is imported without all of the private keys required to spend from that address, it will be watchonly. The 'watchonly' option must be set to true in this case or a warning will be returned.\n" "Conversely, if all the private keys are provided and the address/script is spendable, the watchonly option must be set to false, or a warning will be returned.\n" @@ -1314,8 +1332,8 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) "{ \"scriptPubKey\": { \"address\": \"<my 2nd address>\" }, \"label\": \"example 2\", \"timestamp\": 1455191480 }]'") + HelpExampleCli("importmulti", "'[{ \"scriptPubKey\": { \"address\": \"<my address>\" }, \"timestamp\":1455191478 }]' '{ \"rescan\": false}'") }, - }.Check(mainRequest); - + [&](const RPCHelpMan& self, const JSONRPCRequest& mainRequest) -> UniValue +{ std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(mainRequest); if (!wallet) return NullUniValue; CWallet* const pwallet = wallet.get(); @@ -1423,6 +1441,8 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) } return response; +}, + }; } static UniValue ProcessDescriptorImport(CWallet * const pwallet, const UniValue& data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) @@ -1564,9 +1584,9 @@ static UniValue ProcessDescriptorImport(CWallet * const pwallet, const UniValue& return result; } -UniValue importdescriptors(const JSONRPCRequest& main_request) +RPCHelpMan importdescriptors() { - RPCHelpMan{"importdescriptors", + return RPCHelpMan{"importdescriptors", "\nImport descriptors. This will trigger a rescan of the blockchain based on the earliest timestamp of all descriptors being imported. Requires a new wallet backup.\n" "\nNote: This call can take over an hour to complete if using an early timestamp; during that time, other rpc calls\n" "may report that the imported keys, addresses or scripts exist but related transactions are still missing.\n", @@ -1615,8 +1635,8 @@ UniValue importdescriptors(const JSONRPCRequest& main_request) "{ \"desc\": \"<my desccriptor 2>\", \"label\": \"example 2\", \"timestamp\": 1455191480 }]'") + HelpExampleCli("importdescriptors", "'[{ \"desc\": \"<my descriptor>\", \"timestamp\":1455191478, \"active\": true, \"range\": [0,100], \"label\": \"<my bech32 wallet>\" }]'") }, - }.Check(main_request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& main_request) -> UniValue +{ std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(main_request); if (!wallet) return NullUniValue; CWallet* const pwallet = wallet.get(); @@ -1713,4 +1733,6 @@ UniValue importdescriptors(const JSONRPCRequest& main_request) } return response; +}, + }; } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 9d334063c..62a320680 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -11,6 +11,7 @@ #include <outputtype.h> #include <policy/feerate.h> #include <policy/fees.h> +#include <policy/policy.h> #include <policy/rbf.h> #include <rpc/rawtransaction_util.h> #include <rpc/server.h> @@ -30,6 +31,7 @@ #include <wallet/coincontrol.h> #include <wallet/context.h> #include <wallet/feebumper.h> +#include <wallet/load.h> #include <wallet/rpcwallet.h> #include <wallet/wallet.h> #include <wallet/walletdb.h> @@ -1484,7 +1486,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) {RPCResult::Type::ARR, "removed", "<structure is the same as \"transactions\" above, only present if include_removed=true>\n" "Note: transactions that were re-added in the active chain will appear as-is in this array, and may thus have a positive confirmation count." , {{RPCResult::Type::ELISION, "", ""},}}, - {RPCResult::Type::STR_HEX, "lastblock", "The hash of the block (target_confirmations-1) from the best block on the main chain. This is typically used to feed back into listsinceblock the next time you call it. So you would generally use a target_confirmations of say 6, so you will be continually re-notified of transactions until they've reached 6 confirmations plus any new ones"}, + {RPCResult::Type::STR_HEX, "lastblock", "The hash of the block (target_confirmations-1) from the best block on the main chain, or the genesis hash if the referenced block does not exist yet. This is typically used to feed back into listsinceblock the next time you call it. So you would generally use a target_confirmations of say 6, so you will be continually re-notified of transactions until they've reached 6 confirmations plus any new ones"}, } }, RPCExamples{ @@ -1567,6 +1569,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) } uint256 lastblock; + target_confirms = std::min(target_confirms, wallet.GetLastBlockHeight() + 1); CHECK_NONFATAL(wallet.chain().findAncestorByHeight(wallet.GetLastBlockHash(), wallet.GetLastBlockHeight() + 1 - target_confirms, FoundBlock().hash(lastblock))); UniValue ret(UniValue::VOBJ); @@ -2047,6 +2050,7 @@ static UniValue lockunspent(const JSONRPCRequest& request) "Temporarily lock (unlock=false) or unlock (unlock=true) specified transaction outputs.\n" "If no transaction outputs are specified when unlocking then all current locked transaction outputs are unlocked.\n" "A locked transaction output will not be chosen by automatic coin selection, when spending bitcoins.\n" + "Manually selected coins are automatically unlocked.\n" "Locks are stored in memory only. Nodes start with zero locked outputs, and the locked output list\n" "is always cleared (by virtue of process exit) when a node stops or fails.\n" "Also see the listunspent call\n", @@ -2338,7 +2342,7 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) {RPCResult::Type::NUM_TIME, "keypoololdest", "the " + UNIX_EPOCH_TIME + " of the oldest pre-generated key in the key pool. Legacy wallets only."}, {RPCResult::Type::NUM, "keypoolsize", "how many new keys are pre-generated (only counts external keys)"}, {RPCResult::Type::NUM, "keypoolsize_hd_internal", "how many new keys are pre-generated for internal use (used for change outputs, only appears if the wallet is using this feature, otherwise external keys are used)"}, - {RPCResult::Type::NUM_TIME, "unlocked_until", "the " + UNIX_EPOCH_TIME + " until which the wallet is unlocked for transfers, or 0 if the wallet is locked"}, + {RPCResult::Type::NUM_TIME, "unlocked_until", /* optional */ true, "the " + UNIX_EPOCH_TIME + " until which the wallet is unlocked for transfers, or 0 if the wallet is locked (only present for passphrase-encrypted wallets)"}, {RPCResult::Type::STR_AMOUNT, "paytxfee", "the transaction fee configuration, set in " + CURRENCY_UNIT + "/kB"}, {RPCResult::Type::STR_HEX, "hdseedid", /* optional */ true, "the Hash160 of the HD seed (only present when HD is enabled)"}, {RPCResult::Type::BOOL, "private_keys_enabled", "false if privatekeys are disabled for this wallet (enforced watch-only wallet)"}, @@ -2480,9 +2484,10 @@ static UniValue loadwallet(const JSONRPCRequest& request) RPCHelpMan{"loadwallet", "\nLoads a wallet from a wallet file or directory." "\nNote that all wallet command-line options used when starting bitcoind will be" - "\napplied to the new wallet (eg -zapwallettxes, rescan, etc).\n", + "\napplied to the new wallet (eg -rescan, etc).\n", { {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet directory or .dat file."}, + {"load_on_startup", RPCArg::Type::BOOL, /* default */ "null", "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -2498,22 +2503,21 @@ static UniValue loadwallet(const JSONRPCRequest& request) }.Check(request); WalletContext& context = EnsureWalletContext(request.context); - WalletLocation location(request.params[0].get_str()); - - if (!location.Exists()) { - throw JSONRPCError(RPC_WALLET_NOT_FOUND, "Wallet " + location.GetName() + " not found."); - } else if (fs::is_directory(location.GetPath())) { - // The given filename is a directory. Check that there's a wallet.dat file. - fs::path wallet_dat_file = location.GetPath() / "wallet.dat"; - if (fs::symlink_status(wallet_dat_file).type() == fs::file_not_found) { - throw JSONRPCError(RPC_WALLET_NOT_FOUND, "Directory " + location.GetName() + " does not contain a wallet.dat file."); - } - } + const std::string name(request.params[0].get_str()); + DatabaseOptions options; + DatabaseStatus status; + options.require_existing = true; bilingual_str error; std::vector<bilingual_str> warnings; - std::shared_ptr<CWallet> const wallet = LoadWallet(*context.chain, location, error, warnings); - if (!wallet) throw JSONRPCError(RPC_WALLET_ERROR, error.original); + Optional<bool> load_on_start = request.params[1].isNull() ? nullopt : Optional<bool>(request.params[1].get_bool()); + std::shared_ptr<CWallet> const wallet = LoadWallet(*context.chain, name, load_on_start, options, status, error, warnings); + if (!wallet) { + // Map bad format to not found, since bad format is returned when the + // wallet directory exists, but doesn't contain a data file. + RPCErrorCode code = status == DatabaseStatus::FAILED_NOT_FOUND || status == DatabaseStatus::FAILED_BAD_FORMAT ? RPC_WALLET_NOT_FOUND : RPC_WALLET_ERROR; + throw JSONRPCError(code, error.original); + } UniValue obj(UniValue::VOBJ); obj.pushKV("name", wallet->GetName()); @@ -2599,6 +2603,7 @@ static UniValue createwallet(const JSONRPCRequest& request) {"passphrase", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Encrypt the wallet with this passphrase."}, {"avoid_reuse", RPCArg::Type::BOOL, /* default */ "false", "Keep track of coin reuse, and treat dirty and clean coins differently with privacy considerations in mind."}, {"descriptors", RPCArg::Type::BOOL, /* default */ "false", "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation"}, + {"load_on_startup", RPCArg::Type::BOOL, /* default */ "null", "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -2641,17 +2646,17 @@ static UniValue createwallet(const JSONRPCRequest& request) warnings.emplace_back(Untranslated("Wallet is an experimental descriptor wallet")); } + DatabaseOptions options; + DatabaseStatus status; + options.require_create = true; + options.create_flags = flags; + options.create_passphrase = passphrase; bilingual_str error; - std::shared_ptr<CWallet> wallet; - WalletCreationStatus status = CreateWallet(*context.chain, passphrase, flags, request.params[0].get_str(), error, warnings, wallet); - switch (status) { - case WalletCreationStatus::CREATION_FAILED: - throw JSONRPCError(RPC_WALLET_ERROR, error.original); - case WalletCreationStatus::ENCRYPTION_FAILED: - throw JSONRPCError(RPC_WALLET_ENCRYPTION_FAILED, error.original); - case WalletCreationStatus::SUCCESS: - break; - // no default case, so the compiler can warn about missing cases + Optional<bool> load_on_start = request.params[6].isNull() ? nullopt : Optional<bool>(request.params[6].get_bool()); + std::shared_ptr<CWallet> wallet = CreateWallet(*context.chain, request.params[0].get_str(), load_on_start, options, status, error, warnings); + if (!wallet) { + RPCErrorCode code = status == DatabaseStatus::FAILED_ENCRYPT ? RPC_WALLET_ENCRYPTION_FAILED : RPC_WALLET_ERROR; + throw JSONRPCError(code, error.original); } UniValue obj(UniValue::VOBJ); @@ -2668,8 +2673,11 @@ static UniValue unloadwallet(const JSONRPCRequest& request) "Specifying the wallet name on a wallet endpoint is invalid.", { {"wallet_name", RPCArg::Type::STR, /* default */ "the wallet name from the RPC request", "The name of the wallet to unload."}, + {"load_on_startup", RPCArg::Type::BOOL, /* default */ "null", "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, }, - RPCResult{RPCResult::Type::NONE, "", ""}, + RPCResult{RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "warning", "Warning message if wallet was not unloaded cleanly."}, + }}, RPCExamples{ HelpExampleCli("unloadwallet", "wallet_name") + HelpExampleRpc("unloadwallet", "wallet_name") @@ -2693,13 +2701,17 @@ static UniValue unloadwallet(const JSONRPCRequest& request) // Release the "main" shared pointer and prevent further notifications. // Note that any attempt to load the same wallet would fail until the wallet // is destroyed (see CheckUniqueFileid). - if (!RemoveWallet(wallet)) { + std::vector<bilingual_str> warnings; + Optional<bool> load_on_start = request.params[1].isNull() ? nullopt : Optional<bool>(request.params[1].get_bool()); + if (!RemoveWallet(wallet, load_on_start, warnings)) { throw JSONRPCError(RPC_MISC_ERROR, "Requested wallet already unloaded"); } UnloadWallet(std::move(wallet)); - return NullUniValue; + UniValue result(UniValue::VOBJ); + result.pushKV("warning", Join(warnings, Untranslated("\n")).original); + return result; } static UniValue listunspent(const JSONRPCRequest& request) @@ -2807,6 +2819,15 @@ static UniValue listunspent(const JSONRPCRequest& request) if (!request.params[4].isNull()) { const UniValue& options = request.params[4].get_obj(); + RPCTypeCheckObj(options, + { + {"minimumAmount", UniValueType()}, + {"maximumAmount", UniValueType()}, + {"minimumSumAmount", UniValueType()}, + {"maximumCount", UniValueType(UniValue::VNUM)}, + }, + true, true); + if (options.exists("minimumAmount")) nMinimumAmount = AmountFromValue(options["minimumAmount"]); @@ -2935,13 +2956,22 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f RPCTypeCheckObj(options, { {"add_inputs", UniValueType(UniValue::VBOOL)}, + {"add_to_wallet", UniValueType(UniValue::VBOOL)}, {"changeAddress", UniValueType(UniValue::VSTR)}, + {"change_address", UniValueType(UniValue::VSTR)}, {"changePosition", UniValueType(UniValue::VNUM)}, + {"change_position", UniValueType(UniValue::VNUM)}, {"change_type", UniValueType(UniValue::VSTR)}, {"includeWatching", UniValueType(UniValue::VBOOL)}, + {"include_watching", UniValueType(UniValue::VBOOL)}, + {"inputs", UniValueType(UniValue::VARR)}, {"lockUnspents", UniValueType(UniValue::VBOOL)}, - {"feeRate", UniValueType()}, // will be checked below + {"lock_unspents", UniValueType(UniValue::VBOOL)}, + {"locktime", UniValueType(UniValue::VNUM)}, + {"feeRate", UniValueType()}, // will be checked below, + {"psbt", UniValueType(UniValue::VBOOL)}, {"subtractFeeFromOutputs", UniValueType(UniValue::VARR)}, + {"subtract_fee_from_outputs", UniValueType(UniValue::VARR)}, {"replaceable", UniValueType(UniValue::VBOOL)}, {"conf_target", UniValueType(UniValue::VNUM)}, {"estimate_mode", UniValueType(UniValue::VSTR)}, @@ -2952,22 +2982,24 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f coinControl.m_add_inputs = options["add_inputs"].get_bool(); } - if (options.exists("changeAddress")) { - CTxDestination dest = DecodeDestination(options["changeAddress"].get_str()); + if (options.exists("changeAddress") || options.exists("change_address")) { + const std::string change_address_str = (options.exists("change_address") ? options["change_address"] : options["changeAddress"]).get_str(); + CTxDestination dest = DecodeDestination(change_address_str); if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "changeAddress must be a valid bitcoin address"); + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Change address must be a valid bitcoin address"); } coinControl.destChange = dest; } - if (options.exists("changePosition")) - change_position = options["changePosition"].get_int(); + if (options.exists("changePosition") || options.exists("change_position")) { + change_position = (options.exists("change_position") ? options["change_position"] : options["changePosition"]).get_int(); + } if (options.exists("change_type")) { - if (options.exists("changeAddress")) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both changeAddress and address_type options"); + if (options.exists("changeAddress") || options.exists("change_address")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both change address and address type options"); } OutputType out_type; if (!ParseOutputType(options["change_type"].get_str(), out_type)) { @@ -2976,10 +3008,12 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f coinControl.m_change_type.emplace(out_type); } - coinControl.fAllowWatchOnly = ParseIncludeWatchonly(options["includeWatching"], *pwallet); + const UniValue include_watching_option = options.exists("include_watching") ? options["include_watching"] : options["includeWatching"]; + coinControl.fAllowWatchOnly = ParseIncludeWatchonly(include_watching_option, *pwallet); - if (options.exists("lockUnspents")) - lockUnspents = options["lockUnspents"].get_bool(); + if (options.exists("lockUnspents") || options.exists("lock_unspents")) { + lockUnspents = (options.exists("lock_unspents") ? options["lock_unspents"] : options["lockUnspents"]).get_bool(); + } if (options.exists("feeRate")) { @@ -2993,8 +3027,8 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f coinControl.fOverrideFeeRate = true; } - if (options.exists("subtractFeeFromOutputs")) - subtractFeeFromOutputs = options["subtractFeeFromOutputs"].get_array(); + if (options.exists("subtractFeeFromOutputs") || options.exists("subtract_fee_from_outputs") ) + subtractFeeFromOutputs = (options.exists("subtract_fee_from_outputs") ? options["subtract_fee_from_outputs"] : options["subtractFeeFromOutputs"]).get_array(); if (options.exists("replaceable")) { coinControl.m_signal_bip125_rbf = options["replaceable"].get_bool(); @@ -3165,7 +3199,7 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) { {RPCResult::Type::STR_HEX, "hex", "The hex-encoded raw transaction with signature(s)"}, {RPCResult::Type::BOOL, "complete", "If the transaction has a complete set of signatures"}, - {RPCResult::Type::ARR, "errors", "Script verification errors (if there are any)", + {RPCResult::Type::ARR, "errors", /* optional */ true, "Script verification errors (if there are any)", { {RPCResult::Type::OBJ, "", "", { @@ -3222,59 +3256,73 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) static UniValue bumpfee(const JSONRPCRequest& request) { - RPCHelpMan{"bumpfee", - "\nBumps the fee of an opt-in-RBF transaction T, replacing it with a new transaction B.\n" - "An opt-in RBF transaction with the given txid must be in the wallet.\n" - "The command will pay the additional fee by reducing change outputs or adding inputs when necessary. It may add a new change output if one does not already exist.\n" - "All inputs in the original transaction will be included in the replacement transaction.\n" - "The command will fail if the wallet or mempool contains a transaction that spends one of T's outputs.\n" - "By default, the new fee will be calculated automatically using estimatesmartfee.\n" - "The user can specify a confirmation target for estimatesmartfee.\n" - "Alternatively, the user can specify a fee_rate (" + CURRENCY_UNIT + " per kB) for the new transaction.\n" - "At a minimum, the new fee rate must be high enough to pay an additional new relay fee (incrementalfee\n" - "returned by getnetworkinfo) to enter the node's mempool.\n", + bool want_psbt = request.strMethod == "psbtbumpfee"; + + RPCHelpMan{request.strMethod, + "\nBumps the fee of an opt-in-RBF transaction T, replacing it with a new transaction B.\n" + + std::string(want_psbt ? "Returns a PSBT instead of creating and signing a new transaction.\n" : "") + + "An opt-in RBF transaction with the given txid must be in the wallet.\n" + "The command will pay the additional fee by reducing change outputs or adding inputs when necessary. It may add a new change output if one does not already exist.\n" + "All inputs in the original transaction will be included in the replacement transaction.\n" + "The command will fail if the wallet or mempool contains a transaction that spends one of T's outputs.\n" + "By default, the new fee will be calculated automatically using estimatesmartfee.\n" + "The user can specify a confirmation target for estimatesmartfee.\n" + "Alternatively, the user can specify a fee_rate (" + CURRENCY_UNIT + " per kB) for the new transaction.\n" + "At a minimum, the new fee rate must be high enough to pay an additional new relay fee (incrementalfee\n" + "returned by getnetworkinfo) to enter the node's mempool.\n", + { + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The txid to be bumped"}, + {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", { - {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The txid to be bumped"}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", - { - {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks)"}, - {"fee_rate", RPCArg::Type::NUM, /* default */ "fall back to 'conf_target'", "fee rate (NOT total fee) to pay, in " + CURRENCY_UNIT + " per kB\n" - " Specify a fee rate instead of relying on the built-in fee estimator.\n" - "Must be at least 0.0001 " + CURRENCY_UNIT + " per kB higher than the current transaction fee rate.\n"}, - {"replaceable", RPCArg::Type::BOOL, /* default */ "true", "Whether the new transaction should still be\n" - " marked bip-125 replaceable. If true, the sequence numbers in the transaction will\n" - " be left unchanged from the original. If false, any input sequence numbers in the\n" - " original transaction that were less than 0xfffffffe will be increased to 0xfffffffe\n" - " so the new transaction will not be explicitly bip-125 replaceable (though it may\n" - " still be replaceable in practice, for example if it has unconfirmed ancestors which\n" - " are replaceable)."}, - {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" - " \"" + FeeModes("\"\n\"") + "\""}, - }, - "options"}, - }, - RPCResult{ - RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::STR, "psbt", "The base64-encoded unsigned PSBT of the new transaction. Only returned when wallet private keys are disabled."}, - {RPCResult::Type::STR_HEX, "txid", "The id of the new transaction. Only returned when wallet private keys are enabled."}, - {RPCResult::Type::STR_AMOUNT, "origfee", "The fee of the replaced transaction."}, - {RPCResult::Type::STR_AMOUNT, "fee", "The fee of the new transaction."}, - {RPCResult::Type::ARR, "errors", "Errors encountered during processing (may be empty).", - { - {RPCResult::Type::STR, "", ""}, - }}, - } - }, - RPCExamples{ - "\nBump the fee, get the new transaction\'s txid\n" + - HelpExampleCli("bumpfee", "<txid>") + {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks)"}, + {"fee_rate", RPCArg::Type::NUM, /* default */ "fall back to 'conf_target'", "fee rate (NOT total fee) to pay, in " + CURRENCY_UNIT + " per kB\n" + " Specify a fee rate instead of relying on the built-in fee estimator.\n" + "Must be at least 0.0001 " + CURRENCY_UNIT + " per kB higher than the current transaction fee rate.\n"}, + {"replaceable", RPCArg::Type::BOOL, /* default */ "true", "Whether the new transaction should still be\n" + " marked bip-125 replaceable. If true, the sequence numbers in the transaction will\n" + " be left unchanged from the original. If false, any input sequence numbers in the\n" + " original transaction that were less than 0xfffffffe will be increased to 0xfffffffe\n" + " so the new transaction will not be explicitly bip-125 replaceable (though it may\n" + " still be replaceable in practice, for example if it has unconfirmed ancestors which\n" + " are replaceable)."}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, }, - }.Check(request); + "options"}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", Cat(Cat<std::vector<RPCResult>>( + { + {RPCResult::Type::STR, "psbt", "The base64-encoded unsigned PSBT of the new transaction." + std::string(want_psbt ? "" : " Only returned when wallet private keys are disabled. (DEPRECATED)")}, + }, + want_psbt ? std::vector<RPCResult>{} : std::vector<RPCResult>{{RPCResult::Type::STR_HEX, "txid", "The id of the new transaction. Only returned when wallet private keys are enabled."}} + ), + { + {RPCResult::Type::STR_AMOUNT, "origfee", "The fee of the replaced transaction."}, + {RPCResult::Type::STR_AMOUNT, "fee", "The fee of the new transaction."}, + {RPCResult::Type::ARR, "errors", "Errors encountered during processing (may be empty).", + { + {RPCResult::Type::STR, "", ""}, + }}, + }) + }, + RPCExamples{ + "\nBump the fee, get the new transaction\'s" + std::string(want_psbt ? "psbt" : "txid") + "\n" + + HelpExampleCli(request.strMethod, "<txid>") + }, + }.Check(request); std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); if (!wallet) return NullUniValue; CWallet* const pwallet = wallet.get(); + if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !want_psbt) { + if (!pwallet->chain().rpcEnableDeprecated("bumpfee")) { + throw JSONRPCError(RPC_METHOD_DEPRECATED, "Using bumpfee with wallets that have private keys disabled is deprecated. Use psbtbumpfee instead or restart bitcoind with -deprecatedrpc=bumpfee. This functionality will be removed in 0.22"); + } + want_psbt = true; + } + RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VOBJ}); uint256 hash(ParseHashV(request.params[0], "txid")); @@ -3359,7 +3407,7 @@ static UniValue bumpfee(const JSONRPCRequest& request) // If wallet private keys are enabled, return the new transaction id, // otherwise return the base64-encoded unsigned PSBT of the new transaction. - if (!pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (!want_psbt) { if (!feebumper::SignTransaction(*pwallet, mtx)) { throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); } @@ -3392,6 +3440,11 @@ static UniValue bumpfee(const JSONRPCRequest& request) return result; } +static UniValue psbtbumpfee(const JSONRPCRequest& request) +{ + return bumpfee(request); +} + UniValue rescanblockchain(const JSONRPCRequest& request) { RPCHelpMan{"rescanblockchain", @@ -3688,7 +3741,7 @@ UniValue getaddressinfo(const JSONRPCRequest& request) if (meta->has_key_origin) { ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path)); ret.pushKV("hdseedid", meta->hd_seed_id.GetHex()); - ret.pushKV("hdmasterfingerprint", HexStr(meta->key_origin.fingerprint, meta->key_origin.fingerprint + 4)); + ret.pushKV("hdmasterfingerprint", HexStr(meta->key_origin.fingerprint)); } } } @@ -3818,6 +3871,185 @@ static UniValue listlabels(const JSONRPCRequest& request) return ret; } +static RPCHelpMan send() +{ + return RPCHelpMan{"send", + "\nSend a transaction.\n", + { + {"outputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "a json array with outputs (key-value pairs), where none of the keys are duplicated.\n" + "That is, each address can only appear once and there can only be one 'data' object.\n" + "For convenience, a dictionary, which holds the key-value pairs directly, is also accepted.", + { + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + { + {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "A key-value pair. The key (string) is the bitcoin address, the value (float or string) is the amount in " + CURRENCY_UNIT + ""}, + }, + }, + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + { + {"data", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "A key-value pair. The key must be \"data\", the value is hex-encoded data"}, + }, + }, + }, + }, + {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks), or fee rate (for " + CURRENCY_UNIT + "/kB or " + CURRENCY_ATOM + "/B estimate modes)"}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, + {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", + { + {"add_inputs", RPCArg::Type::BOOL, /* default */ "false", "If inputs are specified, automatically include more if they are not enough."}, + {"add_to_wallet", RPCArg::Type::BOOL, /* default */ "true", "When false, returns a serialized transaction which will not be added to the wallet or broadcast"}, + {"change_address", RPCArg::Type::STR_HEX, /* default */ "pool address", "The bitcoin address to receive the change"}, + {"change_position", RPCArg::Type::NUM, /* default */ "random", "The index of the change output"}, + {"change_type", RPCArg::Type::STR, /* default */ "set by -changetype", "The output type to use. Only valid if change_address is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, + {"conf_target", RPCArg::Type::NUM, /* default */ "wallet default", "Confirmation target (in blocks), or fee rate (for " + CURRENCY_UNIT + "/kB or " + CURRENCY_ATOM + "/B estimate modes)"}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "unset", std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, + {"include_watching", RPCArg::Type::BOOL, /* default */ "true for watch-only wallets, otherwise false", "Also select inputs which are watch only.\n" + "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" + "e.g. with 'importpubkey' or 'importmulti' with the 'pubkeys' or 'desc' field."}, + {"inputs", RPCArg::Type::ARR, /* default */ "empty array", "Specify inputs instead of adding them automatically. A json array of json objects", + { + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, + {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, + {"sequence", RPCArg::Type::NUM, RPCArg::Optional::NO, "The sequence number"}, + }, + }, + {"locktime", RPCArg::Type::NUM, /* default */ "0", "Raw locktime. Non-0 value also locktime-activates inputs"}, + {"lock_unspents", RPCArg::Type::BOOL, /* default */ "false", "Lock selected unspent outputs"}, + {"psbt", RPCArg::Type::BOOL, /* default */ "automatic", "Always return a PSBT, implies add_to_wallet=false."}, + {"subtract_fee_from_outputs", RPCArg::Type::ARR, /* default */ "empty array", "A json array of integers.\n" + "The fee will be equally deducted from the amount of each specified output.\n" + "Those recipients will receive less bitcoins than you enter in their corresponding amount field.\n" + "If no outputs are specified here, the sender pays the fee.", + { + {"vout_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The zero-based output index, before a change output is added."}, + }, + }, + {"replaceable", RPCArg::Type::BOOL, /* default */ "wallet default", "Marks this transaction as BIP125 replaceable.\n" + " Allows this transaction to be replaced by a transaction with higher fees"}, + }, + "options"}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::BOOL, "complete", "If the transaction has a complete set of signatures"}, + {RPCResult::Type::STR_HEX, "txid", "The transaction id for the send. Only 1 transaction is created regardless of the number of addresses."}, + {RPCResult::Type::STR_HEX, "hex", "If add_to_wallet is false, the hex-encoded raw transaction with signature(s)"}, + {RPCResult::Type::STR, "psbt", "If more signatures are needed, or if add_to_wallet is false, the base64-encoded (partially) signed transaction"} + } + }, + RPCExamples{"" + "\nSend with a fee rate of 1 satoshi per byte\n" + + HelpExampleCli("send", "'{\"" + EXAMPLE_ADDRESS[0] + "\": 0.1}' 1 sat/b\n" + + "\nCreate a transaction that should confirm the next block, with a specific input, and return result without adding to wallet or broadcasting to the network\n") + + HelpExampleCli("send", "'{\"" + EXAMPLE_ADDRESS[0] + "\": 0.1}' 1 economical '{\"add_to_wallet\": false, \"inputs\": [{\"txid\":\"a08e6907dbbd3d809776dbfc5d82e371b764ed838b5655e72f463568df1aadf0\", \"vout\":1}]}'") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + RPCTypeCheck(request.params, { + UniValueType(), // ARR or OBJ, checked later + UniValue::VNUM, + UniValue::VSTR, + UniValue::VOBJ + }, true + ); + + std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); + if (!wallet) return NullUniValue; + CWallet* const pwallet = wallet.get(); + + UniValue options = request.params[3]; + if (options.exists("feeRate") || options.exists("fee_rate") || options.exists("estimate_mode") || options.exists("conf_target")) { + if (!request.params[1].isNull() || !request.params[2].isNull()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use either conf_target and estimate_mode or the options dictionary to control fee rate"); + } + } else { + options.pushKV("conf_target", request.params[1]); + options.pushKV("estimate_mode", request.params[2]); + } + if (!options["conf_target"].isNull() && (options["estimate_mode"].isNull() || (options["estimate_mode"].get_str() == "unset"))) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Specify estimate_mode"); + } + if (options.exists("changeAddress")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use change_address"); + } + if (options.exists("changePosition")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use change_position"); + } + if (options.exists("includeWatching")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use include_watching"); + } + if (options.exists("lockUnspents")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use lock_unspents"); + } + if (options.exists("subtractFeeFromOutputs")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Use subtract_fee_from_outputs"); + } + + const bool psbt_opt_in = options.exists("psbt") && options["psbt"].get_bool(); + + CAmount fee; + int change_position; + bool rbf = pwallet->m_signal_rbf; + if (options.exists("replaceable")) { + rbf = options["add_to_wallet"].get_bool(); + } + CMutableTransaction rawTx = ConstructTransaction(options["inputs"], request.params[0], options["locktime"], rbf); + CCoinControl coin_control; + // Automatically select coins, unless at least one is manually selected. Can + // be overriden by options.add_inputs. + coin_control.m_add_inputs = rawTx.vin.size() == 0; + FundTransaction(pwallet, rawTx, fee, change_position, options, coin_control); + + bool add_to_wallet = true; + if (options.exists("add_to_wallet")) { + add_to_wallet = options["add_to_wallet"].get_bool(); + } + + // Make a blank psbt + PartiallySignedTransaction psbtx(rawTx); + + // Fill transaction with out data and sign + bool complete = true; + const TransactionError err = pwallet->FillPSBT(psbtx, complete, SIGHASH_ALL, true, false); + if (err != TransactionError::OK) { + throw JSONRPCTransactionError(err); + } + + CMutableTransaction mtx; + complete = FinalizeAndExtractPSBT(psbtx, mtx); + + UniValue result(UniValue::VOBJ); + + // Serialize the PSBT + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + const std::string result_str = EncodeBase64(ssTx.str()); + + if (psbt_opt_in || !complete || !add_to_wallet) { + result.pushKV("psbt", result_str); + } + + if (complete) { + std::string err_string; + std::string hex = EncodeHexTx(CTransaction(mtx)); + CTransactionRef tx(MakeTransactionRef(std::move(mtx))); + result.pushKV("txid", tx->GetHash().GetHex()); + if (add_to_wallet && !psbt_opt_in) { + pwallet->CommitTransaction(tx, {}, {} /* orderForm */); + } else { + result.pushKV("hex", hex); + } + } + result.pushKV("complete", complete); + + return result; + } + }; +} + UniValue sethdseed(const JSONRPCRequest& request) { RPCHelpMan{"sethdseed", @@ -3958,7 +4190,7 @@ UniValue walletcreatefundedpsbt(const JSONRPCRequest& request) "\nCreates and funds a transaction in the Partially Signed Transaction format.\n" "Implements the Creator and Updater roles.\n", { - {"inputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The inputs. Leave empty to add inputs automatically. See add_inputs option.", + {"inputs", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "Leave empty to add inputs automatically. See add_inputs option.", { {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { @@ -4113,17 +4345,17 @@ static UniValue upgradewallet(const JSONRPCRequest& request) return error.original; } -UniValue abortrescan(const JSONRPCRequest& request); // in rpcdump.cpp -UniValue dumpprivkey(const JSONRPCRequest& request); // in rpcdump.cpp -UniValue importprivkey(const JSONRPCRequest& request); -UniValue importaddress(const JSONRPCRequest& request); -UniValue importpubkey(const JSONRPCRequest& request); -UniValue dumpwallet(const JSONRPCRequest& request); -UniValue importwallet(const JSONRPCRequest& request); -UniValue importprunedfunds(const JSONRPCRequest& request); -UniValue removeprunedfunds(const JSONRPCRequest& request); -UniValue importmulti(const JSONRPCRequest& request); -UniValue importdescriptors(const JSONRPCRequest& request); +RPCHelpMan abortrescan(); +RPCHelpMan dumpprivkey(); +RPCHelpMan importprivkey(); +RPCHelpMan importaddress(); +RPCHelpMan importpubkey(); +RPCHelpMan dumpwallet(); +RPCHelpMan importwallet(); +RPCHelpMan importprunedfunds(); +RPCHelpMan removeprunedfunds(); +RPCHelpMan importmulti(); +RPCHelpMan importdescriptors(); Span<const CRPCCommand> GetWalletRPCCommands() { @@ -4137,7 +4369,8 @@ static const CRPCCommand commands[] = { "wallet", "addmultisigaddress", &addmultisigaddress, {"nrequired","keys","label","address_type"} }, { "wallet", "backupwallet", &backupwallet, {"destination"} }, { "wallet", "bumpfee", &bumpfee, {"txid", "options"} }, - { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys", "blank", "passphrase", "avoid_reuse", "descriptors"} }, + { "wallet", "psbtbumpfee", &psbtbumpfee, {"txid", "options"} }, + { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys", "blank", "passphrase", "avoid_reuse", "descriptors", "load_on_startup"} }, { "wallet", "dumpprivkey", &dumpprivkey, {"address"} }, { "wallet", "dumpwallet", &dumpwallet, {"filename"} }, { "wallet", "encryptwallet", &encryptwallet, {"passphrase"} }, @@ -4170,10 +4403,11 @@ static const CRPCCommand commands[] = { "wallet", "listunspent", &listunspent, {"minconf","maxconf","addresses","include_unsafe","query_options"} }, { "wallet", "listwalletdir", &listwalletdir, {} }, { "wallet", "listwallets", &listwallets, {} }, - { "wallet", "loadwallet", &loadwallet, {"filename"} }, + { "wallet", "loadwallet", &loadwallet, {"filename", "load_on_startup"} }, { "wallet", "lockunspent", &lockunspent, {"unlock","transactions"} }, { "wallet", "removeprunedfunds", &removeprunedfunds, {"txid"} }, { "wallet", "rescanblockchain", &rescanblockchain, {"start_height", "stop_height"} }, + { "wallet", "send", &send, {"outputs","conf_target","estimate_mode","options"} }, { "wallet", "sendmany", &sendmany, {"dummy","amounts","minconf","comment","subtractfeefrom","replaceable","conf_target","estimate_mode"} }, { "wallet", "sendtoaddress", &sendtoaddress, {"address","amount","comment","comment_to","subtractfeefromamount","replaceable","conf_target","estimate_mode","avoid_reuse"} }, { "wallet", "sethdseed", &sethdseed, {"newkeypool","seed"} }, @@ -4182,7 +4416,7 @@ static const CRPCCommand commands[] = { "wallet", "setwalletflag", &setwalletflag, {"flag","value"} }, { "wallet", "signmessage", &signmessage, {"address","message"} }, { "wallet", "signrawtransactionwithwallet", &signrawtransactionwithwallet, {"hexstring","prevtxs","sighashtype"} }, - { "wallet", "unloadwallet", &unloadwallet, {"wallet_name"} }, + { "wallet", "unloadwallet", &unloadwallet, {"wallet_name", "load_on_startup"} }, { "wallet", "upgradewallet", &upgradewallet, {"version"} }, { "wallet", "walletcreatefundedpsbt", &walletcreatefundedpsbt, {"inputs","outputs","locktime","options","bip32derivs"} }, { "wallet", "walletlock", &walletlock, {} }, diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index af57210f0..225b97506 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -16,14 +16,24 @@ static const char *HEADER_END = "HEADER=END"; static const char *DATA_END = "DATA=END"; typedef std::pair<std::vector<unsigned char>, std::vector<unsigned char> > KeyValPair; -bool RecoverDatabaseFile(const fs::path& file_path) +static bool KeyFilter(const std::string& type) { + return WalletBatch::IsKeyType(type) || type == DBKeys::HDCHAIN; +} + +bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::vector<bilingual_str>& warnings) +{ + DatabaseOptions options; + DatabaseStatus status; + options.require_existing = true; + options.verify = false; + std::unique_ptr<WalletDatabase> database = MakeDatabase(file_path, options, status, error); + if (!database) return false; + std::string filename; std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename); - bilingual_str open_err; - if (!env->Open(open_err)) { - tfm::format(std::cerr, "%s\n", open_err.original); + if (!env->Open(error)) { return false; } @@ -39,11 +49,9 @@ bool RecoverDatabaseFile(const fs::path& file_path) int result = env->dbenv->dbrename(nullptr, filename.c_str(), nullptr, newFilename.c_str(), DB_AUTO_COMMIT); - if (result == 0) - LogPrintf("Renamed %s to %s\n", filename, newFilename); - else + if (result != 0) { - LogPrintf("Failed to rename %s to %s\n", filename, newFilename); + error = strprintf(Untranslated("Failed to rename %s to %s"), filename, newFilename); return false; } @@ -60,10 +68,10 @@ bool RecoverDatabaseFile(const fs::path& file_path) Db db(env->dbenv.get(), 0); result = db.verify(newFilename.c_str(), nullptr, &strDump, DB_SALVAGE | DB_AGGRESSIVE); if (result == DB_VERIFY_BAD) { - LogPrintf("Salvage: Database salvage found errors, all data may not be recoverable.\n"); + warnings.push_back(Untranslated("Salvage: Database salvage found errors, all data may not be recoverable.")); } if (result != 0 && result != DB_VERIFY_BAD) { - LogPrintf("Salvage: Database salvage failed with result %d.\n", result); + error = strprintf(Untranslated("Salvage: Database salvage failed with result %d."), result); return false; } @@ -87,7 +95,7 @@ bool RecoverDatabaseFile(const fs::path& file_path) break; getline(strDump, valueHex); if (valueHex == DATA_END) { - LogPrintf("Salvage: WARNING: Number of keys in data does not match number of values.\n"); + warnings.push_back(Untranslated("Salvage: WARNING: Number of keys in data does not match number of values.")); break; } salvagedData.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex))); @@ -96,7 +104,7 @@ bool RecoverDatabaseFile(const fs::path& file_path) bool fSuccess; if (keyHex != DATA_END) { - LogPrintf("Salvage: WARNING: Unexpected end of file while reading salvage output.\n"); + warnings.push_back(Untranslated("Salvage: WARNING: Unexpected end of file while reading salvage output.")); fSuccess = false; } else { fSuccess = (result == 0); @@ -104,10 +112,9 @@ bool RecoverDatabaseFile(const fs::path& file_path) if (salvagedData.empty()) { - LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); + error = strprintf(Untranslated("Salvage(aggressive) found no records in %s."), newFilename); return false; } - LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0); int ret = pdbCopy->open(nullptr, // Txn pointer @@ -117,13 +124,13 @@ bool RecoverDatabaseFile(const fs::path& file_path) DB_CREATE, // Flags 0); if (ret > 0) { - LogPrintf("Cannot create database file %s\n", filename); + error = strprintf(Untranslated("Cannot create database file %s"), filename); pdbCopy->close(0); return false; } DbTxn* ptxn = env->TxnBegin(); - CWallet dummyWallet(nullptr, WalletLocation(), CreateDummyWalletDatabase()); + CWallet dummyWallet(nullptr, "", CreateDummyWalletDatabase()); for (KeyValPair& row : salvagedData) { /* Filter for only private key type KV pairs to be added to the salvaged wallet */ @@ -134,14 +141,14 @@ bool RecoverDatabaseFile(const fs::path& file_path) { // Required in LoadKeyMetadata(): LOCK(dummyWallet.cs_wallet); - fReadOK = ReadKeyValue(&dummyWallet, ssKey, ssValue, strType, strErr); + fReadOK = ReadKeyValue(&dummyWallet, ssKey, ssValue, strType, strErr, KeyFilter); } - if (!WalletBatch::IsKeyType(strType) && strType != DBKeys::HDCHAIN) { + if (!KeyFilter(strType)) { continue; } if (!fReadOK) { - LogPrintf("WARNING: WalletBatch::Recover skipping %s: %s\n", strType, strErr); + warnings.push_back(strprintf(Untranslated("WARNING: WalletBatch::Recover skipping %s: %s"), strType, strErr)); continue; } Dbt datKey(&row.first[0], row.first.size()); diff --git a/src/wallet/salvage.h b/src/wallet/salvage.h index e361930f5..5a8538f94 100644 --- a/src/wallet/salvage.h +++ b/src/wallet/salvage.h @@ -9,6 +9,8 @@ #include <fs.h> #include <streams.h> -bool RecoverDatabaseFile(const fs::path& file_path); +struct bilingual_str; + +bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::vector<bilingual_str>& warnings); #endif // BITCOIN_WALLET_SALVAGE_H diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index a96d97173..14fb1fa89 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -535,7 +535,7 @@ private: //! keeps track of whether Unlock has run a thorough check before bool m_decryption_thoroughly_checked = false; - bool AddDescriptorKeyWithDB(WalletBatch& batch, const CKey& key, const CPubKey &pubkey); + bool AddDescriptorKeyWithDB(WalletBatch& batch, const CKey& key, const CPubKey &pubkey) EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); KeyMap GetKeys() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 1deedede4..f38ccba38 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -29,7 +29,7 @@ typedef std::set<CInputCoin> CoinSet; static std::vector<COutput> vCoins; static NodeContext testNode; static auto testChain = interfaces::MakeChain(testNode); -static CWallet testWallet(testChain.get(), WalletLocation(), CreateDummyWalletDatabase()); +static CWallet testWallet(testChain.get(), "", CreateDummyWalletDatabase()); static CAmount balance = 0; CoinEligibilityFilter filter_standard(1, 6, 0); @@ -283,7 +283,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) // Make sure that can use BnB when there are preset inputs empty_wallet(); { - std::unique_ptr<CWallet> wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), CreateMockWalletDatabase()); + std::unique_ptr<CWallet> wallet = MakeUnique<CWallet>(m_chain.get(), "", CreateMockWalletDatabase()); bool firstRun; wallet->LoadWallet(firstRun); wallet->SetupLegacyScriptPubKeyMan(); diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index 35bd96567..c80310045 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -10,7 +10,7 @@ InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) { - m_chain_client = MakeWalletClient(*m_chain, *Assert(m_node.args), {}); + m_wallet_client = MakeWalletClient(*m_chain, *Assert(m_node.args)); std::string sep; sep += fs::path::preferred_separator; diff --git a/src/wallet/test/init_test_fixture.h b/src/wallet/test/init_test_fixture.h index c95b4f1f6..f5bade77d 100644 --- a/src/wallet/test/init_test_fixture.h +++ b/src/wallet/test/init_test_fixture.h @@ -6,6 +6,7 @@ #define BITCOIN_WALLET_TEST_INIT_TEST_FIXTURE_H #include <interfaces/chain.h> +#include <interfaces/wallet.h> #include <node/context.h> #include <test/util/setup_common.h> @@ -19,7 +20,7 @@ struct InitWalletDirTestingSetup: public BasicTestingSetup { fs::path m_cwd; std::map<std::string, fs::path> m_walletdir_path_cases; std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); - std::unique_ptr<interfaces::ChainClient> m_chain_client; + std::unique_ptr<interfaces::WalletClient> m_wallet_client; }; #endif // BITCOIN_WALLET_TEST_INIT_TEST_FIXTURE_H diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index c228e0600..9b905569f 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -15,7 +15,7 @@ BOOST_FIXTURE_TEST_SUITE(init_tests, InitWalletDirTestingSetup) BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_default) { SetWalletDir(m_walletdir_path_cases["default"]); - bool result = m_chain_client->verify(); + bool result = m_wallet_client->verify(); BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); @@ -25,7 +25,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_default) BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) { SetWalletDir(m_walletdir_path_cases["custom"]); - bool result = m_chain_client->verify(); + bool result = m_wallet_client->verify(); BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["custom"]); @@ -37,7 +37,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_does_not_exist) SetWalletDir(m_walletdir_path_cases["nonexistent"]); { ASSERT_DEBUG_LOG("does not exist"); - bool result = m_chain_client->verify(); + bool result = m_wallet_client->verify(); BOOST_CHECK(result == false); } } @@ -47,7 +47,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_is_not_directory) SetWalletDir(m_walletdir_path_cases["file"]); { ASSERT_DEBUG_LOG("is not a directory"); - bool result = m_chain_client->verify(); + bool result = m_wallet_client->verify(); BOOST_CHECK(result == false); } } @@ -57,7 +57,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_is_not_relative) SetWalletDir(m_walletdir_path_cases["relative"]); { ASSERT_DEBUG_LOG("is a relative path"); - bool result = m_chain_client->verify(); + bool result = m_wallet_client->verify(); BOOST_CHECK(result == false); } } @@ -65,7 +65,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_is_not_relative) BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing) { SetWalletDir(m_walletdir_path_cases["trailing"]); - bool result = m_chain_client->verify(); + bool result = m_wallet_client->verify(); BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); @@ -75,7 +75,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing) BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing2) { SetWalletDir(m_walletdir_path_cases["trailing2"]); - bool result = m_chain_client->verify(); + bool result = m_wallet_client->verify(); BOOST_CHECK(result == true); fs::path walletdir = gArgs.GetArg("-walletdir", ""); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index e416f1604..d5aed99d9 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -35,7 +35,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK compressed { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(pubkeys[0]); @@ -52,7 +52,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK uncompressed { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(uncompressedPubkey); @@ -69,7 +69,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH compressed { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(pubkeys[0])); @@ -86,7 +86,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH uncompressed { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(uncompressedPubkey)); @@ -103,7 +103,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -127,7 +127,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2SH (invalid) { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -145,7 +145,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2WSH (invalid) { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -163,7 +163,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH inside P2WSH (invalid) { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -179,7 +179,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2WSH inside P2WSH (invalid) { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -197,7 +197,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH compressed { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -212,7 +212,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH uncompressed { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -231,7 +231,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // scriptPubKey multisig { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -262,7 +262,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH multisig { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -283,7 +283,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with compressed keys { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -309,7 +309,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with uncompressed key { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -335,7 +335,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig wrapped in P2SH { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -362,7 +362,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // OP_RETURN { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -376,7 +376,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unspendable { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -390,7 +390,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unknown { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -404,7 +404,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Nonstandard { - CWallet keystore(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); diff --git a/src/wallet/test/scriptpubkeyman_tests.cpp b/src/wallet/test/scriptpubkeyman_tests.cpp index 4f1207976..f7c1337b0 100644 --- a/src/wallet/test/scriptpubkeyman_tests.cpp +++ b/src/wallet/test/scriptpubkeyman_tests.cpp @@ -19,7 +19,7 @@ BOOST_AUTO_TEST_CASE(CanProvide) // Set up wallet and keyman variables. NodeContext node; std::unique_ptr<interfaces::Chain> chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); LegacyScriptPubKeyMan& keyman = *wallet.GetOrCreateLegacyScriptPubKeyMan(); // Make a 1 of 2 multisig script diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index 44f9eb5dd..4d6f42761 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -6,10 +6,10 @@ WalletTestingSetup::WalletTestingSetup(const std::string& chainName) : TestingSetup(chainName), - m_wallet(m_chain.get(), WalletLocation(), CreateMockWalletDatabase()) + m_wallet(m_chain.get(), "", CreateMockWalletDatabase()) { bool fFirstRun; m_wallet.LoadWallet(fFirstRun); m_chain_notifications_handler = m_chain->handleNotifications({ &m_wallet, [](CWallet*) {} }); - m_chain_client->registerRpcs(); + m_wallet_client->registerRpcs(); } diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index 99d7cfe92..ba8a5ff1f 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -21,7 +21,7 @@ struct WalletTestingSetup : public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain(m_node); - std::unique_ptr<interfaces::ChainClient> m_chain_client = interfaces::MakeWalletClient(*m_chain, *Assert(m_node.args), {}); + std::unique_ptr<interfaces::WalletClient> m_wallet_client = interfaces::MakeWalletClient(*m_chain, *Assert(m_node.args)); CWallet m_wallet; std::unique_ptr<interfaces::Handler> m_chain_notifications_handler; }; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 7ef06663b..6b98482f9 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -24,9 +24,9 @@ #include <boost/test/unit_test.hpp> #include <univalue.h> -extern UniValue importmulti(const JSONRPCRequest& request); -extern UniValue dumpwallet(const JSONRPCRequest& request); -extern UniValue importwallet(const JSONRPCRequest& request); +RPCHelpMan importmulti(); +RPCHelpMan dumpwallet(); +RPCHelpMan importwallet(); // Ensure that fee levels defined in the wallet are at least as high // as the default levels for node policy. @@ -37,9 +37,12 @@ BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) static std::shared_ptr<CWallet> TestLoadWallet(interfaces::Chain& chain) { + DatabaseOptions options; + DatabaseStatus status; bilingual_str error; std::vector<bilingual_str> warnings; - auto wallet = CWallet::CreateWalletFromFile(chain, WalletLocation(""), error, warnings); + auto database = MakeWalletDatabase("", options, status, error); + auto wallet = CWallet::Create(chain, "", std::move(database), options.create_flags, error, warnings); wallet->postInitProcess(); return wallet; } @@ -85,7 +88,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions fails to read an unknown start block. { - CWallet wallet(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -104,7 +107,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions picks up transactions in both the old // and new block files. { - CWallet wallet(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -130,7 +133,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions only picks transactions in the new block // file. { - CWallet wallet(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -155,7 +158,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions scans no blocks. { - CWallet wallet(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -194,7 +197,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) // before the missing block, and success for a key whose creation time is // after. { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), "", CreateDummyWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); WITH_LOCK(wallet->cs_wallet, wallet->SetLastBlockProcessed(newTip->nHeight, newTip->GetBlockHash())); AddWallet(wallet); @@ -219,7 +222,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) request.params.setArray(); request.params.push_back(keys); - UniValue response = importmulti(request); + UniValue response = importmulti().HandleRequest(request); BOOST_CHECK_EQUAL(response.write(), strprintf("[{\"success\":false,\"error\":{\"code\":-1,\"message\":\"Rescan failed for key with creation " "timestamp %d. There was an error reading a block from time %d, which is after or within %d " @@ -229,7 +232,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) "downloading and rescanning the relevant blocks (see -reindex and -rescan " "options).\"}},{\"success\":true}]", 0, oldTip->GetBlockTimeMax(), TIMESTAMP_WINDOW)); - RemoveWallet(wallet); + RemoveWallet(wallet, nullopt); } } @@ -259,7 +262,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Import key into wallet and call dumpwallet to create backup file. { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), "", CreateDummyWalletDatabase()); { auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); @@ -274,14 +277,14 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) request.params.setArray(); request.params.push_back(backup_file); - ::dumpwallet(request); - RemoveWallet(wallet); + ::dumpwallet().HandleRequest(request); + RemoveWallet(wallet, nullopt); } // Call importwallet RPC and verify all blocks with timestamps >= BLOCK_TIME // were scanned, and no prior blocks were scanned. { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), "", CreateDummyWalletDatabase()); LOCK(wallet->cs_wallet); wallet->SetupLegacyScriptPubKeyMan(); @@ -291,8 +294,8 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) request.params.push_back(backup_file); AddWallet(wallet); wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); - ::importwallet(request); - RemoveWallet(wallet); + ::importwallet().HandleRequest(request); + RemoveWallet(wallet, nullopt); BOOST_CHECK_EQUAL(wallet->mapWallet.size(), 3U); BOOST_CHECK_EQUAL(m_coinbase_txns.size(), 103U); @@ -317,7 +320,7 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) NodeContext node; auto chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); CWalletTx wtx(&wallet, m_coinbase_txns.back()); @@ -492,7 +495,7 @@ public: ListCoinsTestingSetup() { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); - wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), CreateMockWalletDatabase()); + wallet = MakeUnique<CWallet>(m_chain.get(), "", CreateMockWalletDatabase()); { LOCK2(wallet->cs_wallet, ::cs_main); wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); @@ -610,7 +613,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { NodeContext node; auto chain = interfaces::MakeChain(node); - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), CreateDummyWalletDatabase()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), "", CreateDummyWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index cee2f2214..73e11a5b5 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -33,6 +33,8 @@ #include <wallet/coincontrol.h> #include <wallet/fees.h> +#include <univalue.h> + #include <algorithm> #include <assert.h> @@ -54,6 +56,42 @@ static RecursiveMutex cs_wallets; static std::vector<std::shared_ptr<CWallet>> vpwallets GUARDED_BY(cs_wallets); static std::list<LoadWalletFn> g_load_wallet_fns GUARDED_BY(cs_wallets); +bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name) +{ + util::SettingsValue setting_value = chain.getRwSetting("wallet"); + if (!setting_value.isArray()) setting_value.setArray(); + for (const util::SettingsValue& value : setting_value.getValues()) { + if (value.isStr() && value.get_str() == wallet_name) return true; + } + setting_value.push_back(wallet_name); + return chain.updateRwSetting("wallet", setting_value); +} + +bool RemoveWalletSetting(interfaces::Chain& chain, const std::string& wallet_name) +{ + util::SettingsValue setting_value = chain.getRwSetting("wallet"); + if (!setting_value.isArray()) return true; + util::SettingsValue new_value(util::SettingsValue::VARR); + for (const util::SettingsValue& value : setting_value.getValues()) { + if (!value.isStr() || value.get_str() != wallet_name) new_value.push_back(value); + } + if (new_value.size() == setting_value.size()) return true; + return chain.updateRwSetting("wallet", new_value); +} + +static void UpdateWalletSetting(interfaces::Chain& chain, + const std::string& wallet_name, + Optional<bool> load_on_startup, + std::vector<bilingual_str>& warnings) +{ + if (load_on_startup == nullopt) return; + if (load_on_startup.get() && !AddWalletSetting(chain, wallet_name)) { + warnings.emplace_back(Untranslated("Wallet load on startup setting could not be updated, so wallet may not be loaded next node startup.")); + } else if (!load_on_startup.get() && !RemoveWalletSetting(chain, wallet_name)) { + warnings.emplace_back(Untranslated("Wallet load on startup setting could not be updated, so wallet may still be loaded next node startup.")); + } +} + bool AddWallet(const std::shared_ptr<CWallet>& wallet) { LOCK(cs_wallets); @@ -65,18 +103,32 @@ bool AddWallet(const std::shared_ptr<CWallet>& wallet) return true; } -bool RemoveWallet(const std::shared_ptr<CWallet>& wallet) +bool RemoveWallet(const std::shared_ptr<CWallet>& wallet, Optional<bool> load_on_start, std::vector<bilingual_str>& warnings) { assert(wallet); + + interfaces::Chain& chain = wallet->chain(); + std::string name = wallet->GetName(); + // Unregister with the validation interface which also drops shared ponters. wallet->m_chain_notifications_handler.reset(); LOCK(cs_wallets); std::vector<std::shared_ptr<CWallet>>::iterator i = std::find(vpwallets.begin(), vpwallets.end(), wallet); if (i == vpwallets.end()) return false; vpwallets.erase(i); + + // Write the wallet setting + UpdateWalletSetting(chain, name, load_on_start, warnings); + return true; } +bool RemoveWallet(const std::shared_ptr<CWallet>& wallet, Optional<bool> load_on_start) +{ + std::vector<bilingual_str> warnings; + return RemoveWallet(wallet, load_on_start, warnings); +} + std::vector<std::shared_ptr<CWallet>> GetWallets() { LOCK(cs_wallets); @@ -148,48 +200,54 @@ void UnloadWallet(std::shared_ptr<CWallet>&& wallet) } namespace { -std::shared_ptr<CWallet> LoadWalletInternal(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings) +std::shared_ptr<CWallet> LoadWalletInternal(interfaces::Chain& chain, const std::string& name, Optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) { try { - if (!CWallet::Verify(chain, location, error, warnings)) { + std::unique_ptr<WalletDatabase> database = MakeWalletDatabase(name, options, status, error); + if (!database) { error = Untranslated("Wallet file verification failed.") + Untranslated(" ") + error; return nullptr; } - std::shared_ptr<CWallet> wallet = CWallet::CreateWalletFromFile(chain, location, error, warnings); + std::shared_ptr<CWallet> wallet = CWallet::Create(chain, name, std::move(database), options.create_flags, error, warnings); if (!wallet) { error = Untranslated("Wallet loading failed.") + Untranslated(" ") + error; + status = DatabaseStatus::FAILED_LOAD; return nullptr; } AddWallet(wallet); wallet->postInitProcess(); + + // Write the wallet setting + UpdateWalletSetting(chain, name, load_on_start, warnings); + return wallet; } catch (const std::runtime_error& e) { error = Untranslated(e.what()); + status = DatabaseStatus::FAILED_LOAD; return nullptr; } } } // namespace -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings) +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, Optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) { - auto result = WITH_LOCK(g_loading_wallet_mutex, return g_loading_wallet_set.insert(location.GetName())); + auto result = WITH_LOCK(g_loading_wallet_mutex, return g_loading_wallet_set.insert(name)); if (!result.second) { error = Untranslated("Wallet already being loading."); + status = DatabaseStatus::FAILED_LOAD; return nullptr; } - auto wallet = LoadWalletInternal(chain, location, error, warnings); + auto wallet = LoadWalletInternal(chain, name, load_on_start, options, status, error, warnings); WITH_LOCK(g_loading_wallet_mutex, g_loading_wallet_set.erase(result.first)); return wallet; } -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) +std::shared_ptr<CWallet> CreateWallet(interfaces::Chain& chain, const std::string& name, Optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) { - return LoadWallet(chain, WalletLocation(name), error, warnings); -} + uint64_t wallet_creation_flags = options.create_flags; + const SecureString& passphrase = options.create_passphrase; -WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, std::shared_ptr<CWallet>& result) -{ // Indicate that the wallet is actually supposed to be blank and not just blank to make it encrypted bool create_blank = (wallet_creation_flags & WALLET_FLAG_BLANK_WALLET); @@ -198,43 +256,42 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& wallet_creation_flags |= WALLET_FLAG_BLANK_WALLET; } - // Check the wallet file location - WalletLocation location(name); - if (location.Exists()) { - error = strprintf(Untranslated("Wallet %s already exists."), location.GetName()); - return WalletCreationStatus::CREATION_FAILED; - } - // Wallet::Verify will check if we're trying to create a wallet with a duplicate name. - if (!CWallet::Verify(chain, location, error, warnings)) { + std::unique_ptr<WalletDatabase> database = MakeWalletDatabase(name, options, status, error); + if (!database) { error = Untranslated("Wallet file verification failed.") + Untranslated(" ") + error; - return WalletCreationStatus::CREATION_FAILED; + status = DatabaseStatus::FAILED_VERIFY; + return nullptr; } // Do not allow a passphrase when private keys are disabled if (!passphrase.empty() && (wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { error = Untranslated("Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled."); - return WalletCreationStatus::CREATION_FAILED; + status = DatabaseStatus::FAILED_CREATE; + return nullptr; } // Make the wallet - std::shared_ptr<CWallet> wallet = CWallet::CreateWalletFromFile(chain, location, error, warnings, wallet_creation_flags); + std::shared_ptr<CWallet> wallet = CWallet::Create(chain, name, std::move(database), wallet_creation_flags, error, warnings); if (!wallet) { error = Untranslated("Wallet creation failed.") + Untranslated(" ") + error; - return WalletCreationStatus::CREATION_FAILED; + status = DatabaseStatus::FAILED_CREATE; + return nullptr; } // Encrypt the wallet if (!passphrase.empty() && !(wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { if (!wallet->EncryptWallet(passphrase)) { error = Untranslated("Error: Wallet created but failed to encrypt."); - return WalletCreationStatus::ENCRYPTION_FAILED; + status = DatabaseStatus::FAILED_ENCRYPT; + return nullptr; } if (!create_blank) { // Unlock the wallet if (!wallet->Unlock(passphrase)) { error = Untranslated("Error: Wallet was encrypted but could not be unlocked"); - return WalletCreationStatus::ENCRYPTION_FAILED; + status = DatabaseStatus::FAILED_ENCRYPT; + return nullptr; } // Set a seed for the wallet @@ -246,7 +303,8 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& for (auto spk_man : wallet->GetActiveScriptPubKeyMans()) { if (!spk_man->SetupGeneration()) { error = Untranslated("Unable to generate initial keys"); - return WalletCreationStatus::CREATION_FAILED; + status = DatabaseStatus::FAILED_CREATE; + return nullptr; } } } @@ -258,8 +316,12 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& } AddWallet(wallet); wallet->postInitProcess(); - result = wallet; - return WalletCreationStatus::SUCCESS; + + // Write the wallet settings + UpdateWalletSetting(chain, name, load_on_start, warnings); + + status = DatabaseStatus::SUCCESS; + return wallet; } const uint256 CWalletTx::ABANDON_HASH(UINT256_ONE()); @@ -276,7 +338,7 @@ std::string COutput::ToString() const const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const { - LOCK(cs_wallet); + AssertLockHeld(cs_wallet); std::map<uint256, CWalletTx>::const_iterator it = mapWallet.find(hash); if (it == mapWallet.end()) return nullptr; @@ -1210,15 +1272,13 @@ void CWallet::BlockUntilSyncedToCurrentChain() const { isminetype CWallet::IsMine(const CTxIn &txin) const { + AssertLockHeld(cs_wallet); + std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(txin.prevout.hash); + if (mi != mapWallet.end()) { - LOCK(cs_wallet); - std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(txin.prevout.hash); - if (mi != mapWallet.end()) - { - const CWalletTx& prev = (*mi).second; - if (txin.prevout.n < prev.tx->vout.size()) - return IsMine(prev.tx->vout[txin.prevout.n]); - } + const CWalletTx& prev = (*mi).second; + if (txin.prevout.n < prev.tx->vout.size()) + return IsMine(prev.tx->vout[txin.prevout.n]); } return ISMINE_NO; } @@ -1243,16 +1303,19 @@ CAmount CWallet::GetDebit(const CTxIn &txin, const isminefilter& filter) const isminetype CWallet::IsMine(const CTxOut& txout) const { + AssertLockHeld(cs_wallet); return IsMine(txout.scriptPubKey); } isminetype CWallet::IsMine(const CTxDestination& dest) const { + AssertLockHeld(cs_wallet); return IsMine(GetScriptForDestination(dest)); } isminetype CWallet::IsMine(const CScript& script) const { + AssertLockHeld(cs_wallet); isminetype result = ISMINE_NO; for (const auto& spk_man_pair : m_spk_managers) { result = std::max(result, spk_man_pair.second->IsMine(script)); @@ -1264,6 +1327,7 @@ CAmount CWallet::GetCredit(const CTxOut& txout, const isminefilter& filter) cons { if (!MoneyRange(txout.nValue)) throw std::runtime_error(std::string(__func__) + ": value out of range"); + LOCK(cs_wallet); return ((IsMine(txout) & filter) ? txout.nValue : 0); } @@ -1281,13 +1345,12 @@ bool CWallet::IsChange(const CScript& script) const // a better way of identifying which outputs are 'the send' and which are // 'the change' will need to be implemented (maybe extend CWalletTx to remember // which output, if any, was change). + AssertLockHeld(cs_wallet); if (IsMine(script)) { CTxDestination address; if (!ExtractDestination(script, address)) return true; - - LOCK(cs_wallet); if (!FindAddressBookEntry(address)) { return true; } @@ -1297,6 +1360,7 @@ bool CWallet::IsChange(const CScript& script) const CAmount CWallet::GetChange(const CTxOut& txout) const { + AssertLockHeld(cs_wallet); if (!MoneyRange(txout.nValue)) throw std::runtime_error(std::string(__func__) + ": value out of range"); return (IsChange(txout) ? txout.nValue : 0); @@ -1304,6 +1368,7 @@ CAmount CWallet::GetChange(const CTxOut& txout) const bool CWallet::IsMine(const CTransaction& tx) const { + AssertLockHeld(cs_wallet); for (const CTxOut& txout : tx.vout) if (IsMine(txout)) return true; @@ -1362,6 +1427,7 @@ CAmount CWallet::GetCredit(const CTransaction& tx, const isminefilter& filter) c CAmount CWallet::GetChange(const CTransaction& tx) const { + LOCK(cs_wallet); CAmount nChange = 0; for (const CTxOut& txout : tx.vout) { @@ -1597,6 +1663,7 @@ void CWalletTx::GetAmounts(std::list<COutputEntry>& listReceived, nFee = nDebit - nValueOut; } + LOCK(pwallet->cs_wallet); // Sent/received. for (unsigned int i = 0; i < tx->vout.size(); ++i) { @@ -1965,36 +2032,38 @@ bool CWalletTx::InMempool() const bool CWalletTx::IsTrusted() const { - std::set<uint256> s; - return IsTrusted(s); + std::set<uint256> trusted_parents; + LOCK(pwallet->cs_wallet); + return pwallet->IsTrusted(*this, trusted_parents); } -bool CWalletTx::IsTrusted(std::set<uint256>& trusted_parents) const +bool CWallet::IsTrusted(const CWalletTx& wtx, std::set<uint256>& trusted_parents) const { + AssertLockHeld(cs_wallet); // Quick answer in most cases - if (!pwallet->chain().checkFinalTx(*tx)) return false; - int nDepth = GetDepthInMainChain(); + if (!chain().checkFinalTx(*wtx.tx)) return false; + int nDepth = wtx.GetDepthInMainChain(); if (nDepth >= 1) return true; if (nDepth < 0) return false; // using wtx's cached debit - if (!pwallet->m_spend_zero_conf_change || !IsFromMe(ISMINE_ALL)) return false; + if (!m_spend_zero_conf_change || !wtx.IsFromMe(ISMINE_ALL)) return false; // Don't trust unconfirmed transactions from us unless they are in the mempool. - if (!InMempool()) return false; + if (!wtx.InMempool()) return false; // Trusted if all inputs are from us and are in the mempool: - for (const CTxIn& txin : tx->vin) + for (const CTxIn& txin : wtx.tx->vin) { // Transactions not sent by us: not trusted - const CWalletTx* parent = pwallet->GetWalletTx(txin.prevout.hash); + const CWalletTx* parent = GetWalletTx(txin.prevout.hash); if (parent == nullptr) return false; const CTxOut& parentOut = parent->tx->vout[txin.prevout.n]; // Check that this specific input being spent is trusted - if (pwallet->IsMine(parentOut) != ISMINE_SPENDABLE) return false; + if (IsMine(parentOut) != ISMINE_SPENDABLE) return false; // If we've already trusted this parent, continue if (trusted_parents.count(parent->GetHash())) continue; // Recurse to check that the parent is also trusted - if (!parent->IsTrusted(trusted_parents)) return false; + if (!IsTrusted(*parent, trusted_parents)) return false; trusted_parents.insert(parent->GetHash()); } return true; @@ -2080,7 +2149,7 @@ CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) cons for (const auto& entry : mapWallet) { const CWalletTx& wtx = entry.second; - const bool is_trusted{wtx.IsTrusted(trusted_parents)}; + const bool is_trusted{IsTrusted(wtx, trusted_parents)}; const int tx_depth{wtx.GetDepthInMainChain()}; const CAmount tx_credit_mine{wtx.GetAvailableCredit(/* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; const CAmount tx_credit_watchonly{wtx.GetAvailableCredit(/* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; @@ -2148,7 +2217,7 @@ void CWallet::AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe, const if (nDepth == 0 && !wtx.InMempool()) continue; - bool safeTx = wtx.IsTrusted(trusted_parents); + bool safeTx = IsTrusted(wtx, trusted_parents); // We should not consider coins from transactions that are replacing // other transactions. @@ -2284,6 +2353,7 @@ std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins() const const CTxOut& CWallet::FindNonChangeParentOutput(const CTransaction& tx, int output) const { + AssertLockHeld(cs_wallet); const CTransaction* ptx = &tx; int n = output; while (IsChange(ptx->vout[n]) && ptx->vin.size() > 0) { @@ -2320,27 +2390,15 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibil for (OutputGroup& group : groups) { if (!group.EligibleForSpending(eligibility_filter)) continue; - group.fee = 0; - group.long_term_fee = 0; - group.effective_value = 0; - for (auto it = group.m_outputs.begin(); it != group.m_outputs.end(); ) { - const CInputCoin& coin = *it; - CAmount effective_value = coin.txout.nValue - (coin.m_input_bytes < 0 ? 0 : coin_selection_params.effective_fee.GetFee(coin.m_input_bytes)); - // Only include outputs that are positive effective value (i.e. not dust) - if (effective_value > 0) { - group.fee += coin.m_input_bytes < 0 ? 0 : coin_selection_params.effective_fee.GetFee(coin.m_input_bytes); - group.long_term_fee += coin.m_input_bytes < 0 ? 0 : long_term_feerate.GetFee(coin.m_input_bytes); - if (coin_selection_params.m_subtract_fee_outputs) { - group.effective_value += coin.txout.nValue; - } else { - group.effective_value += effective_value; - } - ++it; - } else { - it = group.Discard(coin); - } + if (coin_selection_params.m_subtract_fee_outputs) { + // Set the effective feerate to 0 as we don't want to use the effective value since the fees will be deducted from the output + group.SetFees(CFeeRate(0) /* effective_feerate */, long_term_feerate); + } else { + group.SetFees(coin_selection_params.effective_fee, long_term_feerate); } - if (group.effective_value > 0) utxo_pool.push_back(group); + + OutputGroup pos_group = group.GetPositiveOnlyGroup(); + if (pos_group.effective_value > 0) utxo_pool.push_back(pos_group); } // Calculate the fees for things that aren't inputs CAmount not_input_fees = coin_selection_params.effective_fee.GetFee(coin_selection_params.tx_noinputs_size); @@ -2486,23 +2544,6 @@ bool CWallet::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, } // At this point, one input was not fully signed otherwise we would have exited already - // Find that input and figure out what went wrong. - for (unsigned int i = 0; i < tx.vin.size(); i++) { - // Get the prevout - CTxIn& txin = tx.vin[i]; - auto coin = coins.find(txin.prevout); - if (coin == coins.end() || coin->second.IsSpent()) { - input_errors[i] = "Input not found or already spent"; - continue; - } - - // Check if this input is complete - SignatureData sigdata = DataFromTransaction(tx, i, coin->second.out); - if (!sigdata.complete) { - input_errors[i] = "Unable to sign input, missing keys"; - continue; - } - } return false; } @@ -2609,10 +2650,11 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC if (!coinControl.IsSelected(txin.prevout)) { tx.vin.push_back(txin); - if (lockUnspents) { - LockCoin(txin.prevout); - } } + if (lockUnspents) { + LockCoin(txin.prevout); + } + } return true; @@ -2706,7 +2748,14 @@ OutputType CWallet::TransactionChangeType(const Optional<OutputType>& change_typ return m_default_address_type; } -bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, const CCoinControl& coin_control, bool sign) +bool CWallet::CreateTransactionInternal( + const std::vector<CRecipient>& vecSend, + CTransactionRef& tx, + CAmount& nFeeRet, + int& nChangePosInOut, + bilingual_str& error, + const CCoinControl& coin_control, + bool sign) { CAmount nValue = 0; const OutputType change_type = TransactionChangeType(coin_control.m_change_type ? *coin_control.m_change_type : m_default_change_type, vecSend); @@ -3061,6 +3110,39 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CTransac return true; } +bool CWallet::CreateTransaction( + const std::vector<CRecipient>& vecSend, + CTransactionRef& tx, + CAmount& nFeeRet, + int& nChangePosInOut, + bilingual_str& error, + const CCoinControl& coin_control, + bool sign) +{ + int nChangePosIn = nChangePosInOut; + CTransactionRef tx2 = tx; + bool res = CreateTransactionInternal(vecSend, tx, nFeeRet, nChangePosInOut, error, coin_control, sign); + // try with avoidpartialspends unless it's enabled already + if (res && nFeeRet > 0 /* 0 means non-functional fee rate estimation */ && m_max_aps_fee > -1 && !coin_control.m_avoid_partial_spends) { + CCoinControl tmp_cc = coin_control; + tmp_cc.m_avoid_partial_spends = true; + CAmount nFeeRet2; + int nChangePosInOut2 = nChangePosIn; + bilingual_str error2; // fired and forgotten; if an error occurs, we discard the results + if (CreateTransactionInternal(vecSend, tx2, nFeeRet2, nChangePosInOut2, error2, tmp_cc, sign)) { + // if fee of this alternative one is within the range of the max fee, we use this one + const bool use_aps = nFeeRet2 <= nFeeRet + m_max_aps_fee; + WalletLogPrintf("Fee non-grouped = %lld, grouped = %lld, using %s\n", nFeeRet, nFeeRet2, use_aps ? "grouped" : "non-grouped"); + if (use_aps) { + tx = tx2; + nFeeRet = nFeeRet2; + nChangePosInOut = nChangePosInOut2; + } + } + } + return res; +} + void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm) { LOCK(cs_wallet); @@ -3161,28 +3243,10 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 return DBErrors::LOAD_OK; } -DBErrors CWallet::ZapWalletTx(std::list<CWalletTx>& vWtx) -{ - DBErrors nZapWalletTxRet = WalletBatch(*database,"cr+").ZapWalletTx(vWtx); - if (nZapWalletTxRet == DBErrors::NEED_REWRITE) - { - if (database->Rewrite("\x04pool")) - { - for (const auto& spk_man_pair : m_spk_managers) { - spk_man_pair.second->RewriteDB(); - } - } - } - - if (nZapWalletTxRet != DBErrors::LOAD_OK) - return nZapWalletTxRet; - - return DBErrors::LOAD_OK; -} - bool CWallet::SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& address, const std::string& strName, const std::string& strPurpose) { bool fUpdated = false; + bool is_mine; { LOCK(cs_wallet); std::map<CTxDestination, CAddressBookData>::iterator mi = m_address_book.find(address); @@ -3190,8 +3254,9 @@ bool CWallet::SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& add m_address_book[address].SetLabel(strName); if (!strPurpose.empty()) /* update purpose only if requested */ m_address_book[address].purpose = strPurpose; + is_mine = IsMine(address) != ISMINE_NO; } - NotifyAddressBookChanged(this, address, strName, IsMine(address) != ISMINE_NO, + NotifyAddressBookChanged(this, address, strName, is_mine, strPurpose, (fUpdated ? CT_UPDATED : CT_NEW) ); if (!strPurpose.empty() && !batch.WritePurpose(EncodeDestination(address), strPurpose)) return false; @@ -3206,30 +3271,31 @@ bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& s bool CWallet::DelAddressBook(const CTxDestination& address) { - // If we want to delete receiving addresses, we need to take care that DestData "used" (and possibly newer DestData) gets preserved (and the "deleted" address transformed into a change entry instead of actually being deleted) - // NOTE: This isn't a problem for sending addresses because they never have any DestData yet! - // When adding new DestData, it should be considered here whether to retain or delete it (or move it?). - if (IsMine(address)) { - WalletLogPrintf("%s called with IsMine address, NOT SUPPORTED. Please report this bug! %s\n", __func__, PACKAGE_BUGREPORT); - return false; - } - + bool is_mine; + WalletBatch batch(*database); { LOCK(cs_wallet); - + // If we want to delete receiving addresses, we need to take care that DestData "used" (and possibly newer DestData) gets preserved (and the "deleted" address transformed into a change entry instead of actually being deleted) + // NOTE: This isn't a problem for sending addresses because they never have any DestData yet! + // When adding new DestData, it should be considered here whether to retain or delete it (or move it?). + if (IsMine(address)) { + WalletLogPrintf("%s called with IsMine address, NOT SUPPORTED. Please report this bug! %s\n", __func__, PACKAGE_BUGREPORT); + return false; + } // Delete destdata tuples associated with address std::string strAddress = EncodeDestination(address); for (const std::pair<const std::string, std::string> &item : m_address_book[address].destdata) { - WalletBatch(*database).EraseDestData(strAddress, item.first); + batch.EraseDestData(strAddress, item.first); } m_address_book.erase(address); + is_mine = IsMine(address) != ISMINE_NO; } - NotifyAddressBookChanged(this, address, "", IsMine(address) != ISMINE_NO, "", CT_DELETED); + NotifyAddressBookChanged(this, address, "", is_mine, "", CT_DELETED); - WalletBatch(*database).ErasePurpose(EncodeDestination(address)); - return WalletBatch(*database).EraseName(EncodeDestination(address)); + batch.ErasePurpose(EncodeDestination(address)); + return batch.EraseName(EncodeDestination(address)); } size_t CWallet::KeypoolCountExternalKeys() const @@ -3334,7 +3400,7 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances() const { const CWalletTx& wtx = walletEntry.second; - if (!wtx.IsTrusted(trusted_parents)) + if (!IsTrusted(wtx, trusted_parents)) continue; if (wtx.IsImmatureCoinBase()) @@ -3353,9 +3419,6 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances() const continue; CAmount n = IsSpent(walletEntry.first, i) ? 0 : wtx.tx->vout[i].nValue; - - if (!balances.count(addr)) - balances[addr] = 0; balances[addr] += n; } } @@ -3716,7 +3779,7 @@ std::vector<std::string> CWallet::GetDestValues(const std::string& prefix) const return values; } -bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error_string, std::vector<bilingual_str>& warnings) +std::unique_ptr<WalletDatabase> MakeWalletDatabase(const std::string& name, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error_string) { // Do some checking on wallet path. It should be either a: // @@ -3724,54 +3787,25 @@ bool CWallet::Verify(interfaces::Chain& chain, const WalletLocation& location, b // 2. Path to an existing directory. // 3. Path to a symlink to a directory. // 4. For backwards compatibility, the name of a data file in -walletdir. - LOCK(cs_wallets); - const fs::path& wallet_path = location.GetPath(); + const fs::path& wallet_path = fs::absolute(name, GetWalletDir()); fs::file_type path_type = fs::symlink_status(wallet_path).type(); if (!(path_type == fs::file_not_found || path_type == fs::directory_file || (path_type == fs::symlink_file && fs::is_directory(wallet_path)) || - (path_type == fs::regular_file && fs::path(location.GetName()).filename() == location.GetName()))) { + (path_type == fs::regular_file && fs::path(name).filename() == name))) { error_string = Untranslated(strprintf( "Invalid -wallet path '%s'. -wallet path should point to a directory where wallet.dat and " "database/log.?????????? files can be stored, a location where such a directory could be created, " "or (for backwards compatibility) the name of an existing data file in -walletdir (%s)", - location.GetName(), GetWalletDir())); - return false; - } - - // Make sure that the wallet path doesn't clash with an existing wallet path - if (IsWalletLoaded(wallet_path)) { - error_string = Untranslated(strprintf("Error loading wallet %s. Duplicate -wallet filename specified.", location.GetName())); - return false; - } - - // Keep same database environment instance across Verify/Recover calls below. - std::unique_ptr<WalletDatabase> database = CreateWalletDatabase(wallet_path); - - try { - return database->Verify(error_string); - } catch (const fs::filesystem_error& e) { - error_string = Untranslated(strprintf("Error loading wallet %s. %s", location.GetName(), fsbridge::get_filesystem_error_message(e))); - return false; + name, GetWalletDir())); + status = DatabaseStatus::FAILED_BAD_PATH; + return nullptr; } + return MakeDatabase(wallet_path, options, status, error_string); } -std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings, uint64_t wallet_creation_flags) +std::shared_ptr<CWallet> CWallet::Create(interfaces::Chain& chain, const std::string& name, std::unique_ptr<WalletDatabase> database, uint64_t wallet_creation_flags, bilingual_str& error, std::vector<bilingual_str>& warnings) { - const std::string walletFile = WalletDataFilePath(location.GetPath()).string(); - - // needed to restore wallet transaction meta data after -zapwallettxes - std::list<CWalletTx> vWtx; - - if (gArgs.GetBoolArg("-zapwallettxes", false)) { - chain.initMessage(_("Zapping all transactions from wallet...").translated); - - std::unique_ptr<CWallet> tempWallet = MakeUnique<CWallet>(&chain, location, CreateWalletDatabase(location.GetPath())); - DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); - if (nZapWalletRet != DBErrors::LOAD_OK) { - error = strprintf(_("Error loading %s: Wallet corrupted"), walletFile); - return nullptr; - } - } + const std::string& walletFile = database->Filename(); chain.initMessage(_("Loading wallet...").translated); @@ -3779,7 +3813,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, bool fFirstRun = true; // TODO: Can't use std::make_shared because we need a custom deleter but // should be possible to use std::allocate_shared. - std::shared_ptr<CWallet> walletInstance(new CWallet(&chain, location, CreateWalletDatabase(location.GetPath())), ReleaseWallet); + std::shared_ptr<CWallet> walletInstance(new CWallet(&chain, name, std::move(database)), ReleaseWallet); DBErrors nLoadWalletRet = walletInstance->LoadWallet(fFirstRun); if (nLoadWalletRet != DBErrors::LOAD_OK) { if (nLoadWalletRet == DBErrors::CORRUPT) { @@ -3878,6 +3912,22 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->m_min_fee = CFeeRate(n); } + if (gArgs.IsArgSet("-maxapsfee")) { + const std::string max_aps_fee{gArgs.GetArg("-maxapsfee", "")}; + CAmount n = 0; + if (max_aps_fee == "-1") { + n = -1; + } else if (!ParseMoney(max_aps_fee, n)) { + error = AmountErrMsg("maxapsfee", max_aps_fee); + return nullptr; + } + if (n > HIGH_APS_FEE) { + warnings.push_back(AmountHighWarn("-maxapsfee") + Untranslated(" ") + + _("This is the maximum transaction fee you pay (in addition to the normal fee) to prioritize partial spend avoidance over regular coin selection.")); + } + walletInstance->m_max_aps_fee = n; + } + if (gArgs.IsArgSet("-fallbackfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-fallbackfee", ""), nFeePerK)) { @@ -4033,30 +4083,6 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } walletInstance->chainStateFlushed(chain.getTipLocator()); walletInstance->database->IncrementUpdateCounter(); - - // Restore wallet transaction metadata after -zapwallettxes=1 - if (gArgs.GetBoolArg("-zapwallettxes", false) && gArgs.GetArg("-zapwallettxes", "1") != "2") - { - WalletBatch batch(*walletInstance->database); - - for (const CWalletTx& wtxOld : vWtx) - { - uint256 hash = wtxOld.GetHash(); - std::map<uint256, CWalletTx>::iterator mi = walletInstance->mapWallet.find(hash); - if (mi != walletInstance->mapWallet.end()) - { - const CWalletTx* copyFrom = &wtxOld; - CWalletTx* copyTo = &mi->second; - copyTo->mapValue = copyFrom->mapValue; - copyTo->vOrderForm = copyFrom->vOrderForm; - copyTo->nTimeReceived = copyFrom->nTimeReceived; - copyTo->nTimeSmart = copyFrom->nTimeSmart; - copyTo->fFromMe = copyFrom->fFromMe; - copyTo->nOrderPos = copyFrom->nOrderPos; - batch.WriteTx(*copyTo); - } - } - } } { diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index a761caf38..c54480612 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -50,19 +50,14 @@ struct bilingual_str; void UnloadWallet(std::shared_ptr<CWallet>&& wallet); bool AddWallet(const std::shared_ptr<CWallet>& wallet); -bool RemoveWallet(const std::shared_ptr<CWallet>& wallet); +bool RemoveWallet(const std::shared_ptr<CWallet>& wallet, Optional<bool> load_on_start, std::vector<bilingual_str>& warnings); +bool RemoveWallet(const std::shared_ptr<CWallet>& wallet, Optional<bool> load_on_start); std::vector<std::shared_ptr<CWallet>> GetWallets(); std::shared_ptr<CWallet> GetWallet(const std::string& name); -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings); +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, Optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings); +std::shared_ptr<CWallet> CreateWallet(interfaces::Chain& chain, const std::string& name, Optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings); std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet); - -enum class WalletCreationStatus { - SUCCESS, - CREATION_FAILED, - ENCRYPTION_FAILED -}; - -WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& passphrase, uint64_t wallet_creation_flags, const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings, std::shared_ptr<CWallet>& result); +std::unique_ptr<WalletDatabase> MakeWalletDatabase(const std::string& name, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); //! -paytxfee default constexpr CAmount DEFAULT_PAY_TX_FEE = 0; @@ -72,6 +67,16 @@ static const CAmount DEFAULT_FALLBACK_FEE = 0; static const CAmount DEFAULT_DISCARD_FEE = 10000; //! -mintxfee default static const CAmount DEFAULT_TRANSACTION_MINFEE = 1000; +/** + * maximum fee increase allowed to do partial spend avoidance, even for nodes with this feature disabled by default + * + * A value of -1 disables this feature completely. + * A value of 0 (current default) means to attempt to do partial spend avoidance, and use its results if the fees remain *unchanged* + * A value > 0 means to do partial spend avoidance if the fee difference against a regular coin selection instance is in the range [0..value]. + */ +static const CAmount DEFAULT_MAX_AVOIDPARTIALSPEND_FEE = 0; +//! discourage APS fee higher than this amount +constexpr CAmount HIGH_APS_FEE{COIN / 10000}; //! minimum recommended increment for BIP 125 replacement txs static const CAmount WALLET_INCREMENTAL_RELAY_FEE = 5000; //! Default for -spendzeroconfchange @@ -217,7 +222,7 @@ static inline void ReadOrderPos(int64_t& nOrderPos, mapValue_t& mapValue) nOrderPos = -1; // TODO: calculate elsewhere return; } - nOrderPos = atoi64(mapValue["n"].c_str()); + nOrderPos = atoi64(mapValue["n"]); } @@ -265,7 +270,7 @@ int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, class CWalletTx { private: - const CWallet* pwallet; + const CWallet* const pwallet; /** Constant used in hashBlock to indicate tx has been abandoned, only used at * serialization/deserialization to avoid ambiguity with conflicted. @@ -492,7 +497,6 @@ public: bool InMempool() const; bool IsTrusted() const; - bool IsTrusted(std::set<uint256>& trusted_parents) const; int64_t GetTxTime() const; @@ -690,8 +694,8 @@ private: /** Interface for accessing chain state. */ interfaces::Chain* m_chain; - /** Wallet location which includes wallet name (see WalletLocation). */ - WalletLocation m_location; + /** Wallet name: relative directory name or "" for default wallet. */ + std::string m_name; /** Internal database handle. */ std::unique_ptr<WalletDatabase> database; @@ -719,6 +723,8 @@ private: // ScriptPubKeyMan::GetID. In many cases it will be the hash of an internal structure std::map<uint256, std::unique_ptr<ScriptPubKeyMan>> m_spk_managers; + bool CreateTransactionInternal(const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, const CCoinControl& coin_control, bool sign); + public: /* * Main wallet lock. @@ -743,20 +749,18 @@ public: bool SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl& coin_control, CoinSelectionParams& coin_selection_params, bool& bnb_used) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - const WalletLocation& GetLocation() const { return m_location; } - /** Get a name for this wallet for logging/debugging purposes. */ - const std::string& GetName() const { return m_location.GetName(); } + const std::string& GetName() const { return m_name; } typedef std::map<unsigned int, CMasterKey> MasterKeyMap; MasterKeyMap mapMasterKeys; unsigned int nMasterKeyMaxID = 0; /** Construct wallet with specified name and database implementation. */ - CWallet(interfaces::Chain* chain, const WalletLocation& location, std::unique_ptr<WalletDatabase> database) + CWallet(interfaces::Chain* chain, const std::string& name, std::unique_ptr<WalletDatabase> database) : m_chain(chain), - m_location(location), + m_name(name), database(std::move(database)) { } @@ -793,7 +797,8 @@ public: /** Interface for accessing chain state. */ interfaces::Chain& chain() const { assert(m_chain); return *m_chain; } - const CWalletTx* GetWalletTx(const uint256& hash) const; + const CWalletTx* GetWalletTx(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool IsTrusted(const CWalletTx& wtx, std::set<uint256>& trusted_parents) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! check whether we are allowed to upgrade (or already support) to the named feature bool CanSupportFeature(enum WalletFeature wf) const override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); return nWalletMaxVersion >= wf; } @@ -1008,6 +1013,7 @@ public: */ CFeeRate m_fallback_fee{DEFAULT_FALLBACK_FEE}; CFeeRate m_discard_rate{DEFAULT_DISCARD_FEE}; + CAmount m_max_aps_fee{DEFAULT_MAX_AVOIDPARTIALSPEND_FEE}; //!< note: this is absolute fee, not fee rate OutputType m_default_address_type{DEFAULT_ADDRESS_TYPE}; /** * Default output type for change outputs. When unset, automatically choose type @@ -1038,20 +1044,20 @@ public: bool GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, std::string& error); bool GetNewChangeDestination(const OutputType type, CTxDestination& dest, std::string& error); - isminetype IsMine(const CTxDestination& dest) const; - isminetype IsMine(const CScript& script) const; - isminetype IsMine(const CTxIn& txin) const; + isminetype IsMine(const CTxDestination& dest) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + isminetype IsMine(const CScript& script) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + isminetype IsMine(const CTxIn& txin) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** * Returns amount of debit if the input matches the * filter, otherwise returns 0 */ CAmount GetDebit(const CTxIn& txin, const isminefilter& filter) const; - isminetype IsMine(const CTxOut& txout) const; + isminetype IsMine(const CTxOut& txout) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); CAmount GetCredit(const CTxOut& txout, const isminefilter& filter) const; - bool IsChange(const CTxOut& txout) const; - bool IsChange(const CScript& script) const; - CAmount GetChange(const CTxOut& txout) const; - bool IsMine(const CTransaction& tx) const; + bool IsChange(const CTxOut& txout) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool IsChange(const CScript& script) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + CAmount GetChange(const CTxOut& txout) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool IsMine(const CTransaction& tx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** should probably be renamed to IsRelevantToMe */ bool IsFromMe(const CTransaction& tx) const; CAmount GetDebit(const CTransaction& tx, const isminefilter& filter) const; @@ -1062,7 +1068,6 @@ public: void chainStateFlushed(const CBlockLocator& loc) override; DBErrors LoadWallet(bool& fFirstRunRet); - DBErrors ZapWalletTx(std::list<CWalletTx>& vWtx); DBErrors ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool SetAddressBook(const CTxDestination& address, const std::string& strName, const std::string& purpose); @@ -1140,11 +1145,8 @@ public: /** Mark a transaction as replaced by another transaction (e.g., BIP 125). */ bool MarkReplaced(const uint256& originalHash, const uint256& newHash); - //! Verify wallet naming and perform salvage on the wallet if required - static bool Verify(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error_string, std::vector<bilingual_str>& warnings); - /* Initializes the wallet, returns a new CWallet instance or a null pointer in case of an error */ - static std::shared_ptr<CWallet> CreateWalletFromFile(interfaces::Chain& chain, const WalletLocation& location, bilingual_str& error, std::vector<bilingual_str>& warnings, uint64_t wallet_creation_flags = 0); + static std::shared_ptr<CWallet> Create(interfaces::Chain& chain, const std::string& name, std::unique_ptr<WalletDatabase> database, uint64_t wallet_creation_flags, bilingual_str& error, std::vector<bilingual_str>& warnings); /** * Wallet post-init setup @@ -1166,7 +1168,7 @@ public: * Obviously holding cs_main/cs_wallet when going into this call may cause * deadlock */ - void BlockUntilSyncedToCurrentChain() const LOCKS_EXCLUDED(cs_main, cs_wallet); + void BlockUntilSyncedToCurrentChain() const EXCLUSIVE_LOCKS_REQUIRED(!::cs_main, !cs_wallet); /** set a single wallet flag */ void SetWalletFlag(uint64_t flags); @@ -1272,7 +1274,7 @@ public: void LoadActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal); //! Create new DescriptorScriptPubKeyMans and add them to the wallet - void SetupDescriptorScriptPubKeyMans(); + void SetupDescriptorScriptPubKeyMans() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Return the DescriptorScriptPubKeyMan for a WalletDescriptor if it is already in the wallet DescriptorScriptPubKeyMan* GetDescriptorScriptPubKeyMan(const WalletDescriptor& desc) const; @@ -1327,4 +1329,11 @@ public: // be IsAllFromMe). int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, bool use_max_sig = false) EXCLUSIVE_LOCKS_REQUIRED(wallet->cs_wallet); int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const std::vector<CTxOut>& txouts, bool use_max_sig = false); + +//! Add wallet name to persistent configuration so it will be loaded on startup. +bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); + +//! Remove wallet name from persistent configuration so it will not be loaded on startup. +bool RemoveWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); + #endif // BITCOIN_WALLET_WALLET_H diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index fa6814d0d..5bf21eb91 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -13,6 +13,8 @@ #include <util/bip32.h> #include <util/system.h> #include <util/time.h> +#include <util/translation.h> +#include <wallet/bdb.h> #include <wallet/wallet.h> #include <atomic> @@ -263,13 +265,17 @@ public: static bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, - CWalletScanState &wss, std::string& strType, std::string& strErr) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) + CWalletScanState &wss, std::string& strType, std::string& strErr, const KeyFilterFn& filter_fn = nullptr) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { try { // Unserialize // Taking advantage of the fact that pair serialization // is just the two items serialized one after the other ssKey >> strType; + // If we have a filter, check if this matches the filter + if (filter_fn && !filter_fn(strType)) { + return true; + } if (strType == DBKeys::NAME) { std::string strAddress; ssKey >> strAddress; @@ -668,11 +674,11 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, return true; } -bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr) +bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr, const KeyFilterFn& filter_fn) { CWalletScanState dummy_wss; LOCK(pwallet->cs_wallet); - return ReadKeyValue(pwallet, ssKey, ssValue, dummy_wss, strType, strErr); + return ReadKeyValue(pwallet, ssKey, ssValue, dummy_wss, strType, strErr, filter_fn); } bool WalletBatch::IsKeyType(const std::string& strType) @@ -926,23 +932,6 @@ DBErrors WalletBatch::ZapSelectTx(std::vector<uint256>& vTxHashIn, std::vector<u return DBErrors::LOAD_OK; } -DBErrors WalletBatch::ZapWalletTx(std::list<CWalletTx>& vWtx) -{ - // build list of wallet TXs - std::vector<uint256> vTxHash; - DBErrors err = FindWalletTx(vTxHash, vWtx); - if (err != DBErrors::LOAD_OK) - return err; - - // erase each wallet TX - for (const uint256& hash : vTxHash) { - if (!EraseTx(hash)) - return DBErrors::CORRUPT; - } - - return DBErrors::LOAD_OK; -} - void MaybeCompactWalletDB() { static std::atomic<bool> fOneThread(false); @@ -1006,16 +995,41 @@ bool WalletBatch::TxnAbort() return m_batch->TxnAbort(); } -bool IsWalletLoaded(const fs::path& wallet_path) +std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error) { - return IsBDBWalletLoaded(wallet_path); -} + bool exists; + try { + exists = fs::symlink_status(path).type() != fs::file_not_found; + } catch (const fs::filesystem_error& e) { + error = Untranslated(strprintf("Failed to access database path '%s': %s", path.string(), fsbridge::get_filesystem_error_message(e))); + status = DatabaseStatus::FAILED_BAD_PATH; + return nullptr; + } -/** Return object for accessing database at specified path. */ -std::unique_ptr<WalletDatabase> CreateWalletDatabase(const fs::path& path) -{ - std::string filename; - return MakeUnique<BerkeleyDatabase>(GetWalletEnv(path, filename), std::move(filename)); + Optional<DatabaseFormat> format; + if (exists) { + if (ExistsBerkeleyDatabase(path)) { + format = DatabaseFormat::BERKELEY; + } + } else if (options.require_existing) { + error = Untranslated(strprintf("Failed to load database path '%s'. Path does not exist.", path.string())); + status = DatabaseStatus::FAILED_NOT_FOUND; + return nullptr; + } + + if (!format && options.require_existing) { + error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in recognized format.", path.string())); + status = DatabaseStatus::FAILED_BAD_FORMAT; + return nullptr; + } + + if (format && options.require_create) { + error = Untranslated(strprintf("Failed to create database path '%s'. Database already exists.", path.string())); + status = DatabaseStatus::FAILED_ALREADY_EXISTS; + return nullptr; + } + + return MakeBerkeleyDatabase(path, options, status, error); } /** Return object for accessing dummy database with no read/write capabilities. */ diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 7c5bf7652..eda810ed8 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -257,14 +257,9 @@ public: DBErrors LoadWallet(CWallet* pwallet); DBErrors FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWalletTx>& vWtx); - DBErrors ZapWalletTx(std::list<CWalletTx>& vWtx); DBErrors ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut); /* Function to determine if a certain KV/key-type is a key (cryptographical key) type */ static bool IsKeyType(const std::string& strType); - /* verifies the database environment */ - static bool VerifyEnvironment(const fs::path& wallet_path, bilingual_str& errorStr); - /* verifies the database file */ - static bool VerifyDatabaseFile(const fs::path& wallet_path, bilingual_str& errorStr); //! write the hdchain model (external chain child index counter) bool WriteHDChain(const CHDChain& chain); @@ -284,14 +279,11 @@ private: //! Compacts BDB state so that wallet.dat is self-contained (if there are changes) void MaybeCompactWalletDB(); -//! Unserialize a given Key-Value pair and load it into the wallet -bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr); - -/** Return whether a wallet database is currently loaded. */ -bool IsWalletLoaded(const fs::path& wallet_path); +//! Callback for filtering key types to deserialize in ReadKeyValue +using KeyFilterFn = std::function<bool(const std::string&)>; -/** Return object for accessing database at specified path. */ -std::unique_ptr<WalletDatabase> CreateWalletDatabase(const fs::path& path); +//! Unserialize a given Key-Value pair and load it into the wallet +bool ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr, const KeyFilterFn& filter_fn = nullptr); /** Return object for accessing dummy database with no read/write capabilities. */ std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase(); diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 9f25b1ae7..4452840eb 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -21,21 +21,9 @@ static void WalletToolReleaseWallet(CWallet* wallet) delete wallet; } -static std::shared_ptr<CWallet> CreateWallet(const std::string& name, const fs::path& path) +static void WalletCreate(CWallet* wallet_instance) { - if (fs::exists(path)) { - tfm::format(std::cerr, "Error: File exists already\n"); - return nullptr; - } - // dummy chain interface - std::shared_ptr<CWallet> wallet_instance(new CWallet(nullptr /* chain */, WalletLocation(name), CreateWalletDatabase(path)), WalletToolReleaseWallet); LOCK(wallet_instance->cs_wallet); - bool first_run = true; - DBErrors load_wallet_ret = wallet_instance->LoadWallet(first_run); - if (load_wallet_ret != DBErrors::LOAD_OK) { - tfm::format(std::cerr, "Error creating %s", name); - return nullptr; - } wallet_instance->SetMinVersion(FEATURE_HD_SPLIT); @@ -46,18 +34,26 @@ static std::shared_ptr<CWallet> CreateWallet(const std::string& name, const fs:: tfm::format(std::cout, "Topping up keypool...\n"); wallet_instance->TopUpKeyPool(); - return wallet_instance; } -static std::shared_ptr<CWallet> LoadWallet(const std::string& name, const fs::path& path) +static std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::path& path, bool create) { - if (!fs::exists(path)) { - tfm::format(std::cerr, "Error: Wallet files does not exist\n"); + DatabaseOptions options; + DatabaseStatus status; + if (create) { + options.require_create = true; + } else { + options.require_existing = true; + } + bilingual_str error; + std::unique_ptr<WalletDatabase> database = MakeDatabase(path, options, status, error); + if (!database) { + tfm::format(std::cerr, "%s\n", error.original); return nullptr; } // dummy chain interface - std::shared_ptr<CWallet> wallet_instance(new CWallet(nullptr /* chain */, WalletLocation(name), CreateWalletDatabase(path)), WalletToolReleaseWallet); + std::shared_ptr<CWallet> wallet_instance{new CWallet(nullptr /* chain */, name, std::move(database)), WalletToolReleaseWallet}; DBErrors load_wallet_ret; try { bool first_run; @@ -89,6 +85,8 @@ static std::shared_ptr<CWallet> LoadWallet(const std::string& name, const fs::pa } } + if (create) WalletCreate(wallet_instance.get()); + return wallet_instance; } @@ -104,50 +102,35 @@ static void WalletShowInfo(CWallet* wallet_instance) tfm::format(std::cout, "Address Book: %zu\n", wallet_instance->m_address_book.size()); } -static bool SalvageWallet(const fs::path& path) -{ - // Create a Database handle to allow for the db to be initialized before recovery - std::unique_ptr<WalletDatabase> database = CreateWalletDatabase(path); - - // Initialize the environment before recovery - bilingual_str error_string; - try { - database->Verify(error_string); - } catch (const fs::filesystem_error& e) { - error_string = Untranslated(strprintf("Error loading wallet. %s", fsbridge::get_filesystem_error_message(e))); - } - if (!error_string.original.empty()) { - tfm::format(std::cerr, "Failed to open wallet for salvage :%s\n", error_string.original); - return false; - } - - // Perform the recovery - return RecoverDatabaseFile(path); -} - bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) { fs::path path = fs::absolute(name, GetWalletDir()); if (command == "create") { - std::shared_ptr<CWallet> wallet_instance = CreateWallet(name, path); + std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, /* create= */ true); if (wallet_instance) { WalletShowInfo(wallet_instance.get()); wallet_instance->Close(); } } else if (command == "info" || command == "salvage") { - if (!fs::exists(path)) { - tfm::format(std::cerr, "Error: no wallet file at %s\n", name); - return false; - } - if (command == "info") { - std::shared_ptr<CWallet> wallet_instance = LoadWallet(name, path); + std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, /* create= */ false); if (!wallet_instance) return false; WalletShowInfo(wallet_instance.get()); wallet_instance->Close(); } else if (command == "salvage") { - return SalvageWallet(path); + bilingual_str error; + std::vector<bilingual_str> warnings; + bool ret = RecoverDatabaseFile(path, error, warnings); + if (!ret) { + for (const auto& warning : warnings) { + tfm::format(std::cerr, "%s\n", warning.original); + } + if (!error.empty()) { + tfm::format(std::cerr, "%s\n", error.original); + } + } + return ret; } } else { tfm::format(std::cerr, "Invalid command: %s\n", command); diff --git a/src/wallet/wallettool.h b/src/wallet/wallettool.h index 8ee3355f0..d0b8d6812 100644 --- a/src/wallet/wallettool.h +++ b/src/wallet/wallettool.h @@ -9,8 +9,6 @@ namespace WalletTool { -std::shared_ptr<CWallet> CreateWallet(const std::string& name, const fs::path& path); -std::shared_ptr<CWallet> LoadWallet(const std::string& name, const fs::path& path); void WalletShowInfo(CWallet* wallet_instance); bool ExecuteWalletToolFunc(const std::string& command, const std::string& file); diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index 8bac0608a..e4c72aed9 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -29,7 +29,7 @@ fs::path GetWalletDir() return path; } -static bool IsBerkeleyBtree(const fs::path& path) +bool IsBerkeleyBtree(const fs::path& path) { if (!fs::exists(path)) return false; @@ -91,19 +91,3 @@ std::vector<fs::path> ListWalletDir() return paths; } - -WalletLocation::WalletLocation(const std::string& name) - : m_name(name) - , m_path(fs::absolute(name, GetWalletDir())) -{ -} - -bool WalletLocation::Exists() const -{ - fs::path path = m_path; - // For the default wallet, check specifically for the wallet.dat file - if (m_name.empty()) { - path = fs::absolute("wallet.dat", m_path); - } - return fs::symlink_status(path).type() != fs::file_not_found; -} diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index a4e4fda8a..afdcb2e18 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -67,26 +67,6 @@ fs::path GetWalletDir(); //! Get wallets in wallet directory. std::vector<fs::path> ListWalletDir(); -//! The WalletLocation class provides wallet information. -class WalletLocation final -{ - std::string m_name; - fs::path m_path; - -public: - explicit WalletLocation() {} - explicit WalletLocation(const std::string& name); - - //! Get wallet name. - const std::string& GetName() const { return m_name; } - - //! Get wallet absolute path. - const fs::path& GetPath() const { return m_path; } - - //! Return whether the wallet exists. - bool Exists() const; -}; - /** Descriptor with some wallet metadata */ class WalletDescriptor { diff --git a/src/zmq/zmqabstractnotifier.cpp b/src/zmq/zmqabstractnotifier.cpp index aae760add..0d0428f3c 100644 --- a/src/zmq/zmqabstractnotifier.cpp +++ b/src/zmq/zmqabstractnotifier.cpp @@ -4,6 +4,8 @@ #include <zmq/zmqabstractnotifier.h> +#include <cassert> + const int CZMQAbstractNotifier::DEFAULT_ZMQ_SNDHWM; CZMQAbstractNotifier::~CZMQAbstractNotifier() diff --git a/src/zmq/zmqabstractnotifier.h b/src/zmq/zmqabstractnotifier.h index 887dde7b2..34d7e5ef0 100644 --- a/src/zmq/zmqabstractnotifier.h +++ b/src/zmq/zmqabstractnotifier.h @@ -5,12 +5,16 @@ #ifndef BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H #define BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H -#include <zmq/zmqconfig.h> +#include <util/memory.h> + +#include <memory> +#include <string> class CBlockIndex; +class CTransaction; class CZMQAbstractNotifier; -typedef CZMQAbstractNotifier* (*CZMQNotifierFactory)(); +using CZMQNotifierFactory = std::unique_ptr<CZMQAbstractNotifier> (*)(); class CZMQAbstractNotifier { @@ -21,9 +25,9 @@ public: virtual ~CZMQAbstractNotifier(); template <typename T> - static CZMQAbstractNotifier* Create() + static std::unique_ptr<CZMQAbstractNotifier> Create() { - return new T(); + return MakeUnique<T>(); } std::string GetType() const { return type; } diff --git a/src/zmq/zmqconfig.h b/src/zmq/zmqconfig.h deleted file mode 100644 index 5f0036206..000000000 --- a/src/zmq/zmqconfig.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) 2014-2019 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_ZMQ_ZMQCONFIG_H -#define BITCOIN_ZMQ_ZMQCONFIG_H - -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#include <stdarg.h> - -#if ENABLE_ZMQ -#include <zmq.h> -#endif - -#include <primitives/transaction.h> - -void zmqError(const char *str); - -#endif // BITCOIN_ZMQ_ZMQCONFIG_H diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index d55b106e0..a22772bae 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -4,15 +4,13 @@ #include <zmq/zmqnotificationinterface.h> #include <zmq/zmqpublishnotifier.h> +#include <zmq/zmqutil.h> + +#include <zmq.h> #include <validation.h> #include <util/system.h> -void zmqError(const char *str) -{ - LogPrint(BCLog::ZMQ, "zmq: Error: %s, errno=%s\n", str, zmq_strerror(errno)); -} - CZMQNotificationInterface::CZMQNotificationInterface() : pcontext(nullptr) { } @@ -20,61 +18,52 @@ CZMQNotificationInterface::CZMQNotificationInterface() : pcontext(nullptr) CZMQNotificationInterface::~CZMQNotificationInterface() { Shutdown(); - - for (std::list<CZMQAbstractNotifier*>::iterator i=notifiers.begin(); i!=notifiers.end(); ++i) - { - delete *i; - } } std::list<const CZMQAbstractNotifier*> CZMQNotificationInterface::GetActiveNotifiers() const { std::list<const CZMQAbstractNotifier*> result; - for (const auto* n : notifiers) { - result.push_back(n); + for (const auto& n : notifiers) { + result.push_back(n.get()); } return result; } CZMQNotificationInterface* CZMQNotificationInterface::Create() { - CZMQNotificationInterface* notificationInterface = nullptr; std::map<std::string, CZMQNotifierFactory> factories; - std::list<CZMQAbstractNotifier*> notifiers; - factories["pubhashblock"] = CZMQAbstractNotifier::Create<CZMQPublishHashBlockNotifier>; factories["pubhashtx"] = CZMQAbstractNotifier::Create<CZMQPublishHashTransactionNotifier>; factories["pubrawblock"] = CZMQAbstractNotifier::Create<CZMQPublishRawBlockNotifier>; factories["pubrawtx"] = CZMQAbstractNotifier::Create<CZMQPublishRawTransactionNotifier>; + std::list<std::unique_ptr<CZMQAbstractNotifier>> notifiers; for (const auto& entry : factories) { std::string arg("-zmq" + entry.first); if (gArgs.IsArgSet(arg)) { - CZMQNotifierFactory factory = entry.second; - std::string address = gArgs.GetArg(arg, ""); - CZMQAbstractNotifier *notifier = factory(); + const auto& factory = entry.second; + const std::string address = gArgs.GetArg(arg, ""); + std::unique_ptr<CZMQAbstractNotifier> notifier = factory(); notifier->SetType(entry.first); notifier->SetAddress(address); notifier->SetOutboundMessageHighWaterMark(static_cast<int>(gArgs.GetArg(arg + "hwm", CZMQAbstractNotifier::DEFAULT_ZMQ_SNDHWM))); - notifiers.push_back(notifier); + notifiers.push_back(std::move(notifier)); } } if (!notifiers.empty()) { - notificationInterface = new CZMQNotificationInterface(); - notificationInterface->notifiers = notifiers; + std::unique_ptr<CZMQNotificationInterface> notificationInterface(new CZMQNotificationInterface()); + notificationInterface->notifiers = std::move(notifiers); - if (!notificationInterface->Initialize()) - { - delete notificationInterface; - notificationInterface = nullptr; + if (notificationInterface->Initialize()) { + return notificationInterface.release(); } } - return notificationInterface; + return nullptr; } // Called at startup to conditionally set up ZMQ socket(s) @@ -95,26 +84,15 @@ bool CZMQNotificationInterface::Initialize() return false; } - std::list<CZMQAbstractNotifier*>::iterator i=notifiers.begin(); - for (; i!=notifiers.end(); ++i) - { - CZMQAbstractNotifier *notifier = *i; - if (notifier->Initialize(pcontext)) - { + for (auto& notifier : notifiers) { + if (notifier->Initialize(pcontext)) { LogPrint(BCLog::ZMQ, "zmq: Notifier %s ready (address = %s)\n", notifier->GetType(), notifier->GetAddress()); - } - else - { + } else { LogPrint(BCLog::ZMQ, "zmq: Notifier %s failed (address = %s)\n", notifier->GetType(), notifier->GetAddress()); - break; + return false; } } - if (i!=notifiers.end()) - { - return false; - } - return true; } @@ -124,9 +102,7 @@ void CZMQNotificationInterface::Shutdown() LogPrint(BCLog::ZMQ, "zmq: Shutdown notification interface\n"); if (pcontext) { - for (std::list<CZMQAbstractNotifier*>::iterator i=notifiers.begin(); i!=notifiers.end(); ++i) - { - CZMQAbstractNotifier *notifier = *i; + for (auto& notifier : notifiers) { LogPrint(BCLog::ZMQ, "zmq: Shutdown notifier %s at %s\n", notifier->GetType(), notifier->GetAddress()); notifier->Shutdown(); } @@ -136,45 +112,43 @@ void CZMQNotificationInterface::Shutdown() } } -void CZMQNotificationInterface::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) -{ - if (fInitialDownload || pindexNew == pindexFork) // In IBD or blocks were disconnected without any new ones - return; +namespace { - for (std::list<CZMQAbstractNotifier*>::iterator i = notifiers.begin(); i!=notifiers.end(); ) - { - CZMQAbstractNotifier *notifier = *i; - if (notifier->NotifyBlock(pindexNew)) - { - i++; - } - else - { +template <typename Function> +void TryForEachAndRemoveFailed(std::list<std::unique_ptr<CZMQAbstractNotifier>>& notifiers, const Function& func) +{ + for (auto i = notifiers.begin(); i != notifiers.end(); ) { + CZMQAbstractNotifier* notifier = i->get(); + if (func(notifier)) { + ++i; + } else { notifier->Shutdown(); i = notifiers.erase(i); } } } +} // anonymous namespace + +void CZMQNotificationInterface::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) +{ + if (fInitialDownload || pindexNew == pindexFork) // In IBD or blocks were disconnected without any new ones + return; + + TryForEachAndRemoveFailed(notifiers, [pindexNew](CZMQAbstractNotifier* notifier) { + return notifier->NotifyBlock(pindexNew); + }); +} + void CZMQNotificationInterface::TransactionAddedToMempool(const CTransactionRef& ptx) { // Used by BlockConnected and BlockDisconnected as well, because they're // all the same external callback. const CTransaction& tx = *ptx; - for (std::list<CZMQAbstractNotifier*>::iterator i = notifiers.begin(); i!=notifiers.end(); ) - { - CZMQAbstractNotifier *notifier = *i; - if (notifier->NotifyTransaction(tx)) - { - i++; - } - else - { - notifier->Shutdown(); - i = notifiers.erase(i); - } - } + TryForEachAndRemoveFailed(notifiers, [&tx](CZMQAbstractNotifier* notifier) { + return notifier->NotifyTransaction(tx); + }); } void CZMQNotificationInterface::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index 60f3b6148..0686960ed 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -7,6 +7,7 @@ #include <validationinterface.h> #include <list> +#include <memory> class CBlockIndex; class CZMQAbstractNotifier; @@ -34,7 +35,7 @@ private: CZMQNotificationInterface(); void *pcontext; - std::list<CZMQAbstractNotifier*> notifiers; + std::list<std::unique_ptr<CZMQAbstractNotifier>> notifiers; }; extern CZMQNotificationInterface* g_zmq_notification_interface; diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 04806903c..d4d21b05b 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -2,13 +2,23 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <zmq/zmqpublishnotifier.h> + #include <chain.h> #include <chainparams.h> +#include <rpc/server.h> #include <streams.h> -#include <zmq/zmqpublishnotifier.h> -#include <validation.h> #include <util/system.h> -#include <rpc/server.h> +#include <validation.h> +#include <zmq/zmqutil.h> + +#include <zmq.h> + +#include <cstdarg> +#include <cstddef> +#include <map> +#include <string> +#include <utility> static std::multimap<std::string, CZMQAbstractPublishNotifier*> mapPublishNotifiers; @@ -86,6 +96,14 @@ bool CZMQAbstractPublishNotifier::Initialize(void *pcontext) return false; } + const int so_keepalive_option {1}; + rc = zmq_setsockopt(psocket, ZMQ_TCP_KEEPALIVE, &so_keepalive_option, sizeof(so_keepalive_option)); + if (rc != 0) { + zmqError("Failed to set SO_KEEPALIVE"); + zmq_close(psocket); + return false; + } + rc = zmq_bind(psocket, address.c_str()); if (rc != 0) { @@ -141,7 +159,7 @@ void CZMQAbstractPublishNotifier::Shutdown() psocket = nullptr; } -bool CZMQAbstractPublishNotifier::SendMessage(const char *command, const void* data, size_t size) +bool CZMQAbstractPublishNotifier::SendZmqMessage(const char *command, const void* data, size_t size) { assert(psocket); @@ -165,7 +183,7 @@ bool CZMQPublishHashBlockNotifier::NotifyBlock(const CBlockIndex *pindex) char data[32]; for (unsigned int i = 0; i < 32; i++) data[31 - i] = hash.begin()[i]; - return SendMessage(MSG_HASHBLOCK, data, 32); + return SendZmqMessage(MSG_HASHBLOCK, data, 32); } bool CZMQPublishHashTransactionNotifier::NotifyTransaction(const CTransaction &transaction) @@ -175,7 +193,7 @@ bool CZMQPublishHashTransactionNotifier::NotifyTransaction(const CTransaction &t char data[32]; for (unsigned int i = 0; i < 32; i++) data[31 - i] = hash.begin()[i]; - return SendMessage(MSG_HASHTX, data, 32); + return SendZmqMessage(MSG_HASHTX, data, 32); } bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) @@ -196,7 +214,7 @@ bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) ss << block; } - return SendMessage(MSG_RAWBLOCK, &(*ss.begin()), ss.size()); + return SendZmqMessage(MSG_RAWBLOCK, &(*ss.begin()), ss.size()); } bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &transaction) @@ -205,5 +223,5 @@ bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &tr LogPrint(BCLog::ZMQ, "zmq: Publish rawtx %s\n", hash.GetHex()); CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ss << transaction; - return SendMessage(MSG_RAWTX, &(*ss.begin()), ss.size()); + return SendZmqMessage(MSG_RAWTX, &(*ss.begin()), ss.size()); } diff --git a/src/zmq/zmqpublishnotifier.h b/src/zmq/zmqpublishnotifier.h index 278fdb94d..eb9ae881b 100644 --- a/src/zmq/zmqpublishnotifier.h +++ b/src/zmq/zmqpublishnotifier.h @@ -22,7 +22,7 @@ public: * data * message sequence number */ - bool SendMessage(const char *command, const void* data, size_t size); + bool SendZmqMessage(const char *command, const void* data, size_t size); bool Initialize(void *pcontext) override; void Shutdown() override; diff --git a/src/zmq/zmqrpc.cpp b/src/zmq/zmqrpc.cpp index cce621012..1dd751b49 100644 --- a/src/zmq/zmqrpc.cpp +++ b/src/zmq/zmqrpc.cpp @@ -13,9 +13,9 @@ namespace { -UniValue getzmqnotifications(const JSONRPCRequest& request) +static RPCHelpMan getzmqnotifications() { - RPCHelpMan{"getzmqnotifications", + return RPCHelpMan{"getzmqnotifications", "\nReturns information about the active ZeroMQ notifications.\n", {}, RPCResult{ @@ -33,8 +33,8 @@ UniValue getzmqnotifications(const JSONRPCRequest& request) HelpExampleCli("getzmqnotifications", "") + HelpExampleRpc("getzmqnotifications", "") }, - }.Check(request); - + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ UniValue result(UniValue::VARR); if (g_zmq_notification_interface != nullptr) { for (const auto* n : g_zmq_notification_interface->GetActiveNotifiers()) { @@ -47,6 +47,8 @@ UniValue getzmqnotifications(const JSONRPCRequest& request) } return result; +}, + }; } const CRPCCommand commands[] = diff --git a/src/zmq/zmqutil.cpp b/src/zmq/zmqutil.cpp new file mode 100644 index 000000000..f07a4ae9f --- /dev/null +++ b/src/zmq/zmqutil.cpp @@ -0,0 +1,14 @@ +// Copyright (c) 2014-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <zmq/zmqutil.h> + +#include <logging.h> + +#include <zmq.h> + +void zmqError(const char* str) +{ + LogPrint(BCLog::ZMQ, "zmq: Error: %s, errno=%s\n", str, zmq_strerror(errno)); +} diff --git a/src/zmq/zmqutil.h b/src/zmq/zmqutil.h new file mode 100644 index 000000000..4c1df5d6d --- /dev/null +++ b/src/zmq/zmqutil.h @@ -0,0 +1,10 @@ +// Copyright (c) 2014-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ZMQ_ZMQUTIL_H +#define BITCOIN_ZMQ_ZMQUTIL_H + +void zmqError(const char* str); + +#endif // BITCOIN_ZMQ_ZMQUTIL_H diff --git a/test/functional/README.md b/test/functional/README.md index aff5f714f..0d85a7407 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -127,8 +127,8 @@ Base class for functional tests. #### [util.py](test_framework/util.py) Generally useful functions. -#### [mininode.py](test_framework/mininode.py) -Basic code to support P2P connectivity to a bitcoind. +#### [p2p.py](test_framework/p2p.py) +Test objects for interacting with a bitcoind node over the p2p interface. #### [script.py](test_framework/script.py) Utilities for manipulating transaction scripts (originally from python-bitcoinlib) diff --git a/test/functional/example_test.py b/test/functional/example_test.py index 5d782026d..083deb646 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -16,17 +16,16 @@ from collections import defaultdict # Avoid wildcard * imports from test_framework.blocktools import (create_block, create_coinbase) from test_framework.messages import CInv, MSG_BLOCK -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, - mininode_lock, msg_block, msg_getdata, + p2p_lock, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes, - wait_until, ) # P2PInterface is a class containing callbacks to be executed when a P2P @@ -166,8 +165,8 @@ class ExampleTest(BitcoinTestFramework): height = self.nodes[0].getblockcount() - for i in range(10): - # Use the mininode and blocktools functionality to manually build a block + for _ in range(10): + # Use the blocktools functionality to manually build a block. # Calling the generate() rpc is easier, but this allows us to exactly # control the blocks and transactions. block = create_block(self.tip, create_coinbase(height+1), self.block_time) @@ -203,15 +202,16 @@ class ExampleTest(BitcoinTestFramework): # wait_until() will loop until a predicate condition is met. Use it to test properties of the # P2PInterface objects. - wait_until(lambda: sorted(blocks) == sorted(list(self.nodes[2].p2p.block_receive_map.keys())), timeout=5, lock=mininode_lock) + self.nodes[2].p2p.wait_until(lambda: sorted(blocks) == sorted(list(self.nodes[2].p2p.block_receive_map.keys())), timeout=5) self.log.info("Check that each block was received only once") # The network thread uses a global lock on data access to the P2PConnection objects when sending and receiving # messages. The test thread should acquire the global lock before accessing any P2PConnection data to avoid locking - # and synchronization issues. Note wait_until() acquires this global lock when testing the predicate. - with mininode_lock: + # and synchronization issues. Note p2p.wait_until() acquires this global lock internally when testing the predicate. + with p2p_lock: for block in self.nodes[2].p2p.block_receive_map.values(): assert_equal(block, 1) + if __name__ == '__main__': ExampleTest().main() diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index 75267de80..17fbf50cc 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -11,7 +11,7 @@ """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import wait_until, get_datadir_path, connect_nodes +from test_framework.util import get_datadir_path, connect_nodes import os @@ -41,7 +41,7 @@ class AbortNodeTest(BitcoinTestFramework): # Check that node0 aborted self.log.info("Waiting for crash") - wait_until(lambda: self.nodes[0].is_node_stopped(), timeout=200) + self.nodes[0].wait_until_stopped(timeout=200) self.log.info("Node crashed - now verifying restart fails") self.nodes[0].assert_start_raises_init_error() diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index 79777f558..603d7f5d3 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -42,7 +42,7 @@ from test_framework.messages import ( msg_block, msg_headers, ) -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.script import (CScript, OP_TRUE) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal @@ -123,7 +123,7 @@ class AssumeValidTest(BitcoinTestFramework): height += 1 # Bury the block 100 deep so the coinbase output is spendable - for i in range(100): + for _ in range(100): block = create_block(self.tip, create_coinbase(height), self.block_time) block.solve() self.blocks.append(block) @@ -149,7 +149,7 @@ class AssumeValidTest(BitcoinTestFramework): height += 1 # Bury the assumed valid block 2100 deep - for i in range(2100): + for _ in range(2100): block = create_block(self.tip, create_coinbase(height), self.block_time) block.nVersion = 4 block.solve() diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index 07dd0f8f8..daefb161a 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -6,7 +6,7 @@ Test various backwards compatibility scenarios. Download the previous node binaries: -contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 +test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 v0.15.2 is not required by this test, but it is used in wallet_upgradewallet.py. Due to a hardfork in regtest, it can't be used to sync nodes. @@ -36,12 +36,12 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.num_nodes = 6 # Add new version after each release: self.extra_args = [ - ["-addresstype=bech32"], # Pre-release: use to mine blocks + ["-addresstype=bech32", "-wallet="], # Pre-release: use to mine blocks ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # Pre-release: use to receive coins, swap wallets, etc ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.19.1 ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.18.1 - ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.17.1 - ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.16.3 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32"], # v0.17.2 + ["-nowallet", "-walletrbf=1", "-addresstype=bech32", "-wallet=wallet.dat"], # v0.16.3 ] def skip_test_if_missing_module(self): @@ -54,7 +54,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): None, 190100, 180100, - 170100, + 170200, 160300, ]) diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 19cdc1093..1253c4541 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -141,7 +141,7 @@ class BIP68Test(BitcoinTestFramework): # some of those inputs to be sequence locked (and randomly choose # between height/time locking). Small random chance of making the locks # all pass. - for i in range(400): + for _ in range(400): # Randomly choose up to 10 inputs num_inputs = random.randint(1, 10) random.shuffle(utxos) @@ -260,7 +260,7 @@ class BIP68Test(BitcoinTestFramework): # Use prioritisetransaction to lower the effective feerate to 0 self.nodes[0].prioritisetransaction(txid=tx2.hash, fee_delta=int(-self.relayfee*COIN)) cur_time = int(time.time()) - for i in range(10): + for _ in range(10): self.nodes[0].setmocktime(cur_time + 600) self.nodes[0].generate(1) cur_time += 600 diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 6619d83dc..efafcfaec 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -26,7 +26,7 @@ from test_framework.messages import ( uint256_from_compact, uint256_from_str, ) -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.script import ( CScript, MAX_SCRIPT_ELEMENT_SIZE, @@ -53,7 +53,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal from data import invalid_txs -# Use this class for tests that require behavior other than normal "mininode" behavior. +# Use this class for tests that require behavior other than normal p2p behavior. # For now, it is used to serialize a bloated varint (b64). class CBrokenBlock(CBlock): def initialize(self, base_block): @@ -125,7 +125,7 @@ class FullBlockTest(BitcoinTestFramework): # collect spendable outputs now to avoid cluttering the code later on out = [] - for i in range(NUM_OUTPUTS_TO_COLLECT): + for _ in range(NUM_OUTPUTS_TO_COLLECT): out.append(self.get_spendable_output()) # Start by building a couple of blocks on top (which output is spent is diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index fd0330924..2919b0ea0 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -10,7 +10,7 @@ Test that the CHECKLOCKTIMEVERIFY soft-fork activates at (regtest) block height from test_framework.blocktools import create_coinbase, create_block, create_transaction from test_framework.messages import CTransaction, msg_block, ToHex -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.script import CScript, OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, OP_DROP, CScriptNum from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index c6852ef01..38e95f00e 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -44,7 +44,7 @@ import time from test_framework.blocktools import create_coinbase, create_block, create_transaction from test_framework.messages import ToHex, CTransaction -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.script import ( CScript, OP_CHECKSEQUENCEVERIFY, @@ -161,7 +161,7 @@ class BIP68_112_113Test(BitcoinTestFramework): def generate_blocks(self, number): test_blocks = [] - for i in range(number): + for _ in range(number): block = self.create_test_block([]) test_blocks.append(block) self.last_block_time += 600 @@ -209,22 +209,22 @@ class BIP68_112_113Test(BitcoinTestFramework): # Note we reuse inputs for v1 and v2 txs so must test these separately # 16 normal inputs bip68inputs = [] - for i in range(16): + for _ in range(16): bip68inputs.append(send_generic_input_tx(self.nodes[0], self.coinbase_blocks, self.nodeaddress)) # 2 sets of 16 inputs with 10 OP_CSV OP_DROP (actually will be prepended to spending scriptSig) bip112basicinputs = [] - for j in range(2): + for _ in range(2): inputs = [] - for i in range(16): + for _ in range(16): inputs.append(send_generic_input_tx(self.nodes[0], self.coinbase_blocks, self.nodeaddress)) bip112basicinputs.append(inputs) # 2 sets of 16 varied inputs with (relative_lock_time) OP_CSV OP_DROP (actually will be prepended to spending scriptSig) bip112diverseinputs = [] - for j in range(2): + for _ in range(2): inputs = [] - for i in range(16): + for _ in range(16): inputs.append(send_generic_input_tx(self.nodes[0], self.coinbase_blocks, self.nodeaddress)) bip112diverseinputs.append(inputs) diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index 7b38e09bf..4bc47141a 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -56,7 +56,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): # Set -maxmempool=0 to turn off mempool memory sharing with dbcache # Set -rpcservertimeout=900 to reduce socket disconnects in this # long-running test - self.base_args = ["-limitdescendantsize=0", "-maxmempool=0", "-rpcservertimeout=900", "-dbbatchsize=200000"] + self.base_args = ["-limitdescendantsize=0", "-maxmempool=0", "-rpcservertimeout=900", "-dbbatchsize=200000", "-wallet="] # Set different crash ratios and cache sizes. Note that not all of # -dbcache goes to the in-memory coins cache. @@ -66,7 +66,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): # Node3 is a normal node with default args, except will mine full blocks # and non-standard txs (e.g. txs with "dust" outputs) - self.node3_args = ["-blockmaxweight=4000000", "-acceptnonstdtxn"] + self.node3_args = ["-blockmaxweight=4000000", "-acceptnonstdtxn", "-wallet="] self.extra_args = [self.node0_args, self.node1_args, self.node2_args, self.node3_args] def skip_test_if_missing_module(self): @@ -195,7 +195,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): while len(utxo_list) >= 2 and num_transactions < count: tx = CTransaction() input_amount = 0 - for i in range(2): + for _ in range(2): utxo = utxo_list.pop() tx.vin.append(CTxIn(COutPoint(int(utxo['txid'], 16), utxo['vout']))) input_amount += int(utxo['amount'] * COIN) @@ -205,7 +205,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): # Sanity check -- if we chose inputs that are too small, skip continue - for i in range(3): + for _ in range(3): tx.vout.append(CTxOut(output_amount, hex_str_to_bytes(utxo['scriptPubKey']))) # Sign and send the transaction to get into the mempool diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 05fdacd45..f263c93c8 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -9,7 +9,7 @@ Test that the DERSIG soft-fork activates at (regtest) height 1251. from test_framework.blocktools import create_coinbase, create_block, create_transaction from test_framework.messages import msg_block -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 3cf0fb8f7..d89eeec40 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -145,9 +145,9 @@ class EstimateFeeTest(BitcoinTestFramework): # mine non-standard txs (e.g. txs with "dust" outputs) # Force fSendTrickle to true (via whitelist.noban) self.extra_args = [ - ["-acceptnonstdtxn", "[email protected]"], - ["-acceptnonstdtxn", "[email protected]", "-blockmaxweight=68000"], - ["-acceptnonstdtxn", "[email protected]", "-blockmaxweight=32000"], + ["-acceptnonstdtxn", "[email protected]", "-wallet="], + ["-acceptnonstdtxn", "[email protected]", "-blockmaxweight=68000", "-wallet="], + ["-acceptnonstdtxn", "[email protected]", "-blockmaxweight=32000", "-wallet="], ] def skip_test_if_missing_module(self): @@ -176,9 +176,9 @@ class EstimateFeeTest(BitcoinTestFramework): # We shuffle our confirmed txout set before each set of transactions # small_txpuzzle_randfee will use the transactions that have inputs already in the chain when possible # resorting to tx's that depend on the mempool when those run out - for i in range(numblocks): + for _ in range(numblocks): random.shuffle(self.confutxo) - for j in range(random.randrange(100 - 50, 100 + 50)): + for _ in range(random.randrange(100 - 50, 100 + 50)): from_index = random.randint(1, 2) (txhex, fee) = small_txpuzzle_randfee(self.nodes[from_index], self.confutxo, self.memutxo, Decimal("0.005"), min_fee, min_fee) @@ -243,7 +243,7 @@ class EstimateFeeTest(BitcoinTestFramework): self.confutxo = self.txouts # Start with the set of confirmed txouts after splitting self.log.info("Will output estimates for 1/2/3/6/15/25 blocks") - for i in range(2): + for _ in range(2): self.log.info("Creating transactions and mining them with a block size that can't keep up") # Create transactions and mine 10 small blocks with node 2, but create txs faster than we can mine self.transact_and_mine(10, self.nodes[2]) diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index b56ffe179..e4ceb62c9 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -15,7 +15,7 @@ class FilelockTest(BitcoinTestFramework): def setup_network(self): self.add_nodes(self.num_nodes, extra_args=None) - self.nodes[0].start([]) + self.nodes[0].start(['-wallet=']) self.nodes[0].wait_for_rpc_connection() def run_test(self): @@ -30,7 +30,7 @@ class FilelockTest(BitcoinTestFramework): wallet_dir = os.path.join(datadir, 'wallets') self.log.info("Check that we can't start a second bitcoind instance using the same wallet") expected_msg = "Error: Error initializing wallet database environment" - self.nodes[1].assert_start_raises_init_error(extra_args=['-walletdir={}'.format(wallet_dir), '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX) + self.nodes[1].assert_start_raises_init_error(extra_args=['-walletdir={}'.format(wallet_dir), '-wallet=', '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX) if __name__ == '__main__': FilelockTest().main() diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 5538d6d3b..e5c62d1ea 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -14,7 +14,7 @@ from collections import defaultdict import time from test_framework.messages import CInv, MSG_BLOCK, msg_getdata -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, mine_large_block @@ -104,7 +104,7 @@ class MaxUploadTest(BitcoinTestFramework): assert_equal(len(self.nodes[0].getpeerinfo()), 3) # At most a couple more tries should succeed (depending on how long # the test has been running so far). - for i in range(3): + for _ in range(3): p2p_conns[0].send_message(getdata_request) p2p_conns[0].wait_for_disconnect() assert_equal(len(self.nodes[0].getpeerinfo()), 2) diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index dd4c318ce..20020c323 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -9,7 +9,6 @@ from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE, keyhash_to_p2pkh from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, connect_nodes, disconnect_nodes, hex_str_to_bytes, @@ -19,7 +18,7 @@ from test_framework.util import ( # Windows disallow control characters (0-31) and /\?%:|"<> FILE_CHAR_START = 32 if os.name == 'nt' else 1 FILE_CHAR_END = 128 -FILE_CHAR_BLOCKLIST = '/\\?%*:|"<>' if os.name == 'nt' else '/' +FILE_CHARS_DISALLOWED = '/\\?%*:|"<>' if os.name == 'nt' else '/' def notify_outputname(walletname, txid): @@ -32,7 +31,7 @@ class NotificationsTest(BitcoinTestFramework): self.setup_clean_chain = True def setup_network(self): - self.wallet = ''.join(chr(i) for i in range(FILE_CHAR_START, FILE_CHAR_END) if chr(i) not in FILE_CHAR_BLOCKLIST) + self.wallet = ''.join(chr(i) for i in range(FILE_CHAR_START, FILE_CHAR_END) if chr(i) not in FILE_CHARS_DISALLOWED) self.alertnotify_dir = os.path.join(self.options.tmpdir, "alertnotify") self.blocknotify_dir = os.path.join(self.options.tmpdir, "blocknotify") self.walletnotify_dir = os.path.join(self.options.tmpdir, "walletnotify") @@ -56,7 +55,7 @@ class NotificationsTest(BitcoinTestFramework): blocks = self.nodes[1].generatetoaddress(block_count, self.nodes[1].getnewaddress() if self.is_wallet_compiled() else ADDRESS_BCRT1_UNSPENDABLE) # wait at most 10 seconds for expected number of files before reading the content - wait_until(lambda: len(os.listdir(self.blocknotify_dir)) == block_count, timeout=10) + self.wait_until(lambda: len(os.listdir(self.blocknotify_dir)) == block_count, timeout=10) # directory content should equal the generated blocks hashes assert_equal(sorted(blocks), sorted(os.listdir(self.blocknotify_dir))) @@ -64,7 +63,7 @@ class NotificationsTest(BitcoinTestFramework): if self.is_wallet_compiled(): self.log.info("test -walletnotify") # wait at most 10 seconds for expected number of files before reading the content - wait_until(lambda: len(os.listdir(self.walletnotify_dir)) == block_count, timeout=10) + self.wait_until(lambda: len(os.listdir(self.walletnotify_dir)) == block_count, timeout=10) # directory content should equal the generated transaction hashes txids_rpc = list(map(lambda t: notify_outputname(self.wallet, t['txid']), self.nodes[1].listtransactions("*", block_count))) @@ -78,7 +77,7 @@ class NotificationsTest(BitcoinTestFramework): self.start_node(1) connect_nodes(self.nodes[0], 1) - wait_until(lambda: len(os.listdir(self.walletnotify_dir)) == block_count, timeout=10) + self.wait_until(lambda: len(os.listdir(self.walletnotify_dir)) == block_count, timeout=10) # directory content should equal the generated transaction hashes txids_rpc = list(map(lambda t: notify_outputname(self.wallet, t['txid']), self.nodes[1].listtransactions("*", block_count))) @@ -140,7 +139,7 @@ class NotificationsTest(BitcoinTestFramework): # TODO: add test for `-alertnotify` large fork notifications def expect_wallet_notify(self, tx_ids): - wait_until(lambda: len(os.listdir(self.walletnotify_dir)) >= len(tx_ids), timeout=10) + self.wait_until(lambda: len(os.listdir(self.walletnotify_dir)) >= len(tx_ids), timeout=10) assert_equal(sorted(notify_outputname(self.wallet, tx_id) for tx_id in tx_ids), sorted(os.listdir(self.walletnotify_dir))) for tx_file in os.listdir(self.walletnotify_dir): os.remove(os.path.join(self.walletnotify_dir, tx_file)) diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index e46e5aacc..df54ae777 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -20,7 +20,6 @@ from test_framework.util import ( assert_raises_rpc_error, connect_nodes, disconnect_nodes, - wait_until, ) # Rescans start at the earliest block up to 2 hours before a key timestamp, so @@ -82,16 +81,16 @@ class PruneTest(BitcoinTestFramework): # Create nodes 0 and 1 to mine. # Create node 2 to test pruning. - self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5"] + self.full_node_default_args = ["-maxreceivebuffer=20000", "-checkblocks=5", "-wallet="] # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later) # Create nodes 5 to test wallet in prune mode, but do not connect self.extra_args = [ self.full_node_default_args, self.full_node_default_args, - ["-maxreceivebuffer=20000", "-prune=550"], - ["-maxreceivebuffer=20000"], - ["-maxreceivebuffer=20000"], - ["-prune=550"], + ["-wallet=", "-maxreceivebuffer=20000", "-prune=550"], + ["-wallet=", "-maxreceivebuffer=20000"], + ["-wallet=", "-maxreceivebuffer=20000"], + ["-wallet=", "-prune=550"], ] self.rpc_timeout = 120 @@ -136,7 +135,7 @@ class PruneTest(BitcoinTestFramework): mine_large_blocks(self.nodes[0], 25) # Wait for blk00000.dat to be pruned - wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30) + self.wait_until(lambda: not os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), timeout=30) self.log.info("Success") usage = calc_usage(self.prunedir) @@ -147,7 +146,7 @@ class PruneTest(BitcoinTestFramework): # Create stale blocks in manageable sized chunks self.log.info("Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds") - for j in range(12): + for _ in range(12): # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects disconnect_nodes(self.nodes[0], 1) @@ -250,7 +249,7 @@ class PruneTest(BitcoinTestFramework): self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") # Wait for Node 2 to reorg to proper height - wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900) + self.wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900) assert_equal(self.nodes[2].getbestblockhash(), goalbesthash) # Verify we can now have the data for a block previously pruned assert_equal(self.nodes[2].getblock(self.forkhash)["height"], self.forkheight) diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index acf551ef6..1b531ad51 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -376,7 +376,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): split_value = int((initial_nValue-fee)/(MAX_REPLACEMENT_LIMIT+1)) outputs = [] - for i in range(MAX_REPLACEMENT_LIMIT+1): + for _ in range(MAX_REPLACEMENT_LIMIT+1): outputs.append(CTxOut(split_value, CScript([1]))) splitting_tx = CTransaction() diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 5195d20dc..084297277 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -126,11 +126,11 @@ class SegWitTest(BitcoinTestFramework): assert_equal(bip173_ms_addr, script_to_p2wsh(multiscript)) p2sh_ids.append([]) wit_ids.append([]) - for v in range(2): + for _ in range(2): p2sh_ids[i].append([]) wit_ids[i].append([]) - for i in range(5): + for _ in range(5): for n in range(3): for v in range(2): wit_ids[n][v].append(send_to_witness(v, self.nodes[0], find_spendable_utxo(self.nodes[0], 50), self.pubkey[n], False, Decimal("49.999"))) diff --git a/test/functional/feature_shutdown.py b/test/functional/feature_shutdown.py index d782d3b1d..a76e0f1b5 100755 --- a/test/functional/feature_shutdown.py +++ b/test/functional/feature_shutdown.py @@ -5,7 +5,7 @@ """Test bitcoind shutdown.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, get_rpc_proxy, wait_until +from test_framework.util import assert_equal, get_rpc_proxy from threading import Thread def test_long_call(node): @@ -25,7 +25,7 @@ class ShutdownTest(BitcoinTestFramework): node.getblockcount() Thread(target=test_long_call, args=(node,)).start() # Wait until the server is executing the above `waitfornewblock`. - wait_until(lambda: len(self.nodes[0].getrpcinfo()['active_commands']) == 2) + self.wait_until(lambda: len(self.nodes[0].getrpcinfo()['active_commands']) == 2) # Wait 1 second after requesting shutdown but not before the `stop` call # finishes. This is to ensure event loop waits for current connections # to close. diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index 071392514..e1016e158 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -12,9 +12,8 @@ import re from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import msg_block -from test_framework.mininode import P2PInterface, mininode_lock +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import wait_until VB_PERIOD = 144 # versionbits period length for regtest VB_THRESHOLD = 108 # versionbits activation threshold for regtest @@ -91,14 +90,14 @@ class VersionBitsWarningTest(BitcoinTestFramework): # Generating one block guarantees that we'll get out of IBD node.generatetoaddress(1, node_deterministic_address) - wait_until(lambda: not node.getblockchaininfo()['initialblockdownload'], timeout=10, lock=mininode_lock) + self.wait_until(lambda: not node.getblockchaininfo()['initialblockdownload']) # Generating one more block will be enough to generate an error. node.generatetoaddress(1, node_deterministic_address) # Check that get*info() shows the versionbits unknown rules warning assert WARN_UNKNOWN_RULES_ACTIVE in node.getmininginfo()["warnings"] assert WARN_UNKNOWN_RULES_ACTIVE in node.getnetworkinfo()["warnings"] # Check that the alert file shows the versionbits unknown rules warning - wait_until(lambda: self.versionbits_in_alert_file(), timeout=60) + self.wait_until(lambda: self.versionbits_in_alert_file()) if __name__ == '__main__': VersionBitsWarningTest().main() diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 80003aca0..81c007c27 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -71,7 +71,14 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(cli_get_info['blocks'], blockchain_info['blocks']) assert_equal(cli_get_info['headers'], blockchain_info['headers']) assert_equal(cli_get_info['timeoffset'], network_info['timeoffset']) - assert_equal(cli_get_info['connections'], network_info['connections']) + assert_equal( + cli_get_info['connections'], + { + 'in': network_info['connections_in'], + 'out': network_info['connections_out'], + 'total': network_info['connections'] + } + ) assert_equal(cli_get_info['proxy'], network_info['networks'][0]['proxy']) assert_equal(cli_get_info['difficulty'], blockchain_info['difficulty']) assert_equal(cli_get_info['chain'], blockchain_info['chain']) diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index ac2e73fc5..9c877aaea 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -45,7 +45,7 @@ class RPCInterfaceTest(BitcoinTestFramework): # work fine. {"method": "invalidmethod", "id": 2}, # Another call that should succeed. - {"method": "getbestblockhash", "id": 3}, + {"method": "getblockhash", "id": 3, "params": [0]}, ]) result_by_id = {} diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 89c55f31f..ef4780cac 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -54,28 +54,31 @@ class ZMQTest (BitcoinTestFramework): self.ctx.destroy(linger=None) def test_basic(self): - # All messages are received in the same socket which means - # that this test fails if the publishing order changes. - # Note that the publishing order is not defined in the documentation and - # is subject to change. import zmq # Invalid zmq arguments don't take down the node, see #17185. self.restart_node(0, ["-zmqpubrawtx=foo", "-zmqpubhashtx=bar"]) address = 'tcp://127.0.0.1:28332' - socket = self.ctx.socket(zmq.SUB) - socket.set(zmq.RCVTIMEO, 60000) + sockets = [] + subs = [] + services = [b"hashblock", b"hashtx", b"rawblock", b"rawtx"] + for service in services: + sockets.append(self.ctx.socket(zmq.SUB)) + sockets[-1].set(zmq.RCVTIMEO, 60000) + subs.append(ZMQSubscriber(sockets[-1], service)) # Subscribe to all available topics. - hashblock = ZMQSubscriber(socket, b"hashblock") - hashtx = ZMQSubscriber(socket, b"hashtx") - rawblock = ZMQSubscriber(socket, b"rawblock") - rawtx = ZMQSubscriber(socket, b"rawtx") + hashblock = subs[0] + hashtx = subs[1] + rawblock = subs[2] + rawtx = subs[3] self.restart_node(0, ["-zmqpub%s=%s" % (sub.topic.decode(), address) for sub in [hashblock, hashtx, rawblock, rawtx]]) connect_nodes(self.nodes[0], 1) - socket.connect(address) + for socket in sockets: + socket.connect(address) + # Relax so that the subscriber is ready before publishing zmq messages sleep(0.2) @@ -96,15 +99,16 @@ class ZMQTest (BitcoinTestFramework): tx.calc_sha256() assert_equal(tx.hash, txid.hex()) + # Should receive the generated raw block. + block = rawblock.receive() + assert_equal(genhashes[x], hash256_reversed(block[:80]).hex()) + # Should receive the generated block hash. hash = hashblock.receive().hex() assert_equal(genhashes[x], hash) # The block should only have the coinbase txid. assert_equal([txid.hex()], self.nodes[1].getblock(hash)["tx"]) - # Should receive the generated raw block. - block = rawblock.receive() - assert_equal(genhashes[x], hash256_reversed(block[:80]).hex()) if self.is_wallet_compiled(): self.log.info("Wait for tx from second node") @@ -119,6 +123,13 @@ class ZMQTest (BitcoinTestFramework): hex = rawtx.receive() assert_equal(payment_txid, hash256_reversed(hex).hex()) + # Mining the block with this tx should result in second notification + # after coinbase tx notification + self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) + hashtx.receive() + txid = hashtx.receive() + assert_equal(payment_txid, txid.hex()) + self.log.info("Test the getzmqnotifications RPC") assert_equal(self.nodes[0].getzmqnotifications(), [ @@ -131,30 +142,67 @@ class ZMQTest (BitcoinTestFramework): assert_equal(self.nodes[1].getzmqnotifications(), []) def test_reorg(self): + if not self.is_wallet_compiled(): + self.log.info("Skipping reorg test because wallet is disabled") + return + import zmq address = 'tcp://127.0.0.1:28333' - socket = self.ctx.socket(zmq.SUB) - socket.set(zmq.RCVTIMEO, 60000) - hashblock = ZMQSubscriber(socket, b'hashblock') + + services = [b"hashblock", b"hashtx"] + sockets = [] + subs = [] + for service in services: + sockets.append(self.ctx.socket(zmq.SUB)) + # 2 second timeout to check end of notifications + sockets[-1].set(zmq.RCVTIMEO, 2000) + subs.append(ZMQSubscriber(sockets[-1], service)) + + # Subscribe to all available topics. + hashblock = subs[0] + hashtx = subs[1] # Should only notify the tip if a reorg occurs - self.restart_node(0, ['-zmqpub%s=%s' % (hashblock.topic.decode(), address)]) - socket.connect(address) + self.restart_node(0, ["-zmqpub%s=%s" % (sub.topic.decode(), address) for sub in [hashblock, hashtx]]) + for socket in sockets: + socket.connect(address) # Relax so that the subscriber is ready before publishing zmq messages sleep(0.2) - # Generate 1 block in nodes[0] and receive all notifications - self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) + # Generate 1 block in nodes[0] with 1 mempool tx and receive all notifications + payment_txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) + disconnect_block = self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE)[0] + disconnect_cb = self.nodes[0].getblock(disconnect_block)["tx"][0] assert_equal(self.nodes[0].getbestblockhash(), hashblock.receive().hex()) + assert_equal(hashtx.receive().hex(), payment_txid) + assert_equal(hashtx.receive().hex(), disconnect_cb) # Generate 2 blocks in nodes[1] - self.nodes[1].generatetoaddress(2, ADDRESS_BCRT1_UNSPENDABLE) + connect_blocks = self.nodes[1].generatetoaddress(2, ADDRESS_BCRT1_UNSPENDABLE) # nodes[0] will reorg chain after connecting back nodes[1] connect_nodes(self.nodes[0], 1) + self.sync_blocks() # tx in mempool valid but not advertised # Should receive nodes[1] tip assert_equal(self.nodes[1].getbestblockhash(), hashblock.receive().hex()) + # During reorg: + # Get old payment transaction notification from disconnect and disconnected cb + assert_equal(hashtx.receive().hex(), payment_txid) + assert_equal(hashtx.receive().hex(), disconnect_cb) + # And the payment transaction again due to mempool entry + assert_equal(hashtx.receive().hex(), payment_txid) + assert_equal(hashtx.receive().hex(), payment_txid) + # And the new connected coinbases + for i in [0, 1]: + assert_equal(hashtx.receive().hex(), self.nodes[1].getblock(connect_blocks[i])["tx"][0]) + + # If we do a simple invalidate we announce the disconnected coinbase + self.nodes[0].invalidateblock(connect_blocks[1]) + assert_equal(hashtx.receive().hex(), self.nodes[1].getblock(connect_blocks[1])["tx"][0]) + # And the current tip + assert_equal(hashtx.receive().hex(), self.nodes[1].getblock(connect_blocks[0])["tx"][0]) + if __name__ == '__main__': ZMQTest().main() diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 6df8f1c3e..57a059b7f 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -4,6 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool acceptance of raw transactions.""" +from decimal import Decimal from io import BytesIO import math @@ -91,20 +92,22 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) txid_0 = tx.rehash() self.check_mempool_result( - result_expected=[{'txid': txid_0, 'allowed': True}], + result_expected=[{'txid': txid_0, 'allowed': True, 'vsize': tx.get_vsize(), 'fees': {'base': Decimal(str(fee))}}], rawtxs=[raw_tx_0], ) self.log.info('A final transaction not in the mempool') coin = coins.pop() # Pick a random coin(base) to spend + output_amount = 0.025 raw_tx_final = node.signrawtransactionwithwallet(node.createrawtransaction( inputs=[{'txid': coin['txid'], 'vout': coin['vout'], "sequence": 0xffffffff}], # SEQUENCE_FINAL - outputs=[{node.getnewaddress(): 0.025}], + outputs=[{node.getnewaddress(): output_amount}], locktime=node.getblockcount() + 2000, # Can be anything ))['hex'] tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_final))) + fee_expected = int(coin['amount']) - output_amount self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': True}], + result_expected=[{'txid': tx.rehash(), 'allowed': True, 'vsize': tx.get_vsize(), 'fees': {'base': Decimal(str(fee_expected))}}], rawtxs=[tx.serialize().hex()], maxfeerate=0, ) @@ -127,7 +130,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) txid_0 = tx.rehash() self.check_mempool_result( - result_expected=[{'txid': txid_0, 'allowed': True}], + result_expected=[{'txid': txid_0, 'allowed': True, 'vsize': tx.get_vsize(), 'fees': {'base': Decimal(str(2 * fee))}}], rawtxs=[raw_tx_0], ) @@ -187,7 +190,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) # Reference tx should be valid on itself self.check_mempool_result( - result_expected=[{'txid': tx.rehash(), 'allowed': True}], + result_expected=[{'txid': tx.rehash(), 'allowed': True, 'vsize': tx.get_vsize(), 'fees': { 'base': Decimal(str(0.1 - 0.05))}}], rawtxs=[tx.serialize().hex()], maxfeerate=0, ) diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index 31fb75190..fd3dd47e2 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -8,7 +8,7 @@ NOTE: The test is designed to prevent cases when compatibility is broken acciden In case we need to break mempool compatibility we can continue to use the test by just bumping the version number. Download node binaries: -contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 +test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 Only v0.15.2 is required by this test. The rest is used in other backwards compatibility tests. """ @@ -31,7 +31,7 @@ class MempoolCompatibilityTest(BitcoinTestFramework): 150200, # oldest version supported by the test framework None, ]) - self.start_nodes() + self.start_nodes([[], ["-wallet="]]) self.import_deterministic_coinbase_privkeys() def run_test(self): diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index 0739d7e29..e956fe07d 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -31,7 +31,7 @@ class MempoolPackagesTest(BitcoinTestFramework): for (txid, vout) in zip(parent_txids, vouts): inputs.append({'txid' : txid, 'vout' : vout}) outputs = {} - for i in range(num_outputs): + for _ in range(num_outputs): outputs[node.getnewaddress()] = send_value rawtx = node.createrawtransaction(inputs, outputs, 0, True) signedtx = node.signrawtransactionwithwallet(rawtx) diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 542d24f4b..e74ef8cf1 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -7,13 +7,12 @@ from decimal import Decimal from test_framework.messages import COIN -from test_framework.mininode import P2PTxInvStore +from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, satoshi_round, - wait_until, ) # default limits @@ -48,7 +47,7 @@ class MempoolPackagesTest(BitcoinTestFramework): send_value = satoshi_round((value - fee)/num_outputs) inputs = [ {'txid' : parent_txid, 'vout' : vout} ] outputs = {} - for i in range(num_outputs): + for _ in range(num_outputs): outputs[node.getnewaddress()] = send_value rawtx = node.createrawtransaction(inputs, outputs) signedtx = node.signrawtransactionwithwallet(rawtx) @@ -70,7 +69,7 @@ class MempoolPackagesTest(BitcoinTestFramework): # MAX_ANCESTORS transactions off a confirmed tx should be fine chain = [] witness_chain = [] - for i in range(MAX_ANCESTORS): + for _ in range(MAX_ANCESTORS): (txid, sent_value) = self.chain_transaction(self.nodes[0], txid, 0, value, fee, 1) value = sent_value chain.append(txid) @@ -245,7 +244,7 @@ class MempoolPackagesTest(BitcoinTestFramework): # Sign and send up to MAX_DESCENDANT transactions chained off the parent tx chain = [] # save sent txs for the purpose of checking node1's mempool later (see below) - for i in range(MAX_DESCENDANTS - 1): + for _ in range(MAX_DESCENDANTS - 1): utxo = transaction_package.pop(0) (txid, sent_value) = self.chain_transaction(self.nodes[0], utxo['txid'], utxo['vout'], utxo['amount'], fee, 10) chain.append(txid) @@ -269,8 +268,8 @@ class MempoolPackagesTest(BitcoinTestFramework): # - txs from previous ancestor test (-> custom ancestor limit) # - parent tx for descendant test # - txs chained off parent tx (-> custom descendant limit) - wait_until(lambda: len(self.nodes[1].getrawmempool(False)) == - MAX_ANCESTORS_CUSTOM + 1 + MAX_DESCENDANTS_CUSTOM, timeout=10) + self.wait_until(lambda: len(self.nodes[1].getrawmempool(False)) == + MAX_ANCESTORS_CUSTOM + 1 + MAX_DESCENDANTS_CUSTOM, timeout=10) mempool0 = self.nodes[0].getrawmempool(False) mempool1 = self.nodes[1].getrawmempool(False) assert set(mempool1).issubset(set(mempool0)) @@ -312,7 +311,7 @@ class MempoolPackagesTest(BitcoinTestFramework): send_value = satoshi_round((value - fee)/2) inputs = [ {'txid' : txid, 'vout' : vout} ] outputs = {} - for i in range(2): + for _ in range(2): outputs[self.nodes[0].getnewaddress()] = send_value rawtx = self.nodes[0].createrawtransaction(inputs, outputs) signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx) @@ -326,7 +325,7 @@ class MempoolPackagesTest(BitcoinTestFramework): # Create tx2-7 vout = 1 txid = tx0_id - for i in range(6): + for _ in range(6): (txid, sent_value) = self.chain_transaction(self.nodes[0], txid, vout, value, fee, 1) vout = 0 value = sent_value diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 5d00648ae..f73f1a02a 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -39,15 +39,14 @@ from decimal import Decimal import os import time +from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework -from test_framework.mininode import P2PTxInvStore from test_framework.util import ( assert_equal, assert_greater_than_or_equal, assert_raises_rpc_error, connect_nodes, disconnect_nodes, - wait_until, ) @@ -62,7 +61,7 @@ class MempoolPersistTest(BitcoinTestFramework): def run_test(self): self.log.debug("Send 5 transactions from node2 (to its own address)") tx_creation_time_lower = int(time.time()) - for i in range(5): + for _ in range(5): last_txid = self.nodes[2].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("10")) node2_balance = self.nodes[2].getbalance() self.sync_all() @@ -172,7 +171,7 @@ class MempoolPersistTest(BitcoinTestFramework): # check that txn gets broadcast due to unbroadcast logic conn = node0.add_p2p_connection(P2PTxInvStore()) node0.mockscheduler(16*60) # 15 min + 1 for buffer - wait_until(lambda: len(conn.get_invs()) == 1) + self.wait_until(lambda: len(conn.get_invs()) == 1) if __name__ == '__main__': MempoolPersistTest().main() diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py index 365d01115..abd5a03d9 100755 --- a/test/functional/mempool_unbroadcast.py +++ b/test/functional/mempool_unbroadcast.py @@ -7,7 +7,7 @@ to peers until a GETDATA is received.""" import time -from test_framework.mininode import P2PTxInvStore +from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, diff --git a/test/functional/mempool_updatefromblock.py b/test/functional/mempool_updatefromblock.py index 8a703ef00..8baf974a0 100755 --- a/test/functional/mempool_updatefromblock.py +++ b/test/functional/mempool_updatefromblock.py @@ -73,7 +73,7 @@ class MempoolUpdateFromBlockTest(BitcoinTestFramework): n_outputs = size - tx_count output_value = ((inputs_value - fee) / Decimal(n_outputs)).quantize(Decimal('0.00000001')) outputs = {} - for n in range(0, n_outputs): + for _ in range(n_outputs): outputs[self.nodes[0].getnewaddress()] = output_value else: output_value = (inputs_value - fee).quantize(Decimal('0.00000001')) diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index 63d1ccfb3..b13740750 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -20,7 +20,7 @@ from test_framework.messages import ( CBlockHeader, BLOCK_HEADER_SIZE, ) -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 5c7e27a3a..80f262d0d 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -12,9 +12,7 @@ from test_framework.messages import ( NODE_WITNESS, msg_addr, ) -from test_framework.mininode import ( - P2PInterface, -) +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py index 6d947ac66..84178d0dd 100755 --- a/test/functional/p2p_blockfilters.py +++ b/test/functional/p2p_blockfilters.py @@ -4,12 +4,13 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests NODE_COMPACT_FILTERS (BIP 157/158). -Tests that a node configured with -blockfilterindex and -peerblockfilters can serve -cfheaders and cfcheckpts. +Tests that a node configured with -blockfilterindex and -peerblockfilters signals +NODE_COMPACT_FILTERS and can serve cfilters, cfheaders and cfcheckpts. """ from test_framework.messages import ( FILTER_TYPE_BASIC, + NODE_COMPACT_FILTERS, hash256, msg_getcfcheckpt, msg_getcfheaders, @@ -17,13 +18,12 @@ from test_framework.messages import ( ser_uint256, uint256_from_str, ) -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes, disconnect_nodes, - wait_until, ) class CFiltersClient(P2PInterface): @@ -64,11 +64,19 @@ class CompactFiltersTest(BitcoinTestFramework): disconnect_nodes(self.nodes[0], 1) self.nodes[0].generate(1) - wait_until(lambda: self.nodes[0].getblockcount() == 1000) + self.wait_until(lambda: self.nodes[0].getblockcount() == 1000) stale_block_hash = self.nodes[0].getblockhash(1000) self.nodes[1].generate(1001) - wait_until(lambda: self.nodes[1].getblockcount() == 2000) + self.wait_until(lambda: self.nodes[1].getblockcount() == 2000) + + # Check that nodes have signalled NODE_COMPACT_FILTERS correctly. + assert node0.nServices & NODE_COMPACT_FILTERS != 0 + assert node1.nServices & NODE_COMPACT_FILTERS == 0 + + # Check that the localservices is as expected. + assert int(self.nodes[0].getnetworkinfo()['localservices'], 16) & NODE_COMPACT_FILTERS != 0 + assert int(self.nodes[1].getnetworkinfo()['localservices'], 16) & NODE_COMPACT_FILTERS == 0 self.log.info("get cfcheckpt on chain to be re-orged out.") request = msg_getcfcheckpt( diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 27e6b669f..65259f186 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -5,7 +5,7 @@ """Test p2p blocksonly""" from test_framework.messages import msg_tx, CTransaction, FromHex -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 0b3738b57..fdae7fb68 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -11,10 +11,10 @@ import random from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, FromHex, HeaderAndShortIDs, msg_no_witness_block, msg_no_witness_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, msg_block, msg_blocktxn, MSG_BLOCK, MSG_CMPCT_BLOCK, MSG_WITNESS_FLAG, NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, ser_uint256, ToHex -from test_framework.mininode import mininode_lock, P2PInterface +from test_framework.p2p import p2p_lock, P2PInterface from test_framework.script import CScript, OP_TRUE, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, wait_until, softfork_active +from test_framework.util import assert_equal, softfork_active # TestP2PConn: A peer we use to send messages to bitcoind, and store responses. class TestP2PConn(P2PInterface): @@ -48,12 +48,12 @@ class TestP2PConn(P2PInterface): self.block_announced = True self.announced_blockhashes.add(x.hash) - # Requires caller to hold mininode_lock + # Requires caller to hold p2p_lock def received_block_announcement(self): return self.block_announced def clear_block_announcement(self): - with mininode_lock: + with p2p_lock: self.block_announced = False self.last_message.pop("inv", None) self.last_message.pop("headers", None) @@ -73,7 +73,7 @@ class TestP2PConn(P2PInterface): def request_headers_and_sync(self, locator, hashstop=0): self.clear_block_announcement() self.get_headers(locator, hashstop) - wait_until(self.received_block_announcement, timeout=30, lock=mininode_lock) + self.wait_until(self.received_block_announcement, timeout=30) self.clear_block_announcement() # Block until a block announcement for a particular block hash is @@ -81,7 +81,7 @@ class TestP2PConn(P2PInterface): def wait_for_block_announcement(self, block_hash, timeout=30): def received_hash(): return (block_hash in self.announced_blockhashes) - wait_until(received_hash, timeout=timeout, lock=mininode_lock) + self.wait_until(received_hash, timeout=timeout) def send_await_disconnect(self, message, timeout=30): """Sends a message to the node and wait for disconnect. @@ -89,7 +89,7 @@ class TestP2PConn(P2PInterface): This is used when we want to send a message into the node that we expect will get us disconnected, eg an invalid block.""" self.send_message(message) - wait_until(lambda: not self.is_connected, timeout=timeout, lock=mininode_lock) + self.wait_for_disconnect(timeout) class CompactBlocksTest(BitcoinTestFramework): def set_test_params(self): @@ -125,7 +125,7 @@ class CompactBlocksTest(BitcoinTestFramework): out_value = total_value // 10 tx = CTransaction() tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b'')) - for i in range(10): + for _ in range(10): tx.vout.append(CTxOut(out_value, CScript([OP_TRUE]))) tx.rehash() @@ -154,8 +154,8 @@ class CompactBlocksTest(BitcoinTestFramework): # Make sure we get a SENDCMPCT message from our peer def received_sendcmpct(): return (len(test_node.last_sendcmpct) > 0) - wait_until(received_sendcmpct, timeout=30, lock=mininode_lock) - with mininode_lock: + test_node.wait_until(received_sendcmpct, timeout=30) + with p2p_lock: # Check that the first version received is the preferred one assert_equal(test_node.last_sendcmpct[0].version, preferred_version) # And that we receive versions down to 1. @@ -170,7 +170,7 @@ class CompactBlocksTest(BitcoinTestFramework): peer.wait_for_block_announcement(block_hash, timeout=30) assert peer.block_announced - with mininode_lock: + with p2p_lock: assert predicate(peer), ( "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None))) @@ -266,7 +266,7 @@ class CompactBlocksTest(BitcoinTestFramework): address = node.getnewaddress() segwit_tx_generated = False - for i in range(num_transactions): + for _ in range(num_transactions): txid = node.sendtoaddress(address, 0.1) hex_tx = node.gettransaction(txid)["hex"] tx = FromHex(CTransaction(), hex_tx) @@ -294,12 +294,11 @@ class CompactBlocksTest(BitcoinTestFramework): block.rehash() # Wait until the block was announced (via compact blocks) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) + test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) # Now fetch and check the compact block header_and_shortids = None - with mininode_lock: - assert "cmpctblock" in test_node.last_message + with p2p_lock: # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) @@ -309,12 +308,11 @@ class CompactBlocksTest(BitcoinTestFramework): inv = CInv(MSG_CMPCT_BLOCK, block_hash) test_node.send_message(msg_getdata([inv])) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) + test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) # Now fetch and check the compact block header_and_shortids = None - with mininode_lock: - assert "cmpctblock" in test_node.last_message + with p2p_lock: # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) @@ -380,7 +378,7 @@ class CompactBlocksTest(BitcoinTestFramework): if announce == "inv": test_node.send_message(msg_inv([CInv(MSG_BLOCK, block.sha256)])) - wait_until(lambda: "getheaders" in test_node.last_message, timeout=30, lock=mininode_lock) + test_node.wait_until(lambda: "getheaders" in test_node.last_message, timeout=30) test_node.send_header_for_blocks([block]) else: test_node.send_header_for_blocks([block]) @@ -399,7 +397,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # Expect a getblocktxn message. - with mininode_lock: + with p2p_lock: assert "getblocktxn" in test_node.last_message absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [0]) # should be a coinbase request @@ -418,7 +416,7 @@ class CompactBlocksTest(BitcoinTestFramework): def build_block_with_transactions(self, node, utxo, num_transactions): block = self.build_block_on_tip(node) - for i in range(num_transactions): + for _ in range(num_transactions): tx = CTransaction() tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b'')) tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE]))) @@ -441,7 +439,7 @@ class CompactBlocksTest(BitcoinTestFramework): def test_getblocktxn_response(compact_block, peer, expected_result): msg = msg_cmpctblock(compact_block.to_p2p()) peer.send_and_ping(msg) - with mininode_lock: + with p2p_lock: assert "getblocktxn" in peer.last_message absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, expected_result) @@ -506,13 +504,13 @@ class CompactBlocksTest(BitcoinTestFramework): assert tx.hash in mempool # Clear out last request. - with mininode_lock: + with p2p_lock: test_node.last_message.pop("getblocktxn", None) # Send compact block comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness) test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) - with mininode_lock: + with p2p_lock: # Shouldn't have gotten a request for any transaction assert "getblocktxn" not in test_node.last_message @@ -539,7 +537,7 @@ class CompactBlocksTest(BitcoinTestFramework): comp_block.initialize_from_block(block, prefill_list=[0], use_witness=(version == 2)) test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) absolute_indexes = [] - with mininode_lock: + with p2p_lock: assert "getblocktxn" in test_node.last_message absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) @@ -590,10 +588,10 @@ class CompactBlocksTest(BitcoinTestFramework): num_to_request = random.randint(1, len(block.vtx)) msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) test_node.send_message(msg) - wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10, lock=mininode_lock) + test_node.wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10) [tx.calc_sha256() for tx in block.vtx] - with mininode_lock: + with p2p_lock: assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16)) all_indices = msg.block_txn_request.to_absolute() for index in all_indices: @@ -613,11 +611,11 @@ class CompactBlocksTest(BitcoinTestFramework): # allowed depth for a blocktxn response. block_hash = node.getblockhash(current_height) msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) - with mininode_lock: + with p2p_lock: test_node.last_message.pop("block", None) test_node.last_message.pop("blocktxn", None) test_node.send_and_ping(msg) - with mininode_lock: + with p2p_lock: test_node.last_message["block"].block.calc_sha256() assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16)) assert "blocktxn" not in test_node.last_message @@ -627,24 +625,24 @@ class CompactBlocksTest(BitcoinTestFramework): # Test that requesting old compactblocks doesn't work. MAX_CMPCTBLOCK_DEPTH = 5 new_blocks = [] - for i in range(MAX_CMPCTBLOCK_DEPTH + 1): + for _ in range(MAX_CMPCTBLOCK_DEPTH + 1): test_node.clear_block_announcement() new_blocks.append(node.generate(1)[0]) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) + test_node.wait_until(test_node.received_block_announcement, timeout=30) test_node.clear_block_announcement() test_node.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) - wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30, lock=mininode_lock) + test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) test_node.clear_block_announcement() node.generate(1) - wait_until(test_node.received_block_announcement, timeout=30, lock=mininode_lock) + test_node.wait_until(test_node.received_block_announcement, timeout=30) test_node.clear_block_announcement() - with mininode_lock: + with p2p_lock: test_node.last_message.pop("block", None) test_node.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, int(new_blocks[0], 16))])) - wait_until(lambda: "block" in test_node.last_message, timeout=30, lock=mininode_lock) - with mininode_lock: + test_node.wait_until(lambda: "block" in test_node.last_message, timeout=30) + with p2p_lock: test_node.last_message["block"].block.calc_sha256() assert_equal(test_node.last_message["block"].block.sha256, int(new_blocks[0], 16)) @@ -672,10 +670,10 @@ class CompactBlocksTest(BitcoinTestFramework): # (to avoid fingerprinting attacks). msg = msg_getblocktxn() msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) - with mininode_lock: + with p2p_lock: test_node.last_message.pop("blocktxn", None) test_node.send_and_ping(msg) - with mininode_lock: + with p2p_lock: assert "blocktxn" not in test_node.last_message def test_end_to_end_block_relay(self, listeners): @@ -691,10 +689,9 @@ class CompactBlocksTest(BitcoinTestFramework): node.submitblock(ToHex(block)) for l in listeners: - wait_until(lambda: l.received_block_announcement(), timeout=30, lock=mininode_lock) - with mininode_lock: + l.wait_until(lambda: "cmpctblock" in l.last_message, timeout=30) + with p2p_lock: for l in listeners: - assert "cmpctblock" in l.last_message l.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.sha256, block.sha256) @@ -750,7 +747,7 @@ class CompactBlocksTest(BitcoinTestFramework): cmpct_block.initialize_from_block(block) msg = msg_cmpctblock(cmpct_block.to_p2p()) peer.send_and_ping(msg) - with mininode_lock: + with p2p_lock: assert "getblocktxn" in peer.last_message return block, cmpct_block diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index 09b9ebeb2..b7c2a306e 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -10,7 +10,6 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, connect_nodes, - wait_until, ) class DisconnectBanTest(BitcoinTestFramework): @@ -28,7 +27,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("setban: successfully ban single IP address") assert_equal(len(self.nodes[1].getpeerinfo()), 2) # node1 should have 2 connections to node0 at this point self.nodes[1].setban(subnet="127.0.0.1", command="add") - wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0, timeout=10) + self.wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0, timeout=10) assert_equal(len(self.nodes[1].getpeerinfo()), 0) # all nodes must be disconnected at this point assert_equal(len(self.nodes[1].listbanned()), 1) @@ -95,7 +94,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("disconnectnode: successfully disconnect node by address") address1 = self.nodes[0].getpeerinfo()[0]['addr'] self.nodes[0].disconnectnode(address=address1) - wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) + self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) assert not [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] self.log.info("disconnectnode: successfully reconnect node") @@ -106,7 +105,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("disconnectnode: successfully disconnect node by node id") id1 = self.nodes[0].getpeerinfo()[0]['id'] self.nodes[0].disconnectnode(nodeid=id1) - wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) + self.wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) assert not [node for node in self.nodes[0].getpeerinfo() if node['id'] == id1] if __name__ == '__main__': diff --git a/test/functional/p2p_dos_header_tree.py b/test/functional/p2p_dos_header_tree.py index f8552cf53..7dd8c3146 100755 --- a/test/functional/p2p_dos_header_tree.py +++ b/test/functional/p2p_dos_header_tree.py @@ -8,7 +8,7 @@ from test_framework.messages import ( CBlockHeader, FromHex, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, msg_headers, ) diff --git a/test/functional/p2p_eviction.py b/test/functional/p2p_eviction.py index b2b3a89aa..72a255991 100755 --- a/test/functional/p2p_eviction.py +++ b/test/functional/p2p_eviction.py @@ -15,11 +15,11 @@ Therefore, this test is limited to the remaining protection criteria. import time -from test_framework.test_framework import BitcoinTestFramework -from test_framework.mininode import P2PInterface, P2PDataStore -from test_framework.util import assert_equal, wait_until from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import CTransaction, FromHex, msg_pong, msg_tx +from test_framework.p2p import P2PDataStore, P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal class SlowP2PDataStore(P2PDataStore): @@ -92,7 +92,7 @@ class P2PEvict(BitcoinTestFramework): for _ in range(8): fastpeer = node.add_p2p_connection(P2PInterface()) current_peer += 1 - wait_until(lambda: "ping" in fastpeer.last_message, timeout=10) + self.wait_until(lambda: "ping" in fastpeer.last_message, timeout=10) # Make sure by asking the node what the actual min pings are peerinfo = node.getpeerinfo() diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index 73afe9adc..ea066a984 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -5,26 +5,12 @@ """Test processing of feefilter messages.""" from decimal import Decimal -import time from test_framework.messages import MSG_TX, MSG_WTX, msg_feefilter -from test_framework.mininode import mininode_lock, P2PInterface +from test_framework.p2p import P2PInterface, p2p_lock from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal - - -def hashToHex(hash): - return format(hash, '064x') - - -# Wait up to 60 secs to see if the testnode has received all the expected invs -def allInvsMatch(invsExpected, testnode): - for x in range(60): - with mininode_lock: - if (sorted(invsExpected) == sorted(testnode.txinvs)): - return True - time.sleep(1) - return False +from test_framework.wallet import MiniWallet class FeefilterConn(P2PInterface): @@ -34,7 +20,7 @@ class FeefilterConn(P2PInterface): self.feefilter_received = True def assert_feefilter_received(self, recv: bool): - with mininode_lock: + with p2p_lock: assert_equal(self.feefilter_received, recv) @@ -46,10 +32,14 @@ class TestP2PConn(P2PInterface): def on_inv(self, message): for i in message.inv: if (i.type == MSG_TX) or (i.type == MSG_WTX): - self.txinvs.append(hashToHex(i.hash)) + self.txinvs.append('{:064x}'.format(i.hash)) + + def wait_for_invs_to_match(self, invs_expected): + invs_expected.sort() + self.wait_until(lambda: invs_expected == sorted(self.txinvs)) def clear_invs(self): - with mininode_lock: + with p2p_lock: self.txinvs = [] @@ -61,10 +51,12 @@ class FeeFilterTest(BitcoinTestFramework): # mempool and wallet feerate calculation based on GetFee # rounding down 3 places, leading to stranded transactions. # See issue #16499 - self.extra_args = [["-minrelaytxfee=0.00000100", "-mintxfee=0.00000100"]] * self.num_nodes - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + # grant noban permission to all peers to speed up tx relay / mempool sync + self.extra_args = [[ + "-minrelaytxfee=0.00000100", + "-mintxfee=0.00000100", + "[email protected]", + ]] * self.num_nodes def run_test(self): self.test_feefilter_forcerelay() @@ -85,29 +77,28 @@ class FeeFilterTest(BitcoinTestFramework): def test_feefilter(self): node1 = self.nodes[1] node0 = self.nodes[0] + miniwallet = MiniWallet(node1) + # Add enough mature utxos to the wallet, so that all txs spend confirmed coins + miniwallet.generate(5) + node1.generate(100) conn = self.nodes[0].add_p2p_connection(TestP2PConn()) - # Test that invs are received by test connection for all txs at - # feerate of .2 sat/byte - node1.settxfee(Decimal("0.00000200")) - txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert allInvsMatch(txids, conn) + self.log.info("Test txs paying 0.2 sat/byte are received by test connection") + txids = [miniwallet.send_self_transfer(fee_rate=Decimal('0.00000200'), from_node=node1)['wtxid'] for _ in range(3)] + conn.wait_for_invs_to_match(txids) conn.clear_invs() - # Set a filter of .15 sat/byte on test connection + # Set a fee filter of 0.15 sat/byte on test connection conn.send_and_ping(msg_feefilter(150)) - # Test that txs are still being received by test connection (paying .15 sat/byte) - node1.settxfee(Decimal("0.00000150")) - txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert allInvsMatch(txids, conn) + self.log.info("Test txs paying 0.15 sat/byte are received by test connection") + txids = [miniwallet.send_self_transfer(fee_rate=Decimal('0.00000150'), from_node=node1)['wtxid'] for _ in range(3)] + conn.wait_for_invs_to_match(txids) conn.clear_invs() - # Change tx fee rate to .1 sat/byte and test they are no longer received - # by the test connection - node1.settxfee(Decimal("0.00000100")) - [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] + self.log.info("Test txs paying 0.1 sat/byte are no longer received by test connection") + txids = [miniwallet.send_self_transfer(fee_rate=Decimal('0.00000100'), from_node=node1)['wtxid'] for _ in range(3)] self.sync_mempools() # must be sure node 0 has received all txs # Send one transaction from node0 that should be received, so that we @@ -117,15 +108,14 @@ class FeeFilterTest(BitcoinTestFramework): # to 35 entries in an inv, which means that when this next transaction # is eligible for relay, the prior transactions from node1 are eligible # as well. - node0.settxfee(Decimal("0.00020000")) - txids = [node0.sendtoaddress(node0.getnewaddress(), 1)] - assert allInvsMatch(txids, conn) + txids = [miniwallet.send_self_transfer(fee_rate=Decimal('0.00020000'), from_node=node0)['wtxid'] for _ in range(3)] + conn.wait_for_invs_to_match(txids) conn.clear_invs() - # Remove fee filter and check that txs are received again + self.log.info("Remove fee filter and check txs are received again") conn.send_and_ping(msg_feefilter(0)) - txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert allInvsMatch(txids, conn) + txids = [miniwallet.send_self_transfer(fee_rate=Decimal('0.00020000'), from_node=node1)['wtxid'] for _ in range(3)] + conn.wait_for_invs_to_match(txids) conn.clear_invs() diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 741da3be3..613d96eaa 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -19,7 +19,7 @@ from test_framework.messages import ( msg_mempool, msg_version, ) -from test_framework.mininode import P2PInterface, mininode_lock +from test_framework.p2p import P2PInterface, p2p_lock from test_framework.script import MAX_SCRIPT_ELEMENT_SIZE from test_framework.test_framework import BitcoinTestFramework @@ -60,22 +60,22 @@ class P2PBloomFilter(P2PInterface): @property def tx_received(self): - with mininode_lock: + with p2p_lock: return self._tx_received @tx_received.setter def tx_received(self, value): - with mininode_lock: + with p2p_lock: self._tx_received = value @property def merkleblock_received(self): - with mininode_lock: + with p2p_lock: return self._merkleblock_received @merkleblock_received.setter def merkleblock_received(self, value): - with mininode_lock: + with p2p_lock: self._merkleblock_received = value @@ -218,7 +218,6 @@ class FilterTest(BitcoinTestFramework): # Add peer but do not send version yet filter_peer_without_nrelay = self.nodes[0].add_p2p_connection(P2PBloomFilter(), send_version=False, wait_for_verack=False) # Send version with fRelay=False - filter_peer_without_nrelay.wait_until(lambda: filter_peer_without_nrelay.is_connected, timeout=10) version_without_fRelay = msg_version() version_without_fRelay.nRelay = 0 filter_peer_without_nrelay.send_message(version_without_fRelay) diff --git a/test/functional/p2p_fingerprint.py b/test/functional/p2p_fingerprint.py index d743abe68..aaf862e6c 100755 --- a/test/functional/p2p_fingerprint.py +++ b/test/functional/p2p_fingerprint.py @@ -12,7 +12,7 @@ import time from test_framework.blocktools import (create_block, create_coinbase) from test_framework.messages import CInv, MSG_BLOCK -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, msg_headers, msg_block, @@ -22,9 +22,9 @@ from test_framework.mininode import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) + class P2PFingerprintTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -102,12 +102,12 @@ class P2PFingerprintTest(BitcoinTestFramework): # Check that getdata request for stale block succeeds self.send_block_request(stale_hash, node0) test_function = lambda: self.last_block_equals(stale_hash, node0) - wait_until(test_function, timeout=3) + self.wait_until(test_function, timeout=3) # Check that getheader request for stale block header succeeds self.send_header_request(stale_hash, node0) test_function = lambda: self.last_header_equals(stale_hash, node0) - wait_until(test_function, timeout=3) + self.wait_until(test_function, timeout=3) # Longest chain is extended so stale is much older than chain tip self.nodes[0].setmocktime(0) @@ -138,11 +138,11 @@ class P2PFingerprintTest(BitcoinTestFramework): self.send_block_request(block_hash, node0) test_function = lambda: self.last_block_equals(block_hash, node0) - wait_until(test_function, timeout=3) + self.wait_until(test_function, timeout=3) self.send_header_request(block_hash, node0) test_function = lambda: self.last_header_equals(block_hash, node0) - wait_until(test_function, timeout=3) + self.wait_until(test_function, timeout=3) if __name__ == '__main__': P2PFingerprintTest().main() diff --git a/test/functional/p2p_getaddr_caching.py b/test/functional/p2p_getaddr_caching.py index c9278eab9..6622ea9ec 100755 --- a/test/functional/p2p_getaddr_caching.py +++ b/test/functional/p2p_getaddr_caching.py @@ -12,9 +12,9 @@ from test_framework.messages import ( msg_addr, msg_getaddr, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, - mininode_lock + p2p_lock ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -44,7 +44,7 @@ class AddrReceiver(P2PInterface): self.received_addrs = None def get_received_addrs(self): - with mininode_lock: + with p2p_lock: return self.received_addrs def on_addr(self, message): diff --git a/test/functional/p2p_getdata.py b/test/functional/p2p_getdata.py index d1b11c2c6..51921a8ab 100755 --- a/test/functional/p2p_getdata.py +++ b/test/functional/p2p_getdata.py @@ -9,7 +9,7 @@ from test_framework.messages import ( CInv, msg_getdata, ) -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework diff --git a/test/functional/p2p_invalid_block.py b/test/functional/p2p_invalid_block.py index e280a6299..b2c3c5d45 100755 --- a/test/functional/p2p_invalid_block.py +++ b/test/functional/p2p_invalid_block.py @@ -14,7 +14,7 @@ import copy from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script from test_framework.messages import COIN -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal diff --git a/test/functional/p2p_invalid_locator.py b/test/functional/p2p_invalid_locator.py index 0155eb21f..24328c291 100755 --- a/test/functional/p2p_invalid_locator.py +++ b/test/functional/p2p_invalid_locator.py @@ -6,7 +6,7 @@ """ from test_framework.messages import msg_getheaders, msg_getblocks, MAX_LOCATOR_SZ -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index d9a9ae518..fe57057a8 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -17,14 +17,13 @@ from test_framework.messages import ( MSG_TX, ser_string, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PDataStore, P2PInterface, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) VALID_DATA_LIMIT = MAX_PROTOCOL_MESSAGE_LENGTH - 5 # Account for the 5-byte length prefix @@ -70,7 +69,7 @@ class InvalidMessagesTest(BitcoinTestFramework): before = int(self.nodes[0].getnettotals()['totalbytesrecv']) conn.send_raw_message(msg[:cut_pos]) # Wait until node has processed the first half of the message - wait_until(lambda: int(self.nodes[0].getnettotals()['totalbytesrecv']) != before) + self.wait_until(lambda: int(self.nodes[0].getnettotals()['totalbytesrecv']) != before) middle = int(self.nodes[0].getnettotals()['totalbytesrecv']) # If this assert fails, we've hit an unlikely race # where the test framework sent a message in between the two halves diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index c70a89246..a0ef6c9d6 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -13,11 +13,10 @@ from test_framework.messages import ( CTxIn, CTxOut, ) -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) from data import invalid_txs @@ -146,7 +145,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): # tx_orphan_no_fee, because it has too low fee (p2ps[0] is not disconnected for relaying that tx) # tx_orphan_invaid, because it has negative fee (p2ps[1] is disconnected for relaying that tx) - wait_until(lambda: 1 == len(node.getpeerinfo()), timeout=12) # p2ps[1] is no longer connected + self.wait_until(lambda: 1 == len(node.getpeerinfo()), timeout=12) # p2ps[1] is no longer connected assert_equal(expected_mempool, set(node.getrawmempool())) self.log.info('Test orphan pool overflow') diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index fe6e236fc..4b32d60db 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -15,21 +15,19 @@ import time from test_framework.messages import ( msg_getaddr, msg_ping, - msg_verack, msg_version, ) -from test_framework.mininode import mininode_lock, P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_greater_than_or_equal, - wait_until, ) DISCOURAGEMENT_THRESHOLD = 100 -class CLazyNode(P2PInterface): +class LazyPeer(P2PInterface): def __init__(self): super().__init__() self.unexpected_msg = False @@ -42,6 +40,7 @@ class CLazyNode(P2PInterface): def on_open(self): self.ever_connected = True + # Does not respond to "version" with "verack" def on_version(self, message): self.bad_message(message) def on_verack(self, message): self.bad_message(message) def on_inv(self, message): self.bad_message(message) @@ -63,25 +62,9 @@ class CLazyNode(P2PInterface): def on_getblocktxn(self, message): self.bad_message(message) def on_blocktxn(self, message): self.bad_message(message) -# Node that never sends a version. We'll use this to send a bunch of messages -# anyway, and eventually get disconnected. -class CNodeNoVersionMisbehavior(CLazyNode): - # Send enough veracks without a message to reach the peer discouragement - # threshold. This should get us disconnected. NOTE: implementation-specific - # test; update if our discouragement policy for peer misbehavior changes. - def on_open(self): - super().on_open() - for _ in range(DISCOURAGEMENT_THRESHOLD): - self.send_message(msg_verack()) - -# Node that never sends a version. This one just sits idle and hopes to receive -# any message (it shouldn't!) -class CNodeNoVersionIdle(CLazyNode): - def __init__(self): - super().__init__() -# Node that sends a version but not a verack. -class CNodeNoVerackIdle(CLazyNode): +# Peer that sends a version but not a verack. +class NoVerackIdlePeer(LazyPeer): def __init__(self): self.version_received = False super().__init__() @@ -100,6 +83,7 @@ class P2PVersionStore(P2PInterface): version_received = None def on_version(self, msg): + # Responds with an appropriate verack super().on_version(msg) self.version_received = msg @@ -109,34 +93,45 @@ class P2PLeakTest(BitcoinTestFramework): self.num_nodes = 1 def run_test(self): - no_version_disconnect_node = self.nodes[0].add_p2p_connection( - CNodeNoVersionMisbehavior(), send_version=False, wait_for_verack=False) - no_version_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVersionIdle(), send_version=False, wait_for_verack=False) - no_verack_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVerackIdle(), wait_for_verack=False) + # Peer that never sends a version. We will send a bunch of messages + # from this peer anyway and verify eventual disconnection. + no_version_disconnect_peer = self.nodes[0].add_p2p_connection( + LazyPeer(), send_version=False, wait_for_verack=False) + + # Another peer that never sends a version, nor any other messages. It shouldn't receive anything from the node. + no_version_idle_peer = self.nodes[0].add_p2p_connection(LazyPeer(), send_version=False, wait_for_verack=False) + + # Peer that sends a version but not a verack. + no_verack_idle_peer = self.nodes[0].add_p2p_connection(NoVerackIdlePeer(), wait_for_verack=False) + + # Send enough ping messages (any non-version message will do) prior to sending + # version to reach the peer discouragement threshold. This should get us disconnected. + for _ in range(DISCOURAGEMENT_THRESHOLD): + no_version_disconnect_peer.send_message(msg_ping()) - # Wait until we got the verack in response to the version. Though, don't wait for the other node to receive the + # Wait until we got the verack in response to the version. Though, don't wait for the node to receive the # verack, since we never sent one - no_verack_idlenode.wait_for_verack() + no_verack_idle_peer.wait_for_verack() - wait_until(lambda: no_version_disconnect_node.ever_connected, timeout=10, lock=mininode_lock) - wait_until(lambda: no_version_idlenode.ever_connected, timeout=10, lock=mininode_lock) - wait_until(lambda: no_verack_idlenode.version_received, timeout=10, lock=mininode_lock) + no_version_disconnect_peer.wait_until(lambda: no_version_disconnect_peer.ever_connected, check_connected=False) + no_version_idle_peer.wait_until(lambda: no_version_idle_peer.ever_connected) + no_verack_idle_peer.wait_until(lambda: no_verack_idle_peer.version_received) - # Mine a block and make sure that it's not sent to the connected nodes - self.nodes[0].generatetoaddress(1, self.nodes[0].get_deterministic_priv_key().address) + # Mine a block and make sure that it's not sent to the connected peers + self.nodes[0].generate(nblocks=1) #Give the node enough time to possibly leak out a message time.sleep(5) - # Expect this node to be disconnected for misbehavior - assert not no_version_disconnect_node.is_connected + # Expect this peer to be disconnected for misbehavior + assert not no_version_disconnect_peer.is_connected self.nodes[0].disconnect_p2ps() # Make sure no unexpected messages came in - assert no_version_disconnect_node.unexpected_msg == False - assert no_version_idlenode.unexpected_msg == False - assert no_verack_idlenode.unexpected_msg == False + assert no_version_disconnect_peer.unexpected_msg == False + assert no_version_idle_peer.unexpected_msg == False + assert no_verack_idle_peer.unexpected_msg == False self.log.info('Check that the version message does not leak the local address of the node') p2p_version_store = self.nodes[0].add_p2p_connection(P2PVersionStore()) @@ -149,14 +144,13 @@ class P2PLeakTest(BitcoinTestFramework): assert_equal(ver.nStartingHeight, 201) assert_equal(ver.nRelay, 1) - self.log.info('Check that old nodes are disconnected') - p2p_old_node = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False) + self.log.info('Check that old peers are disconnected') + p2p_old_peer = self.nodes[0].add_p2p_connection(P2PInterface(), send_version=False, wait_for_verack=False) old_version_msg = msg_version() old_version_msg.nVersion = 31799 - wait_until(lambda: p2p_old_node.is_connected) with self.nodes[0].assert_debug_log(['peer=4 using obsolete version 31799; disconnecting']): - p2p_old_node.send_message(old_version_msg) - p2p_old_node.wait_for_disconnect() + p2p_old_peer.send_message(old_version_msg) + p2p_old_peer.wait_for_disconnect() if __name__ == '__main__': diff --git a/test/functional/p2p_leak_tx.py b/test/functional/p2p_leak_tx.py index da30ad597..9e761db03 100755 --- a/test/functional/p2p_leak_tx.py +++ b/test/functional/p2p_leak_tx.py @@ -5,7 +5,7 @@ """Test that we don't leak txs to inbound peers that we haven't yet announced to""" from test_framework.messages import msg_getdata, CInv, MSG_TX -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, diff --git a/test/functional/p2p_nobloomfilter_messages.py b/test/functional/p2p_nobloomfilter_messages.py index accc5dc23..c2311cb19 100755 --- a/test/functional/p2p_nobloomfilter_messages.py +++ b/test/functional/p2p_nobloomfilter_messages.py @@ -12,7 +12,7 @@ Test that, when bloom filters are not enabled, peers are disconnected if: """ from test_framework.messages import msg_mempool, msg_filteradd, msg_filterload, msg_filterclear -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index a2f6ea538..2c9cbea5e 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -9,13 +9,12 @@ and that it responds to getdata requests for blocks correctly: - send a block within 288 + 2 of the tip - disconnect peers who request blocks older than that.""" from test_framework.messages import CInv, MSG_BLOCK, msg_getdata, msg_verack, NODE_NETWORK_LIMITED, NODE_WITNESS -from test_framework.mininode import P2PInterface, mininode_lock +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, disconnect_nodes, connect_nodes, - wait_until, ) @@ -28,7 +27,7 @@ class P2PIgnoreInv(P2PInterface): self.firstAddrnServices = message.addrs[0].nServices def wait_for_addr(self, timeout=5): test_function = lambda: self.last_message.get("addr") - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def send_getdata_for_block(self, blockhash): getdata_request = msg_getdata() getdata_request.inv.append(CInv(MSG_BLOCK, int(blockhash, 16))) diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index 254352c81..3ec36edb4 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -13,7 +13,7 @@ from test_framework.messages import ( CTxInWitness, FromHex, ) -from test_framework.mininode import P2PDataStore +from test_framework.p2p import P2PDataStore from test_framework.script import ( CScript, OP_TRUE, @@ -24,7 +24,6 @@ from test_framework.util import ( assert_equal, connect_nodes, p2p_port, - wait_until, ) @@ -109,7 +108,7 @@ class P2PPermissionsTests(BitcoinTestFramework): self.sync_all() self.log.debug("Create a connection from a forcerelay peer that rebroadcasts raw txs") - # A python mininode is needed to send the raw transaction directly. If a full node was used, it could only + # A test framework p2p connection is needed to send the raw transaction directly. If a full node was used, it could only # rebroadcast via the inv-getdata mechanism. However, even for forcerelay connections, a full node would # currently not request a txid that is already in the mempool. self.restart_node(1, extra_args=["[email protected]"]) @@ -137,7 +136,7 @@ class P2PPermissionsTests(BitcoinTestFramework): connect_nodes(self.nodes[1], 0) with self.nodes[1].assert_debug_log(["Force relaying tx {} from peer=0".format(txid)]): p2p_rebroadcast_wallet.send_txs_and_test([tx], self.nodes[1]) - wait_until(lambda: txid in self.nodes[0].getrawmempool()) + self.wait_until(lambda: txid in self.nodes[0].getrawmempool()) self.log.debug("Check that node[1] will not send an invalid tx to node[0]") tx.vout[0].nValue += 1 diff --git a/test/functional/p2p_ping.py b/test/functional/p2p_ping.py index e00af88cc..888e986fb 100755 --- a/test/functional/p2p_ping.py +++ b/test/functional/p2p_ping.py @@ -7,13 +7,8 @@ import time -from test_framework.messages import ( - msg_pong, -) -from test_framework.mininode import ( - P2PInterface, - wait_until, -) +from test_framework.messages import msg_pong +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal @@ -78,7 +73,7 @@ class PingPongTest(BitcoinTestFramework): with self.nodes[0].assert_debug_log(['pong peer=0: Nonce mismatch']): # mock time PING_INTERVAL ahead to trigger node into sending a ping self.mock_forward(PING_INTERVAL + 1) - wait_until(lambda: 'ping' in no_pong_node.last_message) + no_pong_node.wait_until(lambda: 'ping' in no_pong_node.last_message) self.mock_forward(9) # Send the wrong pong no_pong_node.send_and_ping(msg_pong(no_pong_node.last_message.pop('ping').nonce - 1)) @@ -93,27 +88,27 @@ class PingPongTest(BitcoinTestFramework): assert 'ping' not in no_pong_node.last_message # mock time PING_INTERVAL ahead to trigger node into sending a ping self.mock_forward(PING_INTERVAL + 1) - wait_until(lambda: 'ping' in no_pong_node.last_message) + no_pong_node.wait_until(lambda: 'ping' in no_pong_node.last_message) ping_delay = 29 self.mock_forward(ping_delay) - wait_until(lambda: 'ping' in no_pong_node.last_message) + no_pong_node.wait_until(lambda: 'ping' in no_pong_node.last_message) no_pong_node.send_and_ping(msg_pong(no_pong_node.last_message.pop('ping').nonce)) self.check_peer_info(pingtime=ping_delay, minping=ping_delay, pingwait=None) self.log.info('Check that minping is decreased after a fast roundtrip') # mock time PING_INTERVAL ahead to trigger node into sending a ping self.mock_forward(PING_INTERVAL + 1) - wait_until(lambda: 'ping' in no_pong_node.last_message) + no_pong_node.wait_until(lambda: 'ping' in no_pong_node.last_message) ping_delay = 9 self.mock_forward(ping_delay) - wait_until(lambda: 'ping' in no_pong_node.last_message) + no_pong_node.wait_until(lambda: 'ping' in no_pong_node.last_message) no_pong_node.send_and_ping(msg_pong(no_pong_node.last_message.pop('ping').nonce)) self.check_peer_info(pingtime=ping_delay, minping=ping_delay, pingwait=None) self.log.info('Check that peer is disconnected after ping timeout') assert 'ping' not in no_pong_node.last_message self.nodes[0].ping() - wait_until(lambda: 'ping' in no_pong_node.last_message) + no_pong_node.wait_until(lambda: 'ping' in no_pong_node.last_message) with self.nodes[0].assert_debug_log(['ping timeout: 1201.000000s']): self.mock_forward(20 * 60 + 1) time.sleep(4) # peertimeout + 1 diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 728212ca2..dfbf5d52f 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test segwit transactions and blocks on P2P network.""" +from decimal import Decimal import math import random import struct @@ -25,6 +26,7 @@ from test_framework.messages import ( MSG_BLOCK, MSG_TX, MSG_WITNESS_FLAG, + MSG_WITNESS_TX, MSG_WTX, NODE_NETWORK, NODE_WITNESS, @@ -42,9 +44,9 @@ from test_framework.messages import ( uint256_from_str, FromHex, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, - mininode_lock, + p2p_lock, ) from test_framework.script import ( CScript, @@ -83,7 +85,6 @@ from test_framework.util import ( softfork_active, hex_str_to_bytes, assert_raises_rpc_error, - wait_until, ) # The versionbit bit used to signal activation of SegWit @@ -153,8 +154,8 @@ class TestP2PConn(P2PInterface): self.lastgetdata = [] self.wtxidrelay = wtxidrelay - # Avoid sending out msg_getdata in the mininode thread as a reply to invs. - # They are not needed and would only lead to races because we send msg_getdata out in the test thread + # Don't send getdata message replies to invs automatically. + # We'll send the getdata messages explicitly in the test logic. def on_inv(self, message): pass @@ -174,7 +175,10 @@ class TestP2PConn(P2PInterface): self.last_wtxidrelay.append(message) def announce_tx_and_wait_for_getdata(self, tx, timeout=60, success=True, use_wtxid=False): - with mininode_lock: + if success: + # sanity check + assert (self.wtxidrelay and use_wtxid) or (not self.wtxidrelay and not use_wtxid) + with p2p_lock: self.last_message.pop("getdata", None) if use_wtxid: wtxid = tx.calc_sha256(True) @@ -192,7 +196,7 @@ class TestP2PConn(P2PInterface): assert not self.last_message.get("getdata") def announce_block_and_wait_for_getdata(self, block, use_header, timeout=60): - with mininode_lock: + with p2p_lock: self.last_message.pop("getdata", None) self.last_message.pop("getheaders", None) msg = msg_headers() @@ -206,7 +210,7 @@ class TestP2PConn(P2PInterface): self.wait_for_getdata([block.sha256]) def request_block(self, blockhash, inv_type, timeout=60): - with mininode_lock: + with p2p_lock: self.last_message.pop("block", None) self.send_message(msg_getdata(inv=[CInv(inv_type, blockhash)])) self.wait_for_block(blockhash, timeout) @@ -259,6 +263,8 @@ class SegWitTest(BitcoinTestFramework): self.old_node = self.nodes[0].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK) # self.std_node is for testing node1 (fRequireStandard=true) self.std_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK | NODE_WITNESS) + # self.std_wtx_node is for testing node1 with wtxid relay + self.std_wtx_node = self.nodes[1].add_p2p_connection(TestP2PConn(wtxidrelay=True), services=NODE_NETWORK | NODE_WITNESS) assert self.test_node.nServices & NODE_WITNESS != 0 @@ -690,13 +696,13 @@ class SegWitTest(BitcoinTestFramework): if not self.segwit_active: # Just check mempool acceptance, but don't add the transaction to the mempool, since witness is disallowed # in blocks and the tx is impossible to mine right now. - assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True}]) + assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True, 'vsize': tx3.get_vsize(), 'fees': { 'base': Decimal('0.00001000')}}]) # Create the same output as tx3, but by replacing tx tx3_out = tx3.vout[0] tx3 = tx tx3.vout = [tx3_out] tx3.rehash() - assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True}]) + assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True, 'vsize': tx3.get_vsize(), 'fees': { 'base': Decimal('0.00011000')}}]) test_transaction_acceptance(self.nodes[0], self.test_node, tx3, with_witness=True, accepted=True) self.nodes[0].generate(1) @@ -942,7 +948,7 @@ class SegWitTest(BitcoinTestFramework): parent_tx = CTransaction() parent_tx.vin.append(CTxIn(prevout, b"")) child_value = int(value / NUM_OUTPUTS) - for i in range(NUM_OUTPUTS): + for _ in range(NUM_OUTPUTS): parent_tx.vout.append(CTxOut(child_value, script_pubkey)) parent_tx.vout[0].nValue -= 50000 assert parent_tx.vout[0].nValue > 0 @@ -952,7 +958,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(NUM_OUTPUTS): child_tx.vin.append(CTxIn(COutPoint(parent_tx.sha256, i), b"")) child_tx.vout = [CTxOut(value - 100000, CScript([OP_TRUE]))] - for i in range(NUM_OUTPUTS): + for _ in range(NUM_OUTPUTS): child_tx.wit.vtxinwit.append(CTxInWitness()) child_tx.wit.vtxinwit[-1].scriptWitness.stack = [b'a' * 195] * (2 * NUM_DROPS) + [witness_program] child_tx.rehash() @@ -1199,7 +1205,7 @@ class SegWitTest(BitcoinTestFramework): tx = CTransaction() tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) value = self.utxo[0].nValue - for i in range(10): + for _ in range(10): tx.vout.append(CTxOut(int(value / 10), script_pubkey)) tx.vout[0].nValue -= 1000 assert tx.vout[0].nValue >= 0 @@ -1319,9 +1325,14 @@ class SegWitTest(BitcoinTestFramework): tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_program2] tx3.rehash() - # Node will not be blinded to the transaction + # Node will not be blinded to the transaction, requesting it any number of times + # if it is being announced via txid relay. + # Node will be blinded to the transaction via wtxid, however. self.std_node.announce_tx_and_wait_for_getdata(tx3) + self.std_wtx_node.announce_tx_and_wait_for_getdata(tx3, use_wtxid=True) test_transaction_acceptance(self.nodes[1], self.std_node, tx3, True, False, 'tx-size') + self.std_node.announce_tx_and_wait_for_getdata(tx3) + self.std_wtx_node.announce_tx_and_wait_for_getdata(tx3, use_wtxid=True, success=False) # Remove witness stuffing, instead add extra witness push on stack tx3.vout[0] = CTxOut(tx2.vout[0].nValue - 1000, CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE])) @@ -1372,7 +1383,7 @@ class SegWitTest(BitcoinTestFramework): tx = CTransaction() tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) split_value = (self.utxo[0].nValue - 4000) // NUM_SEGWIT_VERSIONS - for i in range(NUM_SEGWIT_VERSIONS): + for _ in range(NUM_SEGWIT_VERSIONS): tx.vout.append(CTxOut(split_value, CScript([OP_TRUE]))) tx.rehash() block = self.build_next_block() @@ -1418,7 +1429,7 @@ class SegWitTest(BitcoinTestFramework): temp_utxo.pop() # last entry in temp_utxo was the output we just spent temp_utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - # Spend everything in temp_utxo back to an OP_TRUE output. + # Spend everything in temp_utxo into an segwit v1 output. tx3 = CTransaction() total_value = 0 for i in temp_utxo: @@ -1426,8 +1437,16 @@ class SegWitTest(BitcoinTestFramework): tx3.wit.vtxinwit.append(CTxInWitness()) total_value += i.nValue tx3.wit.vtxinwit[-1].scriptWitness.stack = [witness_program] - tx3.vout.append(CTxOut(total_value - 1000, CScript([OP_TRUE]))) + tx3.vout.append(CTxOut(total_value - 1000, script_pubkey)) tx3.rehash() + + # First we test this transaction against fRequireStandard=true node + # making sure the txid is added to the reject filter + self.std_node.announce_tx_and_wait_for_getdata(tx3) + test_transaction_acceptance(self.nodes[1], self.std_node, tx3, with_witness=True, accepted=False, reason="bad-txns-nonstandard-inputs") + # Now the node will no longer ask for getdata of this transaction when advertised by same txid + self.std_node.announce_tx_and_wait_for_getdata(tx3, timeout=5, success=False) + # Spending a higher version witness output is not allowed by policy, # even with fRequireStandard=false. test_transaction_acceptance(self.nodes[0], self.test_node, tx3, with_witness=True, accepted=False, reason="reserved for soft-fork upgrades") @@ -1646,7 +1665,7 @@ class SegWitTest(BitcoinTestFramework): tx = CTransaction() tx.vin.append(CTxIn(COutPoint(prev_utxo.sha256, prev_utxo.n), b"")) split_value = prev_utxo.nValue // NUM_SIGHASH_TESTS - for i in range(NUM_SIGHASH_TESTS): + for _ in range(NUM_SIGHASH_TESTS): tx.vout.append(CTxOut(split_value, script_pubkey)) tx.wit.vtxinwit.append(CTxInWitness()) sign_p2pk_witness_input(witness_program, tx, 0, SIGHASH_ALL, prev_utxo.nValue, key) @@ -1676,7 +1695,7 @@ class SegWitTest(BitcoinTestFramework): tx.wit.vtxinwit.append(CTxInWitness()) total_value += temp_utxos[i].nValue split_value = total_value // num_outputs - for i in range(num_outputs): + for _ in range(num_outputs): tx.vout.append(CTxOut(split_value, script_pubkey)) for i in range(num_inputs): # Now try to sign each input, using a random hashtype. @@ -1974,7 +1993,7 @@ class SegWitTest(BitcoinTestFramework): split_value = self.utxo[0].nValue // outputs tx = CTransaction() tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) - for i in range(outputs): + for _ in range(outputs): tx.vout.append(CTxOut(split_value, script_pubkey)) tx.vout[-2].scriptPubKey = script_pubkey_toomany tx.vout[-1].scriptPubKey = script_pubkey_justright @@ -2060,7 +2079,7 @@ class SegWitTest(BitcoinTestFramework): if (len(tx.wit.vtxinwit) != len(tx.vin)): # vtxinwit must have the same length as vin tx.wit.vtxinwit = tx.wit.vtxinwit[:len(tx.vin)] - for i in range(len(tx.wit.vtxinwit), len(tx.vin)): + for _ in range(len(tx.wit.vtxinwit), len(tx.vin)): tx.wit.vtxinwit.append(CTxInWitness()) r += tx.wit.serialize() r += struct.pack("<I", tx.nLockTime) @@ -2096,7 +2115,7 @@ class SegWitTest(BitcoinTestFramework): # Check wtxidrelay feature negotiation message through connecting a new peer def received_wtxidrelay(): return (len(self.wtx_node.last_wtxidrelay) > 0) - wait_until(received_wtxidrelay, timeout=60, lock=mininode_lock) + self.wtx_node.wait_until(received_wtxidrelay) # Create a Segwit output from the latest UTXO # and announce it to the network @@ -2120,27 +2139,27 @@ class SegWitTest(BitcoinTestFramework): # Announce Segwit transaction with wtxid # and wait for getdata self.wtx_node.announce_tx_and_wait_for_getdata(tx2, use_wtxid=True) - with mininode_lock: + with p2p_lock: lgd = self.wtx_node.lastgetdata[:] assert_equal(lgd, [CInv(MSG_WTX, tx2.calc_sha256(True))]) # Announce Segwit transaction from non wtxidrelay peer # and wait for getdata self.tx_node.announce_tx_and_wait_for_getdata(tx2, use_wtxid=False) - with mininode_lock: + with p2p_lock: lgd = self.tx_node.lastgetdata[:] assert_equal(lgd, [CInv(MSG_TX|MSG_WITNESS_FLAG, tx2.sha256)]) # Send tx2 through; it's an orphan so won't be accepted - with mininode_lock: + with p2p_lock: self.wtx_node.last_message.pop("getdata", None) test_transaction_acceptance(self.nodes[0], self.wtx_node, tx2, with_witness=True, accepted=False) # Expect a request for parent (tx) by txid despite use of WTX peer self.wtx_node.wait_for_getdata([tx.sha256], 60) - with mininode_lock: + with p2p_lock: lgd = self.wtx_node.lastgetdata[:] - assert_equal(lgd, [CInv(MSG_TX|MSG_WITNESS_FLAG, tx.sha256)]) + assert_equal(lgd, [CInv(MSG_WITNESS_TX, tx.sha256)]) # Send tx through test_transaction_acceptance(self.nodes[0], self.wtx_node, tx, with_witness=False, accepted=True) diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 481b1c184..04e6ec417 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -87,11 +87,11 @@ e. Announce one more that doesn't connect. """ from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import CInv -from test_framework.mininode import ( +from test_framework.p2p import ( CBlockHeader, NODE_WITNESS, P2PInterface, - mininode_lock, + p2p_lock, MSG_BLOCK, msg_block, msg_getblocks, @@ -104,7 +104,6 @@ from test_framework.mininode import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) DIRECT_FETCH_RESPONSE_TIME = 0.05 @@ -147,7 +146,7 @@ class BaseNode(P2PInterface): def wait_for_block_announcement(self, block_hash, timeout=60): test_function = lambda: self.last_blockhash_announced == block_hash - wait_until(test_function, timeout=timeout, lock=mininode_lock) + self.wait_until(test_function, timeout=timeout) def on_inv(self, message): self.block_announced = True @@ -163,7 +162,7 @@ class BaseNode(P2PInterface): self.last_blockhash_announced = message.headers[-1].sha256 def clear_block_announcements(self): - with mininode_lock: + with p2p_lock: self.block_announced = False self.last_message.pop("inv", None) self.last_message.pop("headers", None) @@ -174,8 +173,8 @@ class BaseNode(P2PInterface): """Test whether the last headers announcements received are right. Headers may be announced across more than one message.""" test_function = lambda: (len(self.recent_headers_announced) >= len(headers)) - wait_until(test_function, timeout=60, lock=mininode_lock) - with mininode_lock: + self.wait_until(test_function) + with p2p_lock: assert_equal(self.recent_headers_announced, headers) self.block_announced = False self.last_message.pop("headers", None) @@ -186,9 +185,9 @@ class BaseNode(P2PInterface): inv should be a list of block hashes.""" test_function = lambda: self.block_announced - wait_until(test_function, timeout=60, lock=mininode_lock) + self.wait_until(test_function) - with mininode_lock: + with p2p_lock: compare_inv = [] if "inv" in self.last_message: compare_inv = [x.hash for x in self.last_message["inv"].inv] @@ -298,7 +297,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.send_header_for_blocks([new_block]) test_node.wait_for_getdata([new_block.sha256]) test_node.send_and_ping(msg_block(new_block)) # make sure this block is processed - wait_until(lambda: inv_node.block_announced, timeout=60, lock=mininode_lock) + inv_node.wait_until(lambda: inv_node.block_announced) inv_node.clear_block_announcements() test_node.clear_block_announcements() @@ -328,7 +327,7 @@ class SendHeadersTest(BitcoinTestFramework): for j in range(2): self.log.debug("Part 2.{}.{}: starting...".format(i, j)) blocks = [] - for b in range(i + 1): + for _ in range(i + 1): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -443,7 +442,7 @@ class SendHeadersTest(BitcoinTestFramework): # Create 2 blocks. Send the blocks, then send the headers. blocks = [] - for b in range(2): + for _ in range(2): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -456,12 +455,12 @@ class SendHeadersTest(BitcoinTestFramework): test_node.send_header_for_blocks(blocks) test_node.sync_with_ping() # should not have received any getdata messages - with mininode_lock: + with p2p_lock: assert "getdata" not in test_node.last_message # This time, direct fetch should work blocks = [] - for b in range(3): + for _ in range(3): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -482,7 +481,7 @@ class SendHeadersTest(BitcoinTestFramework): blocks = [] # Create extra blocks for later - for b in range(20): + for _ in range(20): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -494,7 +493,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks[0:1]) test_node.sync_with_ping() - with mininode_lock: + with p2p_lock: assert "getdata" not in test_node.last_message # Announcing one more block on fork should trigger direct fetch for @@ -513,7 +512,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks[18:19]) test_node.sync_with_ping() - with mininode_lock: + with p2p_lock: assert "getdata" not in test_node.last_message self.log.info("Part 4: success!") @@ -529,14 +528,14 @@ class SendHeadersTest(BitcoinTestFramework): test_node.last_message.pop("getdata", None) blocks = [] # Create two more blocks. - for j in range(2): + for _ in range(2): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 block_time += 1 height += 1 # Send the header of the second block -> this won't connect. - with mininode_lock: + with p2p_lock: test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[1]]) test_node.wait_for_getheaders() @@ -550,7 +549,7 @@ class SendHeadersTest(BitcoinTestFramework): # Now we test that if we repeatedly don't send connecting headers, we # don't go into an infinite loop trying to get them to connect. MAX_UNCONNECTING_HEADERS = 10 - for j in range(MAX_UNCONNECTING_HEADERS + 1): + for _ in range(MAX_UNCONNECTING_HEADERS + 1): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -559,7 +558,7 @@ class SendHeadersTest(BitcoinTestFramework): for i in range(1, MAX_UNCONNECTING_HEADERS): # Send a header that doesn't connect, check that we get a getheaders. - with mininode_lock: + with p2p_lock: test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[i]]) test_node.wait_for_getheaders() @@ -574,7 +573,7 @@ class SendHeadersTest(BitcoinTestFramework): # before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS for i in range(5 * MAX_UNCONNECTING_HEADERS - 1): # Send a header that doesn't connect, check that we get a getheaders. - with mininode_lock: + with p2p_lock: test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[i % len(blocks)]]) test_node.wait_for_getheaders() diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index 5a4fa4298..ce12ce26c 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -24,7 +24,7 @@ from time import sleep from test_framework.messages import msg_ping -from test_framework.mininode import P2PInterface +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 2527edc13..653c7ae43 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -16,14 +16,13 @@ from test_framework.messages import ( msg_inv, msg_notfound, ) -from test_framework.mininode import ( +from test_framework.p2p import ( P2PInterface, - mininode_lock, + p2p_lock, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - wait_until, ) from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE @@ -73,14 +72,14 @@ class TxDownloadTest(BitcoinTestFramework): def getdata_found(peer_index): p = self.nodes[0].p2ps[peer_index] - with mininode_lock: + with p2p_lock: return p.last_message.get("getdata") and p.last_message["getdata"].inv[-1].hash == txid node_0_mocktime = int(time.time()) while outstanding_peer_index: node_0_mocktime += MAX_GETDATA_INBOUND_WAIT self.nodes[0].setmocktime(node_0_mocktime) - wait_until(lambda: any(getdata_found(i) for i in outstanding_peer_index)) + self.wait_until(lambda: any(getdata_found(i) for i in outstanding_peer_index)) for i in outstanding_peer_index: if getdata_found(i): outstanding_peer_index.remove(i) @@ -134,24 +133,24 @@ class TxDownloadTest(BitcoinTestFramework): p = self.nodes[0].p2ps[0] - with mininode_lock: + with p2p_lock: p.tx_getdata_count = 0 p.send_message(msg_inv([CInv(t=MSG_WTX, h=i) for i in txids])) - wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT, lock=mininode_lock) - with mininode_lock: + p.wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT) + with p2p_lock: assert_equal(p.tx_getdata_count, MAX_GETDATA_IN_FLIGHT) self.log.info("Now check that if we send a NOTFOUND for a transaction, we'll get one more request") p.send_message(msg_notfound(vec=[CInv(t=MSG_WTX, h=txids[0])])) - wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT + 1, timeout=10, lock=mininode_lock) - with mininode_lock: + p.wait_until(lambda: p.tx_getdata_count >= MAX_GETDATA_IN_FLIGHT + 1, timeout=10) + with p2p_lock: assert_equal(p.tx_getdata_count, MAX_GETDATA_IN_FLIGHT + 1) WAIT_TIME = TX_EXPIRY_INTERVAL // 2 + TX_EXPIRY_INTERVAL self.log.info("if we wait about {} minutes, we should eventually get more requests".format(WAIT_TIME / 60)) self.nodes[0].setmocktime(int(time.time() + WAIT_TIME)) - wait_until(lambda: p.tx_getdata_count == MAX_GETDATA_IN_FLIGHT + 2) + p.wait_until(lambda: p.tx_getdata_count == MAX_GETDATA_IN_FLIGHT + 2) self.nodes[0].setmocktime(0) def test_spurious_notfound(self): @@ -162,7 +161,7 @@ class TxDownloadTest(BitcoinTestFramework): # Setup the p2p connections self.peers = [] for node in self.nodes: - for i in range(NUM_INBOUND): + for _ in range(NUM_INBOUND): self.peers.append(node.add_p2p_connection(TestP2PConn())) self.log.info("Nodes are setup with {} incoming connections each".format(NUM_INBOUND)) diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index 71b0b0f63..36b434bce 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -55,7 +55,7 @@ import time from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script from test_framework.messages import CBlockHeader, CInv, MSG_BLOCK, msg_block, msg_headers, msg_inv -from test_framework.mininode import mininode_lock, P2PInterface +from test_framework.p2p import p2p_lock, P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -199,13 +199,13 @@ class AcceptBlockTest(BitcoinTestFramework): # 6. Try to get node to request the missing block. # Poke the node with an inv for block at height 3 and see if that # triggers a getdata on block 2 (it should if block 2 is missing). - with mininode_lock: + with p2p_lock: # Clear state so we can check the getdata request test_node.last_message.pop("getdata", None) test_node.send_message(msg_inv([CInv(MSG_BLOCK, block_h3.sha256)])) test_node.sync_with_ping() - with mininode_lock: + with p2p_lock: getdata = test_node.last_message["getdata"] # Check that the getdata includes the right block diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 7c70f30ca..c00558448 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -22,6 +22,17 @@ from decimal import Decimal import http.client import subprocess +from test_framework.blocktools import ( + create_block, + create_coinbase, + TIME_GENESIS_BLOCK, +) +from test_framework.messages import ( + CBlockHeader, + FromHex, + msg_block, +) +from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -32,17 +43,6 @@ from test_framework.util import ( assert_is_hex_string, assert_is_hash_string, ) -from test_framework.blocktools import ( - create_block, - create_coinbase, - TIME_GENESIS_BLOCK, -) -from test_framework.messages import ( - msg_block, -) -from test_framework.mininode import ( - P2PInterface, -) class BlockchainTest(BitcoinTestFramework): @@ -280,6 +280,14 @@ class BlockchainTest(BitcoinTestFramework): assert isinstance(int(header['versionHex'], 16), int) assert isinstance(header['difficulty'], Decimal) + # Test with verbose=False, which should return the header as hex. + header_hex = node.getblockheader(blockhash=besthash, verbose=False) + assert_is_hex_string(header_hex) + + header = FromHex(CBlockHeader(), header_hex) + header.calc_sha256() + assert_equal(header.hash, besthash) + def _test_getdifficulty(self): difficulty = self.nodes[0].getdifficulty() # 1 hash in 2 should be valid, so difficulty should be 1/2**31 diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 3c81a4a4e..f19c60dc3 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -129,7 +129,8 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): try: node1.loadwallet('wmulti') except JSONRPCException as e: - if e.error['code'] == -18 and 'Wallet wmulti not found' in e.error['message']: + path = os.path.join(self.options.tmpdir, "node1", "regtest", "wallets", "wmulti") + if e.error['code'] == -18 and "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path) in e.error['message']: node1.createwallet(wallet_name='wmulti', disable_private_keys=True) else: raise diff --git a/test/functional/rpc_deprecated.py b/test/functional/rpc_deprecated.py index 9a21998d1..b71854d23 100755 --- a/test/functional/rpc_deprecated.py +++ b/test/functional/rpc_deprecated.py @@ -4,13 +4,13 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test deprecation of RPC calls.""" from test_framework.test_framework import BitcoinTestFramework -# from test_framework.util import assert_raises_rpc_error +from test_framework.util import assert_raises_rpc_error, find_vout_for_address class DeprecatedRpcTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True - self.extra_args = [[], []] + self.extra_args = [[], ['-deprecatedrpc=bumpfee']] def run_test(self): # This test should be used to verify correct behaviour of deprecated @@ -23,7 +23,38 @@ class DeprecatedRpcTest(BitcoinTestFramework): # self.log.info("Test generate RPC") # assert_raises_rpc_error(-32, 'The wallet generate rpc method is deprecated', self.nodes[0].rpc.generate, 1) # self.nodes[1].generate(1) - self.log.info("No tested deprecated RPC methods") + + if self.is_wallet_compiled(): + self.log.info("Test bumpfee RPC") + self.nodes[0].generate(101) + self.nodes[0].createwallet(wallet_name='nopriv', disable_private_keys=True) + noprivs0 = self.nodes[0].get_wallet_rpc('nopriv') + w0 = self.nodes[0].get_wallet_rpc('') + self.nodes[1].createwallet(wallet_name='nopriv', disable_private_keys=True) + noprivs1 = self.nodes[1].get_wallet_rpc('nopriv') + + address = w0.getnewaddress() + desc = w0.getaddressinfo(address)['desc'] + change_addr = w0.getrawchangeaddress() + change_desc = w0.getaddressinfo(change_addr)['desc'] + txid = w0.sendtoaddress(address=address, amount=10) + vout = find_vout_for_address(w0, txid, address) + self.nodes[0].generate(1) + rawtx = w0.createrawtransaction([{'txid': txid, 'vout': vout}], {w0.getnewaddress(): 5}, 0, True) + rawtx = w0.fundrawtransaction(rawtx, {'changeAddress': change_addr}) + signed_tx = w0.signrawtransactionwithwallet(rawtx['hex'])['hex'] + + noprivs0.importmulti([{'desc': desc, 'timestamp': 0}, {'desc': change_desc, 'timestamp': 0, 'internal': True}]) + noprivs1.importmulti([{'desc': desc, 'timestamp': 0}, {'desc': change_desc, 'timestamp': 0, 'internal': True}]) + + txid = w0.sendrawtransaction(signed_tx) + self.sync_all() + + assert_raises_rpc_error(-32, 'Using bumpfee with wallets that have private keys disabled is deprecated. Use psbtbumpfee instead or restart bitcoind with -deprecatedrpc=bumpfee. This functionality will be removed in 0.22', noprivs0.bumpfee, txid) + bumped_psbt = noprivs1.bumpfee(txid) + assert 'psbt' in bumped_psbt + else: + self.log.info("No tested deprecated RPC methods") if __name__ == '__main__': DeprecatedRpcTest().main() diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index 57c8f511a..6dcbec271 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -224,7 +224,7 @@ class RawTransactionsTest(BitcoinTestFramework): dec_tx = self.nodes[2].decoderawtransaction(rawtx) assert_equal(utx['txid'], dec_tx['vin'][0]['txid']) - assert_raises_rpc_error(-5, "changeAddress must be a valid bitcoin address", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':'foobar'}) + assert_raises_rpc_error(-5, "Change address must be a valid bitcoin address", self.nodes[2].fundrawtransaction, rawtx, {'changeAddress':'foobar'}) def test_valid_change_address(self): self.log.info("Test fundrawtxn with a provided change address") @@ -554,7 +554,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[1].generate(1) self.sync_all() - for i in range(0,20): + for _ in range(20): self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.01) self.nodes[0].generate(1) self.sync_all() @@ -582,7 +582,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[1].generate(1) self.sync_all() - for i in range(0,20): + for _ in range(20): self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.01) self.nodes[0].generate(1) self.sync_all() diff --git a/test/functional/rpc_generate.py b/test/functional/rpc_generate.py new file mode 100755 index 000000000..e55f2e6d1 --- /dev/null +++ b/test/functional/rpc_generate.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test generate RPC.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + + +class RPCGenerateTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + message = ( + "generate\n" + "has been replaced by the -generate " + "cli option. Refer to -help for more information." + ) + + self.log.info("Test rpc generate raises with message to use cli option") + assert_raises_rpc_error(-32601, message, self.nodes[0].rpc.generate) + + self.log.info("Test rpc generate help prints message to use cli option") + assert_equal(message, self.nodes[0].help("generate")) + + self.log.info("Test rpc generate is a hidden command not discoverable in general help") + assert message not in self.nodes[0].help() + + +if __name__ == "__main__": + RPCGenerateTest().main() diff --git a/test/functional/rpc_generateblock.py b/test/functional/rpc_generateblock.py index aa58c0af9..08ff0fba5 100755 --- a/test/functional/rpc_generateblock.py +++ b/test/functional/rpc_generateblock.py @@ -55,7 +55,7 @@ class GenerateBlockTest(BitcoinTestFramework): node.generatetoaddress(110, address) # Generate some extra mempool transactions to verify they don't get mined - for i in range(10): + for _ in range(10): node.sendtoaddress(address, 0.001) self.log.info('Generate block with txid') diff --git a/test/functional/rpc_invalidateblock.py b/test/functional/rpc_invalidateblock.py index 1fdc134f9..e788e7555 100755 --- a/test/functional/rpc_invalidateblock.py +++ b/test/functional/rpc_invalidateblock.py @@ -9,7 +9,6 @@ from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR from test_framework.util import ( assert_equal, connect_nodes, - wait_until, ) @@ -57,9 +56,9 @@ class InvalidateTest(BitcoinTestFramework): self.log.info("..and then mine a block") self.nodes[2].generatetoaddress(1, self.nodes[2].get_deterministic_priv_key().address) self.log.info("Verify all nodes are at the right height") - wait_until(lambda: self.nodes[2].getblockcount() == 3, timeout=5) - wait_until(lambda: self.nodes[0].getblockcount() == 4, timeout=5) - wait_until(lambda: self.nodes[1].getblockcount() == 4, timeout=5) + self.wait_until(lambda: self.nodes[2].getblockcount() == 3, timeout=5) + self.wait_until(lambda: self.nodes[0].getblockcount() == 4, timeout=5) + self.wait_until(lambda: self.nodes[1].getblockcount() == 4, timeout=5) self.log.info("Verify that we reconsider all ancestors as well") blocks = self.nodes[1].generatetodescriptor(10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR) diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index c8517d719..0493ceeb6 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -27,8 +27,8 @@ class RpcMiscTest(BitcoinTestFramework): self.log.info("test CHECK_NONFATAL") assert_raises_rpc_error( -1, - "Internal bug detected: 'request.params.size() != 100'", - lambda: node.echo(*[0] * 100), + 'Internal bug detected: \'request.params[9].get_str() != "trigger_internal_bug"\'', + lambda: node.echo(arg9='trigger_internal_bug'), ) self.log.info("test getmemoryinfo") @@ -61,6 +61,34 @@ class RpcMiscTest(BitcoinTestFramework): node.logging(include=['qt']) assert_equal(node.logging()['qt'], True) + self.log.info("test getindexinfo") + # Without any indices running the RPC returns an empty object + assert_equal(node.getindexinfo(), {}) + + # Restart the node with indices and wait for them to sync + self.restart_node(0, ["-txindex", "-blockfilterindex"]) + self.wait_until(lambda: all(i["synced"] for i in node.getindexinfo().values())) + + # Returns a list of all running indices by default + assert_equal( + node.getindexinfo(), + { + "txindex": {"synced": True, "best_block_height": 200}, + "basic block filter index": {"synced": True, "best_block_height": 200} + } + ) + + # Specifying an index by name returns only the status of that index + assert_equal( + node.getindexinfo("txindex"), + { + "txindex": {"synced": True, "best_block_height": 200}, + } + ) + + # Specifying an unknown index name returns an empty result + assert_equal(node.getindexinfo("foo"), {}) + if __name__ == '__main__': RpcMiscTest().main() diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 3336246c8..bc0e5b458 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -8,24 +8,24 @@ Tests correspond to code in rpc/net.cpp. """ from decimal import Decimal +from itertools import product +import time +from test_framework.p2p import P2PInterface +import test_framework.messages +from test_framework.messages import ( + NODE_NETWORK, + NODE_WITNESS, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( + assert_approx, assert_equal, assert_greater_than_or_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes, p2p_port, - wait_until, -) -from test_framework.mininode import P2PInterface -import test_framework.messages -from test_framework.messages import ( - CAddress, - msg_addr, - NODE_NETWORK, - NODE_WITNESS, ) @@ -50,25 +50,27 @@ class NetTest(BitcoinTestFramework): self.supports_cli = False def run_test(self): - self.log.info('Get out of IBD for the minfeefilter test') - self.nodes[0].generate(1) - self.log.info('Connect nodes both way') + # Get out of IBD for the minfeefilter and getpeerinfo tests. + self.nodes[0].generate(101) + # Connect nodes both ways. connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 0) - self._test_connection_count() - self._test_getnettotals() - self._test_getnetworkinfo() - self._test_getaddednodeinfo() - self._test_getpeerinfo() + self.test_connection_count() + self.test_getpeerinfo() + self.test_getnettotals() + self.test_getnetworkinfo() + self.test_getaddednodeinfo() self.test_service_flags() - self._test_getnodeaddresses() + self.test_getnodeaddresses() - def _test_connection_count(self): - # connect_nodes connects each node to the other + def test_connection_count(self): + self.log.info("Test getconnectioncount") + # After using `connect_nodes` to connect nodes 0 and 1 to each other. assert_equal(self.nodes[0].getconnectioncount(), 2) - def _test_getnettotals(self): + def test_getnettotals(self): + self.log.info("Test getnettotals") # getnettotals totalbytesrecv and totalbytessent should be # consistent with getpeerinfo. Since the RPC calls are not atomic, # and messages might have been recvd or sent between RPC calls, call @@ -90,39 +92,47 @@ class NetTest(BitcoinTestFramework): # the bytes sent/received should change # note ping and pong are 32 bytes each self.nodes[0].ping() - wait_until(lambda: (self.nodes[0].getnettotals()['totalbytessent'] >= net_totals_after['totalbytessent'] + 32 * 2), timeout=1) - wait_until(lambda: (self.nodes[0].getnettotals()['totalbytesrecv'] >= net_totals_after['totalbytesrecv'] + 32 * 2), timeout=1) + self.wait_until(lambda: (self.nodes[0].getnettotals()['totalbytessent'] >= net_totals_after['totalbytessent'] + 32 * 2), timeout=1) + self.wait_until(lambda: (self.nodes[0].getnettotals()['totalbytesrecv'] >= net_totals_after['totalbytesrecv'] + 32 * 2), timeout=1) peer_info_after_ping = self.nodes[0].getpeerinfo() for before, after in zip(peer_info, peer_info_after_ping): assert_greater_than_or_equal(after['bytesrecv_per_msg'].get('pong', 0), before['bytesrecv_per_msg'].get('pong', 0) + 32) assert_greater_than_or_equal(after['bytessent_per_msg'].get('ping', 0), before['bytessent_per_msg'].get('ping', 0) + 32) - def _test_getnetworkinfo(self): - assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) - assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) + def test_getnetworkinfo(self): + self.log.info("Test getnetworkinfo") + info = self.nodes[0].getnetworkinfo() + assert_equal(info['networkactive'], True) + assert_equal(info['connections'], 2) + assert_equal(info['connections_in'], 1) + assert_equal(info['connections_out'], 1) with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: false\n']): self.nodes[0].setnetworkactive(state=False) assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], False) # Wait a bit for all sockets to close - wait_until(lambda: self.nodes[0].getnetworkinfo()['connections'] == 0, timeout=3) + self.wait_until(lambda: self.nodes[0].getnetworkinfo()['connections'] == 0, timeout=3) with self.nodes[0].assert_debug_log(expected_msgs=['SetNetworkActive: true\n']): self.nodes[0].setnetworkactive(state=True) - self.log.info('Connect nodes both way') + # Connect nodes both ways. connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 0) - assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) - assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) + info = self.nodes[0].getnetworkinfo() + assert_equal(info['networkactive'], True) + assert_equal(info['connections'], 2) + assert_equal(info['connections_in'], 1) + assert_equal(info['connections_out'], 1) # check the `servicesnames` field network_info = [node.getnetworkinfo() for node in self.nodes] for info in network_info: assert_net_servicesnames(int(info["localservices"], 0x10), info["localservicesnames"]) - def _test_getaddednodeinfo(self): + def test_getaddednodeinfo(self): + self.log.info("Test getaddednodeinfo") assert_equal(self.nodes[0].getaddednodeinfo(), []) # add a node (node2) to node0 ip_port = "127.0.0.1:{}".format(p2p_port(2)) @@ -131,11 +141,30 @@ class NetTest(BitcoinTestFramework): added_nodes = self.nodes[0].getaddednodeinfo(ip_port) assert_equal(len(added_nodes), 1) assert_equal(added_nodes[0]['addednode'], ip_port) + # check that node cannot be added again + assert_raises_rpc_error(-23, "Node already added", self.nodes[0].addnode, node=ip_port, command='add') + # check that node can be removed + self.nodes[0].addnode(node=ip_port, command='remove') + assert_equal(self.nodes[0].getaddednodeinfo(), []) + # check that trying to remove the node again returns an error + assert_raises_rpc_error(-24, "Node could not be removed", self.nodes[0].addnode, node=ip_port, command='remove') # check that a non-existent node returns an error assert_raises_rpc_error(-24, "Node has not been added", self.nodes[0].getaddednodeinfo, '1.1.1.1') - def _test_getpeerinfo(self): + def test_getpeerinfo(self): + self.log.info("Test getpeerinfo") + # Create a few getpeerinfo last_block/last_transaction values. + if self.is_wallet_compiled(): + self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) + self.nodes[1].generate(1) + self.sync_all() + time_now = int(time.time()) peer_info = [x.getpeerinfo() for x in self.nodes] + # Verify last_block and last_transaction keys/values. + for node, peer, field in product(range(self.num_nodes), range(2), ['last_block', 'last_transaction']): + assert field in peer_info[node][peer].keys() + if peer_info[node][peer][field] != 0: + assert_approx(peer_info[node][peer][field], time_now, vspan=60) # check both sides of bidirectional connection between nodes # the address bound to on one side will be the source address for the other node assert_equal(peer_info[0][0]['addrbind'], peer_info[1][0]['addr']) @@ -147,37 +176,43 @@ class NetTest(BitcoinTestFramework): assert_net_servicesnames(int(info[0]["services"], 0x10), info[0]["servicesnames"]) def test_service_flags(self): + self.log.info("Test service flags") self.nodes[0].add_p2p_connection(P2PInterface(), services=(1 << 4) | (1 << 63)) assert_equal(['UNKNOWN[2^4]', 'UNKNOWN[2^63]'], self.nodes[0].getpeerinfo()[-1]['servicesnames']) self.nodes[0].disconnect_p2ps() - def _test_getnodeaddresses(self): + def test_getnodeaddresses(self): + self.log.info("Test getnodeaddresses") self.nodes[0].add_p2p_connection(P2PInterface()) - # send some addresses to the node via the p2p message addr - msg = msg_addr() + # Add some addresses to the Address Manager over RPC. Due to the way + # bucket and bucket position are calculated, some of these addresses + # will collide. imported_addrs = [] - for i in range(256): - a = "123.123.123.{}".format(i) + for i in range(10000): + first_octet = i >> 8 + second_octet = i % 256 + a = "{}.{}.1.1".format(first_octet, second_octet) imported_addrs.append(a) - addr = CAddress() - addr.time = 100000000 - addr.nServices = NODE_NETWORK | NODE_WITNESS - addr.ip = a - addr.port = 8333 - msg.addrs.append(addr) - self.nodes[0].p2p.send_and_ping(msg) - - # obtain addresses via rpc call and check they were ones sent in before - REQUEST_COUNT = 10 - node_addresses = self.nodes[0].getnodeaddresses(REQUEST_COUNT) - assert_equal(len(node_addresses), REQUEST_COUNT) + self.nodes[0].addpeeraddress(a, 8333) + + # Obtain addresses via rpc call and check they were ones sent in before. + # + # Maximum possible addresses in addrman is 10000, although actual + # number will usually be less due to bucket and bucket position + # collisions. + node_addresses = self.nodes[0].getnodeaddresses(0) + assert_greater_than(len(node_addresses), 5000) + assert_greater_than(10000, len(node_addresses)) for a in node_addresses: - assert_greater_than(a["time"], 1527811200) # 1st June 2018 + assert_greater_than(a["time"], 1527811200) # 1st June 2018 assert_equal(a["services"], NODE_NETWORK | NODE_WITNESS) assert a["address"] in imported_addrs assert_equal(a["port"], 8333) + node_addresses = self.nodes[0].getnodeaddresses(1) + assert_equal(len(node_addresses), 1) + assert_raises_rpc_error(-8, "Address count out of range", self.nodes[0].getnodeaddresses, -1) # addrman's size cannot be known reliably after insertion, as hash collisions may occur diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 4d985dd1b..781a49dfa 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -94,6 +94,9 @@ class PSBTTest(BitcoinTestFramework): psbtx1 = self.nodes[0].walletcreatefundedpsbt([{"txid": utxo1['txid'], "vout": utxo1['vout']}], {self.nodes[2].getnewaddress():90}, 0, {"add_inputs": True})['psbt'] assert_equal(len(self.nodes[0].decodepsbt(psbtx1)['tx']['vin']), 2) + # Inputs argument can be null + self.nodes[0].walletcreatefundedpsbt(None, {self.nodes[2].getnewaddress():10}) + # Node 1 should not be able to add anything to it but still return the psbtx same as before psbtx = self.nodes[1].walletprocesspsbt(psbtx1)['psbt'] assert_equal(psbtx1, psbtx) @@ -103,7 +106,16 @@ class PSBTTest(BitcoinTestFramework): final_tx = self.nodes[0].finalizepsbt(signed_tx)['hex'] self.nodes[0].sendrawtransaction(final_tx) - # Get pubkeys + # Manually selected inputs can be locked: + assert_equal(len(self.nodes[0].listlockunspent()), 0) + utxo1 = self.nodes[0].listunspent()[0] + psbtx1 = self.nodes[0].walletcreatefundedpsbt([{"txid": utxo1['txid'], "vout": utxo1['vout']}], {self.nodes[2].getnewaddress():1}, 0,{"lockUnspents": True})["psbt"] + assert_equal(len(self.nodes[0].listlockunspent()), 1) + + # Locks are ignored for manually selected inputs + self.nodes[0].walletcreatefundedpsbt([{"txid": utxo1['txid'], "vout": utxo1['vout']}], {self.nodes[2].getnewaddress():1}, 0) + + # Create p2sh, p2wpkh, and p2wsh addresses pubkey0 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress())['pubkey'] pubkey1 = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress())['pubkey'] pubkey2 = self.nodes[2].getaddressinfo(self.nodes[2].getnewaddress())['pubkey'] @@ -430,7 +442,7 @@ class PSBTTest(BitcoinTestFramework): # Check that joining shuffles the inputs and outputs # 10 attempts should be enough to get a shuffled join shuffled = False - for i in range(0, 10): + for _ in range(10): shuffled_joined = self.nodes[0].joinpsbts([psbt, psbt2]) shuffled |= joined != shuffled_joined if shuffled: diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index 3d0820272..704b65c06 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -198,10 +198,30 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert_equal(spending_tx_signed['complete'], True) self.nodes[0].sendrawtransaction(spending_tx_signed['hex']) + def OP_1NEGATE_test(self): + self.log.info("Test OP_1NEGATE (0x4f) satisfies BIP62 minimal push standardness rule") + hex_str = ( + "0200000001FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" + "FFFFFFFF00000000044F024F9CFDFFFFFF01F0B9F5050000000023210277777777" + "77777777777777777777777777777777777777777777777777777777AC66030000" + ) + prev_txs = [ + { + "txid": "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + "vout": 0, + "scriptPubKey": "A914AE44AB6E9AA0B71F1CD2B453B69340E9BFBAEF6087", + "redeemScript": "4F9C", + "amount": 1, + } + ] + txn = self.nodes[0].signrawtransactionwithwallet(hex_str, prev_txs) + assert txn["complete"] + def run_test(self): self.successful_signing_test() self.script_verification_error_test() self.witness_script_test() + self.OP_1NEGATE_test() self.test_with_lock_outputs() diff --git a/test/functional/rpc_txoutproof.py b/test/functional/rpc_txoutproof.py index ca8be42d3..93fb62c5d 100755 --- a/test/functional/rpc_txoutproof.py +++ b/test/functional/rpc_txoutproof.py @@ -6,41 +6,31 @@ from test_framework.messages import CMerkleBlock, FromHex, ToHex from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes +from test_framework.util import assert_equal, assert_raises_rpc_error +from test_framework.wallet import MiniWallet + class MerkleBlockTest(BitcoinTestFramework): def set_test_params(self): - self.num_nodes = 4 + self.num_nodes = 2 self.setup_clean_chain = True - # Nodes 0/1 are "wallet" nodes, Nodes 2/3 are used for testing - self.extra_args = [[], [], [], ["-txindex"]] - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - - def setup_network(self): - self.setup_nodes() - connect_nodes(self.nodes[0], 1) - connect_nodes(self.nodes[0], 2) - connect_nodes(self.nodes[0], 3) - - self.sync_all() + self.extra_args = [ + [], + ["-txindex"], + ] def run_test(self): - self.log.info("Mining blocks...") - self.nodes[0].generate(105) + miniwallet = MiniWallet(self.nodes[0]) + # Add enough mature utxos to the wallet, so that all txs spend confirmed coins + miniwallet.generate(5) + self.nodes[0].generate(100) self.sync_all() chain_height = self.nodes[1].getblockcount() assert_equal(chain_height, 105) - assert_equal(self.nodes[1].getbalance(), 0) - assert_equal(self.nodes[2].getbalance(), 0) - - node0utxos = self.nodes[0].listunspent(1) - tx1 = self.nodes[0].createrawtransaction([node0utxos.pop()], {self.nodes[1].getnewaddress(): 49.99}) - txid1 = self.nodes[0].sendrawtransaction(self.nodes[0].signrawtransactionwithwallet(tx1)["hex"]) - tx2 = self.nodes[0].createrawtransaction([node0utxos.pop()], {self.nodes[1].getnewaddress(): 49.99}) - txid2 = self.nodes[0].sendrawtransaction(self.nodes[0].signrawtransactionwithwallet(tx2)["hex"]) + + txid1 = miniwallet.send_self_transfer(from_node=self.nodes[0])['txid'] + txid2 = miniwallet.send_self_transfer(from_node=self.nodes[0])['txid'] # This will raise an exception because the transaction is not yet in a block assert_raises_rpc_error(-5, "Transaction not yet in block", self.nodes[0].gettxoutproof, [txid1]) @@ -53,50 +43,54 @@ class MerkleBlockTest(BitcoinTestFramework): txlist.append(blocktxn[1]) txlist.append(blocktxn[2]) - assert_equal(self.nodes[2].verifytxoutproof(self.nodes[2].gettxoutproof([txid1])), [txid1]) - assert_equal(self.nodes[2].verifytxoutproof(self.nodes[2].gettxoutproof([txid1, txid2])), txlist) - assert_equal(self.nodes[2].verifytxoutproof(self.nodes[2].gettxoutproof([txid1, txid2], blockhash)), txlist) + assert_equal(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid1])), [txid1]) + assert_equal(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid1, txid2])), txlist) + assert_equal(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid1, txid2], blockhash)), txlist) - txin_spent = self.nodes[1].listunspent(1).pop() - tx3 = self.nodes[1].createrawtransaction([txin_spent], {self.nodes[0].getnewaddress(): 49.98}) - txid3 = self.nodes[0].sendrawtransaction(self.nodes[1].signrawtransactionwithwallet(tx3)["hex"]) + txin_spent = miniwallet.get_utxo() # Get the change from txid2 + tx3 = miniwallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=txin_spent) + txid3 = tx3['txid'] self.nodes[0].generate(1) self.sync_all() txid_spent = txin_spent["txid"] - txid_unspent = txid1 if txin_spent["txid"] != txid1 else txid2 + txid_unspent = txid1 # Input was change from txid2, so txid1 should be unspent # Invalid txids - assert_raises_rpc_error(-8, "txid must be of length 64 (not 32, for '00000000000000000000000000000000')", self.nodes[2].gettxoutproof, ["00000000000000000000000000000000"], blockhash) - assert_raises_rpc_error(-8, "txid must be hexadecimal string (not 'ZZZ0000000000000000000000000000000000000000000000000000000000000')", self.nodes[2].gettxoutproof, ["ZZZ0000000000000000000000000000000000000000000000000000000000000"], blockhash) + assert_raises_rpc_error(-8, "txid must be of length 64 (not 32, for '00000000000000000000000000000000')", self.nodes[0].gettxoutproof, ["00000000000000000000000000000000"], blockhash) + assert_raises_rpc_error(-8, "txid must be hexadecimal string (not 'ZZZ0000000000000000000000000000000000000000000000000000000000000')", self.nodes[0].gettxoutproof, ["ZZZ0000000000000000000000000000000000000000000000000000000000000"], blockhash) # Invalid blockhashes - assert_raises_rpc_error(-8, "blockhash must be of length 64 (not 32, for '00000000000000000000000000000000')", self.nodes[2].gettxoutproof, [txid_spent], "00000000000000000000000000000000") - assert_raises_rpc_error(-8, "blockhash must be hexadecimal string (not 'ZZZ0000000000000000000000000000000000000000000000000000000000000')", self.nodes[2].gettxoutproof, [txid_spent], "ZZZ0000000000000000000000000000000000000000000000000000000000000") + assert_raises_rpc_error(-8, "blockhash must be of length 64 (not 32, for '00000000000000000000000000000000')", self.nodes[0].gettxoutproof, [txid_spent], "00000000000000000000000000000000") + assert_raises_rpc_error(-8, "blockhash must be hexadecimal string (not 'ZZZ0000000000000000000000000000000000000000000000000000000000000')", self.nodes[0].gettxoutproof, [txid_spent], "ZZZ0000000000000000000000000000000000000000000000000000000000000") # We can't find the block from a fully-spent tx - assert_raises_rpc_error(-5, "Transaction not yet in block", self.nodes[2].gettxoutproof, [txid_spent]) + assert_raises_rpc_error(-5, "Transaction not yet in block", self.nodes[0].gettxoutproof, [txid_spent]) # We can get the proof if we specify the block - assert_equal(self.nodes[2].verifytxoutproof(self.nodes[2].gettxoutproof([txid_spent], blockhash)), [txid_spent]) + assert_equal(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid_spent], blockhash)), [txid_spent]) # We can't get the proof if we specify a non-existent block - assert_raises_rpc_error(-5, "Block not found", self.nodes[2].gettxoutproof, [txid_spent], "0000000000000000000000000000000000000000000000000000000000000000") + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].gettxoutproof, [txid_spent], "0000000000000000000000000000000000000000000000000000000000000000") # We can get the proof if the transaction is unspent - assert_equal(self.nodes[2].verifytxoutproof(self.nodes[2].gettxoutproof([txid_unspent])), [txid_unspent]) + assert_equal(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid_unspent])), [txid_unspent]) # We can get the proof if we provide a list of transactions and one of them is unspent. The ordering of the list should not matter. - assert_equal(sorted(self.nodes[2].verifytxoutproof(self.nodes[2].gettxoutproof([txid1, txid2]))), sorted(txlist)) - assert_equal(sorted(self.nodes[2].verifytxoutproof(self.nodes[2].gettxoutproof([txid2, txid1]))), sorted(txlist)) + assert_equal(sorted(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid1, txid2]))), sorted(txlist)) + assert_equal(sorted(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid2, txid1]))), sorted(txlist)) # We can always get a proof if we have a -txindex - assert_equal(self.nodes[2].verifytxoutproof(self.nodes[3].gettxoutproof([txid_spent])), [txid_spent]) + assert_equal(self.nodes[0].verifytxoutproof(self.nodes[1].gettxoutproof([txid_spent])), [txid_spent]) # We can't get a proof if we specify transactions from different blocks - assert_raises_rpc_error(-5, "Not all transactions found in specified or retrieved block", self.nodes[2].gettxoutproof, [txid1, txid3]) + assert_raises_rpc_error(-5, "Not all transactions found in specified or retrieved block", self.nodes[0].gettxoutproof, [txid1, txid3]) + # Test empty list + assert_raises_rpc_error(-5, "Transaction not yet in block", self.nodes[0].gettxoutproof, []) + # Test duplicate txid + assert_raises_rpc_error(-8, 'Invalid parameter, duplicated txid', self.nodes[0].gettxoutproof, [txid1, txid1]) # Now we'll try tweaking a proof. - proof = self.nodes[3].gettxoutproof([txid1, txid2]) + proof = self.nodes[1].gettxoutproof([txid1, txid2]) assert txid1 in self.nodes[0].verifytxoutproof(proof) assert txid2 in self.nodes[1].verifytxoutproof(proof) tweaked_proof = FromHex(CMerkleBlock(), proof) # Make sure that our serialization/deserialization is working - assert txid1 in self.nodes[2].verifytxoutproof(ToHex(tweaked_proof)) + assert txid1 in self.nodes[0].verifytxoutproof(ToHex(tweaked_proof)) # Check to see if we can go up the merkle tree and pass this off as a # single-transaction block diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index 912c0ca97..adbffb7dc 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -8,22 +8,7 @@ keys, and is trivially vulnerable to side channel attacks. Do not use for anything but tests.""" import random -def modinv(a, n): - """Compute the modular inverse of a modulo n - - See https://en.wikipedia.org/wiki/Extended_Euclidean_algorithm#Modular_integers. - """ - t1, t2 = 0, 1 - r1, r2 = n, a - while r2 != 0: - q = r1 // r2 - t1, t2 = t2, t1 - q * t2 - r1, r2 = r2, r1 - q * r2 - if r1 > 1: - return None - if t1 < 0: - t1 += n - return t1 +from .util import modinv def jacobi_symbol(n, k): """Compute the Jacobi symbol of n modulo k diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 2462a9a6d..1e062ab9a 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -22,6 +22,7 @@ from codecs import encode import copy import hashlib from io import BytesIO +import math import random import socket import struct @@ -32,7 +33,7 @@ from test_framework.util import hex_str_to_bytes, assert_equal MIN_VERSION_SUPPORTED = 60001 MY_VERSION = 70016 # past wtxid relay -MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" +MY_SUBVERSION = b"/python-p2p-tester:0.0.3/" MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) MAX_LOCATOR_SZ = 101 @@ -53,6 +54,7 @@ NODE_NETWORK = (1 << 0) NODE_GETUTXO = (1 << 1) NODE_BLOOM = (1 << 2) NODE_WITNESS = (1 << 3) +NODE_COMPACT_FILTERS = (1 << 6) NODE_NETWORK_LIMITED = (1 << 10) MSG_TX = 1 @@ -62,9 +64,12 @@ MSG_CMPCT_BLOCK = 4 MSG_WTX = 5 MSG_WITNESS_FLAG = 1 << 30 MSG_TYPE_MASK = 0xffffffff >> 2 +MSG_WITNESS_TX = MSG_TX | MSG_WITNESS_FLAG FILTER_TYPE_BASIC = 0 +WITNESS_SCALE_FACTOR = 4 + # Serialization/deserialization tools def sha256(s): return hashlib.new('sha256', s).digest() @@ -111,7 +116,7 @@ def deser_uint256(f): def ser_uint256(u): rs = b"" - for i in range(8): + for _ in range(8): rs += struct.pack("<I", u & 0xFFFFFFFF) u >>= 32 return rs @@ -134,7 +139,7 @@ def uint256_from_compact(c): def deser_vector(f, c): nit = deser_compact_size(f) r = [] - for i in range(nit): + for _ in range(nit): t = c() t.deserialize(f) r.append(t) @@ -157,7 +162,7 @@ def ser_vector(l, ser_function_name=None): def deser_uint256_vector(f): nit = deser_compact_size(f) r = [] - for i in range(nit): + for _ in range(nit): t = deser_uint256(f) r.append(t) return r @@ -173,7 +178,7 @@ def ser_uint256_vector(l): def deser_string_vector(f): nit = deser_compact_size(f) r = [] - for i in range(nit): + for _ in range(nit): t = deser_string(f) r.append(t) return r @@ -243,8 +248,8 @@ class CInv: MSG_TX | MSG_WITNESS_FLAG: "WitnessTx", MSG_BLOCK | MSG_WITNESS_FLAG: "WitnessBlock", MSG_FILTERED_BLOCK: "filtered Block", - 4: "CompactBlock", - 5: "WTX", + MSG_CMPCT_BLOCK: "CompactBlock", + MSG_WTX: "WTX", } def __init__(self, t=0, h=0): @@ -252,12 +257,12 @@ class CInv: self.hash = h def deserialize(self, f): - self.type = struct.unpack("<i", f.read(4))[0] + self.type = struct.unpack("<I", f.read(4))[0] self.hash = deser_uint256(f) def serialize(self): r = b"" - r += struct.pack("<i", self.type) + r += struct.pack("<I", self.type) r += ser_uint256(self.hash) return r @@ -467,7 +472,7 @@ class CTransaction: else: self.vout = deser_vector(f, CTxOut) if flags != 0: - self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))] + self.wit.vtxinwit = [CTxInWitness() for _ in range(len(self.vin))] self.wit.deserialize(f) else: self.wit = CTxWitness() @@ -500,7 +505,7 @@ class CTransaction: if (len(self.wit.vtxinwit) != len(self.vin)): # vtxinwit must have the same length as vin self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)] - for i in range(len(self.wit.vtxinwit), len(self.vin)): + for _ in range(len(self.wit.vtxinwit), len(self.vin)): self.wit.vtxinwit.append(CTxInWitness()) r += self.wit.serialize() r += struct.pack("<I", self.nLockTime) @@ -535,6 +540,13 @@ class CTransaction: return False return True + # Calculate the virtual transaction size using witness and non-witness + # serialization size (does NOT use sigops). + def get_vsize(self): + with_witness_size = len(self.serialize_with_witness()) + without_witness_size = len(self.serialize_without_witness()) + return math.ceil(((WITNESS_SCALE_FACTOR - 1) * without_witness_size + with_witness_size) / WITNESS_SCALE_FACTOR) + def __repr__(self): return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime) @@ -735,7 +747,7 @@ class P2PHeaderAndShortIDs: self.header.deserialize(f) self.nonce = struct.unpack("<Q", f.read(8))[0] self.shortids_length = deser_compact_size(f) - for i in range(self.shortids_length): + for _ in range(self.shortids_length): # shortids are defined to be 6 bytes in the spec, so append # two zero bytes and read it in as an 8-byte number self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0]) @@ -852,7 +864,7 @@ class BlockTransactionsRequest: def deserialize(self, f): self.blockhash = deser_uint256(f) indexes_length = deser_compact_size(f) - for i in range(indexes_length): + for _ in range(indexes_length): self.indexes.append(deser_compact_size(f)) def serialize(self): diff --git a/test/functional/test_framework/muhash.py b/test/functional/test_framework/muhash.py new file mode 100644 index 000000000..97d02359c --- /dev/null +++ b/test/functional/test_framework/muhash.py @@ -0,0 +1,110 @@ +# Copyright (c) 2020 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Native Python MuHash3072 implementation.""" + +import hashlib +import unittest + +from .util import modinv + +def rot32(v, bits): + """Rotate the 32-bit value v left by bits bits.""" + bits %= 32 # Make sure the term below does not throw an exception + return ((v << bits) & 0xffffffff) | (v >> (32 - bits)) + +def chacha20_doubleround(s): + """Apply a ChaCha20 double round to 16-element state array s. + + See https://cr.yp.to/chacha/chacha-20080128.pdf and https://tools.ietf.org/html/rfc8439 + """ + QUARTER_ROUNDS = [(0, 4, 8, 12), + (1, 5, 9, 13), + (2, 6, 10, 14), + (3, 7, 11, 15), + (0, 5, 10, 15), + (1, 6, 11, 12), + (2, 7, 8, 13), + (3, 4, 9, 14)] + + for a, b, c, d in QUARTER_ROUNDS: + s[a] = (s[a] + s[b]) & 0xffffffff + s[d] = rot32(s[d] ^ s[a], 16) + s[c] = (s[c] + s[d]) & 0xffffffff + s[b] = rot32(s[b] ^ s[c], 12) + s[a] = (s[a] + s[b]) & 0xffffffff + s[d] = rot32(s[d] ^ s[a], 8) + s[c] = (s[c] + s[d]) & 0xffffffff + s[b] = rot32(s[b] ^ s[c], 7) + +def chacha20_32_to_384(key32): + """Specialized ChaCha20 implementation with 32-byte key, 0 IV, 384-byte output.""" + # See RFC 8439 section 2.3 for chacha20 parameters + CONSTANTS = [0x61707865, 0x3320646e, 0x79622d32, 0x6b206574] + + key_bytes = [0]*8 + for i in range(8): + key_bytes[i] = int.from_bytes(key32[(4 * i):(4 * (i+1))], 'little') + + INITIALIZATION_VECTOR = [0] * 4 + init = CONSTANTS + key_bytes + INITIALIZATION_VECTOR + out = bytearray() + for counter in range(6): + init[12] = counter + s = init.copy() + for _ in range(10): + chacha20_doubleround(s) + for i in range(16): + out.extend(((s[i] + init[i]) & 0xffffffff).to_bytes(4, 'little')) + return bytes(out) + +def data_to_num3072(data): + """Hash a 32-byte array data to a 3072-bit number using 6 Chacha20 operations.""" + bytes384 = chacha20_32_to_384(data) + return int.from_bytes(bytes384, 'little') + +class MuHash3072: + """Class representing the MuHash3072 computation of a set. + + See https://cseweb.ucsd.edu/~mihir/papers/inchash.pdf and https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2017-May/014337.html + """ + + MODULUS = 2**3072 - 1103717 + + def __init__(self): + """Initialize for an empty set.""" + self.numerator = 1 + self.denominator = 1 + + def insert(self, data): + """Insert a byte array data in the set.""" + self.numerator = (self.numerator * data_to_num3072(data)) % self.MODULUS + + def remove(self, data): + """Remove a byte array from the set.""" + self.denominator = (self.denominator * data_to_num3072(data)) % self.MODULUS + + def digest(self): + """Extract the final hash. Does not modify this object.""" + val = (self.numerator * modinv(self.denominator, self.MODULUS)) % self.MODULUS + bytes384 = val.to_bytes(384, 'little') + return hashlib.sha256(bytes384).digest() + +class TestFrameworkMuhash(unittest.TestCase): + def test_muhash(self): + muhash = MuHash3072() + muhash.insert([0]*32) + muhash.insert([1] + [0]*31) + muhash.remove([2] + [0]*31) + finalized = muhash.digest() + # This mirrors the result in the C++ MuHash3072 unit test + self.assertEqual(finalized[::-1].hex(), "a44e16d5e34d259b349af21c06e65d653915d2e208e4e03f389af750dc0bfdc3") + + def test_chacha20(self): + def chacha_check(key, result): + self.assertEqual(chacha20_32_to_384(key)[:64].hex(), result) + + # Test vectors from https://tools.ietf.org/html/draft-agl-tls-chacha20poly1305-04#section-7 + # Since the nonce is hardcoded to 0 in our function we only use those vectors. + chacha_check([0]*32, "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586") + chacha_check([0]*31 + [1], "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41bbe2a0b6ea7566d2a5d1e7e20d42af2c53d792b1c43fea817e9ad275ae546963") diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/p2p.py index 07811667a..963a507f5 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/p2p.py @@ -4,10 +4,14 @@ # Copyright (c) 2010-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Bitcoin P2P network half-a-node. +"""Test objects for interacting with a bitcoind node over the p2p protocol. -This python code was modified from ArtForz' public domain half-a-node, as -found in the mini-node branch of http://github.com/jgarzik/pynode. +The P2PInterface objects interact with the bitcoind nodes under test using the +node's p2p interface. They can be used to send messages to the node, and +callbacks can be registered that execute when messages are received from the +node. Messages are sent to/received from the node on an asyncio event loop. +State held inside the objects must be guarded by the p2p_lock to avoid data +races between the main testing thread and the event loop. P2PConnection: A low-level connection object to a node's P2P interface P2PInterface: A high-level interface object for communicating to a node over P2P @@ -65,9 +69,9 @@ from test_framework.messages import ( NODE_WITNESS, sha256, ) -from test_framework.util import wait_until +from test_framework.util import wait_until_helper -logger = logging.getLogger("TestFramework.mininode") +logger = logging.getLogger("TestFramework.p2p") MESSAGEMAP = { b"addr": msg_addr, @@ -289,7 +293,7 @@ class P2PInterface(P2PConnection): # Track the most recent message of each type. # To wait for a message to be received, pop that message from - # this and use wait_until. + # this and use self.wait_until. self.last_message = {} # A count of the number of ping messages we've sent to the node @@ -320,7 +324,7 @@ class P2PInterface(P2PConnection): We keep a count of how many of each message type has been received and the most recent message of each type.""" - with mininode_lock: + with p2p_lock: try: msgtype = message.msgtype.decode('ascii') self.message_count[msgtype] += 1 @@ -388,18 +392,22 @@ class P2PInterface(P2PConnection): # Connection helper methods - def wait_until(self, test_function, timeout=60): - wait_until(test_function, timeout=timeout, lock=mininode_lock, timeout_factor=self.timeout_factor) + def wait_until(self, test_function_in, *, timeout=60, check_connected=True): + def test_function(): + if check_connected: + assert self.is_connected + return test_function_in() + + wait_until_helper(test_function, timeout=timeout, lock=p2p_lock, timeout_factor=self.timeout_factor) def wait_for_disconnect(self, timeout=60): test_function = lambda: not self.is_connected - self.wait_until(test_function, timeout=timeout) + self.wait_until(test_function, timeout=timeout, check_connected=False) # Message receiving helper methods def wait_for_tx(self, txid, timeout=60): def test_function(): - assert self.is_connected if not self.last_message.get('tx'): return False return self.last_message['tx'].tx.rehash() == txid @@ -408,14 +416,12 @@ class P2PInterface(P2PConnection): def wait_for_block(self, blockhash, timeout=60): def test_function(): - assert self.is_connected return self.last_message.get("block") and self.last_message["block"].block.rehash() == blockhash self.wait_until(test_function, timeout=timeout) def wait_for_header(self, blockhash, timeout=60): def test_function(): - assert self.is_connected last_headers = self.last_message.get('headers') if not last_headers: return False @@ -425,7 +431,6 @@ class P2PInterface(P2PConnection): def wait_for_merkleblock(self, blockhash, timeout=60): def test_function(): - assert self.is_connected last_filtered_block = self.last_message.get('merkleblock') if not last_filtered_block: return False @@ -437,9 +442,7 @@ class P2PInterface(P2PConnection): """Waits for a getdata message. The object hashes in the inventory vector must match the provided hash_list.""" - def test_function(): - assert self.is_connected last_data = self.last_message.get("getdata") if not last_data: return False @@ -454,9 +457,7 @@ class P2PInterface(P2PConnection): value must be explicitly cleared before calling this method, or this will return immediately with success. TODO: change this method to take a hash value and only return true if the correct block header has been requested.""" - def test_function(): - assert self.is_connected return self.last_message.get("getheaders") self.wait_until(test_function, timeout=timeout) @@ -467,7 +468,6 @@ class P2PInterface(P2PConnection): raise NotImplementedError("wait_for_inv() will only verify the first inv object") def test_function(): - assert self.is_connected return self.last_message.get("inv") and \ self.last_message["inv"].inv[0].type == expected_inv[0].type and \ self.last_message["inv"].inv[0].hash == expected_inv[0].hash @@ -491,7 +491,6 @@ class P2PInterface(P2PConnection): self.send_message(msg_ping(nonce=self.ping_counter)) def test_function(): - assert self.is_connected return self.last_message.get("pong") and self.last_message["pong"].nonce == self.ping_counter self.wait_until(test_function, timeout=timeout) @@ -503,7 +502,7 @@ class P2PInterface(P2PConnection): # P2PConnection acquires this lock whenever delivering a message to a P2PInterface. # This lock should be acquired in the thread running the test logic to synchronize # access to any data shared with the P2PInterface or P2PConnection. -mininode_lock = threading.Lock() +p2p_lock = threading.Lock() class NetworkThread(threading.Thread): @@ -523,7 +522,7 @@ class NetworkThread(threading.Thread): def close(self, timeout=10): """Close the connections and network event loop.""" self.network_event_loop.call_soon_threadsafe(self.network_event_loop.stop) - wait_until(lambda: not self.network_event_loop.is_running(), timeout=timeout) + wait_until_helper(lambda: not self.network_event_loop.is_running(), timeout=timeout) self.network_event_loop.close() self.join(timeout) # Safe to remove event loop. @@ -597,7 +596,7 @@ class P2PDataStore(P2PInterface): - if success is False: assert that the node's tip doesn't advance - if reject_reason is set: assert that the correct reject message is logged""" - with mininode_lock: + with p2p_lock: for block in blocks: self.block_store[block.sha256] = block self.last_block_hash = block.sha256 @@ -609,7 +608,11 @@ class P2PDataStore(P2PInterface): self.send_message(msg_block(block=b)) else: self.send_message(msg_headers([CBlockHeader(block) for block in blocks])) - self.wait_until(lambda: blocks[-1].sha256 in self.getdata_requests, timeout=timeout) + self.wait_until( + lambda: blocks[-1].sha256 in self.getdata_requests, + timeout=timeout, + check_connected=success, + ) if expect_disconnect: self.wait_for_disconnect(timeout=timeout) @@ -630,7 +633,7 @@ class P2PDataStore(P2PInterface): - if expect_disconnect is True: Skip the sync with ping - if reject_reason is set: assert that the correct reject message is logged.""" - with mininode_lock: + with p2p_lock: for tx in txs: self.tx_store[tx.sha256] = tx @@ -669,7 +672,7 @@ class P2PTxInvStore(P2PInterface): self.tx_invs_received[i.hash] += 1 def get_invs(self): - with mininode_lock: + with p2p_lock: return list(self.tx_invs_received.keys()) def wait_for_broadcast(self, txns, timeout=60): @@ -677,6 +680,6 @@ class P2PTxInvStore(P2PInterface): The mempool should mark unbroadcast=False for these transactions. """ # Wait until invs have been received (and getdatas sent) for each txid. - self.wait_until(lambda: set(self.tx_invs_received.keys()) == set([int(tx, 16) for tx in txns]), timeout) + self.wait_until(lambda: set(self.tx_invs_received.keys()) == set([int(tx, 16) for tx in txns]), timeout=timeout) # Flush messages and wait for the getdatas to be processed self.sync_with_ping() diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index cc5f8307d..5e35ba0fc 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -646,7 +646,7 @@ def LegacySignatureHash(script, txTo, inIdx, hashtype): tmp = txtmp.vout[outIdx] txtmp.vout = [] - for i in range(outIdx): + for _ in range(outIdx): txtmp.vout.append(CTxOut(-1)) txtmp.vout.append(tmp) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 8d402d488..f41f5129b 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -20,8 +20,8 @@ import time from .authproxy import JSONRPCException from . import coverage +from .p2p import NetworkThread from .test_node import TestNode -from .mininode import NetworkThread from .util import ( MAX_NODES, PortSeed, @@ -31,7 +31,7 @@ from .util import ( disconnect_nodes, get_datadir_path, initialize_datadir, - wait_until, + wait_until_helper, ) @@ -603,8 +603,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.sync_blocks(nodes) self.sync_mempools(nodes) - def wait_until(self, test_function, timeout=60, lock=None): - return wait_until(test_function, timeout=timeout, lock=lock, timeout_factor=self.options.timeout_factor) + def wait_until(self, test_function, timeout=60): + return wait_until_helper(test_function, timeout=timeout, timeout_factor=self.options.timeout_factor) # Private helper methods. These should not be accessed by the subclass test scripts. diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 66bb2c89b..d03498682 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -31,7 +31,7 @@ from .util import ( get_auth_cookie, get_rpc_proxy, rpc_url, - wait_until, + wait_until_helper, p2p_port, EncodeDecimal, ) @@ -231,7 +231,7 @@ class TestNode(): if self.version_is_at_least(190000): # getmempoolinfo.loaded is available since commit # bb8ae2c (version 0.19.0) - wait_until(lambda: rpc.getmempoolinfo()['loaded']) + wait_until_helper(lambda: rpc.getmempoolinfo()['loaded'], timeout_factor=self.timeout_factor) # Wait for the node to finish reindex, block import, and # loading the mempool. Usually importing happens fast or # even "immediate" when the node is started. However, there @@ -359,7 +359,7 @@ class TestNode(): return True def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT): - wait_until(self.is_node_stopped, timeout=timeout, timeout_factor=self.timeout_factor) + wait_until_helper(self.is_node_stopped, timeout=timeout, timeout_factor=self.timeout_factor) @contextlib.contextmanager def assert_debug_log(self, expected_msgs, unexpected_msgs=None, timeout=2): @@ -524,6 +524,7 @@ class TestNode(): p2p_conn.peer_connect(**kwargs, net=self.chain, timeout_factor=self.timeout_factor)() self.p2ps.append(p2p_conn) + p2p_conn.wait_until(lambda: p2p_conn.is_connected, check_connected=False) if wait_for_verack: # Wait for the node to send us the version and verack p2p_conn.wait_for_verack() @@ -550,7 +551,7 @@ class TestNode(): assert self.p2ps, self._node_msg("No p2p connection") return self.p2ps[0] - def num_connected_mininodes(self): + def num_test_p2p_connections(self): """Return number of test framework p2p connections to the node.""" return len([peer for peer in self.getpeerinfo() if peer['subver'] == MY_SUBVERSION]) @@ -559,7 +560,7 @@ class TestNode(): for p in self.p2ps: p.peer_disconnect() del self.p2ps[:] - wait_until(lambda: self.num_connected_mininodes() == 0) + wait_until_helper(lambda: self.num_test_p2p_connections() == 0, timeout_factor=self.timeout_factor) class TestNodeCLIAttr: @@ -637,7 +638,7 @@ class TestNodeCLI(): raise subprocess.CalledProcessError(returncode, self.binary, output=cli_stderr) try: return json.loads(cli_stdout, parse_float=decimal.Decimal) - except json.JSONDecodeError: + except (json.JSONDecodeError, decimal.InvalidOperation): return cli_stdout.rstrip("\n") class RPCOverloadWrapper(): @@ -649,10 +650,10 @@ class RPCOverloadWrapper(): def __getattr__(self, name): return getattr(self.rpc, name) - def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase='', avoid_reuse=None, descriptors=None): + def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase='', avoid_reuse=None, descriptors=None, load_on_startup=None): if descriptors is None: descriptors = self.descriptors - return self.__getattr__('createwallet')(wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors) + return self.__getattr__('createwallet')(wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors, load_on_startup) def importprivkey(self, privkey, label=None, rescan=None): wallet_info = self.getwalletinfo() diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 506057f1f..af7f0b62f 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -15,6 +15,7 @@ import os import random import re import time +import unittest from . import coverage from .authproxy import AuthServiceProxy, JSONRPCException @@ -225,7 +226,15 @@ def satoshi_round(amount): return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) -def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None, timeout_factor=1.0): +def wait_until_helper(predicate, *, attempts=float('inf'), timeout=float('inf'), lock=None, timeout_factor=1.0): + """Sleep until the predicate resolves to be True. + + Warning: Note that this method is not recommended to be used in tests as it is + not aware of the context of the test framework. Using the `wait_until()` members + from `BitcoinTestFramework` or `P2PInterface` class ensures the timeout is + properly scaled. Furthermore, `wait_until()` from `P2PInterface` class in + `p2p.py` has a preset lock. + """ if attempts == float('inf') and timeout == float('inf'): timeout = 60 timeout = timeout * timeout_factor @@ -429,7 +438,7 @@ def disconnect_nodes(from_connection, node_num): raise # wait to disconnect - wait_until(lambda: not get_peer_ids(), timeout=5) + wait_until_helper(lambda: not get_peer_ids(), timeout=5) def connect_nodes(from_connection, node_num): @@ -440,8 +449,8 @@ def connect_nodes(from_connection, node_num): # See comments in net_processing: # * Must have a version message before anything else # * Must have a verack message before anything else - wait_until(lambda: all(peer['version'] != 0 for peer in from_connection.getpeerinfo())) - wait_until(lambda: all(peer['bytesrecv_per_msg'].pop('verack', 0) == 24 for peer in from_connection.getpeerinfo())) + wait_until_helper(lambda: all(peer['version'] != 0 for peer in from_connection.getpeerinfo())) + wait_until_helper(lambda: all(peer['bytesrecv_per_msg'].pop('verack', 0) == 24 for peer in from_connection.getpeerinfo())) # Transaction/Block functions @@ -529,7 +538,7 @@ def create_confirmed_utxos(fee, node, count): addr2 = node.getnewaddress() if iterations <= 0: return utxos - for i in range(iterations): + for _ in range(iterations): t = utxos.pop() inputs = [] inputs.append({"txid": t["txid"], "vout": t["vout"]}) @@ -556,7 +565,7 @@ def gen_return_txouts(): # So we have big transactions (and therefore can't fit very many into each block) # create one script_pubkey script_pubkey = "6a4d0200" # OP_RETURN OP_PUSH2 512 bytes - for i in range(512): + for _ in range(512): script_pubkey = script_pubkey + "01" # concatenate 128 txouts of above script_pubkey which we'll insert before the txout for change txouts = [] @@ -564,7 +573,7 @@ def gen_return_txouts(): txout = CTxOut() txout.nValue = 0 txout.scriptPubKey = hex_str_to_bytes(script_pubkey) - for k in range(128): + for _ in range(128): txouts.append(txout) return txouts @@ -617,3 +626,33 @@ def find_vout_for_address(node, txid, addr): if any([addr == a for a in tx["vout"][i]["scriptPubKey"]["addresses"]]): return i raise RuntimeError("Vout not found for address: txid=%s, addr=%s" % (txid, addr)) + +def modinv(a, n): + """Compute the modular inverse of a modulo n using the extended Euclidean + Algorithm. See https://en.wikipedia.org/wiki/Extended_Euclidean_algorithm#Modular_integers. + """ + # TODO: Change to pow(a, -1, n) available in Python 3.8 + t1, t2 = 0, 1 + r1, r2 = n, a + while r2 != 0: + q = r1 // r2 + t1, t2 = t2, t1 - q * t2 + r1, r2 = r2, r1 - q * r2 + if r1 > 1: + return None + if t1 < 0: + t1 += n + return t1 + +class TestFrameworkUtil(unittest.TestCase): + def test_modinv(self): + test_vectors = [ + [7, 11], + [11, 29], + [90, 13], + [1891, 3797], + [6003722857, 77695236973], + ] + + for a, n in test_vectors: + self.assertEqual(modinv(a, n), pow(a, n-2, n)) diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py new file mode 100644 index 000000000..39b3bf2a5 --- /dev/null +++ b/test/functional/test_framework/wallet.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""A limited-functionality wallet, which may replace a real wallet in tests""" + +from decimal import Decimal +from test_framework.address import ADDRESS_BCRT1_P2WSH_OP_TRUE +from test_framework.messages import ( + COIN, + COutPoint, + CTransaction, + CTxIn, + CTxInWitness, + CTxOut, +) +from test_framework.script import ( + CScript, + OP_TRUE, +) +from test_framework.util import ( + assert_equal, + hex_str_to_bytes, + satoshi_round, +) + + +class MiniWallet: + def __init__(self, test_node): + self._test_node = test_node + self._utxos = [] + self._address = ADDRESS_BCRT1_P2WSH_OP_TRUE + self._scriptPubKey = hex_str_to_bytes(self._test_node.validateaddress(self._address)['scriptPubKey']) + + def generate(self, num_blocks): + """Generate blocks with coinbase outputs to the internal address, and append the outputs to the internal list""" + blocks = self._test_node.generatetoaddress(num_blocks, self._address) + for b in blocks: + cb_tx = self._test_node.getblock(blockhash=b, verbosity=2)['tx'][0] + self._utxos.append({'txid': cb_tx['txid'], 'vout': 0, 'value': cb_tx['vout'][0]['value']}) + return blocks + + def get_utxo(self): + """Return the last utxo. Can be used to get the change output immediately after a send_self_transfer""" + return self._utxos.pop() + + def send_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node, utxo_to_spend=None): + """Create and send a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" + self._utxos = sorted(self._utxos, key=lambda k: k['value']) + utxo_to_spend = utxo_to_spend or self._utxos.pop() # Pick the largest utxo (if none provided) and hope it covers the fee + vsize = Decimal(96) + send_value = satoshi_round(utxo_to_spend['value'] - fee_rate * (vsize / 1000)) + fee = utxo_to_spend['value'] - send_value + assert send_value > 0 + + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(utxo_to_spend['txid'], 16), utxo_to_spend['vout']))] + tx.vout = [CTxOut(int(send_value * COIN), self._scriptPubKey)] + tx.wit.vtxinwit = [CTxInWitness()] + tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] + tx_hex = tx.serialize().hex() + + txid = from_node.sendrawtransaction(tx_hex) + self._utxos.append({'txid': txid, 'vout': 0, 'value': send_value}) + tx_info = from_node.getmempoolentry(txid) + assert_equal(tx_info['vsize'], vsize) + assert_equal(tx_info['fee'], fee) + return {'txid': txid, 'wtxid': tx_info['wtxid'], 'hex': tx_hex} diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index b090e9339..a3e160f12 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -69,7 +69,9 @@ TEST_EXIT_SKIPPED = 77 TEST_FRAMEWORK_MODULES = [ "address", "blocktools", + "muhash", "script", + "util", ] EXTENDED_SCRIPTS = [ @@ -105,7 +107,6 @@ BASE_SCRIPTS = [ 'wallet_listtransactions.py', # vv Tests less than 60s vv 'p2p_sendheaders.py', - 'wallet_zapwallettxes.py', 'wallet_importmulti.py', 'mempool_limit.py', 'rpc_txoutproof.py', @@ -194,6 +195,7 @@ BASE_SCRIPTS = [ 'p2p_eviction.py', 'rpc_signmessage.py', 'rpc_generateblock.py', + 'rpc_generate.py', 'wallet_balance.py', 'feature_nulldummy.py', 'mempool_accept.py', @@ -223,6 +225,7 @@ BASE_SCRIPTS = [ 'rpc_estimatefee.py', 'rpc_getblockstats.py', 'wallet_create_tx.py', + 'wallet_send.py', 'p2p_fingerprint.py', 'feature_uacomment.py', 'wallet_coinbase_category.py', @@ -243,6 +246,7 @@ BASE_SCRIPTS = [ 'p2p_node_network_limited.py', 'p2p_permissions.py', 'feature_blocksdir.py', + 'wallet_startup.py', 'feature_config_args.py', 'feature_settings.py', 'rpc_getdescriptorinfo.py', @@ -713,14 +717,16 @@ class RPCCoverage(): Return a set of currently untested RPC commands. """ - # This is shared from `test/functional/test-framework/coverage.py` + # This is shared from `test/functional/test_framework/coverage.py` reference_filename = 'rpc_interface.txt' coverage_file_prefix = 'coverage.' coverage_ref_filename = os.path.join(self.dir, reference_filename) coverage_filenames = set() all_cmds = set() - covered_cmds = set() + # Consider RPC generate covered, because it is overloaded in + # test_framework/test_node.py and not seen by the coverage check. + covered_cmds = set({'generate'}) if not os.path.isfile(coverage_ref_filename): raise RuntimeError("No coverage reference found") diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 18f0beb59..fa5b5c10f 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -70,12 +70,14 @@ class ToolWalletTest(BitcoinTestFramework): self.assert_raises_tool_error('Invalid command: help', 'help') self.assert_raises_tool_error('Error: two methods provided (info and create). Only one method should be provided.', 'info', 'create') self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo') + locked_dir = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets") self.assert_raises_tool_error( - 'Error loading wallet.dat. Is wallet being used by another process?', + 'Error initializing wallet database environment "{}"!'.format(locked_dir), '-wallet=wallet.dat', 'info', ) - self.assert_raises_tool_error('Error: no wallet file at nonexistent.dat', '-wallet=nonexistent.dat', 'info') + path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "nonexistent.dat") + self.assert_raises_tool_error("Failed to load database path '{}'. Path does not exist.".format(path), '-wallet=nonexistent.dat', 'info') def test_tool_wallet_info(self): # Stop the node to close the wallet to call the info command. diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index 9dd91b249..bcbac18d5 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -50,10 +50,10 @@ class WalletBackupTest(BitcoinTestFramework): # nodes 1, 2,3 are spenders, let's give them a keypool=100 # whitelist all peers to speed up tx relay / mempool sync self.extra_args = [ - ["[email protected]", "-keypool=100"], - ["[email protected]", "-keypool=100"], - ["[email protected]", "-keypool=100"], - ["[email protected]"], + ["[email protected]", "-keypool=100", "-wallet="], + ["[email protected]", "-keypool=100", "-wallet="], + ["[email protected]", "-keypool=100", "-wallet="], + ["[email protected]", "-wallet="], ] self.rpc_timeout = 120 @@ -129,7 +129,7 @@ class WalletBackupTest(BitcoinTestFramework): self.log.info("Creating transactions") # Five rounds of sending each other transactions. - for i in range(5): + for _ in range(5): self.do_one_round() self.log.info("Backing up") @@ -142,7 +142,7 @@ class WalletBackupTest(BitcoinTestFramework): self.nodes[2].dumpwallet(os.path.join(self.nodes[2].datadir, 'wallet.dump')) self.log.info("More transactions") - for i in range(5): + for _ in range(5): self.do_one_round() # Generate 101 more blocks, so any fees paid mature diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 81382d94a..bb208341a 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -12,7 +12,6 @@ from test_framework.util import ( assert_fee_amount, assert_raises_rpc_error, connect_nodes, - wait_until, ) from test_framework.wallet_util import test_address @@ -136,11 +135,19 @@ class WalletTest(BitcoinTestFramework): self.nodes[2].lockunspent, False, [{"txid": unspent_0["txid"], "vout": 999}]) - # An output should be unlocked when spent + # The lock on a manually selected output is ignored unspent_0 = self.nodes[1].listunspent()[0] self.nodes[1].lockunspent(False, [unspent_0]) tx = self.nodes[1].createrawtransaction([unspent_0], { self.nodes[1].getnewaddress() : 1 }) - tx = self.nodes[1].fundrawtransaction(tx)['hex'] + self.nodes[1].fundrawtransaction(tx,{"lockUnspents": True}) + + # fundrawtransaction can lock an input + self.nodes[1].lockunspent(True, [unspent_0]) + assert_equal(len(self.nodes[1].listlockunspent()), 0) + tx = self.nodes[1].fundrawtransaction(tx,{"lockUnspents": True})['hex'] + assert_equal(len(self.nodes[1].listlockunspent()), 1) + + # Send transaction tx = self.nodes[1].signrawtransactionwithwallet(tx)["hex"] self.nodes[1].sendrawtransaction(tx) assert_equal(len(self.nodes[1].listlockunspent()), 0) @@ -527,8 +534,6 @@ class WalletTest(BitcoinTestFramework): maintenance = [ '-rescan', '-reindex', - '-zapwallettxes=1', - '-zapwallettxes=2', ] chainlimit = 6 for m in maintenance: @@ -540,7 +545,7 @@ class WalletTest(BitcoinTestFramework): self.start_node(2, [m, "-limitancestorcount=" + str(chainlimit)]) if m == '-reindex': # reindex will leave rpc warm up "early"; Wait for it to finish - wait_until(lambda: [block_count] * 3 == [self.nodes[i].getblockcount() for i in range(3)]) + self.wait_until(lambda: [block_count] * 3 == [self.nodes[i].getblockcount() for i in range(3)]) assert_equal(balance_nodes, [self.nodes[i].getbalance() for i in range(3)]) # Exercise listsinceblock with the last two blocks @@ -569,7 +574,7 @@ class WalletTest(BitcoinTestFramework): # So we should be able to generate exactly chainlimit txs for each original output sending_addr = self.nodes[1].getnewaddress() txid_list = [] - for i in range(chainlimit * 2): + for _ in range(chainlimit * 2): txid_list.append(self.nodes[0].sendtoaddress(sending_addr, Decimal('0.0001'))) assert_equal(self.nodes[0].getmempoolinfo()['size'], chainlimit * 2) assert_equal(len(txid_list), chainlimit * 2) @@ -589,7 +594,10 @@ class WalletTest(BitcoinTestFramework): self.start_node(0, extra_args=extra_args) # wait until the wallet has submitted all transactions to the mempool - wait_until(lambda: len(self.nodes[0].getrawmempool()) == chainlimit * 2) + self.wait_until(lambda: len(self.nodes[0].getrawmempool()) == chainlimit * 2) + + # Prevent potential race condition when calling wallet RPCs right after restart + self.nodes[0].syncwithvalidationinterfacequeue() node0_balance = self.nodes[0].getbalance() # With walletrejectlongchains we will not create the tx and store it in our wallet. diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 72c85b883..56d1da60b 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -50,6 +50,11 @@ class BumpFeeTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() + def clear_mempool(self): + # Clear mempool between subtests. The subtests may only depend on chainstate (utxos) + self.nodes[1].generate(1) + self.sync_all() + def run_test(self): # Encrypt wallet for test_locked_wallet_fails test self.nodes[1].encryptwallet(WALLET_PASSPHRASE) @@ -62,7 +67,7 @@ class BumpFeeTest(BitcoinTestFramework): self.log.info("Mining blocks...") peer_node.generate(110) self.sync_all() - for i in range(25): + for _ in range(25): peer_node.sendtoaddress(rbf_node_address, 0.001) self.sync_all() peer_node.generate(1) @@ -71,7 +76,7 @@ class BumpFeeTest(BitcoinTestFramework): self.log.info("Running tests") dest_address = peer_node.getnewaddress() - test_invalid_parameters(rbf_node, dest_address) + self.test_invalid_parameters(rbf_node, dest_address) test_simple_bumpfee_succeeds(self, "default", rbf_node, peer_node, dest_address) test_simple_bumpfee_succeeds(self, "fee_rate", rbf_node, peer_node, dest_address) test_feerate_args(self, rbf_node, peer_node, dest_address) @@ -93,28 +98,30 @@ class BumpFeeTest(BitcoinTestFramework): test_small_output_with_feerate_succeeds(self, rbf_node, dest_address) test_no_more_inputs_fails(self, rbf_node, dest_address) -def test_invalid_parameters(node, dest_address): - txid = spend_one_input(node, dest_address) - # invalid estimate mode - assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { - "estimate_mode": "moo", - }) - assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { - "estimate_mode": 38, - }) - assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { - "estimate_mode": { - "foo": "bar", - }, - }) - assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { - "estimate_mode": Decimal("3.141592"), - }) - # confTarget and conf_target - assert_raises_rpc_error(-8, "confTarget and conf_target options should not both be set", node.bumpfee, txid, { - "confTarget": 123, - "conf_target": 456, - }) + def test_invalid_parameters(self, node, dest_address): + txid = spend_one_input(node, dest_address) + # invalid estimate mode + assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { + "estimate_mode": "moo", + }) + assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { + "estimate_mode": 38, + }) + assert_raises_rpc_error(-3, "Expected type string", node.bumpfee, txid, { + "estimate_mode": { + "foo": "bar", + }, + }) + assert_raises_rpc_error(-8, "Invalid estimate_mode parameter", node.bumpfee, txid, { + "estimate_mode": Decimal("3.141592"), + }) + # confTarget and conf_target + assert_raises_rpc_error(-8, "confTarget and conf_target options should not both be set", node.bumpfee, txid, { + "confTarget": 123, + "conf_target": 456, + }) + self.clear_mempool() + def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): self.log.info('Test simple bumpfee: {}'.format(mode)) @@ -123,13 +130,19 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): self.sync_mempools((rbf_node, peer_node)) assert rbfid in rbf_node.getrawmempool() and rbfid in peer_node.getrawmempool() if mode == "fee_rate": + bumped_psbt = rbf_node.psbtbumpfee(rbfid, {"fee_rate": NORMAL}) bumped_tx = rbf_node.bumpfee(rbfid, {"fee_rate": NORMAL}) else: + bumped_psbt = rbf_node.psbtbumpfee(rbfid) bumped_tx = rbf_node.bumpfee(rbfid) assert_equal(bumped_tx["errors"], []) assert bumped_tx["fee"] > -rbftx["fee"] assert_equal(bumped_tx["origfee"], -rbftx["fee"]) assert "psbt" not in bumped_tx + assert_equal(bumped_psbt["errors"], []) + assert bumped_psbt["fee"] > -rbftx["fee"] + assert_equal(bumped_psbt["origfee"], -rbftx["fee"]) + assert "psbt" in bumped_psbt # check that bumped_tx propagates, original tx was evicted and has a wallet conflict self.sync_mempools((rbf_node, peer_node)) assert bumped_tx["txid"] in rbf_node.getrawmempool() @@ -142,6 +155,7 @@ def test_simple_bumpfee_succeeds(self, mode, rbf_node, peer_node, dest_address): bumpedwtx = rbf_node.gettransaction(bumped_tx["txid"]) assert_equal(oldwtx["replaced_by_txid"], bumped_tx["txid"]) assert_equal(bumpedwtx["replaces_txid"], rbfid) + self.clear_mempool() def test_feerate_args(self, rbf_node, peer_node, dest_address): @@ -161,6 +175,7 @@ def test_feerate_args(self, rbf_node, peer_node, dest_address): assert_raises_rpc_error(-3, "Amount out of range", rbf_node.bumpfee, rbfid, {"fee_rate": -1}) assert_raises_rpc_error(-4, "is too high (cannot be higher than", rbf_node.bumpfee, rbfid, {"fee_rate": TOO_HIGH}) + self.clear_mempool() def test_segwit_bumpfee_succeeds(self, rbf_node, dest_address): @@ -192,12 +207,14 @@ def test_segwit_bumpfee_succeeds(self, rbf_node, dest_address): bumped_tx = rbf_node.bumpfee(rbfid) assert bumped_tx["txid"] in rbf_node.getrawmempool() assert rbfid not in rbf_node.getrawmempool() + self.clear_mempool() def test_nonrbf_bumpfee_fails(self, peer_node, dest_address): self.log.info('Test that we cannot replace a non RBF transaction') not_rbfid = peer_node.sendtoaddress(dest_address, Decimal("0.00090000")) assert_raises_rpc_error(-4, "not BIP 125 replaceable", peer_node.bumpfee, not_rbfid) + self.clear_mempool() def test_notmine_bumpfee_fails(self, rbf_node, peer_node, dest_address): @@ -205,20 +222,22 @@ def test_notmine_bumpfee_fails(self, rbf_node, peer_node, dest_address): # here, the rbftx has a peer_node coin and then adds a rbf_node input # Note that this test depends upon the RPC code checking input ownership prior to change outputs # (since it can't use fundrawtransaction, it lacks a proper change output) - utxos = [node.listunspent()[-1] for node in (rbf_node, peer_node)] + fee = Decimal("0.001") + utxos = [node.listunspent(query_options={'minimumAmount': fee})[-1] for node in (rbf_node, peer_node)] inputs = [{ "txid": utxo["txid"], "vout": utxo["vout"], "address": utxo["address"], "sequence": BIP125_SEQUENCE_NUMBER } for utxo in utxos] - output_val = sum(utxo["amount"] for utxo in utxos) - Decimal("0.001") + output_val = sum(utxo["amount"] for utxo in utxos) - fee rawtx = rbf_node.createrawtransaction(inputs, {dest_address: output_val}) signedtx = rbf_node.signrawtransactionwithwallet(rawtx) signedtx = peer_node.signrawtransactionwithwallet(signedtx["hex"]) rbfid = rbf_node.sendrawtransaction(signedtx["hex"]) assert_raises_rpc_error(-4, "Transaction contains inputs that don't belong to this wallet", rbf_node.bumpfee, rbfid) + self.clear_mempool() def test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_address): @@ -229,6 +248,7 @@ def test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_ad tx = rbf_node.signrawtransactionwithwallet(tx) rbf_node.sendrawtransaction(tx["hex"]) assert_raises_rpc_error(-8, "Transaction has descendants in the wallet", rbf_node.bumpfee, parent_id) + self.clear_mempool() def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): @@ -270,6 +290,7 @@ def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): rbf_node.generatetoaddress(1, rbf_node.getnewaddress()) assert_equal(rbf_node.gettransaction(rbfid)["confirmations"], 1) + self.clear_mempool() def test_dust_to_fee(self, rbf_node, dest_address): @@ -292,6 +313,7 @@ def test_dust_to_fee(self, rbf_node, dest_address): assert_equal(len(fulltx["vout"]), 2) assert_equal(len(full_bumped_tx["vout"]), 1) # change output is eliminated assert_equal(full_bumped_tx["vout"][0]['value'], Decimal("0.00050000")) + self.clear_mempool() def test_settxfee(self, rbf_node, dest_address): @@ -314,6 +336,8 @@ def test_settxfee(self, rbf_node, dest_address): assert_raises_rpc_error(-8, "txfee cannot be more than wallet max tx fee", rbf_node.settxfee, Decimal('0.00003')) self.restart_node(1, self.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + self.connect_nodes(1, 0) + self.clear_mempool() def test_maxtxfee_fails(self, rbf_node, dest_address): @@ -327,6 +351,8 @@ def test_maxtxfee_fails(self, rbf_node, dest_address): assert_raises_rpc_error(-4, "Unable to create transaction. Fee exceeds maximum configured by -maxtxfee", rbf_node.bumpfee, rbfid) self.restart_node(1, self.extra_args[1]) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + self.connect_nodes(1, 0) + self.clear_mempool() def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): @@ -391,7 +417,7 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): assert_equal(len(watcher.decodepsbt(psbt)["tx"]["vin"]), 1) # Bump fee, obnoxiously high to add additional watchonly input - bumped_psbt = watcher.bumpfee(original_txid, {"fee_rate": HIGH}) + bumped_psbt = watcher.psbtbumpfee(original_txid, {"fee_rate": HIGH}) assert_greater_than(len(watcher.decodepsbt(bumped_psbt['psbt'])["tx"]["vin"]), 1) assert "txid" not in bumped_psbt assert_equal(bumped_psbt["origfee"], -watcher.gettransaction(original_txid)["fee"]) @@ -409,6 +435,7 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): rbf_node.unloadwallet("watcher") rbf_node.unloadwallet("signer") + self.clear_mempool() def test_rebumping(self, rbf_node, dest_address): @@ -417,6 +444,7 @@ def test_rebumping(self, rbf_node, dest_address): bumped = rbf_node.bumpfee(rbfid, {"fee_rate": ECONOMICAL}) assert_raises_rpc_error(-4, "already bumped", rbf_node.bumpfee, rbfid, {"fee_rate": NORMAL}) rbf_node.bumpfee(bumped["txid"], {"fee_rate": NORMAL}) + self.clear_mempool() def test_rebumping_not_replaceable(self, rbf_node, dest_address): @@ -425,6 +453,7 @@ def test_rebumping_not_replaceable(self, rbf_node, dest_address): bumped = rbf_node.bumpfee(rbfid, {"fee_rate": ECONOMICAL, "replaceable": False}) assert_raises_rpc_error(-4, "Transaction is not BIP 125 replaceable", rbf_node.bumpfee, bumped["txid"], {"fee_rate": NORMAL}) + self.clear_mempool() def test_unconfirmed_not_spendable(self, rbf_node, rbf_node_address): @@ -464,6 +493,7 @@ def test_unconfirmed_not_spendable(self, rbf_node, rbf_node_address): assert_equal( sum(1 for t in rbf_node.listunspent(minconf=0, include_unsafe=False) if t["txid"] == rbfid and t["address"] == rbf_node_address and t["spendable"]), 1) + self.clear_mempool() def test_bumpfee_metadata(self, rbf_node, dest_address): @@ -475,6 +505,7 @@ def test_bumpfee_metadata(self, rbf_node, dest_address): bumped_wtx = rbf_node.gettransaction(bumped_tx["txid"]) assert_equal(bumped_wtx["comment"], "comment value") assert_equal(bumped_wtx["to"], "to value") + self.clear_mempool() def test_locked_wallet_fails(self, rbf_node, dest_address): @@ -484,6 +515,7 @@ def test_locked_wallet_fails(self, rbf_node, dest_address): assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first.", rbf_node.bumpfee, rbfid) rbf_node.walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) + self.clear_mempool() def test_change_script_match(self, rbf_node, dest_address): @@ -504,6 +536,7 @@ def test_change_script_match(self, rbf_node, dest_address): assert_equal(change_addresses, get_change_address(bumped_total_tx['txid'])) bumped_rate_tx = rbf_node.bumpfee(bumped_total_tx["txid"]) assert_equal(change_addresses, get_change_address(bumped_rate_tx['txid'])) + self.clear_mempool() def spend_one_input(node, dest_address, change_size=Decimal("0.00049000")): @@ -542,6 +575,7 @@ def test_no_more_inputs_fails(self, rbf_node, dest_address): # spend all funds, no change output rbfid = rbf_node.sendtoaddress(rbf_node.getnewaddress(), rbf_node.getbalance(), "", "", True) assert_raises_rpc_error(-4, "Unable to create transaction. Insufficient funds", rbf_node.bumpfee, rbfid) + self.clear_mempool() if __name__ == "__main__": diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py index 330de8b0f..ed9159726 100755 --- a/test/functional/wallet_create_tx.py +++ b/test/functional/wallet_create_tx.py @@ -45,7 +45,7 @@ class CreateTxWalletTest(BitcoinTestFramework): def test_tx_size_too_large(self): # More than 10kB of outputs, so that we hit -maxtxfee with a high feerate - outputs = {self.nodes[0].getnewaddress(address_type='bech32'): 0.000025 for i in range(400)} + outputs = {self.nodes[0].getnewaddress(address_type='bech32'): 0.000025 for _ in range(400)} raw_tx = self.nodes[0].createrawtransaction(inputs=[], outputs=outputs) for fee_setting in ['-minrelaytxfee=0.01', '-mintxfee=0.01', '-paytxfee=0.01']: diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index 289ccf43e..9c63e8f7d 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -107,7 +107,7 @@ class WalletDescriptorTest(BitcoinTestFramework): assert_equal(info2['desc'], info3['desc']) self.log.info("Test that getnewaddress still works after keypool is exhausted in an encrypted wallet") - for i in range(0, 500): + for _ in range(500): send_wrpc.getnewaddress() self.log.info("Test that unlock is needed when deriving only hardened keys in an encrypted wallet") @@ -120,7 +120,7 @@ class WalletDescriptorTest(BitcoinTestFramework): }]) send_wrpc.walletlock() # Exhaust keypool of 100 - for i in range(0, 100): + for _ in range(100): send_wrpc.getnewaddress(address_type='bech32') # This should now error assert_raises_rpc_error(-12, "Keypool ran out, please call keypoolrefill first", send_wrpc.getnewaddress, '', 'bech32') diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index 6bfb46882..09581d864 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -95,7 +95,7 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): class WalletDumpTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [["-keypool=90", "-addresstype=legacy"]] + self.extra_args = [["-keypool=90", "-addresstype=legacy", "-wallet=dump"]] self.rpc_timeout = 120 def skip_test_if_missing_module(self): @@ -116,7 +116,7 @@ class WalletDumpTest(BitcoinTestFramework): test_addr_count = 10 addrs = [] for address_type in ['legacy', 'p2sh-segwit', 'bech32']: - for i in range(0, test_addr_count): + for _ in range(test_addr_count): addr = self.nodes[0].getnewaddress(address_type=address_type) vaddr = self.nodes[0].getaddressinfo(addr) # required to get hd keypath addrs.append(vaddr) diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index 9dd55b4ab..e5c4f12f2 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -15,8 +15,14 @@ from test_framework.util import ( class WalletGroupTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 3 - self.extra_args = [[], [], ['-avoidpartialspends']] + self.num_nodes = 5 + self.extra_args = [ + [], + [], + ["-avoidpartialspends"], + ["-maxapsfee=0.00002719"], + ["-maxapsfee=0.00002720"], + ] self.rpc_timeout = 480 def skip_test_if_missing_module(self): @@ -27,8 +33,8 @@ class WalletGroupTest(BitcoinTestFramework): self.nodes[0].generate(110) # Get some addresses from the two nodes - addr1 = [self.nodes[1].getnewaddress() for i in range(3)] - addr2 = [self.nodes[2].getnewaddress() for i in range(3)] + addr1 = [self.nodes[1].getnewaddress() for _ in range(3)] + addr2 = [self.nodes[2].getnewaddress() for _ in range(3)] addrs = addr1 + addr2 # Send 1 + 0.5 coin to each address @@ -50,8 +56,8 @@ class WalletGroupTest(BitcoinTestFramework): # one output should be 0.2, the other should be ~0.3 v = [vout["value"] for vout in tx1["vout"]] v.sort() - assert_approx(v[0], 0.2) - assert_approx(v[1], 0.3, 0.0001) + assert_approx(v[0], vexp=0.2, vspan=0.0001) + assert_approx(v[1], vexp=0.3, vspan=0.0001) txid2 = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 0.2) tx2 = self.nodes[2].getrawtransaction(txid2, True) @@ -61,8 +67,80 @@ class WalletGroupTest(BitcoinTestFramework): # one output should be 0.2, the other should be ~1.3 v = [vout["value"] for vout in tx2["vout"]] v.sort() - assert_approx(v[0], 0.2) - assert_approx(v[1], 1.3, 0.0001) + assert_approx(v[0], vexp=0.2, vspan=0.0001) + assert_approx(v[1], vexp=1.3, vspan=0.0001) + + # Test 'avoid partial if warranted, even if disabled' + self.sync_all() + self.nodes[0].generate(1) + # Nodes 1-2 now have confirmed UTXOs (letters denote destinations): + # Node #1: Node #2: + # - A 1.0 - D0 1.0 + # - B0 1.0 - D1 0.5 + # - B1 0.5 - E0 1.0 + # - C0 1.0 - E1 0.5 + # - C1 0.5 - F ~1.3 + # - D ~0.3 + assert_approx(self.nodes[1].getbalance(), vexp=4.3, vspan=0.0001) + assert_approx(self.nodes[2].getbalance(), vexp=4.3, vspan=0.0001) + # Sending 1.4 btc should pick one 1.0 + one more. For node #1, + # this could be (A / B0 / C0) + (B1 / C1 / D). We ensure that it is + # B0 + B1 or C0 + C1, because this avoids partial spends while not being + # detrimental to transaction cost + txid3 = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.4) + tx3 = self.nodes[1].getrawtransaction(txid3, True) + # tx3 should have 2 inputs and 2 outputs + assert_equal(2, len(tx3["vin"])) + assert_equal(2, len(tx3["vout"])) + # the accumulated value should be 1.5, so the outputs should be + # ~0.1 and 1.4 and should come from the same destination + values = [vout["value"] for vout in tx3["vout"]] + values.sort() + assert_approx(values[0], vexp=0.1, vspan=0.0001) + assert_approx(values[1], vexp=1.4, vspan=0.0001) + + input_txids = [vin["txid"] for vin in tx3["vin"]] + input_addrs = [self.nodes[1].gettransaction(txid)['details'][0]['address'] for txid in input_txids] + assert_equal(input_addrs[0], input_addrs[1]) + # Node 2 enforces avoidpartialspends so needs no checking here + + # Test wallet option maxapsfee with Node 3 + addr_aps = self.nodes[3].getnewaddress() + self.nodes[0].sendtoaddress(addr_aps, 1.0) + self.nodes[0].sendtoaddress(addr_aps, 1.0) + self.nodes[0].generate(1) + self.sync_all() + with self.nodes[3].assert_debug_log(['Fee non-grouped = 2820, grouped = 4160, using grouped']): + txid4 = self.nodes[3].sendtoaddress(self.nodes[0].getnewaddress(), 0.1) + tx4 = self.nodes[3].getrawtransaction(txid4, True) + # tx4 should have 2 inputs and 2 outputs although one output would + # have been enough and the transaction caused higher fees + assert_equal(2, len(tx4["vin"])) + assert_equal(2, len(tx4["vout"])) + + addr_aps2 = self.nodes[3].getnewaddress() + [self.nodes[0].sendtoaddress(addr_aps2, 1.0) for _ in range(5)] + self.nodes[0].generate(1) + self.sync_all() + with self.nodes[3].assert_debug_log(['Fee non-grouped = 5520, grouped = 8240, using non-grouped']): + txid5 = self.nodes[3].sendtoaddress(self.nodes[0].getnewaddress(), 2.95) + tx5 = self.nodes[3].getrawtransaction(txid5, True) + # tx5 should have 3 inputs (1.0, 1.0, 1.0) and 2 outputs + assert_equal(3, len(tx5["vin"])) + assert_equal(2, len(tx5["vout"])) + + # Test wallet option maxapsfee with node 4, which sets maxapsfee + # 1 sat higher, crossing the threshold from non-grouped to grouped. + addr_aps3 = self.nodes[4].getnewaddress() + [self.nodes[0].sendtoaddress(addr_aps3, 1.0) for _ in range(5)] + self.nodes[0].generate(1) + self.sync_all() + with self.nodes[4].assert_debug_log(['Fee non-grouped = 5520, grouped = 8240, using grouped']): + txid6 = self.nodes[4].sendtoaddress(self.nodes[0].getnewaddress(), 2.95) + tx6 = self.nodes[4].getrawtransaction(txid6, True) + # tx6 should have 5 inputs and 2 outputs + assert_equal(5, len(tx6["vin"])) + assert_equal(2, len(tx6["vout"])) # Empty out node2's wallet self.nodes[2].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=self.nodes[2].getbalance(), subtractfeefromamount=True) @@ -71,7 +149,7 @@ class WalletGroupTest(BitcoinTestFramework): # Fill node2's wallet with 10000 outputs corresponding to the same # scriptPubKey - for i in range(5): + for _ in range(5): raw_tx = self.nodes[0].createrawtransaction([{"txid":"0"*64, "vout":0}], [{addr2[0]: 0.05}]) tx = FromHex(CTransaction(), raw_tx) tx.vin = [] diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index 4ff7f1d52..87deaded0 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -151,7 +151,7 @@ class ImportRescanTest(BitcoinTestFramework): self.skip_if_no_wallet() def setup_network(self): - self.extra_args = [[] for _ in range(self.num_nodes)] + self.extra_args = [["-wallet="] for _ in range(self.num_nodes)] for i, import_node in enumerate(IMPORT_NODES, 2): if import_node.prune: self.extra_args[i] += ["-prune=1"] @@ -159,7 +159,7 @@ class ImportRescanTest(BitcoinTestFramework): self.add_nodes(self.num_nodes, extra_args=self.extra_args) # Import keys with pruning disabled - self.start_nodes(extra_args=[[]] * self.num_nodes) + self.start_nodes(extra_args=[["-wallet="]] * self.num_nodes) for n in self.nodes: n.importprivkey(privkey=n.get_deterministic_priv_key().key, label='coinbase') self.stop_nodes() diff --git a/test/functional/wallet_labels.py b/test/functional/wallet_labels.py index fb4a1f979..cff59bd1c 100755 --- a/test/functional/wallet_labels.py +++ b/test/functional/wallet_labels.py @@ -118,7 +118,7 @@ class WalletLabelsTest(BitcoinTestFramework): if not self.options.descriptors: for label in labels: addresses = [] - for x in range(10): + for _ in range(10): addresses.append(node.getnewaddress()) multisig_address = node.addmultisigaddress(5, addresses, label.name)['address'] label.add_address(multisig_address) diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index 6d51ca6c9..d4131deab 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -36,6 +36,7 @@ class ListSinceBlockTest(BitcoinTestFramework): self.test_double_spend() self.test_double_send() self.double_spends_filtered() + self.test_targetconfirmations() def test_no_blockhash(self): self.log.info("Test no blockhash") @@ -74,6 +75,27 @@ class ListSinceBlockTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "blockhash must be hexadecimal string (not 'Z000000000000000000000000000000000000000000000000000000000000000')", self.nodes[0].listsinceblock, "Z000000000000000000000000000000000000000000000000000000000000000") + def test_targetconfirmations(self): + ''' + This tests when the value of target_confirmations exceeds the number of + blocks in the main chain. In this case, the genesis block hash should be + given for the `lastblock` property. If target_confirmations is < 1, then + a -8 invalid parameter error is thrown. + ''' + self.log.info("Test target_confirmations") + blockhash, = self.nodes[2].generate(1) + blockheight = self.nodes[2].getblockheader(blockhash)['height'] + self.sync_all() + + assert_equal( + self.nodes[0].getblockhash(0), + self.nodes[0].listsinceblock(blockhash, blockheight + 1)['lastblock']) + assert_equal( + self.nodes[0].getblockhash(0), + self.nodes[0].listsinceblock(blockhash, blockheight + 1000)['lastblock']) + assert_raises_rpc_error(-8, "Invalid parameter", + self.nodes[0].listsinceblock, blockhash, 0) + def test_reorg(self): ''' `listsinceblock` did not behave correctly when handed a block that was diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index a54396cad..aaf050ebf 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -26,7 +26,7 @@ FEATURE_LATEST = 169900 got_loading_error = False def test_load_unload(node, name): global got_loading_error - for i in range(10): + for _ in range(10): if got_loading_error: return try: @@ -43,6 +43,7 @@ class MultiWalletTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 2 self.rpc_timeout = 120 + self.extra_args = [["-wallet="], ["-wallet="]] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -82,7 +83,7 @@ class MultiWalletTest(BitcoinTestFramework): os.rename(wallet_dir("wallet.dat"), wallet_dir("w8")) # create another dummy wallet for use in testing backups later - self.start_node(0, []) + self.start_node(0, ["-wallet="]) self.stop_nodes() empty_wallet = os.path.join(self.options.tmpdir, 'empty.dat') os.rename(wallet_dir("wallet.dat"), empty_wallet) @@ -134,11 +135,6 @@ class MultiWalletTest(BitcoinTestFramework): open(not_a_dir, 'a', encoding="utf8").close() self.nodes[0].assert_start_raises_init_error(['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory') - self.log.info("Do not allow -zapwallettxes with multiwallet") - self.nodes[0].assert_start_raises_init_error(['-zapwallettxes', '-wallet=w1', '-wallet=w2'], "Error: -zapwallettxes is only allowed with a single wallet file") - self.nodes[0].assert_start_raises_init_error(['-zapwallettxes=1', '-wallet=w1', '-wallet=w2'], "Error: -zapwallettxes is only allowed with a single wallet file") - self.nodes[0].assert_start_raises_init_error(['-zapwallettxes=2', '-wallet=w1', '-wallet=w2'], "Error: -zapwallettxes is only allowed with a single wallet file") - # if wallets/ doesn't exist, datadir should be the default wallet dir wallet_dir2 = data_dir('walletdir') os.rename(wallet_dir(), wallet_dir2) @@ -157,7 +153,7 @@ class MultiWalletTest(BitcoinTestFramework): competing_wallet_dir = os.path.join(self.options.tmpdir, 'competing_walletdir') os.mkdir(competing_wallet_dir) - self.restart_node(0, ['-walletdir=' + competing_wallet_dir]) + self.restart_node(0, ['-walletdir=' + competing_wallet_dir, '-wallet=']) exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\"!" self.nodes[1].assert_start_raises_init_error(['-walletdir=' + competing_wallet_dir], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) @@ -249,13 +245,16 @@ class MultiWalletTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].listwallets()), set(wallet_names)) # Fail to load if wallet doesn't exist - assert_raises_rpc_error(-18, 'Wallet wallets not found.', self.nodes[0].loadwallet, 'wallets') + path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "wallets") + assert_raises_rpc_error(-18, "Wallet file verification failed. Failed to load database path '{}'. Path does not exist.".format(path), self.nodes[0].loadwallet, 'wallets') # Fail to load duplicate wallets - assert_raises_rpc_error(-4, 'Wallet file verification failed. Error loading wallet w1. Duplicate -wallet filename specified.', self.nodes[0].loadwallet, wallet_names[0]) + path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "w1", "wallet.dat") + assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, wallet_names[0]) # Fail to load duplicate wallets by different ways (directory and filepath) - assert_raises_rpc_error(-4, "Wallet file verification failed. Error loading wallet wallet.dat. Duplicate -wallet filename specified.", self.nodes[0].loadwallet, 'wallet.dat') + path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "wallet.dat") + assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, 'wallet.dat') # Fail to load if one wallet is a copy of another assert_raises_rpc_error(-4, "BerkeleyDatabase: Can't open database w8_copy (duplicates fileid", self.nodes[0].loadwallet, 'w8_copy') @@ -269,12 +268,14 @@ class MultiWalletTest(BitcoinTestFramework): # Fail to load if a directory is specified that doesn't contain a wallet os.mkdir(wallet_dir('empty_wallet_dir')) - assert_raises_rpc_error(-18, "Directory empty_wallet_dir does not contain a wallet.dat file", self.nodes[0].loadwallet, 'empty_wallet_dir') + path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "empty_wallet_dir") + assert_raises_rpc_error(-18, "Wallet file verification failed. Failed to load database path '{}'. Data is not in recognized format.".format(path), self.nodes[0].loadwallet, 'empty_wallet_dir') self.log.info("Test dynamic wallet creation.") # Fail to create a wallet if it already exists. - assert_raises_rpc_error(-4, "Wallet w2 already exists.", self.nodes[0].createwallet, 'w2') + path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "w2") + assert_raises_rpc_error(-4, "Failed to create database path '{}'. Database already exists.".format(path), self.nodes[0].createwallet, 'w2') # Successfully create a wallet with a new name loadwallet_name = self.nodes[0].createwallet('w9') diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 3417616d7..0327c9e07 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -7,9 +7,9 @@ import time from test_framework.blocktools import create_block, create_coinbase from test_framework.messages import ToHex -from test_framework.mininode import P2PTxInvStore, mininode_lock +from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, wait_until +from test_framework.util import assert_equal class ResendWalletTransactionsTest(BitcoinTestFramework): def set_test_params(self): @@ -24,7 +24,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): node.add_p2p_connection(P2PTxInvStore()) self.log.info("Create a new transaction and wait until it's broadcast") - txid = int(node.sendtoaddress(node.getnewaddress(), 1), 16) + txid = node.sendtoaddress(node.getnewaddress(), 1) # Wallet rebroadcast is first scheduled 1 sec after startup (see # nNextResend in ResendWalletTransactions()). Sleep for just over a @@ -33,7 +33,7 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): time.sleep(1.1) # Can take a few seconds due to transaction trickling - wait_until(lambda: node.p2p.tx_invs_received[txid] >= 1, lock=mininode_lock) + node.p2p.wait_for_broadcast([txid]) # Add a second peer since txs aren't rebroadcast to the same peer (see filterInventoryKnown) node.add_p2p_connection(P2PTxInvStore()) @@ -58,13 +58,13 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): two_min = 2 * 60 node.setmocktime(now + twelve_hrs - two_min) time.sleep(2) # ensure enough time has passed for rebroadcast attempt to occur - assert_equal(txid in node.p2ps[1].get_invs(), False) + assert_equal(int(txid, 16) in node.p2ps[1].get_invs(), False) self.log.info("Bump time & check that transaction is rebroadcast") # Transaction should be rebroadcast approximately 24 hours in the future, # but can range from 12-36. So bump 36 hours to be sure. node.setmocktime(now + 36 * 60 * 60) - wait_until(lambda: node.p2ps[1].tx_invs_received[txid] >= 1, lock=mininode_lock) + node.p2p.wait_for_broadcast([txid]) if __name__ == '__main__': diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py new file mode 100755 index 000000000..b64d2030a --- /dev/null +++ b/test/functional/wallet_send.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the send RPC command.""" + +from decimal import Decimal, getcontext +from test_framework.authproxy import JSONRPCException +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_fee_amount, + assert_greater_than, + assert_raises_rpc_error +) + +class WalletSendTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + # whitelist all peers to speed up tx relay / mempool sync + self.extra_args = [ + ["-whitelist=127.0.0.1","-walletrbf=1"], + ["-whitelist=127.0.0.1","-walletrbf=1"], + ] + getcontext().prec = 8 # Satoshi precision for Decimal + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def test_send(self, from_wallet, to_wallet=None, amount=None, data=None, + arg_conf_target=None, arg_estimate_mode=None, + conf_target=None, estimate_mode=None, add_to_wallet=None,psbt=None, + inputs=None,add_inputs=None,change_address=None,change_position=None,change_type=None, + include_watching=None,locktime=None,lock_unspents=None,replaceable=None,subtract_fee_from_outputs=None, + expect_error=None): + assert (amount is None) != (data is None) + + from_balance_before = from_wallet.getbalance() + if to_wallet is None: + assert amount is None + else: + to_untrusted_pending_before = to_wallet.getbalances()["mine"]["untrusted_pending"] + + if amount: + dest = to_wallet.getnewaddress() + outputs = {dest: amount} + else: + outputs = {"data": data} + + # Construct options dictionary + options = {} + if add_to_wallet is not None: + options["add_to_wallet"] = add_to_wallet + else: + if psbt: + add_to_wallet = False + else: + add_to_wallet = from_wallet.getwalletinfo()["private_keys_enabled"] # Default value + if psbt is not None: + options["psbt"] = psbt + if conf_target is not None: + options["conf_target"] = conf_target + if estimate_mode is not None: + options["estimate_mode"] = estimate_mode + if inputs is not None: + options["inputs"] = inputs + if add_inputs is not None: + options["add_inputs"] = add_inputs + if change_address is not None: + options["change_address"] = change_address + if change_position is not None: + options["change_position"] = change_position + if change_type is not None: + options["change_type"] = change_type + if include_watching is not None: + options["include_watching"] = include_watching + if locktime is not None: + options["locktime"] = locktime + if lock_unspents is not None: + options["lock_unspents"] = lock_unspents + if replaceable is None: + replaceable = True # default + else: + options["replaceable"] = replaceable + if subtract_fee_from_outputs is not None: + options["subtract_fee_from_outputs"] = subtract_fee_from_outputs + + if len(options.keys()) == 0: + options = None + + if expect_error is None: + res = from_wallet.send(outputs=outputs, conf_target=arg_conf_target, estimate_mode=arg_estimate_mode, options=options) + else: + try: + assert_raises_rpc_error(expect_error[0],expect_error[1],from_wallet.send, + outputs=outputs,conf_target=arg_conf_target,estimate_mode=arg_estimate_mode,options=options) + except AssertionError: + # Provide debug info if the test fails + self.log.error("Unexpected successful result:") + self.log.error(options) + res = from_wallet.send(outputs=outputs,conf_target=arg_conf_target,estimate_mode=arg_estimate_mode,options=options) + self.log.error(res) + if "txid" in res and add_to_wallet: + self.log.error("Transaction details:") + try: + tx = from_wallet.gettransaction(res["txid"]) + self.log.error(tx) + self.log.error("testmempoolaccept (transaction may already be in mempool):") + self.log.error(from_wallet.testmempoolaccept([tx["hex"]])) + except JSONRPCException as exc: + self.log.error(exc) + + raise + + return + + if locktime: + return res + + if from_wallet.getwalletinfo()["private_keys_enabled"] and not include_watching: + assert_equal(res["complete"], True) + assert "txid" in res + else: + assert_equal(res["complete"], False) + assert not "txid" in res + assert "psbt" in res + + if add_to_wallet and not include_watching: + # Ensure transaction exists in the wallet: + tx = from_wallet.gettransaction(res["txid"]) + assert tx + assert_equal(tx["bip125-replaceable"], "yes" if replaceable else "no") + # Ensure transaction exists in the mempool: + tx = from_wallet.getrawtransaction(res["txid"],True) + assert tx + if amount: + if subtract_fee_from_outputs: + assert_equal(from_balance_before - from_wallet.getbalance(), amount) + else: + assert_greater_than(from_balance_before - from_wallet.getbalance(), amount) + else: + assert next((out for out in tx["vout"] if out["scriptPubKey"]["asm"] == "OP_RETURN 35"), None) + else: + assert_equal(from_balance_before, from_wallet.getbalance()) + + if to_wallet: + self.sync_mempools() + if add_to_wallet: + if not subtract_fee_from_outputs: + assert_equal(to_wallet.getbalances()["mine"]["untrusted_pending"], to_untrusted_pending_before + Decimal(amount if amount else 0)) + else: + assert_equal(to_wallet.getbalances()["mine"]["untrusted_pending"], to_untrusted_pending_before) + + return res + + def run_test(self): + self.log.info("Setup wallets...") + # w0 is a wallet with coinbase rewards + w0 = self.nodes[0].get_wallet_rpc("") + # w1 is a regular wallet + self.nodes[1].createwallet(wallet_name="w1") + w1 = self.nodes[1].get_wallet_rpc("w1") + # w2 contains the private keys for w3 + self.nodes[1].createwallet(wallet_name="w2") + w2 = self.nodes[1].get_wallet_rpc("w2") + # w3 is a watch-only wallet, based on w2 + self.nodes[1].createwallet(wallet_name="w3",disable_private_keys=True) + w3 = self.nodes[1].get_wallet_rpc("w3") + for _ in range(3): + a2_receive = w2.getnewaddress() + a2_change = w2.getrawchangeaddress() # doesn't actually use change derivation + res = w3.importmulti([{ + "desc": w2.getaddressinfo(a2_receive)["desc"], + "timestamp": "now", + "keypool": True, + "watchonly": True + },{ + "desc": w2.getaddressinfo(a2_change)["desc"], + "timestamp": "now", + "keypool": True, + "internal": True, + "watchonly": True + }]) + assert_equal(res, [{"success": True}, {"success": True}]) + + w0.sendtoaddress(a2_receive, 10) # fund w3 + self.nodes[0].generate(1) + self.sync_blocks() + + # w4 has private keys enabled, but only contains watch-only keys (from w2) + self.nodes[1].createwallet(wallet_name="w4",disable_private_keys=False) + w4 = self.nodes[1].get_wallet_rpc("w4") + for _ in range(3): + a2_receive = w2.getnewaddress() + res = w4.importmulti([{ + "desc": w2.getaddressinfo(a2_receive)["desc"], + "timestamp": "now", + "keypool": False, + "watchonly": True + }]) + assert_equal(res, [{"success": True}]) + + w0.sendtoaddress(a2_receive, 10) # fund w4 + self.nodes[0].generate(1) + self.sync_blocks() + + self.log.info("Send to address...") + self.test_send(from_wallet=w0, to_wallet=w1, amount=1) + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, add_to_wallet=True) + + self.log.info("Don't broadcast...") + res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, add_to_wallet=False) + assert(res["hex"]) + + self.log.info("Return PSBT...") + res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, psbt=True) + assert(res["psbt"]) + + self.log.info("Create transaction that spends to address, but don't broadcast...") + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, add_to_wallet=False) + # conf_target & estimate_mode can be set as argument or option + res1 = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_conf_target=1, arg_estimate_mode="economical", add_to_wallet=False) + res2 = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, conf_target=1, estimate_mode="economical", add_to_wallet=False) + assert_equal(self.nodes[1].decodepsbt(res1["psbt"])["fee"], + self.nodes[1].decodepsbt(res2["psbt"])["fee"]) + # but not at the same time + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, arg_conf_target=1, arg_estimate_mode="economical", + conf_target=1, estimate_mode="economical", add_to_wallet=False, expect_error=(-8,"Use either conf_target and estimate_mode or the options dictionary to control fee rate")) + + self.log.info("Create PSBT from watch-only wallet w3, sign with w2...") + res = self.test_send(from_wallet=w3, to_wallet=w1, amount=1) + res = w2.walletprocesspsbt(res["psbt"]) + assert res["complete"] + + self.log.info("Create PSBT from wallet w4 with watch-only keys, sign with w2...") + self.test_send(from_wallet=w4, to_wallet=w1, amount=1, expect_error=(-4, "Insufficient funds")) + res = self.test_send(from_wallet=w4, to_wallet=w1, amount=1, include_watching=True, add_to_wallet=False) + res = w2.walletprocesspsbt(res["psbt"]) + assert res["complete"] + + self.log.info("Create OP_RETURN...") + self.test_send(from_wallet=w0, to_wallet=w1, amount=1) + self.test_send(from_wallet=w0, data="Hello World", expect_error=(-8, "Data must be hexadecimal string (not 'Hello World')")) + self.test_send(from_wallet=w0, data="23") + res = self.test_send(from_wallet=w3, data="23") + res = w2.walletprocesspsbt(res["psbt"]) + assert res["complete"] + + self.log.info("Set fee rate...") + res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, conf_target=2, estimate_mode="sat/b", add_to_wallet=False) + fee = self.nodes[1].decodepsbt(res["psbt"])["fee"] + assert_fee_amount(fee, Decimal(len(res["hex"]) / 2), Decimal("0.00002")) + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, conf_target=-1, estimate_mode="sat/b", + expect_error=(-3, "Amount out of range")) + # Fee rate of 0.1 satoshi per byte should throw an error + # TODO: error should say 1.000 sat/b + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, conf_target=0.1, estimate_mode="sat/b", + expect_error=(-4, "Fee rate (0.00000100 BTC/kB) is lower than the minimum fee rate setting (0.00001000 BTC/kB)")) + + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, conf_target=0.000001, estimate_mode="BTC/KB", + expect_error=(-4, "Fee rate (0.00000100 BTC/kB) is lower than the minimum fee rate setting (0.00001000 BTC/kB)")) + + # TODO: Return hex if fee rate is below -maxmempool + # res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, conf_target=0.1, estimate_mode="sat/b", add_to_wallet=False) + # assert res["hex"] + # hex = res["hex"] + # res = self.nodes[0].testmempoolaccept([hex]) + # assert not res[0]["allowed"] + # assert_equal(res[0]["reject-reason"], "...") # low fee + # assert_fee_amount(fee, Decimal(len(res["hex"]) / 2), Decimal("0.000001")) + + self.log.info("If inputs are specified, do not automatically add more...") + res = self.test_send(from_wallet=w0, to_wallet=w1, amount=51, inputs=[], add_to_wallet=False) + assert res["complete"] + utxo1 = w0.listunspent()[0] + assert_equal(utxo1["amount"], 50) + self.test_send(from_wallet=w0, to_wallet=w1, amount=51, inputs=[utxo1], + expect_error=(-4, "Insufficient funds")) + self.test_send(from_wallet=w0, to_wallet=w1, amount=51, inputs=[utxo1], add_inputs=False, + expect_error=(-4, "Insufficient funds")) + res = self.test_send(from_wallet=w0, to_wallet=w1, amount=51, inputs=[utxo1], add_inputs=True, add_to_wallet=False) + assert res["complete"] + + self.log.info("Manual change address and position...") + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, change_address="not an address", + expect_error=(-5, "Change address must be a valid bitcoin address")) + change_address = w0.getnewaddress() + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, add_to_wallet=False, change_address=change_address) + assert res["complete"] + res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, add_to_wallet=False, change_address=change_address, change_position=0) + assert res["complete"] + assert_equal(self.nodes[0].decodepsbt(res["psbt"])["tx"]["vout"][0]["scriptPubKey"]["addresses"], [change_address]) + res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, add_to_wallet=False, change_type="legacy", change_position=0) + assert res["complete"] + change_address = self.nodes[0].decodepsbt(res["psbt"])["tx"]["vout"][0]["scriptPubKey"]["addresses"][0] + assert change_address[0] == "m" or change_address[0] == "n" + + self.log.info("Set lock time...") + height = self.nodes[0].getblockchaininfo()["blocks"] + res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, locktime=height + 1) + assert res["complete"] + assert res["txid"] + txid = res["txid"] + # Although the wallet finishes the transaction, it can't be added to the mempool yet: + hex = self.nodes[0].gettransaction(res["txid"])["hex"] + res = self.nodes[0].testmempoolaccept([hex]) + assert not res[0]["allowed"] + assert_equal(res[0]["reject-reason"], "non-final") + # It shouldn't be confirmed in the next block + self.nodes[0].generate(1) + assert_equal(self.nodes[0].gettransaction(txid)["confirmations"], 0) + # The mempool should allow it now: + res = self.nodes[0].testmempoolaccept([hex]) + assert res[0]["allowed"] + # Don't wait for wallet to add it to the mempool: + res = self.nodes[0].sendrawtransaction(hex) + self.nodes[0].generate(1) + assert_equal(self.nodes[0].gettransaction(txid)["confirmations"], 1) + + self.log.info("Lock unspents...") + utxo1 = w0.listunspent()[0] + assert_greater_than(utxo1["amount"], 1) + res = self.test_send(from_wallet=w0, to_wallet=w1, amount=1, inputs=[utxo1], add_to_wallet=False, lock_unspents=True) + assert res["complete"] + locked_coins = w0.listlockunspent() + assert_equal(len(locked_coins), 1) + # Locked coins are automatically unlocked when manually selected + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, inputs=[utxo1],add_to_wallet=False) + + self.log.info("Replaceable...") + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, add_to_wallet=False, replaceable=True) + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, add_to_wallet=False, replaceable=False) + + self.log.info("Subtract fee from output") + self.test_send(from_wallet=w0, to_wallet=w1, amount=1, subtract_fee_from_outputs=[0]) + + +if __name__ == '__main__': + WalletSendTest().main() diff --git a/test/functional/wallet_startup.py b/test/functional/wallet_startup.py new file mode 100755 index 000000000..d3119925f --- /dev/null +++ b/test/functional/wallet_startup.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test wallet load on startup. + +Verify that a bitcoind node can maintain list of wallets loading on startup +""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) + + +class WalletStartupTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.supports_cli = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def setup_nodes(self): + self.add_nodes(self.num_nodes) + self.start_nodes() + + def run_test(self): + self.log.info('Should start without any wallets') + assert_equal(self.nodes[0].listwallets(), []) + assert_equal(self.nodes[0].listwalletdir(), {'wallets': []}) + + self.log.info('New default wallet should load by default when there are no other wallets') + self.nodes[0].createwallet(wallet_name='', load_on_startup=False) + self.restart_node(0) + assert_equal(self.nodes[0].listwallets(), ['']) + + self.log.info('Test load on startup behavior') + self.nodes[0].createwallet(wallet_name='w0', load_on_startup=True) + self.nodes[0].createwallet(wallet_name='w1', load_on_startup=False) + self.nodes[0].createwallet(wallet_name='w2', load_on_startup=True) + self.nodes[0].createwallet(wallet_name='w3', load_on_startup=False) + self.nodes[0].createwallet(wallet_name='w4', load_on_startup=False) + self.nodes[0].unloadwallet(wallet_name='w0', load_on_startup=False) + self.nodes[0].unloadwallet(wallet_name='w4', load_on_startup=False) + self.nodes[0].loadwallet(filename='w4', load_on_startup=True) + assert_equal(set(self.nodes[0].listwallets()), set(('', 'w1', 'w2', 'w3', 'w4'))) + self.restart_node(0) + assert_equal(set(self.nodes[0].listwallets()), set(('', 'w2', 'w4'))) + self.nodes[0].unloadwallet(wallet_name='', load_on_startup=False) + self.nodes[0].unloadwallet(wallet_name='w4', load_on_startup=False) + self.nodes[0].loadwallet(filename='w3', load_on_startup=True) + self.nodes[0].loadwallet(filename='') + self.restart_node(0) + assert_equal(set(self.nodes[0].listwallets()), set(('w2', 'w3'))) + +if __name__ == '__main__': + WalletStartupTest().main() diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index 1a76f6521..031da8da8 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -6,7 +6,7 @@ Test upgradewallet RPC. Download node binaries: -contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 +test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 Only v0.15.2 and v0.16.3 are required by this test. The others are used in feature_backwards_compatibility.py """ @@ -27,7 +27,7 @@ class UpgradeWalletTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 self.extra_args = [ - ["-addresstype=bech32"], # current wallet version + ["-addresstype=bech32", "-wallet="], # current wallet version ["-usehd=1"], # v0.16.3 wallet ["-usehd=0"] # v0.15.2 wallet ] diff --git a/test/functional/wallet_zapwallettxes.py b/test/functional/wallet_zapwallettxes.py deleted file mode 100755 index 7f1cdbd20..000000000 --- a/test/functional/wallet_zapwallettxes.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test the zapwallettxes functionality. - -- start two bitcoind nodes -- create two transactions on node 0 - one is confirmed and one is unconfirmed. -- restart node 0 and verify that both the confirmed and the unconfirmed - transactions are still available. -- restart node 0 with zapwallettxes and persistmempool, and verify that both - the confirmed and the unconfirmed transactions are still available. -- restart node 0 with just zapwallettxes and verify that the confirmed - transactions are still available, but that the unconfirmed transaction has - been zapped. -""" -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - assert_raises_rpc_error, - wait_until, -) - -class ZapWalletTXesTest (BitcoinTestFramework): - def set_test_params(self): - self.setup_clean_chain = True - self.num_nodes = 2 - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - - def run_test(self): - self.log.info("Mining blocks...") - self.nodes[0].generate(1) - self.sync_all() - self.nodes[1].generate(100) - self.sync_all() - - # This transaction will be confirmed - txid1 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 10) - - self.nodes[0].generate(1) - self.sync_all() - - # This transaction will not be confirmed - txid2 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 20) - - # Confirmed and unconfirmed transactions are now in the wallet. - assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) - assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) - - # Restart node0. Both confirmed and unconfirmed transactions remain in the wallet. - self.restart_node(0) - - assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) - assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) - - # Restart node0 with zapwallettxes and persistmempool. The unconfirmed - # transaction is zapped from the wallet, but is re-added when the mempool is reloaded. - self.restart_node(0, ["-persistmempool=1", "-zapwallettxes=2"]) - - wait_until(lambda: self.nodes[0].getmempoolinfo()['size'] == 1, timeout=3) - self.nodes[0].syncwithvalidationinterfacequeue() # Flush mempool to wallet - - assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) - assert_equal(self.nodes[0].gettransaction(txid2)['txid'], txid2) - - # Restart node0 with zapwallettxes, but not persistmempool. - # The unconfirmed transaction is zapped and is no longer in the wallet. - self.restart_node(0, ["-zapwallettxes=2"]) - - # tx1 is still be available because it was confirmed - assert_equal(self.nodes[0].gettransaction(txid1)['txid'], txid1) - - # This will raise an exception because the unconfirmed transaction has been zapped - assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', self.nodes[0].gettransaction, txid2) - -if __name__ == '__main__': - ZapWalletTXesTest().main() diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index 56b18752e..c7895edbc 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -56,6 +56,14 @@ def main(): '--m_dir', help='Merge inputs from this directory into the seed_dir. Needs /target subdirectory.', ) + parser.add_argument( + '-g', + '--generate', + action='store_true', + help='Create new corpus seeds (or extend the existing ones) by running' + ' the given targets for a finite number of times. Outputs them to' + ' the passed seed_dir.' + ) args = parser.parse_args() @@ -100,19 +108,20 @@ def main(): logging.info("{} of {} detected fuzz target(s) selected: {}".format(len(test_list_selection), len(test_list_all), " ".join(test_list_selection))) - test_list_seedless = [] - for t in test_list_selection: - corpus_path = os.path.join(args.seed_dir, t) - if not os.path.exists(corpus_path) or len(os.listdir(corpus_path)) == 0: - test_list_seedless.append(t) - test_list_seedless.sort() - if test_list_seedless: - logging.info( - "Fuzzing harnesses lacking a seed corpus: {}".format( - " ".join(test_list_seedless) + if not args.generate: + test_list_seedless = [] + for t in test_list_selection: + corpus_path = os.path.join(args.seed_dir, t) + if not os.path.exists(corpus_path) or len(os.listdir(corpus_path)) == 0: + test_list_seedless.append(t) + test_list_seedless.sort() + if test_list_seedless: + logging.info( + "Fuzzing harnesses lacking a seed corpus: {}".format( + " ".join(test_list_seedless) + ) ) - ) - logging.info("Please consider adding a fuzz seed corpus at https://github.com/bitcoin-core/qa-assets") + logging.info("Please consider adding a fuzz seed corpus at https://github.com/bitcoin-core/qa-assets") try: help_output = subprocess.run( @@ -133,6 +142,14 @@ def main(): sys.exit(1) with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool: + if args.generate: + return generate_corpus_seeds( + fuzz_pool=fuzz_pool, + build_dir=config["environment"]["BUILDDIR"], + seed_dir=args.seed_dir, + targets=test_list_selection, + ) + if args.m_dir: merge_inputs( fuzz_pool=fuzz_pool, @@ -152,6 +169,37 @@ def main(): ) +def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets): + """Generates new corpus seeds. + + Run {targets} without input, and outputs the generated corpus seeds to + {seed_dir}. + """ + logging.info("Generating corpus seeds to {}".format(seed_dir)) + + def job(command): + logging.debug("Running '{}'\n".format(" ".join(command))) + logging.debug("Command '{}' output:\n'{}'\n".format( + ' '.join(command), + subprocess.run(command, check=True, stderr=subprocess.PIPE, + universal_newlines=True).stderr + )) + + futures = [] + for target in targets: + target_seed_dir = os.path.join(seed_dir, target) + os.makedirs(target_seed_dir, exist_ok=True) + command = [ + os.path.join(build_dir, "src", "test", "fuzz", target), + "-runs=100000", + target_seed_dir, + ] + futures.append(fuzz_pool.submit(job, command)) + + for future in as_completed(futures): + future.result() + + def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): logging.info("Merge the inputs in the passed dir into the seed_dir. Passed dir {}".format(merge_dir)) jobs = [] diff --git a/contrib/devtools/previous_release.py b/test/get_previous_releases.py index 5599051cf..1348b8246 100755 --- a/contrib/devtools/previous_release.py +++ b/test/get_previous_releases.py @@ -20,6 +20,40 @@ import sys import hashlib +SHA256_SUMS = { +"d40f18b4e43c6e6370ef7db9131f584fbb137276ec2e3dba67a4b267f81cb644": "bitcoin-0.15.2-aarch64-linux-gnu.tar.gz", +"54fb877a148a6ad189a1e1ab1ff8b11181e58ff2aaf430da55b3fd46ae549a6b": "bitcoin-0.15.2-arm-linux-gnueabihf.tar.gz", +"2b843506c3f1af0eeca5854a920264f9a829f02d0d50328005950ddcbe88874d": "bitcoin-0.15.2-i686-pc-linux-gnu.tar.gz", +"87e9340ff3d382d543b2b69112376077f0c8b4f7450d372e83b68f5a1e22b2df": "bitcoin-0.15.2-osx64.tar.gz", +"566be44190fd76daa01f13d428939dadfb8e3daacefc8fa17f433cad28f73bd5": "bitcoin-0.15.2-x86_64-linux-gnu.tar.gz", + +"0768c6c15caffbaca6524824c9563b42c24f70633c681c2744649158aa3fd484": "bitcoin-0.16.3-aarch64-linux-gnu.tar.gz", +"fb2818069854a6ad20ea03b28b55dbd35d8b1f7d453e90b83eace5d0098a2a87": "bitcoin-0.16.3-arm-linux-gnueabihf.tar.gz", +"75a537844313b0a84bdb61ffcdc5c4ce19a738f7ddf71007cd2edf664efd7c37": "bitcoin-0.16.3-i686-pc-linux-gnu.tar.gz", +"78c3bff3b619a19aed575961ea43cc9e142959218835cf51aede7f0b764fc25d": "bitcoin-0.16.3-osx64.tar.gz", +"5d422a9d544742bc0df12427383f9c2517433ce7b58cf672b9a9b17c2ef51e4f": "bitcoin-0.16.3-x86_64-linux-gnu.tar.gz", + +"5a6b35d1a348a402f2d2d6ab5aed653a1a1f13bc63aaaf51605e3501b0733b7a": "bitcoin-0.17.2-aarch64-linux-gnu.tar.gz", +"d1913a5d19c8e8da4a67d1bd5205d03c8614dfd2e02bba2fe3087476643a729e": "bitcoin-0.17.2-arm-linux-gnueabihf.tar.gz", +"d295fc93f39bbf0fd937b730a93184899a2eb6c3a6d53f3d857cbe77ef89b98c": "bitcoin-0.17.2-i686-pc-linux-gnu.tar.gz", +"a783ba20706dbfd5b47fbedf42165fce70fbbc7d78003305d964f6b3da14887f": "bitcoin-0.17.2-osx64.tar.gz", +"943f9362b9f11130177839116f48f809d83478b4c28591d486ee9a7e35179da6": "bitcoin-0.17.2-x86_64-linux-gnu.tar.gz", + +"88f343af72803b851c7da13874cc5525026b0b55e63e1b5e1298390c4688adc6": "bitcoin-0.18.1-aarch64-linux-gnu.tar.gz", +"cc7d483e4b20c5dabd4dcaf304965214cf4934bcc029ca99cbc9af00d3771a1f": "bitcoin-0.18.1-arm-linux-gnueabihf.tar.gz", +"989e847b3e95fc9fedc0b109cae1b4fa43348f2f712e187a118461876af9bd16": "bitcoin-0.18.1-i686-pc-linux-gnu.tar.gz", +"b7bbcee7a7540f711b171d6981f939ca8482005fde22689bc016596d80548bb1": "bitcoin-0.18.1-osx64.tar.gz", +"425ee5ec631ae8da71ebc1c3f5c0269c627cf459379b9b030f047107a28e3ef8": "bitcoin-0.18.1-riscv64-linux-gnu.tar.gz", +"600d1db5e751fa85903e935a01a74f5cc57e1e7473c15fd3e17ed21e202cfe5a": "bitcoin-0.18.1-x86_64-linux-gnu.tar.gz", + +"3a80431717842672df682bdb619e66523b59541483297772a7969413be3502ff": "bitcoin-0.19.1-aarch64-linux-gnu.tar.gz", +"657f28213823d240dd3324d14829702f9ad6f0710f8bdd1c379cb3c447197f48": "bitcoin-0.19.1-arm-linux-gnueabihf.tar.gz", +"10d1e53208aa7603022f4acc084a046299ab4ccf25fe01e81b3fb6f856772589": "bitcoin-0.19.1-i686-pc-linux-gnu.tar.gz", +"1ae1b87de26487075cd2fd22e0d4ead87d969bd55c44f2f1d873ecdc6147ebb3": "bitcoin-0.19.1-osx64.tar.gz", +"aa7a9563b48aa79252c8e7b6a41c07a5441bd9f14c5e4562cc72720ea6cb0ee5": "bitcoin-0.19.1-riscv64-linux-gnu.tar.gz", +"5fcac9416e486d4960e1a946145566350ca670f9aaba99de6542080851122e4c": "bitcoin-0.19.1-x86_64-linux-gnu.tar.gz" +} + @contextlib.contextmanager def pushd(new_dir) -> None: previous_dir = os.getcwd() @@ -44,14 +78,10 @@ def download_binary(tag, args) -> int: match.group(1), match.group(2)) tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format( tag=tag[1:], platform=args.platform) - sha256Sums = "SHA256SUMS-{tag}.asc".format(tag=tag[1:]) tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format( bin_path=bin_path, tarball=tarball) - sha256SumsUrl = 'https://bitcoincore.org/{bin_path}/SHA256SUMS.asc'.format( - bin_path=bin_path) print('Fetching: {tarballUrl}'.format(tarballUrl=tarballUrl)) - print('Fetching: {sha256SumsUrl}'.format(sha256SumsUrl=sha256SumsUrl)) header, status = subprocess.Popen( ['curl', '--head', tarballUrl], stdout=subprocess.PIPE).communicate() @@ -60,8 +90,7 @@ def download_binary(tag, args) -> int: return 1 curlCmds = [ - ['curl', '--remote-name', tarballUrl], - ['curl', '--output', sha256Sums, sha256SumsUrl], + ['curl', '--remote-name', tarballUrl] ] for cmd in curlCmds: @@ -73,29 +102,12 @@ def download_binary(tag, args) -> int: with open(tarball, "rb") as afile: hasher.update(afile.read()) tarballHash = hasher.hexdigest() - tarballHash = '{} {}\n'.format(tarballHash, tarball) - with open(sha256Sums, 'r', encoding="utf-8") as afile: - shasums = afile.readlines() - if tarballHash not in shasums: + if tarballHash not in SHA256_SUMS or SHA256_SUMS[tarballHash] != tarball: print("Checksum did not match") - Path(tarball).unlink() return 1 print("Checksum matched") - # Bitcoin Core Release Signing Keys v0.11.0+ - signingKey = "01EA5486DE18A882D4C2684590C8019E36C2E964" - - isKeyPresent = subprocess.run( - ["gpg", "--list-keys", signingKey]).returncode - if isKeyPresent: - return isKeyPresent - - isVerified = subprocess.run( - ["gpg", "--verify", sha256Sums]).returncode - if isVerified: - return isVerified - # Extract tarball ret = subprocess.run(['tar', '-zxf', tarball, '-C', tag, '--strip-components=1', @@ -104,7 +116,6 @@ def download_binary(tag, args) -> int: return ret Path(tarball).unlink() - Path(sha256Sums).unlink() return 0 diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py index bd947d194..f77242d33 100755 --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -23,7 +23,7 @@ CMD_GREP_WALLET_ARGS = r"git grep --function-context 'void WalletInit::AddWallet CMD_GREP_WALLET_HIDDEN_ARGS = r"git grep --function-context 'void DummyWalletInit::AddWalletOptions' -- {}".format(CMD_ROOT_DIR) CMD_GREP_DOCS = r"git grep --perl-regexp '{}' {}".format(REGEX_DOC, CMD_ROOT_DIR) # list unsupported, deprecated and duplicate args as they need no documentation -SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb']) +SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb', '-zapwallettxes']) def lint_missing_argument_documentation(): diff --git a/test/lint/lint-cpp.sh b/test/lint/lint-cpp.sh new file mode 100755 index 000000000..cac57b968 --- /dev/null +++ b/test/lint/lint-cpp.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for various C++ code patterns we want to avoid. + +export LC_ALL=C + +EXIT_CODE=0 + +OUTPUT=$(git grep -E "boost::bind\(" -- "*.cpp" "*.h") +if [[ ${OUTPUT} != "" ]]; then + echo "Use of boost::bind detected. Use std::bind instead." + echo + echo "${OUTPUT}" + EXIT_CODE=1 +fi + +exit ${EXIT_CODE}
\ No newline at end of file diff --git a/test/lint/lint-git-commit-check.sh b/test/lint/lint-git-commit-check.sh index 7cffd267d..8947f67bf 100755 --- a/test/lint/lint-git-commit-check.sh +++ b/test/lint/lint-git-commit-check.sh @@ -14,21 +14,22 @@ while getopts "?" opt; do case $opt in ?) echo "Usage: $0 [N]" - echo " TRAVIS_COMMIT_RANGE='<commit range>' $0" + echo " COMMIT_RANGE='<commit range>' $0" echo " $0 -?" echo "Checks unmerged commits, the previous N commits, or a commit range." - echo "TRAVIS_COMMIT_RANGE='47ba2c3...ee50c9e' $0" + echo "COMMIT_RANGE='47ba2c3...ee50c9e' $0" exit ${EXIT_CODE} ;; esac done -if [ -z "${TRAVIS_COMMIT_RANGE}" ]; then - if [ -n "$1" ]; then - TRAVIS_COMMIT_RANGE="HEAD~$1...HEAD" - else - TRAVIS_COMMIT_RANGE="origin/master..HEAD" - fi +if [ -z "${COMMIT_RANGE}" ]; then + if [ -n "$1" ]; then + COMMIT_RANGE="HEAD~$1...HEAD" + else + MERGE_BASE=$(git merge-base HEAD master) + COMMIT_RANGE="$MERGE_BASE..HEAD" + fi fi while IFS= read -r commit_hash || [[ -n "$commit_hash" ]]; do @@ -41,6 +42,6 @@ while IFS= read -r commit_hash || [[ -n "$commit_hash" ]]; do EXIT_CODE=1 fi done < <(git log --format=%B -n 1 "$commit_hash") -done < <(git log "${TRAVIS_COMMIT_RANGE}" --format=%H) +done < <(git log "${COMMIT_RANGE}" --format=%H) exit ${EXIT_CODE} diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index 611bd4a8c..fde77aea2 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -63,6 +63,7 @@ EXPECTED_BOOST_INCLUDES=( boost/optional.hpp boost/preprocessor/cat.hpp boost/preprocessor/stringize.hpp + boost/process.hpp boost/signals2/connection.hpp boost/signals2/optional_last_value.hpp boost/signals2/signal.hpp diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index 2e5b80184..e5657f755 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -4,6 +4,39 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C + +# Be aware that bitcoind and bitcoin-qt differ in terms of localization: Qt +# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup, +# whereas no such call is made in bitcoind. +# +# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale +# specified by the user's LC_ALL (or LC_*) environment variable as the new +# C locale. +# +# In contrast, bitcoind does not opt in to localization -- no call to +# setlocale(LC_ALL, "") is made and the environment variables LC_* are +# thus ignored. +# +# This results in situations where bitcoind is guaranteed to be running +# with the classic locale ("C") whereas the locale of bitcoin-qt will vary +# depending on the user's environment variables. +# +# An example: Assuming the environment variable LC_ALL=de_DE then the +# call std::to_string(1.23) will return "1.230000" in bitcoind but +# "1,230000" in bitcoin-qt. +# +# From the Qt documentation: +# "On Unix/Linux Qt is configured to use the system locale settings by default. +# This can cause a conflict when using POSIX functions, for instance, when +# converting between data types such as floats and strings, since the notation +# may differ between locales. To get around this problem, call the POSIX function +# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication +# or QCoreApplication to reset the locale that is used for number formatting to +# "C"-locale." +# +# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and +# https://stackoverflow.com/a/34878283 for more details. + KNOWN_VIOLATIONS=( "src/bitcoin-tx.cpp.*stoul" "src/bitcoin-tx.cpp.*trim_right" diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 3d0ac7f99..625085c55 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -11,7 +11,7 @@ mutex:CConnman::ThreadOpenConnections mutex:CConnman::ThreadOpenAddedConnections mutex:CConnman::SocketHandler mutex:UpdateTip -mutex:PeerLogicValidation::UpdatedBlockTip +mutex:PeerManager::UpdatedBlockTip mutex:g_best_block_mutex # race (TODO fix) race:CConnman::WakeMessageHandler @@ -24,6 +24,7 @@ race:WalletBatch::WriteHDChain race:BerkeleyBatch race:BerkeleyDatabase race:DatabaseBatch +race:leveldb::DBImpl::DeleteObsoleteFiles race:zmq::* race:bitcoin-qt # deadlock (TODO fix) |