diff options
Diffstat (limited to 'src/validation.cpp')
| -rw-r--r-- | src/validation.cpp | 4994 |
1 files changed, 2972 insertions, 2022 deletions
diff --git a/src/validation.cpp b/src/validation.cpp index 4ce0723b2..396fc0a1b 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1,133 +1,157 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "validation.h" - -#include "arith_uint256.h" -#include "chainparams.h" -#include "checkpoints.h" -#include "checkqueue.h" -#include "consensus/consensus.h" -#include "consensus/merkle.h" -#include "consensus/validation.h" -#include "hash.h" -#include "init.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "pow.h" -#include "primitives/block.h" -#include "primitives/transaction.h" -#include "random.h" -#include "script/script.h" -#include "script/sigcache.h" -#include "script/standard.h" -#include "timedata.h" -#include "tinyformat.h" -#include "txdb.h" -#include "txmempool.h" -#include "ui_interface.h" -#include "undo.h" -#include "util.h" -#include "utilmoneystr.h" -#include "utilstrencodings.h" -#include "validationinterface.h" -#include "versionbits.h" -#include "warnings.h" - -#include <atomic> -#include <sstream> +#include <validation.h> + +#include <arith_uint256.h> +#include <chain.h> +#include <chainparams.h> +#include <checkqueue.h> +#include <consensus/consensus.h> +#include <consensus/merkle.h> +#include <consensus/tx_check.h> +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <cuckoocache.h> +#include <flatfile.h> +#include <hash.h> +#include <index/txindex.h> +#include <logging.h> +#include <logging/timer.h> +#include <optional.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <policy/settings.h> +#include <pow.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <random.h> +#include <reverse_iterator.h> +#include <script/script.h> +#include <script/sigcache.h> +#include <shutdown.h> +#include <timedata.h> +#include <tinyformat.h> +#include <txdb.h> +#include <txmempool.h> +#include <ui_interface.h> +#include <uint256.h> +#include <undo.h> +#include <util/moneystr.h> +#include <util/rbf.h> +#include <util/strencodings.h> +#include <util/system.h> +#include <util/translation.h> +#include <validationinterface.h> +#include <warnings.h> + +#include <string> #include <boost/algorithm/string/replace.hpp> -#include <boost/algorithm/string/join.hpp> -#include <boost/filesystem.hpp> -#include <boost/filesystem/fstream.hpp> -#include <boost/math/distributions/poisson.hpp> #include <boost/thread.hpp> #if defined(NDEBUG) # error "Bitcoin cannot be compiled without assertions." #endif +#define MICRO 0.000001 +#define MILLI 0.001 + /** - * Global state + * An extra transaction can be added to a package, as long as it only has one + * ancestor and is no larger than this. Not really any reason to make this + * configurable as it doesn't materially change DoS parameters. */ +static const unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT = 10000; +/** Maximum kilobytes for transactions to store for processing during reorg */ +static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; +/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ +static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB +/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ +static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB +/** Time to wait (in seconds) between writing blocks/block index to disk. */ +static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; +/** Time to wait (in seconds) between flushing chainstate to disk. */ +static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; +/** Maximum age of our tip in seconds for us to be considered current for fee estimation */ +static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60; + +bool CBlockIndexWorkComparator::operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { + // First sort by most total work, ... + if (pa->nChainWork > pb->nChainWork) return false; + if (pa->nChainWork < pb->nChainWork) return true; + + // ... then by earliest time received, ... + if (pa->nSequenceId < pb->nSequenceId) return false; + if (pa->nSequenceId > pb->nSequenceId) return true; + + // Use pointer address as tie breaker (should only happen with blocks + // loaded from disk, as those all have id 0). + if (pa < pb) return false; + if (pa > pb) return true; + + // Identical blocks. + return false; +} + +ChainstateManager g_chainman; + +CChainState& ChainstateActive() +{ + LOCK(::cs_main); + assert(g_chainman.m_active_chainstate); + return *g_chainman.m_active_chainstate; +} -CCriticalSection cs_main; +CChain& ChainActive() +{ + LOCK(::cs_main); + return ::ChainstateActive().m_chain; +} + +/** + * Mutex to guard access to validation specific variables, such as reading + * or changing the chainstate. + * + * This may also need to be locked when updating the transaction pool, e.g. on + * AcceptToMemoryPool. See CTxMemPool::cs comment for details. + * + * The transaction pool has a separate lock to allow reading from it and the + * chainstate at the same time. + */ +RecursiveMutex cs_main; -BlockMap mapBlockIndex; -CChain chainActive; -CBlockIndex *pindexBestHeader = NULL; -CWaitableCriticalSection csBestBlock; -CConditionVariable cvBlockChange; -int nScriptCheckThreads = 0; +CBlockIndex *pindexBestHeader = nullptr; +Mutex g_best_block_mutex; +std::condition_variable g_best_block_cv; +uint256 g_best_block; +bool g_parallel_script_checks{false}; std::atomic_bool fImporting(false); -bool fReindex = false; -bool fTxIndex = false; +std::atomic_bool fReindex(false); bool fHavePruned = false; bool fPruneMode = false; -bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; bool fRequireStandard = true; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; size_t nCoinCacheUsage = 5000 * 300; uint64_t nPruneTarget = 0; int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE; -bool fEnableReplacement = DEFAULT_ENABLE_REPLACEMENT; uint256 hashAssumeValid; +arith_uint256 nMinimumChainWork; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); -CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; -CTxMemPool mempool(::minRelayTxFee); - -static void CheckBlockIndex(const Consensus::Params& consensusParams); - -/** Constant stuff for coinbase transactions we create: */ -CScript COINBASE_FLAGS; - -const std::string strMessageMagic = "Bitcoin Signed Message:\n"; +CBlockPolicyEstimator feeEstimator; +CTxMemPool mempool(&feeEstimator); // Internal stuff namespace { + CBlockIndex* pindexBestInvalid = nullptr; - struct CBlockIndexWorkComparator - { - bool operator()(CBlockIndex *pa, CBlockIndex *pb) const { - // First sort by most total work, ... - if (pa->nChainWork > pb->nChainWork) return false; - if (pa->nChainWork < pb->nChainWork) return true; - - // ... then by earliest time received, ... - if (pa->nSequenceId < pb->nSequenceId) return false; - if (pa->nSequenceId > pb->nSequenceId) return true; - - // Use pointer address as tie breaker (should only happen with blocks - // loaded from disk, as those all have id 0). - if (pa < pb) return false; - if (pa > pb) return true; - - // Identical blocks. - return false; - } - }; - - CBlockIndex *pindexBestInvalid; - - /** - * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and - * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be - * missing the data for the block. - */ - std::set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates; - /** All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions. - * Pruned nodes may have entries where B is missing data. - */ - std::multimap<CBlockIndex*, CBlockIndex*> mapBlocksUnlinked; - - CCriticalSection cs_LastBlockFile; + RecursiveMutex cs_LastBlockFile; std::vector<CBlockFileInfo> vinfoBlockFile; int nLastBlockFile = 0; /** Global flag to indicate we should check to see if there are @@ -136,18 +160,6 @@ namespace { */ bool fCheckForPruning = false; - /** - * Every received block is assigned a unique and increasing identifier, so we - * know which one to give priority in case of a fork. - */ - CCriticalSection cs_nBlockSequenceId; - /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ - int32_t nBlockSequenceId = 1; - /** Decreasing counter (used by subsequent preciousblock calls). */ - int32_t nBlockReverseSequenceId = -1; - /** chainwork for the last block that preciousblock has been applied to. */ - arith_uint256 nLastPreciousChainwork = 0; - /** Dirty block index entries. */ std::set<CBlockIndex*> setDirtyBlockIndex; @@ -155,47 +167,22 @@ namespace { std::set<int> setDirtyFileInfo; } // anon namespace -/* Use this class to start tracking transactions that are removed from the - * mempool and pass all those transactions through SyncTransaction when the - * object goes out of scope. This is currently only used to call SyncTransaction - * on conflicts removed from the mempool during block connection. Applied in - * ActivateBestChain around ActivateBestStep which in turn calls: - * ConnectTip->removeForBlock->removeConflicts - */ -class MemPoolConflictRemovalTracker +CBlockIndex* LookupBlockIndex(const uint256& hash) { -private: - std::vector<CTransactionRef> conflictedTxs; - CTxMemPool &pool; - -public: - MemPoolConflictRemovalTracker(CTxMemPool &_pool) : pool(_pool) { - pool.NotifyEntryRemoved.connect(boost::bind(&MemPoolConflictRemovalTracker::NotifyEntryRemoved, this, _1, _2)); - } - - void NotifyEntryRemoved(CTransactionRef txRemoved, MemPoolRemovalReason reason) { - if (reason == MemPoolRemovalReason::CONFLICT) { - conflictedTxs.push_back(txRemoved); - } - } - - ~MemPoolConflictRemovalTracker() { - pool.NotifyEntryRemoved.disconnect(boost::bind(&MemPoolConflictRemovalTracker::NotifyEntryRemoved, this, _1, _2)); - for (const auto& tx : conflictedTxs) { - GetMainSignals().SyncTransaction(*tx, NULL, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); - } - conflictedTxs.clear(); - } -}; + AssertLockHeld(cs_main); + BlockMap::const_iterator it = g_chainman.BlockIndex().find(hash); + return it == g_chainman.BlockIndex().end() ? nullptr : it->second; +} CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) { - // Find the first block the caller has in the main chain - BOOST_FOREACH(const uint256& hash, locator.vHave) { - BlockMap::iterator mi = mapBlockIndex.find(hash); - if (mi != mapBlockIndex.end()) - { - CBlockIndex* pindex = (*mi).second; + AssertLockHeld(cs_main); + + // Find the latest block common to locator and chain - we expect that + // locator.vHave is sorted descending by height. + for (const uint256& hash : locator.vHave) { + CBlockIndex* pindex = LookupBlockIndex(hash); + if (pindex) { if (chain.Contains(pindex)) return pindex; if (pindex->GetAncestor(chain.Height()) == chain.Tip()) { @@ -206,32 +193,15 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc return chain.Genesis(); } -CCoinsViewCache *pcoinsTip = NULL; -CBlockTreeDB *pblocktree = NULL; - -enum FlushStateMode { - FLUSH_STATE_NONE, - FLUSH_STATE_IF_NEEDED, - FLUSH_STATE_PERIODIC, - FLUSH_STATE_ALWAYS -}; +std::unique_ptr<CBlockTreeDB> pblocktree; // See definition for documentation -bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int nManualPruneHeight=0); -void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight); - -bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) -{ - if (tx.nLockTime == 0) - return true; - if ((int64_t)tx.nLockTime < ((int64_t)tx.nLockTime < LOCKTIME_THRESHOLD ? (int64_t)nBlockHeight : nBlockTime)) - return true; - for (const auto& txin : tx.vin) { - if (!(txin.nSequence == CTxIn::SEQUENCE_FINAL)) - return false; - } - return true; -} +static void FindFilesToPruneManual(ChainstateManager& chainman, std::set<int>& setFilesToPrune, int nManualPruneHeight); +static void FindFilesToPrune(ChainstateManager& chainman, std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight); +bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); +static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); +static FlatFileSeq BlockFileSeq(); +static FlatFileSeq UndoFileSeq(); bool CheckFinalTx(const CTransaction &tx, int flags) { @@ -245,109 +215,26 @@ bool CheckFinalTx(const CTransaction &tx, int flags) // scheduled, so no flags are set. flags = std::max(flags, 0); - // CheckFinalTx() uses chainActive.Height()+1 to evaluate + // CheckFinalTx() uses ::ChainActive().Height()+1 to evaluate // nLockTime because when IsFinalTx() is called within // CBlock::AcceptBlock(), the height of the block *being* // evaluated is what is used. Thus if we want to know if a // transaction can be part of the *next* block, we need to call - // IsFinalTx() with one more than chainActive.Height(). - const int nBlockHeight = chainActive.Height() + 1; + // IsFinalTx() with one more than ::ChainActive().Height(). + const int nBlockHeight = ::ChainActive().Height() + 1; - // BIP113 will require that time-locked transactions have nLockTime set to + // BIP113 requires that time-locked transactions have nLockTime set to // less than the median time of the previous block they're contained in. // When the next block is created its previous block will be the current // chain tip, so we use that to calculate the median time passed to // IsFinalTx() if LOCKTIME_MEDIAN_TIME_PAST is set. const int64_t nBlockTime = (flags & LOCKTIME_MEDIAN_TIME_PAST) - ? chainActive.Tip()->GetMedianTimePast() + ? ::ChainActive().Tip()->GetMedianTimePast() : GetAdjustedTime(); return IsFinalTx(tx, nBlockHeight, nBlockTime); } -/** - * Calculates the block height and previous block's median time past at - * which the transaction will be considered final in the context of BIP 68. - * Also removes from the vector of input heights any entries which did not - * correspond to sequence locked inputs as they do not affect the calculation. - */ -static std::pair<int, int64_t> CalculateSequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) -{ - assert(prevHeights->size() == tx.vin.size()); - - // Will be set to the equivalent height- and time-based nLockTime - // values that would be necessary to satisfy all relative lock- - // time constraints given our view of block chain history. - // The semantics of nLockTime are the last invalid height/time, so - // use -1 to have the effect of any height or time being valid. - int nMinHeight = -1; - int64_t nMinTime = -1; - - // tx.nVersion is signed integer so requires cast to unsigned otherwise - // we would be doing a signed comparison and half the range of nVersion - // wouldn't support BIP 68. - bool fEnforceBIP68 = static_cast<uint32_t>(tx.nVersion) >= 2 - && flags & LOCKTIME_VERIFY_SEQUENCE; - - // Do not enforce sequence numbers as a relative lock time - // unless we have been instructed to - if (!fEnforceBIP68) { - return std::make_pair(nMinHeight, nMinTime); - } - - for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { - const CTxIn& txin = tx.vin[txinIndex]; - - // Sequence numbers with the most significant bit set are not - // treated as relative lock-times, nor are they given any - // consensus-enforced meaning at this point. - if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_DISABLE_FLAG) { - // The height of this input is not relevant for sequence locks - (*prevHeights)[txinIndex] = 0; - continue; - } - - int nCoinHeight = (*prevHeights)[txinIndex]; - - if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG) { - int64_t nCoinTime = block.GetAncestor(std::max(nCoinHeight-1, 0))->GetMedianTimePast(); - // NOTE: Subtract 1 to maintain nLockTime semantics - // BIP 68 relative lock times have the semantics of calculating - // the first block or time at which the transaction would be - // valid. When calculating the effective block time or height - // for the entire transaction, we switch to using the - // semantics of nLockTime which is the last invalid block - // time or height. Thus we subtract 1 from the calculated - // time or height. - - // Time-based relative lock-times are measured from the - // smallest allowed timestamp of the block containing the - // txout being spent, which is the median time past of the - // block prior. - nMinTime = std::max(nMinTime, nCoinTime + (int64_t)((txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) << CTxIn::SEQUENCE_LOCKTIME_GRANULARITY) - 1); - } else { - nMinHeight = std::max(nMinHeight, nCoinHeight + (int)(txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) - 1); - } - } - - return std::make_pair(nMinHeight, nMinTime); -} - -static bool EvaluateSequenceLocks(const CBlockIndex& block, std::pair<int, int64_t> lockPair) -{ - assert(block.pprev); - int64_t nBlockTime = block.pprev->GetMedianTimePast(); - if (lockPair.first >= block.nHeight || lockPair.second >= nBlockTime) - return false; - - return true; -} - -bool SequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) -{ - return EvaluateSequenceLocks(block, CalculateSequenceLocks(tx, flags, prevHeights, block)); -} - bool TestLockPointValidity(const LockPoints* lp) { AssertLockHeld(cs_main); @@ -355,9 +242,9 @@ bool TestLockPointValidity(const LockPoints* lp) // If there are relative lock times then the maxInputBlock will be set // If there are no relative lock times, the LockPoints don't depend on the chain if (lp->maxInputBlock) { - // Check whether chainActive is an extension of the block at which the LockPoints + // Check whether ::ChainActive() is an extension of the block at which the LockPoints // calculation was valid. If not LockPoints are no longer valid - if (!chainActive.Contains(lp->maxInputBlock)) { + if (!::ChainActive().Contains(lp->maxInputBlock)) { return false; } } @@ -366,20 +253,22 @@ bool TestLockPointValidity(const LockPoints* lp) return true; } -bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool useExistingLockPoints) +bool CheckSequenceLocks(const CTxMemPool& pool, const CTransaction& tx, int flags, LockPoints* lp, bool useExistingLockPoints) { AssertLockHeld(cs_main); - AssertLockHeld(mempool.cs); + AssertLockHeld(pool.cs); + + CBlockIndex* tip = ::ChainActive().Tip(); + assert(tip != nullptr); - CBlockIndex* tip = chainActive.Tip(); CBlockIndex index; index.pprev = tip; - // CheckSequenceLocks() uses chainActive.Height()+1 to evaluate + // CheckSequenceLocks() uses ::ChainActive().Height()+1 to evaluate // height based locks because when SequenceLocks() is called within // ConnectBlock(), the height of the block *being* // evaluated is what is used. // Thus if we want to know if a transaction can be part of the - // *next* block, we need to use one more than chainActive.Height() + // *next* block, we need to use one more than ::ChainActive().Height() index.nHeight = tip->nHeight + 1; std::pair<int, int64_t> lockPair; @@ -389,21 +278,21 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool lockPair.second = lp->time; } else { - // pcoinsTip contains the UTXO set for chainActive.Tip() - CCoinsViewMemPool viewMemPool(pcoinsTip, mempool); + // CoinsTip() contains the UTXO set for ::ChainActive().Tip() + CCoinsViewMemPool viewMemPool(&::ChainstateActive().CoinsTip(), pool); std::vector<int> prevheights; prevheights.resize(tx.vin.size()); for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { const CTxIn& txin = tx.vin[txinIndex]; - CCoins coins; - if (!viewMemPool.GetCoins(txin.prevout.hash, coins)) { + Coin coin; + if (!viewMemPool.GetCoin(txin.prevout, coin)) { return error("%s: Missing input", __func__); } - if (coins.nHeight == MEMPOOL_HEIGHT) { + if (coin.nHeight == MEMPOOL_HEIGHT) { // Assume all mempool transaction confirm in the next block prevheights[txinIndex] = tip->nHeight + 1; } else { - prevheights[txinIndex] = coins.nHeight; + prevheights[txinIndex] = coin.nHeight; } } lockPair = CalculateSequenceLocks(tx, flags, &prevheights, index); @@ -424,7 +313,7 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool // lock on a mempool input, so we can use the return value of // CheckSequenceLocks to indicate the LockPoints validity int maxInputHeight = 0; - BOOST_FOREACH(int height, prevheights) { + for (const int height : prevheights) { // Can ignore mempool inputs since we'll fail if they had non-zero locks if (height != tip->nHeight+1) { maxInputHeight = std::max(maxInputHeight, height); @@ -436,194 +325,291 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool return EvaluateSequenceLocks(index, lockPair); } +// Returns the script flags which should be checked for a given block +static unsigned int GetBlockScriptFlags(const CBlockIndex* pindex, const Consensus::Params& chainparams); -unsigned int GetLegacySigOpCount(const CTransaction& tx) +static void LimitMempoolSize(CTxMemPool& pool, size_t limit, std::chrono::seconds age) + EXCLUSIVE_LOCKS_REQUIRED(pool.cs, ::cs_main) { - unsigned int nSigOps = 0; - for (const auto& txin : tx.vin) - { - nSigOps += txin.scriptSig.GetSigOpCount(false); - } - for (const auto& txout : tx.vout) - { - nSigOps += txout.scriptPubKey.GetSigOpCount(false); + int expired = pool.Expire(GetTime<std::chrono::seconds>() - age); + if (expired != 0) { + LogPrint(BCLog::MEMPOOL, "Expired %i transactions from the memory pool\n", expired); } - return nSigOps; + + std::vector<COutPoint> vNoSpendsRemaining; + pool.TrimToSize(limit, &vNoSpendsRemaining); + for (const COutPoint& removed : vNoSpendsRemaining) + ::ChainstateActive().CoinsTip().Uncache(removed); } -unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& inputs) +static bool IsCurrentForFeeEstimation() EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - if (tx.IsCoinBase()) - return 0; - - unsigned int nSigOps = 0; - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); - if (prevout.scriptPubKey.IsPayToScriptHash()) - nSigOps += prevout.scriptPubKey.GetSigOpCount(tx.vin[i].scriptSig); - } - return nSigOps; + AssertLockHeld(cs_main); + if (::ChainstateActive().IsInitialBlockDownload()) + return false; + if (::ChainActive().Tip()->GetBlockTime() < (GetTime() - MAX_FEE_ESTIMATION_TIP_AGE)) + return false; + if (::ChainActive().Height() < pindexBestHeader->nHeight - 1) + return false; + return true; } -int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags) +/* Make mempool consistent after a reorg, by re-adding or recursively erasing + * disconnected block transactions from the mempool, and also removing any + * other transactions from the mempool that are no longer valid given the new + * tip/height. + * + * Note: we assume that disconnectpool only contains transactions that are NOT + * confirmed in the current chain nor already in the mempool (otherwise, + * in-mempool descendants of such transactions would be removed). + * + * Passing fAddToMempool=false will skip trying to add the transactions back, + * and instead just erase from the mempool as needed. + */ + +static void UpdateMempoolForReorg(DisconnectedBlockTransactions& disconnectpool, bool fAddToMempool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs) { - int64_t nSigOps = GetLegacySigOpCount(tx) * WITNESS_SCALE_FACTOR; + AssertLockHeld(cs_main); + std::vector<uint256> vHashUpdate; + // disconnectpool's insertion_order index sorts the entries from + // oldest to newest, but the oldest entry will be the last tx from the + // latest mined block that was disconnected. + // Iterate disconnectpool in reverse, so that we add transactions + // back to the mempool starting with the earliest transaction that had + // been previously seen in a block. + auto it = disconnectpool.queuedTx.get<insertion_order>().rbegin(); + while (it != disconnectpool.queuedTx.get<insertion_order>().rend()) { + // ignore validation errors in resurrected transactions + TxValidationState stateDummy; + if (!fAddToMempool || (*it)->IsCoinBase() || + !AcceptToMemoryPool(mempool, stateDummy, *it, + nullptr /* plTxnReplaced */, true /* bypass_limits */, 0 /* nAbsurdFee */)) { + // If the transaction doesn't make it in to the mempool, remove any + // transactions that depend on it (which would now be orphans). + mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + } else if (mempool.exists((*it)->GetHash())) { + vHashUpdate.push_back((*it)->GetHash()); + } + ++it; + } + disconnectpool.queuedTx.clear(); + // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have + // no in-mempool children, which is generally not true when adding + // previously-confirmed transactions back to the mempool. + // UpdateTransactionsFromBlock finds descendants of any transactions in + // the disconnectpool that were added back and cleans up the mempool state. + mempool.UpdateTransactionsFromBlock(vHashUpdate); + + // We also need to remove any now-immature transactions + mempool.removeForReorg(&::ChainstateActive().CoinsTip(), ::ChainActive().Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + // Re-limit mempool size, in case we added any transactions + LimitMempoolSize(mempool, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); +} - if (tx.IsCoinBase()) - return nSigOps; +// Used to avoid mempool polluting consensus critical paths if CCoinsViewMempool +// were somehow broken and returning the wrong scriptPubKeys +static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, TxValidationState& state, const CCoinsViewCache& view, const CTxMemPool& pool, + unsigned int flags, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { + AssertLockHeld(cs_main); - if (flags & SCRIPT_VERIFY_P2SH) { - nSigOps += GetP2SHSigOpCount(tx, inputs) * WITNESS_SCALE_FACTOR; + // pool.cs should be locked already, but go ahead and re-take the lock here + // to enforce that mempool doesn't change between when we check the view + // and when we actually call through to CheckInputScripts + LOCK(pool.cs); + + assert(!tx.IsCoinBase()); + for (const CTxIn& txin : tx.vin) { + const Coin& coin = view.AccessCoin(txin.prevout); + + // AcceptToMemoryPoolWorker has already checked that the coins are + // available, so this shouldn't fail. If the inputs are not available + // here then return false. + if (coin.IsSpent()) return false; + + // Check equivalence for available inputs. + const CTransactionRef& txFrom = pool.get(txin.prevout.hash); + if (txFrom) { + assert(txFrom->GetHash() == txin.prevout.hash); + assert(txFrom->vout.size() > txin.prevout.n); + assert(txFrom->vout[txin.prevout.n] == coin.out); + } else { + const Coin& coinFromDisk = ::ChainstateActive().CoinsTip().AccessCoin(txin.prevout); + assert(!coinFromDisk.IsSpent()); + assert(coinFromDisk.out == coin.out); + } } - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); - nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, &tx.vin[i].scriptWitness, flags); - } - return nSigOps; + // Call CheckInputScripts() to cache signature and script validity against current tip consensus rules. + return CheckInputScripts(tx, state, view, flags, /* cacheSigStore = */ true, /* cacheFullSciptStore = */ true, txdata); } +namespace { +class MemPoolAccept +{ +public: + MemPoolAccept(CTxMemPool& mempool) : m_pool(mempool), m_view(&m_dummy), m_viewmempool(&::ChainstateActive().CoinsTip(), m_pool), + m_limit_ancestors(gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT)), + m_limit_ancestor_size(gArgs.GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000), + m_limit_descendants(gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT)), + m_limit_descendant_size(gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000) {} + + // We put the arguments we're handed into a struct, so we can pass them + // around easier. + struct ATMPArgs { + const CChainParams& m_chainparams; + TxValidationState &m_state; + const int64_t m_accept_time; + std::list<CTransactionRef>* m_replaced_transactions; + const bool m_bypass_limits; + const CAmount& m_absurd_fee; + /* + * Return any outpoints which were not previously present in the coins + * cache, but were added as a result of validating the tx for mempool + * acceptance. This allows the caller to optionally remove the cache + * additions if the associated transaction ends up being rejected by + * the mempool. + */ + std::vector<COutPoint>& m_coins_to_uncache; + const bool m_test_accept; + }; + // Single transaction acceptance + bool AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs& args) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +private: + // All the intermediate state that gets passed between the various levels + // of checking a given transaction. + struct Workspace { + Workspace(const CTransactionRef& ptx) : m_ptx(ptx), m_hash(ptx->GetHash()) {} + std::set<uint256> m_conflicts; + CTxMemPool::setEntries m_all_conflicting; + CTxMemPool::setEntries m_ancestors; + std::unique_ptr<CTxMemPoolEntry> m_entry; + + bool m_replacement_transaction; + CAmount m_modified_fees; + CAmount m_conflicting_fees; + size_t m_conflicting_size; + + const CTransactionRef& m_ptx; + const uint256& m_hash; + }; -bool CheckTransaction(const CTransaction& tx, CValidationState &state, bool fCheckDuplicateInputs) -{ - // Basic checks that don't depend on any context - if (tx.vin.empty()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-vin-empty"); - if (tx.vout.empty()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-vout-empty"); - // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) - if (::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) > MAX_BLOCK_BASE_SIZE) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-oversize"); - - // Check for negative or overflow output values - CAmount nValueOut = 0; - for (const auto& txout : tx.vout) + // Run the policy checks on a given transaction, excluding any script checks. + // Looks up inputs, calculates feerate, considers replacement, evaluates + // package limits, etc. As this function can be invoked for "free" by a peer, + // only tests that are fast should be done here (to avoid CPU DoS). + bool PreChecks(ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + + // Run the script checks using our policy flags. As this can be slow, we should + // only invoke this on transactions that have otherwise passed policy checks. + bool PolicyScriptChecks(ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + // Re-run the script checks, using consensus flags, and try to cache the + // result in the scriptcache. This should be done after + // PolicyScriptChecks(). This requires that all inputs either be in our + // utxo set or in the mempool. + bool ConsensusScriptChecks(ATMPArgs& args, Workspace& ws, PrecomputedTransactionData &txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + // Try to add the transaction to the mempool, removing any conflicts first. + // Returns true if the transaction is in the mempool after any size + // limiting is performed, false otherwise. + bool Finalize(ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + + // Compare a package's feerate against minimum allowed. + bool CheckFeeRate(size_t package_size, CAmount package_fee, TxValidationState& state) { - if (txout.nValue < 0) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-negative"); - if (txout.nValue > MAX_MONEY) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-toolarge"); - nValueOut += txout.nValue; - if (!MoneyRange(nValueOut)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-txouttotal-toolarge"); - } - - // Check for duplicate inputs - note that this check is slow so we skip it in CheckBlock - if (fCheckDuplicateInputs) { - std::set<COutPoint> vInOutPoints; - for (const auto& txin : tx.vin) - { - if (!vInOutPoints.insert(txin.prevout).second) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); + CAmount mempoolRejectFee = m_pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(package_size); + if (mempoolRejectFee > 0 && package_fee < mempoolRejectFee) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); } - } - if (tx.IsCoinBase()) - { - if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) - return state.DoS(100, false, REJECT_INVALID, "bad-cb-length"); - } - else - { - for (const auto& txin : tx.vin) - if (txin.prevout.IsNull()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-prevout-null"); + if (package_fee < ::minRelayTxFee.GetFee(package_size)) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "min relay fee not met", strprintf("%d < %d", package_fee, ::minRelayTxFee.GetFee(package_size))); + } + return true; } - return true; -} - -void LimitMempoolSize(CTxMemPool& pool, size_t limit, unsigned long age) { - int expired = pool.Expire(GetTime() - age); - if (expired != 0) - LogPrint("mempool", "Expired %i transactions from the memory pool\n", expired); - - std::vector<uint256> vNoSpendsRemaining; - pool.TrimToSize(limit, &vNoSpendsRemaining); - BOOST_FOREACH(const uint256& removed, vNoSpendsRemaining) - pcoinsTip->Uncache(removed); -} - -/** Convert CValidationState to a human-readable message for logging */ -std::string FormatStateMessage(const CValidationState &state) -{ - return strprintf("%s%s (code %i)", - state.GetRejectReason(), - state.GetDebugMessage().empty() ? "" : ", "+state.GetDebugMessage(), - state.GetRejectCode()); -} - -static bool IsCurrentForFeeEstimation() -{ - AssertLockHeld(cs_main); - if (IsInitialBlockDownload()) - return false; - if (chainActive.Tip()->GetBlockTime() < (GetTime() - MAX_FEE_ESTIMATION_TIP_AGE)) - return false; - if (chainActive.Height() < pindexBestHeader->nHeight - 1) - return false; - return true; -} +private: + CTxMemPool& m_pool; + CCoinsViewCache m_view; + CCoinsViewMemPool m_viewmempool; + CCoinsView m_dummy; + + // The package limits in effect at the time of invocation. + const size_t m_limit_ancestors; + const size_t m_limit_ancestor_size; + // These may be modified while evaluating a transaction (eg to account for + // in-mempool conflicts; see below). + size_t m_limit_descendants; + size_t m_limit_descendant_size; +}; -bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransactionRef& ptx, bool fLimitFree, - bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, - bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<uint256>& vHashTxnToUncache) +bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) { - const CTransaction& tx = *ptx; - const uint256 hash = tx.GetHash(); - AssertLockHeld(cs_main); - if (pfMissingInputs) - *pfMissingInputs = false; + const CTransactionRef& ptx = ws.m_ptx; + const CTransaction& tx = *ws.m_ptx; + const uint256& hash = ws.m_hash; + + // Copy/alias what we need out of args + TxValidationState &state = args.m_state; + const int64_t nAcceptTime = args.m_accept_time; + const bool bypass_limits = args.m_bypass_limits; + const CAmount& nAbsurdFee = args.m_absurd_fee; + std::vector<COutPoint>& coins_to_uncache = args.m_coins_to_uncache; + + // Alias what we need out of ws + std::set<uint256>& setConflicts = ws.m_conflicts; + CTxMemPool::setEntries& allConflicting = ws.m_all_conflicting; + CTxMemPool::setEntries& setAncestors = ws.m_ancestors; + std::unique_ptr<CTxMemPoolEntry>& entry = ws.m_entry; + bool& fReplacementTransaction = ws.m_replacement_transaction; + CAmount& nModifiedFees = ws.m_modified_fees; + CAmount& nConflictingFees = ws.m_conflicting_fees; + size_t& nConflictingSize = ws.m_conflicting_size; if (!CheckTransaction(tx, state)) return false; // state filled in by CheckTransaction // Coinbase is only valid in a block, not as a loose transaction if (tx.IsCoinBase()) - return state.DoS(100, false, REJECT_INVALID, "coinbase"); - - // Reject transactions with witness before segregated witness activates (override with -prematurewitness) - bool witnessEnabled = IsWitnessEnabled(chainActive.Tip(), Params().GetConsensus()); - if (!GetBoolArg("-prematurewitness",false) && tx.HasWitness() && !witnessEnabled) { - return state.DoS(0, false, REJECT_NONSTANDARD, "no-witness-yet", true); - } + return state.Invalid(TxValidationResult::TX_CONSENSUS, "coinbase"); // Rather not work on nonstandard transactions (unless -testnet/-regtest) std::string reason; - if (fRequireStandard && !IsStandardTx(tx, reason, witnessEnabled)) - return state.DoS(0, false, REJECT_NONSTANDARD, reason); + if (fRequireStandard && !IsStandardTx(tx, reason)) + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, reason); + + // Do not work on transactions that are too small. + // A transaction with 1 segwit input and 1 P2WPHK output has non-witness size of 82 bytes. + // Transactions smaller than this are not relayed to mitigate CVE-2017-12842 by not relaying + // 64-byte transactions. + if (::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) < MIN_STANDARD_TX_NONWITNESS_SIZE) + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "tx-size-small"); // Only accept nLockTime-using transactions that can be mined in the next // block; we don't want our mempool filled up with transactions that can't // be mined yet. if (!CheckFinalTx(tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) - return state.DoS(0, false, REJECT_NONSTANDARD, "non-final"); + return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-final"); // is it already in the memory pool? - if (pool.exists(hash)) - return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-in-mempool"); + if (m_pool.exists(hash)) { + return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-already-in-mempool"); + } // Check for conflicts with in-memory transactions - std::set<uint256> setConflicts; + for (const CTxIn &txin : tx.vin) { - LOCK(pool.cs); // protect pool.mapNextTx - BOOST_FOREACH(const CTxIn &txin, tx.vin) - { - auto itConflicting = pool.mapNextTx.find(txin.prevout); - if (itConflicting != pool.mapNextTx.end()) - { - const CTransaction *ptxConflicting = itConflicting->second; + const CTransaction* ptxConflicting = m_pool.GetConflictTx(txin.prevout); + if (ptxConflicting) { if (!setConflicts.count(ptxConflicting->GetHash())) { // Allow opt-out of transaction replacement by setting - // nSequence >= maxint-1 on all inputs. + // nSequence > MAX_BIP125_RBF_SEQUENCE (SEQUENCE_FINAL-2) on all inputs. // - // maxint-1 is picked to still allow use of nLockTime by + // SEQUENCE_FINAL-1 is picked to still allow use of nLockTime by // non-replaceable transactions. All inputs rather than just one // is for the sake of multi-party protocols, where we don't // want a single party to be able to disable replacement. @@ -633,462 +619,496 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C // unconfirmed ancestors anyway; doing otherwise is hopelessly // insecure. bool fReplacementOptOut = true; - if (fEnableReplacement) + for (const CTxIn &_txin : ptxConflicting->vin) { - BOOST_FOREACH(const CTxIn &_txin, ptxConflicting->vin) + if (_txin.nSequence <= MAX_BIP125_RBF_SEQUENCE) { - if (_txin.nSequence < std::numeric_limits<unsigned int>::max()-1) - { - fReplacementOptOut = false; - break; - } + fReplacementOptOut = false; + break; } } - if (fReplacementOptOut) - return state.Invalid(false, REJECT_CONFLICT, "txn-mempool-conflict"); + if (fReplacementOptOut) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "txn-mempool-conflict"); + } setConflicts.insert(ptxConflicting->GetHash()); } } } - } - { - CCoinsView dummy; - CCoinsViewCache view(&dummy); + LockPoints lp; + m_view.SetBackend(m_viewmempool); - CAmount nValueIn = 0; - LockPoints lp; - { - LOCK(pool.cs); - CCoinsViewMemPool viewMemPool(pcoinsTip, pool); - view.SetBackend(viewMemPool); - - // do we already have it? - bool fHadTxInCache = pcoinsTip->HaveCoinsInCache(hash); - if (view.HaveCoins(hash)) { - if (!fHadTxInCache) - vHashTxnToUncache.push_back(hash); - return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); - } - - // do all inputs exist? - // Note that this does not check for the presence of actual outputs (see the next check for that), - // and only helps with filling in pfMissingInputs (to determine missing vs spent). - BOOST_FOREACH(const CTxIn txin, tx.vin) { - if (!pcoinsTip->HaveCoinsInCache(txin.prevout.hash)) - vHashTxnToUncache.push_back(txin.prevout.hash); - if (!view.HaveCoins(txin.prevout.hash)) { - if (pfMissingInputs) - *pfMissingInputs = true; - return false; // fMissingInputs and !state.IsInvalid() is used to detect this condition, don't set state.Invalid() - } + CCoinsViewCache& coins_cache = ::ChainstateActive().CoinsTip(); + // do all inputs exist? + for (const CTxIn& txin : tx.vin) { + if (!coins_cache.HaveCoinInCache(txin.prevout)) { + coins_to_uncache.push_back(txin.prevout); } - // are the actual inputs available? - if (!view.HaveInputs(tx)) - return state.Invalid(false, REJECT_DUPLICATE, "bad-txns-inputs-spent"); - - // Bring the best block into scope - view.GetBestBlock(); + // Note: this call may add txin.prevout to the coins cache + // (coins_cache.cacheCoins) by way of FetchCoin(). It should be removed + // later (via coins_to_uncache) if this tx turns out to be invalid. + if (!m_view.HaveCoin(txin.prevout)) { + // Are inputs missing because we already have the tx? + for (size_t out = 0; out < tx.vout.size(); out++) { + // Optimistically just do efficient check of cache for outputs + if (coins_cache.HaveCoinInCache(COutPoint(hash, out))) { + return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-already-known"); + } + } + // Otherwise assume this might be an orphan tx for which we just haven't seen parents yet + return state.Invalid(TxValidationResult::TX_MISSING_INPUTS, "bad-txns-inputs-missingorspent"); + } + } - nValueIn = view.GetValueIn(tx); + // Bring the best block into scope + m_view.GetBestBlock(); - // we have all inputs cached now, so switch back to dummy, so we don't need to keep lock on mempool - view.SetBackend(dummy); + // we have all inputs cached now, so switch back to dummy (to protect + // against bugs where we pull more inputs from disk that miss being added + // to coins_to_uncache) + m_view.SetBackend(m_dummy); - // Only accept BIP68 sequence locked transactions that can be mined in the next - // block; we don't want our mempool filled up with transactions that can't - // be mined yet. - // Must keep pool.cs for this unless we change CheckSequenceLocks to take a - // CoinsViewCache instead of create its own - if (!CheckSequenceLocks(tx, STANDARD_LOCKTIME_VERIFY_FLAGS, &lp)) - return state.DoS(0, false, REJECT_NONSTANDARD, "non-BIP68-final"); - } + // Only accept BIP68 sequence locked transactions that can be mined in the next + // block; we don't want our mempool filled up with transactions that can't + // be mined yet. + // Must keep pool.cs for this unless we change CheckSequenceLocks to take a + // CoinsViewCache instead of create its own + if (!CheckSequenceLocks(m_pool, tx, STANDARD_LOCKTIME_VERIFY_FLAGS, &lp)) + return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-BIP68-final"); - // Check for non-standard pay-to-script-hash in inputs - if (fRequireStandard && !AreInputsStandard(tx, view)) - return state.Invalid(false, REJECT_NONSTANDARD, "bad-txns-nonstandard-inputs"); + CAmount nFees = 0; + if (!Consensus::CheckTxInputs(tx, state, m_view, GetSpendHeight(m_view), nFees)) { + return error("%s: Consensus::CheckTxInputs: %s, %s", __func__, tx.GetHash().ToString(), state.ToString()); + } - // Check for non-standard witness in P2WSH - if (tx.HasWitness() && fRequireStandard && !IsWitnessStandard(tx, view)) - return state.DoS(0, false, REJECT_NONSTANDARD, "bad-witness-nonstandard", true); + // Check for non-standard pay-to-script-hash in inputs + if (fRequireStandard && !AreInputsStandard(tx, m_view)) + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "bad-txns-nonstandard-inputs"); - int64_t nSigOpsCost = GetTransactionSigOpCost(tx, view, STANDARD_SCRIPT_VERIFY_FLAGS); + // Check for non-standard witness in P2WSH + if (tx.HasWitness() && fRequireStandard && !IsWitnessStandard(tx, m_view)) + return state.Invalid(TxValidationResult::TX_WITNESS_MUTATED, "bad-witness-nonstandard"); - CAmount nValueOut = tx.GetValueOut(); - CAmount nFees = nValueIn-nValueOut; - // nModifiedFees includes any fee deltas from PrioritiseTransaction - CAmount nModifiedFees = nFees; - double nPriorityDummy = 0; - pool.ApplyDeltas(hash, nPriorityDummy, nModifiedFees); + int64_t nSigOpsCost = GetTransactionSigOpCost(tx, m_view, STANDARD_SCRIPT_VERIFY_FLAGS); - CAmount inChainInputValue; - double dPriority = view.GetPriority(tx, chainActive.Height(), inChainInputValue); + // nModifiedFees includes any fee deltas from PrioritiseTransaction + nModifiedFees = nFees; + m_pool.ApplyDelta(hash, nModifiedFees); - // Keep track of transactions that spend a coinbase, which we re-scan - // during reorgs to ensure COINBASE_MATURITY is still met. - bool fSpendsCoinbase = false; - BOOST_FOREACH(const CTxIn &txin, tx.vin) { - const CCoins *coins = view.AccessCoins(txin.prevout.hash); - if (coins->IsCoinBase()) { - fSpendsCoinbase = true; - break; - } + // Keep track of transactions that spend a coinbase, which we re-scan + // during reorgs to ensure COINBASE_MATURITY is still met. + bool fSpendsCoinbase = false; + for (const CTxIn &txin : tx.vin) { + const Coin &coin = m_view.AccessCoin(txin.prevout); + if (coin.IsCoinBase()) { + fSpendsCoinbase = true; + break; } + } - CTxMemPoolEntry entry(ptx, nFees, nAcceptTime, dPriority, chainActive.Height(), - inChainInputValue, fSpendsCoinbase, nSigOpsCost, lp); - unsigned int nSize = entry.GetTxSize(); + entry.reset(new CTxMemPoolEntry(ptx, nFees, nAcceptTime, ::ChainActive().Height(), + fSpendsCoinbase, nSigOpsCost, lp)); + unsigned int nSize = entry->GetTxSize(); - // Check that the transaction doesn't have an excessive number of - // sigops, making it impossible to mine. Since the coinbase transaction - // itself can contain sigops MAX_STANDARD_TX_SIGOPS is less than - // MAX_BLOCK_SIGOPS; we still consider this an invalid rather than - // merely non-standard transaction. - if (nSigOpsCost > MAX_STANDARD_TX_SIGOPS_COST) - return state.DoS(0, false, REJECT_NONSTANDARD, "bad-txns-too-many-sigops", false, + if (nSigOpsCost > MAX_STANDARD_TX_SIGOPS_COST) + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "bad-txns-too-many-sigops", strprintf("%d", nSigOpsCost)); - CAmount mempoolRejectFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(nSize); - if (mempoolRejectFee > 0 && nModifiedFees < mempoolRejectFee) { - return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool min fee not met", false, strprintf("%d < %d", nFees, mempoolRejectFee)); - } else if (GetBoolArg("-relaypriority", DEFAULT_RELAYPRIORITY) && nModifiedFees < ::minRelayTxFee.GetFee(nSize) && !AllowFree(entry.GetPriority(chainActive.Height() + 1))) { - // Require that free transactions have sufficient priority to be mined in the next block. - return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "insufficient priority"); + // No transactions are allowed below minRelayTxFee except from disconnected + // blocks + if (!bypass_limits && !CheckFeeRate(nSize, nModifiedFees, state)) return false; + + if (nAbsurdFee && nFees > nAbsurdFee) + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, + "absurdly-high-fee", strprintf("%d > %d", nFees, nAbsurdFee)); + + const CTxMemPool::setEntries setIterConflicting = m_pool.GetIterSet(setConflicts); + // Calculate in-mempool ancestors, up to a limit. + if (setConflicts.size() == 1) { + // In general, when we receive an RBF transaction with mempool conflicts, we want to know whether we + // would meet the chain limits after the conflicts have been removed. However, there isn't a practical + // way to do this short of calculating the ancestor and descendant sets with an overlay cache of + // changed mempool entries. Due to both implementation and runtime complexity concerns, this isn't + // very realistic, thus we only ensure a limited set of transactions are RBF'able despite mempool + // conflicts here. Importantly, we need to ensure that some transactions which were accepted using + // the below carve-out are able to be RBF'ed, without impacting the security the carve-out provides + // for off-chain contract systems (see link in the comment below). + // + // Specifically, the subset of RBF transactions which we allow despite chain limits are those which + // conflict directly with exactly one other transaction (but may evict children of said transaction), + // and which are not adding any new mempool dependencies. Note that the "no new mempool dependencies" + // check is accomplished later, so we don't bother doing anything about it here, but if BIP 125 is + // amended, we may need to move that check to here instead of removing it wholesale. + // + // Such transactions are clearly not merging any existing packages, so we are only concerned with + // ensuring that (a) no package is growing past the package size (not count) limits and (b) we are + // not allowing something to effectively use the (below) carve-out spot when it shouldn't be allowed + // to. + // + // To check these we first check if we meet the RBF criteria, above, and increment the descendant + // limits by the direct conflict and its descendants (as these are recalculated in + // CalculateMempoolAncestors by assuming the new transaction being added is a new descendant, with no + // removals, of each parent's existing dependent set). The ancestor count limits are unmodified (as + // the ancestor limits should be the same for both our new transaction and any conflicts). + // We don't bother incrementing m_limit_descendants by the full removal count as that limit never comes + // into force here (as we're only adding a single transaction). + assert(setIterConflicting.size() == 1); + CTxMemPool::txiter conflict = *setIterConflicting.begin(); + + m_limit_descendants += 1; + m_limit_descendant_size += conflict->GetSizeWithDescendants(); + } + + std::string errString; + if (!m_pool.CalculateMemPoolAncestors(*entry, setAncestors, m_limit_ancestors, m_limit_ancestor_size, m_limit_descendants, m_limit_descendant_size, errString)) { + setAncestors.clear(); + // If CalculateMemPoolAncestors fails second time, we want the original error string. + std::string dummy_err_string; + // Contracting/payment channels CPFP carve-out: + // If the new transaction is relatively small (up to 40k weight) + // and has at most one ancestor (ie ancestor limit of 2, including + // the new transaction), allow it if its parent has exactly the + // descendant limit descendants. + // + // This allows protocols which rely on distrusting counterparties + // being able to broadcast descendants of an unconfirmed transaction + // to be secure by simply only having two immediately-spendable + // outputs - one for each counterparty. For more info on the uses for + // this, see https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2018-November/016518.html + if (nSize > EXTRA_DESCENDANT_TX_SIZE_LIMIT || + !m_pool.CalculateMemPoolAncestors(*entry, setAncestors, 2, m_limit_ancestor_size, m_limit_descendants + 1, m_limit_descendant_size + EXTRA_DESCENDANT_TX_SIZE_LIMIT, dummy_err_string)) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too-long-mempool-chain", errString); } + } - // Continuously rate-limit free (really, very-low-fee) transactions - // This mitigates 'penny-flooding' -- sending thousands of free transactions just to - // be annoying or make others' transactions take longer to confirm. - if (fLimitFree && nModifiedFees < ::minRelayTxFee.GetFee(nSize)) - { - static CCriticalSection csFreeLimiter; - static double dFreeCount; - static int64_t nLastTime; - int64_t nNow = GetTime(); - - LOCK(csFreeLimiter); - - // Use an exponentially decaying ~10-minute window: - dFreeCount *= pow(1.0 - 1.0/600.0, (double)(nNow - nLastTime)); - nLastTime = nNow; - // -limitfreerelay unit is thousand-bytes-per-minute - // At default rate it would take over a month to fill 1GB - if (dFreeCount + nSize >= GetArg("-limitfreerelay", DEFAULT_LIMITFREERELAY) * 10 * 1000) - return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "rate limited free transaction"); - LogPrint("mempool", "Rate limit dFreeCount: %g => %g\n", dFreeCount, dFreeCount+nSize); - dFreeCount += nSize; - } - - if (nAbsurdFee && nFees > nAbsurdFee) - return state.Invalid(false, - REJECT_HIGHFEE, "absurdly-high-fee", - strprintf("%d > %d", nFees, nAbsurdFee)); - - // Calculate in-mempool ancestors, up to a limit. - CTxMemPool::setEntries setAncestors; - size_t nLimitAncestors = GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); - size_t nLimitAncestorSize = GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000; - size_t nLimitDescendants = GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); - size_t nLimitDescendantSize = GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000; - std::string errString; - if (!pool.CalculateMemPoolAncestors(entry, setAncestors, nLimitAncestors, nLimitAncestorSize, nLimitDescendants, nLimitDescendantSize, errString)) { - return state.DoS(0, false, REJECT_NONSTANDARD, "too-long-mempool-chain", false, errString); - } - - // A transaction that spends outputs that would be replaced by it is invalid. Now - // that we have the set of all ancestors we can detect this - // pathological case by making sure setConflicts and setAncestors don't - // intersect. - BOOST_FOREACH(CTxMemPool::txiter ancestorIt, setAncestors) + // A transaction that spends outputs that would be replaced by it is invalid. Now + // that we have the set of all ancestors we can detect this + // pathological case by making sure setConflicts and setAncestors don't + // intersect. + for (CTxMemPool::txiter ancestorIt : setAncestors) + { + const uint256 &hashAncestor = ancestorIt->GetTx().GetHash(); + if (setConflicts.count(hashAncestor)) { - const uint256 &hashAncestor = ancestorIt->GetTx().GetHash(); - if (setConflicts.count(hashAncestor)) - { - return state.DoS(10, false, - REJECT_INVALID, "bad-txns-spends-conflicting-tx", false, - strprintf("%s spends conflicting transaction %s", - hash.ToString(), - hashAncestor.ToString())); - } + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-spends-conflicting-tx", + strprintf("%s spends conflicting transaction %s", + hash.ToString(), + hashAncestor.ToString())); } + } - // Check if it's economically rational to mine this transaction rather - // than the ones it replaces. - CAmount nConflictingFees = 0; - size_t nConflictingSize = 0; - uint64_t nConflictingCount = 0; - CTxMemPool::setEntries allConflicting; + // Check if it's economically rational to mine this transaction rather + // than the ones it replaces. + nConflictingFees = 0; + nConflictingSize = 0; + uint64_t nConflictingCount = 0; - // If we don't hold the lock allConflicting might be incomplete; the - // subsequent RemoveStaged() and addUnchecked() calls don't guarantee - // mempool consistency for us. - LOCK(pool.cs); - const bool fReplacementTransaction = setConflicts.size(); - if (fReplacementTransaction) - { - CFeeRate newFeeRate(nModifiedFees, nSize); - std::set<uint256> setConflictsParents; - const int maxDescendantsToVisit = 100; - CTxMemPool::setEntries setIterConflicting; - BOOST_FOREACH(const uint256 &hashConflicting, setConflicts) + // If we don't hold the lock allConflicting might be incomplete; the + // subsequent RemoveStaged() and addUnchecked() calls don't guarantee + // mempool consistency for us. + fReplacementTransaction = setConflicts.size(); + if (fReplacementTransaction) + { + CFeeRate newFeeRate(nModifiedFees, nSize); + std::set<uint256> setConflictsParents; + const int maxDescendantsToVisit = 100; + for (const auto& mi : setIterConflicting) { + // Don't allow the replacement to reduce the feerate of the + // mempool. + // + // We usually don't want to accept replacements with lower + // feerates than what they replaced as that would lower the + // feerate of the next block. Requiring that the feerate always + // be increased is also an easy-to-reason about way to prevent + // DoS attacks via replacements. + // + // We only consider the feerates of transactions being directly + // replaced, not their indirect descendants. While that does + // mean high feerate children are ignored when deciding whether + // or not to replace, we do require the replacement to pay more + // overall fees too, mitigating most cases. + CFeeRate oldFeeRate(mi->GetModifiedFee(), mi->GetTxSize()); + if (newFeeRate <= oldFeeRate) { - CTxMemPool::txiter mi = pool.mapTx.find(hashConflicting); - if (mi == pool.mapTx.end()) - continue; - - // Save these to avoid repeated lookups - setIterConflicting.insert(mi); - - // Don't allow the replacement to reduce the feerate of the - // mempool. - // - // We usually don't want to accept replacements with lower - // feerates than what they replaced as that would lower the - // feerate of the next block. Requiring that the feerate always - // be increased is also an easy-to-reason about way to prevent - // DoS attacks via replacements. - // - // The mining code doesn't (currently) take children into - // account (CPFP) so we only consider the feerates of - // transactions being directly replaced, not their indirect - // descendants. While that does mean high feerate children are - // ignored when deciding whether or not to replace, we do - // require the replacement to pay more overall fees too, - // mitigating most cases. - CFeeRate oldFeeRate(mi->GetModifiedFee(), mi->GetTxSize()); - if (newFeeRate <= oldFeeRate) - { - return state.DoS(0, false, - REJECT_INSUFFICIENTFEE, "insufficient fee", false, - strprintf("rejecting replacement %s; new feerate %s <= old feerate %s", - hash.ToString(), - newFeeRate.ToString(), - oldFeeRate.ToString())); - } - - BOOST_FOREACH(const CTxIn &txin, mi->GetTx().vin) - { - setConflictsParents.insert(txin.prevout.hash); - } - - nConflictingCount += mi->GetCountWithDescendants(); - } - // This potentially overestimates the number of actual descendants - // but we just want to be conservative to avoid doing too much - // work. - if (nConflictingCount <= maxDescendantsToVisit) { - // If not too many to replace, then calculate the set of - // transactions that would have to be evicted - BOOST_FOREACH(CTxMemPool::txiter it, setIterConflicting) { - pool.CalculateDescendants(it, allConflicting); - } - BOOST_FOREACH(CTxMemPool::txiter it, allConflicting) { - nConflictingFees += it->GetModifiedFee(); - nConflictingSize += it->GetTxSize(); - } - } else { - return state.DoS(0, false, - REJECT_NONSTANDARD, "too many potential replacements", false, - strprintf("rejecting replacement %s; too many potential replacements (%d > %d)\n", + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", + strprintf("rejecting replacement %s; new feerate %s <= old feerate %s", hash.ToString(), - nConflictingCount, - maxDescendantsToVisit)); + newFeeRate.ToString(), + oldFeeRate.ToString())); } - for (unsigned int j = 0; j < tx.vin.size(); j++) + for (const CTxIn &txin : mi->GetTx().vin) { - // We don't want to accept replacements that require low - // feerate junk to be mined first. Ideally we'd keep track of - // the ancestor feerates and make the decision based on that, - // but for now requiring all new inputs to be confirmed works. - if (!setConflictsParents.count(tx.vin[j].prevout.hash)) - { - // Rather than check the UTXO set - potentially expensive - - // it's cheaper to just check if the new input refers to a - // tx that's in the mempool. - if (pool.mapTx.find(tx.vin[j].prevout.hash) != pool.mapTx.end()) - return state.DoS(0, false, - REJECT_NONSTANDARD, "replacement-adds-unconfirmed", false, - strprintf("replacement %s adds unconfirmed input, idx %d", - hash.ToString(), j)); - } + setConflictsParents.insert(txin.prevout.hash); } - // The replacement must pay greater fees than the transactions it - // replaces - if we did the bandwidth used by those conflicting - // transactions would not be paid for. - if (nModifiedFees < nConflictingFees) - { - return state.DoS(0, false, - REJECT_INSUFFICIENTFEE, "insufficient fee", false, - strprintf("rejecting replacement %s, less fees than conflicting txs; %s < %s", - hash.ToString(), FormatMoney(nModifiedFees), FormatMoney(nConflictingFees))); + nConflictingCount += mi->GetCountWithDescendants(); + } + // This potentially overestimates the number of actual descendants + // but we just want to be conservative to avoid doing too much + // work. + if (nConflictingCount <= maxDescendantsToVisit) { + // If not too many to replace, then calculate the set of + // transactions that would have to be evicted + for (CTxMemPool::txiter it : setIterConflicting) { + m_pool.CalculateDescendants(it, allConflicting); } - - // Finally in addition to paying more fees than the conflicts the - // new transaction must pay for its own bandwidth. - CAmount nDeltaFees = nModifiedFees - nConflictingFees; - if (nDeltaFees < ::incrementalRelayFee.GetFee(nSize)) - { - return state.DoS(0, false, - REJECT_INSUFFICIENTFEE, "insufficient fee", false, - strprintf("rejecting replacement %s, not enough additional fees to relay; %s < %s", - hash.ToString(), - FormatMoney(nDeltaFees), - FormatMoney(::incrementalRelayFee.GetFee(nSize)))); + for (CTxMemPool::txiter it : allConflicting) { + nConflictingFees += it->GetModifiedFee(); + nConflictingSize += it->GetTxSize(); } + } else { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too many potential replacements", + strprintf("rejecting replacement %s; too many potential replacements (%d > %d)\n", + hash.ToString(), + nConflictingCount, + maxDescendantsToVisit)); } - unsigned int scriptVerifyFlags = STANDARD_SCRIPT_VERIFY_FLAGS; - if (!Params().RequireStandard()) { - scriptVerifyFlags = GetArg("-promiscuousmempoolflags", scriptVerifyFlags); - } - - // Check against previous transactions - // This is done last to help prevent CPU exhaustion denial-of-service attacks. - PrecomputedTransactionData txdata(tx); - if (!CheckInputs(tx, state, view, true, scriptVerifyFlags, true, txdata)) { - // SCRIPT_VERIFY_CLEANSTACK requires SCRIPT_VERIFY_WITNESS, so we - // need to turn both off, and compare against just turning off CLEANSTACK - // to see if the failure is specifically due to witness validation. - CValidationState stateDummy; // Want reported failures to be from first CheckInputs - if (!tx.HasWitness() && CheckInputs(tx, stateDummy, view, true, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, txdata) && - !CheckInputs(tx, stateDummy, view, true, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, txdata)) { - // Only the witness is missing, so the transaction itself may be fine. - state.SetCorruptionPossible(); + for (unsigned int j = 0; j < tx.vin.size(); j++) + { + // We don't want to accept replacements that require low + // feerate junk to be mined first. Ideally we'd keep track of + // the ancestor feerates and make the decision based on that, + // but for now requiring all new inputs to be confirmed works. + // + // Note that if you relax this to make RBF a little more useful, + // this may break the CalculateMempoolAncestors RBF relaxation, + // above. See the comment above the first CalculateMempoolAncestors + // call for more info. + if (!setConflictsParents.count(tx.vin[j].prevout.hash)) + { + // Rather than check the UTXO set - potentially expensive - + // it's cheaper to just check if the new input refers to a + // tx that's in the mempool. + if (m_pool.exists(tx.vin[j].prevout.hash)) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "replacement-adds-unconfirmed", + strprintf("replacement %s adds unconfirmed input, idx %d", + hash.ToString(), j)); + } } - return false; // state filled in by CheckInputs } - // Check again against just the consensus-critical mandatory script - // verification flags, in case of bugs in the standard flags that cause - // transactions to pass as valid when they're actually invalid. For - // instance the STRICTENC flag was incorrectly allowing certain - // CHECKSIG NOT scripts to pass, even though they were invalid. - // - // There is a similar check in CreateNewBlock() to prevent creating - // invalid blocks, however allowing such transactions into the mempool - // can be exploited as a DoS attack. - if (!CheckInputs(tx, state, view, true, MANDATORY_SCRIPT_VERIFY_FLAGS, true, txdata)) + // The replacement must pay greater fees than the transactions it + // replaces - if we did the bandwidth used by those conflicting + // transactions would not be paid for. + if (nModifiedFees < nConflictingFees) { - return error("%s: BUG! PLEASE REPORT THIS! ConnectInputs failed against MANDATORY but not STANDARD flags %s, %s", - __func__, hash.ToString(), FormatStateMessage(state)); + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", + strprintf("rejecting replacement %s, less fees than conflicting txs; %s < %s", + hash.ToString(), FormatMoney(nModifiedFees), FormatMoney(nConflictingFees))); } - // Remove conflicting transactions from the mempool - BOOST_FOREACH(const CTxMemPool::txiter it, allConflicting) + // Finally in addition to paying more fees than the conflicts the + // new transaction must pay for its own bandwidth. + CAmount nDeltaFees = nModifiedFees - nConflictingFees; + if (nDeltaFees < ::incrementalRelayFee.GetFee(nSize)) { - LogPrint("mempool", "replacing tx %s with %s for %s BTC additional fees, %d delta bytes\n", - it->GetTx().GetHash().ToString(), - hash.ToString(), - FormatMoney(nModifiedFees - nConflictingFees), - (int)nSize - (int)nConflictingSize); - if (plTxnReplaced) - plTxnReplaced->push_back(it->GetSharedTx()); + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", + strprintf("rejecting replacement %s, not enough additional fees to relay; %s < %s", + hash.ToString(), + FormatMoney(nDeltaFees), + FormatMoney(::incrementalRelayFee.GetFee(nSize)))); + } + } + return true; +} + +bool MemPoolAccept::PolicyScriptChecks(ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) +{ + const CTransaction& tx = *ws.m_ptx; + + TxValidationState &state = args.m_state; + + constexpr unsigned int scriptVerifyFlags = STANDARD_SCRIPT_VERIFY_FLAGS; + + // Check input scripts and signatures. + // This is done last to help prevent CPU exhaustion denial-of-service attacks. + if (!CheckInputScripts(tx, state, m_view, scriptVerifyFlags, true, false, txdata)) { + // SCRIPT_VERIFY_CLEANSTACK requires SCRIPT_VERIFY_WITNESS, so we + // need to turn both off, and compare against just turning off CLEANSTACK + // to see if the failure is specifically due to witness validation. + TxValidationState state_dummy; // Want reported failures to be from first CheckInputScripts + if (!tx.HasWitness() && CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, false, txdata) && + !CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, false, txdata)) { + // Only the witness is missing, so the transaction itself may be fine. + state.Invalid(TxValidationResult::TX_WITNESS_MUTATED, + state.GetRejectReason(), state.GetDebugMessage()); } - pool.RemoveStaged(allConflicting, false, MemPoolRemovalReason::REPLACED); + return false; // state filled in by CheckInputScripts + } - // This transaction should only count for fee estimation if it isn't a - // BIP 125 replacement transaction (may not be widely supported), the - // node is not behind, and the transaction is not dependent on any other - // transactions in the mempool. - bool validForFeeEstimation = !fReplacementTransaction && IsCurrentForFeeEstimation() && pool.HasNoInputsOf(tx); + return true; +} - // Store transaction in memory - pool.addUnchecked(hash, entry, setAncestors, validForFeeEstimation); +bool MemPoolAccept::ConsensusScriptChecks(ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) +{ + const CTransaction& tx = *ws.m_ptx; + const uint256& hash = ws.m_hash; + + TxValidationState &state = args.m_state; + const CChainParams& chainparams = args.m_chainparams; + + // Check again against the current block tip's script verification + // flags to cache our script execution flags. This is, of course, + // useless if the next block has different script flags from the + // previous one, but because the cache tracks script flags for us it + // will auto-invalidate and we'll just have a few blocks of extra + // misses on soft-fork activation. + // + // This is also useful in case of bugs in the standard flags that cause + // transactions to pass as valid when they're actually invalid. For + // instance the STRICTENC flag was incorrectly allowing certain + // CHECKSIG NOT scripts to pass, even though they were invalid. + // + // There is a similar check in CreateNewBlock() to prevent creating + // invalid blocks (using TestBlockValidity), however allowing such + // transactions into the mempool can be exploited as a DoS attack. + unsigned int currentBlockScriptVerifyFlags = GetBlockScriptFlags(::ChainActive().Tip(), chainparams.GetConsensus()); + if (!CheckInputsFromMempoolAndCache(tx, state, m_view, m_pool, currentBlockScriptVerifyFlags, txdata)) { + return error("%s: BUG! PLEASE REPORT THIS! CheckInputScripts failed against latest-block but not STANDARD flags %s, %s", + __func__, hash.ToString(), state.ToString()); + } - // trim mempool and check if tx was trimmed - if (!fOverrideMempoolLimit) { - LimitMempoolSize(pool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); - if (!pool.exists(hash)) - return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool full"); - } + return true; +} + +bool MemPoolAccept::Finalize(ATMPArgs& args, Workspace& ws) +{ + const CTransaction& tx = *ws.m_ptx; + const uint256& hash = ws.m_hash; + TxValidationState &state = args.m_state; + const bool bypass_limits = args.m_bypass_limits; + + CTxMemPool::setEntries& allConflicting = ws.m_all_conflicting; + CTxMemPool::setEntries& setAncestors = ws.m_ancestors; + const CAmount& nModifiedFees = ws.m_modified_fees; + const CAmount& nConflictingFees = ws.m_conflicting_fees; + const size_t& nConflictingSize = ws.m_conflicting_size; + const bool fReplacementTransaction = ws.m_replacement_transaction; + std::unique_ptr<CTxMemPoolEntry>& entry = ws.m_entry; + + // Remove conflicting transactions from the mempool + for (CTxMemPool::txiter it : allConflicting) + { + LogPrint(BCLog::MEMPOOL, "replacing tx %s with %s for %s additional fees, %d delta bytes\n", + it->GetTx().GetHash().ToString(), + hash.ToString(), + FormatMoney(nModifiedFees - nConflictingFees), + (int)entry->GetTxSize() - (int)nConflictingSize); + if (args.m_replaced_transactions) + args.m_replaced_transactions->push_back(it->GetSharedTx()); + } + m_pool.RemoveStaged(allConflicting, false, MemPoolRemovalReason::REPLACED); + + // This transaction should only count for fee estimation if: + // - it isn't a BIP 125 replacement transaction (may not be widely supported) + // - it's not being re-added during a reorg which bypasses typical mempool fee limits + // - the node is not behind + // - the transaction is not dependent on any other transactions in the mempool + bool validForFeeEstimation = !fReplacementTransaction && !bypass_limits && IsCurrentForFeeEstimation() && m_pool.HasNoInputsOf(tx); + + // Store transaction in memory + m_pool.addUnchecked(*entry, setAncestors, validForFeeEstimation); + + // trim mempool and check if tx was trimmed + if (!bypass_limits) { + LimitMempoolSize(m_pool, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); + if (!m_pool.exists(hash)) + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool full"); } + return true; +} + +bool MemPoolAccept::AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs& args) +{ + AssertLockHeld(cs_main); + LOCK(m_pool.cs); // mempool "read lock" (held through GetMainSignals().TransactionAddedToMempool()) + + Workspace workspace(ptx); + + if (!PreChecks(args, workspace)) return false; + + // Only compute the precomputed transaction data if we need to verify + // scripts (ie, other policy checks pass). We perform the inexpensive + // checks first and avoid hashing and signature verification unless those + // checks pass, to mitigate CPU exhaustion denial-of-service attacks. + PrecomputedTransactionData txdata; + + if (!PolicyScriptChecks(args, workspace, txdata)) return false; + + if (!ConsensusScriptChecks(args, workspace, txdata)) return false; - GetMainSignals().SyncTransaction(tx, NULL, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); + // Tx was accepted, but not added + if (args.m_test_accept) return true; + + if (!Finalize(args, workspace)) return false; + + GetMainSignals().TransactionAddedToMempool(ptx); return true; } -bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree, - bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, - bool fOverrideMempoolLimit, const CAmount nAbsurdFee) +} // anon namespace + +/** (try to) add transaction to memory pool with a specified acceptance time **/ +static bool AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, + int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, + bool bypass_limits, const CAmount nAbsurdFee, bool test_accept) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - std::vector<uint256> vHashTxToUncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); + std::vector<COutPoint> coins_to_uncache; + MemPoolAccept::ATMPArgs args { chainparams, state, nAcceptTime, plTxnReplaced, bypass_limits, nAbsurdFee, coins_to_uncache, test_accept }; + bool res = MemPoolAccept(pool).AcceptSingleTransaction(tx, args); if (!res) { - BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache) - pcoinsTip->Uncache(hashTx); + // Remove coins that were not present in the coins cache before calling ATMPW; + // this is to prevent memory DoS in case we receive a large number of + // invalid transactions that attempt to overrun the in-memory coins cache + // (`CCoinsViewCache::cacheCoins`). + + for (const COutPoint& hashTx : coins_to_uncache) + ::ChainstateActive().CoinsTip().Uncache(hashTx); } // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits - CValidationState stateDummy; - FlushStateToDisk(stateDummy, FLUSH_STATE_PERIODIC); + BlockValidationState state_dummy; + ::ChainstateActive().FlushStateToDisk(chainparams, state_dummy, FlushStateMode::PERIODIC); return res; } -bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree, - bool* pfMissingInputs, std::list<CTransactionRef>* plTxnReplaced, - bool fOverrideMempoolLimit, const CAmount nAbsurdFee) +bool AcceptToMemoryPool(CTxMemPool& pool, TxValidationState &state, const CTransactionRef &tx, + std::list<CTransactionRef>* plTxnReplaced, + bool bypass_limits, const CAmount nAbsurdFee, bool test_accept) { - return AcceptToMemoryPoolWithTime(pool, state, tx, fLimitFree, pfMissingInputs, GetTime(), plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee); + const CChainParams& chainparams = Params(); + return AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, GetTime(), plTxnReplaced, bypass_limits, nAbsurdFee, test_accept); } -/** Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock */ -bool GetTransaction(const uint256 &hash, CTransactionRef &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) +/** + * Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock. + * If blockIndex is provided, the transaction is fetched from the corresponding block. + */ +bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus::Params& consensusParams, uint256& hashBlock, const CBlockIndex* const block_index) { - CBlockIndex *pindexSlow = NULL; - LOCK(cs_main); - CTransactionRef ptx = mempool.get(hash); - if (ptx) - { - txOut = ptx; - return true; - } - - if (fTxIndex) { - CDiskTxPos postx; - if (pblocktree->ReadTxIndex(hash, postx)) { - CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); - if (file.IsNull()) - return error("%s: OpenBlockFile failed", __func__); - CBlockHeader header; - try { - file >> header; - fseek(file.Get(), postx.nTxOffset, SEEK_CUR); - file >> txOut; - } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); - } - hashBlock = header.GetHash(); - if (txOut->GetHash() != hash) - return error("%s: txid mismatch", __func__); + if (!block_index) { + CTransactionRef ptx = mempool.get(hash); + if (ptx) { + txOut = ptx; return true; } - } - if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it - int nHeight = -1; - { - const CCoinsViewCache& view = *pcoinsTip; - const CCoins* coins = view.AccessCoins(hash); - if (coins) - nHeight = coins->nHeight; + if (g_txindex) { + return g_txindex->FindTx(hash, hashBlock, txOut); } - if (nHeight > 0) - pindexSlow = chainActive[nHeight]; - } - - if (pindexSlow) { + } else { CBlock block; - if (ReadBlockFromDisk(block, pindexSlow, consensusParams)) { + if (ReadBlockFromDisk(block, block_index, consensusParams)) { for (const auto& tx : block.vtx) { if (tx->GetHash() == hash) { txOut = tx; - hashBlock = pindexSlow->GetBlockHash(); + hashBlock = block_index->GetBlockHash(); return true; } } @@ -1108,7 +1128,7 @@ bool GetTransaction(const uint256 &hash, CTransactionRef &txOut, const Consensus // CBlock and CBlockIndex // -bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart) +static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); @@ -1116,8 +1136,8 @@ bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHea return error("WriteBlockToDisk: OpenBlockFile failed"); // Write index header - unsigned int nSize = GetSerializeSize(fileout, block); - fileout << FLATDATA(messageStart) << nSize; + unsigned int nSize = GetSerializeSize(block, fileout.GetVersion()); + fileout << messageStart << nSize; // Write block long fileOutPos = ftell(fileout.Get()); @@ -1129,7 +1149,7 @@ bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHea return true; } -bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) +bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) { block.SetNull(); @@ -1155,7 +1175,13 @@ bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus: bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams) { - if (!ReadBlockFromDisk(block, pindex->GetBlockPos(), consensusParams)) + FlatFilePos blockPos; + { + LOCK(cs_main); + blockPos = pindex->GetBlockPos(); + } + + if (!ReadBlockFromDisk(block, blockPos, consensusParams)) return false; if (block.GetHash() != pindex->GetBlockHash()) return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s", @@ -1163,6 +1189,52 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus return true; } +bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) +{ + FlatFilePos hpos = pos; + hpos.nPos -= 8; // Seek back 8 bytes for meta header + CAutoFile filein(OpenBlockFile(hpos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) { + return error("%s: OpenBlockFile failed for %s", __func__, pos.ToString()); + } + + try { + CMessageHeader::MessageStartChars blk_start; + unsigned int blk_size; + + filein >> blk_start >> blk_size; + + if (memcmp(blk_start, message_start, CMessageHeader::MESSAGE_START_SIZE)) { + return error("%s: Block magic mismatch for %s: %s versus expected %s", __func__, pos.ToString(), + HexStr(blk_start, blk_start + CMessageHeader::MESSAGE_START_SIZE), + HexStr(message_start, message_start + CMessageHeader::MESSAGE_START_SIZE)); + } + + if (blk_size > MAX_SIZE) { + return error("%s: Block data is larger than maximum deserialization size for %s: %s versus %s", __func__, pos.ToString(), + blk_size, MAX_SIZE); + } + + block.resize(blk_size); // Zeroing of memory is intentional here + filein.read((char*)block.data(), blk_size); + } catch(const std::exception& e) { + return error("%s: Read from block file failed: %s for %s", __func__, e.what(), pos.ToString()); + } + + return true; +} + +bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start) +{ + FlatFilePos block_pos; + { + LOCK(cs_main); + block_pos = pindex->GetBlockPos(); + } + + return ReadRawBlockFromDisk(block, block_pos, message_start); +} + CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams) { int halvings = nHeight / consensusParams.nSubsidyHalvingInterval; @@ -1176,37 +1248,83 @@ CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams) return nSubsidy; } -bool IsInitialBlockDownload() +CoinsViews::CoinsViews( + std::string ldb_name, + size_t cache_size_bytes, + bool in_memory, + bool should_wipe) : m_dbview( + GetDataDir() / ldb_name, cache_size_bytes, in_memory, should_wipe), + m_catcherview(&m_dbview) {} + +void CoinsViews::InitCache() +{ + m_cacheview = MakeUnique<CCoinsViewCache>(&m_catcherview); +} + +CChainState::CChainState(BlockManager& blockman, uint256 from_snapshot_blockhash) + : m_blockman(blockman), + m_from_snapshot_blockhash(from_snapshot_blockhash) {} + +void CChainState::InitCoinsDB( + size_t cache_size_bytes, + bool in_memory, + bool should_wipe, + std::string leveldb_name) +{ + if (!m_from_snapshot_blockhash.IsNull()) { + leveldb_name += "_" + m_from_snapshot_blockhash.ToString(); + } + + m_coins_views = MakeUnique<CoinsViews>( + leveldb_name, cache_size_bytes, in_memory, should_wipe); +} + +void CChainState::InitCoinsCache() { - const CChainParams& chainParams = Params(); + assert(m_coins_views != nullptr); + m_coins_views->InitCache(); +} - // Once this function has returned false, it must remain false. - static std::atomic<bool> latchToFalse{false}; +// Note that though this is marked const, we may end up modifying `m_cached_finished_ibd`, which +// is a performance-related implementation detail. This function must be marked +// `const` so that `CValidationInterface` clients (which are given a `const CChainState*`) +// can call it. +// +bool CChainState::IsInitialBlockDownload() const +{ // Optimization: pre-test latch before taking the lock. - if (latchToFalse.load(std::memory_order_relaxed)) + if (m_cached_finished_ibd.load(std::memory_order_relaxed)) return false; LOCK(cs_main); - if (latchToFalse.load(std::memory_order_relaxed)) + if (m_cached_finished_ibd.load(std::memory_order_relaxed)) return false; if (fImporting || fReindex) return true; - if (chainActive.Tip() == NULL) + if (m_chain.Tip() == nullptr) return true; - if (chainActive.Tip()->nChainWork < UintToArith256(chainParams.GetConsensus().nMinimumChainWork)) + if (m_chain.Tip()->nChainWork < nMinimumChainWork) return true; - if (chainActive.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) + if (m_chain.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) return true; - latchToFalse.store(true, std::memory_order_relaxed); + LogPrintf("Leaving InitialBlockDownload (latching to false)\n"); + m_cached_finished_ibd.store(true, std::memory_order_relaxed); return false; } -CBlockIndex *pindexBestForkTip = NULL, *pindexBestForkBase = NULL; +static CBlockIndex *pindexBestForkTip = nullptr, *pindexBestForkBase = nullptr; + +BlockMap& BlockIndex() +{ + LOCK(::cs_main); + return g_chainman.m_blockman.m_block_index; +} static void AlertNotify(const std::string& strMessage) { uiInterface.NotifyAlertChanged(); - std::string strCmd = GetArg("-alertnotify", ""); +#if HAVE_SYSTEM + std::string strCmd = gArgs.GetArg("-alertnotify", ""); if (strCmd.empty()) return; // Alert text should be plain ascii coming from a trusted source, but to @@ -1217,23 +1335,25 @@ static void AlertNotify(const std::string& strMessage) safeStatus = singleQuote+safeStatus+singleQuote; boost::replace_all(strCmd, "%s", safeStatus); - boost::thread t(runCommand, strCmd); // thread runs free + std::thread t(runCommand, strCmd); + t.detach(); // thread runs free +#endif } -void CheckForkWarningConditions() +static void CheckForkWarningConditions() EXCLUSIVE_LOCKS_REQUIRED(cs_main) { AssertLockHeld(cs_main); // Before we get past initial download, we cannot reliably alert about forks // (we assume we don't get stuck on a fork before finishing our initial sync) - if (IsInitialBlockDownload()) + if (::ChainstateActive().IsInitialBlockDownload()) return; // If our best fork is no longer within 72 blocks (+/- 12 hours if no one mines it) // of our head, drop it - if (pindexBestForkTip && chainActive.Height() - pindexBestForkTip->nHeight >= 72) - pindexBestForkTip = NULL; + if (pindexBestForkTip && ::ChainActive().Height() - pindexBestForkTip->nHeight >= 72) + pindexBestForkTip = nullptr; - if (pindexBestForkTip || (pindexBestInvalid && pindexBestInvalid->nChainWork > chainActive.Tip()->nChainWork + (GetBlockProof(*chainActive.Tip()) * 6))) + if (pindexBestForkTip || (pindexBestInvalid && pindexBestInvalid->nChainWork > ::ChainActive().Tip()->nChainWork + (GetBlockProof(*::ChainActive().Tip()) * 6))) { if (!GetfLargeWorkForkFound() && pindexBestForkBase) { @@ -1261,12 +1381,12 @@ void CheckForkWarningConditions() } } -void CheckForkWarningConditionsOnNewFork(CBlockIndex* pindexNewForkTip) +static void CheckForkWarningConditionsOnNewFork(CBlockIndex* pindexNewForkTip) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { AssertLockHeld(cs_main); // If we are on a fork that is sufficiently large, set a warning flag CBlockIndex* pfork = pindexNewForkTip; - CBlockIndex* plonger = chainActive.Tip(); + CBlockIndex* plonger = ::ChainActive().Tip(); while (pfork && pfork != plonger) { while (plonger && plonger->nHeight > pfork->nHeight) @@ -1283,9 +1403,9 @@ void CheckForkWarningConditionsOnNewFork(CBlockIndex* pindexNewForkTip) // or a chain that is entirely longer than ours and invalid (note that this should be detected by both) // We define it this way because it allows us to only store the highest fork tip (+ base) which meets // the 7-block condition and from this always have the most-likely-to-cause-warning fork - if (pfork && (!pindexBestForkTip || (pindexBestForkTip && pindexNewForkTip->nHeight > pindexBestForkTip->nHeight)) && + if (pfork && (!pindexBestForkTip || pindexNewForkTip->nHeight > pindexBestForkTip->nHeight) && pindexNewForkTip->nChainWork - pfork->nChainWork > (GetBlockProof(*pfork) * 7) && - chainActive.Height() - pindexNewForkTip->nHeight < 72) + ::ChainActive().Height() - pindexNewForkTip->nHeight < 72) { pindexBestForkTip = pindexNewForkTip; pindexBestForkBase = pfork; @@ -1294,26 +1414,32 @@ void CheckForkWarningConditionsOnNewFork(CBlockIndex* pindexNewForkTip) CheckForkWarningConditions(); } -void static InvalidChainFound(CBlockIndex* pindexNew) +// Called both upon regular invalid block discovery *and* InvalidateBlock +void static InvalidChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { if (!pindexBestInvalid || pindexNew->nChainWork > pindexBestInvalid->nChainWork) pindexBestInvalid = pindexNew; + if (pindexBestHeader != nullptr && pindexBestHeader->GetAncestor(pindexNew->nHeight) == pindexNew) { + pindexBestHeader = ::ChainActive().Tip(); + } LogPrintf("%s: invalid block=%s height=%d log2_work=%.8g date=%s\n", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, - log(pindexNew->nChainWork.getdouble())/log(2.0), DateTimeStrFormat("%Y-%m-%d %H:%M:%S", - pindexNew->GetBlockTime())); - CBlockIndex *tip = chainActive.Tip(); + log(pindexNew->nChainWork.getdouble())/log(2.0), FormatISO8601DateTime(pindexNew->GetBlockTime())); + CBlockIndex *tip = ::ChainActive().Tip(); assert (tip); LogPrintf("%s: current best=%s height=%d log2_work=%.8g date=%s\n", __func__, - tip->GetBlockHash().ToString(), chainActive.Height(), log(tip->nChainWork.getdouble())/log(2.0), - DateTimeStrFormat("%Y-%m-%d %H:%M:%S", tip->GetBlockTime())); + tip->GetBlockHash().ToString(), ::ChainActive().Height(), log(tip->nChainWork.getdouble())/log(2.0), + FormatISO8601DateTime(tip->GetBlockTime())); CheckForkWarningConditions(); } -void static InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) { - if (!state.CorruptionPossible()) { +// Same as InvalidChainFound, above, except not called directly from InvalidateBlock, +// which does its own setBlockIndexCandidates manageent. +void CChainState::InvalidBlockFound(CBlockIndex *pindex, const BlockValidationState &state) { + if (state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; + m_blockman.m_failed_blocks.insert(pindex); setDirtyBlockIndex.insert(pindex); setBlockIndexCandidates.erase(pindex); InvalidChainFound(pindex); @@ -1325,25 +1451,14 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund // mark inputs spent if (!tx.IsCoinBase()) { txundo.vprevout.reserve(tx.vin.size()); - BOOST_FOREACH(const CTxIn &txin, tx.vin) { - CCoinsModifier coins = inputs.ModifyCoins(txin.prevout.hash); - unsigned nPos = txin.prevout.n; - - if (nPos >= coins->vout.size() || coins->vout[nPos].IsNull()) - assert(false); - // mark an outpoint spent, and construct undo information - txundo.vprevout.push_back(CTxInUndo(coins->vout[nPos])); - coins->Spend(nPos); - if (coins->vout.size() == 0) { - CTxInUndo& undo = txundo.vprevout.back(); - undo.nHeight = coins->nHeight; - undo.fCoinBase = coins->fCoinBase; - undo.nVersion = coins->nVersion; - } + for (const CTxIn &txin : tx.vin) { + txundo.vprevout.emplace_back(); + bool is_spent = inputs.SpendCoin(txin.prevout, &txundo.vprevout.back()); + assert(is_spent); } } // add outputs - inputs.ModifyNewCoins(tx.GetHash(), tx.IsCoinBase())->FromTx(tx, nHeight); + AddCoins(inputs, tx, nHeight); } void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) @@ -1355,127 +1470,129 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) bool CScriptCheck::operator()() { const CScript &scriptSig = ptxTo->vin[nIn].scriptSig; const CScriptWitness *witness = &ptxTo->vin[nIn].scriptWitness; - if (!VerifyScript(scriptSig, scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, amount, cacheStore, *txdata), &error)) { - return false; - } - return true; + return VerifyScript(scriptSig, m_tx_out.scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, m_tx_out.nValue, cacheStore, *txdata), &error); } int GetSpendHeight(const CCoinsViewCache& inputs) { LOCK(cs_main); - CBlockIndex* pindexPrev = mapBlockIndex.find(inputs.GetBestBlock())->second; + CBlockIndex* pindexPrev = LookupBlockIndex(inputs.GetBestBlock()); return pindexPrev->nHeight + 1; } -namespace Consensus { -bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight) -{ - // This doesn't trigger the DoS code on purpose; if it did, it would make it easier - // for an attacker to attempt to split the network. - if (!inputs.HaveInputs(tx)) - return state.Invalid(false, 0, "", "Inputs unavailable"); - - CAmount nValueIn = 0; - CAmount nFees = 0; - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const COutPoint &prevout = tx.vin[i].prevout; - const CCoins *coins = inputs.AccessCoins(prevout.hash); - assert(coins); - - // If prev is coinbase, check that it's matured - if (coins->IsCoinBase()) { - if (nSpendHeight - coins->nHeight < COINBASE_MATURITY) - return state.Invalid(false, - REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", - strprintf("tried to spend coinbase at depth %d", nSpendHeight - coins->nHeight)); - } - - // Check for negative or overflow input values - nValueIn += coins->vout[prevout.n].nValue; - if (!MoneyRange(coins->vout[prevout.n].nValue) || !MoneyRange(nValueIn)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); - } +static CuckooCache::cache<uint256, SignatureCacheHasher> scriptExecutionCache; +static uint256 scriptExecutionCacheNonce(GetRandHash()); - if (nValueIn < tx.GetValueOut()) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-in-belowout", false, - strprintf("value in (%s) < value out (%s)", FormatMoney(nValueIn), FormatMoney(tx.GetValueOut()))); - - // Tally transaction fees - CAmount nTxFee = nValueIn - tx.GetValueOut(); - if (nTxFee < 0) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-negative"); - nFees += nTxFee; - if (!MoneyRange(nFees)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-outofrange"); - return true; +void InitScriptExecutionCache() { + // nMaxCacheSize is unsigned. If -maxsigcachesize is set to zero, + // setup_bytes creates the minimum possible cache (2 elements). + size_t nMaxCacheSize = std::min(std::max((int64_t)0, gArgs.GetArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_SIZE) / 2), MAX_MAX_SIG_CACHE_SIZE) * ((size_t) 1 << 20); + size_t nElems = scriptExecutionCache.setup_bytes(nMaxCacheSize); + LogPrintf("Using %zu MiB out of %zu/2 requested for script execution cache, able to store %zu elements\n", + (nElems*sizeof(uint256)) >>20, (nMaxCacheSize*2)>>20, nElems); } -}// namespace Consensus -bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks) +/** + * Check whether all of this transaction's input scripts succeed. + * + * This involves ECDSA signature checks so can be computationally intensive. This function should + * only be called after the cheap sanity checks in CheckTxInputs passed. + * + * If pvChecks is not nullptr, script checks are pushed onto it instead of being performed inline. Any + * script checks which are not necessary (eg due to script execution cache hits) are, obviously, + * not pushed onto pvChecks/run. + * + * Setting cacheSigStore/cacheFullScriptStore to false will remove elements from the corresponding cache + * which are matched. This is useful for checking blocks where we will likely never need the cache + * entry again. + * + * Note that we may set state.reason to NOT_STANDARD for extra soft-fork flags in flags, block-checking + * callers should probably reset it to CONSENSUS in such cases. + * + * Non-static (and re-declared) in src/test/txvalidationcache_tests.cpp + */ +bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const CCoinsViewCache &inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - if (!tx.IsCoinBase()) - { - if (!Consensus::CheckTxInputs(tx, state, inputs, GetSpendHeight(inputs))) - return false; + if (tx.IsCoinBase()) return true; + + if (pvChecks) { + pvChecks->reserve(tx.vin.size()); + } + + // First check if script executions have been cached with the same + // flags. Note that this assumes that the inputs provided are + // correct (ie that the transaction hash which is in tx's prevouts + // properly commits to the scriptPubKey in the inputs view of that + // transaction). + uint256 hashCacheEntry; + // We only use the first 19 bytes of nonce to avoid a second SHA + // round - giving us 19 + 32 + 4 = 55 bytes (+ 8 + 1 = 64) + static_assert(55 - sizeof(flags) - 32 >= 128/8, "Want at least 128 bits of nonce for script execution cache"); + CSHA256().Write(scriptExecutionCacheNonce.begin(), 55 - sizeof(flags) - 32).Write(tx.GetWitnessHash().begin(), 32).Write((unsigned char*)&flags, sizeof(flags)).Finalize(hashCacheEntry.begin()); + AssertLockHeld(cs_main); //TODO: Remove this requirement by making CuckooCache not require external locks + if (scriptExecutionCache.contains(hashCacheEntry, !cacheFullScriptStore)) { + return true; + } - if (pvChecks) - pvChecks->reserve(tx.vin.size()); - - // The first loop above does all the inexpensive checks. - // Only if ALL inputs pass do we perform expensive ECDSA signature checks. - // Helps prevent CPU exhaustion attacks. - - // Skip script verification when connecting blocks under the - // assumedvalid block. Assuming the assumedvalid block is valid this - // is safe because block merkle hashes are still computed and checked, - // Of course, if an assumed valid block is invalid due to false scriptSigs - // this optimization would allow an invalid chain to be accepted. - if (fScriptChecks) { - for (unsigned int i = 0; i < tx.vin.size(); i++) { - const COutPoint &prevout = tx.vin[i].prevout; - const CCoins* coins = inputs.AccessCoins(prevout.hash); - assert(coins); - - // Verify signature - CScriptCheck check(*coins, tx, i, flags, cacheStore, &txdata); - if (pvChecks) { - pvChecks->push_back(CScriptCheck()); - check.swap(pvChecks->back()); - } else if (!check()) { - if (flags & STANDARD_NOT_MANDATORY_VERIFY_FLAGS) { - // Check whether the failure was caused by a - // non-mandatory script verification check, such as - // non-standard DER encodings or non-null dummy - // arguments; if so, don't trigger DoS protection to - // avoid splitting the network between upgraded and - // non-upgraded nodes. - CScriptCheck check2(*coins, tx, i, - flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheStore, &txdata); - if (check2()) - return state.Invalid(false, REJECT_NONSTANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); - } - // Failures of other flags indicate a transaction that is - // invalid in new blocks, e.g. a invalid P2SH. We DoS ban - // such nodes as they are not following the protocol. That - // said during an upgrade careful thought should be taken - // as to the correct behavior - we may want to continue - // peering with non-upgraded nodes even after soft-fork - // super-majority signaling has occurred. - return state.DoS(100,false, REJECT_INVALID, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); - } + if (!txdata.m_ready) { + txdata.Init(tx); + } + + for (unsigned int i = 0; i < tx.vin.size(); i++) { + const COutPoint &prevout = tx.vin[i].prevout; + const Coin& coin = inputs.AccessCoin(prevout); + assert(!coin.IsSpent()); + + // We very carefully only pass in things to CScriptCheck which + // are clearly committed to by tx' witness hash. This provides + // a sanity check that our caching is not introducing consensus + // failures through additional data in, eg, the coins being + // spent being checked as a part of CScriptCheck. + + // Verify signature + CScriptCheck check(coin.out, tx, i, flags, cacheSigStore, &txdata); + if (pvChecks) { + pvChecks->push_back(CScriptCheck()); + check.swap(pvChecks->back()); + } else if (!check()) { + if (flags & STANDARD_NOT_MANDATORY_VERIFY_FLAGS) { + // Check whether the failure was caused by a + // non-mandatory script verification check, such as + // non-standard DER encodings or non-null dummy + // arguments; if so, ensure we return NOT_STANDARD + // instead of CONSENSUS to avoid downstream users + // splitting the network between upgraded and + // non-upgraded nodes by banning CONSENSUS-failing + // data providers. + CScriptCheck check2(coin.out, tx, i, + flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheSigStore, &txdata); + if (check2()) + return state.Invalid(TxValidationResult::TX_NOT_STANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); } + // MANDATORY flag failures correspond to + // TxValidationResult::TX_CONSENSUS. Because CONSENSUS + // failures are the most serious case of validation + // failures, we may need to consider using + // RECENT_CONSENSUS_CHANGE for any script failure that + // could be due to non-upgraded nodes which we may want to + // support, to avoid splitting the network (but this + // depends on the details of how net_processing handles + // such errors). + return state.Invalid(TxValidationResult::TX_CONSENSUS, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); } } + if (cacheFullScriptStore && !pvChecks) { + // We executed all of the provided scripts, and were told to + // cache the result. Do so now. + scriptExecutionCache.insert(hashCacheEntry); + } + return true; } -namespace { - -bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); @@ -1483,8 +1600,8 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint return error("%s: OpenUndoFile failed", __func__); // Write index header - unsigned int nSize = GetSerializeSize(fileout, blockundo); - fileout << FLATDATA(messageStart) << nSize; + unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion()); + fileout << messageStart << nSize; // Write undo data long fileOutPos = ftell(fileout.Get()); @@ -1502,8 +1619,13 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint return true; } -bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uint256& hashBlock) +bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) { + FlatFilePos pos = pindex->GetUndoPos(); + if (pos.IsNull()) { + return error("%s: no undo data available", __func__); + } + // Open history file to read CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); if (filein.IsNull()) @@ -1511,8 +1633,10 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin // Read block uint256 hashChecksum; + CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data try { - filein >> blockundo; + verifier << pindex->pprev->GetBlockHash(); + verifier >> blockundo; filein >> hashChecksum; } catch (const std::exception& e) { @@ -1520,170 +1644,171 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin } // Verify checksum - CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); - hasher << hashBlock; - hasher << blockundo; - if (hashChecksum != hasher.GetHash()) + if (hashChecksum != verifier.GetHash()) return error("%s: Checksum mismatch", __func__); return true; } /** Abort with a message */ -bool AbortNode(const std::string& strMessage, const std::string& userMessage="") +// TODO: AbortNode() should take bilingual_str userMessage parameter. +static bool AbortNode(const std::string& strMessage, const std::string& userMessage = "", unsigned int prefix = 0) { SetMiscWarning(strMessage); LogPrintf("*** %s\n", strMessage); - uiInterface.ThreadSafeMessageBox( - userMessage.empty() ? _("Error: A fatal internal error occurred, see debug.log for details") : userMessage, - "", CClientUIInterface::MSG_ERROR); + if (!userMessage.empty()) { + uiInterface.ThreadSafeMessageBox(Untranslated(userMessage), "", CClientUIInterface::MSG_ERROR | prefix); + } else { + uiInterface.ThreadSafeMessageBox(_("Error: A fatal internal error occurred, see debug.log for details"), "", CClientUIInterface::MSG_ERROR | CClientUIInterface::MSG_NOPREFIX); + } StartShutdown(); return false; } -bool AbortNode(CValidationState& state, const std::string& strMessage, const std::string& userMessage="") +static bool AbortNode(BlockValidationState& state, const std::string& strMessage, const std::string& userMessage = "", unsigned int prefix = 0) { - AbortNode(strMessage, userMessage); + AbortNode(strMessage, userMessage, prefix); return state.Error(strMessage); } -} // anon namespace - /** - * Apply the undo operation of a CTxInUndo to the given chain state. - * @param undo The undo object. + * Restore the UTXO in a Coin at a given COutPoint + * @param undo The Coin to be restored. * @param view The coins view to which to apply the changes. * @param out The out point that corresponds to the tx input. - * @return True on success. + * @return A DisconnectResult as an int */ -bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out) +int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) { bool fClean = true; - CCoinsModifier coins = view.ModifyCoins(out.hash); - if (undo.nHeight != 0) { - // undo data contains height: this is the last output of the prevout tx being spent - if (!coins->IsPruned()) - fClean = fClean && error("%s: undo data overwriting existing transaction", __func__); - coins->Clear(); - coins->fCoinBase = undo.fCoinBase; - coins->nHeight = undo.nHeight; - coins->nVersion = undo.nVersion; - } else { - if (coins->IsPruned()) - fClean = fClean && error("%s: undo data adding output to missing transaction", __func__); + if (view.HaveCoin(out)) fClean = false; // overwriting transaction output + + if (undo.nHeight == 0) { + // Missing undo metadata (height and coinbase). Older versions included this + // information only in undo records for the last spend of a transactions' + // outputs. This implies that it must be present for some other output of the same tx. + const Coin& alternate = AccessByTxid(view, out.hash); + if (!alternate.IsSpent()) { + undo.nHeight = alternate.nHeight; + undo.fCoinBase = alternate.fCoinBase; + } else { + return DISCONNECT_FAILED; // adding output for transaction without known metadata + } } - if (coins->IsAvailable(out.n)) - fClean = fClean && error("%s: undo data overwriting existing output", __func__); - if (coins->vout.size() < out.n+1) - coins->vout.resize(out.n+1); - coins->vout[out.n] = undo.txout; + // If the coin already exists as an unspent coin in the cache, then the + // possible_overwrite parameter to AddCoin must be set to true. We have + // already checked whether an unspent coin exists above using HaveCoin, so + // we don't need to guess. When fClean is false, an unspent coin already + // existed and it is an overwrite. + view.AddCoin(out, std::move(undo), !fClean); - return fClean; + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } -bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean) +/** Undo the effects of this block (with given index) on the UTXO set represented by coins. + * When FAILED is returned, view is left in an indeterminate state. */ +DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) { - assert(pindex->GetBlockHash() == view.GetBestBlock()); - - if (pfClean) - *pfClean = false; - bool fClean = true; CBlockUndo blockUndo; - CDiskBlockPos pos = pindex->GetUndoPos(); - if (pos.IsNull()) - return error("DisconnectBlock(): no undo data available"); - if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) - return error("DisconnectBlock(): failure reading undo data"); + if (!UndoReadFromDisk(blockUndo, pindex)) { + error("DisconnectBlock(): failure reading undo data"); + return DISCONNECT_FAILED; + } - if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) - return error("DisconnectBlock(): block and undo data inconsistent"); + if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) { + error("DisconnectBlock(): block and undo data inconsistent"); + return DISCONNECT_FAILED; + } // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { const CTransaction &tx = *(block.vtx[i]); uint256 hash = tx.GetHash(); + bool is_coinbase = tx.IsCoinBase(); // Check that all outputs are available and match the outputs in the block itself // exactly. - { - CCoinsModifier outs = view.ModifyCoins(hash); - outs->ClearUnspendable(); - - CCoins outsBlock(tx, pindex->nHeight); - // The CCoins serialization does not serialize negative numbers. - // No network rules currently depend on the version here, so an inconsistency is harmless - // but it must be corrected before txout nversion ever influences a network rule. - if (outsBlock.nVersion < 0) - outs->nVersion = outsBlock.nVersion; - if (*outs != outsBlock) - fClean = fClean && error("DisconnectBlock(): added transaction mismatch? database corrupted"); - - // remove outputs - outs->Clear(); + for (size_t o = 0; o < tx.vout.size(); o++) { + if (!tx.vout[o].scriptPubKey.IsUnspendable()) { + COutPoint out(hash, o); + Coin coin; + bool is_spent = view.SpendCoin(out, &coin); + if (!is_spent || tx.vout[o] != coin.out || pindex->nHeight != coin.nHeight || is_coinbase != coin.fCoinBase) { + fClean = false; // transaction output mismatch + } + } } // restore inputs if (i > 0) { // not coinbases - const CTxUndo &txundo = blockUndo.vtxundo[i-1]; - if (txundo.vprevout.size() != tx.vin.size()) - return error("DisconnectBlock(): transaction and undo data inconsistent"); + CTxUndo &txundo = blockUndo.vtxundo[i-1]; + if (txundo.vprevout.size() != tx.vin.size()) { + error("DisconnectBlock(): transaction and undo data inconsistent"); + return DISCONNECT_FAILED; + } for (unsigned int j = tx.vin.size(); j-- > 0;) { const COutPoint &out = tx.vin[j].prevout; - const CTxInUndo &undo = txundo.vprevout[j]; - if (!ApplyTxInUndo(undo, view, out)) - fClean = false; + int res = ApplyTxInUndo(std::move(txundo.vprevout[j]), view, out); + if (res == DISCONNECT_FAILED) return DISCONNECT_FAILED; + fClean = fClean && res != DISCONNECT_UNCLEAN; } + // At this point, all of txundo.vprevout should have been moved out. } } // move best block pointer to prevout block view.SetBestBlock(pindex->pprev->GetBlockHash()); - if (pfClean) { - *pfClean = fClean; - return true; - } - - return fClean; + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } void static FlushBlockFile(bool fFinalize = false) { LOCK(cs_LastBlockFile); - CDiskBlockPos posOld(nLastBlockFile, 0); + FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); + FlatFilePos undo_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nUndoSize); - FILE *fileOld = OpenBlockFile(posOld); - if (fileOld) { - if (fFinalize) - TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nSize); - FileCommit(fileOld); - fclose(fileOld); + bool status = true; + status &= BlockFileSeq().Flush(block_pos_old, fFinalize); + status &= UndoFileSeq().Flush(undo_pos_old, fFinalize); + if (!status) { + AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); } +} - fileOld = OpenUndoFile(posOld); - if (fileOld) { - if (fFinalize) - TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nUndoSize); - FileCommit(fileOld); - fclose(fileOld); +static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); + +static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) +{ + // Write undo information to disk + if (pindex->GetUndoPos().IsNull()) { + FlatFilePos _pos; + if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) + return error("ConnectBlock(): FindUndoPos failed"); + if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) + return AbortNode(state, "Failed to write undo data"); + + // update nUndoPos in block index + pindex->nUndoPos = _pos.nPos; + pindex->nStatus |= BLOCK_HAVE_UNDO; + setDirtyBlockIndex.insert(pindex); } -} -bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); + return true; +} static CCheckQueue<CScriptCheck> scriptcheckqueue(128); -void ThreadScriptCheck() { - RenameThread("bitcoin-scriptch"); +void ThreadScriptCheck(int worker_num) { + util::ThreadRename(strprintf("scriptch.%i", worker_num)); scriptcheckqueue.Thread(); } -// Protected by cs_main -VersionBitsCache versionbitscache; +VersionBitsCache versionbitscache GUARDED_BY(cs_main); int32_t ComputeBlockVersion(const CBlockIndex* pindexPrev, const Consensus::Params& params) { @@ -1691,9 +1816,9 @@ int32_t ComputeBlockVersion(const CBlockIndex* pindexPrev, const Consensus::Para int32_t nVersion = VERSIONBITS_TOP_BITS; for (int i = 0; i < (int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; i++) { - ThresholdState state = VersionBitsState(pindexPrev, params, (Consensus::DeploymentPos)i, versionbitscache); - if (state == THRESHOLD_LOCKED_IN || state == THRESHOLD_STARTED) { - nVersion |= VersionBitsMask(params, (Consensus::DeploymentPos)i); + ThresholdState state = VersionBitsState(pindexPrev, params, static_cast<Consensus::DeploymentPos>(i), versionbitscache); + if (state == ThresholdState::LOCKED_IN || state == ThresholdState::STARTED) { + nVersion |= VersionBitsMask(params, static_cast<Consensus::DeploymentPos>(i)); } } @@ -1709,23 +1834,80 @@ private: int bit; public: - WarningBitsConditionChecker(int bitIn) : bit(bitIn) {} + explicit WarningBitsConditionChecker(int bitIn) : bit(bitIn) {} - int64_t BeginTime(const Consensus::Params& params) const { return 0; } - int64_t EndTime(const Consensus::Params& params) const { return std::numeric_limits<int64_t>::max(); } - int Period(const Consensus::Params& params) const { return params.nMinerConfirmationWindow; } - int Threshold(const Consensus::Params& params) const { return params.nRuleChangeActivationThreshold; } + int64_t BeginTime(const Consensus::Params& params) const override { return 0; } + int64_t EndTime(const Consensus::Params& params) const override { return std::numeric_limits<int64_t>::max(); } + int Period(const Consensus::Params& params) const override { return params.nMinerConfirmationWindow; } + int Threshold(const Consensus::Params& params) const override { return params.nRuleChangeActivationThreshold; } - bool Condition(const CBlockIndex* pindex, const Consensus::Params& params) const + bool Condition(const CBlockIndex* pindex, const Consensus::Params& params) const override { - return ((pindex->nVersion & VERSIONBITS_TOP_MASK) == VERSIONBITS_TOP_BITS) && + return pindex->nHeight >= params.MinBIP9WarningHeight && + ((pindex->nVersion & VERSIONBITS_TOP_MASK) == VERSIONBITS_TOP_BITS) && ((pindex->nVersion >> bit) & 1) != 0 && ((ComputeBlockVersion(pindex->pprev, params) >> bit) & 1) == 0; } }; -// Protected by cs_main -static ThresholdConditionCache warningcache[VERSIONBITS_NUM_BITS]; +static ThresholdConditionCache warningcache[VERSIONBITS_NUM_BITS] GUARDED_BY(cs_main); + +// 0.13.0 was shipped with a segwit deployment defined for testnet, but not for +// mainnet. We no longer need to support disabling the segwit deployment +// except for testing purposes, due to limitations of the functional test +// environment. See test/functional/p2p-segwit.py. +static bool IsScriptWitnessEnabled(const Consensus::Params& params) +{ + return params.SegwitHeight != std::numeric_limits<int>::max(); +} + +static unsigned int GetBlockScriptFlags(const CBlockIndex* pindex, const Consensus::Params& consensusparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { + AssertLockHeld(cs_main); + + unsigned int flags = SCRIPT_VERIFY_NONE; + + // BIP16 didn't become active until Apr 1 2012 (on mainnet, and + // retroactively applied to testnet) + // However, only one historical block violated the P2SH rules (on both + // mainnet and testnet), so for simplicity, always leave P2SH + // on except for the one violating block. + if (consensusparams.BIP16Exception.IsNull() || // no bip16 exception on this chain + pindex->phashBlock == nullptr || // this is a new candidate block, eg from TestBlockValidity() + *pindex->phashBlock != consensusparams.BIP16Exception) // this block isn't the historical exception + { + flags |= SCRIPT_VERIFY_P2SH; + } + + // Enforce WITNESS rules whenever P2SH is in effect (and the segwit + // deployment is defined). + if (flags & SCRIPT_VERIFY_P2SH && IsScriptWitnessEnabled(consensusparams)) { + flags |= SCRIPT_VERIFY_WITNESS; + } + + // Start enforcing the DERSIG (BIP66) rule + if (pindex->nHeight >= consensusparams.BIP66Height) { + flags |= SCRIPT_VERIFY_DERSIG; + } + + // Start enforcing CHECKLOCKTIMEVERIFY (BIP65) rule + if (pindex->nHeight >= consensusparams.BIP65Height) { + flags |= SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY; + } + + // Start enforcing BIP112 (CHECKSEQUENCEVERIFY) + if (pindex->nHeight >= consensusparams.CSVHeight) { + flags |= SCRIPT_VERIFY_CHECKSEQUENCEVERIFY; + } + + // Start enforcing BIP147 NULLDUMMY (activated simultaneously with segwit) + if (IsWitnessEnabled(pindex->pprev, consensusparams)) { + flags |= SCRIPT_VERIFY_NULLDUMMY; + } + + return flags; +} + + static int64_t nTimeCheck = 0; static int64_t nTimeForks = 0; @@ -1734,22 +1916,48 @@ static int64_t nTimeConnect = 0; static int64_t nTimeIndex = 0; static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; +static int64_t nBlocksTotal = 0; -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, +/** Apply the effects of this block (with given index) on the UTXO set represented by coins. + * Validity checks that depend on the UTXO set are also done; ConnectBlock() + * can fail if those validity checks fail (among other reasons). */ +bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck) { AssertLockHeld(cs_main); - + assert(pindex); + assert(*pindex->phashBlock == block.GetHash()); int64_t nTimeStart = GetTimeMicros(); // Check it again in case a previous version let a bad block in - if (!CheckBlock(block, state, chainparams.GetConsensus(), !fJustCheck, !fJustCheck)) - return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); + // NOTE: We don't currently (re-)invoke ContextualCheckBlock() or + // ContextualCheckBlockHeader() here. This means that if we add a new + // consensus rule that is enforced in one of those two functions, then we + // may have let in a block that violates the rule prior to updating the + // software, and we would NOT be enforcing the rule here. Fully solving + // upgrade from one software version to the next after a consensus rule + // change is potentially tricky and issue-specific (see RewindBlockIndex() + // for one general approach that was used for BIP 141 deployment). + // Also, currently the rule against blocks more than 2 hours in the future + // is enforced in ContextualCheckBlockHeader(); we wouldn't want to + // re-enforce that rule here (at least until we make it impossible for + // GetAdjustedTime() to go backward). + if (!CheckBlock(block, state, chainparams.GetConsensus(), !fJustCheck, !fJustCheck)) { + if (state.GetResult() == BlockValidationResult::BLOCK_MUTATED) { + // We don't write down blocks to disk if they may have been + // corrupted, so this should be impossible unless we're having hardware + // problems. + return AbortNode(state, "Corrupt block found indicating potential hardware failure; shutting down"); + } + return error("%s: Consensus::CheckBlock: %s", __func__, state.ToString()); + } // verify that the view's current state corresponds to the previous block - uint256 hashPrevBlock = pindex->pprev == NULL ? uint256() : pindex->pprev->GetBlockHash(); + uint256 hashPrevBlock = pindex->pprev == nullptr ? uint256() : pindex->pprev->GetBlockHash(); assert(hashPrevBlock == view.GetBestBlock()); + nBlocksTotal++; + // Special case for the genesis block, skipping connection of its transactions // (its coinbase is unspendable) if (block.GetHash() == chainparams.GetConsensus().hashGenesisBlock) { @@ -1765,13 +1973,18 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin // relative to a piece of software is an objective fact these defaults can be easily reviewed. // This setting doesn't force the selection of any particular chain but makes validating some faster by // effectively caching the result of part of the verification. - BlockMap::const_iterator it = mapBlockIndex.find(hashAssumeValid); - if (it != mapBlockIndex.end()) { + BlockMap::const_iterator it = m_blockman.m_block_index.find(hashAssumeValid); + if (it != m_blockman.m_block_index.end()) { if (it->second->GetAncestor(pindex->nHeight) == pindex && pindexBestHeader->GetAncestor(pindex->nHeight) == pindex && - pindexBestHeader->nChainWork >= UintToArith256(chainparams.GetConsensus().nMinimumChainWork)) { + pindexBestHeader->nChainWork >= nMinimumChainWork) { // This block is a member of the assumed verified chain and an ancestor of the best header. - // The equivalent time check discourages hashpower from extorting the network via DOS attack + // Script verification is skipped when connecting blocks under the + // assumevalid block. Assuming the assumevalid block is valid this + // is safe because block merkle hashes are still computed and checked, + // Of course, if an assumed valid block is invalid due to false scriptSigs + // this optimization would allow an invalid chain to be accepted. + // The equivalent time check discourages hash power from extorting the network via DOS attack // into accepting an invalid block through telling users they must manually set assumevalid. // Requiring a software change or burying the invalid block, regardless of the setting, makes // it hard to hide the implication of the demand. This also avoids having release candidates @@ -1785,89 +1998,125 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } int64_t nTime1 = GetTimeMicros(); nTimeCheck += nTime1 - nTimeStart; - LogPrint("bench", " - Sanity checks: %.2fms [%.2fs]\n", 0.001 * (nTime1 - nTimeStart), nTimeCheck * 0.000001); + LogPrint(BCLog::BENCH, " - Sanity checks: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime1 - nTimeStart), nTimeCheck * MICRO, nTimeCheck * MILLI / nBlocksTotal); // Do not allow blocks that contain transactions which 'overwrite' older transactions, // unless those are already completely spent. // If such overwrites are allowed, coinbases and transactions depending upon those // can be duplicated to remove the ability to spend the first instance -- even after // being sent to another address. - // See BIP30 and http://r6.ca/blog/20120206T005236Z.html for more information. + // See BIP30, CVE-2012-1909, and http://r6.ca/blog/20120206T005236Z.html for more information. // This logic is not necessary for memory pool transactions, as AcceptToMemoryPool // already refuses previously-known transaction ids entirely. // This rule was originally applied to all blocks with a timestamp after March 15, 2012, 0:00 UTC. // Now that the whole chain is irreversibly beyond that time it is applied to all blocks except the // two in the chain that violate it. This prevents exploiting the issue against nodes during their // initial block download. - bool fEnforceBIP30 = (!pindex->phashBlock) || // Enforce on CreateNewBlock invocations which don't have a hash. - !((pindex->nHeight==91842 && pindex->GetBlockHash() == uint256S("0x00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec")) || + bool fEnforceBIP30 = !((pindex->nHeight==91842 && pindex->GetBlockHash() == uint256S("0x00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec")) || (pindex->nHeight==91880 && pindex->GetBlockHash() == uint256S("0x00000000000743f190a18c5577a3c2d2a1f610ae9601ac046a38084ccb7cd721"))); // Once BIP34 activated it was not possible to create new duplicate coinbases and thus other than starting // with the 2 existing duplicate coinbase pairs, not possible to create overwriting txs. But by the // time BIP34 activated, in each of the existing pairs the duplicate coinbase had overwritten the first - // before the first had been spent. Since those coinbases are sufficiently buried its no longer possible to create further + // before the first had been spent. Since those coinbases are sufficiently buried it's no longer possible to create further // duplicate transactions descending from the known pairs either. // If we're on the known chain at height greater than where BIP34 activated, we can save the db accesses needed for the BIP30 check. + + // BIP34 requires that a block at height X (block X) has its coinbase + // scriptSig start with a CScriptNum of X (indicated height X). The above + // logic of no longer requiring BIP30 once BIP34 activates is flawed in the + // case that there is a block X before the BIP34 height of 227,931 which has + // an indicated height Y where Y is greater than X. The coinbase for block + // X would also be a valid coinbase for block Y, which could be a BIP30 + // violation. An exhaustive search of all mainnet coinbases before the + // BIP34 height which have an indicated height greater than the block height + // reveals many occurrences. The 3 lowest indicated heights found are + // 209,921, 490,897, and 1,983,702 and thus coinbases for blocks at these 3 + // heights would be the first opportunity for BIP30 to be violated. + + // The search reveals a great many blocks which have an indicated height + // greater than 1,983,702, so we simply remove the optimization to skip + // BIP30 checking for blocks at height 1,983,702 or higher. Before we reach + // that block in another 25 years or so, we should take advantage of a + // future consensus change to do a new and improved version of BIP34 that + // will actually prevent ever creating any duplicate coinbases in the + // future. + static constexpr int BIP34_IMPLIES_BIP30_LIMIT = 1983702; + + // There is no potential to create a duplicate coinbase at block 209,921 + // because this is still before the BIP34 height and so explicit BIP30 + // checking is still active. + + // The final case is block 176,684 which has an indicated height of + // 490,897. Unfortunately, this issue was not discovered until about 2 weeks + // before block 490,897 so there was not much opportunity to address this + // case other than to carefully analyze it and determine it would not be a + // problem. Block 490,897 was, in fact, mined with a different coinbase than + // block 176,684, but it is important to note that even if it hadn't been or + // is remined on an alternate fork with a duplicate coinbase, we would still + // not run into a BIP30 violation. This is because the coinbase for 176,684 + // is spent in block 185,956 in transaction + // d4f7fbbf92f4a3014a230b2dc70b8058d02eb36ac06b4a0736d9d60eaa9e8781. This + // spending transaction can't be duplicated because it also spends coinbase + // 0328dd85c331237f18e781d692c92de57649529bd5edf1d01036daea32ffde29. This + // coinbase has an indicated height of over 4.2 billion, and wouldn't be + // duplicatable until that height, and it's currently impossible to create a + // chain that long. Nevertheless we may wish to consider a future soft fork + // which retroactively prevents block 490,897 from creating a duplicate + // coinbase. The two historical BIP30 violations often provide a confusing + // edge case when manipulating the UTXO and it would be simpler not to have + // another edge case to deal with. + + // testnet3 has no blocks before the BIP34 height with indicated heights + // post BIP34 before approximately height 486,000,000 and presumably will + // be reset before it reaches block 1,983,702 and starts doing unnecessary + // BIP30 checking again. + assert(pindex->pprev); CBlockIndex *pindexBIP34height = pindex->pprev->GetAncestor(chainparams.GetConsensus().BIP34Height); //Only continue to enforce if we're below BIP34 activation height or the block hash at that height doesn't correspond. fEnforceBIP30 = fEnforceBIP30 && (!pindexBIP34height || !(pindexBIP34height->GetBlockHash() == chainparams.GetConsensus().BIP34Hash)); - if (fEnforceBIP30) { + // TODO: Remove BIP30 checking from block height 1,983,702 on, once we have a + // consensus change that ensures coinbases at those heights can not + // duplicate earlier coinbases. + if (fEnforceBIP30 || pindex->nHeight >= BIP34_IMPLIES_BIP30_LIMIT) { for (const auto& tx : block.vtx) { - const CCoins* coins = view.AccessCoins(tx->GetHash()); - if (coins && !coins->IsPruned()) - return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), - REJECT_INVALID, "bad-txns-BIP30"); + for (size_t o = 0; o < tx->vout.size(); o++) { + if (view.HaveCoin(COutPoint(tx->GetHash(), o))) { + LogPrintf("ERROR: ConnectBlock(): tried to overwrite transaction\n"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-BIP30"); + } + } } } - // BIP16 didn't become active until Apr 1 2012 - int64_t nBIP16SwitchTime = 1333238400; - bool fStrictPayToScriptHash = (pindex->GetBlockTime() >= nBIP16SwitchTime); - - unsigned int flags = fStrictPayToScriptHash ? SCRIPT_VERIFY_P2SH : SCRIPT_VERIFY_NONE; - - // Start enforcing the DERSIG (BIP66) rule - if (pindex->nHeight >= chainparams.GetConsensus().BIP66Height) { - flags |= SCRIPT_VERIFY_DERSIG; - } - - // Start enforcing CHECKLOCKTIMEVERIFY (BIP65) rule - if (pindex->nHeight >= chainparams.GetConsensus().BIP65Height) { - flags |= SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY; - } - - // Start enforcing BIP68 (sequence locks) and BIP112 (CHECKSEQUENCEVERIFY) using versionbits logic. + // Start enforcing BIP68 (sequence locks) int nLockTimeFlags = 0; - if (VersionBitsState(pindex->pprev, chainparams.GetConsensus(), Consensus::DEPLOYMENT_CSV, versionbitscache) == THRESHOLD_ACTIVE) { - flags |= SCRIPT_VERIFY_CHECKSEQUENCEVERIFY; + if (pindex->nHeight >= chainparams.GetConsensus().CSVHeight) { nLockTimeFlags |= LOCKTIME_VERIFY_SEQUENCE; } - // Start enforcing WITNESS rules using versionbits logic. - if (IsWitnessEnabled(pindex->pprev, chainparams.GetConsensus())) { - flags |= SCRIPT_VERIFY_WITNESS; - flags |= SCRIPT_VERIFY_NULLDUMMY; - } + // Get the script flags for this block + unsigned int flags = GetBlockScriptFlags(pindex, chainparams.GetConsensus()); int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1; - LogPrint("bench", " - Fork checks: %.2fms [%.2fs]\n", 0.001 * (nTime2 - nTime1), nTimeForks * 0.000001); + LogPrint(BCLog::BENCH, " - Fork checks: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime2 - nTime1), nTimeForks * MICRO, nTimeForks * MILLI / nBlocksTotal); CBlockUndo blockundo; - CCheckQueueControl<CScriptCheck> control(fScriptChecks && nScriptCheckThreads ? &scriptcheckqueue : NULL); + // Precomputed transaction data pointers must not be invalidated + // until after `control` has run the script checks (potentially + // in multiple threads). Preallocate the vector size so a new allocation + // doesn't invalidate pointers into the vector, and keep txsdata in scope + // for as long as `control`. + CCheckQueueControl<CScriptCheck> control(fScriptChecks && g_parallel_script_checks ? &scriptcheckqueue : nullptr); + std::vector<PrecomputedTransactionData> txsdata(block.vtx.size()); std::vector<int> prevheights; CAmount nFees = 0; int nInputs = 0; int64_t nSigOpsCost = 0; - CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); - std::vector<std::pair<uint256, CDiskTxPos> > vPos; - vPos.reserve(block.vtx.size()); blockundo.vtxundo.reserve(block.vtx.size() - 1); - std::vector<PrecomputedTransactionData> txdata; - txdata.reserve(block.vtx.size()); // Required so that pointers to individual PrecomputedTransactionData don't get invalidated for (unsigned int i = 0; i < block.vtx.size(); i++) { const CTransaction &tx = *(block.vtx[i]); @@ -1876,21 +2125,31 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (!tx.IsCoinBase()) { - if (!view.HaveInputs(tx)) - return state.DoS(100, error("ConnectBlock(): inputs missing/spent"), - REJECT_INVALID, "bad-txns-inputs-missingorspent"); + CAmount txfee = 0; + TxValidationState tx_state; + if (!Consensus::CheckTxInputs(tx, tx_state, view, pindex->nHeight, txfee)) { + // Any transaction validation failure in ConnectBlock is a block consensus failure + state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, + tx_state.GetRejectReason(), tx_state.GetDebugMessage()); + return error("%s: Consensus::CheckTxInputs: %s, %s", __func__, tx.GetHash().ToString(), state.ToString()); + } + nFees += txfee; + if (!MoneyRange(nFees)) { + LogPrintf("ERROR: %s: accumulated fee in the block out of range.\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-accumulated-fee-outofrange"); + } // Check that transaction is BIP68 final // BIP68 lock checks (as opposed to nLockTime checks) must // be in ConnectBlock because they require the UTXO set prevheights.resize(tx.vin.size()); for (size_t j = 0; j < tx.vin.size(); j++) { - prevheights[j] = view.AccessCoins(tx.vin[j].prevout.hash)->nHeight; + prevheights[j] = view.AccessCoin(tx.vin[j].prevout).nHeight; } if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { - return state.DoS(100, error("%s: contains a non-BIP68-final transaction", __func__), - REJECT_INVALID, "bad-txns-nonfinal"); + LogPrintf("ERROR: %s: contains a non-BIP68-final transaction\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-nonfinal"); } } @@ -1899,20 +2158,23 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin // * p2sh (when P2SH enabled in flags and excludes coinbase) // * witness (when witness enabled in flags and excludes coinbase) nSigOpsCost += GetTransactionSigOpCost(tx, view, flags); - if (nSigOpsCost > MAX_BLOCK_SIGOPS_COST) - return state.DoS(100, error("ConnectBlock(): too many sigops"), - REJECT_INVALID, "bad-blk-sigops"); + if (nSigOpsCost > MAX_BLOCK_SIGOPS_COST) { + LogPrintf("ERROR: ConnectBlock(): too many sigops\n"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-sigops"); + } - txdata.emplace_back(tx); if (!tx.IsCoinBase()) { - nFees += view.GetValueIn(tx)-tx.GetValueOut(); - std::vector<CScriptCheck> vChecks; bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */ - if (!CheckInputs(tx, state, view, fScriptChecks, flags, fCacheResults, txdata[i], nScriptCheckThreads ? &vChecks : NULL)) - return error("ConnectBlock(): CheckInputs on %s failed with %s", - tx.GetHash().ToString(), FormatStateMessage(state)); + TxValidationState tx_state; + if (fScriptChecks && !CheckInputScripts(tx, tx_state, view, flags, fCacheResults, fCacheResults, txsdata[i], g_parallel_script_checks ? &vChecks : nullptr)) { + // Any transaction validation failure in ConnectBlock is a block consensus failure + state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, + tx_state.GetRejectReason(), tx_state.GetDebugMessage()); + return error("ConnectBlock(): CheckInputScripts on %s failed with %s", + tx.GetHash().ToString(), state.ToString()); + } control.Add(vChecks); } @@ -1921,172 +2183,202 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin blockundo.vtxundo.push_back(CTxUndo()); } UpdateCoins(tx, view, i == 0 ? undoDummy : blockundo.vtxundo.back(), pindex->nHeight); - - vPos.push_back(std::make_pair(tx.GetHash(), pos)); - pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION); } int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2; - LogPrint("bench", " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs]\n", (unsigned)block.vtx.size(), 0.001 * (nTime3 - nTime2), 0.001 * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * 0.000001); + LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), MILLI * (nTime3 - nTime2), MILLI * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : MILLI * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * MICRO, nTimeConnect * MILLI / nBlocksTotal); CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, chainparams.GetConsensus()); - if (block.vtx[0]->GetValueOut() > blockReward) - return state.DoS(100, - error("ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)", - block.vtx[0]->GetValueOut(), blockReward), - REJECT_INVALID, "bad-cb-amount"); - - if (!control.Wait()) - return state.DoS(100, false); + if (block.vtx[0]->GetValueOut() > blockReward) { + LogPrintf("ERROR: ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)\n", block.vtx[0]->GetValueOut(), blockReward); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-amount"); + } + + if (!control.Wait()) { + LogPrintf("ERROR: %s: CheckQueue failed\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "block-validation-failed"); + } int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2; - LogPrint("bench", " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs]\n", nInputs - 1, 0.001 * (nTime4 - nTime2), nInputs <= 1 ? 0 : 0.001 * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * 0.000001); + LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs (%.2fms/blk)]\n", nInputs - 1, MILLI * (nTime4 - nTime2), nInputs <= 1 ? 0 : MILLI * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * MICRO, nTimeVerify * MILLI / nBlocksTotal); if (fJustCheck) return true; - // Write undo information to disk - if (pindex->GetUndoPos().IsNull() || !pindex->IsValid(BLOCK_VALID_SCRIPTS)) - { - if (pindex->GetUndoPos().IsNull()) { - CDiskBlockPos _pos; - if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 40)) - return error("ConnectBlock(): FindUndoPos failed"); - if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) - return AbortNode(state, "Failed to write undo data"); - - // update nUndoPos in block index - pindex->nUndoPos = _pos.nPos; - pindex->nStatus |= BLOCK_HAVE_UNDO; - } + if (!WriteUndoDataForBlock(blockundo, state, pindex, chainparams)) + return false; + if (!pindex->IsValid(BLOCK_VALID_SCRIPTS)) { pindex->RaiseValidity(BLOCK_VALID_SCRIPTS); setDirtyBlockIndex.insert(pindex); } - if (fTxIndex) - if (!pblocktree->WriteTxIndex(vPos)) - return AbortNode(state, "Failed to write transaction index"); - + assert(pindex->phashBlock); // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; - LogPrint("bench", " - Index writing: %.2fms [%.2fs]\n", 0.001 * (nTime5 - nTime4), nTimeIndex * 0.000001); - - // Watch for changes to the previous coinbase transaction. - static uint256 hashPrevBestCoinBase; - GetMainSignals().UpdatedTransaction(hashPrevBestCoinBase); - hashPrevBestCoinBase = block.vtx[0]->GetHash(); - + LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime5 - nTime4), nTimeIndex * MICRO, nTimeIndex * MILLI / nBlocksTotal); int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5; - LogPrint("bench", " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime6 - nTime5), nTimeCallbacks * 0.000001); + LogPrint(BCLog::BENCH, " - Callbacks: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime6 - nTime5), nTimeCallbacks * MICRO, nTimeCallbacks * MILLI / nBlocksTotal); return true; } -/** - * Update the on-disk chain state. - * The caches and indexes are flushed depending on the mode we're called with - * if they're too large, if it's been a while since the last write, - * or always and in all cases if we're in prune mode and are deleting files. - */ -bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int nManualPruneHeight) { - int64_t nMempoolUsage = mempool.DynamicMemoryUsage(); - const CChainParams& chainparams = Params(); - LOCK2(cs_main, cs_LastBlockFile); +CoinsCacheSizeState CChainState::GetCoinsCacheSizeState(const CTxMemPool& tx_pool) +{ + return this->GetCoinsCacheSizeState( + tx_pool, + nCoinCacheUsage, + gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); +} + +CoinsCacheSizeState CChainState::GetCoinsCacheSizeState( + const CTxMemPool& tx_pool, + size_t max_coins_cache_size_bytes, + size_t max_mempool_size_bytes) +{ + int64_t nMempoolUsage = tx_pool.DynamicMemoryUsage(); + int64_t cacheSize = CoinsTip().DynamicMemoryUsage(); + int64_t nTotalSpace = + max_coins_cache_size_bytes + std::max<int64_t>(max_mempool_size_bytes - nMempoolUsage, 0); + + //! No need to periodic flush if at least this much space still available. + static constexpr int64_t MAX_BLOCK_COINSDB_USAGE_BYTES = 10 * 1024 * 1024; // 10MB + int64_t large_threshold = + std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE_BYTES); + + if (cacheSize > nTotalSpace) { + LogPrintf("Cache size (%s) exceeds total space (%s)\n", cacheSize, nTotalSpace); + return CoinsCacheSizeState::CRITICAL; + } else if (cacheSize > large_threshold) { + return CoinsCacheSizeState::LARGE; + } + return CoinsCacheSizeState::OK; +} + +bool CChainState::FlushStateToDisk( + const CChainParams& chainparams, + BlockValidationState &state, + FlushStateMode mode, + int nManualPruneHeight) +{ + LOCK(cs_main); + assert(this->CanFlushToDisk()); static int64_t nLastWrite = 0; static int64_t nLastFlush = 0; - static int64_t nLastSetChain = 0; std::set<int> setFilesToPrune; - bool fFlushForPrune = false; + bool full_flush_completed = false; + + const size_t coins_count = CoinsTip().GetCacheSize(); + const size_t coins_mem_usage = CoinsTip().DynamicMemoryUsage(); + try { - if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { - if (nManualPruneHeight > 0) { - FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight); - } else { - FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight()); - fCheckForPruning = false; - } - if (!setFilesToPrune.empty()) { - fFlushForPrune = true; - if (!fHavePruned) { - pblocktree->WriteFlag("prunedblockfiles", true); - fHavePruned = true; + { + bool fFlushForPrune = false; + bool fDoFullFlush = false; + CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(::mempool); + LOCK(cs_LastBlockFile); + if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { + if (nManualPruneHeight > 0) { + LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune (manual)", BCLog::BENCH); + + FindFilesToPruneManual(g_chainman, setFilesToPrune, nManualPruneHeight); + } else { + LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCH); + + FindFilesToPrune(g_chainman, setFilesToPrune, chainparams.PruneAfterHeight()); + fCheckForPruning = false; + } + if (!setFilesToPrune.empty()) { + fFlushForPrune = true; + if (!fHavePruned) { + pblocktree->WriteFlag("prunedblockfiles", true); + fHavePruned = true; + } } } - } - int64_t nNow = GetTimeMicros(); - // Avoid writing/flushing immediately after startup. - if (nLastWrite == 0) { - nLastWrite = nNow; - } - if (nLastFlush == 0) { - nLastFlush = nNow; - } - if (nLastSetChain == 0) { - nLastSetChain = nNow; - } - int64_t nMempoolSizeMax = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - int64_t cacheSize = pcoinsTip->DynamicMemoryUsage(); - int64_t nTotalSpace = nCoinCacheUsage + std::max<int64_t>(nMempoolSizeMax - nMempoolUsage, 0); - // The cache is large and we're within 10% and 100 MiB of the limit, but we have time now (not in the middle of a block processing). - bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - 100 * 1024 * 1024); - // The cache is over the limit, we have to write now. - bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nTotalSpace; - // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. - bool fPeriodicWrite = mode == FLUSH_STATE_PERIODIC && nNow > nLastWrite + (int64_t)DATABASE_WRITE_INTERVAL * 1000000; - // It's been very long since we flushed the cache. Do this infrequently, to optimize cache usage. - bool fPeriodicFlush = mode == FLUSH_STATE_PERIODIC && nNow > nLastFlush + (int64_t)DATABASE_FLUSH_INTERVAL * 1000000; - // Combine all conditions that result in a full cache flush. - bool fDoFullFlush = (mode == FLUSH_STATE_ALWAYS) || fCacheLarge || fCacheCritical || fPeriodicFlush || fFlushForPrune; - // Write blocks and block index to disk. - if (fDoFullFlush || fPeriodicWrite) { - // Depend on nMinDiskSpace to ensure we can write block index - if (!CheckDiskSpace(0)) - return state.Error("out of disk space"); - // First make sure all block and undo data is flushed to disk. - FlushBlockFile(); - // Then update all block file information (which may refer to block and undo files). - { - std::vector<std::pair<int, const CBlockFileInfo*> > vFiles; - vFiles.reserve(setDirtyFileInfo.size()); - for (std::set<int>::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end(); ) { - vFiles.push_back(std::make_pair(*it, &vinfoBlockFile[*it])); - setDirtyFileInfo.erase(it++); + int64_t nNow = GetTimeMicros(); + // Avoid writing/flushing immediately after startup. + if (nLastWrite == 0) { + nLastWrite = nNow; + } + if (nLastFlush == 0) { + nLastFlush = nNow; + } + // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FlushStateMode::PERIODIC && cache_state >= CoinsCacheSizeState::LARGE; + // The cache is over the limit, we have to write now. + bool fCacheCritical = mode == FlushStateMode::IF_NEEDED && cache_state >= CoinsCacheSizeState::CRITICAL; + // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. + bool fPeriodicWrite = mode == FlushStateMode::PERIODIC && nNow > nLastWrite + (int64_t)DATABASE_WRITE_INTERVAL * 1000000; + // It's been very long since we flushed the cache. Do this infrequently, to optimize cache usage. + bool fPeriodicFlush = mode == FlushStateMode::PERIODIC && nNow > nLastFlush + (int64_t)DATABASE_FLUSH_INTERVAL * 1000000; + // Combine all conditions that result in a full cache flush. + fDoFullFlush = (mode == FlushStateMode::ALWAYS) || fCacheLarge || fCacheCritical || fPeriodicFlush || fFlushForPrune; + // Write blocks and block index to disk. + if (fDoFullFlush || fPeriodicWrite) { + // Depend on nMinDiskSpace to ensure we can write block index + if (!CheckDiskSpace(GetBlocksDir())) { + return AbortNode(state, "Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX); + } + { + LOG_TIME_MILLIS_WITH_CATEGORY("write block and undo data to disk", BCLog::BENCH); + + // First make sure all block and undo data is flushed to disk. + FlushBlockFile(); + } + + // Then update all block file information (which may refer to block and undo files). + { + LOG_TIME_MILLIS_WITH_CATEGORY("write block index to disk", BCLog::BENCH); + + std::vector<std::pair<int, const CBlockFileInfo*> > vFiles; + vFiles.reserve(setDirtyFileInfo.size()); + for (std::set<int>::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end(); ) { + vFiles.push_back(std::make_pair(*it, &vinfoBlockFile[*it])); + setDirtyFileInfo.erase(it++); + } + std::vector<const CBlockIndex*> vBlocks; + vBlocks.reserve(setDirtyBlockIndex.size()); + for (std::set<CBlockIndex*>::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) { + vBlocks.push_back(*it); + setDirtyBlockIndex.erase(it++); + } + if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { + return AbortNode(state, "Failed to write to block index database"); + } } - std::vector<const CBlockIndex*> vBlocks; - vBlocks.reserve(setDirtyBlockIndex.size()); - for (std::set<CBlockIndex*>::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) { - vBlocks.push_back(*it); - setDirtyBlockIndex.erase(it++); + // Finally remove any pruned files + if (fFlushForPrune) { + LOG_TIME_MILLIS_WITH_CATEGORY("unlink pruned files", BCLog::BENCH); + + UnlinkPrunedFiles(setFilesToPrune); } - if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { - return AbortNode(state, "Failed to write to block index database"); + nLastWrite = nNow; + } + // Flush best chain related state. This can only be done if the blocks / block index write was also done. + if (fDoFullFlush && !CoinsTip().GetBestBlock().IsNull()) { + LOG_TIME_SECONDS(strprintf("write coins cache to disk (%d coins, %.2fkB)", + coins_count, coins_mem_usage / 1000)); + + // Typical Coin structures on disk are around 48 bytes in size. + // Pushing a new one to the database can cause it to be written + // twice (once in the log, and once in the tables). This is already + // an overestimation, as most will delete an existing entry or + // overwrite one. Still, use a conservative safety factor of 2. + if (!CheckDiskSpace(GetDataDir(), 48 * 2 * 2 * CoinsTip().GetCacheSize())) { + return AbortNode(state, "Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX); } + // Flush the chainstate (which may refer to block index entries). + if (!CoinsTip().Flush()) + return AbortNode(state, "Failed to write to coin database"); + nLastFlush = nNow; + full_flush_completed = true; } - // Finally remove any pruned files - if (fFlushForPrune) - UnlinkPrunedFiles(setFilesToPrune); - nLastWrite = nNow; - } - // Flush best chain related state. This can only be done if the blocks / block index write was also done. - if (fDoFullFlush) { - // Typical CCoins structures on disk are around 128 bytes in size. - // Pushing a new one to the database can cause it to be written - // twice (once in the log, and once in the tables). This is already - // an overestimation, as most will delete an existing entry or - // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(128 * 2 * 2 * pcoinsTip->GetCacheSize())) - return state.Error("out of disk space"); - // Flush the chainstate (which may refer to block index entries). - if (!pcoinsTip->Flush()) - return AbortNode(state, "Failed to write to coin database"); - nLastFlush = nNow; - } - if (fDoFullFlush || ((mode == FLUSH_STATE_ALWAYS || mode == FLUSH_STATE_PERIODIC) && nNow > nLastSetChain + (int64_t)DATABASE_WRITE_INTERVAL * 1000000)) { + } + if (full_flush_completed) { // Update best block in wallet (so we can detect restored wallets). - GetMainSignals().SetBestChain(chainActive.GetLocator()); - nLastSetChain = nNow; + GetMainSignals().ChainStateFlushed(m_chain.GetLocator()); } } catch (const std::runtime_error& e) { return AbortNode(state, std::string("System error while flushing: ") + e.what()); @@ -2094,50 +2386,73 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n return true; } -void FlushStateToDisk() { - CValidationState state; - FlushStateToDisk(state, FLUSH_STATE_ALWAYS); +void CChainState::ForceFlushStateToDisk() { + BlockValidationState state; + const CChainParams& chainparams = Params(); + if (!this->FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS)) { + LogPrintf("%s: failed to flush state (%s)\n", __func__, state.ToString()); + } } -void PruneAndFlush() { - CValidationState state; +void CChainState::PruneAndFlush() { + BlockValidationState state; fCheckForPruning = true; - FlushStateToDisk(state, FLUSH_STATE_NONE); + const CChainParams& chainparams = Params(); + + if (!this->FlushStateToDisk(chainparams, state, FlushStateMode::NONE)) { + LogPrintf("%s: failed to flush state (%s)\n", __func__, state.ToString()); + } } -/** Update chainActive and related internal data structures. */ -void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { - chainActive.SetTip(pindexNew); +static void DoWarning(const std::string& strWarning) +{ + static bool fWarned = false; + SetMiscWarning(strWarning); + if (!fWarned) { + AlertNotify(strWarning); + fWarned = true; + } +} + +/** Private helper function that concatenates warning messages. */ +static void AppendWarning(std::string& res, const std::string& warn) +{ + if (!res.empty()) res += ", "; + res += warn; +} +/** Check warning conditions and do some notifications on new chain tip set. */ +void static UpdateTip(const CBlockIndex* pindexNew, const CChainParams& chainParams) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main) +{ // New best block mempool.AddTransactionsUpdated(1); - cvBlockChange.notify_all(); + { + LOCK(g_best_block_mutex); + g_best_block = pindexNew->GetBlockHash(); + g_best_block_cv.notify_all(); + } - static bool fWarned = false; - std::vector<std::string> warningMessages; - if (!IsInitialBlockDownload()) + std::string warningMessages; + if (!::ChainstateActive().IsInitialBlockDownload()) { int nUpgraded = 0; - const CBlockIndex* pindex = chainActive.Tip(); + const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(bit); ThresholdState state = checker.GetStateFor(pindex, chainParams.GetConsensus(), warningcache[bit]); - if (state == THRESHOLD_ACTIVE || state == THRESHOLD_LOCKED_IN) { - if (state == THRESHOLD_ACTIVE) { - std::string strWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)"), bit); - SetMiscWarning(strWarning); - if (!fWarned) { - AlertNotify(strWarning); - fWarned = true; - } + if (state == ThresholdState::ACTIVE || state == ThresholdState::LOCKED_IN) { + const std::string strWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)").translated, bit); + if (state == ThresholdState::ACTIVE) { + DoWarning(strWarning); } else { - warningMessages.push_back(strprintf("unknown new rules are about to activate (versionbit %i)", bit)); + AppendWarning(warningMessages, strWarning); } } } // Check the version of the last 100 blocks to see if we need to upgrade: - for (int i = 0; i < 100 && pindex != NULL; i++) + for (int i = 0; i < 100 && pindex != nullptr; i++) { int32_t nExpectedVersion = ComputeBlockVersion(pindex->pprev, chainParams.GetConsensus()); if (pindex->nVersion > VERSIONBITS_LAST_OLD_BLOCK_VERSION && (pindex->nVersion & ~nExpectedVersion) != 0) @@ -2145,80 +2460,70 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { pindex = pindex->pprev; } if (nUpgraded > 0) - warningMessages.push_back(strprintf("%d of last 100 blocks have unexpected version", nUpgraded)); - if (nUpgraded > 100/2) - { - std::string strWarning = _("Warning: Unknown block versions being mined! It's possible unknown rules are in effect"); - // notify GetWarnings(), called by Qt and the JSON-RPC code to warn the user: - SetMiscWarning(strWarning); - if (!fWarned) { - AlertNotify(strWarning); - fWarned = true; - } - } + AppendWarning(warningMessages, strprintf(_("%d of last 100 blocks have unexpected version").translated, nUpgraded)); } - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utx)", __func__, - chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, - log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, - DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), - GuessVerificationProgress(chainParams.TxData(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); - if (!warningMessages.empty()) - LogPrintf(" warning='%s'", boost::algorithm::join(warningMessages, ", ")); - LogPrintf("\n"); + LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s\n", __func__, + pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, + log(pindexNew->nChainWork.getdouble())/log(2.0), (unsigned long)pindexNew->nChainTx, + FormatISO8601DateTime(pindexNew->GetBlockTime()), + GuessVerificationProgress(chainParams.TxData(), pindexNew), ::ChainstateActive().CoinsTip().DynamicMemoryUsage() * (1.0 / (1<<20)), ::ChainstateActive().CoinsTip().GetCacheSize(), + !warningMessages.empty() ? strprintf(" warning='%s'", warningMessages) : ""); } -/** Disconnect chainActive's tip. You probably want to call mempool.removeForReorg and manually re-limit mempool size after this, with cs_main held. */ -bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams, bool fBare = false) +/** Disconnect m_chain's tip. + * After calling, the mempool will be in an inconsistent state, with + * transactions from disconnected blocks being added to disconnectpool. You + * should make the mempool consistent again by calling UpdateMempoolForReorg. + * with cs_main held. + * + * If disconnectpool is nullptr, then no disconnected transactions are added to + * disconnectpool (note that the caller is responsible for mempool consistency + * in any case). + */ +bool CChainState::DisconnectTip(BlockValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool) { - CBlockIndex *pindexDelete = chainActive.Tip(); + CBlockIndex *pindexDelete = m_chain.Tip(); assert(pindexDelete); // Read block from disk. - CBlock block; + std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); + CBlock& block = *pblock; if (!ReadBlockFromDisk(block, pindexDelete, chainparams.GetConsensus())) - return AbortNode(state, "Failed to read block"); + return error("DisconnectTip(): Failed to read block"); // Apply the block atomically to the chain state. int64_t nStart = GetTimeMicros(); { - CCoinsViewCache view(pcoinsTip); - if (!DisconnectBlock(block, state, pindexDelete, view)) + CCoinsViewCache view(&CoinsTip()); + assert(view.GetBestBlock() == pindexDelete->GetBlockHash()); + if (DisconnectBlock(block, pindexDelete, view) != DISCONNECT_OK) return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); bool flushed = view.Flush(); assert(flushed); } - LogPrint("bench", "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); + LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * MILLI); // Write the chain state to disk, if necessary. - if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) + if (!FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED)) return false; - if (!fBare) { - // Resurrect mempool transactions from the disconnected block. - std::vector<uint256> vHashUpdate; - for (const auto& it : block.vtx) { - const CTransaction& tx = *it; - // ignore validation errors in resurrected transactions - CValidationState stateDummy; - if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, it, false, NULL, NULL, true)) { - mempool.removeRecursive(tx, MemPoolRemovalReason::REORG); - } else if (mempool.exists(tx.GetHash())) { - vHashUpdate.push_back(tx.GetHash()); - } + if (disconnectpool) { + // Save transactions to re-add to mempool at end of reorg + for (auto it = block.vtx.rbegin(); it != block.vtx.rend(); ++it) { + disconnectpool->addTransaction(*it); + } + while (disconnectpool->DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE * 1000) { + // Drop the earliest entry, and remove its children from the mempool. + auto it = disconnectpool->queuedTx.get<insertion_order>().begin(); + mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + disconnectpool->removeEntry(it); } - // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have - // no in-mempool children, which is generally not true when adding - // previously-confirmed transactions back to the mempool. - // UpdateTransactionsFromBlock finds descendants of any transactions in this - // block that were added back and cleans up the mempool state. - mempool.UpdateTransactionsFromBlock(vHashUpdate); } - // Update chainActive and related variables. + m_chain.SetTip(pindexDelete->pprev); + UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: - for (const auto& tx : block.vtx) { - GetMainSignals().SyncTransaction(*tx, pindexDelete->pprev, CMainSignals::SYNC_TRANSACTION_NOT_IN_BLOCK); - } + GetMainSignals().BlockDisconnected(pblock, pindexDelete); return true; } @@ -2228,69 +2533,105 @@ static int64_t nTimeFlush = 0; static int64_t nTimeChainState = 0; static int64_t nTimePostConnect = 0; +struct PerBlockConnectTrace { + CBlockIndex* pindex = nullptr; + std::shared_ptr<const CBlock> pblock; + PerBlockConnectTrace() {} +}; /** * Used to track blocks whose transactions were applied to the UTXO state as a * part of a single ActivateBestChainStep call. + * + * This class is single-use, once you call GetBlocksConnected() you have to throw + * it away and make a new one. */ -struct ConnectTrace { - std::vector<std::pair<CBlockIndex*, std::shared_ptr<const CBlock> > > blocksConnected; +class ConnectTrace { +private: + std::vector<PerBlockConnectTrace> blocksConnected; + +public: + explicit ConnectTrace() : blocksConnected(1) {} + + void BlockConnected(CBlockIndex* pindex, std::shared_ptr<const CBlock> pblock) { + assert(!blocksConnected.back().pindex); + assert(pindex); + assert(pblock); + blocksConnected.back().pindex = pindex; + blocksConnected.back().pblock = std::move(pblock); + blocksConnected.emplace_back(); + } + + std::vector<PerBlockConnectTrace>& GetBlocksConnected() { + // We always keep one extra block at the end of our list because + // blocks are added after all the conflicted transactions have + // been filled in. Thus, the last entry should always be an empty + // one waiting for the transactions from the next block. We pop + // the last entry here to make sure the list we return is sane. + assert(!blocksConnected.back().pindex); + blocksConnected.pop_back(); + return blocksConnected; + } }; /** - * Connect a new block to chainActive. pblock is either NULL or a pointer to a CBlock + * Connect a new block to m_chain. pblock is either nullptr or a pointer to a CBlock * corresponding to pindexNew, to bypass loading it again from disk. * - * The block is always added to connectTrace (either after loading from disk or by copying - * pblock) - if that is not intended, care must be taken to remove the last entry in - * blocksConnected in case of failure. + * The block is added to connectTrace if connection succeeds. */ -bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace) +bool CChainState::ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) { - assert(pindexNew->pprev == chainActive.Tip()); + assert(pindexNew->pprev == m_chain.Tip()); // Read block from disk. int64_t nTime1 = GetTimeMicros(); + std::shared_ptr<const CBlock> pthisBlock; if (!pblock) { std::shared_ptr<CBlock> pblockNew = std::make_shared<CBlock>(); - connectTrace.blocksConnected.emplace_back(pindexNew, pblockNew); if (!ReadBlockFromDisk(*pblockNew, pindexNew, chainparams.GetConsensus())) return AbortNode(state, "Failed to read block"); + pthisBlock = pblockNew; } else { - connectTrace.blocksConnected.emplace_back(pindexNew, pblock); + pthisBlock = pblock; } - const CBlock& blockConnecting = *connectTrace.blocksConnected.back().second; + const CBlock& blockConnecting = *pthisBlock; // Apply the block atomically to the chain state. int64_t nTime2 = GetTimeMicros(); nTimeReadFromDisk += nTime2 - nTime1; int64_t nTime3; - LogPrint("bench", " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); + LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * MILLI, nTimeReadFromDisk * MICRO); { - CCoinsViewCache view(pcoinsTip); + CCoinsViewCache view(&CoinsTip()); bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, chainparams); GetMainSignals().BlockChecked(blockConnecting, state); if (!rv) { if (state.IsInvalid()) InvalidBlockFound(pindexNew, state); - return error("ConnectTip(): ConnectBlock %s failed", pindexNew->GetBlockHash().ToString()); + return error("%s: ConnectBlock %s failed, %s", __func__, pindexNew->GetBlockHash().ToString(), state.ToString()); } nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; - LogPrint("bench", " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); + assert(nBlocksTotal > 0); + LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime3 - nTime2) * MILLI, nTimeConnectTotal * MICRO, nTimeConnectTotal * MILLI / nBlocksTotal); bool flushed = view.Flush(); assert(flushed); } int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; - LogPrint("bench", " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); + LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime4 - nTime3) * MILLI, nTimeFlush * MICRO, nTimeFlush * MILLI / nBlocksTotal); // Write the chain state to disk, if necessary. - if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) + if (!FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED)) return false; int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; - LogPrint("bench", " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); + LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime5 - nTime4) * MILLI, nTimeChainState * MICRO, nTimeChainState * MILLI / nBlocksTotal); // Remove conflicting transactions from the mempool.; mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); - // Update chainActive & related variables. + disconnectpool.removeForBlock(blockConnecting.vtx); + // Update m_chain & related variables. + m_chain.SetTip(pindexNew); UpdateTip(pindexNew, chainparams); int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; - LogPrint("bench", " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); - LogPrint("bench", "- Connect block: %.2fms [%.2fs]\n", (nTime6 - nTime1) * 0.001, nTimeTotal * 0.000001); + LogPrint(BCLog::BENCH, " - Connect postprocess: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime6 - nTime5) * MILLI, nTimePostConnect * MICRO, nTimePostConnect * MILLI / nBlocksTotal); + LogPrint(BCLog::BENCH, "- Connect block: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime6 - nTime1) * MILLI, nTimeTotal * MICRO, nTimeTotal * MILLI / nBlocksTotal); + + connectTrace.BlockConnected(pindexNew, std::move(pthisBlock)); return true; } @@ -2298,15 +2639,15 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, * Return the tip of the chain with the most work in it, that isn't * known to be invalid (it's however far from certain to be valid). */ -static CBlockIndex* FindMostWorkChain() { +CBlockIndex* CChainState::FindMostWorkChain() { do { - CBlockIndex *pindexNew = NULL; + CBlockIndex *pindexNew = nullptr; // Find the best candidate header. { std::set<CBlockIndex*, CBlockIndexWorkComparator>::reverse_iterator it = setBlockIndexCandidates.rbegin(); if (it == setBlockIndexCandidates.rend()) - return NULL; + return nullptr; pindexNew = *it; } @@ -2314,8 +2655,8 @@ static CBlockIndex* FindMostWorkChain() { // Just going until the active chain is an optimization, as we know all blocks in it are valid already. CBlockIndex *pindexTest = pindexNew; bool fInvalidAncestor = false; - while (pindexTest && !chainActive.Contains(pindexTest)) { - assert(pindexTest->nChainTx || pindexTest->nHeight == 0); + while (pindexTest && !m_chain.Contains(pindexTest)) { + assert(pindexTest->HaveTxsDownloaded() || pindexTest->nHeight == 0); // Pruned nodes may have entries in setBlockIndexCandidates for // which block files have been deleted. Remove those as candidates @@ -2325,7 +2666,7 @@ static CBlockIndex* FindMostWorkChain() { bool fMissingData = !(pindexTest->nStatus & BLOCK_HAVE_DATA); if (fFailedChain || fMissingData) { // Candidate chain is not usable (either invalid or missing data) - if (fFailedChain && (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork)) + if (fFailedChain && (pindexBestInvalid == nullptr || pindexNew->nChainWork > pindexBestInvalid->nChainWork)) pindexBestInvalid = pindexNew; CBlockIndex *pindexFailed = pindexNew; // Remove the entire chain from the set. @@ -2333,10 +2674,11 @@ static CBlockIndex* FindMostWorkChain() { if (fFailedChain) { pindexFailed->nStatus |= BLOCK_FAILED_CHILD; } else if (fMissingData) { - // If we're missing data, then add back to mapBlocksUnlinked, + // If we're missing data, then add back to m_blocks_unlinked, // so that if the block arrives in the future we can try adding // to setBlockIndexCandidates again. - mapBlocksUnlinked.insert(std::make_pair(pindexFailed->pprev, pindexFailed)); + m_blockman.m_blocks_unlinked.insert( + std::make_pair(pindexFailed->pprev, pindexFailed)); } setBlockIndexCandidates.erase(pindexFailed); pindexFailed = pindexFailed->pprev; @@ -2353,11 +2695,11 @@ static CBlockIndex* FindMostWorkChain() { } /** Delete all entries in setBlockIndexCandidates that are worse than the current tip. */ -static void PruneBlockIndexCandidates() { +void CChainState::PruneBlockIndexCandidates() { // Note that we can't delete the current block itself, as we may need to return to it later in case a // reorganization to a better block fails. std::set<CBlockIndex*, CBlockIndexWorkComparator>::iterator it = setBlockIndexCandidates.begin(); - while (it != setBlockIndexCandidates.end() && setBlockIndexCandidates.value_comp()(*it, chainActive.Tip())) { + while (it != setBlockIndexCandidates.end() && setBlockIndexCandidates.value_comp()(*it, m_chain.Tip())) { setBlockIndexCandidates.erase(it++); } // Either the current tip or a successor of it we're working towards is left in setBlockIndexCandidates. @@ -2366,19 +2708,32 @@ static void PruneBlockIndexCandidates() { /** * Try to make some progress towards making pindexMostWork the active block. - * pblock is either NULL or a pointer to a CBlock corresponding to pindexMostWork. + * pblock is either nullptr or a pointer to a CBlock corresponding to pindexMostWork. + * + * @returns true unless a system error occurred */ -static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) +bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) { AssertLockHeld(cs_main); - const CBlockIndex *pindexOldTip = chainActive.Tip(); - const CBlockIndex *pindexFork = chainActive.FindFork(pindexMostWork); + + const CBlockIndex *pindexOldTip = m_chain.Tip(); + const CBlockIndex *pindexFork = m_chain.FindFork(pindexMostWork); // Disconnect active blocks which are no longer in the best chain. bool fBlocksDisconnected = false; - while (chainActive.Tip() && chainActive.Tip() != pindexFork) { - if (!DisconnectTip(state, chainparams)) + DisconnectedBlockTransactions disconnectpool; + while (m_chain.Tip() && m_chain.Tip() != pindexFork) { + if (!DisconnectTip(state, chainparams, &disconnectpool)) { + // This is likely a fatal error, but keep the mempool consistent, + // just in case. Only remove from the mempool in this case. + UpdateMempoolForReorg(disconnectpool, false); + + // If we're unable to disconnect a block during normal operation, + // then that is a failure of our local system -- we should abort + // rather than stay on a less work chain. + AbortNode(state, "Failed to disconnect block; see debug.log for details"); return false; + } fBlocksDisconnected = true; } @@ -2400,25 +2755,27 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c nHeight = nTargetHeight; // Connect new blocks. - BOOST_REVERSE_FOREACH(CBlockIndex *pindexConnect, vpindexToConnect) { - if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace)) { + for (CBlockIndex *pindexConnect : reverse_iterate(vpindexToConnect)) { + if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace, disconnectpool)) { if (state.IsInvalid()) { // The block violates a consensus rule. - if (!state.CorruptionPossible()) - InvalidChainFound(vpindexToConnect.back()); - state = CValidationState(); + if (state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { + InvalidChainFound(vpindexToConnect.front()); + } + state = BlockValidationState(); fInvalidFound = true; fContinue = false; - // If we didn't actually connect the block, don't notify listeners about it - connectTrace.blocksConnected.pop_back(); break; } else { // A system error occurred (disk space, database error, ...). + // Make the mempool consistent with the current tip, just in case + // any observers try to use it before shutdown. + UpdateMempoolForReorg(disconnectpool, false); return false; } } else { PruneBlockIndexCandidates(); - if (!pindexOldTip || chainActive.Tip()->nChainWork > pindexOldTip->nChainWork) { + if (!pindexOldTip || m_chain.Tip()->nChainWork > pindexOldTip->nChainWork) { // We're in a better position than we were. Return temporarily to release the lock. fContinue = false; break; @@ -2428,10 +2785,11 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c } if (fBlocksDisconnected) { - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); - LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); + // If any blocks were disconnected, disconnectpool may be non empty. Add + // any disconnected transactions back to the mempool. + UpdateMempoolForReorg(disconnectpool, true); } - mempool.check(pcoinsTip); + mempool.check(&CoinsTip()); // Callbacks/notifications for a new best chain. if (fInvalidFound) @@ -2442,128 +2800,160 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c return true; } -static void NotifyHeaderTip() { +static SynchronizationState GetSynchronizationState(bool init) +{ + if (!init) return SynchronizationState::POST_INIT; + if (::fReindex) return SynchronizationState::INIT_REINDEX; + return SynchronizationState::INIT_DOWNLOAD; +} + +static bool NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { bool fNotify = false; bool fInitialBlockDownload = false; - static CBlockIndex* pindexHeaderOld = NULL; - CBlockIndex* pindexHeader = NULL; + static CBlockIndex* pindexHeaderOld = nullptr; + CBlockIndex* pindexHeader = nullptr; { LOCK(cs_main); pindexHeader = pindexBestHeader; if (pindexHeader != pindexHeaderOld) { fNotify = true; - fInitialBlockDownload = IsInitialBlockDownload(); + fInitialBlockDownload = ::ChainstateActive().IsInitialBlockDownload(); pindexHeaderOld = pindexHeader; } } // Send block tip changed notifications without cs_main if (fNotify) { - uiInterface.NotifyHeaderTip(fInitialBlockDownload, pindexHeader); + uiInterface.NotifyHeaderTip(GetSynchronizationState(fInitialBlockDownload), pindexHeader); } + return fNotify; } -/** - * Make the best chain active, in multiple steps. The result is either failure - * or an activated best chain. pblock is either NULL or a pointer to a block - * that is already loaded (to avoid loading it again from disk). - */ -bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { +static void LimitValidationInterfaceQueue() LOCKS_EXCLUDED(cs_main) { + AssertLockNotHeld(cs_main); + + if (GetMainSignals().CallbacksPending() > 10) { + SyncWithValidationInterfaceQueue(); + } +} + +bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { // Note that while we're often called here from ProcessNewBlock, this is // far from a guarantee. Things in the P2P/RPC will often end up calling // us in the middle of ProcessNewBlock - do not assume pblock is set // sanely for performance or correctness! + AssertLockNotHeld(cs_main); - CBlockIndex *pindexMostWork = NULL; - CBlockIndex *pindexNewTip = NULL; + // ABC maintains a fair degree of expensive-to-calculate internal state + // because this function periodically releases cs_main so that it does not lock up other threads for too long + // during large connects - and to allow for e.g. the callback queue to drain + // we use m_cs_chainstate to enforce mutual exclusion so that only one caller may execute this function at a time + LOCK(m_cs_chainstate); + + CBlockIndex *pindexMostWork = nullptr; + CBlockIndex *pindexNewTip = nullptr; + int nStopAtHeight = gArgs.GetArg("-stopatheight", DEFAULT_STOPATHEIGHT); do { boost::this_thread::interruption_point(); - if (ShutdownRequested()) - break; - const CBlockIndex *pindexFork; - ConnectTrace connectTrace; - bool fInitialDownload; + // Block until the validation queue drains. This should largely + // never happen in normal operation, however may happen during + // reindex, causing memory blowup if we run too far ahead. + // Note that if a validationinterface callback ends up calling + // ActivateBestChain this may lead to a deadlock! We should + // probably have a DEBUG_LOCKORDER test for this in the future. + LimitValidationInterfaceQueue(); + { - LOCK(cs_main); - { // TODO: Tempoarily ensure that mempool removals are notified before - // connected transactions. This shouldn't matter, but the abandoned - // state of transactions in our wallet is currently cleared when we - // receive another notification and there is a race condition where - // notification of a connected conflict might cause an outside process - // to abandon a transaction and then have it inadvertantly cleared by - // the notification that the conflicted transaction was evicted. - MemPoolConflictRemovalTracker mrt(mempool); - CBlockIndex *pindexOldTip = chainActive.Tip(); - if (pindexMostWork == NULL) { - pindexMostWork = FindMostWorkChain(); - } + LOCK2(cs_main, ::mempool.cs); // Lock transaction pool for at least as long as it takes for connectTrace to be consumed + CBlockIndex* starting_tip = m_chain.Tip(); + bool blocks_connected = false; + do { + // We absolutely may not unlock cs_main until we've made forward progress + // (with the exception of shutdown due to hardware issues, low disk space, etc). + ConnectTrace connectTrace; // Destructed before cs_main is unlocked + + if (pindexMostWork == nullptr) { + pindexMostWork = FindMostWorkChain(); + } - // Whether we have anything to do at all. - if (pindexMostWork == NULL || pindexMostWork == chainActive.Tip()) - return true; + // Whether we have anything to do at all. + if (pindexMostWork == nullptr || pindexMostWork == m_chain.Tip()) { + break; + } - bool fInvalidFound = false; - std::shared_ptr<const CBlock> nullBlockPtr; - if (!ActivateBestChainStep(state, chainparams, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : nullBlockPtr, fInvalidFound, connectTrace)) - return false; + bool fInvalidFound = false; + std::shared_ptr<const CBlock> nullBlockPtr; + if (!ActivateBestChainStep(state, chainparams, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : nullBlockPtr, fInvalidFound, connectTrace)) { + // A system error occurred + return false; + } + blocks_connected = true; - if (fInvalidFound) { - // Wipe cache, we may need another branch now. - pindexMostWork = NULL; - } - pindexNewTip = chainActive.Tip(); - pindexFork = chainActive.FindFork(pindexOldTip); - fInitialDownload = IsInitialBlockDownload(); + if (fInvalidFound) { + // Wipe cache, we may need another branch now. + pindexMostWork = nullptr; + } + pindexNewTip = m_chain.Tip(); + + for (const PerBlockConnectTrace& trace : connectTrace.GetBlocksConnected()) { + assert(trace.pblock && trace.pindex); + GetMainSignals().BlockConnected(trace.pblock, trace.pindex); + } + } while (!m_chain.Tip() || (starting_tip && CBlockIndexWorkComparator()(m_chain.Tip(), starting_tip))); + if (!blocks_connected) return true; - // throw all transactions though the signal-interface + const CBlockIndex* pindexFork = m_chain.FindFork(starting_tip); + bool fInitialDownload = IsInitialBlockDownload(); - } // MemPoolConflictRemovalTracker destroyed and conflict evictions are notified + // Notify external listeners about the new tip. + // Enqueue while holding cs_main to ensure that UpdatedBlockTip is called in the order in which blocks are connected + if (pindexFork != pindexNewTip) { + // Notify ValidationInterface subscribers + GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload); - // Transactions in the connnected block are notified - for (const auto& pair : connectTrace.blocksConnected) { - assert(pair.second); - const CBlock& block = *(pair.second); - for (unsigned int i = 0; i < block.vtx.size(); i++) - GetMainSignals().SyncTransaction(*block.vtx[i], pair.first, i); + // Always notify the UI if a new block tip was connected + uiInterface.NotifyBlockTip(GetSynchronizationState(fInitialDownload), pindexNewTip); } } // When we reach this point, we switched to a new tip (stored in pindexNewTip). - // Notifications/callbacks that can run without cs_main - - // Notify external listeners about the new tip. - GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload); + if (nStopAtHeight && pindexNewTip && pindexNewTip->nHeight >= nStopAtHeight) StartShutdown(); - // Always notify the UI if a new block tip was connected - if (pindexFork != pindexNewTip) { - uiInterface.NotifyBlockTip(fInitialDownload, pindexNewTip); - } + // We check shutdown only after giving ActivateBestChainStep a chance to run once so that we + // never shutdown before connecting the genesis block during LoadChainTip(). Previously this + // caused an assert() failure during shutdown in such cases as the UTXO DB flushing checks + // that the best block hash is non-null. + if (ShutdownRequested()) + break; } while (pindexNewTip != pindexMostWork); CheckBlockIndex(chainparams.GetConsensus()); // Write changes periodically to disk, after relay. - if (!FlushStateToDisk(state, FLUSH_STATE_PERIODIC)) { + if (!FlushStateToDisk(chainparams, state, FlushStateMode::PERIODIC)) { return false; } return true; } +bool ActivateBestChain(BlockValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { + return ::ChainstateActive().ActivateBestChain(state, chainparams, std::move(pblock)); +} -bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) +bool CChainState::PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex *pindex) { { LOCK(cs_main); - if (pindex->nChainWork < chainActive.Tip()->nChainWork) { + if (pindex->nChainWork < m_chain.Tip()->nChainWork) { // Nothing to do, this block is not at the tip. return true; } - if (chainActive.Tip()->nChainWork > nLastPreciousChainwork) { + if (m_chain.Tip()->nChainWork > nLastPreciousChainwork) { // The chain has been extended since the last call, reset the counter. nBlockReverseSequenceId = -1; } - nLastPreciousChainwork = chainActive.Tip()->nChainWork; + nLastPreciousChainwork = m_chain.Tip()->nChainWork; setBlockIndexCandidates.erase(pindex); pindex->nSequenceId = nBlockReverseSequenceId; if (nBlockReverseSequenceId > std::numeric_limits<int32_t>::min()) { @@ -2571,107 +2961,216 @@ bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIn // call preciousblock 2**31-1 times on the same set of tips... nBlockReverseSequenceId--; } - if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && pindex->nChainTx) { + if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && pindex->HaveTxsDownloaded()) { setBlockIndexCandidates.insert(pindex); PruneBlockIndexCandidates(); } } - return ActivateBestChain(state, params); + return ActivateBestChain(state, params, std::shared_ptr<const CBlock>()); +} +bool PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex *pindex) { + return ::ChainstateActive().PreciousBlock(state, params, pindex); } -bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) +bool CChainState::InvalidateBlock(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { - AssertLockHeld(cs_main); + CBlockIndex* to_mark_failed = pindex; + bool pindex_was_in_chain = false; + int disconnected = 0; + + // We do not allow ActivateBestChain() to run while InvalidateBlock() is + // running, as that could cause the tip to change while we disconnect + // blocks. + LOCK(m_cs_chainstate); + + // We'll be acquiring and releasing cs_main below, to allow the validation + // callbacks to run. However, we should keep the block index in a + // consistent state as we disconnect blocks -- in particular we need to + // add equal-work blocks to setBlockIndexCandidates as we disconnect. + // To avoid walking the block index repeatedly in search of candidates, + // build a map once so that we can look up candidate blocks by chain + // work as we go. + std::multimap<const arith_uint256, CBlockIndex *> candidate_blocks_by_work; + + { + LOCK(cs_main); + for (const auto& entry : m_blockman.m_block_index) { + CBlockIndex *candidate = entry.second; + // We don't need to put anything in our active chain into the + // multimap, because those candidates will be found and considered + // as we disconnect. + // Instead, consider only non-active-chain blocks that have at + // least as much work as where we expect the new tip to end up. + if (!m_chain.Contains(candidate) && + !CBlockIndexWorkComparator()(candidate, pindex->pprev) && + candidate->IsValid(BLOCK_VALID_TRANSACTIONS) && + candidate->HaveTxsDownloaded()) { + candidate_blocks_by_work.insert(std::make_pair(candidate->nChainWork, candidate)); + } + } + } - // Mark the block itself as invalid. - pindex->nStatus |= BLOCK_FAILED_VALID; - setDirtyBlockIndex.insert(pindex); - setBlockIndexCandidates.erase(pindex); - - while (chainActive.Contains(pindex)) { - CBlockIndex *pindexWalk = chainActive.Tip(); - pindexWalk->nStatus |= BLOCK_FAILED_CHILD; - setDirtyBlockIndex.insert(pindexWalk); - setBlockIndexCandidates.erase(pindexWalk); - // ActivateBestChain considers blocks already in chainActive + // Disconnect (descendants of) pindex, and mark them invalid. + while (true) { + if (ShutdownRequested()) break; + + // Make sure the queue of validation callbacks doesn't grow unboundedly. + LimitValidationInterfaceQueue(); + + LOCK(cs_main); + LOCK(::mempool.cs); // Lock for as long as disconnectpool is in scope to make sure UpdateMempoolForReorg is called after DisconnectTip without unlocking in between + if (!m_chain.Contains(pindex)) break; + pindex_was_in_chain = true; + CBlockIndex *invalid_walk_tip = m_chain.Tip(); + + // ActivateBestChain considers blocks already in m_chain // unconditionally valid already, so force disconnect away from it. - if (!DisconnectTip(state, chainparams)) { - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); - return false; + DisconnectedBlockTransactions disconnectpool; + bool ret = DisconnectTip(state, chainparams, &disconnectpool); + // DisconnectTip will add transactions to disconnectpool. + // Adjust the mempool to be consistent with the new tip, adding + // transactions back to the mempool if disconnecting was successful, + // and we're not doing a very deep invalidation (in which case + // keeping the mempool up to date is probably futile anyway). + UpdateMempoolForReorg(disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); + if (!ret) return false; + assert(invalid_walk_tip->pprev == m_chain.Tip()); + + // We immediately mark the disconnected blocks as invalid. + // This prevents a case where pruned nodes may fail to invalidateblock + // and be left unable to start as they have no tip candidates (as there + // are no blocks that meet the "have data and are not invalid per + // nStatus" criteria for inclusion in setBlockIndexCandidates). + invalid_walk_tip->nStatus |= BLOCK_FAILED_VALID; + setDirtyBlockIndex.insert(invalid_walk_tip); + setBlockIndexCandidates.erase(invalid_walk_tip); + setBlockIndexCandidates.insert(invalid_walk_tip->pprev); + if (invalid_walk_tip->pprev == to_mark_failed && (to_mark_failed->nStatus & BLOCK_FAILED_VALID)) { + // We only want to mark the last disconnected block as BLOCK_FAILED_VALID; its children + // need to be BLOCK_FAILED_CHILD instead. + to_mark_failed->nStatus = (to_mark_failed->nStatus ^ BLOCK_FAILED_VALID) | BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(to_mark_failed); + } + + // Add any equal or more work headers to setBlockIndexCandidates + auto candidate_it = candidate_blocks_by_work.lower_bound(invalid_walk_tip->pprev->nChainWork); + while (candidate_it != candidate_blocks_by_work.end()) { + if (!CBlockIndexWorkComparator()(candidate_it->second, invalid_walk_tip->pprev)) { + setBlockIndexCandidates.insert(candidate_it->second); + candidate_it = candidate_blocks_by_work.erase(candidate_it); + } else { + ++candidate_it; + } } + + // Track the last disconnected block, so we can correct its BLOCK_FAILED_CHILD status in future + // iterations, or, if it's the last one, call InvalidChainFound on it. + to_mark_failed = invalid_walk_tip; } - LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); + CheckBlockIndex(chainparams.GetConsensus()); + + { + LOCK(cs_main); + if (m_chain.Contains(to_mark_failed)) { + // If the to-be-marked invalid block is in the active chain, something is interfering and we can't proceed. + return false; + } - // The resulting new best tip may not be in setBlockIndexCandidates anymore, so - // add it again. - BlockMap::iterator it = mapBlockIndex.begin(); - while (it != mapBlockIndex.end()) { - if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->nChainTx && !setBlockIndexCandidates.value_comp()(it->second, chainActive.Tip())) { - setBlockIndexCandidates.insert(it->second); + // Mark pindex (or the last disconnected block) as invalid, even when it never was in the main chain + to_mark_failed->nStatus |= BLOCK_FAILED_VALID; + setDirtyBlockIndex.insert(to_mark_failed); + setBlockIndexCandidates.erase(to_mark_failed); + m_blockman.m_failed_blocks.insert(to_mark_failed); + + // If any new blocks somehow arrived while we were disconnecting + // (above), then the pre-calculation of what should go into + // setBlockIndexCandidates may have missed entries. This would + // technically be an inconsistency in the block index, but if we clean + // it up here, this should be an essentially unobservable error. + // Loop back over all block index entries and add any missing entries + // to setBlockIndexCandidates. + BlockMap::iterator it = m_blockman.m_block_index.begin(); + while (it != m_blockman.m_block_index.end()) { + if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->HaveTxsDownloaded() && !setBlockIndexCandidates.value_comp()(it->second, m_chain.Tip())) { + setBlockIndexCandidates.insert(it->second); + } + it++; } - it++; + + InvalidChainFound(to_mark_failed); } - InvalidChainFound(pindex); - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); - uiInterface.NotifyBlockTip(IsInitialBlockDownload(), pindex->pprev); + // Only notify about a new block tip if the active chain was modified. + if (pindex_was_in_chain) { + uiInterface.NotifyBlockTip(GetSynchronizationState(IsInitialBlockDownload()), to_mark_failed->pprev); + } return true; } -bool ResetBlockFailureFlags(CBlockIndex *pindex) { +bool InvalidateBlock(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { + return ::ChainstateActive().InvalidateBlock(state, chainparams, pindex); +} + +void CChainState::ResetBlockFailureFlags(CBlockIndex *pindex) { AssertLockHeld(cs_main); int nHeight = pindex->nHeight; // Remove the invalidity flag from this block and all its descendants. - BlockMap::iterator it = mapBlockIndex.begin(); - while (it != mapBlockIndex.end()) { + BlockMap::iterator it = m_blockman.m_block_index.begin(); + while (it != m_blockman.m_block_index.end()) { if (!it->second->IsValid() && it->second->GetAncestor(nHeight) == pindex) { it->second->nStatus &= ~BLOCK_FAILED_MASK; setDirtyBlockIndex.insert(it->second); - if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->nChainTx && setBlockIndexCandidates.value_comp()(chainActive.Tip(), it->second)) { + if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->HaveTxsDownloaded() && setBlockIndexCandidates.value_comp()(m_chain.Tip(), it->second)) { setBlockIndexCandidates.insert(it->second); } if (it->second == pindexBestInvalid) { // Reset invalid block marker if it was pointing to one of those. - pindexBestInvalid = NULL; + pindexBestInvalid = nullptr; } + m_blockman.m_failed_blocks.erase(it->second); } it++; } // Remove the invalidity flag from all ancestors too. - while (pindex != NULL) { + while (pindex != nullptr) { if (pindex->nStatus & BLOCK_FAILED_MASK) { pindex->nStatus &= ~BLOCK_FAILED_MASK; setDirtyBlockIndex.insert(pindex); + m_blockman.m_failed_blocks.erase(pindex); } pindex = pindex->pprev; } - return true; } -CBlockIndex* AddToBlockIndex(const CBlockHeader& block) +void ResetBlockFailureFlags(CBlockIndex *pindex) { + return ::ChainstateActive().ResetBlockFailureFlags(pindex); +} + +CBlockIndex* BlockManager::AddToBlockIndex(const CBlockHeader& block) { + AssertLockHeld(cs_main); + // Check for duplicate uint256 hash = block.GetHash(); - BlockMap::iterator it = mapBlockIndex.find(hash); - if (it != mapBlockIndex.end()) + BlockMap::iterator it = m_block_index.find(hash); + if (it != m_block_index.end()) return it->second; // Construct new block index object CBlockIndex* pindexNew = new CBlockIndex(block); - assert(pindexNew); // We assign the sequence id to blocks only when the full data is available, // to avoid miners withholding blocks but broadcasting headers, to get a // competitive advantage. pindexNew->nSequenceId = 0; - BlockMap::iterator mi = mapBlockIndex.insert(std::make_pair(hash, pindexNew)).first; + BlockMap::iterator mi = m_block_index.insert(std::make_pair(hash, pindexNew)).first; pindexNew->phashBlock = &((*mi).first); - BlockMap::iterator miPrev = mapBlockIndex.find(block.hashPrevBlock); - if (miPrev != mapBlockIndex.end()) + BlockMap::iterator miPrev = m_block_index.find(block.hashPrevBlock); + if (miPrev != m_block_index.end()) { pindexNew->pprev = (*miPrev).second; pindexNew->nHeight = pindexNew->pprev->nHeight + 1; @@ -2680,7 +3179,7 @@ CBlockIndex* AddToBlockIndex(const CBlockHeader& block) pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime); pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew); pindexNew->RaiseValidity(BLOCK_VALID_TREE); - if (pindexBestHeader == NULL || pindexBestHeader->nChainWork < pindexNew->nChainWork) + if (pindexBestHeader == nullptr || pindexBestHeader->nChainWork < pindexNew->nChainWork) pindexBestHeader = pindexNew; setDirtyBlockIndex.insert(pindexNew); @@ -2689,7 +3188,7 @@ CBlockIndex* AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos) +void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2697,13 +3196,13 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; - if (IsWitnessEnabled(pindexNew->pprev, Params().GetConsensus())) { + if (IsWitnessEnabled(pindexNew->pprev, consensusParams)) { pindexNew->nStatus |= BLOCK_OPT_WITNESS; } pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); setDirtyBlockIndex.insert(pindexNew); - if (pindexNew->pprev == NULL || pindexNew->pprev->nChainTx) { + if (pindexNew->pprev == nullptr || pindexNew->pprev->HaveTxsDownloaded()) { // If pindexNew is the genesis block or all parents are BLOCK_VALID_TRANSACTIONS. std::deque<CBlockIndex*> queue; queue.push_back(pindexNew); @@ -2717,27 +3216,25 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl LOCK(cs_nBlockSequenceId); pindex->nSequenceId = nBlockSequenceId++; } - if (chainActive.Tip() == NULL || !setBlockIndexCandidates.value_comp()(pindex, chainActive.Tip())) { + if (m_chain.Tip() == nullptr || !setBlockIndexCandidates.value_comp()(pindex, m_chain.Tip())) { setBlockIndexCandidates.insert(pindex); } - std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> range = mapBlocksUnlinked.equal_range(pindex); + std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> range = m_blockman.m_blocks_unlinked.equal_range(pindex); while (range.first != range.second) { std::multimap<CBlockIndex*, CBlockIndex*>::iterator it = range.first; queue.push_back(it->second); range.first++; - mapBlocksUnlinked.erase(it); + m_blockman.m_blocks_unlinked.erase(it); } } } else { if (pindexNew->pprev && pindexNew->pprev->IsValid(BLOCK_VALID_TREE)) { - mapBlocksUnlinked.insert(std::make_pair(pindexNew->pprev, pindexNew)); + m_blockman.m_blocks_unlinked.insert(std::make_pair(pindexNew->pprev, pindexNew)); } } - - return true; } -bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) +static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) { LOCK(cs_LastBlockFile); @@ -2772,21 +3269,13 @@ bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAdd vinfoBlockFile[nFile].nSize += nAddSize; if (!fKnown) { - unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - if (fPruneMode) - fCheckForPruning = true; - if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos)) { - FILE *file = OpenBlockFile(pos); - if (file) { - LogPrintf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos); - fclose(file); - } - } - else - return state.Error("out of disk space"); + bool out_of_space; + size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode("Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; } } @@ -2794,47 +3283,38 @@ bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAdd return true; } -bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize) +static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize) { pos.nFile = nFile; LOCK(cs_LastBlockFile); - unsigned int nNewSize; pos.nPos = vinfoBlockFile[nFile].nUndoSize; - nNewSize = vinfoBlockFile[nFile].nUndoSize += nAddSize; + vinfoBlockFile[nFile].nUndoSize += nAddSize; setDirtyFileInfo.insert(nFile); - unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; - unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - if (fPruneMode) - fCheckForPruning = true; - if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos)) { - FILE *file = OpenUndoFile(pos); - if (file) { - LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos); - fclose(file); - } - } - else - return state.Error("out of disk space"); + bool out_of_space; + size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode(state, "Disk space is too low!", _("Error: Disk space is too low!").translated, CClientUIInterface::MSG_NOPREFIX); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; } return true; } -bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW) +static bool CheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true) { // Check proof of work matches claimed amount if (fCheckPOW && !CheckProofOfWork(block.GetHash(), block.nBits, consensusParams)) - return state.DoS(50, false, REJECT_INVALID, "high-hash", false, "proof of work failed"); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, "high-hash", "proof of work failed"); return true; } -bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW, bool fCheckMerkleRoot) +bool CheckBlock(const CBlock& block, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW, bool fCheckMerkleRoot) { // These are checks that are independent of context. @@ -2851,13 +3331,13 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P bool mutated; uint256 hashMerkleRoot2 = BlockMerkleRoot(block, &mutated); if (block.hashMerkleRoot != hashMerkleRoot2) - return state.DoS(100, false, REJECT_INVALID, "bad-txnmrklroot", true, "hashMerkleRoot mismatch"); + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "bad-txnmrklroot", "hashMerkleRoot mismatch"); // Check for merkle tree malleability (CVE-2012-2459): repeating sequences // of transactions in a block without affecting the merkle root of a block, // while still invalidating it. if (mutated) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-duplicate", true, "duplicate transaction"); + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "bad-txns-duplicate", "duplicate transaction"); } // All potential-corruption validation must be done before we do any @@ -2867,29 +3347,35 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P // checks that use witness data may be performed here. // Size limits - if (block.vtx.empty() || block.vtx.size() > MAX_BLOCK_BASE_SIZE || ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) > MAX_BLOCK_BASE_SIZE) - return state.DoS(100, false, REJECT_INVALID, "bad-blk-length", false, "size limits failed"); + if (block.vtx.empty() || block.vtx.size() * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT || ::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-length", "size limits failed"); // First transaction must be coinbase, the rest must not be if (block.vtx.empty() || !block.vtx[0]->IsCoinBase()) - return state.DoS(100, false, REJECT_INVALID, "bad-cb-missing", false, "first tx is not coinbase"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-missing", "first tx is not coinbase"); for (unsigned int i = 1; i < block.vtx.size(); i++) if (block.vtx[i]->IsCoinBase()) - return state.DoS(100, false, REJECT_INVALID, "bad-cb-multiple", false, "more than one coinbase"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-multiple", "more than one coinbase"); // Check transactions - for (const auto& tx : block.vtx) - if (!CheckTransaction(*tx, state, false)) - return state.Invalid(false, state.GetRejectCode(), state.GetRejectReason(), - strprintf("Transaction check failed (tx hash %s) %s", tx->GetHash().ToString(), state.GetDebugMessage())); - + // Must check for duplicate inputs (see CVE-2018-17144) + for (const auto& tx : block.vtx) { + TxValidationState tx_state; + if (!CheckTransaction(*tx, tx_state)) { + // CheckBlock() does context-free validation checks. The only + // possible failures are consensus failures. + assert(tx_state.GetResult() == TxValidationResult::TX_CONSENSUS); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, tx_state.GetRejectReason(), + strprintf("Transaction check failed (tx hash %s) %s", tx->GetHash().ToString(), tx_state.GetDebugMessage())); + } + } unsigned int nSigOps = 0; for (const auto& tx : block.vtx) { nSigOps += GetLegacySigOpCount(*tx); } if (nSigOps * WITNESS_SCALE_FACTOR > MAX_BLOCK_SIGOPS_COST) - return state.DoS(100, false, REJECT_INVALID, "bad-blk-sigops", false, "out-of-bounds SigOpCount"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-sigops", "out-of-bounds SigOpCount"); if (fCheckPOW && fCheckMerkleRoot) block.fChecked = true; @@ -2897,34 +3383,27 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P return true; } -static bool CheckIndexAgainstCheckpoint(const CBlockIndex* pindexPrev, CValidationState& state, const CChainParams& chainparams, const uint256& hash) -{ - if (*pindexPrev->phashBlock == chainparams.GetConsensus().hashGenesisBlock) - return true; - - int nHeight = pindexPrev->nHeight+1; - // Don't accept any forks from the main chain prior to last checkpoint - CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(chainparams.Checkpoints()); - if (pcheckpoint && nHeight < pcheckpoint->nHeight) - return state.DoS(100, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight)); - - return true; -} - bool IsWitnessEnabled(const CBlockIndex* pindexPrev, const Consensus::Params& params) { - LOCK(cs_main); - return (VersionBitsState(pindexPrev, params, Consensus::DEPLOYMENT_SEGWIT, versionbitscache) == THRESHOLD_ACTIVE); + int height = pindexPrev == nullptr ? 0 : pindexPrev->nHeight + 1; + return (height >= params.SegwitHeight); } -// Compute at which vout of the block's coinbase transaction the witness -// commitment occurs, or -1 if not found. -static int GetWitnessCommitmentIndex(const CBlock& block) +int GetWitnessCommitmentIndex(const CBlock& block) { int commitpos = -1; - for (size_t o = 0; o < block.vtx[0]->vout.size(); o++) { - if (block.vtx[0]->vout[o].scriptPubKey.size() >= 38 && block.vtx[0]->vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0]->vout[o].scriptPubKey[1] == 0x24 && block.vtx[0]->vout[o].scriptPubKey[2] == 0xaa && block.vtx[0]->vout[o].scriptPubKey[3] == 0x21 && block.vtx[0]->vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0]->vout[o].scriptPubKey[5] == 0xed) { - commitpos = o; + if (!block.vtx.empty()) { + for (size_t o = 0; o < block.vtx[0]->vout.size(); o++) { + const CTxOut& vout = block.vtx[0]->vout[o]; + if (vout.scriptPubKey.size() >= MINIMUM_WITNESS_COMMITMENT && + vout.scriptPubKey[0] == OP_RETURN && + vout.scriptPubKey[1] == 0x24 && + vout.scriptPubKey[2] == 0xaa && + vout.scriptPubKey[3] == 0x21 && + vout.scriptPubKey[4] == 0xa9 && + vout.scriptPubKey[5] == 0xed) { + commitpos = o; + } } } return commitpos; @@ -2947,13 +3426,13 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc std::vector<unsigned char> commitment; int commitpos = GetWitnessCommitmentIndex(block); std::vector<unsigned char> ret(32, 0x00); - if (consensusParams.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout != 0) { + if (consensusParams.SegwitHeight != std::numeric_limits<int>::max()) { if (commitpos == -1) { - uint256 witnessroot = BlockWitnessMerkleRoot(block, NULL); - CHash256().Write(witnessroot.begin(), 32).Write(&ret[0], 32).Finalize(witnessroot.begin()); + uint256 witnessroot = BlockWitnessMerkleRoot(block, nullptr); + CHash256().Write(witnessroot.begin(), 32).Write(ret.data(), 32).Finalize(witnessroot.begin()); CTxOut out; out.nValue = 0; - out.scriptPubKey.resize(38); + out.scriptPubKey.resize(MINIMUM_WITNESS_COMMITMENT); out.scriptPubKey[0] = OP_RETURN; out.scriptPubKey[1] = 0x24; out.scriptPubKey[2] = 0xaa; @@ -2971,39 +3450,86 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc return commitment; } -bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) +//! Returns last CBlockIndex* that is a checkpoint +static CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + const MapCheckpoints& checkpoints = data.mapCheckpoints; + + for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) + { + const uint256& hash = i.second; + CBlockIndex* pindex = LookupBlockIndex(hash); + if (pindex) { + return pindex; + } + } + return nullptr; +} + +/** Context-dependent validity checks. + * By "context", we mean only the previous block headers, but not the UTXO + * set; UTXO-related validity checks are done in ConnectBlock(). + * NOTE: This function is not currently invoked by ConnectBlock(), so we + * should consider upgrade issues if we change which consensus rules are + * enforced in this function (eg by adding a new consensus rule). See comment + * in ConnectBlock(). + * Note that -reindex-chainstate skips the validation that happens here! + */ +static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, const CChainParams& params, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - const int nHeight = pindexPrev == NULL ? 0 : pindexPrev->nHeight + 1; + assert(pindexPrev != nullptr); + const int nHeight = pindexPrev->nHeight + 1; + // Check proof of work + const Consensus::Params& consensusParams = params.GetConsensus(); if (block.nBits != GetNextWorkRequired(pindexPrev, &block, consensusParams)) - return state.DoS(100, false, REJECT_INVALID, "bad-diffbits", false, "incorrect proof of work"); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, "bad-diffbits", "incorrect proof of work"); + + // Check against checkpoints + if (fCheckpointsEnabled) { + // Don't accept any forks from the main chain prior to last checkpoint. + // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our + // BlockIndex(). + CBlockIndex* pcheckpoint = GetLastCheckpoint(params.Checkpoints()); + if (pcheckpoint && nHeight < pcheckpoint->nHeight) { + LogPrintf("ERROR: %s: forked chain older than last checkpoint (height %d)\n", __func__, nHeight); + return state.Invalid(BlockValidationResult::BLOCK_CHECKPOINT, "bad-fork-prior-to-checkpoint"); + } + } // Check timestamp against prev if (block.GetBlockTime() <= pindexPrev->GetMedianTimePast()) - return state.Invalid(false, REJECT_INVALID, "time-too-old", "block's timestamp is too early"); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, "time-too-old", "block's timestamp is too early"); // Check timestamp - if (block.GetBlockTime() > nAdjustedTime + 2 * 60 * 60) - return state.Invalid(false, REJECT_INVALID, "time-too-new", "block timestamp too far in the future"); + if (block.GetBlockTime() > nAdjustedTime + MAX_FUTURE_BLOCK_TIME) + return state.Invalid(BlockValidationResult::BLOCK_TIME_FUTURE, "time-too-new", "block timestamp too far in the future"); // Reject outdated version blocks when 95% (75% on testnet) of the network has upgraded: // check for version 2, 3 and 4 upgrades if((block.nVersion < 2 && nHeight >= consensusParams.BIP34Height) || (block.nVersion < 3 && nHeight >= consensusParams.BIP66Height) || (block.nVersion < 4 && nHeight >= consensusParams.BIP65Height)) - return state.Invalid(false, REJECT_OBSOLETE, strprintf("bad-version(0x%08x)", block.nVersion), + return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, strprintf("bad-version(0x%08x)", block.nVersion), strprintf("rejected nVersion=0x%08x block", block.nVersion)); return true; } -bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) +/** NOTE: This function is not currently invoked by ConnectBlock(), so we + * should consider upgrade issues if we change which consensus rules are + * enforced in this function (eg by adding a new consensus rule). See comment + * in ConnectBlock(). + * Note that -reindex-chainstate skips the validation that happens here! + */ +static bool ContextualCheckBlock(const CBlock& block, BlockValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) { - const int nHeight = pindexPrev == NULL ? 0 : pindexPrev->nHeight + 1; + const int nHeight = pindexPrev == nullptr ? 0 : pindexPrev->nHeight + 1; - // Start enforcing BIP113 (Median Time Past) using versionbits logic. + // Start enforcing BIP113 (Median Time Past). int nLockTimeFlags = 0; - if (VersionBitsState(pindexPrev, consensusParams, Consensus::DEPLOYMENT_CSV, versionbitscache) == THRESHOLD_ACTIVE) { + if (nHeight >= consensusParams.CSVHeight) { + assert(pindexPrev != nullptr); nLockTimeFlags |= LOCKTIME_MEDIAN_TIME_PAST; } @@ -3014,7 +3540,7 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Co // Check that all transactions are finalized for (const auto& tx : block.vtx) { if (!IsFinalTx(*tx, nHeight, nLockTimeCutoff)) { - return state.DoS(10, false, REJECT_INVALID, "bad-txns-nonfinal", false, "non-final transaction"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-txns-nonfinal", "non-final transaction"); } } @@ -3024,20 +3550,20 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Co CScript expect = CScript() << nHeight; if (block.vtx[0]->vin[0].scriptSig.size() < expect.size() || !std::equal(expect.begin(), expect.end(), block.vtx[0]->vin[0].scriptSig.begin())) { - return state.DoS(100, false, REJECT_INVALID, "bad-cb-height", false, "block height mismatch in coinbase"); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-height", "block height mismatch in coinbase"); } } // Validation for witness commitments. // * We compute the witness hash (which is the hash including witnesses) of all the block's transactions, except the // coinbase (where 0x0000....0000 is used instead). - // * The coinbase scriptWitness is a stack of a single 32-byte vector, containing a witness nonce (unconstrained). + // * The coinbase scriptWitness is a stack of a single 32-byte vector, containing a witness reserved value (unconstrained). // * We build a merkle tree with all those witness hashes as leaves (similar to the hashMerkleRoot in the block header). // * There must be at least one output whose scriptPubKey is a single 36-byte push, the first 4 bytes of which are - // {0xaa, 0x21, 0xa9, 0xed}, and the following 32 bytes are SHA256^2(witness root, witness nonce). In case there are + // {0xaa, 0x21, 0xa9, 0xed}, and the following 32 bytes are SHA256^2(witness root, witness reserved value). In case there are // multiple, the last one is used. bool fHaveWitness = false; - if (VersionBitsState(pindexPrev, consensusParams, Consensus::DEPLOYMENT_SEGWIT, versionbitscache) == THRESHOLD_ACTIVE) { + if (nHeight >= consensusParams.SegwitHeight) { int commitpos = GetWitnessCommitmentIndex(block); if (commitpos != -1) { bool malleated = false; @@ -3046,11 +3572,11 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Co // already does not permit it, it is impossible to trigger in the // witness tree. if (block.vtx[0]->vin[0].scriptWitness.stack.size() != 1 || block.vtx[0]->vin[0].scriptWitness.stack[0].size() != 32) { - return state.DoS(100, false, REJECT_INVALID, "bad-witness-nonce-size", true, strprintf("%s : invalid witness nonce size", __func__)); + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "bad-witness-nonce-size", strprintf("%s : invalid witness reserved value size", __func__)); } CHash256().Write(hashWitness.begin(), 32).Write(&block.vtx[0]->vin[0].scriptWitness.stack[0][0], 32).Finalize(hashWitness.begin()); if (memcmp(hashWitness.begin(), &block.vtx[0]->vout[commitpos].scriptPubKey[6], 32)) { - return state.DoS(100, false, REJECT_INVALID, "bad-witness-merkle-match", true, strprintf("%s : witness merkle commitment mismatch", __func__)); + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "bad-witness-merkle-match", strprintf("%s : witness merkle commitment mismatch", __func__)); } fHaveWitness = true; } @@ -3058,83 +3584,125 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Co // No witness data is allowed in blocks that don't commit to witness data, as this would otherwise leave room for spam if (!fHaveWitness) { - for (size_t i = 0; i < block.vtx.size(); i++) { - if (block.vtx[i]->HasWitness()) { - return state.DoS(100, false, REJECT_INVALID, "unexpected-witness", true, strprintf("%s : unexpected witness data found", __func__)); + for (const auto& tx : block.vtx) { + if (tx->HasWitness()) { + return state.Invalid(BlockValidationResult::BLOCK_MUTATED, "unexpected-witness", strprintf("%s : unexpected witness data found", __func__)); } } } - // After the coinbase witness nonce and commitment are verified, + // After the coinbase witness reserved value and commitment are verified, // we can check if the block weight passes (before we've checked the // coinbase witness, it would be possible for the weight to be too // large by filling up the coinbase witness, which doesn't change // the block hash, so we couldn't mark the block as permanently // failed). if (GetBlockWeight(block) > MAX_BLOCK_WEIGHT) { - return state.DoS(100, false, REJECT_INVALID, "bad-blk-weight", false, strprintf("%s : weight limit failed", __func__)); + return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-weight", strprintf("%s : weight limit failed", __func__)); } return true; } -static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) +bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) { AssertLockHeld(cs_main); // Check for duplicate uint256 hash = block.GetHash(); - BlockMap::iterator miSelf = mapBlockIndex.find(hash); - CBlockIndex *pindex = NULL; + BlockMap::iterator miSelf = m_block_index.find(hash); + CBlockIndex *pindex = nullptr; if (hash != chainparams.GetConsensus().hashGenesisBlock) { - - if (miSelf != mapBlockIndex.end()) { + if (miSelf != m_block_index.end()) { // Block header is already known. pindex = miSelf->second; if (ppindex) *ppindex = pindex; - if (pindex->nStatus & BLOCK_FAILED_MASK) - return state.Invalid(error("%s: block %s is marked invalid", __func__, hash.ToString()), 0, "duplicate"); + if (pindex->nStatus & BLOCK_FAILED_MASK) { + LogPrintf("ERROR: %s: block %s is marked invalid\n", __func__, hash.ToString()); + return state.Invalid(BlockValidationResult::BLOCK_CACHED_INVALID, "duplicate"); + } return true; } if (!CheckBlockHeader(block, state, chainparams.GetConsensus())) - return error("%s: Consensus::CheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); + return error("%s: Consensus::CheckBlockHeader: %s, %s", __func__, hash.ToString(), state.ToString()); // Get prev block index - CBlockIndex* pindexPrev = NULL; - BlockMap::iterator mi = mapBlockIndex.find(block.hashPrevBlock); - if (mi == mapBlockIndex.end()) - return state.DoS(10, error("%s: prev block not found", __func__), 0, "bad-prevblk"); + CBlockIndex* pindexPrev = nullptr; + BlockMap::iterator mi = m_block_index.find(block.hashPrevBlock); + if (mi == m_block_index.end()) { + LogPrintf("ERROR: %s: prev block not found\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_MISSING_PREV, "prev-blk-not-found"); + } pindexPrev = (*mi).second; - if (pindexPrev->nStatus & BLOCK_FAILED_MASK) - return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); - - assert(pindexPrev); - if (fCheckpointsEnabled && !CheckIndexAgainstCheckpoint(pindexPrev, state, chainparams, hash)) - return error("%s: CheckIndexAgainstCheckpoint(): %s", __func__, state.GetRejectReason().c_str()); - - if (!ContextualCheckBlockHeader(block, state, chainparams.GetConsensus(), pindexPrev, GetAdjustedTime())) - return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); + if (pindexPrev->nStatus & BLOCK_FAILED_MASK) { + LogPrintf("ERROR: %s: prev block invalid\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); + } + if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) + return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), state.ToString()); + + /* Determine if this block descends from any block which has been found + * invalid (m_failed_blocks), then mark pindexPrev and any blocks between + * them as failed. For example: + * + * D3 + * / + * B2 - C2 + * / \ + * A D2 - E2 - F2 + * \ + * B1 - C1 - D1 - E1 + * + * In the case that we attempted to reorg from E1 to F2, only to find + * C2 to be invalid, we would mark D2, E2, and F2 as BLOCK_FAILED_CHILD + * but NOT D3 (it was not in any of our candidate sets at the time). + * + * In any case D3 will also be marked as BLOCK_FAILED_CHILD at restart + * in LoadBlockIndex. + */ + if (!pindexPrev->IsValid(BLOCK_VALID_SCRIPTS)) { + // The above does not mean "invalid": it checks if the previous block + // hasn't been validated up to BLOCK_VALID_SCRIPTS. This is a performance + // optimization, in the common case of adding a new block to the tip, + // we don't need to iterate over the failed blocks list. + for (const CBlockIndex* failedit : m_failed_blocks) { + if (pindexPrev->GetAncestor(failedit->nHeight) == failedit) { + assert(failedit->nStatus & BLOCK_FAILED_VALID); + CBlockIndex* invalid_walk = pindexPrev; + while (invalid_walk != failedit) { + invalid_walk->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(invalid_walk); + invalid_walk = invalid_walk->pprev; + } + LogPrintf("ERROR: %s: prev block invalid\n", __func__); + return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); + } + } + } } - if (pindex == NULL) + if (pindex == nullptr) pindex = AddToBlockIndex(block); if (ppindex) *ppindex = pindex; - CheckBlockIndex(chainparams.GetConsensus()); - return true; } // Exposed wrapper for AcceptBlockHeader -bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) +bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) { + AssertLockNotHeld(cs_main); { LOCK(cs_main); for (const CBlockHeader& header : headers) { - CBlockIndex *pindex = NULL; // Use a temp pindex instead of ppindex to avoid a const_cast - if (!AcceptBlockHeader(header, state, chainparams, &pindex)) { + CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast + bool accepted = m_blockman.AcceptBlockHeader( + header, state, chainparams, &pindex); + ::ChainstateActive().CheckBlockIndex(chainparams.GetConsensus()); + + if (!accepted) { return false; } if (ppindex) { @@ -3142,38 +3710,64 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidatio } } } - NotifyHeaderTip(); + if (NotifyHeaderTip()) { + if (::ChainstateActive().IsInitialBlockDownload() && ppindex && *ppindex) { + LogPrintf("Synchronizing blockheaders, height: %d (~%.2f%%)\n", (*ppindex)->nHeight, 100.0/((*ppindex)->nHeight+(GetAdjustedTime() - (*ppindex)->GetBlockTime()) / Params().GetConsensus().nPowTargetSpacing) * (*ppindex)->nHeight); + } + } return true; } -/** Store block on disk. If dbp is non-NULL, the file is known to already reside on disk */ -static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) +/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ +static FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const FlatFilePos* dbp) { + unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); + FlatFilePos blockPos; + if (dbp != nullptr) + blockPos = *dbp; + if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != nullptr)) { + error("%s: FindBlockPos failed", __func__); + return FlatFilePos(); + } + if (dbp == nullptr) { + if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { + AbortNode("Failed to write block"); + return FlatFilePos(); + } + } + return blockPos; +} + +/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ +bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) { const CBlock& block = *pblock; if (fNewBlock) *fNewBlock = false; AssertLockHeld(cs_main); - CBlockIndex *pindexDummy = NULL; + CBlockIndex *pindexDummy = nullptr; CBlockIndex *&pindex = ppindex ? *ppindex : pindexDummy; - if (!AcceptBlockHeader(block, state, chainparams, &pindex)) + bool accepted_header = m_blockman.AcceptBlockHeader(block, state, chainparams, &pindex); + CheckBlockIndex(chainparams.GetConsensus()); + + if (!accepted_header) return false; // Try to process all requested blocks that we don't have, but only // process an unrequested block if it's new and has enough work to // advance our tip, and isn't too many blocks ahead. bool fAlreadyHave = pindex->nStatus & BLOCK_HAVE_DATA; - bool fHasMoreWork = (chainActive.Tip() ? pindex->nChainWork > chainActive.Tip()->nChainWork : true); + bool fHasMoreOrSameWork = (m_chain.Tip() ? pindex->nChainWork >= m_chain.Tip()->nChainWork : true); // Blocks that are too out-of-order needlessly limit the effectiveness of // pruning, because pruning will not delete block files that contain any // blocks which are too close in height to the tip. Apply this test // regardless of whether pruning is enabled; it should generally be safe to // not process unrequested blocks. - bool fTooFarAhead = (pindex->nHeight > int(chainActive.Height() + MIN_BLOCKS_TO_KEEP)); + bool fTooFarAhead = (pindex->nHeight > int(m_chain.Height() + MIN_BLOCKS_TO_KEEP)); // TODO: Decouple this function from the block download logic by removing fRequested - // This requires some new chain datastructure to efficiently look up if a + // This requires some new chain data structure to efficiently look up if a // block is in a chain leading to a candidate for best tip, despite not // being such a candidate itself. @@ -3181,103 +3775,105 @@ static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidation // and unrequested blocks. if (fAlreadyHave) return true; if (!fRequested) { // If we didn't ask for it: - if (pindex->nTx != 0) return true; // This is a previously-processed block that was pruned - if (!fHasMoreWork) return true; // Don't process less-work chains - if (fTooFarAhead) return true; // Block height is too high + if (pindex->nTx != 0) return true; // This is a previously-processed block that was pruned + if (!fHasMoreOrSameWork) return true; // Don't process less-work chains + if (fTooFarAhead) return true; // Block height is too high + + // Protect against DoS attacks from low-work chains. + // If our tip is behind, a peer could try to send us + // low-work blocks on a fake chain that we would never + // request; don't process these. + if (pindex->nChainWork < nMinimumChainWork) return true; } - if (fNewBlock) *fNewBlock = true; if (!CheckBlock(block, state, chainparams.GetConsensus()) || !ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindex->pprev)) { - if (state.IsInvalid() && !state.CorruptionPossible()) { + if (state.IsInvalid() && state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(pindex); } - return error("%s: %s", __func__, FormatStateMessage(state)); + return error("%s: %s", __func__, state.ToString()); } // Header is valid/has work, merkle tree and segwit merkle tree are good...RELAY NOW // (but if it does not build on our best tip, let the SendMessages loop relay it) - if (!IsInitialBlockDownload() && chainActive.Tip() == pindex->pprev) + if (!IsInitialBlockDownload() && m_chain.Tip() == pindex->pprev) GetMainSignals().NewPoWValidBlock(pindex, pblock); - int nHeight = pindex->nHeight; - // Write block to history file + if (fNewBlock) *fNewBlock = true; try { - unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); - CDiskBlockPos blockPos; - if (dbp != NULL) - blockPos = *dbp; - if (!FindBlockPos(state, blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != NULL)) - return error("AcceptBlock(): FindBlockPos failed"); - if (dbp == NULL) - if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) - AbortNode(state, "Failed to write block"); - if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) - return error("AcceptBlock(): ReceivedBlockTransactions failed"); + FlatFilePos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp); + if (blockPos.IsNull()) { + state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); + return false; + } + ReceivedBlockTransactions(block, pindex, blockPos, chainparams.GetConsensus()); } catch (const std::runtime_error& e) { return AbortNode(state, std::string("System error: ") + e.what()); } - if (fCheckForPruning) - FlushStateToDisk(state, FLUSH_STATE_NONE); // we just allocated more disk space for block files + FlushStateToDisk(chainparams, state, FlushStateMode::NONE); + + CheckBlockIndex(chainparams.GetConsensus()); return true; } -bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool *fNewBlock) +bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock) { + AssertLockNotHeld(cs_main); + { - CBlockIndex *pindex = NULL; + CBlockIndex *pindex = nullptr; if (fNewBlock) *fNewBlock = false; - CValidationState state; - // Ensure that CheckBlock() passes before calling AcceptBlock, as - // belt-and-suspenders. - bool ret = CheckBlock(*pblock, state, chainparams.GetConsensus()); + BlockValidationState state; + // CheckBlock() does not support multi-threaded block validation because CBlock::fChecked can cause data race. + // Therefore, the following critical section must include the CheckBlock() call as well. LOCK(cs_main); + // Ensure that CheckBlock() passes before calling AcceptBlock, as + // belt-and-suspenders. + bool ret = CheckBlock(*pblock, state, chainparams.GetConsensus()); if (ret) { // Store to disk - ret = AcceptBlock(pblock, state, chainparams, &pindex, fForceProcessing, NULL, fNewBlock); + ret = ::ChainstateActive().AcceptBlock(pblock, state, chainparams, &pindex, fForceProcessing, nullptr, fNewBlock); } - CheckBlockIndex(chainparams.GetConsensus()); if (!ret) { GetMainSignals().BlockChecked(*pblock, state); - return error("%s: AcceptBlock FAILED", __func__); + return error("%s: AcceptBlock FAILED (%s)", __func__, state.ToString()); } } NotifyHeaderTip(); - CValidationState state; // Only used to report errors, not invalidity - ignore it - if (!ActivateBestChain(state, chainparams, pblock)) - return error("%s: ActivateBestChain failed", __func__); + BlockValidationState state; // Only used to report errors, not invalidity - ignore it + if (!::ChainstateActive().ActivateBestChain(state, chainparams, pblock)) + return error("%s: ActivateBestChain failed (%s)", __func__, state.ToString()); return true; } -bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW, bool fCheckMerkleRoot) +bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW, bool fCheckMerkleRoot) { AssertLockHeld(cs_main); - assert(pindexPrev && pindexPrev == chainActive.Tip()); - if (fCheckpointsEnabled && !CheckIndexAgainstCheckpoint(pindexPrev, state, chainparams, block.GetHash())) - return error("%s: CheckIndexAgainstCheckpoint(): %s", __func__, state.GetRejectReason().c_str()); - - CCoinsViewCache viewNew(pcoinsTip); + assert(pindexPrev && pindexPrev == ::ChainActive().Tip()); + CCoinsViewCache viewNew(&::ChainstateActive().CoinsTip()); + uint256 block_hash(block.GetHash()); CBlockIndex indexDummy(block); indexDummy.pprev = pindexPrev; indexDummy.nHeight = pindexPrev->nHeight + 1; + indexDummy.phashBlock = &block_hash; // NOTE: CheckBlockHeader is called by CheckBlock - if (!ContextualCheckBlockHeader(block, state, chainparams.GetConsensus(), pindexPrev, GetAdjustedTime())) - return error("%s: Consensus::ContextualCheckBlockHeader: %s", __func__, FormatStateMessage(state)); + if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) + return error("%s: Consensus::ContextualCheckBlockHeader: %s", __func__, state.ToString()); if (!CheckBlock(block, state, chainparams.GetConsensus(), fCheckPOW, fCheckMerkleRoot)) - return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); + return error("%s: Consensus::CheckBlock: %s", __func__, state.ToString()); if (!ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindexPrev)) - return error("%s: Consensus::ContextualCheckBlock: %s", __func__, FormatStateMessage(state)); - if (!ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) + return error("%s: Consensus::ContextualCheckBlock: %s", __func__, state.ToString()); + if (!::ChainstateActive().ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) return false; assert(state.IsValid()); @@ -3291,18 +3887,22 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, /* Calculate the amount of disk space the block & undo files currently use */ uint64_t CalculateCurrentUsage() { + LOCK(cs_LastBlockFile); + uint64_t retval = 0; - BOOST_FOREACH(const CBlockFileInfo &file, vinfoBlockFile) { + for (const CBlockFileInfo &file : vinfoBlockFile) { retval += file.nSize + file.nUndoSize; } return retval; } -/* Prune a block file (modify associated database entries)*/ -void PruneOneBlockFile(const int fileNumber) +void ChainstateManager::PruneOneBlockFile(const int fileNumber) { - for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); ++it) { - CBlockIndex* pindex = it->second; + AssertLockHeld(cs_main); + LOCK(cs_LastBlockFile); + + for (const auto& entry : m_blockman.m_block_index) { + CBlockIndex* pindex = entry.second; if (pindex->nFile == fileNumber) { pindex->nStatus &= ~BLOCK_HAVE_DATA; pindex->nStatus &= ~BLOCK_HAVE_UNDO; @@ -3311,16 +3911,16 @@ void PruneOneBlockFile(const int fileNumber) pindex->nUndoPos = 0; setDirtyBlockIndex.insert(pindex); - // Prune from mapBlocksUnlinked -- any block we prune would have + // Prune from m_blocks_unlinked -- any block we prune would have // to be downloaded again in order to consider its chain, at which // point it would be considered as a candidate for - // mapBlocksUnlinked or setBlockIndexCandidates. - std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> range = mapBlocksUnlinked.equal_range(pindex->pprev); + // m_blocks_unlinked or setBlockIndexCandidates. + auto range = m_blockman.m_blocks_unlinked.equal_range(pindex->pprev); while (range.first != range.second) { std::multimap<CBlockIndex *, CBlockIndex *>::iterator _it = range.first; range.first++; if (_it->second == pindex) { - mapBlocksUnlinked.erase(_it); + m_blockman.m_blocks_unlinked.erase(_it); } } } @@ -3331,32 +3931,32 @@ void PruneOneBlockFile(const int fileNumber) } -void UnlinkPrunedFiles(std::set<int>& setFilesToPrune) +void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) { for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { - CDiskBlockPos pos(*it, 0); - boost::filesystem::remove(GetBlockPosFilename(pos, "blk")); - boost::filesystem::remove(GetBlockPosFilename(pos, "rev")); + FlatFilePos pos(*it, 0); + fs::remove(BlockFileSeq().FileName(pos)); + fs::remove(UndoFileSeq().FileName(pos)); LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); } } /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */ -void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight) +static void FindFilesToPruneManual(ChainstateManager& chainman, std::set<int>& setFilesToPrune, int nManualPruneHeight) { assert(fPruneMode && nManualPruneHeight > 0); LOCK2(cs_main, cs_LastBlockFile); - if (chainActive.Tip() == NULL) + if (::ChainActive().Tip() == nullptr) return; // last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip) - unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP); + unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, ::ChainActive().Tip()->nHeight - MIN_BLOCKS_TO_KEEP); int count=0; for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { if (vinfoBlockFile[fileNumber].nSize == 0 || vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) continue; - PruneOneBlockFile(fileNumber); + chainman.PruneOneBlockFile(fileNumber); setFilesToPrune.insert(fileNumber); count++; } @@ -3366,22 +3966,40 @@ void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeig /* This function is called from the RPC code for pruneblockchain */ void PruneBlockFilesManual(int nManualPruneHeight) { - CValidationState state; - FlushStateToDisk(state, FLUSH_STATE_NONE, nManualPruneHeight); + BlockValidationState state; + const CChainParams& chainparams = Params(); + if (!::ChainstateActive().FlushStateToDisk( + chainparams, state, FlushStateMode::NONE, nManualPruneHeight)) { + LogPrintf("%s: failed to flush state (%s)\n", __func__, state.ToString()); + } } -/* Calculate the block/rev files that should be deleted to remain under target*/ -void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight) +/** + * Prune block and undo files (blk???.dat and undo???.dat) so that the disk space used is less than a user-defined target. + * The user sets the target (in MB) on the command line or in config file. This will be run on startup and whenever new + * space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex + * (which in this case means the blockchain must be re-downloaded.) + * + * Pruning functions are called from FlushStateToDisk when the global fCheckForPruning flag has been set. + * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.) + * Pruning cannot take place until the longest chain is at least a certain length (100000 on mainnet, 1000 on testnet, 1000 on regtest). + * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip. + * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files. + * A db flag records the fact that at least some block files have been pruned. + * + * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned + */ +static void FindFilesToPrune(ChainstateManager& chainman, std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight) { LOCK2(cs_main, cs_LastBlockFile); - if (chainActive.Tip() == NULL || nPruneTarget == 0) { + if (::ChainActive().Tip() == nullptr || nPruneTarget == 0) { return; } - if ((uint64_t)chainActive.Tip()->nHeight <= nPruneAfterHeight) { + if ((uint64_t)::ChainActive().Tip()->nHeight <= nPruneAfterHeight) { return; } - unsigned int nLastBlockWeCanPrune = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP; + unsigned int nLastBlockWeCanPrune = ::ChainActive().Tip()->nHeight - MIN_BLOCKS_TO_KEEP; uint64_t nCurrentUsage = CalculateCurrentUsage(); // We don't check to prune until after we've allocated new space for files // So we should leave a buffer under our target to account for another allocation @@ -3391,6 +4009,15 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight int count=0; if (nCurrentUsage + nBuffer >= nPruneTarget) { + // On a prune event, the chainstate DB is flushed. + // To avoid excessive prune events negating the benefit of high dbcache + // values, we should not prune too rapidly. + // So when pruning in IBD, increase the buffer a bit to avoid a re-prune too soon. + if (::ChainstateActive().IsInitialBlockDownload()) { + // Since this is only relevant during IBD, we use a fixed 10% + nBuffer += nPruneTarget / 10; + } + for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize; @@ -3404,7 +4031,7 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) continue; - PruneOneBlockFile(fileNumber); + chainman.PruneOneBlockFile(fileNumber); // Queue up the files for removal setFilesToPrune.insert(fileNumber); nCurrentUsage -= nBytesToPrune; @@ -3412,97 +4039,76 @@ void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight } } - LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", + LogPrint(BCLog::PRUNE, "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", nPruneTarget/1024/1024, nCurrentUsage/1024/1024, ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, nLastBlockWeCanPrune, count); } -bool CheckDiskSpace(uint64_t nAdditionalBytes) +static FlatFileSeq BlockFileSeq() { - uint64_t nFreeBytesAvailable = boost::filesystem::space(GetDataDir()).available; - - // Check for nMinDiskSpace bytes (currently 50MB) - if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes) - return AbortNode("Disk space is low!", _("Error: Disk space is low!")); - - return true; + return FlatFileSeq(GetBlocksDir(), "blk", BLOCKFILE_CHUNK_SIZE); } -FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) +static FlatFileSeq UndoFileSeq() { - if (pos.IsNull()) - return NULL; - boost::filesystem::path path = GetBlockPosFilename(pos, prefix); - boost::filesystem::create_directories(path.parent_path()); - FILE* file = fopen(path.string().c_str(), "rb+"); - if (!file && !fReadOnly) - file = fopen(path.string().c_str(), "wb+"); - if (!file) { - LogPrintf("Unable to open file %s\n", path.string()); - return NULL; - } - if (pos.nPos) { - if (fseek(file, pos.nPos, SEEK_SET)) { - LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); - fclose(file); - return NULL; - } - } - return file; + return FlatFileSeq(GetBlocksDir(), "rev", UNDOFILE_CHUNK_SIZE); } -FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) { - return OpenDiskFile(pos, "blk", fReadOnly); +FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) { + return BlockFileSeq().Open(pos, fReadOnly); } -FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { - return OpenDiskFile(pos, "rev", fReadOnly); +/** Open an undo file (rev?????.dat) */ +static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly) { + return UndoFileSeq().Open(pos, fReadOnly); } -boost::filesystem::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) +fs::path GetBlockPosFilename(const FlatFilePos &pos) { - return GetDataDir() / "blocks" / strprintf("%s%05u.dat", prefix, pos.nFile); + return BlockFileSeq().FileName(pos); } -CBlockIndex * InsertBlockIndex(uint256 hash) +CBlockIndex * BlockManager::InsertBlockIndex(const uint256& hash) { + AssertLockHeld(cs_main); + if (hash.IsNull()) - return NULL; + return nullptr; // Return existing - BlockMap::iterator mi = mapBlockIndex.find(hash); - if (mi != mapBlockIndex.end()) + BlockMap::iterator mi = m_block_index.find(hash); + if (mi != m_block_index.end()) return (*mi).second; // Create new CBlockIndex* pindexNew = new CBlockIndex(); - if (!pindexNew) - throw std::runtime_error(std::string(__func__) + ": new CBlockIndex failed"); - mi = mapBlockIndex.insert(std::make_pair(hash, pindexNew)).first; + mi = m_block_index.insert(std::make_pair(hash, pindexNew)).first; pindexNew->phashBlock = &((*mi).first); return pindexNew; } -bool static LoadBlockIndexDB(const CChainParams& chainparams) +bool BlockManager::LoadBlockIndex( + const Consensus::Params& consensus_params, + CBlockTreeDB& blocktree, + std::set<CBlockIndex*, CBlockIndexWorkComparator>& block_index_candidates) { - if (!pblocktree->LoadBlockIndexGuts(InsertBlockIndex)) + if (!blocktree.LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) return false; - boost::this_thread::interruption_point(); - // Calculate nChainWork std::vector<std::pair<int, CBlockIndex*> > vSortedByHeight; - vSortedByHeight.reserve(mapBlockIndex.size()); - BOOST_FOREACH(const PAIRTYPE(uint256, CBlockIndex*)& item, mapBlockIndex) + vSortedByHeight.reserve(m_block_index.size()); + for (const std::pair<const uint256, CBlockIndex*>& item : m_block_index) { CBlockIndex* pindex = item.second; vSortedByHeight.push_back(std::make_pair(pindex->nHeight, pindex)); } sort(vSortedByHeight.begin(), vSortedByHeight.end()); - BOOST_FOREACH(const PAIRTYPE(int, CBlockIndex*)& item, vSortedByHeight) + for (const std::pair<int, CBlockIndex*>& item : vSortedByHeight) { + if (ShutdownRequested()) return false; CBlockIndex* pindex = item.second; pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime); @@ -3510,26 +4116,53 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) // Pruned nodes may have deleted the block. if (pindex->nTx > 0) { if (pindex->pprev) { - if (pindex->pprev->nChainTx) { + if (pindex->pprev->HaveTxsDownloaded()) { pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; } else { pindex->nChainTx = 0; - mapBlocksUnlinked.insert(std::make_pair(pindex->pprev, pindex)); + m_blocks_unlinked.insert(std::make_pair(pindex->pprev, pindex)); } } else { pindex->nChainTx = pindex->nTx; } } - if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->nChainTx || pindex->pprev == NULL)) - setBlockIndexCandidates.insert(pindex); + if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) { + pindex->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(pindex); + } + if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr)) { + block_index_candidates.insert(pindex); + } if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork)) pindexBestInvalid = pindex; if (pindex->pprev) pindex->BuildSkip(); - if (pindex->IsValid(BLOCK_VALID_TREE) && (pindexBestHeader == NULL || CBlockIndexWorkComparator()(pindexBestHeader, pindex))) + if (pindex->IsValid(BLOCK_VALID_TREE) && (pindexBestHeader == nullptr || CBlockIndexWorkComparator()(pindexBestHeader, pindex))) pindexBestHeader = pindex; } + return true; +} + +void BlockManager::Unload() { + m_failed_blocks.clear(); + m_blocks_unlinked.clear(); + + for (const BlockMap::value_type& entry : m_block_index) { + delete entry.second; + } + + m_block_index.clear(); +} + +bool static LoadBlockIndexDB(ChainstateManager& chainman, const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + if (!chainman.m_blockman.LoadBlockIndex( + chainparams.GetConsensus(), *pblocktree, + ::ChainstateActive().setBlockIndexCandidates)) { + return false; + } + // Load block file info pblocktree->ReadLastBlockFile(nLastBlockFile); vinfoBlockFile.resize(nLastBlockFile + 1); @@ -3550,8 +4183,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) // Check presence of blk files LogPrintf("Checking all blk files are present...\n"); std::set<int> setBlkDataFiles; - BOOST_FOREACH(const PAIRTYPE(uint256, CBlockIndex*)& item, mapBlockIndex) - { + for (const std::pair<const uint256, CBlockIndex*>& item : chainman.BlockIndex()) { CBlockIndex* pindex = item.second; if (pindex->nStatus & BLOCK_HAVE_DATA) { setBlkDataFiles.insert(pindex->nFile); @@ -3559,7 +4191,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) } for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) { - CDiskBlockPos pos(*it, 0); + FlatFilePos pos(*it, 0); if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) { return false; } @@ -3573,69 +4205,77 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) // Check whether we need to continue reindexing bool fReindexing = false; pblocktree->ReadReindexing(fReindexing); - fReindex |= fReindexing; + if(fReindexing) fReindex = true; + + return true; +} - // Check whether we have a transaction index - pblocktree->ReadFlag("txindex", fTxIndex); - LogPrintf("%s: transaction index %s\n", __func__, fTxIndex ? "enabled" : "disabled"); +bool CChainState::LoadChainTip(const CChainParams& chainparams) +{ + AssertLockHeld(cs_main); + const CCoinsViewCache& coins_cache = CoinsTip(); + assert(!coins_cache.GetBestBlock().IsNull()); // Never called when the coins view is empty + const CBlockIndex* tip = m_chain.Tip(); - // Load pointer to end of best chain - BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); - if (it == mapBlockIndex.end()) + if (tip && tip->GetBlockHash() == coins_cache.GetBestBlock()) { return true; - chainActive.SetTip(it->second); + } + // Load pointer to end of best chain + CBlockIndex* pindex = LookupBlockIndex(coins_cache.GetBestBlock()); + if (!pindex) { + return false; + } + m_chain.SetTip(pindex); PruneBlockIndexCandidates(); - LogPrintf("%s: hashBestChain=%s height=%d date=%s progress=%f\n", __func__, - chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), - DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), - GuessVerificationProgress(chainparams.TxData(), chainActive.Tip())); - + tip = m_chain.Tip(); + LogPrintf("Loaded best chain: hashBestChain=%s height=%d date=%s progress=%f\n", + tip->GetBlockHash().ToString(), + m_chain.Height(), + FormatISO8601DateTime(tip->GetBlockTime()), + GuessVerificationProgress(chainparams.TxData(), tip)); return true; } CVerifyDB::CVerifyDB() { - uiInterface.ShowProgress(_("Verifying blocks..."), 0); + uiInterface.ShowProgress(_("Verifying blocks...").translated, 0, false); } CVerifyDB::~CVerifyDB() { - uiInterface.ShowProgress("", 100); + uiInterface.ShowProgress("", 100, false); } bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, int nCheckLevel, int nCheckDepth) { LOCK(cs_main); - if (chainActive.Tip() == NULL || chainActive.Tip()->pprev == NULL) + if (::ChainActive().Tip() == nullptr || ::ChainActive().Tip()->pprev == nullptr) return true; // Verify blocks in the best chain - if (nCheckDepth <= 0) - nCheckDepth = 1000000000; // suffices until the year 19000 - if (nCheckDepth > chainActive.Height()) - nCheckDepth = chainActive.Height(); + if (nCheckDepth <= 0 || nCheckDepth > ::ChainActive().Height()) + nCheckDepth = ::ChainActive().Height(); nCheckLevel = std::max(0, std::min(4, nCheckLevel)); LogPrintf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); CCoinsViewCache coins(coinsview); - CBlockIndex* pindexState = chainActive.Tip(); - CBlockIndex* pindexFailure = NULL; + CBlockIndex* pindex; + CBlockIndex* pindexFailure = nullptr; int nGoodTransactions = 0; - CValidationState state; + BlockValidationState state; int reportDone = 0; - LogPrintf("[0%%]..."); - for (CBlockIndex* pindex = chainActive.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) - { + LogPrintf("[0%%]..."); /* Continued */ + for (pindex = ::ChainActive().Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { boost::this_thread::interruption_point(); - int percentageDone = std::max(1, std::min(99, (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); + const int percentageDone = std::max(1, std::min(99, (int)(((double)(::ChainActive().Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); if (reportDone < percentageDone/10) { // report every 10% step - LogPrintf("[%d%%]...", percentageDone); + LogPrintf("[%d%%]...", percentageDone); /* Continued */ reportDone = percentageDone/10; } - uiInterface.ShowProgress(_("Verifying blocks..."), percentageDone); - if (pindex->nHeight < chainActive.Height()-nCheckDepth) + uiInterface.ShowProgress(_("Verifying blocks...").translated, percentageDone, false); + if (pindex->nHeight <= ::ChainActive().Height()-nCheckDepth) break; if (fPruneMode && !(pindex->nStatus & BLOCK_HAVE_DATA)) { // If pruning, only go back as far as we have data. @@ -3649,218 +4289,374 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, // check level 1: verify block validity if (nCheckLevel >= 1 && !CheckBlock(block, state, chainparams.GetConsensus())) return error("%s: *** found bad block at %d, hash=%s (%s)\n", __func__, - pindex->nHeight, pindex->GetBlockHash().ToString(), FormatStateMessage(state)); + pindex->nHeight, pindex->GetBlockHash().ToString(), state.ToString()); // check level 2: verify undo validity if (nCheckLevel >= 2 && pindex) { CBlockUndo undo; - CDiskBlockPos pos = pindex->GetUndoPos(); - if (!pos.IsNull()) { - if (!UndoReadFromDisk(undo, pos, pindex->pprev->GetBlockHash())) + if (!pindex->GetUndoPos().IsNull()) { + if (!UndoReadFromDisk(undo, pindex)) { return error("VerifyDB(): *** found bad undo data at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + } } } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks - if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { - bool fClean = true; - if (!DisconnectBlock(block, state, pindex, coins, &fClean)) + if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + ::ChainstateActive().CoinsTip().DynamicMemoryUsage()) <= nCoinCacheUsage) { + assert(coins.GetBestBlock() == pindex->GetBlockHash()); + DisconnectResult res = ::ChainstateActive().DisconnectBlock(block, pindex, coins); + if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); - pindexState = pindex->pprev; - if (!fClean) { + } + if (res == DISCONNECT_UNCLEAN) { nGoodTransactions = 0; pindexFailure = pindex; - } else + } else { nGoodTransactions += block.vtx.size(); + } } if (ShutdownRequested()) return true; } if (pindexFailure) - return error("VerifyDB(): *** coin database inconsistencies found (last %i blocks, %i good transactions before that)\n", chainActive.Height() - pindexFailure->nHeight + 1, nGoodTransactions); + return error("VerifyDB(): *** coin database inconsistencies found (last %i blocks, %i good transactions before that)\n", ::ChainActive().Height() - pindexFailure->nHeight + 1, nGoodTransactions); + + // store block count as we move pindex at check level >= 4 + int block_count = ::ChainActive().Height() - pindex->nHeight; // check level 4: try reconnecting blocks if (nCheckLevel >= 4) { - CBlockIndex *pindex = pindexState; - while (pindex != chainActive.Tip()) { + while (pindex != ::ChainActive().Tip()) { boost::this_thread::interruption_point(); - uiInterface.ShowProgress(_("Verifying blocks..."), std::max(1, std::min(99, 100 - (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * 50)))); - pindex = chainActive.Next(pindex); + const int percentageDone = std::max(1, std::min(99, 100 - (int)(((double)(::ChainActive().Height() - pindex->nHeight)) / (double)nCheckDepth * 50))); + if (reportDone < percentageDone/10) { + // report every 10% step + LogPrintf("[%d%%]...", percentageDone); /* Continued */ + reportDone = percentageDone/10; + } + uiInterface.ShowProgress(_("Verifying blocks...").translated, percentageDone, false); + pindex = ::ChainActive().Next(pindex); CBlock block; if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); - if (!ConnectBlock(block, state, pindex, coins, chainparams)) - return error("VerifyDB(): *** found unconnectable block at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + if (!::ChainstateActive().ConnectBlock(block, state, pindex, coins, chainparams)) + return error("VerifyDB(): *** found unconnectable block at %d, hash=%s (%s)", pindex->nHeight, pindex->GetBlockHash().ToString(), state.ToString()); } } LogPrintf("[DONE].\n"); - LogPrintf("No coin database inconsistencies in last %i blocks (%i transactions)\n", chainActive.Height() - pindexState->nHeight, nGoodTransactions); + LogPrintf("No coin database inconsistencies in last %i blocks (%i transactions)\n", block_count, nGoodTransactions); + + return true; +} + +/** Apply the effects of a block on the utxo cache, ignoring that it may already have been applied. */ +bool CChainState::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) +{ + // TODO: merge with ConnectBlock + CBlock block; + if (!ReadBlockFromDisk(block, pindex, params.GetConsensus())) { + return error("ReplayBlock(): ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + } + for (const CTransactionRef& tx : block.vtx) { + if (!tx->IsCoinBase()) { + for (const CTxIn &txin : tx->vin) { + inputs.SpendCoin(txin.prevout); + } + } + // Pass check = true as every addition may be an overwrite. + AddCoins(inputs, *tx, pindex->nHeight, true); + } return true; } -bool RewindBlockIndex(const CChainParams& params) +bool CChainState::ReplayBlocks(const CChainParams& params) { LOCK(cs_main); - int nHeight = 1; - while (nHeight <= chainActive.Height()) { - if (IsWitnessEnabled(chainActive[nHeight - 1], params.GetConsensus()) && !(chainActive[nHeight]->nStatus & BLOCK_OPT_WITNESS)) { - break; + CCoinsView& db = this->CoinsDB(); + CCoinsViewCache cache(&db); + + std::vector<uint256> hashHeads = db.GetHeadBlocks(); + if (hashHeads.empty()) return true; // We're already in a consistent state. + if (hashHeads.size() != 2) return error("ReplayBlocks(): unknown inconsistent state"); + + uiInterface.ShowProgress(_("Replaying blocks...").translated, 0, false); + LogPrintf("Replaying blocks\n"); + + const CBlockIndex* pindexOld = nullptr; // Old tip during the interrupted flush. + const CBlockIndex* pindexNew; // New tip during the interrupted flush. + const CBlockIndex* pindexFork = nullptr; // Latest block common to both the old and the new tip. + + if (m_blockman.m_block_index.count(hashHeads[0]) == 0) { + return error("ReplayBlocks(): reorganization to unknown block requested"); + } + pindexNew = m_blockman.m_block_index[hashHeads[0]]; + + if (!hashHeads[1].IsNull()) { // The old tip is allowed to be 0, indicating it's the first flush. + if (m_blockman.m_block_index.count(hashHeads[1]) == 0) { + return error("ReplayBlocks(): reorganization from unknown block requested"); } - nHeight++; + pindexOld = m_blockman.m_block_index[hashHeads[1]]; + pindexFork = LastCommonAncestor(pindexOld, pindexNew); + assert(pindexFork != nullptr); } - // nHeight is now the height of the first insufficiently-validated block, or tipheight + 1 - CValidationState state; - CBlockIndex* pindex = chainActive.Tip(); - while (chainActive.Height() >= nHeight) { - if (fPruneMode && !(chainActive.Tip()->nStatus & BLOCK_HAVE_DATA)) { - // If pruning, don't try rewinding past the HAVE_DATA point; - // since older blocks can't be served anyway, there's - // no need to walk further, and trying to DisconnectTip() - // will fail (and require a needless reindex/redownload - // of the blockchain). - break; + // Rollback along the old branch. + while (pindexOld != pindexFork) { + if (pindexOld->nHeight > 0) { // Never disconnect the genesis block. + CBlock block; + if (!ReadBlockFromDisk(block, pindexOld, params.GetConsensus())) { + return error("RollbackBlock(): ReadBlockFromDisk() failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); + } + LogPrintf("Rolling back %s (%i)\n", pindexOld->GetBlockHash().ToString(), pindexOld->nHeight); + DisconnectResult res = DisconnectBlock(block, pindexOld, cache); + if (res == DISCONNECT_FAILED) { + return error("RollbackBlock(): DisconnectBlock failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); + } + // If DISCONNECT_UNCLEAN is returned, it means a non-existing UTXO was deleted, or an existing UTXO was + // overwritten. It corresponds to cases where the block-to-be-disconnect never had all its operations + // applied to the UTXO set. However, as both writing a UTXO and deleting a UTXO are idempotent operations, + // the result is still a version of the UTXO set with the effects of that block undone. } - if (!DisconnectTip(state, params, true)) { - return error("RewindBlockIndex: unable to disconnect block at height %i", pindex->nHeight); + pindexOld = pindexOld->pprev; + } + + // Roll forward from the forking point to the new tip. + int nForkHeight = pindexFork ? pindexFork->nHeight : 0; + for (int nHeight = nForkHeight + 1; nHeight <= pindexNew->nHeight; ++nHeight) { + const CBlockIndex* pindex = pindexNew->GetAncestor(nHeight); + LogPrintf("Rolling forward %s (%i)\n", pindex->GetBlockHash().ToString(), nHeight); + uiInterface.ShowProgress(_("Replaying blocks...").translated, (int) ((nHeight - nForkHeight) * 100.0 / (pindexNew->nHeight - nForkHeight)) , false); + if (!RollforwardBlock(pindex, cache, params)) return false; + } + + cache.SetBestBlock(pindexNew->GetBlockHash()); + cache.Flush(); + uiInterface.ShowProgress("", 100, false); + return true; +} + +//! Helper for CChainState::RewindBlockIndex +void CChainState::EraseBlockData(CBlockIndex* index) +{ + AssertLockHeld(cs_main); + assert(!m_chain.Contains(index)); // Make sure this block isn't active + + // Reduce validity + index->nStatus = std::min<unsigned int>(index->nStatus & BLOCK_VALID_MASK, BLOCK_VALID_TREE) | (index->nStatus & ~BLOCK_VALID_MASK); + // Remove have-data flags. + index->nStatus &= ~(BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO); + // Remove storage location. + index->nFile = 0; + index->nDataPos = 0; + index->nUndoPos = 0; + // Remove various other things + index->nTx = 0; + index->nChainTx = 0; + index->nSequenceId = 0; + // Make sure it gets written. + setDirtyBlockIndex.insert(index); + // Update indexes + setBlockIndexCandidates.erase(index); + auto ret = m_blockman.m_blocks_unlinked.equal_range(index->pprev); + while (ret.first != ret.second) { + if (ret.first->second == index) { + m_blockman.m_blocks_unlinked.erase(ret.first++); + } else { + ++ret.first; } - // Occasionally flush state to disk. - if (!FlushStateToDisk(state, FLUSH_STATE_PERIODIC)) - return false; } + // Mark parent as eligible for main chain again + if (index->pprev && index->pprev->IsValid(BLOCK_VALID_TRANSACTIONS) && index->pprev->HaveTxsDownloaded()) { + setBlockIndexCandidates.insert(index->pprev); + } +} - // Reduce validity flag and have-data flags. - // We do this after actual disconnecting, otherwise we'll end up writing the lack of data - // to disk before writing the chainstate, resulting in a failure to continue if interrupted. - for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { - CBlockIndex* pindexIter = it->second; - - // Note: If we encounter an insufficiently validated block that - // is on chainActive, it must be because we are a pruning node, and - // this block or some successor doesn't HAVE_DATA, so we were unable to - // rewind all the way. Blocks remaining on chainActive at this point - // must not have their validity reduced. - if (IsWitnessEnabled(pindexIter->pprev, params.GetConsensus()) && !(pindexIter->nStatus & BLOCK_OPT_WITNESS) && !chainActive.Contains(pindexIter)) { - // Reduce validity - pindexIter->nStatus = std::min<unsigned int>(pindexIter->nStatus & BLOCK_VALID_MASK, BLOCK_VALID_TREE) | (pindexIter->nStatus & ~BLOCK_VALID_MASK); - // Remove have-data flags. - pindexIter->nStatus &= ~(BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO); - // Remove storage location. - pindexIter->nFile = 0; - pindexIter->nDataPos = 0; - pindexIter->nUndoPos = 0; - // Remove various other things - pindexIter->nTx = 0; - pindexIter->nChainTx = 0; - pindexIter->nSequenceId = 0; - // Make sure it gets written. - setDirtyBlockIndex.insert(pindexIter); - // Update indexes - setBlockIndexCandidates.erase(pindexIter); - std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> ret = mapBlocksUnlinked.equal_range(pindexIter->pprev); - while (ret.first != ret.second) { - if (ret.first->second == pindexIter) { - mapBlocksUnlinked.erase(ret.first++); - } else { - ++ret.first; - } +bool CChainState::RewindBlockIndex(const CChainParams& params) +{ + // Note that during -reindex-chainstate we are called with an empty m_chain! + + // First erase all post-segwit blocks without witness not in the main chain, + // as this can we done without costly DisconnectTip calls. Active + // blocks will be dealt with below (releasing cs_main in between). + { + LOCK(cs_main); + for (const auto& entry : m_blockman.m_block_index) { + if (IsWitnessEnabled(entry.second->pprev, params.GetConsensus()) && !(entry.second->nStatus & BLOCK_OPT_WITNESS) && !m_chain.Contains(entry.second)) { + EraseBlockData(entry.second); } - } else if (pindexIter->IsValid(BLOCK_VALID_TRANSACTIONS) && pindexIter->nChainTx) { - setBlockIndexCandidates.insert(pindexIter); } } - PruneBlockIndexCandidates(); + // Find what height we need to reorganize to. + CBlockIndex *tip; + int nHeight = 1; + { + LOCK(cs_main); + while (nHeight <= m_chain.Height()) { + // Although SCRIPT_VERIFY_WITNESS is now generally enforced on all + // blocks in ConnectBlock, we don't need to go back and + // re-download/re-verify blocks from before segwit actually activated. + if (IsWitnessEnabled(m_chain[nHeight - 1], params.GetConsensus()) && !(m_chain[nHeight]->nStatus & BLOCK_OPT_WITNESS)) { + break; + } + nHeight++; + } - CheckBlockIndex(params.GetConsensus()); + tip = m_chain.Tip(); + } + // nHeight is now the height of the first insufficiently-validated block, or tipheight + 1 - if (!FlushStateToDisk(state, FLUSH_STATE_ALWAYS)) { - return false; + BlockValidationState state; + // Loop until the tip is below nHeight, or we reach a pruned block. + while (!ShutdownRequested()) { + { + LOCK2(cs_main, ::mempool.cs); + // Make sure nothing changed from under us (this won't happen because RewindBlockIndex runs before importing/network are active) + assert(tip == m_chain.Tip()); + if (tip == nullptr || tip->nHeight < nHeight) break; + if (fPruneMode && !(tip->nStatus & BLOCK_HAVE_DATA)) { + // If pruning, don't try rewinding past the HAVE_DATA point; + // since older blocks can't be served anyway, there's + // no need to walk further, and trying to DisconnectTip() + // will fail (and require a needless reindex/redownload + // of the blockchain). + break; + } + + // Disconnect block + if (!DisconnectTip(state, params, nullptr)) { + return error("RewindBlockIndex: unable to disconnect block at height %i (%s)", tip->nHeight, state.ToString()); + } + + // Reduce validity flag and have-data flags. + // We do this after actual disconnecting, otherwise we'll end up writing the lack of data + // to disk before writing the chainstate, resulting in a failure to continue if interrupted. + // Note: If we encounter an insufficiently validated block that + // is on m_chain, it must be because we are a pruning node, and + // this block or some successor doesn't HAVE_DATA, so we were unable to + // rewind all the way. Blocks remaining on m_chain at this point + // must not have their validity reduced. + EraseBlockData(tip); + + tip = tip->pprev; + } + // Make sure the queue of validation callbacks doesn't grow unboundedly. + LimitValidationInterfaceQueue(); + + // Occasionally flush state to disk. + if (!FlushStateToDisk(params, state, FlushStateMode::PERIODIC)) { + LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", state.ToString()); + return false; + } + } + + { + LOCK(cs_main); + if (m_chain.Tip() != nullptr) { + // We can't prune block index candidates based on our tip if we have + // no tip due to m_chain being empty! + PruneBlockIndexCandidates(); + + CheckBlockIndex(params.GetConsensus()); + + // FlushStateToDisk can possibly read ::ChainActive(). Be conservative + // and skip it here, we're about to -reindex-chainstate anyway, so + // it'll get called a bunch real soon. + BlockValidationState state; + if (!FlushStateToDisk(params, state, FlushStateMode::ALWAYS)) { + LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", state.ToString()); + return false; + } + } } return true; } +void CChainState::UnloadBlockIndex() { + nBlockSequenceId = 1; + setBlockIndexCandidates.clear(); +} + // May NOT be used after any connections are up as much // of the peer-processing logic assumes a consistent // block index state void UnloadBlockIndex() { LOCK(cs_main); - setBlockIndexCandidates.clear(); - chainActive.SetTip(NULL); - pindexBestInvalid = NULL; - pindexBestHeader = NULL; + g_chainman.Unload(); + pindexBestInvalid = nullptr; + pindexBestHeader = nullptr; mempool.clear(); - mapBlocksUnlinked.clear(); vinfoBlockFile.clear(); nLastBlockFile = 0; - nBlockSequenceId = 1; setDirtyBlockIndex.clear(); setDirtyFileInfo.clear(); versionbitscache.Clear(); for (int b = 0; b < VERSIONBITS_NUM_BITS; b++) { warningcache[b].clear(); } - - BOOST_FOREACH(BlockMap::value_type& entry, mapBlockIndex) { - delete entry.second; - } - mapBlockIndex.clear(); fHavePruned = false; } -bool LoadBlockIndex(const CChainParams& chainparams) +bool ChainstateManager::LoadBlockIndex(const CChainParams& chainparams) { + AssertLockHeld(cs_main); // Load block index from databases - if (!fReindex && !LoadBlockIndexDB(chainparams)) - return false; + bool needs_init = fReindex; + if (!fReindex) { + bool ret = LoadBlockIndexDB(*this, chainparams); + if (!ret) return false; + needs_init = m_blockman.m_block_index.empty(); + } + + if (needs_init) { + // Everything here is for *new* reindex/DBs. Thus, though + // LoadBlockIndexDB may have set fReindex if we shut down + // mid-reindex previously, we don't check fReindex and + // instead only check it prior to LoadBlockIndexDB to set + // needs_init. + + LogPrintf("Initializing databases...\n"); + } return true; } -bool InitBlockIndex(const CChainParams& chainparams) +bool CChainState::LoadGenesisBlock(const CChainParams& chainparams) { LOCK(cs_main); - // Check whether we're already initialized - if (chainActive.Genesis() != NULL) + // Check whether we're already initialized by checking for genesis in + // m_blockman.m_block_index. Note that we can't use m_chain here, since it is + // set based on the coins db, not the block index db, which is the only + // thing loaded at this point. + if (m_blockman.m_block_index.count(chainparams.GenesisBlock().GetHash())) return true; - // Use the provided setting for -txindex in the new database - fTxIndex = GetBoolArg("-txindex", DEFAULT_TXINDEX); - pblocktree->WriteFlag("txindex", fTxIndex); - LogPrintf("Initializing databases...\n"); - - // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) - if (!fReindex) { - try { - CBlock &block = const_cast<CBlock&>(chainparams.GenesisBlock()); - // Start new block file - unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); - CDiskBlockPos blockPos; - CValidationState state; - if (!FindBlockPos(state, blockPos, nBlockSize+8, 0, block.GetBlockTime())) - return error("LoadBlockIndex(): FindBlockPos failed"); - if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) - return error("LoadBlockIndex(): writing genesis block to disk failed"); - CBlockIndex *pindex = AddToBlockIndex(block); - if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) - return error("LoadBlockIndex(): genesis block not accepted"); - // Force a chainstate write so that when we VerifyDB in a moment, it doesn't check stale data - return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); - } catch (const std::runtime_error& e) { - return error("LoadBlockIndex(): failed to initialize block database: %s", e.what()); - } + try { + const CBlock& block = chainparams.GenesisBlock(); + FlatFilePos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr); + if (blockPos.IsNull()) + return error("%s: writing genesis block to disk failed", __func__); + CBlockIndex *pindex = m_blockman.AddToBlockIndex(block); + ReceivedBlockTransactions(block, pindex, blockPos, chainparams.GetConsensus()); + } catch (const std::runtime_error& e) { + return error("%s: failed to write genesis block: %s", __func__, e.what()); } return true; } -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp) +bool LoadGenesisBlock(const CChainParams& chainparams) +{ + return ::ChainstateActive().LoadGenesisBlock(chainparams); +} + +void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) - static std::multimap<uint256, CDiskBlockPos> mapBlocksUnknownParent; + static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent; int64_t nStart = GetTimeMillis(); int nLoaded = 0; @@ -3869,7 +4665,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB CBufferedFile blkdat(fileIn, 2*MAX_BLOCK_SERIALIZED_SIZE, MAX_BLOCK_SERIALIZED_SIZE+8, SER_DISK, CLIENT_VERSION); uint64_t nRewind = blkdat.GetPos(); while (!blkdat.eof()) { - boost::this_thread::interruption_point(); + if (ShutdownRequested()) return; blkdat.SetPos(nRewind); nRewind++; // start one byte further next time, in case of failure @@ -3880,7 +4676,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB unsigned char buf[CMessageHeader::MESSAGE_START_SIZE]; blkdat.FindByte(chainparams.MessageStart()[0]); nRewind = blkdat.GetPos()+1; - blkdat >> FLATDATA(buf); + blkdat >> buf; if (memcmp(buf, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE)) continue; // read size @@ -3903,32 +4699,37 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB blkdat >> block; nRewind = blkdat.GetPos(); - // detect out of order blocks, and store them for later uint256 hash = block.GetHash(); - if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex.find(block.hashPrevBlock) == mapBlockIndex.end()) { - LogPrint("reindex", "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), - block.hashPrevBlock.ToString()); - if (dbp) - mapBlocksUnknownParent.insert(std::make_pair(block.hashPrevBlock, *dbp)); - continue; - } - - // process in case the block isn't known yet - if (mapBlockIndex.count(hash) == 0 || (mapBlockIndex[hash]->nStatus & BLOCK_HAVE_DATA) == 0) { + { LOCK(cs_main); - CValidationState state; - if (AcceptBlock(pblock, state, chainparams, NULL, true, dbp, NULL)) - nLoaded++; - if (state.IsError()) - break; - } else if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex[hash]->nHeight % 1000 == 0) { - LogPrint("reindex", "Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight); + // detect out of order blocks, and store them for later + if (hash != chainparams.GetConsensus().hashGenesisBlock && !LookupBlockIndex(block.hashPrevBlock)) { + LogPrint(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), + block.hashPrevBlock.ToString()); + if (dbp) + mapBlocksUnknownParent.insert(std::make_pair(block.hashPrevBlock, *dbp)); + continue; + } + + // process in case the block isn't known yet + CBlockIndex* pindex = LookupBlockIndex(hash); + if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { + BlockValidationState state; + if (::ChainstateActive().AcceptBlock(pblock, state, chainparams, nullptr, true, dbp, nullptr)) { + nLoaded++; + } + if (state.IsError()) { + break; + } + } else if (hash != chainparams.GetConsensus().hashGenesisBlock && pindex->nHeight % 1000 == 0) { + LogPrint(BCLog::REINDEX, "Block Import: already had block %s at height %d\n", hash.ToString(), pindex->nHeight); + } } // Activate the genesis block so normal node progress can continue if (hash == chainparams.GetConsensus().hashGenesisBlock) { - CValidationState state; - if (!ActivateBestChain(state, chainparams)) { + BlockValidationState state; + if (!ActivateBestChain(state, chainparams, nullptr)) { break; } } @@ -3941,17 +4742,17 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB while (!queue.empty()) { uint256 head = queue.front(); queue.pop_front(); - std::pair<std::multimap<uint256, CDiskBlockPos>::iterator, std::multimap<uint256, CDiskBlockPos>::iterator> range = mapBlocksUnknownParent.equal_range(head); + std::pair<std::multimap<uint256, FlatFilePos>::iterator, std::multimap<uint256, FlatFilePos>::iterator> range = mapBlocksUnknownParent.equal_range(head); while (range.first != range.second) { - std::multimap<uint256, CDiskBlockPos>::iterator it = range.first; + std::multimap<uint256, FlatFilePos>::iterator it = range.first; std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>(); if (ReadBlockFromDisk(*pblockrecursive, it->second, chainparams.GetConsensus())) { - LogPrint("reindex", "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), + LogPrint(BCLog::REINDEX, "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), head.ToString()); LOCK(cs_main); - CValidationState dummy; - if (AcceptBlock(pblockrecursive, dummy, chainparams, NULL, true, &it->second, NULL)) + BlockValidationState dummy; + if (::ChainstateActive().AcceptBlock(pblockrecursive, dummy, chainparams, nullptr, true, &it->second, nullptr)) { nLoaded++; queue.push_back(pblockrecursive->GetHash()); @@ -3969,12 +4770,10 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB } catch (const std::runtime_error& e) { AbortNode(std::string("System error: ") + e.what()); } - if (nLoaded > 0) - LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, GetTimeMillis() - nStart); - return nLoaded > 0; + LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, GetTimeMillis() - nStart); } -void static CheckBlockIndex(const Consensus::Params& consensusParams) +void CChainState::CheckBlockIndex(const Consensus::Params& consensusParams) { if (!fCheckBlockIndex) { return; @@ -3983,55 +4782,55 @@ void static CheckBlockIndex(const Consensus::Params& consensusParams) LOCK(cs_main); // During a reindex, we read the genesis block and call CheckBlockIndex before ActivateBestChain, - // so we have the genesis block in mapBlockIndex but no active chain. (A few of the tests when - // iterating the block tree require that chainActive has been initialized.) - if (chainActive.Height() < 0) { - assert(mapBlockIndex.size() <= 1); + // so we have the genesis block in m_blockman.m_block_index but no active chain. (A few of the + // tests when iterating the block tree require that m_chain has been initialized.) + if (m_chain.Height() < 0) { + assert(m_blockman.m_block_index.size() <= 1); return; } // Build forward-pointing map of the entire block tree. std::multimap<CBlockIndex*,CBlockIndex*> forward; - for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { - forward.insert(std::make_pair(it->second->pprev, it->second)); + for (const std::pair<const uint256, CBlockIndex*>& entry : m_blockman.m_block_index) { + forward.insert(std::make_pair(entry.second->pprev, entry.second)); } - assert(forward.size() == mapBlockIndex.size()); + assert(forward.size() == m_blockman.m_block_index.size()); - std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangeGenesis = forward.equal_range(NULL); + std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangeGenesis = forward.equal_range(nullptr); CBlockIndex *pindex = rangeGenesis.first->second; rangeGenesis.first++; - assert(rangeGenesis.first == rangeGenesis.second); // There is only one index entry with parent NULL. + assert(rangeGenesis.first == rangeGenesis.second); // There is only one index entry with parent nullptr. // Iterate over the entire block tree, using depth-first search. // Along the way, remember whether there are blocks on the path from genesis // block being explored which are the first to have certain properties. size_t nNodes = 0; int nHeight = 0; - CBlockIndex* pindexFirstInvalid = NULL; // Oldest ancestor of pindex which is invalid. - CBlockIndex* pindexFirstMissing = NULL; // Oldest ancestor of pindex which does not have BLOCK_HAVE_DATA. - CBlockIndex* pindexFirstNeverProcessed = NULL; // Oldest ancestor of pindex for which nTx == 0. - CBlockIndex* pindexFirstNotTreeValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TREE (regardless of being valid or not). - CBlockIndex* pindexFirstNotTransactionsValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not). - CBlockIndex* pindexFirstNotChainValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not). - CBlockIndex* pindexFirstNotScriptsValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_SCRIPTS (regardless of being valid or not). - while (pindex != NULL) { + CBlockIndex* pindexFirstInvalid = nullptr; // Oldest ancestor of pindex which is invalid. + CBlockIndex* pindexFirstMissing = nullptr; // Oldest ancestor of pindex which does not have BLOCK_HAVE_DATA. + CBlockIndex* pindexFirstNeverProcessed = nullptr; // Oldest ancestor of pindex for which nTx == 0. + CBlockIndex* pindexFirstNotTreeValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_TREE (regardless of being valid or not). + CBlockIndex* pindexFirstNotTransactionsValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not). + CBlockIndex* pindexFirstNotChainValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not). + CBlockIndex* pindexFirstNotScriptsValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_SCRIPTS (regardless of being valid or not). + while (pindex != nullptr) { nNodes++; - if (pindexFirstInvalid == NULL && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; - if (pindexFirstMissing == NULL && !(pindex->nStatus & BLOCK_HAVE_DATA)) pindexFirstMissing = pindex; - if (pindexFirstNeverProcessed == NULL && pindex->nTx == 0) pindexFirstNeverProcessed = pindex; - if (pindex->pprev != NULL && pindexFirstNotTreeValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex; - if (pindex->pprev != NULL && pindexFirstNotTransactionsValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) pindexFirstNotTransactionsValid = pindex; - if (pindex->pprev != NULL && pindexFirstNotChainValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) pindexFirstNotChainValid = pindex; - if (pindex->pprev != NULL && pindexFirstNotScriptsValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) pindexFirstNotScriptsValid = pindex; + if (pindexFirstInvalid == nullptr && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; + if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA)) pindexFirstMissing = pindex; + if (pindexFirstNeverProcessed == nullptr && pindex->nTx == 0) pindexFirstNeverProcessed = pindex; + if (pindex->pprev != nullptr && pindexFirstNotTreeValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex; + if (pindex->pprev != nullptr && pindexFirstNotTransactionsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) pindexFirstNotTransactionsValid = pindex; + if (pindex->pprev != nullptr && pindexFirstNotChainValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) pindexFirstNotChainValid = pindex; + if (pindex->pprev != nullptr && pindexFirstNotScriptsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) pindexFirstNotScriptsValid = pindex; // Begin: actual consistency checks. - if (pindex->pprev == NULL) { + if (pindex->pprev == nullptr) { // Genesis block checks. assert(pindex->GetBlockHash() == consensusParams.hashGenesisBlock); // Genesis block's hash must match. - assert(pindex == chainActive.Genesis()); // The current active chain's genesis block must be this block. + assert(pindex == m_chain.Genesis()); // The current active chain's genesis block must be this block. } - if (pindex->nChainTx == 0) assert(pindex->nSequenceId <= 0); // nSequenceId can't be set positive for blocks that aren't linked (negative is used for preciousblock) + if (!pindex->HaveTxsDownloaded()) assert(pindex->nSequenceId <= 0); // nSequenceId can't be set positive for blocks that aren't linked (negative is used for preciousblock) // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred). // HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred. if (!fHavePruned) { @@ -4044,38 +4843,38 @@ void static CheckBlockIndex(const Consensus::Params& consensusParams) } if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA); assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. - // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to nChainTx being set. - assert((pindexFirstNeverProcessed != NULL) == (pindex->nChainTx == 0)); // nChainTx != 0 is used to signal that all parent blocks have been processed (but may have been pruned). - assert((pindexFirstNotTransactionsValid != NULL) == (pindex->nChainTx == 0)); + // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to HaveTxsDownloaded(). + assert((pindexFirstNeverProcessed == nullptr) == pindex->HaveTxsDownloaded()); + assert((pindexFirstNotTransactionsValid == nullptr) == pindex->HaveTxsDownloaded()); assert(pindex->nHeight == nHeight); // nHeight must be consistent. - assert(pindex->pprev == NULL || pindex->nChainWork >= pindex->pprev->nChainWork); // For every block except the genesis block, the chainwork must be larger than the parent's. + assert(pindex->pprev == nullptr || pindex->nChainWork >= pindex->pprev->nChainWork); // For every block except the genesis block, the chainwork must be larger than the parent's. assert(nHeight < 2 || (pindex->pskip && (pindex->pskip->nHeight < nHeight))); // The pskip pointer must point back for all but the first 2 blocks. - assert(pindexFirstNotTreeValid == NULL); // All mapBlockIndex entries must at least be TREE valid - if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TREE) assert(pindexFirstNotTreeValid == NULL); // TREE valid implies all parents are TREE valid - if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_CHAIN) assert(pindexFirstNotChainValid == NULL); // CHAIN valid implies all parents are CHAIN valid - if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_SCRIPTS) assert(pindexFirstNotScriptsValid == NULL); // SCRIPTS valid implies all parents are SCRIPTS valid - if (pindexFirstInvalid == NULL) { + assert(pindexFirstNotTreeValid == nullptr); // All m_blockman.m_block_index entries must at least be TREE valid + if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TREE) assert(pindexFirstNotTreeValid == nullptr); // TREE valid implies all parents are TREE valid + if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_CHAIN) assert(pindexFirstNotChainValid == nullptr); // CHAIN valid implies all parents are CHAIN valid + if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_SCRIPTS) assert(pindexFirstNotScriptsValid == nullptr); // SCRIPTS valid implies all parents are SCRIPTS valid + if (pindexFirstInvalid == nullptr) { // Checks for not-invalid blocks. assert((pindex->nStatus & BLOCK_FAILED_MASK) == 0); // The failed mask cannot be set for blocks without invalid parents. } - if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstNeverProcessed == NULL) { - if (pindexFirstInvalid == NULL) { + if (!CBlockIndexWorkComparator()(pindex, m_chain.Tip()) && pindexFirstNeverProcessed == nullptr) { + if (pindexFirstInvalid == nullptr) { // If this block sorts at least as good as the current tip and // is valid and we have all data for its parents, it must be in - // setBlockIndexCandidates. chainActive.Tip() must also be there + // setBlockIndexCandidates. m_chain.Tip() must also be there // even if some data has been pruned. - if (pindexFirstMissing == NULL || pindex == chainActive.Tip()) { + if (pindexFirstMissing == nullptr || pindex == m_chain.Tip()) { assert(setBlockIndexCandidates.count(pindex)); } // If some parent is missing, then it could be that this block was in // setBlockIndexCandidates but had to be removed because of the missing data. - // In this case it must be in mapBlocksUnlinked -- see test below. + // In this case it must be in m_blocks_unlinked -- see test below. } } else { // If this block sorts worse than the current tip or some ancestor's block has never been seen, it cannot be in setBlockIndexCandidates. assert(setBlockIndexCandidates.count(pindex) == 0); } - // Check whether this block is in mapBlocksUnlinked. - std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangeUnlinked = mapBlocksUnlinked.equal_range(pindex->pprev); + // Check whether this block is in m_blocks_unlinked. + std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangeUnlinked = m_blockman.m_blocks_unlinked.equal_range(pindex->pprev); bool foundInUnlinked = false; while (rangeUnlinked.first != rangeUnlinked.second) { assert(rangeUnlinked.first->first == pindex->pprev); @@ -4085,25 +4884,25 @@ void static CheckBlockIndex(const Consensus::Params& consensusParams) } rangeUnlinked.first++; } - if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed != NULL && pindexFirstInvalid == NULL) { - // If this block has block data available, some parent was never received, and has no invalid parents, it must be in mapBlocksUnlinked. + if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed != nullptr && pindexFirstInvalid == nullptr) { + // If this block has block data available, some parent was never received, and has no invalid parents, it must be in m_blocks_unlinked. assert(foundInUnlinked); } - if (!(pindex->nStatus & BLOCK_HAVE_DATA)) assert(!foundInUnlinked); // Can't be in mapBlocksUnlinked if we don't HAVE_DATA - if (pindexFirstMissing == NULL) assert(!foundInUnlinked); // We aren't missing data for any parent -- cannot be in mapBlocksUnlinked. - if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == NULL && pindexFirstMissing != NULL) { + if (!(pindex->nStatus & BLOCK_HAVE_DATA)) assert(!foundInUnlinked); // Can't be in m_blocks_unlinked if we don't HAVE_DATA + if (pindexFirstMissing == nullptr) assert(!foundInUnlinked); // We aren't missing data for any parent -- cannot be in m_blocks_unlinked. + if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == nullptr && pindexFirstMissing != nullptr) { // We HAVE_DATA for this block, have received data for all parents at some point, but we're currently missing data for some parent. assert(fHavePruned); // We must have pruned. - // This block may have entered mapBlocksUnlinked if: + // This block may have entered m_blocks_unlinked if: // - it has a descendant that at some point had more work than the // tip, and // - we tried switching to that descendant but were missing - // data for some intermediate block between chainActive and the + // data for some intermediate block between m_chain and the // tip. - // So if this block is itself better than chainActive.Tip() and it wasn't in - // setBlockIndexCandidates, then it must be in mapBlocksUnlinked. - if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && setBlockIndexCandidates.count(pindex) == 0) { - if (pindexFirstInvalid == NULL) { + // So if this block is itself better than m_chain.Tip() and it wasn't in + // setBlockIndexCandidates, then it must be in m_blocks_unlinked. + if (!CBlockIndexWorkComparator()(pindex, m_chain.Tip()) && setBlockIndexCandidates.count(pindex) == 0) { + if (pindexFirstInvalid == nullptr) { assert(foundInUnlinked); } } @@ -4124,13 +4923,13 @@ void static CheckBlockIndex(const Consensus::Params& consensusParams) while (pindex) { // We are going to either move to a parent or a sibling of pindex. // If pindex was the first with a certain property, unset the corresponding variable. - if (pindex == pindexFirstInvalid) pindexFirstInvalid = NULL; - if (pindex == pindexFirstMissing) pindexFirstMissing = NULL; - if (pindex == pindexFirstNeverProcessed) pindexFirstNeverProcessed = NULL; - if (pindex == pindexFirstNotTreeValid) pindexFirstNotTreeValid = NULL; - if (pindex == pindexFirstNotTransactionsValid) pindexFirstNotTransactionsValid = NULL; - if (pindex == pindexFirstNotChainValid) pindexFirstNotChainValid = NULL; - if (pindex == pindexFirstNotScriptsValid) pindexFirstNotScriptsValid = NULL; + if (pindex == pindexFirstInvalid) pindexFirstInvalid = nullptr; + if (pindex == pindexFirstMissing) pindexFirstMissing = nullptr; + if (pindex == pindexFirstNeverProcessed) pindexFirstNeverProcessed = nullptr; + if (pindex == pindexFirstNotTreeValid) pindexFirstNotTreeValid = nullptr; + if (pindex == pindexFirstNotTransactionsValid) pindexFirstNotTransactionsValid = nullptr; + if (pindex == pindexFirstNotChainValid) pindexFirstNotChainValid = nullptr; + if (pindex == pindexFirstNotScriptsValid) pindexFirstNotScriptsValid = nullptr; // Find our parent. CBlockIndex* pindexPar = pindex->pprev; // Find which child we just visited. @@ -4158,29 +4957,51 @@ void static CheckBlockIndex(const Consensus::Params& consensusParams) assert(nNodes == forward.size()); } +std::string CChainState::ToString() +{ + CBlockIndex* tip = m_chain.Tip(); + return strprintf("Chainstate [%s] @ height %d (%s)", + m_from_snapshot_blockhash.IsNull() ? "ibd" : "snapshot", + tip ? tip->nHeight : -1, tip ? tip->GetBlockHash().ToString() : "null"); +} + std::string CBlockFileInfo::ToString() const { - return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, DateTimeStrFormat("%Y-%m-%d", nTimeFirst), DateTimeStrFormat("%Y-%m-%d", nTimeLast)); + return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); +} + +CBlockFileInfo* GetBlockFileInfo(size_t n) +{ + LOCK(cs_LastBlockFile); + + return &vinfoBlockFile.at(n); } ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::DeploymentPos pos) { LOCK(cs_main); - return VersionBitsState(chainActive.Tip(), params, pos, versionbitscache); + return VersionBitsState(::ChainActive().Tip(), params, pos, versionbitscache); +} + +BIP9Stats VersionBitsTipStatistics(const Consensus::Params& params, Consensus::DeploymentPos pos) +{ + LOCK(cs_main); + return VersionBitsStatistics(::ChainActive().Tip(), params, pos); } int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::DeploymentPos pos) { LOCK(cs_main); - return VersionBitsStateSinceHeight(chainActive.Tip(), params, pos, versionbitscache); + return VersionBitsStateSinceHeight(::ChainActive().Tip(), params, pos, versionbitscache); } static const uint64_t MEMPOOL_DUMP_VERSION = 1; -bool LoadMempool(void) +bool LoadMempool(CTxMemPool& pool) { - int64_t nExpiryTimeout = GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; - FILE* filestr = fopen((GetDataDir() / "mempool.dat").string().c_str(), "r"); + const CChainParams& chainparams = Params(); + int64_t nExpiryTimeout = gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; + FILE* filestr = fsbridge::fopen(GetDataDir() / "mempool.dat", "rb"); CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); if (file.IsNull()) { LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); @@ -4188,8 +5009,10 @@ bool LoadMempool(void) } int64_t count = 0; - int64_t skipped = 0; + int64_t expired = 0; int64_t failed = 0; + int64_t already_there = 0; + int64_t unbroadcast = 0; int64_t nNow = GetTime(); try { @@ -4200,7 +5023,6 @@ bool LoadMempool(void) } uint64_t num; file >> num; - double prioritydummy = 0; while (num--) { CTransactionRef tx; int64_t nTime; @@ -4211,19 +5033,29 @@ bool LoadMempool(void) CAmount amountdelta = nFeeDelta; if (amountdelta) { - mempool.PrioritiseTransaction(tx->GetHash(), tx->GetHash().ToString(), prioritydummy, amountdelta); + pool.PrioritiseTransaction(tx->GetHash(), amountdelta); } - CValidationState state; + TxValidationState state; if (nTime + nExpiryTimeout > nNow) { LOCK(cs_main); - AcceptToMemoryPoolWithTime(mempool, state, tx, true, NULL, nTime); + AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, nTime, + nullptr /* plTxnReplaced */, false /* bypass_limits */, 0 /* nAbsurdFee */, + false /* test_accept */); if (state.IsValid()) { ++count; } else { - ++failed; + // mempool may contain the transaction already, e.g. from + // wallet(s) having loaded it while we were processing + // mempool transactions; consider these as valid, instead of + // failed, but mark them as 'already there' + if (pool.exists(tx->GetHash())) { + ++already_there; + } else { + ++failed; + } } } else { - ++skipped; + ++expired; } if (ShutdownRequested()) return false; @@ -4232,38 +5064,58 @@ bool LoadMempool(void) file >> mapDeltas; for (const auto& i : mapDeltas) { - mempool.PrioritiseTransaction(i.first, i.first.ToString(), prioritydummy, i.second); + pool.PrioritiseTransaction(i.first, i.second); + } + + // TODO: remove this try except in v0.22 + try { + std::set<uint256> unbroadcast_txids; + file >> unbroadcast_txids; + unbroadcast = unbroadcast_txids.size(); + + for (const auto& txid : unbroadcast_txids) { + pool.AddUnbroadcastTx(txid); + } + } catch (const std::exception&) { + // mempool.dat files created prior to v0.21 will not have an + // unbroadcast set. No need to log a failure if parsing fails here. } + } catch (const std::exception& e) { LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); return false; } - LogPrintf("Imported mempool transactions from disk: %i successes, %i failed, %i expired\n", count, failed, skipped); + LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); return true; } -void DumpMempool(void) +bool DumpMempool(const CTxMemPool& pool) { int64_t start = GetTimeMicros(); std::map<uint256, CAmount> mapDeltas; std::vector<TxMempoolInfo> vinfo; + std::set<uint256> unbroadcast_txids; + + static Mutex dump_mutex; + LOCK(dump_mutex); { - LOCK(mempool.cs); - for (const auto &i : mempool.mapDeltas) { - mapDeltas[i.first] = i.second.second; + LOCK(pool.cs); + for (const auto &i : pool.mapDeltas) { + mapDeltas[i.first] = i.second; } - vinfo = mempool.infoAll(); + vinfo = pool.infoAll(); + unbroadcast_txids = pool.GetUnbroadcastTxs(); } int64_t mid = GetTimeMicros(); try { - FILE* filestr = fopen((GetDataDir() / "mempool.dat.new").string().c_str(), "w"); + FILE* filestr = fsbridge::fopen(GetDataDir() / "mempool.dat.new", "wb"); if (!filestr) { - return; + return false; } CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); @@ -4274,28 +5126,36 @@ void DumpMempool(void) file << (uint64_t)vinfo.size(); for (const auto& i : vinfo) { file << *(i.tx); - file << (int64_t)i.nTime; - file << (int64_t)i.nFeeDelta; + file << int64_t{count_seconds(i.m_time)}; + file << int64_t{i.nFeeDelta}; mapDeltas.erase(i.tx->GetHash()); } file << mapDeltas; - FileCommit(file.Get()); + + LogPrintf("Writing %d unbroadcast transactions to disk.\n", unbroadcast_txids.size()); + file << unbroadcast_txids; + + if (!FileCommit(file.Get())) + throw std::runtime_error("FileCommit failed"); file.fclose(); RenameOver(GetDataDir() / "mempool.dat.new", GetDataDir() / "mempool.dat"); int64_t last = GetTimeMicros(); - LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", (mid-start)*0.000001, (last-mid)*0.000001); + LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", (mid-start)*MICRO, (last-mid)*MICRO); } catch (const std::exception& e) { LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); + return false; } + return true; } //! Guess how far we are in the verification process at the given block index -double GuessVerificationProgress(const ChainTxData& data, CBlockIndex *pindex) { - if (pindex == NULL) +//! require cs_main if pindex has not been validated yet (because nChainTx might be unset) +double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pindex) { + if (pindex == nullptr) return 0.0; - int64_t nNow = time(NULL); + int64_t nNow = time(nullptr); double fTxTotal; @@ -4305,7 +5165,7 @@ double GuessVerificationProgress(const ChainTxData& data, CBlockIndex *pindex) { fTxTotal = pindex->nChainTx + (nNow - pindex->GetBlockTime()) * data.dTxRate; } - return pindex->nChainTx / fTxTotal; + return std::min<double>(pindex->nChainTx / fTxTotal, 1.0); } class CMainCleanup @@ -4314,9 +5174,99 @@ public: CMainCleanup() {} ~CMainCleanup() { // block headers - BlockMap::iterator it1 = mapBlockIndex.begin(); - for (; it1 != mapBlockIndex.end(); it1++) + BlockMap::iterator it1 = g_chainman.BlockIndex().begin(); + for (; it1 != g_chainman.BlockIndex().end(); it1++) delete (*it1).second; - mapBlockIndex.clear(); + g_chainman.BlockIndex().clear(); + } +}; +static CMainCleanup instance_of_cmaincleanup; + +Optional<uint256> ChainstateManager::SnapshotBlockhash() const { + if (m_active_chainstate != nullptr) { + // If a snapshot chainstate exists, it will always be our active. + return m_active_chainstate->m_from_snapshot_blockhash; + } + return {}; +} + +std::vector<CChainState*> ChainstateManager::GetAll() +{ + std::vector<CChainState*> out; + + if (!IsSnapshotValidated() && m_ibd_chainstate) { + out.push_back(m_ibd_chainstate.get()); + } + + if (m_snapshot_chainstate) { + out.push_back(m_snapshot_chainstate.get()); + } + + return out; +} + +CChainState& ChainstateManager::InitializeChainstate(const uint256& snapshot_blockhash) +{ + bool is_snapshot = !snapshot_blockhash.IsNull(); + std::unique_ptr<CChainState>& to_modify = + is_snapshot ? m_snapshot_chainstate : m_ibd_chainstate; + + if (to_modify) { + throw std::logic_error("should not be overwriting a chainstate"); } -} instance_of_cmaincleanup; + + to_modify.reset(new CChainState(m_blockman, snapshot_blockhash)); + + // Snapshot chainstates and initial IBD chaintates always become active. + if (is_snapshot || (!is_snapshot && !m_active_chainstate)) { + LogPrintf("Switching active chainstate to %s\n", to_modify->ToString()); + m_active_chainstate = to_modify.get(); + } else { + throw std::logic_error("unexpected chainstate activation"); + } + + return *to_modify; +} + +CChain& ChainstateManager::ActiveChain() const +{ + assert(m_active_chainstate); + return m_active_chainstate->m_chain; +} + +bool ChainstateManager::IsSnapshotActive() const +{ + return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); +} + +CChainState& ChainstateManager::ValidatedChainstate() const +{ + if (m_snapshot_chainstate && IsSnapshotValidated()) { + return *m_snapshot_chainstate.get(); + } + assert(m_ibd_chainstate); + return *m_ibd_chainstate.get(); +} + +bool ChainstateManager::IsBackgroundIBD(CChainState* chainstate) const +{ + return (m_snapshot_chainstate && chainstate == m_ibd_chainstate.get()); +} + +void ChainstateManager::Unload() +{ + for (CChainState* chainstate : this->GetAll()) { + chainstate->m_chain.SetTip(nullptr); + chainstate->UnloadBlockIndex(); + } + + m_blockman.Unload(); +} + +void ChainstateManager::Reset() +{ + m_ibd_chainstate.reset(); + m_snapshot_chainstate.reset(); + m_active_chainstate = nullptr; + m_snapshot_validated = false; +} |