diff options
Diffstat (limited to 'src/main.cpp')
| -rw-r--r-- | src/main.cpp | 1510 |
1 files changed, 1136 insertions, 374 deletions
diff --git a/src/main.cpp b/src/main.cpp index ee30954b4..db457f6f5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -7,6 +7,7 @@ #include "addrman.h" #include "arith_uint256.h" +#include "blockencodings.h" #include "chainparams.h" #include "checkpoints.h" #include "checkqueue.h" @@ -37,9 +38,11 @@ #include "validationinterface.h" #include "versionbits.h" +#include <atomic> #include <sstream> #include <boost/algorithm/string/replace.hpp> +#include <boost/algorithm/string/join.hpp> #include <boost/filesystem.hpp> #include <boost/filesystem/fstream.hpp> #include <boost/math/distributions/poisson.hpp> @@ -71,7 +74,6 @@ bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; bool fRequireStandard = true; -unsigned int nBytesPerSigOp = DEFAULT_BYTES_PER_SIGOP; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; size_t nCoinCacheUsage = 5000 * 300; @@ -79,25 +81,31 @@ uint64_t nPruneTarget = 0; int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE; bool fEnableReplacement = DEFAULT_ENABLE_REPLACEMENT; + CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; CTxMemPool mempool(::minRelayTxFee); FeeFilterRounder filterRounder(::minRelayTxFee); +struct IteratorComparator +{ + template<typename I> + bool operator()(const I& a, const I& b) + { + return &(*a) < &(*b); + } +}; + struct COrphanTx { CTransaction tx; NodeId fromPeer; + int64_t nTimeExpire; }; map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(cs_main); -map<uint256, set<uint256> > mapOrphanTransactionsByPrev GUARDED_BY(cs_main); +map<COutPoint, set<map<uint256, COrphanTx>::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(cs_main); void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/** - * Returns true if there are nRequired or more blocks of minVersion or above - * in the last Consensus::Params::nMajorityWindow blocks, starting at pstart and going backwards. - */ -static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned nRequired, const Consensus::Params& consensusParams); static void CheckBlockIndex(const Consensus::Params& consensusParams); /** Constant stuff for coinbase transactions we create: */ @@ -194,11 +202,15 @@ namespace { /** Blocks that are in flight, and that are in the queue to be downloaded. Protected by cs_main. */ struct QueuedBlock { uint256 hash; - CBlockIndex* pindex; //!< Optional. - bool fValidatedHeaders; //!< Whether this block has validated headers at the time of request. + CBlockIndex* pindex; //!< Optional. + bool fValidatedHeaders; //!< Whether this block has validated headers at the time of request. + std::unique_ptr<PartiallyDownloadedBlock> partialBlock; //!< Optional, used for CMPCTBLOCK downloads }; map<uint256, pair<NodeId, list<QueuedBlock>::iterator> > mapBlocksInFlight; + /** Stack of nodes which we have set to announce using compact blocks */ + list<NodeId> lNodesAnnouncingHeaderAndIDs; + /** Number of preferable block download peers. */ int nPreferredDownload = 0; @@ -210,6 +222,12 @@ namespace { /** Number of peers from which we're downloading blocks. */ int nPeersWithValidatedDownloads = 0; + + /** Relay map, protected by cs_main. */ + typedef std::map<uint256, std::shared_ptr<const CTransaction>> MapRelay; + MapRelay mapRelay; + /** Expiration-time ordered list of (expire time, relay map entry) pairs, protected by cs_main). */ + std::deque<std::pair<int64_t, MapRelay::iterator>> vRelayExpiration; } // anon namespace ////////////////////////////////////////////////////////////////////////////// @@ -252,6 +270,8 @@ struct CNodeState { CBlockIndex *pindexLastCommonBlock; //! The best header we have sent our peer. CBlockIndex *pindexBestHeaderSent; + //! Length of current-streak of unconnecting headers announcements + int nUnconnectingHeaders; //! Whether we've started headers synchronization with this peer. bool fSyncStarted; //! Since when we're stalling block download progress (in microseconds), or 0. @@ -265,6 +285,12 @@ struct CNodeState { bool fPreferredDownload; //! Whether this peer wants invs or headers (when possible) for block announcements. bool fPreferHeaders; + //! Whether this peer wants invs or cmpctblocks (when possible) for block announcements. + bool fPreferHeaderAndIDs; + //! Whether this peer will send us cmpctblocks if we request them + bool fProvidesHeaderAndIDs; + //! Whether this peer can give us witnesses + bool fHaveWitness; CNodeState() { fCurrentlyConnected = false; @@ -274,6 +300,7 @@ struct CNodeState { hashLastUnknownBlock.SetNull(); pindexLastCommonBlock = NULL; pindexBestHeaderSent = NULL; + nUnconnectingHeaders = 0; fSyncStarted = false; nStallingSince = 0; nDownloadingSince = 0; @@ -281,6 +308,9 @@ struct CNodeState { nBlocksInFlightValidHeaders = 0; fPreferredDownload = false; fPreferHeaders = false; + fPreferHeaderAndIDs = false; + fProvidesHeaderAndIDs = false; + fHaveWitness = false; } }; @@ -349,6 +379,7 @@ void FinalizeNode(NodeId nodeid) { // Requires cs_main. // Returns a bool indicating whether we requested this block. +// Also used if a block was /not/ received and timed out or started with another peer bool MarkBlockAsReceived(const uint256& hash) { map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); if (itInFlight != mapBlocksInFlight.end()) { @@ -372,17 +403,26 @@ bool MarkBlockAsReceived(const uint256& hash) { } // Requires cs_main. -void MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, CBlockIndex *pindex = NULL) { +// returns false, still setting pit, if the block was already in flight from the same peer +// pit will only be valid as long as the same cs_main lock is being held +bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, CBlockIndex *pindex = NULL, list<QueuedBlock>::iterator **pit = NULL) { CNodeState *state = State(nodeid); assert(state != NULL); + // Short-circuit most stuff in case its from the same node + map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); + if (itInFlight != mapBlocksInFlight.end() && itInFlight->second.first == nodeid) { + *pit = &itInFlight->second.second; + return false; + } + // Make sure it's not listed somewhere already. MarkBlockAsReceived(hash); - QueuedBlock newentry = {hash, pindex, pindex != NULL}; - list<QueuedBlock>::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), newentry); + list<QueuedBlock>::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), + {hash, pindex, pindex != NULL, std::unique_ptr<PartiallyDownloadedBlock>(pit ? new PartiallyDownloadedBlock(&mempool) : NULL)}); state->nBlocksInFlight++; - state->nBlocksInFlightValidHeaders += newentry.fValidatedHeaders; + state->nBlocksInFlightValidHeaders += it->fValidatedHeaders; if (state->nBlocksInFlight == 1) { // We're starting a block download (batch) from this peer. state->nDownloadingSince = GetTimeMicros(); @@ -390,7 +430,10 @@ void MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Pa if (state->nBlocksInFlightValidHeaders == 1 && pindex != NULL) { nPeersWithValidatedDownloads++; } - mapBlocksInFlight[hash] = std::make_pair(nodeid, it); + itInFlight = mapBlocksInFlight.insert(std::make_pair(hash, std::make_pair(nodeid, it))).first; + if (pit) + *pit = &itInFlight->second.second; + return true; } /** Check whether the last unknown block a peer advertised is not yet known. */ @@ -426,6 +469,32 @@ void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { } } +void MaybeSetPeerAsAnnouncingHeaderAndIDs(const CNodeState* nodestate, CNode* pfrom) { + if (nLocalServices & NODE_WITNESS) { + // Don't ever request compact blocks when segwit is enabled. + return; + } + if (nodestate->fProvidesHeaderAndIDs) { + BOOST_FOREACH(const NodeId nodeid, lNodesAnnouncingHeaderAndIDs) + if (nodeid == pfrom->GetId()) + return; + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 1; + if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { + // As per BIP152, we only get 3 of our peers to announce + // blocks using compact encodings. + CNode* pnodeStop = FindNode(lNodesAnnouncingHeaderAndIDs.front()); + if (pnodeStop) { + pnodeStop->PushMessage(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + lNodesAnnouncingHeaderAndIDs.pop_front(); + } + } + fAnnounceUsingCMPCTBLOCK = true; + pfrom->PushMessage(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); + } +} + // Requires cs_main bool CanDirectFetch(const Consensus::Params &consensusParams) { @@ -590,6 +659,9 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc CBlockIndex* pindex = (*mi).second; if (chain.Contains(pindex)) return pindex; + if (pindex->GetAncestor(chain.Height()) == chain.Tip()) { + return chain.Tip(); + } } } return chain.Genesis(); @@ -614,40 +686,42 @@ bool AddOrphanTx(const CTransaction& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(c // large transaction with a missing parent then we assume // it will rebroadcast it later, after the parent transaction(s) // have been mined or received. - // 10,000 orphans, each of which is at most 5,000 bytes big is - // at most 500 megabytes of orphans: - unsigned int sz = tx.GetSerializeSize(SER_NETWORK, CTransaction::CURRENT_VERSION); - if (sz > 5000) + // 100 orphans, each of which is at most 99,999 bytes big is + // at most 10 megabytes of orphans and somewhat more byprev index (in the worst case): + unsigned int sz = GetTransactionWeight(tx); + if (sz >= MAX_STANDARD_TX_WEIGHT) { LogPrint("mempool", "ignoring large orphan tx (size: %u, hash: %s)\n", sz, hash.ToString()); return false; } - mapOrphanTransactions[hash].tx = tx; - mapOrphanTransactions[hash].fromPeer = peer; - BOOST_FOREACH(const CTxIn& txin, tx.vin) - mapOrphanTransactionsByPrev[txin.prevout.hash].insert(hash); + auto ret = mapOrphanTransactions.emplace(hash, COrphanTx{tx, peer, GetTime() + ORPHAN_TX_EXPIRE_TIME}); + assert(ret.second); + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + mapOrphanTransactionsByPrev[txin.prevout].insert(ret.first); + } - LogPrint("mempool", "stored orphan tx %s (mapsz %u prevsz %u)\n", hash.ToString(), + LogPrint("mempool", "stored orphan tx %s (mapsz %u outsz %u)\n", hash.ToString(), mapOrphanTransactions.size(), mapOrphanTransactionsByPrev.size()); return true; } -void static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +int static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.find(hash); if (it == mapOrphanTransactions.end()) - return; + return 0; BOOST_FOREACH(const CTxIn& txin, it->second.tx.vin) { - map<uint256, set<uint256> >::iterator itPrev = mapOrphanTransactionsByPrev.find(txin.prevout.hash); + auto itPrev = mapOrphanTransactionsByPrev.find(txin.prevout); if (itPrev == mapOrphanTransactionsByPrev.end()) continue; - itPrev->second.erase(hash); + itPrev->second.erase(it); if (itPrev->second.empty()) mapOrphanTransactionsByPrev.erase(itPrev); } mapOrphanTransactions.erase(it); + return 1; } void EraseOrphansFor(NodeId peer) @@ -659,8 +733,7 @@ void EraseOrphansFor(NodeId peer) map<uint256, COrphanTx>::iterator maybeErase = iter++; // increment to avoid iterator becoming invalid if (maybeErase->second.fromPeer == peer) { - EraseOrphanTx(maybeErase->second.tx.GetHash()); - ++nErased; + nErased += EraseOrphanTx(maybeErase->second.tx.GetHash()); } } if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx from peer %d\n", nErased, peer); @@ -670,6 +743,26 @@ void EraseOrphansFor(NodeId peer) unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { unsigned int nEvicted = 0; + static int64_t nNextSweep; + int64_t nNow = GetTime(); + if (nNextSweep <= nNow) { + // Sweep out expired orphan pool entries: + int nErased = 0; + int64_t nMinExpTime = nNow + ORPHAN_TX_EXPIRE_TIME - ORPHAN_TX_EXPIRE_INTERVAL; + map<uint256, COrphanTx>::iterator iter = mapOrphanTransactions.begin(); + while (iter != mapOrphanTransactions.end()) + { + map<uint256, COrphanTx>::iterator maybeErase = iter++; + if (maybeErase->second.nTimeExpire <= nNow) { + nErased += EraseOrphanTx(maybeErase->second.tx.GetHash()); + } else { + nMinExpTime = std::min(maybeErase->second.nTimeExpire, nMinExpTime); + } + } + // Sweep again 5 minutes after the next entry that expires in order to batch the linear scan. + nNextSweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; + if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx due to expiration\n", nErased); + } while (mapOrphanTransactions.size() > nMaxOrphans) { // Evict a random orphan: @@ -689,7 +782,7 @@ bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) return true; if ((int64_t)tx.nLockTime < ((int64_t)tx.nLockTime < LOCKTIME_THRESHOLD ? (int64_t)nBlockHeight : nBlockTime)) return true; - BOOST_FOREACH(const CTxIn& txin, tx.vin) { + for (const auto& txin : tx.vin) { if (!(txin.nSequence == CTxIn::SEQUENCE_FINAL)) return false; } @@ -903,11 +996,11 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool unsigned int GetLegacySigOpCount(const CTransaction& tx) { unsigned int nSigOps = 0; - BOOST_FOREACH(const CTxIn& txin, tx.vin) + for (const auto& txin : tx.vin) { nSigOps += txin.scriptSig.GetSigOpCount(false); } - BOOST_FOREACH(const CTxOut& txout, tx.vout) + for (const auto& txout : tx.vout) { nSigOps += txout.scriptPubKey.GetSigOpCount(false); } @@ -929,8 +1022,24 @@ unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& in return nSigOps; } +int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags) +{ + int64_t nSigOps = GetLegacySigOpCount(tx) * WITNESS_SCALE_FACTOR; + + if (tx.IsCoinBase()) + return nSigOps; + if (flags & SCRIPT_VERIFY_P2SH) { + nSigOps += GetP2SHSigOpCount(tx, inputs) * WITNESS_SCALE_FACTOR; + } + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); + nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, i < tx.wit.vtxinwit.size() ? &tx.wit.vtxinwit[i].scriptWitness : NULL, flags); + } + return nSigOps; +} @@ -943,13 +1052,13 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) return state.DoS(10, false, REJECT_INVALID, "bad-txns-vin-empty"); if (tx.vout.empty()) return state.DoS(10, false, REJECT_INVALID, "bad-txns-vout-empty"); - // Size limits - if (::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION) > MAX_BLOCK_SIZE) + // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) + if (::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) > MAX_BLOCK_BASE_SIZE) return state.DoS(100, false, REJECT_INVALID, "bad-txns-oversize"); // Check for negative or overflow output values CAmount nValueOut = 0; - BOOST_FOREACH(const CTxOut& txout, tx.vout) + for (const auto& txout : tx.vout) { if (txout.nValue < 0) return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-negative"); @@ -962,7 +1071,7 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) // Check for duplicate inputs set<COutPoint> vInOutPoints; - BOOST_FOREACH(const CTxIn& txin, tx.vin) + for (const auto& txin : tx.vin) { if (vInOutPoints.count(txin.prevout)) return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); @@ -976,7 +1085,7 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) } else { - BOOST_FOREACH(const CTxIn& txin, tx.vin) + for (const auto& txin : tx.vin) if (txin.prevout.IsNull()) return state.DoS(10, false, REJECT_INVALID, "bad-txns-prevout-null"); } @@ -1005,7 +1114,7 @@ std::string FormatStateMessage(const CValidationState &state) } bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransaction& tx, bool fLimitFree, - bool* pfMissingInputs, CFeeRate* txFeeRate, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, + bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<uint256>& vHashTxnToUncache) { const uint256 hash = tx.GetHash(); @@ -1020,11 +1129,6 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C if (tx.IsCoinBase()) return state.DoS(100, false, REJECT_INVALID, "coinbase"); - // Rather not work on nonstandard transactions (unless -testnet/-regtest) - string reason; - if (fRequireStandard && !IsStandardTx(tx, reason)) - return state.DoS(0, false, REJECT_NONSTANDARD, reason); - // Don't relay version 2 transactions until CSV is active, and we can be // sure that such transactions will be mined (unless we're on // -testnet/-regtest). @@ -1033,6 +1137,17 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C return state.DoS(0, false, REJECT_NONSTANDARD, "premature-version2-tx"); } + // Reject transactions with witness before segregated witness activates (override with -prematurewitness) + bool witnessEnabled = IsWitnessEnabled(chainActive.Tip(), Params().GetConsensus()); + if (!GetBoolArg("-prematurewitness",false) && !tx.wit.IsNull() && !witnessEnabled) { + return state.DoS(0, false, REJECT_NONSTANDARD, "no-witness-yet", true); + } + + // Rather not work on nonstandard transactions (unless -testnet/-regtest) + string reason; + if (fRequireStandard && !IsStandardTx(tx, reason, witnessEnabled)) + return state.DoS(0, false, REJECT_NONSTANDARD, reason); + // Only accept nLockTime-using transactions that can be mined in the next // block; we don't want our mempool filled up with transactions that can't // be mined yet. @@ -1049,9 +1164,10 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C LOCK(pool.cs); // protect pool.mapNextTx BOOST_FOREACH(const CTxIn &txin, tx.vin) { - if (pool.mapNextTx.count(txin.prevout)) + auto itConflicting = pool.mapNextTx.find(txin.prevout); + if (itConflicting != pool.mapNextTx.end()) { - const CTransaction *ptxConflicting = pool.mapNextTx[txin.prevout].ptx; + const CTransaction *ptxConflicting = itConflicting->second; if (!setConflicts.count(ptxConflicting->GetHash())) { // Allow opt-out of transaction replacement by setting @@ -1144,8 +1260,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C if (fRequireStandard && !AreInputsStandard(tx, view)) return state.Invalid(false, REJECT_NONSTANDARD, "bad-txns-nonstandard-inputs"); - unsigned int nSigOps = GetLegacySigOpCount(tx); - nSigOps += GetP2SHSigOpCount(tx, view); + int64_t nSigOpsCost = GetTransactionSigOpCost(tx, view, STANDARD_SCRIPT_VERIFY_FLAGS); CAmount nValueOut = tx.GetValueOut(); CAmount nFees = nValueIn-nValueOut; @@ -1168,20 +1283,17 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C } } - CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), pool.HasNoInputsOf(tx), inChainInputValue, fSpendsCoinbase, nSigOps, lp); + CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), pool.HasNoInputsOf(tx), inChainInputValue, fSpendsCoinbase, nSigOpsCost, lp); unsigned int nSize = entry.GetTxSize(); - if (txFeeRate) { - *txFeeRate = CFeeRate(nFees, nSize); - } // Check that the transaction doesn't have an excessive number of // sigops, making it impossible to mine. Since the coinbase transaction // itself can contain sigops MAX_STANDARD_TX_SIGOPS is less than // MAX_BLOCK_SIGOPS; we still consider this an invalid rather than // merely non-standard transaction. - if ((nSigOps > MAX_STANDARD_TX_SIGOPS) || (nBytesPerSigOp && nSigOps > nSize / nBytesPerSigOp)) + if (nSigOpsCost > MAX_STANDARD_TX_SIGOPS_COST) return state.DoS(0, false, REJECT_NONSTANDARD, "bad-txns-too-many-sigops", false, - strprintf("%d", nSigOps)); + strprintf("%d", nSigOpsCost)); CAmount mempoolRejectFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(nSize); if (mempoolRejectFee > 0 && nModifiedFees < mempoolRejectFee) { @@ -1373,10 +1485,24 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C } } + unsigned int scriptVerifyFlags = STANDARD_SCRIPT_VERIFY_FLAGS; + if (!Params().RequireStandard()) { + scriptVerifyFlags = GetArg("-promiscuousmempoolflags", scriptVerifyFlags); + } + // Check against previous transactions // This is done last to help prevent CPU exhaustion denial-of-service attacks. - if (!CheckInputs(tx, state, view, true, STANDARD_SCRIPT_VERIFY_FLAGS, true)) - return false; // state filled in by CheckInputs + if (!CheckInputs(tx, state, view, true, scriptVerifyFlags, true)) { + // SCRIPT_VERIFY_CLEANSTACK requires SCRIPT_VERIFY_WITNESS, so we + // need to turn both off, and compare against just turning off CLEANSTACK + // to see if the failure is specifically due to witness validation. + if (CheckInputs(tx, state, view, true, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true) && + !CheckInputs(tx, state, view, true, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true)) { + // Only the witness is wrong, so the transaction itself may be fine. + state.SetCorruptionPossible(); + } + return false; + } // Check again against just the consensus-critical mandatory script // verification flags, in case of bugs in the standard flags that cause @@ -1415,16 +1541,16 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C } } - SyncWithWallets(tx, NULL, NULL); + SyncWithWallets(tx, NULL); return true; } bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, - bool* pfMissingInputs, CFeeRate* txFeeRate, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) + bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) { std::vector<uint256> vHashTxToUncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, txFeeRate, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); if (!res) { BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache) pcoinsTip->Uncache(hashTx); @@ -1439,8 +1565,10 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::P LOCK(cs_main); - if (mempool.lookup(hash, txOut)) + std::shared_ptr<const CTransaction> ptx = mempool.get(hash); + if (ptx) { + txOut = *ptx; return true; } @@ -1574,18 +1702,24 @@ CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams) bool IsInitialBlockDownload() { const CChainParams& chainParams = Params(); + + // Once this function has returned false, it must remain false. + static std::atomic<bool> latchToFalse{false}; + // Optimization: pre-test latch before taking the lock. + if (latchToFalse.load(std::memory_order_relaxed)) + return false; + LOCK(cs_main); + if (latchToFalse.load(std::memory_order_relaxed)) + return false; if (fImporting || fReindex) return true; if (fCheckpointsEnabled && chainActive.Height() < Checkpoints::GetTotalBlocksEstimate(chainParams.Checkpoints())) return true; - static bool lockIBDState = false; - if (lockIBDState) - return false; bool state = (chainActive.Height() < pindexBestHeader->nHeight - 24 * 6 || std::max(chainActive.Tip()->GetBlockTime(), pindexBestHeader->GetBlockTime()) < GetTime() - nMaxTipAge); if (!state) - lockIBDState = true; + latchToFalse.store(true, std::memory_order_relaxed); return state; } @@ -1775,7 +1909,8 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) bool CScriptCheck::operator()() { const CScript &scriptSig = ptxTo->vin[nIn].scriptSig; - if (!VerifyScript(scriptSig, scriptPubKey, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, cacheStore), &error)) { + const CScriptWitness *witness = (nIn < ptxTo->wit.vtxinwit.size()) ? &ptxTo->wit.vtxinwit[nIn].scriptWitness : NULL; + if (!VerifyScript(scriptSig, scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, amount, cacheStore), &error)) { return false; } return true; @@ -1883,8 +2018,8 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi // such nodes as they are not following the protocol. That // said during an upgrade careful thought should be taken // as to the correct behavior - we may want to continue - // peering with non-upgraded nodes even after a soft-fork - // super-majority vote has passed. + // peering with non-upgraded nodes even after soft-fork + // super-majority signaling has occurred. return state.DoS(100,false, REJECT_INVALID, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); } } @@ -2103,70 +2238,8 @@ void ThreadScriptCheck() { scriptcheckqueue.Thread(); } -// -// Called periodically asynchronously; alerts if it smells like -// we're being fed a bad chain (blocks being generated much -// too slowly or too quickly). -// -void PartitionCheck(bool (*initialDownloadCheck)(), CCriticalSection& cs, const CBlockIndex *const &bestHeader, - int64_t nPowTargetSpacing) -{ - if (bestHeader == NULL || initialDownloadCheck()) return; - - static int64_t lastAlertTime = 0; - int64_t now = GetAdjustedTime(); - if (lastAlertTime > now-60*60*24) return; // Alert at most once per day - - const int SPAN_HOURS=4; - const int SPAN_SECONDS=SPAN_HOURS*60*60; - int BLOCKS_EXPECTED = SPAN_SECONDS / nPowTargetSpacing; - - boost::math::poisson_distribution<double> poisson(BLOCKS_EXPECTED); - - std::string strWarning; - int64_t startTime = GetAdjustedTime()-SPAN_SECONDS; - - LOCK(cs); - const CBlockIndex* i = bestHeader; - int nBlocks = 0; - while (i->GetBlockTime() >= startTime) { - ++nBlocks; - i = i->pprev; - if (i == NULL) return; // Ran out of chain, we must not be fully sync'ed - } - - // How likely is it to find that many by chance? - double p = boost::math::pdf(poisson, nBlocks); - - LogPrint("partitioncheck", "%s: Found %d blocks in the last %d hours\n", __func__, nBlocks, SPAN_HOURS); - LogPrint("partitioncheck", "%s: likelihood: %g\n", __func__, p); - - // Aim for one false-positive about every fifty years of normal running: - const int FIFTY_YEARS = 50*365*24*60*60; - double alertThreshold = 1.0 / (FIFTY_YEARS / SPAN_SECONDS); - - if (p <= alertThreshold && nBlocks < BLOCKS_EXPECTED) - { - // Many fewer blocks than expected: alert! - strWarning = strprintf(_("WARNING: check your network connection, %d blocks received in the last %d hours (%d expected)"), - nBlocks, SPAN_HOURS, BLOCKS_EXPECTED); - } - else if (p <= alertThreshold && nBlocks > BLOCKS_EXPECTED) - { - // Many more blocks than expected: alert! - strWarning = strprintf(_("WARNING: abnormally high number of blocks generated, %d blocks received in the last %d hours (%d expected)"), - nBlocks, SPAN_HOURS, BLOCKS_EXPECTED); - } - if (!strWarning.empty()) - { - strMiscWarning = strWarning; - AlertNotify(strWarning); - lastAlertTime = now; - } -} - // Protected by cs_main -static VersionBitsCache versionbitscache; +VersionBitsCache versionbitscache; int32_t ComputeBlockVersion(const CBlockIndex* pindexPrev, const Consensus::Params& params) { @@ -2226,7 +2299,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin int64_t nTimeStart = GetTimeMicros(); // Check it again in case a previous version let a bad block in - if (!CheckBlock(block, state, !fJustCheck, !fJustCheck)) + if (!CheckBlock(block, state, chainparams.GetConsensus(), !fJustCheck, !fJustCheck)) return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); // verify that the view's current state corresponds to the previous block @@ -2294,15 +2367,13 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin unsigned int flags = fStrictPayToScriptHash ? SCRIPT_VERIFY_P2SH : SCRIPT_VERIFY_NONE; - // Start enforcing the DERSIG (BIP66) rules, for block.nVersion=3 blocks, - // when 75% of the network has upgraded: - if (block.nVersion >= 3 && IsSuperMajority(3, pindex->pprev, chainparams.GetConsensus().nMajorityEnforceBlockUpgrade, chainparams.GetConsensus())) { + // Start enforcing the DERSIG (BIP66) rule + if (pindex->nHeight >= chainparams.GetConsensus().BIP66Height) { flags |= SCRIPT_VERIFY_DERSIG; } - // Start enforcing CHECKLOCKTIMEVERIFY, (BIP65) for block.nVersion=4 - // blocks, when 75% of the network has upgraded: - if (block.nVersion >= 4 && IsSuperMajority(4, pindex->pprev, chainparams.GetConsensus().nMajorityEnforceBlockUpgrade, chainparams.GetConsensus())) { + // Start enforcing CHECKLOCKTIMEVERIFY (BIP65) rule + if (pindex->nHeight >= chainparams.GetConsensus().BIP65Height) { flags |= SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY; } @@ -2313,6 +2384,11 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin nLockTimeFlags |= LOCKTIME_VERIFY_SEQUENCE; } + // Start enforcing WITNESS rules using versionbits logic. + if (IsWitnessEnabled(pindex->pprev, chainparams.GetConsensus())) { + flags |= SCRIPT_VERIFY_WITNESS; + } + int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1; LogPrint("bench", " - Fork checks: %.2fms [%.2fs]\n", 0.001 * (nTime2 - nTime1), nTimeForks * 0.000001); @@ -2320,10 +2396,11 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin CCheckQueueControl<CScriptCheck> control(fScriptChecks && nScriptCheckThreads ? &scriptcheckqueue : NULL); + std::vector<uint256> vOrphanErase; std::vector<int> prevheights; CAmount nFees = 0; int nInputs = 0; - unsigned int nSigOps = 0; + int64_t nSigOpsCost = 0; CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); std::vector<std::pair<uint256, CDiskTxPos> > vPos; vPos.reserve(block.vtx.size()); @@ -2333,10 +2410,6 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin const CTransaction &tx = block.vtx[i]; nInputs += tx.vin.size(); - nSigOps += GetLegacySigOpCount(tx); - if (nSigOps > MAX_BLOCK_SIGOPS) - return state.DoS(100, error("ConnectBlock(): too many sigops"), - REJECT_INVALID, "bad-blk-sigops"); if (!tx.IsCoinBase()) { @@ -2352,22 +2425,34 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin prevheights[j] = view.AccessCoins(tx.vin[j].prevout.hash)->nHeight; } + // Which orphan pool entries must we evict? + for (size_t j = 0; j < tx.vin.size(); j++) { + auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); + if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { + const CTransaction& orphanTx = (*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + vOrphanErase.push_back(orphanHash); + } + } + if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { return state.DoS(100, error("%s: contains a non-BIP68-final transaction", __func__), REJECT_INVALID, "bad-txns-nonfinal"); } + } - if (fStrictPayToScriptHash) - { - // Add in sigops done by pay-to-script-hash inputs; - // this is to prevent a "rogue miner" from creating - // an incredibly-expensive-to-validate block. - nSigOps += GetP2SHSigOpCount(tx, view); - if (nSigOps > MAX_BLOCK_SIGOPS) - return state.DoS(100, error("ConnectBlock(): too many sigops"), - REJECT_INVALID, "bad-blk-sigops"); - } + // GetTransactionSigOpCost counts 3 types of sigops: + // * legacy (always) + // * p2sh (when P2SH enabled in flags and excludes coinbase) + // * witness (when witness enabled in flags and excludes coinbase) + nSigOpsCost += GetTransactionSigOpCost(tx, view, flags); + if (nSigOpsCost > MAX_BLOCK_SIGOPS_COST) + return state.DoS(100, error("ConnectBlock(): too many sigops"), + REJECT_INVALID, "bad-blk-sigops"); + if (!tx.IsCoinBase()) + { nFees += view.GetValueIn(tx)-tx.GetValueOut(); std::vector<CScriptCheck> vChecks; @@ -2439,6 +2524,15 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin GetMainSignals().UpdatedTransaction(hashPrevBestCoinBase); hashPrevBestCoinBase = block.vtx[0].GetHash(); + // Erase orphan transactions include or precluded by this block + if (vOrphanErase.size()) { + int nErased = 0; + BOOST_FOREACH(uint256 &orphanHash, vOrphanErase) { + nErased += EraseOrphanTx(orphanHash); + } + LogPrint("mempool", "Erased %d orphan tx included or conflicted by block\n", nErased); + } + int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5; LogPrint("bench", " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime6 - nTime5), nTimeCallbacks * 0.000001); @@ -2574,16 +2668,10 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { nTimeBestReceived = GetTime(); mempool.AddTransactionsUpdated(1); - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utx)\n", __func__, - chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, - log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, - DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), - Checkpoints::GuessVerificationProgress(chainParams.Checkpoints(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); - cvBlockChange.notify_all(); - // Check the version of the last 100 blocks to see if we need to upgrade: static bool fWarned = false; + std::vector<std::string> warningMessages; if (!IsInitialBlockDownload()) { int nUpgraded = 0; @@ -2599,10 +2687,11 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { fWarned = true; } } else { - LogPrintf("%s: unknown new rules are about to activate (versionbit %i)\n", __func__, bit); + warningMessages.push_back(strprintf("unknown new rules are about to activate (versionbit %i)", bit)); } } } + // Check the version of the last 100 blocks to see if we need to upgrade: for (int i = 0; i < 100 && pindex != NULL; i++) { int32_t nExpectedVersion = ComputeBlockVersion(pindex->pprev, chainParams.GetConsensus()); @@ -2611,7 +2700,7 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { pindex = pindex->pprev; } if (nUpgraded > 0) - LogPrintf("%s: %d of last 100 blocks have unexpected version\n", __func__, nUpgraded); + warningMessages.push_back(strprintf("%d of last 100 blocks have unexpected version", nUpgraded)); if (nUpgraded > 100/2) { // strMiscWarning is read by GetWarnings(), called by Qt and the JSON-RPC code to warn the user: @@ -2622,10 +2711,19 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { } } } + LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utx)", __func__, + chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, + log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, + DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), + Checkpoints::GuessVerificationProgress(chainParams.Checkpoints(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); + if (!warningMessages.empty()) + LogPrintf(" warning='%s'", boost::algorithm::join(warningMessages, ", ")); + LogPrintf("\n"); + } /** Disconnect chainActive's tip. You probably want to call mempool.removeForReorg and manually re-limit mempool size after this, with cs_main held. */ -bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams) +bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams, bool fBare = false) { CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); @@ -2645,30 +2743,34 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara // Write the chain state to disk, if necessary. if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; - // Resurrect mempool transactions from the disconnected block. - std::vector<uint256> vHashUpdate; - BOOST_FOREACH(const CTransaction &tx, block.vtx) { - // ignore validation errors in resurrected transactions - list<CTransaction> removed; - CValidationState stateDummy; - if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL, NULL, true)) { - mempool.removeRecursive(tx, removed); - } else if (mempool.exists(tx.GetHash())) { - vHashUpdate.push_back(tx.GetHash()); - } - } - // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have - // no in-mempool children, which is generally not true when adding - // previously-confirmed transactions back to the mempool. - // UpdateTransactionsFromBlock finds descendants of any transactions in this - // block that were added back and cleans up the mempool state. - mempool.UpdateTransactionsFromBlock(vHashUpdate); + + if (!fBare) { + // Resurrect mempool transactions from the disconnected block. + std::vector<uint256> vHashUpdate; + BOOST_FOREACH(const CTransaction &tx, block.vtx) { + // ignore validation errors in resurrected transactions + list<CTransaction> removed; + CValidationState stateDummy; + if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL, true)) { + mempool.removeRecursive(tx, removed); + } else if (mempool.exists(tx.GetHash())) { + vHashUpdate.push_back(tx.GetHash()); + } + } + // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have + // no in-mempool children, which is generally not true when adding + // previously-confirmed transactions back to the mempool. + // UpdateTransactionsFromBlock finds descendants of any transactions in this + // block that were added back and cleans up the mempool state. + mempool.UpdateTransactionsFromBlock(vHashUpdate); + } + // Update chainActive and related variables. UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: BOOST_FOREACH(const CTransaction &tx, block.vtx) { - SyncWithWallets(tx, pindexDelete->pprev, NULL); + SyncWithWallets(tx, pindexDelete->pprev); } return true; } @@ -2683,7 +2785,7 @@ static int64_t nTimePostConnect = 0; * Connect a new block to chainActive. pblock is either NULL or a pointer to a CBlock * corresponding to pindexNew, to bypass loading it again from disk. */ -bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const CBlock* pblock) +bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const CBlock* pblock, std::list<CTransaction> &txConflicted, std::vector<std::tuple<CTransaction,CBlockIndex*,int> > &txChanged) { assert(pindexNew->pprev == chainActive.Tip()); // Read block from disk. @@ -2719,20 +2821,13 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, return false; int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; LogPrint("bench", " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); - // Remove conflicting transactions from the mempool. - list<CTransaction> txConflicted; + // Remove conflicting transactions from the mempool.; mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, txConflicted, !IsInitialBlockDownload()); // Update chainActive & related variables. UpdateTip(pindexNew, chainparams); - // Tell wallet about transactions that went from mempool - // to conflicted: - BOOST_FOREACH(const CTransaction &tx, txConflicted) { - SyncWithWallets(tx, pindexNew, NULL); - } - // ... and about transactions that got confirmed: - BOOST_FOREACH(const CTransaction &tx, pblock->vtx) { - SyncWithWallets(tx, pindexNew, pblock); - } + + for(unsigned int i=0; i < pblock->vtx.size(); i++) + txChanged.push_back(std::make_tuple(pblock->vtx[i], pindexNew, i)); int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; LogPrint("bench", " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); @@ -2814,10 +2909,9 @@ static void PruneBlockIndexCandidates() { * Try to make some progress towards making pindexMostWork the active block. * pblock is either NULL or a pointer to a CBlock corresponding to pindexMostWork. */ -static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock) +static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock, bool& fInvalidFound, std::list<CTransaction>& txConflicted, std::vector<std::tuple<CTransaction,CBlockIndex*,int> >& txChanged) { AssertLockHeld(cs_main); - bool fInvalidFound = false; const CBlockIndex *pindexOldTip = chainActive.Tip(); const CBlockIndex *pindexFork = chainActive.FindFork(pindexMostWork); @@ -2848,7 +2942,7 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c // Connect new blocks. BOOST_REVERSE_FOREACH(CBlockIndex *pindexConnect, vpindexToConnect) { - if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : NULL)) { + if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : NULL, txConflicted, txChanged)) { if (state.IsInvalid()) { // The block violates a consensus rule. if (!state.CorruptionPossible()) @@ -2887,6 +2981,28 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c return true; } +static void NotifyHeaderTip() { + bool fNotify = false; + bool fInitialBlockDownload = false; + static CBlockIndex* pindexHeaderOld = NULL; + CBlockIndex* pindexHeader = NULL; + { + LOCK(cs_main); + if (!setBlockIndexCandidates.empty()) { + pindexHeader = *setBlockIndexCandidates.rbegin(); + } + if (pindexHeader != pindexHeaderOld) { + fNotify = true; + fInitialBlockDownload = IsInitialBlockDownload(); + pindexHeaderOld = pindexHeader; + } + } + // Send block tip changed notifications without cs_main + if (fNotify) { + uiInterface.NotifyHeaderTip(fInitialBlockDownload, pindexHeader); + } +} + /** * Make the best chain active, in multiple steps. The result is either failure * or an activated best chain. pblock is either NULL or a pointer to a block @@ -2894,33 +3010,55 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c */ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, const CBlock *pblock) { CBlockIndex *pindexMostWork = NULL; + CBlockIndex *pindexNewTip = NULL; do { boost::this_thread::interruption_point(); if (ShutdownRequested()) break; - CBlockIndex *pindexNewTip = NULL; const CBlockIndex *pindexFork; + std::list<CTransaction> txConflicted; + std::vector<std::tuple<CTransaction,CBlockIndex*,int> > txChanged; bool fInitialDownload; + int nNewHeight; { LOCK(cs_main); CBlockIndex *pindexOldTip = chainActive.Tip(); - pindexMostWork = FindMostWorkChain(); + if (pindexMostWork == NULL) { + pindexMostWork = FindMostWorkChain(); + } // Whether we have anything to do at all. if (pindexMostWork == NULL || pindexMostWork == chainActive.Tip()) return true; - if (!ActivateBestChainStep(state, chainparams, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : NULL)) + bool fInvalidFound = false; + if (!ActivateBestChainStep(state, chainparams, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : NULL, fInvalidFound, txConflicted, txChanged)) return false; + if (fInvalidFound) { + // Wipe cache, we may need another branch now. + pindexMostWork = NULL; + } pindexNewTip = chainActive.Tip(); pindexFork = chainActive.FindFork(pindexOldTip); fInitialDownload = IsInitialBlockDownload(); + nNewHeight = chainActive.Height(); } // When we reach this point, we switched to a new tip (stored in pindexNewTip). // Notifications/callbacks that can run without cs_main + + // throw all transactions though the signal-interface + // while _not_ holding the cs_main lock + BOOST_FOREACH(const CTransaction &tx, txConflicted) + { + SyncWithWallets(tx, pindexNewTip); + } + // ... and about transactions that got confirmed: + for(unsigned int i = 0; i < txChanged.size(); i++) + SyncWithWallets(std::get<0>(txChanged[i]), std::get<1>(txChanged[i]), std::get<2>(txChanged[i])); + // Always notify the UI if a new block tip was connected if (pindexFork != pindexNewTip) { uiInterface.NotifyBlockTip(fInitialDownload, pindexNewTip); @@ -2945,7 +3083,7 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) { - if (chainActive.Height() > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate)) { + if (nNewHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate)) { BOOST_REVERSE_FOREACH(const uint256& hash, vHashes) { pnode->PushBlockHash(hash); } @@ -2958,7 +3096,7 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, } } } - } while(pindexMostWork != chainActive.Tip()); + } while (pindexNewTip != pindexMostWork); CheckBlockIndex(chainparams.GetConsensus()); // Write changes periodically to disk, after relay. @@ -3084,6 +3222,9 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; + if (IsWitnessEnabled(pindexNew->pprev, Params().GetConsensus())) { + pindexNew->nStatus |= BLOCK_OPT_WITNESS; + } pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); setDirtyBlockIndex.insert(pindexNew); @@ -3209,20 +3350,16 @@ bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigne return true; } -bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, bool fCheckPOW) +bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW) { // Check proof of work matches claimed amount - if (fCheckPOW && !CheckProofOfWork(block.GetHash(), block.nBits, Params().GetConsensus())) + if (fCheckPOW && !CheckProofOfWork(block.GetHash(), block.nBits, consensusParams)) return state.DoS(50, false, REJECT_INVALID, "high-hash", false, "proof of work failed"); - // Check timestamp - if (block.GetBlockTime() > GetAdjustedTime() + 2 * 60 * 60) - return state.Invalid(false, REJECT_INVALID, "time-too-new", "block timestamp too far in the future"); - return true; } -bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bool fCheckMerkleRoot) +bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW, bool fCheckMerkleRoot) { // These are checks that are independent of context. @@ -3231,7 +3368,7 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo // Check that the header is valid (particularly PoW). This is mostly // redundant with the call in AcceptBlockHeader. - if (!CheckBlockHeader(block, state, fCheckPOW)) + if (!CheckBlockHeader(block, state, consensusParams, fCheckPOW)) return false; // Check the merkle root. @@ -3251,9 +3388,11 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo // All potential-corruption validation must be done before we do any // transaction validation, as otherwise we may mark the header as invalid // because we receive the wrong transactions for it. + // Note that witness malleability is checked in ContextualCheckBlock, so no + // checks that use witness data may be performed here. // Size limits - if (block.vtx.empty() || block.vtx.size() > MAX_BLOCK_SIZE || ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION) > MAX_BLOCK_SIZE) + if (block.vtx.empty() || block.vtx.size() > MAX_BLOCK_BASE_SIZE || ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) > MAX_BLOCK_BASE_SIZE) return state.DoS(100, false, REJECT_INVALID, "bad-blk-length", false, "size limits failed"); // First transaction must be coinbase, the rest must not be @@ -3264,17 +3403,17 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo return state.DoS(100, false, REJECT_INVALID, "bad-cb-multiple", false, "more than one coinbase"); // Check transactions - BOOST_FOREACH(const CTransaction& tx, block.vtx) + for (const auto& tx : block.vtx) if (!CheckTransaction(tx, state)) return state.Invalid(false, state.GetRejectCode(), state.GetRejectReason(), strprintf("Transaction check failed (tx hash %s) %s", tx.GetHash().ToString(), state.GetDebugMessage())); unsigned int nSigOps = 0; - BOOST_FOREACH(const CTransaction& tx, block.vtx) + for (const auto& tx : block.vtx) { nSigOps += GetLegacySigOpCount(tx); } - if (nSigOps > MAX_BLOCK_SIGOPS) + if (nSigOps * WITNESS_SCALE_FACTOR > MAX_BLOCK_SIGOPS_COST) return state.DoS(100, false, REJECT_INVALID, "bad-blk-sigops", false, "out-of-bounds SigOpCount"); if (fCheckPOW && fCheckMerkleRoot) @@ -3297,9 +3436,74 @@ static bool CheckIndexAgainstCheckpoint(const CBlockIndex* pindexPrev, CValidati return true; } -bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex * const pindexPrev) +bool IsWitnessEnabled(const CBlockIndex* pindexPrev, const Consensus::Params& params) { - const Consensus::Params& consensusParams = Params().GetConsensus(); + LOCK(cs_main); + return (VersionBitsState(pindexPrev, params, Consensus::DEPLOYMENT_SEGWIT, versionbitscache) == THRESHOLD_ACTIVE); +} + +// Compute at which vout of the block's coinbase transaction the witness +// commitment occurs, or -1 if not found. +static int GetWitnessCommitmentIndex(const CBlock& block) +{ + int commitpos = -1; + for (size_t o = 0; o < block.vtx[0].vout.size(); o++) { + if (block.vtx[0].vout[o].scriptPubKey.size() >= 38 && block.vtx[0].vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0].vout[o].scriptPubKey[1] == 0x24 && block.vtx[0].vout[o].scriptPubKey[2] == 0xaa && block.vtx[0].vout[o].scriptPubKey[3] == 0x21 && block.vtx[0].vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0].vout[o].scriptPubKey[5] == 0xed) { + commitpos = o; + } + } + return commitpos; +} + +void UpdateUncommittedBlockStructures(CBlock& block, const CBlockIndex* pindexPrev, const Consensus::Params& consensusParams) +{ + int commitpos = GetWitnessCommitmentIndex(block); + static const std::vector<unsigned char> nonce(32, 0x00); + if (commitpos != -1 && IsWitnessEnabled(pindexPrev, consensusParams) && block.vtx[0].wit.IsEmpty()) { + block.vtx[0].wit.vtxinwit.resize(1); + block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.resize(1); + block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0] = nonce; + } +} + +std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBlockIndex* pindexPrev, const Consensus::Params& consensusParams) +{ + std::vector<unsigned char> commitment; + int commitpos = GetWitnessCommitmentIndex(block); + bool fHaveWitness = false; + for (size_t t = 1; t < block.vtx.size(); t++) { + if (!block.vtx[t].wit.IsNull()) { + fHaveWitness = true; + break; + } + } + std::vector<unsigned char> ret(32, 0x00); + if (fHaveWitness && IsWitnessEnabled(pindexPrev, consensusParams)) { + if (commitpos == -1) { + uint256 witnessroot = BlockWitnessMerkleRoot(block, NULL); + CHash256().Write(witnessroot.begin(), 32).Write(&ret[0], 32).Finalize(witnessroot.begin()); + CTxOut out; + out.nValue = 0; + out.scriptPubKey.resize(38); + out.scriptPubKey[0] = OP_RETURN; + out.scriptPubKey[1] = 0x24; + out.scriptPubKey[2] = 0xaa; + out.scriptPubKey[3] = 0x21; + out.scriptPubKey[4] = 0xa9; + out.scriptPubKey[5] = 0xed; + memcpy(&out.scriptPubKey[6], witnessroot.begin(), 32); + commitment = std::vector<unsigned char>(out.scriptPubKey.begin(), out.scriptPubKey.end()); + const_cast<std::vector<CTxOut>*>(&block.vtx[0].vout)->push_back(out); + block.vtx[0].UpdateHash(); + } + } + UpdateUncommittedBlockStructures(block, pindexPrev, consensusParams); + return commitment; +} + +bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) +{ + const int nHeight = pindexPrev == NULL ? 0 : pindexPrev->nHeight + 1; // Check proof of work if (block.nBits != GetNextWorkRequired(pindexPrev, &block, consensusParams)) return state.DoS(100, false, REJECT_INVALID, "bad-diffbits", false, "incorrect proof of work"); @@ -3308,19 +3512,24 @@ bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& sta if (block.GetBlockTime() <= pindexPrev->GetMedianTimePast()) return state.Invalid(false, REJECT_INVALID, "time-too-old", "block's timestamp is too early"); + // Check timestamp + if (block.GetBlockTime() > nAdjustedTime + 2 * 60 * 60) + return state.Invalid(false, REJECT_INVALID, "time-too-new", "block timestamp too far in the future"); + // Reject outdated version blocks when 95% (75% on testnet) of the network has upgraded: - for (int32_t version = 2; version < 5; ++version) // check for version 2, 3 and 4 upgrades - if (block.nVersion < version && IsSuperMajority(version, pindexPrev, consensusParams.nMajorityRejectBlockOutdated, consensusParams)) - return state.Invalid(false, REJECT_OBSOLETE, strprintf("bad-version(0x%08x)", version - 1), - strprintf("rejected nVersion=0x%08x block", version - 1)); + // check for version 2, 3 and 4 upgrades + if((block.nVersion < 2 && nHeight >= consensusParams.BIP34Height) || + (block.nVersion < 3 && nHeight >= consensusParams.BIP66Height) || + (block.nVersion < 4 && nHeight >= consensusParams.BIP65Height)) + return state.Invalid(false, REJECT_OBSOLETE, strprintf("bad-version(0x%08x)", block.nVersion), + strprintf("rejected nVersion=0x%08x block", block.nVersion)); return true; } -bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIndex * const pindexPrev) +bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) { const int nHeight = pindexPrev == NULL ? 0 : pindexPrev->nHeight + 1; - const Consensus::Params& consensusParams = Params().GetConsensus(); // Start enforcing BIP113 (Median Time Past) using versionbits logic. int nLockTimeFlags = 0; @@ -3333,15 +3542,14 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIn : block.GetBlockTime(); // Check that all transactions are finalized - BOOST_FOREACH(const CTransaction& tx, block.vtx) { + for (const auto& tx : block.vtx) { if (!IsFinalTx(tx, nHeight, nLockTimeCutoff)) { return state.DoS(10, false, REJECT_INVALID, "bad-txns-nonfinal", false, "non-final transaction"); } } - // Enforce block.nVersion=2 rule that the coinbase starts with serialized block height - // if 750 of the last 1,000 blocks are version 2 or greater (51/100 if testnet): - if (block.nVersion >= 2 && IsSuperMajority(2, pindexPrev, consensusParams.nMajorityEnforceBlockUpgrade, consensusParams)) + // Enforce rule that the coinbase starts with serialized block height + if (nHeight >= consensusParams.BIP34Height) { CScript expect = CScript() << nHeight; if (block.vtx[0].vin[0].scriptSig.size() < expect.size() || @@ -3350,6 +3558,53 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIn } } + // Validation for witness commitments. + // * We compute the witness hash (which is the hash including witnesses) of all the block's transactions, except the + // coinbase (where 0x0000....0000 is used instead). + // * The coinbase scriptWitness is a stack of a single 32-byte vector, containing a witness nonce (unconstrained). + // * We build a merkle tree with all those witness hashes as leaves (similar to the hashMerkleRoot in the block header). + // * There must be at least one output whose scriptPubKey is a single 36-byte push, the first 4 bytes of which are + // {0xaa, 0x21, 0xa9, 0xed}, and the following 32 bytes are SHA256^2(witness root, witness nonce). In case there are + // multiple, the last one is used. + bool fHaveWitness = false; + if (VersionBitsState(pindexPrev, consensusParams, Consensus::DEPLOYMENT_SEGWIT, versionbitscache) == THRESHOLD_ACTIVE) { + int commitpos = GetWitnessCommitmentIndex(block); + if (commitpos != -1) { + bool malleated = false; + uint256 hashWitness = BlockWitnessMerkleRoot(block, &malleated); + // The malleation check is ignored; as the transaction tree itself + // already does not permit it, it is impossible to trigger in the + // witness tree. + if (block.vtx[0].wit.vtxinwit.size() != 1 || block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.size() != 1 || block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0].size() != 32) { + return state.DoS(100, false, REJECT_INVALID, "bad-witness-nonce-size", true, strprintf("%s : invalid witness nonce size", __func__)); + } + CHash256().Write(hashWitness.begin(), 32).Write(&block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0][0], 32).Finalize(hashWitness.begin()); + if (memcmp(hashWitness.begin(), &block.vtx[0].vout[commitpos].scriptPubKey[6], 32)) { + return state.DoS(100, false, REJECT_INVALID, "bad-witness-merkle-match", true, strprintf("%s : witness merkle commitment mismatch", __func__)); + } + fHaveWitness = true; + } + } + + // No witness data is allowed in blocks that don't commit to witness data, as this would otherwise leave room for spam + if (!fHaveWitness) { + for (size_t i = 0; i < block.vtx.size(); i++) { + if (!block.vtx[i].wit.IsNull()) { + return state.DoS(100, false, REJECT_INVALID, "unexpected-witness", true, strprintf("%s : unexpected witness data found", __func__)); + } + } + } + + // After the coinbase witness nonce and commitment are verified, + // we can check if the block weight passes (before we've checked the + // coinbase witness, it would be possible for the weight to be too + // large by filling up the coinbase witness, which doesn't change + // the block hash, so we couldn't mark the block as permanently + // failed). + if (GetBlockWeight(block) > MAX_BLOCK_WEIGHT) { + return state.DoS(100, false, REJECT_INVALID, "bad-blk-weight", false, strprintf("%s : weight limit failed", __func__)); + } + return true; } @@ -3372,7 +3627,7 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state return true; } - if (!CheckBlockHeader(block, state)) + if (!CheckBlockHeader(block, state, chainparams.GetConsensus())) return error("%s: Consensus::CheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); // Get prev block index @@ -3388,7 +3643,7 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state if (fCheckpointsEnabled && !CheckIndexAgainstCheckpoint(pindexPrev, state, chainparams, hash)) return error("%s: CheckIndexAgainstCheckpoint(): %s", __func__, state.GetRejectReason().c_str()); - if (!ContextualCheckBlockHeader(block, state, pindexPrev)) + if (!ContextualCheckBlockHeader(block, state, chainparams.GetConsensus(), pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); } if (pindex == NULL) @@ -3401,11 +3656,13 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state } /** Store block on disk. If dbp is non-NULL, the file is known to already reside on disk */ -static bool AcceptBlock(const CBlock& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, CDiskBlockPos* dbp) +static bool AcceptBlock(const CBlock& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) { + if (fNewBlock) *fNewBlock = false; AssertLockHeld(cs_main); - CBlockIndex *&pindex = *ppindex; + CBlockIndex *pindexDummy = NULL; + CBlockIndex *&pindex = ppindex ? *ppindex : pindexDummy; if (!AcceptBlockHeader(block, state, chainparams, &pindex)) return false; @@ -3430,8 +3687,10 @@ static bool AcceptBlock(const CBlock& block, CValidationState& state, const CCha if (!fHasMoreWork) return true; // Don't process less-work chains if (fTooFarAhead) return true; // Block height is too high } + if (fNewBlock) *fNewBlock = true; - if ((!CheckBlock(block, state)) || !ContextualCheckBlock(block, state, pindex->pprev)) { + if (!CheckBlock(block, state, chainparams.GetConsensus(), GetAdjustedTime()) || + !ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindex->pprev)) { if (state.IsInvalid() && !state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(pindex); @@ -3464,20 +3723,7 @@ static bool AcceptBlock(const CBlock& block, CValidationState& state, const CCha return true; } -static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned nRequired, const Consensus::Params& consensusParams) -{ - unsigned int nFound = 0; - for (int i = 0; i < consensusParams.nMajorityWindow && nFound < nRequired && pstart != NULL; i++) - { - if (pstart->nVersion >= minVersion) - ++nFound; - pstart = pstart->pprev; - } - return (nFound >= nRequired); -} - - -bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, CDiskBlockPos* dbp) +bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp) { { LOCK(cs_main); @@ -3486,15 +3732,19 @@ bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, c // Store to disk CBlockIndex *pindex = NULL; - bool ret = AcceptBlock(*pblock, state, chainparams, &pindex, fRequested, dbp); + bool fNewBlock = false; + bool ret = AcceptBlock(*pblock, state, chainparams, &pindex, fRequested, dbp, &fNewBlock); if (pindex && pfrom) { mapBlockSource[pindex->GetBlockHash()] = pfrom->GetId(); + if (fNewBlock) pfrom->nLastBlockTime = GetTime(); } CheckBlockIndex(chainparams.GetConsensus()); if (!ret) return error("%s: AcceptBlock FAILED", __func__); } + NotifyHeaderTip(); + if (!ActivateBestChain(state, chainparams, pblock)) return error("%s: ActivateBestChain failed", __func__); @@ -3514,11 +3764,11 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, indexDummy.nHeight = pindexPrev->nHeight + 1; // NOTE: CheckBlockHeader is called by CheckBlock - if (!ContextualCheckBlockHeader(block, state, pindexPrev)) + if (!ContextualCheckBlockHeader(block, state, chainparams.GetConsensus(), pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s", __func__, FormatStateMessage(state)); - if (!CheckBlock(block, state, fCheckPOW, fCheckMerkleRoot)) + if (!CheckBlock(block, state, chainparams.GetConsensus(), fCheckPOW, fCheckMerkleRoot)) return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); - if (!ContextualCheckBlock(block, state, pindexPrev)) + if (!ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlock: %s", __func__, FormatStateMessage(state)); if (!ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) return false; @@ -3837,18 +4087,31 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, CBlockIndex* pindexFailure = NULL; int nGoodTransactions = 0; CValidationState state; + int reportDone = 0; + LogPrintf("[0%%]..."); for (CBlockIndex* pindex = chainActive.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { boost::this_thread::interruption_point(); - uiInterface.ShowProgress(_("Verifying blocks..."), std::max(1, std::min(99, (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100))))); + int percentageDone = std::max(1, std::min(99, (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); + if (reportDone < percentageDone/10) { + // report every 10% step + LogPrintf("[%d%%]...", percentageDone); + reportDone = percentageDone/10; + } + uiInterface.ShowProgress(_("Verifying blocks..."), percentageDone); if (pindex->nHeight < chainActive.Height()-nCheckDepth) break; + if (fPruneMode && !(pindex->nStatus & BLOCK_HAVE_DATA)) { + // If pruning, only go back as far as we have data. + LogPrintf("VerifyDB(): block verification stopping at height %d (pruning, no data)\n", pindex->nHeight); + break; + } CBlock block; // check level 0: read from disk if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); // check level 1: verify block validity - if (nCheckLevel >= 1 && !CheckBlock(block, state)) + if (nCheckLevel >= 1 && !CheckBlock(block, state, chainparams.GetConsensus())) return error("%s: *** found bad block at %d, hash=%s (%s)\n", __func__, pindex->nHeight, pindex->GetBlockHash().ToString(), FormatStateMessage(state)); // check level 2: verify undo validity @@ -3893,11 +4156,96 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, } } + LogPrintf("[DONE].\n"); LogPrintf("No coin database inconsistencies in last %i blocks (%i transactions)\n", chainActive.Height() - pindexState->nHeight, nGoodTransactions); return true; } +bool RewindBlockIndex(const CChainParams& params) +{ + LOCK(cs_main); + + int nHeight = 1; + while (nHeight <= chainActive.Height()) { + if (IsWitnessEnabled(chainActive[nHeight - 1], params.GetConsensus()) && !(chainActive[nHeight]->nStatus & BLOCK_OPT_WITNESS)) { + break; + } + nHeight++; + } + + // nHeight is now the height of the first insufficiently-validated block, or tipheight + 1 + CValidationState state; + CBlockIndex* pindex = chainActive.Tip(); + while (chainActive.Height() >= nHeight) { + if (fPruneMode && !(chainActive.Tip()->nStatus & BLOCK_HAVE_DATA)) { + // If pruning, don't try rewinding past the HAVE_DATA point; + // since older blocks can't be served anyway, there's + // no need to walk further, and trying to DisconnectTip() + // will fail (and require a needless reindex/redownload + // of the blockchain). + break; + } + if (!DisconnectTip(state, params, true)) { + return error("RewindBlockIndex: unable to disconnect block at height %i", pindex->nHeight); + } + // Occasionally flush state to disk. + if (!FlushStateToDisk(state, FLUSH_STATE_PERIODIC)) + return false; + } + + // Reduce validity flag and have-data flags. + // We do this after actual disconnecting, otherwise we'll end up writing the lack of data + // to disk before writing the chainstate, resulting in a failure to continue if interrupted. + for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { + CBlockIndex* pindexIter = it->second; + + // Note: If we encounter an insufficiently validated block that + // is on chainActive, it must be because we are a pruning node, and + // this block or some successor doesn't HAVE_DATA, so we were unable to + // rewind all the way. Blocks remaining on chainActive at this point + // must not have their validity reduced. + if (IsWitnessEnabled(pindexIter->pprev, params.GetConsensus()) && !(pindexIter->nStatus & BLOCK_OPT_WITNESS) && !chainActive.Contains(pindexIter)) { + // Reduce validity + pindexIter->nStatus = std::min<unsigned int>(pindexIter->nStatus & BLOCK_VALID_MASK, BLOCK_VALID_TREE) | (pindexIter->nStatus & ~BLOCK_VALID_MASK); + // Remove have-data flags. + pindexIter->nStatus &= ~(BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO); + // Remove storage location. + pindexIter->nFile = 0; + pindexIter->nDataPos = 0; + pindexIter->nUndoPos = 0; + // Remove various other things + pindexIter->nTx = 0; + pindexIter->nChainTx = 0; + pindexIter->nSequenceId = 0; + // Make sure it gets written. + setDirtyBlockIndex.insert(pindexIter); + // Update indexes + setBlockIndexCandidates.erase(pindexIter); + std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> ret = mapBlocksUnlinked.equal_range(pindexIter->pprev); + while (ret.first != ret.second) { + if (ret.first->second == pindexIter) { + mapBlocksUnlinked.erase(ret.first++); + } else { + ++ret.first; + } + } + } else if (pindexIter->IsValid(BLOCK_VALID_TRANSACTIONS) && pindexIter->nChainTx) { + setBlockIndexCandidates.insert(pindexIter); + } + } + + PruneBlockIndexCandidates(); + + CheckBlockIndex(params.GetConsensus()); + + if (!FlushStateToDisk(state, FLUSH_STATE_ALWAYS)) { + return false; + } + + return true; +} + void UnloadBlockIndex() { LOCK(cs_main); @@ -3971,8 +4319,6 @@ bool InitBlockIndex(const CChainParams& chainparams) CBlockIndex *pindex = AddToBlockIndex(block); if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) return error("LoadBlockIndex(): genesis block not accepted"); - if (!ActivateBestChain(state, chainparams, &block)) - return error("LoadBlockIndex(): genesis block cannot be activated"); // Force a chainstate write so that when we VerifyDB in a moment, it doesn't check stale data return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); } catch (const std::runtime_error& e) { @@ -3992,7 +4338,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB int nLoaded = 0; try { // This takes over fileIn and calls fclose() on it in the CBufferedFile destructor - CBufferedFile blkdat(fileIn, 2*MAX_BLOCK_SIZE, MAX_BLOCK_SIZE+8, SER_DISK, CLIENT_VERSION); + CBufferedFile blkdat(fileIn, 2*MAX_BLOCK_SERIALIZED_SIZE, MAX_BLOCK_SERIALIZED_SIZE+8, SER_DISK, CLIENT_VERSION); uint64_t nRewind = blkdat.GetPos(); while (!blkdat.eof()) { boost::this_thread::interruption_point(); @@ -4011,7 +4357,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB continue; // read size blkdat >> nSize; - if (nSize < 80 || nSize > MAX_BLOCK_SIZE) + if (nSize < 80 || nSize > MAX_BLOCK_SERIALIZED_SIZE) continue; } catch (const std::exception&) { // no valid block header found; don't complain @@ -4040,15 +4386,26 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB // process in case the block isn't known yet if (mapBlockIndex.count(hash) == 0 || (mapBlockIndex[hash]->nStatus & BLOCK_HAVE_DATA) == 0) { + LOCK(cs_main); CValidationState state; - if (ProcessNewBlock(state, chainparams, NULL, &block, true, dbp)) + if (AcceptBlock(block, state, chainparams, NULL, true, dbp, NULL)) nLoaded++; if (state.IsError()) break; } else if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex[hash]->nHeight % 1000 == 0) { - LogPrintf("Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight); + LogPrint("reindex", "Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight); } + // Activate the genesis block so normal node progress can continue + if (hash == chainparams.GetConsensus().hashGenesisBlock) { + CValidationState state; + if (!ActivateBestChain(state, chainparams)) { + break; + } + } + + NotifyHeaderTip(); + // Recursively process earlier encountered successors of this block deque<uint256> queue; queue.push_back(hash); @@ -4060,10 +4417,11 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB std::multimap<uint256, CDiskBlockPos>::iterator it = range.first; if (ReadBlockFromDisk(block, it->second, chainparams.GetConsensus())) { - LogPrintf("%s: Processing out of order child %s of %s\n", __func__, block.GetHash().ToString(), + LogPrint("reindex", "%s: Processing out of order child %s of %s\n", __func__, block.GetHash().ToString(), head.ToString()); + LOCK(cs_main); CValidationState dummy; - if (ProcessNewBlock(dummy, chainparams, NULL, &block, true, &it->second)) + if (AcceptBlock(block, dummy, chainparams, NULL, true, &it->second, NULL)) { nLoaded++; queue.push_back(block.GetHash()); @@ -4071,6 +4429,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB } range.first++; mapBlocksUnknownParent.erase(it); + NotifyHeaderTip(); } } } catch (const std::exception& e) { @@ -4328,6 +4687,7 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) switch (inv.type) { case MSG_TX: + case MSG_WITNESS_TX: { assert(recentRejects); if (chainActive.Tip()->GetBlockHash() != hashRecentRejectsChainTip) @@ -4348,6 +4708,7 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) pcoinsTip->HaveCoinsInCache(inv.hash); } case MSG_BLOCK: + case MSG_WITNESS_BLOCK: return mapBlockIndex.count(inv.hash); } // Don't know what it is, just say we already got one @@ -4372,7 +4733,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam boost::this_thread::interruption_point(); it++; - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK) + if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK || inv.type == MSG_WITNESS_BLOCK) { bool send = false; BlockMap::iterator mi = mapBlockIndex.find(inv.hash); @@ -4413,8 +4774,10 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam if (!ReadBlockFromDisk(block, (*mi).second, consensusParams)) assert(!"cannot load block from disk"); if (inv.type == MSG_BLOCK) + pfrom->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block); + else if (inv.type == MSG_WITNESS_BLOCK) pfrom->PushMessage(NetMsgType::BLOCK, block); - else // MSG_FILTERED_BLOCK) + else if (inv.type == MSG_FILTERED_BLOCK) { LOCK(pfrom->cs_filter); if (pfrom->pfilter) @@ -4429,11 +4792,23 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // however we MUST always provide at least what the remote peer needs typedef std::pair<unsigned int, uint256> PairType; BOOST_FOREACH(PairType& pair, merkleBlock.vMatchedTxn) - pfrom->PushMessage(NetMsgType::TX, block.vtx[pair.first]); + pfrom->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, block.vtx[pair.first]); } // else // no response } + else if (inv.type == MSG_CMPCT_BLOCK) + { + // If a peer is asking for old blocks, we're almost guaranteed + // they wont have a useful mempool to match against a compact block, + // and we dont feel like constructing the object for them, so + // instead we respond with the full, non-compact block. + if (mi->second->nHeight >= chainActive.Height() - 10) { + CBlockHeaderAndShortTxIDs cmpctblock(block); + pfrom->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::CMPCTBLOCK, cmpctblock); + } else + pfrom->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block); + } // Trigger the peer node to send a getblocks request for the next batch of inventory if (inv.hash == pfrom->hashContinue) @@ -4448,26 +4823,24 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } } } - else if (inv.IsKnownType()) + else if (inv.type == MSG_TX || inv.type == MSG_WITNESS_TX) { // Send stream from relay memory - bool pushed = false; - { - LOCK(cs_mapRelay); - map<uint256, CTransaction>::iterator mi = mapRelay.find(inv.hash); - if (mi != mapRelay.end()) { - pfrom->PushMessage(inv.GetCommand(), (*mi).second); - pushed = true; - } - } - if (!pushed && inv.type == MSG_TX) { - CTransaction tx; - if (mempool.lookup(inv.hash, tx)) { - pfrom->PushMessage(NetMsgType::TX, tx); - pushed = true; + bool push = false; + auto mi = mapRelay.find(inv.hash); + if (mi != mapRelay.end()) { + pfrom->PushMessageWithFlag(inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0, NetMsgType::TX, *mi->second); + push = true; + } else if (pfrom->timeLastMempoolReq) { + auto txinfo = mempool.info(inv.hash); + // To protect privacy, do not answer getdata using the mempool when + // that TX couldn't have been INVed in reply to a MEMPOOL request. + if (txinfo.tx && txinfo.nTime <= pfrom->timeLastMempoolReq) { + pfrom->PushMessageWithFlag(inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0, NetMsgType::TX, *txinfo.tx); + push = true; } } - if (!pushed) { + if (!push) { vNotFound.push_back(inv); } } @@ -4475,7 +4848,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // Track requests for our stuff. GetMainSignals().Inventory(inv.hash); - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK) + if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK || inv.type == MSG_WITNESS_BLOCK) break; } } @@ -4494,9 +4867,16 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } } +uint32_t GetFetchFlags(CNode* pfrom, CBlockIndex* pprev, const Consensus::Params& chainparams) { + uint32_t nFetchFlags = 0; + if (IsWitnessEnabled(pprev, chainparams) && State(pfrom->GetId())->fHaveWitness) { + nFetchFlags |= MSG_WITNESS_FLAG; + } + return nFetchFlags; +} + bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams) { - RandAddSeedPerfmon(); LogPrint("net", "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); if (mapArgs.count("-dropmessagestest") && GetRand(atoi(mapArgs["-dropmessagestest"])) == 0) { @@ -4511,6 +4891,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, strCommand == NetMsgType::FILTERCLEAR)) { if (pfrom->nVersion >= NO_BLOOM_VERSION) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); return false; } else { @@ -4526,6 +4907,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (pfrom->nVersion != 0) { pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, string("Duplicate version message")); + LOCK(cs_main); Misbehaving(pfrom->GetId(), 1); return false; } @@ -4534,7 +4916,22 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, CAddress addrMe; CAddress addrFrom; uint64_t nNonce = 1; - vRecv >> pfrom->nVersion >> pfrom->nServices >> nTime >> addrMe; + uint64_t nServiceInt; + vRecv >> pfrom->nVersion >> nServiceInt >> nTime >> addrMe; + pfrom->nServices = ServiceFlags(nServiceInt); + if (!pfrom->fInbound) + { + addrman.SetServices(pfrom->addr, pfrom->nServices); + } + if (pfrom->nServicesExpected & ~pfrom->nServices) + { + LogPrint("net", "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->id, pfrom->nServices, pfrom->nServicesExpected); + pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, + strprintf("Expected to offer services %08x", pfrom->nServicesExpected)); + pfrom->fDisconnect = true; + return false; + } + if (pfrom->nVersion < MIN_PEER_PROTO_VERSION) { // disconnect from peers older than this proto version @@ -4584,8 +4981,17 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, pfrom->fClient = !(pfrom->nServices & NODE_NETWORK); + if((pfrom->nServices & NODE_WITNESS)) + { + LOCK(cs_main); + State(pfrom->GetId())->fHaveWitness = true; + } + // Potentially mark this peer as a preferred download peer. + { + LOCK(cs_main); UpdatePreferredDownload(pfrom, State(pfrom->GetId())); + } // Change version pfrom->PushMessage(NetMsgType::VERACK); @@ -4643,6 +5049,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, else if (pfrom->nVersion == 0) { // Must have a version message before anything else + LOCK(cs_main); Misbehaving(pfrom->GetId(), 1); return false; } @@ -4665,6 +5072,16 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // nodes) pfrom->PushMessage(NetMsgType::SENDHEADERS); } + if (pfrom->nVersion >= SHORT_IDS_BLOCKS_VERSION) { + // Tell our peer we are willing to provide version-1 cmpctblocks + // However, we do not request new block announcements using + // cmpctblock messages. + // We send this to non-NODE NETWORK peers as well, because + // they may wish to request compact blocks from us + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 1; + pfrom->PushMessage(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + } } @@ -4678,6 +5095,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return true; if (vAddr.size() > 1000) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 20); return error("message addr size() = %u", vAddr.size()); } @@ -4690,6 +5108,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, { boost::this_thread::interruption_point(); + if ((addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) + continue; + if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) addr.nTime = nNow - 5 * 24 * 60 * 60; pfrom->AddAddressKnown(addr); @@ -4701,25 +5122,20 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, LOCK(cs_vNodes); // Use deterministic randomness to send to the same nodes for 24 hours // at a time so the addrKnowns of the chosen nodes prevent repeats - static uint256 hashSalt; - if (hashSalt.IsNull()) - hashSalt = GetRandHash(); + static const uint64_t salt0 = GetRand(std::numeric_limits<uint64_t>::max()); + static const uint64_t salt1 = GetRand(std::numeric_limits<uint64_t>::max()); uint64_t hashAddr = addr.GetHash(); - uint256 hashRand = ArithToUint256(UintToArith256(hashSalt) ^ (hashAddr<<32) ^ ((GetTime()+hashAddr)/(24*60*60))); - hashRand = Hash(BEGIN(hashRand), END(hashRand)); - multimap<uint256, CNode*> mapMix; + multimap<uint64_t, CNode*> mapMix; + const CSipHasher hasher = CSipHasher(salt0, salt1).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); BOOST_FOREACH(CNode* pnode, vNodes) { if (pnode->nVersion < CADDR_TIME_VERSION) continue; - unsigned int nPointer; - memcpy(&nPointer, &pnode, sizeof(nPointer)); - uint256 hashKey = ArithToUint256(UintToArith256(hashRand) ^ nPointer); - hashKey = Hash(BEGIN(hashKey), END(hashKey)); + uint64_t hashKey = CSipHasher(hasher).Write(pnode->id).Finalize(); mapMix.insert(make_pair(hashKey, pnode)); } int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s) - for (multimap<uint256, CNode*>::iterator mi = mapMix.begin(); mi != mapMix.end() && nRelayNodes-- > 0; ++mi) + for (multimap<uint64_t, CNode*>::iterator mi = mapMix.begin(); mi != mapMix.end() && nRelayNodes-- > 0; ++mi) ((*mi).second)->PushAddress(addr); } } @@ -4740,6 +5156,18 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, State(pfrom->GetId())->fPreferHeaders = true; } + else if (strCommand == NetMsgType::SENDCMPCT) + { + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 1; + vRecv >> fAnnounceUsingCMPCTBLOCK >> nCMPCTBLOCKVersion; + if (nCMPCTBLOCKVersion == 1) { + LOCK(cs_main); + State(pfrom->GetId())->fProvidesHeaderAndIDs = true; + State(pfrom->GetId())->fPreferHeaderAndIDs = fAnnounceUsingCMPCTBLOCK; + } + } + else if (strCommand == NetMsgType::INV) { @@ -4747,11 +5175,12 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 20); return error("message inv size() = %u", vInv.size()); } - bool fBlocksOnly = GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY); + bool fBlocksOnly = !fRelayTxes; // Allow whitelisted peers to send data other than blocks in blocks only mode if whitelistrelay is true if (pfrom->fWhitelisted && GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY)) @@ -4759,18 +5188,23 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, LOCK(cs_main); + uint32_t nFetchFlags = GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()); + std::vector<CInv> vToFetch; for (unsigned int nInv = 0; nInv < vInv.size(); nInv++) { - const CInv &inv = vInv[nInv]; + CInv &inv = vInv[nInv]; boost::this_thread::interruption_point(); - pfrom->AddInventoryKnown(inv); bool fAlreadyHave = AlreadyHave(inv); LogPrint("net", "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); + if (inv.type == MSG_TX) { + inv.type |= nFetchFlags; + } + if (inv.type == MSG_BLOCK) { UpdateBlockAvailability(pfrom->GetId(), inv.hash); if (!fAlreadyHave && !fImporting && !fReindex && !mapBlocksInFlight.count(inv.hash)) { @@ -4785,8 +5219,13 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash); CNodeState *nodestate = State(pfrom->GetId()); if (CanDirectFetch(chainparams.GetConsensus()) && - nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - vToFetch.push_back(inv); + nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER && + (!IsWitnessEnabled(chainActive.Tip(), chainparams.GetConsensus()) || State(pfrom->GetId())->fHaveWitness)) { + inv.type |= nFetchFlags; + if (nodestate->fProvidesHeaderAndIDs && !(nLocalServices & NODE_WITNESS)) + vToFetch.push_back(CInv(MSG_CMPCT_BLOCK, inv.hash)); + else + vToFetch.push_back(inv); // Mark block as in flight already, even though the actual "getdata" message only goes out // later (within the same cs_main lock, though). MarkBlockAsInFlight(pfrom->GetId(), inv.hash, chainparams.GetConsensus()); @@ -4796,6 +5235,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } else { + pfrom->AddInventoryKnown(inv); if (fBlocksOnly) LogPrint("net", "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->id); else if (!fAlreadyHave && !fImporting && !fReindex && !IsInitialBlockDownload()) @@ -4822,6 +5262,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 20); return error("message getdata size() = %u", vInv.size()); } @@ -4881,6 +5322,38 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } + else if (strCommand == NetMsgType::GETBLOCKTXN) + { + BlockTransactionsRequest req; + vRecv >> req; + + BlockMap::iterator it = mapBlockIndex.find(req.blockhash); + if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { + LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); + return true; + } + + if (it->second->nHeight < chainActive.Height() - 15) { + LogPrint("net", "Peer %d sent us a getblocktxn for a block > 15 deep", pfrom->id); + return true; + } + + CBlock block; + assert(ReadBlockFromDisk(block, it->second, chainparams.GetConsensus())); + + BlockTransactions resp(req); + for (size_t i = 0; i < req.indexes.size(); i++) { + if (req.indexes[i] >= block.vtx.size()) { + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->id); + return true; + } + resp.txn[i] = block.vtx[req.indexes[i]]; + } + pfrom->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCKTXN, resp); + } + + else if (strCommand == NetMsgType::GETHEADERS) { CBlockLocator locator; @@ -4934,13 +5407,13 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, { // Stop processing the transaction early if // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off - if (GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY) && (!pfrom->fWhitelisted || !GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY))) + if (!fRelayTxes && (!pfrom->fWhitelisted || !GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY))) { LogPrint("net", "transaction sent in violation of protocol peer=%d\n", pfrom->id); return true; } - vector<uint256> vWorkQueue; + deque<COutPoint> vWorkQueue; vector<uint256> vEraseQueue; CTransaction tx; vRecv >> tx; @@ -4956,11 +5429,14 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, pfrom->setAskFor.erase(inv.hash); mapAlreadyAskedFor.erase(inv.hash); - CFeeRate txFeeRate = CFeeRate(0); - if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, tx, true, &fMissingInputs, &txFeeRate)) { + if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, tx, true, &fMissingInputs)) { mempool.check(pcoinsTip); - RelayTransaction(tx, txFeeRate); - vWorkQueue.push_back(inv.hash); + RelayTransaction(tx); + for (unsigned int i = 0; i < tx.vout.size(); i++) { + vWorkQueue.emplace_back(inv.hash, i); + } + + pfrom->nLastTXTime = GetTime(); LogPrint("mempool", "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", pfrom->id, @@ -4969,18 +5445,18 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Recursively process any orphan transactions that depended on this one set<NodeId> setMisbehaving; - for (unsigned int i = 0; i < vWorkQueue.size(); i++) - { - map<uint256, set<uint256> >::iterator itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue[i]); + while (!vWorkQueue.empty()) { + auto itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue.front()); + vWorkQueue.pop_front(); if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; - for (set<uint256>::iterator mi = itByPrev->second.begin(); + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { - const uint256& orphanHash = *mi; - const CTransaction& orphanTx = mapOrphanTransactions[orphanHash].tx; - NodeId fromPeer = mapOrphanTransactions[orphanHash].fromPeer; + const CTransaction& orphanTx = (*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + NodeId fromPeer = (*mi)->second.fromPeer; bool fMissingInputs2 = false; // Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan // resolution (that is, feeding people an invalid transaction based on LegitTxX in order to get @@ -4990,17 +5466,18 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (setMisbehaving.count(fromPeer)) continue; - CFeeRate orphanFeeRate = CFeeRate(0); - if (AcceptToMemoryPool(mempool, stateDummy, orphanTx, true, &fMissingInputs2, &orphanFeeRate)) { + if (AcceptToMemoryPool(mempool, stateDummy, orphanTx, true, &fMissingInputs2)) { LogPrint("mempool", " accepted orphan tx %s\n", orphanHash.ToString()); - RelayTransaction(orphanTx, orphanFeeRate); - vWorkQueue.push_back(orphanHash); + RelayTransaction(orphanTx); + for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { + vWorkQueue.emplace_back(orphanHash, i); + } vEraseQueue.push_back(orphanHash); } else if (!fMissingInputs2) { int nDos = 0; - if (stateDummy.IsInvalid(nDos) && nDos > 0) + if (stateDummy.IsInvalid(nDos) && nDos > 0 && (!state.CorruptionPossible() || State(fromPeer)->fHaveWitness)) { // Punish peer that gave us an invalid orphan tx Misbehaving(fromPeer, nDos); @@ -5011,8 +5488,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Probably non-standard or insufficient fee/priority LogPrint("mempool", " removed orphan tx %s\n", orphanHash.ToString()); vEraseQueue.push_back(orphanHash); - assert(recentRejects); - recentRejects->insert(orphanHash); + if (!stateDummy.CorruptionPossible()) { + assert(recentRejects); + recentRejects->insert(orphanHash); + } } mempool.check(pcoinsTip); } @@ -5023,16 +5502,34 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } else if (fMissingInputs) { - AddOrphanTx(tx, pfrom->GetId()); + bool fRejectedParents = false; // It may be the case that the orphans parents have all been rejected + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + if (recentRejects->contains(txin.prevout.hash)) { + fRejectedParents = true; + break; + } + } + if (!fRejectedParents) { + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + CInv inv(MSG_TX, txin.prevout.hash); + pfrom->AddInventoryKnown(inv); + if (!AlreadyHave(inv)) pfrom->AskFor(inv); + } + AddOrphanTx(tx, pfrom->GetId()); - // DoS prevention: do not allow mapOrphanTransactions to grow unbounded - unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); - unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx); - if (nEvicted > 0) - LogPrint("mempool", "mapOrphan overflow, removed %u tx\n", nEvicted); + // DoS prevention: do not allow mapOrphanTransactions to grow unbounded + unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); + unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx); + if (nEvicted > 0) + LogPrint("mempool", "mapOrphan overflow, removed %u tx\n", nEvicted); + } else { + LogPrint("mempool", "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString()); + } } else { - assert(recentRejects); - recentRejects->insert(tx.GetHash()); + if (!state.CorruptionPossible()) { + assert(recentRejects); + recentRejects->insert(tx.GetHash()); + } if (pfrom->fWhitelisted && GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY)) { // Always relay transactions received from whitelisted peers, even @@ -5046,7 +5543,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, int nDoS = 0; if (!state.IsInvalid(nDoS) || nDoS == 0) { LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->id); - RelayTransaction(tx, txFeeRate); + RelayTransaction(tx); } else { LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->id, FormatStateMessage(state)); } @@ -5061,13 +5558,184 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash); - if (nDoS > 0) + if (nDoS > 0 && (!state.CorruptionPossible() || State(pfrom->id)->fHaveWitness)) { + // When a non-witness-supporting peer gives us a transaction that would + // be accepted if witness validation was off, we can't blame them for it. Misbehaving(pfrom->GetId(), nDoS); + } } FlushStateToDisk(state, FLUSH_STATE_PERIODIC); } + else if (strCommand == NetMsgType::CMPCTBLOCK && !fImporting && !fReindex) // Ignore blocks received while importing + { + CBlockHeaderAndShortTxIDs cmpctblock; + vRecv >> cmpctblock; + + LOCK(cs_main); + + if (mapBlockIndex.find(cmpctblock.header.hashPrevBlock) == mapBlockIndex.end()) { + // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers + if (!IsInitialBlockDownload()) + pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256()); + return true; + } + + CBlockIndex *pindex = NULL; + CValidationState state; + if (!AcceptBlockHeader(cmpctblock.header, state, chainparams, &pindex)) { + int nDoS; + if (state.IsInvalid(nDoS)) { + if (nDoS > 0) + Misbehaving(pfrom->GetId(), nDoS); + LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->id); + return true; + } + } + + // If AcceptBlockHeader returned true, it set pindex + assert(pindex); + UpdateBlockAvailability(pfrom->GetId(), pindex->GetBlockHash()); + + std::map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator blockInFlightIt = mapBlocksInFlight.find(pindex->GetBlockHash()); + bool fAlreadyInFlight = blockInFlightIt != mapBlocksInFlight.end(); + + if (pindex->nStatus & BLOCK_HAVE_DATA) // Nothing to do here + return true; + + if (pindex->nChainWork <= chainActive.Tip()->nChainWork || // We know something better + pindex->nTx != 0) { // We had this block at some point, but pruned it + if (fAlreadyInFlight) { + // We requested this block for some reason, but our mempool will probably be useless + // so we just grab the block via normal getdata + std::vector<CInv> vInv(1); + vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); + pfrom->PushMessage(NetMsgType::GETDATA, vInv); + } + return true; + } + + // If we're not close to tip yet, give up and let parallel block fetch work its magic + if (!fAlreadyInFlight && !CanDirectFetch(chainparams.GetConsensus())) + return true; + + CNodeState *nodestate = State(pfrom->GetId()); + + // We want to be a bit conservative just to be extra careful about DoS + // possibilities in compact block processing... + if (pindex->nHeight <= chainActive.Height() + 2) { + if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || + (fAlreadyInFlight && blockInFlightIt->second.first == pfrom->GetId())) { + list<QueuedBlock>::iterator *queuedBlockIt = NULL; + if (!MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex, &queuedBlockIt)) { + if (!(*queuedBlockIt)->partialBlock) + (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&mempool)); + else { + // The block was already in flight using compact blocks from the same peer + LogPrint("net", "Peer sent us compact block we were already syncing!\n"); + return true; + } + } + + PartiallyDownloadedBlock& partialBlock = *(*queuedBlockIt)->partialBlock; + ReadStatus status = partialBlock.InitData(cmpctblock); + if (status == READ_STATUS_INVALID) { + MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us invalid compact block\n", pfrom->id); + return true; + } else if (status == READ_STATUS_FAILED) { + // Duplicate txindexes, the block is now in-flight, so just request it + std::vector<CInv> vInv(1); + vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); + pfrom->PushMessage(NetMsgType::GETDATA, vInv); + return true; + } + + BlockTransactionsRequest req; + for (size_t i = 0; i < cmpctblock.BlockTxCount(); i++) { + if (!partialBlock.IsTxAvailable(i)) + req.indexes.push_back(i); + } + if (req.indexes.empty()) { + // Dirty hack to jump to BLOCKTXN code (TODO: move message handling into their own functions) + BlockTransactions txn; + txn.blockhash = cmpctblock.header.GetHash(); + CDataStream blockTxnMsg(SER_NETWORK, PROTOCOL_VERSION); + blockTxnMsg << txn; + return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams); + } else { + req.blockhash = pindex->GetBlockHash(); + pfrom->PushMessage(NetMsgType::GETBLOCKTXN, req); + } + } + } else { + if (fAlreadyInFlight) { + // We requested this block, but its far into the future, so our + // mempool will probably be useless - request the block normally + std::vector<CInv> vInv(1); + vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); + pfrom->PushMessage(NetMsgType::GETDATA, vInv); + return true; + } else { + // If this was an announce-cmpctblock, we want the same treatment as a header message + // Dirty hack to process as if it were just a headers message (TODO: move message handling into their own functions) + std::vector<CBlock> headers; + headers.push_back(cmpctblock.header); + CDataStream vHeadersMsg(SER_NETWORK, PROTOCOL_VERSION); + vHeadersMsg << headers; + return ProcessMessage(pfrom, NetMsgType::HEADERS, vHeadersMsg, nTimeReceived, chainparams); + } + } + + CheckBlockIndex(chainparams.GetConsensus()); + } + + else if (strCommand == NetMsgType::BLOCKTXN && !fImporting && !fReindex) // Ignore blocks received while importing + { + BlockTransactions resp; + vRecv >> resp; + + LOCK(cs_main); + + map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); + if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || + it->second.first != pfrom->GetId()) { + LogPrint("net", "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); + return true; + } + + PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; + CBlock block; + ReadStatus status = partialBlock.FillBlock(block, resp.txn); + if (status == READ_STATUS_INVALID) { + MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); + return true; + } else if (status == READ_STATUS_FAILED) { + // Might have collided, fall back to getdata now :( + std::vector<CInv> invs; + invs.push_back(CInv(MSG_BLOCK, resp.blockhash)); + pfrom->PushMessage(NetMsgType::GETDATA, invs); + } else { + CValidationState state; + ProcessNewBlock(state, chainparams, pfrom, &block, false, NULL); + int nDoS; + if (state.IsInvalid(nDoS)) { + assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes + pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), + state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), block.GetHash()); + if (nDoS > 0) { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), nDoS); + } + } + } + } + + else if (strCommand == NetMsgType::HEADERS && !fImporting && !fReindex) // Ignore headers received while importing { std::vector<CBlockHeader> headers; @@ -5075,6 +5743,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. unsigned int nCount = ReadCompactSize(vRecv); if (nCount > MAX_HEADERS_RESULTS) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 20); return error("headers message size = %u", nCount); } @@ -5084,6 +5753,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } + { LOCK(cs_main); if (nCount == 0) { @@ -5091,6 +5761,35 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return true; } + CNodeState *nodestate = State(pfrom->GetId()); + + // If this looks like it could be a block announcement (nCount < + // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that + // don't connect: + // - Send a getheaders message in response to try to connect the chain. + // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that + // don't connect before giving DoS points + // - Once a headers message is received that is valid and does connect, + // nUnconnectingHeaders gets reset back to 0. + if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { + nodestate->nUnconnectingHeaders++; + pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256()); + LogPrint("net", "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + headers[0].GetHash().ToString(), + headers[0].hashPrevBlock.ToString(), + pindexBestHeader->nHeight, + pfrom->id, nodestate->nUnconnectingHeaders); + // Set hashLastUnknownBlock for this peer, so that if we + // eventually get the headers - even from a different peer - + // we can use this peer to download. + UpdateBlockAvailability(pfrom->GetId(), headers.back().GetHash()); + + if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { + Misbehaving(pfrom->GetId(), 20); + } + return true; + } + CBlockIndex *pindexLast = NULL; BOOST_FOREACH(const CBlockHeader& header, headers) { CValidationState state; @@ -5108,10 +5807,15 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } } - if (pindexLast) - UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); + if (nodestate->nUnconnectingHeaders > 0) { + LogPrint("net", "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->id, nodestate->nUnconnectingHeaders); + } + nodestate->nUnconnectingHeaders = 0; + + assert(pindexLast); + UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); - if (nCount == MAX_HEADERS_RESULTS && pindexLast) { + if (nCount == MAX_HEADERS_RESULTS) { // Headers message had its maximum size; the peer may have more headers. // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue // from there instead. @@ -5120,7 +5824,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); - CNodeState *nodestate = State(pfrom->GetId()); // If this set of headers is valid and ends in a block with at least as // much work as our tip, download as much as possible. if (fCanDirectFetch && pindexLast->IsValid(BLOCK_VALID_TREE) && chainActive.Tip()->nChainWork <= pindexLast->nChainWork) { @@ -5129,7 +5832,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. while (pindexWalk && !chainActive.Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && - !mapBlocksInFlight.count(pindexWalk->GetBlockHash())) { + !mapBlocksInFlight.count(pindexWalk->GetBlockHash()) && + (!IsWitnessEnabled(pindexWalk->pprev, chainparams.GetConsensus()) || State(pfrom->GetId())->fHaveWitness)) { // We don't have this block, and it's not yet in flight. vToFetch.push_back(pindexWalk); } @@ -5151,7 +5855,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Can't download any more from this peer break; } - vGetData.push_back(CInv(MSG_BLOCK, pindex->GetBlockHash())); + uint32_t nFetchFlags = GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()); + vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex); LogPrint("net", "Requesting block %s from peer=%d\n", pindex->GetBlockHash().ToString(), pfrom->id); @@ -5161,12 +5866,22 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); } if (vGetData.size() > 0) { + if (nodestate->fProvidesHeaderAndIDs && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN) && !(nLocalServices & NODE_WITNESS)) { + // We seem to be rather well-synced, so it appears pfrom was the first to provide us + // with this block! Let's get them to announce using compact blocks in the future. + MaybeSetPeerAsAnnouncingHeaderAndIDs(nodestate, pfrom); + // In any case, we want to download using a compact block, not a regular one + vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); + } pfrom->PushMessage(NetMsgType::GETDATA, vGetData); } } } CheckBlockIndex(chainparams.GetConsensus()); + } + + NotifyHeaderTip(); } else if (strCommand == NetMsgType::BLOCK && !fImporting && !fReindex) // Ignore blocks received while importing @@ -5174,10 +5889,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, CBlock block; vRecv >> block; - CInv inv(MSG_BLOCK, block.GetHash()); - LogPrint("net", "received block %s peer=%d\n", inv.hash.ToString(), pfrom->id); - - pfrom->AddInventoryKnown(inv); + LogPrint("net", "received block %s peer=%d\n", block.GetHash().ToString(), pfrom->id); CValidationState state; // Process all blocks from whitelisted peers, even if not requested, @@ -5190,7 +5902,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (state.IsInvalid(nDoS)) { assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), - state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash); + state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), block.GetHash()); if (nDoS > 0) { LOCK(cs_main); Misbehaving(pfrom->GetId(), nDoS); @@ -5229,6 +5941,13 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, else if (strCommand == NetMsgType::MEMPOOL) { + if (!(nLocalServices & NODE_BLOOM) && !pfrom->fWhitelisted) + { + LogPrint("net", "mempool request with bloom filters disabled, disconnect peer=%d\n", pfrom->GetId()); + pfrom->fDisconnect = true; + return true; + } + if (CNode::OutboundTargetReached(false) && !pfrom->fWhitelisted) { LogPrint("net", "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom->GetId()); @@ -5328,8 +6047,11 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, LOCK(pfrom->cs_filter); if (!filter.IsWithinSizeConstraints()) + { // There is no excuse for sending a too-large filter + LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); + } else { delete pfrom->pfilter; @@ -5349,13 +6071,17 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // and thus, the maximum size any matched object can have) in a filteradd message if (vData.size() > MAX_SCRIPT_ELEMENT_SIZE) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); } else { LOCK(pfrom->cs_filter); if (pfrom->pfilter) pfrom->pfilter->insert(vData); else + { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); + } } } @@ -5405,6 +6131,11 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } } + else if (strCommand == NetMsgType::NOTFOUND) { + // We do not care about the NOTFOUND message, but logging an Unknown Command + // message would be undesirable as we transmit it ourselves. + } + else { // Ignore unknown commands for extensibility LogPrint("net", "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->id); @@ -5509,6 +6240,11 @@ bool ProcessMessages(CNode* pfrom) // Allow exceptions from over-long size LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } + else if (strstr(e.what(), "non-canonical ReadCompactSize()")) + { + // Allow exceptions from non-canonical encoding + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); + } else { PrintExceptionContinue(&e, "ProcessMessages()"); @@ -5625,6 +6361,9 @@ bool SendMessages(CNode* pto) pto->vAddrToSend.clear(); if (!vAddr.empty()) pto->PushMessage(NetMsgType::ADDR, vAddr); + // we only send the big addr message once + if (pto->vAddrToSend.capacity() > 40) + pto->vAddrToSend.shrink_to_fit(); } CNodeState &state = *State(pto->GetId()); @@ -5692,7 +6431,9 @@ bool SendMessages(CNode* pto) // add all to the inv queue. LOCK(pto->cs_inventory); vector<CBlock> vHeaders; - bool fRevertToInv = (!state.fPreferHeaders || pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); + bool fRevertToInv = ((!state.fPreferHeaders && + (!state.fPreferHeaderAndIDs || pto->vBlockHashesToAnnounce.size() > 1)) || + pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); CBlockIndex *pBestIndex = NULL; // last header queued for delivery ProcessBlockAvailability(pto->id); // ensure pindexBestKnownBlock is up-to-date @@ -5744,6 +6485,33 @@ bool SendMessages(CNode* pto) } } } + if (!fRevertToInv && !vHeaders.empty()) { + if (vHeaders.size() == 1 && state.fPreferHeaderAndIDs) { + // We only send up to 1 block as header-and-ids, as otherwise + // probably means we're doing an initial-ish-sync or they're slow + LogPrint("net", "%s sending header-and-ids %s to peer %d\n", __func__, + vHeaders.front().GetHash().ToString(), pto->id); + //TODO: Shouldn't need to reload block from disk, but requires refactor + CBlock block; + assert(ReadBlockFromDisk(block, pBestIndex, consensusParams)); + CBlockHeaderAndShortTxIDs cmpctblock(block); + pto->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::CMPCTBLOCK, cmpctblock); + state.pindexBestHeaderSent = pBestIndex; + } else if (state.fPreferHeaders) { + if (vHeaders.size() > 1) { + LogPrint("net", "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, + vHeaders.size(), + vHeaders.front().GetHash().ToString(), + vHeaders.back().GetHash().ToString(), pto->id); + } else { + LogPrint("net", "%s: sending header %s to peer=%d\n", __func__, + vHeaders.front().GetHash().ToString(), pto->id); + } + pto->PushMessage(NetMsgType::HEADERS, vHeaders); + state.pindexBestHeaderSent = pBestIndex; + } else + fRevertToInv = true; + } if (fRevertToInv) { // If falling back to using an inv, just try to inv the tip. // The last entry in vBlockHashesToAnnounce was our tip at some point @@ -5762,27 +6530,13 @@ bool SendMessages(CNode* pto) hashToAnnounce.ToString(), chainActive.Tip()->GetBlockHash().ToString()); } - // If the peer announced this block to us, don't inv it back. - // (Since block announcements may not be via inv's, we can't solely rely on - // setInventoryKnown to track this.) + // If the peer's chain has this block, don't inv it back. if (!PeerHasHeader(&state, pindex)) { pto->PushInventory(CInv(MSG_BLOCK, hashToAnnounce)); LogPrint("net", "%s: sending inv peer=%d hash=%s\n", __func__, pto->id, hashToAnnounce.ToString()); } } - } else if (!vHeaders.empty()) { - if (vHeaders.size() > 1) { - LogPrint("net", "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, - vHeaders.size(), - vHeaders.front().GetHash().ToString(), - vHeaders.back().GetHash().ToString(), pto->id); - } else { - LogPrint("net", "%s: sending header %s to peer=%d\n", __func__, - vHeaders.front().GetHash().ToString(), pto->id); - } - pto->PushMessage(NetMsgType::HEADERS, vHeaders); - state.pindexBestHeaderSent = pBestIndex; } pto->vBlockHashesToAnnounce.clear(); } @@ -5821,8 +6575,7 @@ bool SendMessages(CNode* pto) // Respond to BIP35 mempool requests if (fSendTrickle && pto->fSendMempool) { - std::vector<uint256> vtxid; - mempool.queryHashes(vtxid); + auto vtxinfo = mempool.infoAll(); pto->fSendMempool = false; CAmount filterrate = 0; { @@ -5832,20 +6585,16 @@ bool SendMessages(CNode* pto) LOCK(pto->cs_filter); - BOOST_FOREACH(const uint256& hash, vtxid) { + for (const auto& txinfo : vtxinfo) { + const uint256& hash = txinfo.tx->GetHash(); CInv inv(MSG_TX, hash); pto->setInventoryTxToSend.erase(hash); if (filterrate) { - CFeeRate feeRate; - mempool.lookupFeeRate(hash, feeRate); - if (feeRate.GetFeePerK() < filterrate) + if (txinfo.feeRate.GetFeePerK() < filterrate) continue; } if (pto->pfilter) { - CTransaction tx; - bool fInMemPool = mempool.lookup(hash, tx); - if (!fInMemPool) continue; // another thread removed since queryHashes, maybe... - if (!pto->pfilter->IsRelevantAndUpdate(tx)) continue; + if (!pto->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; } pto->filterInventoryKnown.insert(hash); vInv.push_back(inv); @@ -5854,6 +6603,7 @@ bool SendMessages(CNode* pto) vInv.clear(); } } + pto->timeLastMempoolReq = GetTime(); } // Determine transactions to relay @@ -5890,21 +6640,30 @@ bool SendMessages(CNode* pto) continue; } // Not in the mempool anymore? don't bother sending it. - CFeeRate feeRate; - if (!mempool.lookupFeeRate(hash, feeRate)) { + auto txinfo = mempool.info(hash); + if (!txinfo.tx) { continue; } - if (filterrate && feeRate.GetFeePerK() < filterrate) { + if (filterrate && txinfo.feeRate.GetFeePerK() < filterrate) { continue; } - if (pto->pfilter) { - CTransaction tx; - if (!mempool.lookup(hash, tx)) continue; - if (!pto->pfilter->IsRelevantAndUpdate(tx)) continue; - } + if (pto->pfilter && !pto->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; // Send vInv.push_back(CInv(MSG_TX, hash)); nRelayedTransactions++; + { + // Expire old relay messages + while (!vRelayExpiration.empty() && vRelayExpiration.front().first < nNow) + { + mapRelay.erase(vRelayExpiration.front().second); + vRelayExpiration.pop_front(); + } + + auto ret = mapRelay.insert(std::make_pair(hash, std::move(txinfo.tx))); + if (ret.second) { + vRelayExpiration.push_back(std::make_pair(nNow + 15 * 60 * 1000000, ret.first)); + } + } if (vInv.size() == MAX_INV_SZ) { pto->PushMessage(NetMsgType::INV, vInv); vInv.clear(); @@ -5948,10 +6707,13 @@ bool SendMessages(CNode* pto) NodeId staller = -1; FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); BOOST_FOREACH(CBlockIndex *pindex, vToDownload) { - vGetData.push_back(CInv(MSG_BLOCK, pindex->GetBlockHash())); - MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), consensusParams, pindex); - LogPrint("net", "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), - pindex->nHeight, pto->id); + if (State(pto->GetId())->fHaveWitness || !IsWitnessEnabled(pindex->pprev, consensusParams)) { + uint32_t nFetchFlags = GetFetchFlags(pto, pindex->pprev, consensusParams); + vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); + MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), consensusParams, pindex); + LogPrint("net", "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), + pindex->nHeight, pto->id); + } } if (state.nBlocksInFlight == 0 && staller != -1) { if (State(staller)->nStallingSince == 0) { |