diff options
Diffstat (limited to 'src/main.cpp')
| -rw-r--r-- | src/main.cpp | 4156 |
1 files changed, 2483 insertions, 1673 deletions
diff --git a/src/main.cpp b/src/main.cpp index 7afaa9e7e..efb7fb3f8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,172 +1,177 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2014 The Bitcoin developers -// Distributed under the MIT/X11 software license, see the accompanying +// Copyright (c) 2009-2014 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "main.h" #include "addrman.h" #include "alert.h" +#include "arith_uint256.h" #include "chainparams.h" #include "checkpoints.h" #include "checkqueue.h" +#include "consensus/validation.h" #include "init.h" +#include "merkleblock.h" #include "net.h" +#include "pow.h" #include "txdb.h" #include "txmempool.h" #include "ui_interface.h" +#include "undo.h" #include "util.h" +#include "utilmoneystr.h" +#include "validationinterface.h" #include <sstream> #include <boost/algorithm/string/replace.hpp> #include <boost/filesystem.hpp> #include <boost/filesystem/fstream.hpp> +#include <boost/math/distributions/poisson.hpp> +#include <boost/thread.hpp> using namespace std; -using namespace boost; #if defined(NDEBUG) # error "Bitcoin cannot be compiled without assertions." #endif -// -// Global state -// +/** + * Global state + */ CCriticalSection cs_main; -CTxMemPool mempool; - -map<uint256, CBlockIndex*> mapBlockIndex; +BlockMap mapBlockIndex; CChain chainActive; -CChain chainMostWork; +CBlockIndex *pindexBestHeader = NULL; int64_t nTimeBestReceived = 0; +CWaitableCriticalSection csBestBlock; +CConditionVariable cvBlockChange; int nScriptCheckThreads = 0; bool fImporting = false; bool fReindex = false; -bool fBenchmark = false; bool fTxIndex = false; -unsigned int nCoinCacheSize = 5000; - -/** Fees smaller than this (in satoshi) are considered zero fee (for transaction creation) */ -int64_t CTransaction::nMinTxFee = 10000; // Override with -mintxfee -/** Fees smaller than this (in satoshi) are considered zero fee (for relaying) */ -int64_t CTransaction::nMinRelayTxFee = 1000; - -static CMedianFilter<int> cPeerBlockCounts(8, 0); // Amount of blocks that other nodes claim to have - -struct COrphanBlock { - uint256 hashBlock; - uint256 hashPrev; - vector<unsigned char> vchBlock; +bool fHavePruned = false; +bool fPruneMode = false; +bool fIsBareMultisigStd = true; +bool fCheckBlockIndex = false; +bool fCheckpointsEnabled = true; +size_t nCoinCacheUsage = 5000 * 300; +uint64_t nPruneTarget = 0; + +/** Fees smaller than this (in satoshi) are considered zero fee (for relaying and mining) */ +CFeeRate minRelayTxFee = CFeeRate(1000); + +CTxMemPool mempool(::minRelayTxFee); + +struct COrphanTx { + CTransaction tx; + NodeId fromPeer; }; -map<uint256, COrphanBlock*> mapOrphanBlocks; -multimap<uint256, COrphanBlock*> mapOrphanBlocksByPrev; - -map<uint256, CTransaction> mapOrphanTransactions; +map<uint256, COrphanTx> mapOrphanTransactions; map<uint256, set<uint256> > mapOrphanTransactionsByPrev; +void EraseOrphansFor(NodeId peer); -// Constant stuff for coinbase transactions we create: +/** + * Returns true if there are nRequired or more blocks of minVersion or above + * in the last Consensus::Params::nMajorityWindow blocks, starting at pstart and going backwards. + */ +static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned nRequired, const Consensus::Params& consensusParams); +static void CheckBlockIndex(); + +/** Constant stuff for coinbase transactions we create: */ CScript COINBASE_FLAGS; const string strMessageMagic = "Bitcoin Signed Message:\n"; // Internal stuff namespace { -struct CBlockIndexWorkComparator -{ - bool operator()(CBlockIndex *pa, CBlockIndex *pb) { - // First sort by most total work, ... - if (pa->nChainWork > pb->nChainWork) return false; - if (pa->nChainWork < pb->nChainWork) return true; - - // ... then by earliest time received, ... - if (pa->nSequenceId < pb->nSequenceId) return false; - if (pa->nSequenceId > pb->nSequenceId) return true; - - // Use pointer address as tie breaker (should only happen with blocks - // loaded from disk, as those all have id 0). - if (pa < pb) return false; - if (pa > pb) return true; - - // Identical blocks. - return false; - } -}; - -CBlockIndex *pindexBestInvalid; -set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexValid; // may contain all CBlockIndex*'s that have validness >=BLOCK_VALID_TRANSACTIONS, and must contain those who aren't failed - -CCriticalSection cs_LastBlockFile; -CBlockFileInfo infoLastBlockFile; -int nLastBlockFile = 0; - -// Every received block is assigned a unique and increasing identifier, so we -// know which one to give priority in case of a fork. -CCriticalSection cs_nBlockSequenceId; -// Blocks loaded from disk are assigned id 0, so start the counter at 1. -uint32_t nBlockSequenceId = 1; - -// Sources of received blocks, to be able to send them reject messages or ban -// them, if processing happens afterwards. Protected by cs_main. -map<uint256, NodeId> mapBlockSource; -} - -////////////////////////////////////////////////////////////////////////////// -// -// dispatching functions -// - -// These functions dispatch to one or all registered wallets - -namespace { -struct CMainSignals { - // Notifies listeners of updated transaction data (passing hash, transaction, and optionally the block it is found in. - boost::signals2::signal<void (const uint256 &, const CTransaction &, const CBlock *)> SyncTransaction; - // Notifies listeners of an erased transaction (currently disabled, requires transaction replacement). - boost::signals2::signal<void (const uint256 &)> EraseTransaction; - // Notifies listeners of an updated transaction without new data (for now: a coinbase potentially becoming visible). - boost::signals2::signal<void (const uint256 &)> UpdatedTransaction; - // Notifies listeners of a new active block chain. - boost::signals2::signal<void (const CBlockLocator &)> SetBestChain; - // Notifies listeners about an inventory item being seen on the network. - boost::signals2::signal<void (const uint256 &)> Inventory; - // Tells listeners to broadcast their data. - boost::signals2::signal<void ()> Broadcast; -} g_signals; -} -void RegisterWallet(CWalletInterface* pwalletIn) { - g_signals.SyncTransaction.connect(boost::bind(&CWalletInterface::SyncTransaction, pwalletIn, _1, _2, _3)); - g_signals.EraseTransaction.connect(boost::bind(&CWalletInterface::EraseFromWallet, pwalletIn, _1)); - g_signals.UpdatedTransaction.connect(boost::bind(&CWalletInterface::UpdatedTransaction, pwalletIn, _1)); - g_signals.SetBestChain.connect(boost::bind(&CWalletInterface::SetBestChain, pwalletIn, _1)); - g_signals.Inventory.connect(boost::bind(&CWalletInterface::Inventory, pwalletIn, _1)); - g_signals.Broadcast.connect(boost::bind(&CWalletInterface::ResendWalletTransactions, pwalletIn)); -} + struct CBlockIndexWorkComparator + { + bool operator()(CBlockIndex *pa, CBlockIndex *pb) const { + // First sort by most total work, ... + if (pa->nChainWork > pb->nChainWork) return false; + if (pa->nChainWork < pb->nChainWork) return true; -void UnregisterWallet(CWalletInterface* pwalletIn) { - g_signals.Broadcast.disconnect(boost::bind(&CWalletInterface::ResendWalletTransactions, pwalletIn)); - g_signals.Inventory.disconnect(boost::bind(&CWalletInterface::Inventory, pwalletIn, _1)); - g_signals.SetBestChain.disconnect(boost::bind(&CWalletInterface::SetBestChain, pwalletIn, _1)); - g_signals.UpdatedTransaction.disconnect(boost::bind(&CWalletInterface::UpdatedTransaction, pwalletIn, _1)); - g_signals.EraseTransaction.disconnect(boost::bind(&CWalletInterface::EraseFromWallet, pwalletIn, _1)); - g_signals.SyncTransaction.disconnect(boost::bind(&CWalletInterface::SyncTransaction, pwalletIn, _1, _2, _3)); -} + // ... then by earliest time received, ... + if (pa->nSequenceId < pb->nSequenceId) return false; + if (pa->nSequenceId > pb->nSequenceId) return true; -void UnregisterAllWallets() { - g_signals.Broadcast.disconnect_all_slots(); - g_signals.Inventory.disconnect_all_slots(); - g_signals.SetBestChain.disconnect_all_slots(); - g_signals.UpdatedTransaction.disconnect_all_slots(); - g_signals.EraseTransaction.disconnect_all_slots(); - g_signals.SyncTransaction.disconnect_all_slots(); -} + // Use pointer address as tie breaker (should only happen with blocks + // loaded from disk, as those all have id 0). + if (pa < pb) return false; + if (pa > pb) return true; -void SyncWithWallets(const uint256 &hash, const CTransaction &tx, const CBlock *pblock) { - g_signals.SyncTransaction(hash, tx, pblock); -} + // Identical blocks. + return false; + } + }; + + CBlockIndex *pindexBestInvalid; + + /** + * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and + * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be + * missing the data for the block. + */ + set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates; + /** Number of nodes with fSyncStarted. */ + int nSyncStarted = 0; + /** All pairs A->B, where A (or one if its ancestors) misses transactions, but B has transactions. + * Pruned nodes may have entries where B is missing data. + */ + multimap<CBlockIndex*, CBlockIndex*> mapBlocksUnlinked; + + CCriticalSection cs_LastBlockFile; + std::vector<CBlockFileInfo> vinfoBlockFile; + int nLastBlockFile = 0; + /** Global flag to indicate we should check to see if there are + * block/undo files that should be deleted. Set on startup + * or if we allocate more file space when we're in prune mode + */ + bool fCheckForPruning = false; + + /** + * Every received block is assigned a unique and increasing identifier, so we + * know which one to give priority in case of a fork. + */ + CCriticalSection cs_nBlockSequenceId; + /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ + uint32_t nBlockSequenceId = 1; + + /** + * Sources of received blocks, saved to be able to send them reject + * messages or ban them when processing happens afterwards. Protected by + * cs_main. + */ + map<uint256, NodeId> mapBlockSource; + + /** Blocks that are in flight, and that are in the queue to be downloaded. Protected by cs_main. */ + struct QueuedBlock { + uint256 hash; + CBlockIndex *pindex; //! Optional. + int64_t nTime; //! Time of "getdata" request in microseconds. + bool fValidatedHeaders; //! Whether this block has validated headers at the time of request. + int64_t nTimeDisconnect; //! The timeout for this block request (for disconnecting a slow peer) + }; + map<uint256, pair<NodeId, list<QueuedBlock>::iterator> > mapBlocksInFlight; + + /** Number of blocks in flight with validated headers. */ + int nQueuedValidatedHeaders = 0; + + /** Number of preferable block download peers. */ + int nPreferredDownload = 0; + + /** Dirty block index entries. */ + set<CBlockIndex*> setDirtyBlockIndex; + + /** Dirty block file entries. */ + set<int> setDirtyFileInfo; +} // anon namespace ////////////////////////////////////////////////////////////////////////////// // @@ -181,27 +186,57 @@ struct CBlockReject { uint256 hashBlock; }; -// Maintain validation-specific state about nodes, protected by cs_main, instead -// by CNode's own locks. This simplifies asynchronous operation, where -// processing of incoming data is done after the ProcessMessage call returns, -// and we're no longer holding the node's locks. +/** + * Maintain validation-specific state about nodes, protected by cs_main, instead + * by CNode's own locks. This simplifies asynchronous operation, where + * processing of incoming data is done after the ProcessMessage call returns, + * and we're no longer holding the node's locks. + */ struct CNodeState { - // Accumulated misbehaviour score for this peer. + //! The peer's address + CService address; + //! Whether we have a fully established connection. + bool fCurrentlyConnected; + //! Accumulated misbehaviour score for this peer. int nMisbehavior; - // Whether this peer should be disconnected and banned. + //! Whether this peer should be disconnected and banned (unless whitelisted). bool fShouldBan; - // String name of this peer (debugging/logging purposes). + //! String name of this peer (debugging/logging purposes). std::string name; - // List of asynchronously-determined block rejections to notify this peer about. + //! List of asynchronously-determined block rejections to notify this peer about. std::vector<CBlockReject> rejects; + //! The best known block we know this peer has announced. + CBlockIndex *pindexBestKnownBlock; + //! The hash of the last unknown block this peer has announced. + uint256 hashLastUnknownBlock; + //! The last full block we both have. + CBlockIndex *pindexLastCommonBlock; + //! Whether we've started headers synchronization with this peer. + bool fSyncStarted; + //! Since when we're stalling block download progress (in microseconds), or 0. + int64_t nStallingSince; + list<QueuedBlock> vBlocksInFlight; + int nBlocksInFlight; + int nBlocksInFlightValidHeaders; + //! Whether we consider this a preferred download peer. + bool fPreferredDownload; CNodeState() { + fCurrentlyConnected = false; nMisbehavior = 0; fShouldBan = false; + pindexBestKnownBlock = NULL; + hashLastUnknownBlock.SetNull(); + pindexLastCommonBlock = NULL; + fSyncStarted = false; + nStallingSince = 0; + nBlocksInFlight = 0; + nBlocksInFlightValidHeaders = 0; + fPreferredDownload = false; } }; -// Map maintaining per-node state. Requires cs_main. +/** Map maintaining per-node state. Requires cs_main. */ map<NodeId, CNodeState> mapNodeState; // Requires cs_main. @@ -218,24 +253,228 @@ int GetHeight() return chainActive.Height(); } +void UpdatePreferredDownload(CNode* node, CNodeState* state) +{ + nPreferredDownload -= state->fPreferredDownload; + + // Whether this node should be marked as a preferred download node. + state->fPreferredDownload = (!node->fInbound || node->fWhitelisted) && !node->fOneShot && !node->fClient; + + nPreferredDownload += state->fPreferredDownload; +} + +// Returns time at which to timeout block request (nTime in microseconds) +int64_t GetBlockTimeout(int64_t nTime, int nValidatedQueuedBefore, const Consensus::Params &consensusParams) +{ + return nTime + 500000 * consensusParams.nPowTargetSpacing * (4 + nValidatedQueuedBefore); +} + void InitializeNode(NodeId nodeid, const CNode *pnode) { LOCK(cs_main); CNodeState &state = mapNodeState.insert(std::make_pair(nodeid, CNodeState())).first->second; state.name = pnode->addrName; + state.address = pnode->addr; } void FinalizeNode(NodeId nodeid) { LOCK(cs_main); + CNodeState *state = State(nodeid); + + if (state->fSyncStarted) + nSyncStarted--; + + if (state->nMisbehavior == 0 && state->fCurrentlyConnected) { + AddressCurrentlyConnected(state->address); + } + + BOOST_FOREACH(const QueuedBlock& entry, state->vBlocksInFlight) + mapBlocksInFlight.erase(entry.hash); + EraseOrphansFor(nodeid); + nPreferredDownload -= state->fPreferredDownload; + mapNodeState.erase(nodeid); } + +// Requires cs_main. +void MarkBlockAsReceived(const uint256& hash) { + map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); + if (itInFlight != mapBlocksInFlight.end()) { + CNodeState *state = State(itInFlight->second.first); + nQueuedValidatedHeaders -= itInFlight->second.second->fValidatedHeaders; + state->nBlocksInFlightValidHeaders -= itInFlight->second.second->fValidatedHeaders; + state->vBlocksInFlight.erase(itInFlight->second.second); + state->nBlocksInFlight--; + state->nStallingSince = 0; + mapBlocksInFlight.erase(itInFlight); + } +} + +// Requires cs_main. +void MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, CBlockIndex *pindex = NULL) { + CNodeState *state = State(nodeid); + assert(state != NULL); + + // Make sure it's not listed somewhere already. + MarkBlockAsReceived(hash); + + int64_t nNow = GetTimeMicros(); + QueuedBlock newentry = {hash, pindex, nNow, pindex != NULL, GetBlockTimeout(nNow, nQueuedValidatedHeaders, consensusParams)}; + nQueuedValidatedHeaders += newentry.fValidatedHeaders; + list<QueuedBlock>::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), newentry); + state->nBlocksInFlight++; + state->nBlocksInFlightValidHeaders += newentry.fValidatedHeaders; + mapBlocksInFlight[hash] = std::make_pair(nodeid, it); +} + +/** Check whether the last unknown block a peer advertized is not yet known. */ +void ProcessBlockAvailability(NodeId nodeid) { + CNodeState *state = State(nodeid); + assert(state != NULL); + + if (!state->hashLastUnknownBlock.IsNull()) { + BlockMap::iterator itOld = mapBlockIndex.find(state->hashLastUnknownBlock); + if (itOld != mapBlockIndex.end() && itOld->second->nChainWork > 0) { + if (state->pindexBestKnownBlock == NULL || itOld->second->nChainWork >= state->pindexBestKnownBlock->nChainWork) + state->pindexBestKnownBlock = itOld->second; + state->hashLastUnknownBlock.SetNull(); + } + } +} + +/** Update tracking information about which blocks a peer is assumed to have. */ +void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { + CNodeState *state = State(nodeid); + assert(state != NULL); + + ProcessBlockAvailability(nodeid); + + BlockMap::iterator it = mapBlockIndex.find(hash); + if (it != mapBlockIndex.end() && it->second->nChainWork > 0) { + // An actually better block was announced. + if (state->pindexBestKnownBlock == NULL || it->second->nChainWork >= state->pindexBestKnownBlock->nChainWork) + state->pindexBestKnownBlock = it->second; + } else { + // An unknown block was announced; just assume that the latest one is the best one. + state->hashLastUnknownBlock = hash; + } +} + +/** Find the last common ancestor two blocks have. + * Both pa and pb must be non-NULL. */ +CBlockIndex* LastCommonAncestor(CBlockIndex* pa, CBlockIndex* pb) { + if (pa->nHeight > pb->nHeight) { + pa = pa->GetAncestor(pb->nHeight); + } else if (pb->nHeight > pa->nHeight) { + pb = pb->GetAncestor(pa->nHeight); + } + + while (pa != pb && pa && pb) { + pa = pa->pprev; + pb = pb->pprev; + } + + // Eventually all chain branches meet at the genesis block. + assert(pa == pb); + return pa; } +/** Update pindexLastCommonBlock and add not-in-flight missing successors to vBlocks, until it has + * at most count entries. */ +void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<CBlockIndex*>& vBlocks, NodeId& nodeStaller) { + if (count == 0) + return; + + vBlocks.reserve(vBlocks.size() + count); + CNodeState *state = State(nodeid); + assert(state != NULL); + + // Make sure pindexBestKnownBlock is up to date, we'll need it. + ProcessBlockAvailability(nodeid); + + if (state->pindexBestKnownBlock == NULL || state->pindexBestKnownBlock->nChainWork < chainActive.Tip()->nChainWork) { + // This peer has nothing interesting. + return; + } + + if (state->pindexLastCommonBlock == NULL) { + // Bootstrap quickly by guessing a parent of our best tip is the forking point. + // Guessing wrong in either direction is not a problem. + state->pindexLastCommonBlock = chainActive[std::min(state->pindexBestKnownBlock->nHeight, chainActive.Height())]; + } + + // If the peer reorganized, our previous pindexLastCommonBlock may not be an ancestor + // of its current tip anymore. Go back enough to fix that. + state->pindexLastCommonBlock = LastCommonAncestor(state->pindexLastCommonBlock, state->pindexBestKnownBlock); + if (state->pindexLastCommonBlock == state->pindexBestKnownBlock) + return; + + std::vector<CBlockIndex*> vToFetch; + CBlockIndex *pindexWalk = state->pindexLastCommonBlock; + // Never fetch further than the best block we know the peer has, or more than BLOCK_DOWNLOAD_WINDOW + 1 beyond the last + // linked block we have in common with this peer. The +1 is so we can detect stalling, namely if we would be able to + // download that next block if the window were 1 larger. + int nWindowEnd = state->pindexLastCommonBlock->nHeight + BLOCK_DOWNLOAD_WINDOW; + int nMaxHeight = std::min<int>(state->pindexBestKnownBlock->nHeight, nWindowEnd + 1); + NodeId waitingfor = -1; + while (pindexWalk->nHeight < nMaxHeight) { + // Read up to 128 (or more, if more blocks than that are needed) successors of pindexWalk (towards + // pindexBestKnownBlock) into vToFetch. We fetch 128, because CBlockIndex::GetAncestor may be as expensive + // as iterating over ~100 CBlockIndex* entries anyway. + int nToFetch = std::min(nMaxHeight - pindexWalk->nHeight, std::max<int>(count - vBlocks.size(), 128)); + vToFetch.resize(nToFetch); + pindexWalk = state->pindexBestKnownBlock->GetAncestor(pindexWalk->nHeight + nToFetch); + vToFetch[nToFetch - 1] = pindexWalk; + for (unsigned int i = nToFetch - 1; i > 0; i--) { + vToFetch[i - 1] = vToFetch[i]->pprev; + } + + // Iterate over those blocks in vToFetch (in forward direction), adding the ones that + // are not yet downloaded and not in flight to vBlocks. In the mean time, update + // pindexLastCommonBlock as long as all ancestors are already downloaded. + BOOST_FOREACH(CBlockIndex* pindex, vToFetch) { + if (!pindex->IsValid(BLOCK_VALID_TREE)) { + // We consider the chain that this peer is on invalid. + return; + } + if (pindex->nStatus & BLOCK_HAVE_DATA) { + if (pindex->nChainTx) + state->pindexLastCommonBlock = pindex; + } else if (mapBlocksInFlight.count(pindex->GetBlockHash()) == 0) { + // The block is not already downloaded, and not yet in flight. + if (pindex->nHeight > nWindowEnd) { + // We reached the end of the window. + if (vBlocks.size() == 0 && waitingfor != nodeid) { + // We aren't able to fetch anything, but we would be if the download window was one larger. + nodeStaller = waitingfor; + } + return; + } + vBlocks.push_back(pindex); + if (vBlocks.size() == count) { + return; + } + } else if (waitingfor == -1) { + // This is the first already-in-flight block. + waitingfor = mapBlocksInFlight[pindex->GetBlockHash()].first; + } + } + } +} + +} // anon namespace + bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { LOCK(cs_main); CNodeState *state = State(nodeid); if (state == NULL) return false; stats.nMisbehavior = state->nMisbehavior; + stats.nSyncHeight = state->pindexBestKnownBlock ? state->pindexBestKnownBlock->nHeight : -1; + stats.nCommonHeight = state->pindexLastCommonBlock ? state->pindexLastCommonBlock->nHeight : -1; + BOOST_FOREACH(const QueuedBlock& queue, state->vBlocksInFlight) { + if (queue.pindex) + stats.vHeightInFlight.push_back(queue.pindex->nHeight); + } return true; } @@ -257,63 +496,19 @@ void UnregisterNodeSignals(CNodeSignals& nodeSignals) nodeSignals.FinalizeNode.disconnect(&FinalizeNode); } -////////////////////////////////////////////////////////////////////////////// -// -// CChain implementation -// - -CBlockIndex *CChain::SetTip(CBlockIndex *pindex) { - if (pindex == NULL) { - vChain.clear(); - return NULL; - } - vChain.resize(pindex->nHeight + 1); - while (pindex && vChain[pindex->nHeight] != pindex) { - vChain[pindex->nHeight] = pindex; - pindex = pindex->pprev; - } - return pindex; -} - -CBlockLocator CChain::GetLocator(const CBlockIndex *pindex) const { - int nStep = 1; - std::vector<uint256> vHave; - vHave.reserve(32); - - if (!pindex) - pindex = Tip(); - while (pindex) { - vHave.push_back(pindex->GetBlockHash()); - // Stop when we have added the genesis block. - if (pindex->nHeight == 0) - break; - // Exponentially larger steps back, plus the genesis block. - int nHeight = std::max(pindex->nHeight - nStep, 0); - // In case pindex is not in this chain, iterate pindex->pprev to find blocks. - while (pindex->nHeight > nHeight && !Contains(pindex)) - pindex = pindex->pprev; - // If pindex is in this chain, use direct height-based access. - if (pindex->nHeight > nHeight) - pindex = (*this)[nHeight]; - if (vHave.size() > 10) - nStep *= 2; - } - - return CBlockLocator(vHave); -} - -CBlockIndex *CChain::FindFork(const CBlockLocator &locator) const { +CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) +{ // Find the first block the caller has in the main chain BOOST_FOREACH(const uint256& hash, locator.vHave) { - std::map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(hash); + BlockMap::iterator mi = mapBlockIndex.find(hash); if (mi != mapBlockIndex.end()) { CBlockIndex* pindex = (*mi).second; - if (Contains(pindex)) + if (chain.Contains(pindex)) return pindex; } } - return Genesis(); + return chain.Genesis(); } CCoinsViewCache *pcoinsTip = NULL; @@ -324,7 +519,7 @@ CBlockTreeDB *pblocktree = NULL; // mapOrphanTransactions // -bool AddOrphanTx(const CTransaction& tx) +bool AddOrphanTx(const CTransaction& tx, NodeId peer) { uint256 hash = tx.GetHash(); if (mapOrphanTransactions.count(hash)) @@ -344,29 +539,50 @@ bool AddOrphanTx(const CTransaction& tx) return false; } - mapOrphanTransactions[hash] = tx; + mapOrphanTransactions[hash].tx = tx; + mapOrphanTransactions[hash].fromPeer = peer; BOOST_FOREACH(const CTxIn& txin, tx.vin) mapOrphanTransactionsByPrev[txin.prevout.hash].insert(hash); - LogPrint("mempool", "stored orphan tx %s (mapsz %"PRIszu")\n", hash.ToString(), - mapOrphanTransactions.size()); + LogPrint("mempool", "stored orphan tx %s (mapsz %u prevsz %u)\n", hash.ToString(), + mapOrphanTransactions.size(), mapOrphanTransactionsByPrev.size()); return true; } void static EraseOrphanTx(uint256 hash) { - if (!mapOrphanTransactions.count(hash)) + map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.find(hash); + if (it == mapOrphanTransactions.end()) return; - const CTransaction& tx = mapOrphanTransactions[hash]; - BOOST_FOREACH(const CTxIn& txin, tx.vin) + BOOST_FOREACH(const CTxIn& txin, it->second.tx.vin) { - mapOrphanTransactionsByPrev[txin.prevout.hash].erase(hash); - if (mapOrphanTransactionsByPrev[txin.prevout.hash].empty()) - mapOrphanTransactionsByPrev.erase(txin.prevout.hash); + map<uint256, set<uint256> >::iterator itPrev = mapOrphanTransactionsByPrev.find(txin.prevout.hash); + if (itPrev == mapOrphanTransactionsByPrev.end()) + continue; + itPrev->second.erase(hash); + if (itPrev->second.empty()) + mapOrphanTransactionsByPrev.erase(itPrev); } - mapOrphanTransactions.erase(hash); + mapOrphanTransactions.erase(it); } +void EraseOrphansFor(NodeId peer) +{ + int nErased = 0; + map<uint256, COrphanTx>::iterator iter = mapOrphanTransactions.begin(); + while (iter != mapOrphanTransactions.end()) + { + map<uint256, COrphanTx>::iterator maybeErase = iter++; // increment to avoid iterator becoming invalid + if (maybeErase->second.fromPeer == peer) + { + EraseOrphanTx(maybeErase->second.tx.GetHash()); + ++nErased; + } + } + if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx from peer %d\n", nErased, peer); +} + + unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) { unsigned int nEvicted = 0; @@ -374,7 +590,7 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) { // Evict a random orphan: uint256 randomhash = GetRandHash(); - map<uint256, CTransaction>::iterator it = mapOrphanTransactions.lower_bound(randomhash); + map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.lower_bound(randomhash); if (it == mapOrphanTransactions.end()) it = mapOrphanTransactions.begin(); EraseOrphanTx(it->first); @@ -396,28 +612,6 @@ bool IsStandardTx(const CTransaction& tx, string& reason) return false; } - // Treat non-final transactions as non-standard to prevent a specific type - // of double-spend attack, as well as DoS attacks. (if the transaction - // can't be mined, the attacker isn't expending resources broadcasting it) - // Basically we don't want to propagate transactions that can't included in - // the next block. - // - // However, IsFinalTx() is confusing... Without arguments, it uses - // chainActive.Height() to evaluate nLockTime; when a block is accepted, chainActive.Height() - // is set to the value of nHeight in the block. However, when IsFinalTx() - // is called within CBlock::AcceptBlock(), the height of the block *being* - // evaluated is what is used. Thus if we want to know if a transaction can - // be part of the *next* block, we need to call IsFinalTx() with one more - // than chainActive.Height(). - // - // Timestamps on the other hand don't get any special treatment, because we - // can't know what timestamp the next block will have, and there aren't - // timestamp applications where it matters. - if (!IsFinalTx(tx, chainActive.Height() + 1)) { - reason = "non-final"; - return false; - } - // Extremely large transactions with lots of inputs can cost the network // almost as much to process as they cost the sender in fees, because // computing signature hashes is O(ninputs*txsize). Limiting transactions @@ -430,10 +624,14 @@ bool IsStandardTx(const CTransaction& tx, string& reason) BOOST_FOREACH(const CTxIn& txin, tx.vin) { - // Biggest 'standard' txin is a 3-signature 3-of-3 CHECKMULTISIG - // pay-to-script-hash, which is 3 ~80-byte signatures, 3 - // ~65-byte public keys, plus a few script ops. - if (txin.scriptSig.size() > 500) { + // Biggest 'standard' txin is a 15-of-15 P2SH multisig with compressed + // keys. (remember the 520 byte limit on redeemScript size) That works + // out to a (15*(33+1))+3=513 byte redeemScript, 513+1+15*(73+1)+3=1627 + // bytes of scriptSig, which we round off to 1650 bytes for some minor + // future-proofing. That's also enough to spend a 20-of-20 + // CHECKMULTISIG scriptPubKey, though such a scriptPubKey is not + // considered standard) + if (txin.scriptSig.size() > 1650) { reason = "scriptsig-size"; return false; } @@ -441,10 +639,6 @@ bool IsStandardTx(const CTransaction& tx, string& reason) reason = "scriptsig-not-pushonly"; return false; } - if (!txin.scriptSig.HasCanonicalPushes()) { - reason = "scriptsig-non-canonical-push"; - return false; - } } unsigned int nDataOut = 0; @@ -454,9 +648,13 @@ bool IsStandardTx(const CTransaction& tx, string& reason) reason = "scriptpubkey"; return false; } + if (whichType == TX_NULL_DATA) nDataOut++; - else if (txout.IsDust(CTransaction::nMinRelayTxFee)) { + else if ((whichType == TX_MULTISIG) && (!fIsBareMultisigStd)) { + reason = "bare-multisig"; + return false; + } else if (txout.IsDust(::minRelayTxFee)) { reason = "dust"; return false; } @@ -473,13 +671,8 @@ bool IsStandardTx(const CTransaction& tx, string& reason) bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) { - // Time based nLockTime implemented in 0.1.6 if (tx.nLockTime == 0) return true; - if (nBlockHeight == 0) - nBlockHeight = chainActive.Height(); - if (nBlockTime == 0) - nBlockTime = GetAdjustedTime(); if ((int64_t)tx.nLockTime < ((int64_t)tx.nLockTime < LOCKTIME_THRESHOLD ? (int64_t)nBlockHeight : nBlockTime)) return true; BOOST_FOREACH(const CTxIn& txin, tx.vin) @@ -488,18 +681,22 @@ bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) return true; } -// -// Check transaction inputs, and make sure any -// pay-to-script-hash transactions are evaluating IsStandard scripts -// -// Why bother? To avoid denial-of-service attacks; an attacker -// can submit a standard HASH... OP_EQUAL transaction, -// which will get accepted into blocks. The redemption -// script can be anything; an attacker could use a very -// expensive-to-check-upon-redemption script like: -// DUP CHECKSIG DROP ... repeated 100 times... OP_1 -// -bool AreInputsStandard(const CTransaction& tx, CCoinsViewCache& mapInputs) +bool CheckFinalTx(const CTransaction &tx) +{ + AssertLockHeld(cs_main); + return IsFinalTx(tx, chainActive.Height() + 1, GetAdjustedTime()); +} + +/** + * Check transaction inputs to mitigate two + * potential denial-of-service attacks: + * + * 1. scriptSigs with extra data stuffed into them, + * not consumed by scriptPubKey (or P2SH script) + * 2. P2SH scripts with a crazy number of expensive + * CHECKSIG/CHECKMULTISIG operations + */ +bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) { if (tx.IsCoinBase()) return true; // Coinbases don't use vin normally @@ -521,10 +718,11 @@ bool AreInputsStandard(const CTransaction& tx, CCoinsViewCache& mapInputs) // Transactions with extra stuff in their scriptSigs are // non-standard. Note that this EvalScript() call will // be quick, because if there are any operations - // beside "push data" in the scriptSig the - // IsStandard() call returns false + // beside "push data" in the scriptSig + // IsStandardTx() will have already returned false + // and this method isn't called. vector<vector<unsigned char> > stack; - if (!EvalScript(stack, tx.vin[i].scriptSig, tx, i, false, 0)) + if (!EvalScript(stack, tx.vin[i].scriptSig, SCRIPT_VERIFY_NONE, BaseSignatureChecker())) return false; if (whichType == TX_SCRIPTHASH) @@ -534,16 +732,20 @@ bool AreInputsStandard(const CTransaction& tx, CCoinsViewCache& mapInputs) CScript subscript(stack.back().begin(), stack.back().end()); vector<vector<unsigned char> > vSolutions2; txnouttype whichType2; - if (!Solver(subscript, whichType2, vSolutions2)) - return false; - if (whichType2 == TX_SCRIPTHASH) - return false; - - int tmpExpected; - tmpExpected = ScriptSigArgsExpected(whichType2, vSolutions2); - if (tmpExpected < 0) - return false; - nArgsExpected += tmpExpected; + if (Solver(subscript, whichType2, vSolutions2)) + { + int tmpExpected = ScriptSigArgsExpected(whichType2, vSolutions2); + if (tmpExpected < 0) + return false; + nArgsExpected += tmpExpected; + } + else + { + // Any other Script with less than 15 sigops OK: + unsigned int sigops = subscript.GetSigOpCount(true); + // ... extra data left on the stack after execution is OK, too: + return (sigops <= MAX_P2SH_SIGOPS); + } } if (stack.size() != (unsigned int)nArgsExpected) @@ -567,7 +769,7 @@ unsigned int GetLegacySigOpCount(const CTransaction& tx) return nSigOps; } -unsigned int GetP2SHSigOpCount(const CTransaction& tx, CCoinsViewCache& inputs) +unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& inputs) { if (tx.IsCoinBase()) return 0; @@ -582,52 +784,6 @@ unsigned int GetP2SHSigOpCount(const CTransaction& tx, CCoinsViewCache& inputs) return nSigOps; } -int CMerkleTx::SetMerkleBranch(const CBlock* pblock) -{ - CBlock blockTmp; - - if (pblock == NULL) { - CCoins coins; - if (pcoinsTip->GetCoins(GetHash(), coins)) { - CBlockIndex *pindex = chainActive[coins.nHeight]; - if (pindex) { - if (!ReadBlockFromDisk(blockTmp, pindex)) - return 0; - pblock = &blockTmp; - } - } - } - - if (pblock) { - // Update the tx's hashBlock - hashBlock = pblock->GetHash(); - - // Locate the transaction - for (nIndex = 0; nIndex < (int)pblock->vtx.size(); nIndex++) - if (pblock->vtx[nIndex] == *(CTransaction*)this) - break; - if (nIndex == (int)pblock->vtx.size()) - { - vMerkleBranch.clear(); - nIndex = -1; - LogPrintf("ERROR: SetMerkleBranch() : couldn't find tx in block\n"); - return 0; - } - - // Fill in merkle branch - vMerkleBranch = pblock->GetMerkleBranch(nIndex); - } - - // Is the tx in a block that's in the main chain - map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(hashBlock); - if (mi == mapBlockIndex.end()) - return 0; - CBlockIndex* pindex = (*mi).second; - if (!pindex || !chainActive.Contains(pindex)) - return 0; - - return chainActive.Height() - pindex->nHeight + 1; -} @@ -639,29 +795,29 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) { // Basic checks that don't depend on any context if (tx.vin.empty()) - return state.DoS(10, error("CheckTransaction() : vin empty"), + return state.DoS(10, error("CheckTransaction(): vin empty"), REJECT_INVALID, "bad-txns-vin-empty"); if (tx.vout.empty()) - return state.DoS(10, error("CheckTransaction() : vout empty"), + return state.DoS(10, error("CheckTransaction(): vout empty"), REJECT_INVALID, "bad-txns-vout-empty"); // Size limits if (::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION) > MAX_BLOCK_SIZE) - return state.DoS(100, error("CheckTransaction() : size limits failed"), + return state.DoS(100, error("CheckTransaction(): size limits failed"), REJECT_INVALID, "bad-txns-oversize"); // Check for negative or overflow output values - int64_t nValueOut = 0; + CAmount nValueOut = 0; BOOST_FOREACH(const CTxOut& txout, tx.vout) { if (txout.nValue < 0) - return state.DoS(100, error("CheckTransaction() : txout.nValue negative"), + return state.DoS(100, error("CheckTransaction(): txout.nValue negative"), REJECT_INVALID, "bad-txns-vout-negative"); if (txout.nValue > MAX_MONEY) - return state.DoS(100, error("CheckTransaction() : txout.nValue too high"), + return state.DoS(100, error("CheckTransaction(): txout.nValue too high"), REJECT_INVALID, "bad-txns-vout-toolarge"); nValueOut += txout.nValue; if (!MoneyRange(nValueOut)) - return state.DoS(100, error("CheckTransaction() : txout total out of range"), + return state.DoS(100, error("CheckTransaction(): txout total out of range"), REJECT_INVALID, "bad-txns-txouttotal-toolarge"); } @@ -670,7 +826,7 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) BOOST_FOREACH(const CTxIn& txin, tx.vin) { if (vInOutPoints.count(txin.prevout)) - return state.DoS(100, error("CheckTransaction() : duplicate inputs"), + return state.DoS(100, error("CheckTransaction(): duplicate inputs"), REJECT_INVALID, "bad-txns-inputs-duplicate"); vInOutPoints.insert(txin.prevout); } @@ -678,26 +834,33 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) if (tx.IsCoinBase()) { if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) - return state.DoS(100, error("CheckTransaction() : coinbase script size"), + return state.DoS(100, error("CheckTransaction(): coinbase script size"), REJECT_INVALID, "bad-cb-length"); } else { BOOST_FOREACH(const CTxIn& txin, tx.vin) if (txin.prevout.IsNull()) - return state.DoS(10, error("CheckTransaction() : prevout is null"), + return state.DoS(10, error("CheckTransaction(): prevout is null"), REJECT_INVALID, "bad-txns-prevout-null"); } return true; } -int64_t GetMinFee(const CTransaction& tx, unsigned int nBytes, bool fAllowFree, enum GetMinFee_mode mode) +CAmount GetMinRelayFee(const CTransaction& tx, unsigned int nBytes, bool fAllowFree) { - // Base fee is either nMinTxFee or nMinRelayTxFee - int64_t nBaseFee = (mode == GMF_RELAY) ? tx.nMinRelayTxFee : tx.nMinTxFee; + { + LOCK(mempool.cs); + uint256 hash = tx.GetHash(); + double dPriorityDelta = 0; + CAmount nFeeDelta = 0; + mempool.ApplyDeltas(hash, dPriorityDelta, nFeeDelta); + if (dPriorityDelta > 0 || nFeeDelta > 0) + return 0; + } - int64_t nMinFee = (1 + (int64_t)nBytes / 1000) * nBaseFee; + CAmount nMinFee = ::minRelayTxFee.GetFee(nBytes); if (fAllowFree) { @@ -705,22 +868,10 @@ int64_t GetMinFee(const CTransaction& tx, unsigned int nBytes, bool fAllowFree, // * If we are relaying we allow transactions up to DEFAULT_BLOCK_PRIORITY_SIZE - 1000 // to be considered to fall into this category. We don't want to encourage sending // multiple transactions instead of one big transaction to avoid fees. - // * If we are creating a transaction we allow transactions up to 1,000 bytes - // to be considered safe and assume they can likely make it into this section. - if (nBytes < (mode == GMF_SEND ? 1000 : (DEFAULT_BLOCK_PRIORITY_SIZE - 1000))) + if (nBytes < (DEFAULT_BLOCK_PRIORITY_SIZE - 1000)) nMinFee = 0; } - // This code can be removed after enough miners have upgraded to version 0.9. - // Until then, be safe when sending and require a fee if any output - // is less than CENT: - if (nMinFee < nBaseFee && mode == GMF_SEND) - { - BOOST_FOREACH(const CTxOut& txout, tx.vout) - if (txout.nValue < CENT) - nMinFee = nBaseFee; - } - if (!MoneyRange(nMinFee)) nMinFee = MAX_MONEY; return nMinFee; @@ -728,26 +879,34 @@ int64_t GetMinFee(const CTransaction& tx, unsigned int nBytes, bool fAllowFree, bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, - bool* pfMissingInputs, bool fRejectInsaneFee) + bool* pfMissingInputs, bool fRejectAbsurdFee) { + AssertLockHeld(cs_main); if (pfMissingInputs) *pfMissingInputs = false; if (!CheckTransaction(tx, state)) - return error("AcceptToMemoryPool: : CheckTransaction failed"); + return error("AcceptToMemoryPool: CheckTransaction failed"); // Coinbase is only valid in a block, not as a loose transaction if (tx.IsCoinBase()) - return state.DoS(100, error("AcceptToMemoryPool: : coinbase as individual tx"), + return state.DoS(100, error("AcceptToMemoryPool: coinbase as individual tx"), REJECT_INVALID, "coinbase"); // Rather not work on nonstandard transactions (unless -testnet/-regtest) string reason; - if (Params().NetworkID() == CChainParams::MAIN && !IsStandardTx(tx, reason)) + if (Params().RequireStandard() && !IsStandardTx(tx, reason)) return state.DoS(0, - error("AcceptToMemoryPool : nonstandard transaction: %s", reason), + error("AcceptToMemoryPool: nonstandard transaction: %s", reason), REJECT_NONSTANDARD, reason); + // Only accept nLockTime-using transactions that can be mined in the next + // block; we don't want our mempool filled up with transactions that can't + // be mined yet. + if (!CheckFinalTx(tx)) + return state.DoS(0, error("AcceptToMemoryPool: non-final"), + REJECT_NONSTANDARD, "non-final"); + // is it already in the memory pool? uint256 hash = tx.GetHash(); if (pool.exists(hash)) @@ -769,11 +928,12 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa { CCoinsView dummy; - CCoinsViewCache view(dummy); + CCoinsViewCache view(&dummy); + CAmount nValueIn = 0; { LOCK(pool.cs); - CCoinsViewMemPool viewMemPool(*pcoinsTip, pool); + CCoinsViewMemPool viewMemPool(pcoinsTip, pool); view.SetBackend(viewMemPool); // do we already have it? @@ -782,7 +942,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa // do all inputs exist? // Note that this does not check for the presence of actual outputs (see the next check for that), - // only helps filling in pfMissingInputs (to determine missing vs spent). + // and only helps with filling in pfMissingInputs (to determine missing vs spent). BOOST_FOREACH(const CTxIn txin, tx.vin) { if (!view.HaveCoins(txin.prevout.hash)) { if (pfMissingInputs) @@ -793,43 +953,58 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa // are the actual inputs available? if (!view.HaveInputs(tx)) - return state.Invalid(error("AcceptToMemoryPool : inputs already spent"), + return state.Invalid(error("AcceptToMemoryPool: inputs already spent"), REJECT_DUPLICATE, "bad-txns-inputs-spent"); // Bring the best block into scope view.GetBestBlock(); + nValueIn = view.GetValueIn(tx); + // we have all inputs cached now, so switch back to dummy, so we don't need to keep lock on mempool view.SetBackend(dummy); } // Check for non-standard pay-to-script-hash in inputs - if (Params().NetworkID() == CChainParams::MAIN && !AreInputsStandard(tx, view)) - return error("AcceptToMemoryPool: : nonstandard transaction input"); - - // Note: if you modify this code to accept non-standard transactions, then - // you should add code here to check that the transaction does a - // reasonable number of ECDSA signature verifications. - - int64_t nValueIn = view.GetValueIn(tx); - int64_t nValueOut = tx.GetValueOut(); - int64_t nFees = nValueIn-nValueOut; + if (Params().RequireStandard() && !AreInputsStandard(tx, view)) + return error("AcceptToMemoryPool: nonstandard transaction input"); + + // Check that the transaction doesn't have an excessive number of + // sigops, making it impossible to mine. Since the coinbase transaction + // itself can contain sigops MAX_STANDARD_TX_SIGOPS is less than + // MAX_BLOCK_SIGOPS; we still consider this an invalid rather than + // merely non-standard transaction. + unsigned int nSigOps = GetLegacySigOpCount(tx); + nSigOps += GetP2SHSigOpCount(tx, view); + if (nSigOps > MAX_STANDARD_TX_SIGOPS) + return state.DoS(0, + error("AcceptToMemoryPool: too many sigops %s, %d > %d", + hash.ToString(), nSigOps, MAX_STANDARD_TX_SIGOPS), + REJECT_NONSTANDARD, "bad-txns-too-many-sigops"); + + CAmount nValueOut = tx.GetValueOut(); + CAmount nFees = nValueIn-nValueOut; double dPriority = view.GetPriority(tx, chainActive.Height()); - CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height()); + CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), mempool.HasNoInputsOf(tx)); unsigned int nSize = entry.GetTxSize(); // Don't accept it if it can't get into a block - int64_t txMinFee = GetMinFee(tx, nSize, true, GMF_RELAY); + CAmount txMinFee = GetMinRelayFee(tx, nSize, true); if (fLimitFree && nFees < txMinFee) - return state.DoS(0, error("AcceptToMemoryPool : not enough fees %s, %d < %d", + return state.DoS(0, error("AcceptToMemoryPool: not enough fees %s, %d < %d", hash.ToString(), nFees, txMinFee), REJECT_INSUFFICIENTFEE, "insufficient fee"); - // Continuously rate-limit free transactions + // Require that free transactions have sufficient priority to be mined in the next block. + if (GetBoolArg("-relaypriority", true) && nFees < ::minRelayTxFee.GetFee(nSize) && !AllowFree(view.GetPriority(tx, chainActive.Height() + 1))) { + return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "insufficient priority"); + } + + // Continuously rate-limit free (really, very-low-fee) transactions // This mitigates 'penny-flooding' -- sending thousands of free transactions just to // be annoying or make others' transactions take longer to confirm. - if (fLimitFree && nFees < CTransaction::nMinRelayTxFee) + if (fLimitFree && nFees < ::minRelayTxFee.GetFee(nSize)) { static CCriticalSection csFreeLimiter; static double dFreeCount; @@ -844,83 +1019,48 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa // -limitfreerelay unit is thousand-bytes-per-minute // At default rate it would take over a month to fill 1GB if (dFreeCount >= GetArg("-limitfreerelay", 15)*10*1000) - return state.DoS(0, error("AcceptToMemoryPool : free transaction rejected by rate limiter"), - REJECT_INSUFFICIENTFEE, "insufficient priority"); + return state.DoS(0, error("AcceptToMemoryPool: free transaction rejected by rate limiter"), + REJECT_INSUFFICIENTFEE, "rate limited free transaction"); LogPrint("mempool", "Rate limit dFreeCount: %g => %g\n", dFreeCount, dFreeCount+nSize); dFreeCount += nSize; } - if (fRejectInsaneFee && nFees > CTransaction::nMinRelayTxFee * 10000) - return error("AcceptToMemoryPool: : insane fees %s, %d > %d", + if (fRejectAbsurdFee && nFees > ::minRelayTxFee.GetFee(nSize) * 10000) + return error("AcceptToMemoryPool: absurdly high fees %s, %d > %d", hash.ToString(), - nFees, CTransaction::nMinRelayTxFee * 10000); + nFees, ::minRelayTxFee.GetFee(nSize) * 10000); // Check against previous transactions // This is done last to help prevent CPU exhaustion denial-of-service attacks. - if (!CheckInputs(tx, state, view, true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC)) + if (!CheckInputs(tx, state, view, true, STANDARD_SCRIPT_VERIFY_FLAGS, true)) { - return error("AcceptToMemoryPool: : ConnectInputs failed %s", hash.ToString()); + return error("AcceptToMemoryPool: ConnectInputs failed %s", hash.ToString()); } - // Store transaction in memory - pool.addUnchecked(hash, entry); - } - - g_signals.SyncTransaction(hash, tx, NULL); - - return true; -} - -int CMerkleTx::GetDepthInMainChainINTERNAL(CBlockIndex* &pindexRet) const -{ - if (hashBlock == 0 || nIndex == -1) - return 0; - - // Find the block it claims to be in - map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(hashBlock); - if (mi == mapBlockIndex.end()) - return 0; - CBlockIndex* pindex = (*mi).second; - if (!pindex || !chainActive.Contains(pindex)) - return 0; + // Check again against just the consensus-critical mandatory script + // verification flags, in case of bugs in the standard flags that cause + // transactions to pass as valid when they're actually invalid. For + // instance the STRICTENC flag was incorrectly allowing certain + // CHECKSIG NOT scripts to pass, even though they were invalid. + // + // There is a similar check in CreateNewBlock() to prevent creating + // invalid blocks, however allowing such transactions into the mempool + // can be exploited as a DoS attack. + if (!CheckInputs(tx, state, view, true, MANDATORY_SCRIPT_VERIFY_FLAGS, true)) + { + return error("AcceptToMemoryPool: BUG! PLEASE REPORT THIS! ConnectInputs failed against MANDATORY but not STANDARD flags %s", hash.ToString()); + } - // Make sure the merkle branch connects to this block - if (!fMerkleVerified) - { - if (CBlock::CheckMerkleBranch(GetHash(), vMerkleBranch, nIndex) != pindex->hashMerkleRoot) - return 0; - fMerkleVerified = true; + // Store transaction in memory + pool.addUnchecked(hash, entry, !IsInitialBlockDownload()); } - pindexRet = pindex; - return chainActive.Height() - pindex->nHeight + 1; -} - -int CMerkleTx::GetDepthInMainChain(CBlockIndex* &pindexRet) const -{ - int nResult = GetDepthInMainChainINTERNAL(pindexRet); - if (nResult == 0 && !mempool.exists(GetHash())) - return -1; // Not in chain, not in mempool - - return nResult; -} - -int CMerkleTx::GetBlocksToMaturity() const -{ - if (!IsCoinBase()) - return 0; - return max(0, (COINBASE_MATURITY+1) - GetDepthInMainChain()); -} - + SyncWithWallets(tx, NULL); -bool CMerkleTx::AcceptToMemoryPool(bool fLimitFree) -{ - CValidationState state; - return ::AcceptToMemoryPool(mempool, state, *this, fLimitFree, NULL); + return true; } - -// Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock +/** Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock */ bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock, bool fAllowSlow) { CBlockIndex *pindexSlow = NULL; @@ -937,17 +1077,19 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock CDiskTxPos postx; if (pblocktree->ReadTxIndex(hash, postx)) { CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); + if (file.IsNull()) + return error("%s: OpenBlockFile failed", __func__); CBlockHeader header; try { file >> header; - fseek(file, postx.nTxOffset, SEEK_CUR); + fseek(file.Get(), postx.nTxOffset, SEEK_CUR); file >> txOut; - } catch (std::exception &e) { - return error("%s : Deserialize or I/O error - %s", __PRETTY_FUNCTION__, e.what()); + } catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); } hashBlock = header.GetHash(); if (txOut.GetHash() != hash) - return error("%s : txid mismatch", __PRETTY_FUNCTION__); + return error("%s: txid mismatch", __func__); return true; } } @@ -956,9 +1098,9 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock int nHeight = -1; { CCoinsViewCache &view = *pcoinsTip; - CCoins coins; - if (view.GetCoins(hash, coins)) - nHeight = coins.nHeight; + const CCoins* coins = view.AccessCoins(hash); + if (coins) + nHeight = coins->nHeight; } if (nHeight > 0) pindexSlow = chainActive[nHeight]; @@ -991,29 +1133,24 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock // CBlock and CBlockIndex // -bool WriteBlockToDisk(CBlock& block, CDiskBlockPos& pos) +bool WriteBlockToDisk(CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append - CAutoFile fileout = CAutoFile(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); - if (!fileout) - return error("WriteBlockToDisk : OpenBlockFile failed"); + CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) + return error("WriteBlockToDisk: OpenBlockFile failed"); // Write index header unsigned int nSize = fileout.GetSerializeSize(block); - fileout << FLATDATA(Params().MessageStart()) << nSize; + fileout << FLATDATA(messageStart) << nSize; // Write block - long fileOutPos = ftell(fileout); + long fileOutPos = ftell(fileout.Get()); if (fileOutPos < 0) - return error("WriteBlockToDisk : ftell failed"); + return error("WriteBlockToDisk: ftell failed"); pos.nPos = (unsigned int)fileOutPos; fileout << block; - // Flush stdio buffers and commit to disk before returning - fflush(fileout); - if (!IsInitialBlockDownload()) - FileCommit(fileout); - return true; } @@ -1022,21 +1159,21 @@ bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos) block.SetNull(); // Open history file to read - CAutoFile filein = CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION); - if (!filein) - return error("ReadBlockFromDisk : OpenBlockFile failed"); + CAutoFile filein(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) + return error("ReadBlockFromDisk: OpenBlockFile failed for %s", pos.ToString()); // Read block try { filein >> block; } - catch (std::exception &e) { - return error("%s : Deserialize or I/O error - %s", __PRETTY_FUNCTION__, e.what()); + catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s at %s", __func__, e.what(), pos.ToString()); } // Check the header - if (!CheckProofOfWork(block.GetHash(), block.nBits)) - return error("ReadBlockFromDisk : Errors in block header"); + if (!CheckProofOfWork(block.GetHash(), block.nBits, Params().GetConsensus())) + return error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString()); return true; } @@ -1046,187 +1183,40 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex) if (!ReadBlockFromDisk(block, pindex->GetBlockPos())) return false; if (block.GetHash() != pindex->GetBlockHash()) - return error("ReadBlockFromDisk(CBlock&, CBlockIndex*) : GetHash() doesn't match index"); + return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s", + pindex->ToString(), pindex->GetBlockPos().ToString()); return true; } -uint256 static GetOrphanRoot(const uint256& hash) +CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams) { - map<uint256, COrphanBlock*>::iterator it = mapOrphanBlocks.find(hash); - if (it == mapOrphanBlocks.end()) - return hash; - - // Work back to the first block in the orphan chain - do { - map<uint256, COrphanBlock*>::iterator it2 = mapOrphanBlocks.find(it->second->hashPrev); - if (it2 == mapOrphanBlocks.end()) - return it->first; - it = it2; - } while(true); -} - -// Remove a random orphan block (which does not have any dependent orphans). -void static PruneOrphanBlocks() -{ - if (mapOrphanBlocksByPrev.size() <= MAX_ORPHAN_BLOCKS) - return; - - // Pick a random orphan block. - int pos = insecure_rand() % mapOrphanBlocksByPrev.size(); - std::multimap<uint256, COrphanBlock*>::iterator it = mapOrphanBlocksByPrev.begin(); - while (pos--) it++; - - // As long as this block has other orphans depending on it, move to one of those successors. - do { - std::multimap<uint256, COrphanBlock*>::iterator it2 = mapOrphanBlocksByPrev.find(it->second->hashBlock); - if (it2 == mapOrphanBlocksByPrev.end()) - break; - it = it2; - } while(1); - - uint256 hash = it->second->hashBlock; - delete it->second; - mapOrphanBlocksByPrev.erase(it); - mapOrphanBlocks.erase(hash); -} - -int64_t GetBlockValue(int nHeight, int64_t nFees) -{ - int64_t nSubsidy = 50 * COIN; + int halvings = nHeight / consensusParams.nSubsidyHalvingInterval; + // Force block reward to zero when right shift is undefined. + if (halvings >= 64) + return 0; + CAmount nSubsidy = 50 * COIN; // Subsidy is cut in half every 210,000 blocks which will occur approximately every 4 years. - nSubsidy >>= (nHeight / Params().SubsidyHalvingInterval()); - - return nSubsidy + nFees; -} - -static const int64_t nTargetTimespan = 14 * 24 * 60 * 60; // two weeks -static const int64_t nTargetSpacing = 10 * 60; -static const int64_t nInterval = nTargetTimespan / nTargetSpacing; - -// -// minimum amount of work that could possibly be required nTime after -// minimum work required was nBase -// -unsigned int ComputeMinWork(unsigned int nBase, int64_t nTime) -{ - const CBigNum &bnLimit = Params().ProofOfWorkLimit(); - // Testnet has min-difficulty blocks - // after nTargetSpacing*2 time between blocks: - if (TestNet() && nTime > nTargetSpacing*2) - return bnLimit.GetCompact(); - - CBigNum bnResult; - bnResult.SetCompact(nBase); - while (nTime > 0 && bnResult < bnLimit) - { - // Maximum 400% adjustment... - bnResult *= 4; - // ... in best-case exactly 4-times-normal target time - nTime -= nTargetTimespan*4; - } - if (bnResult > bnLimit) - bnResult = bnLimit; - return bnResult.GetCompact(); -} - -unsigned int GetNextWorkRequired(const CBlockIndex* pindexLast, const CBlockHeader *pblock) -{ - unsigned int nProofOfWorkLimit = Params().ProofOfWorkLimit().GetCompact(); - - // Genesis block - if (pindexLast == NULL) - return nProofOfWorkLimit; - - // Only change once per interval - if ((pindexLast->nHeight+1) % nInterval != 0) - { - if (TestNet()) - { - // Special difficulty rule for testnet: - // If the new block's timestamp is more than 2* 10 minutes - // then allow mining of a min-difficulty block. - if (pblock->nTime > pindexLast->nTime + nTargetSpacing*2) - return nProofOfWorkLimit; - else - { - // Return the last non-special-min-difficulty-rules-block - const CBlockIndex* pindex = pindexLast; - while (pindex->pprev && pindex->nHeight % nInterval != 0 && pindex->nBits == nProofOfWorkLimit) - pindex = pindex->pprev; - return pindex->nBits; - } - } - return pindexLast->nBits; - } - - // Go back by what we want to be 14 days worth of blocks - const CBlockIndex* pindexFirst = pindexLast; - for (int i = 0; pindexFirst && i < nInterval-1; i++) - pindexFirst = pindexFirst->pprev; - assert(pindexFirst); - - // Limit adjustment step - int64_t nActualTimespan = pindexLast->GetBlockTime() - pindexFirst->GetBlockTime(); - LogPrintf(" nActualTimespan = %d before bounds\n", nActualTimespan); - if (nActualTimespan < nTargetTimespan/4) - nActualTimespan = nTargetTimespan/4; - if (nActualTimespan > nTargetTimespan*4) - nActualTimespan = nTargetTimespan*4; - - // Retarget - CBigNum bnNew; - bnNew.SetCompact(pindexLast->nBits); - bnNew *= nActualTimespan; - bnNew /= nTargetTimespan; - - if (bnNew > Params().ProofOfWorkLimit()) - bnNew = Params().ProofOfWorkLimit(); - - /// debug print - LogPrintf("GetNextWorkRequired RETARGET\n"); - LogPrintf("nTargetTimespan = %d nActualTimespan = %d\n", nTargetTimespan, nActualTimespan); - LogPrintf("Before: %08x %s\n", pindexLast->nBits, CBigNum().SetCompact(pindexLast->nBits).getuint256().ToString()); - LogPrintf("After: %08x %s\n", bnNew.GetCompact(), bnNew.getuint256().ToString()); - - return bnNew.GetCompact(); -} - -bool CheckProofOfWork(uint256 hash, unsigned int nBits) -{ - CBigNum bnTarget; - bnTarget.SetCompact(nBits); - - // Check range - if (bnTarget <= 0 || bnTarget > Params().ProofOfWorkLimit()) - return error("CheckProofOfWork() : nBits below minimum work"); - - // Check proof of work matches claimed amount - if (hash > bnTarget.getuint256()) - return error("CheckProofOfWork() : hash doesn't match nBits"); - - return true; -} - -// Return maximum amount of blocks that other nodes claim to have -int GetNumBlocksOfPeers() -{ - return std::max(cPeerBlockCounts.median(), Checkpoints::GetTotalBlocksEstimate()); + nSubsidy >>= halvings; + return nSubsidy; } bool IsInitialBlockDownload() { - if (fImporting || fReindex || chainActive.Height() < Checkpoints::GetTotalBlocksEstimate()) + const CChainParams& chainParams = Params(); + LOCK(cs_main); + if (fImporting || fReindex) return true; - static int64_t nLastUpdate; - static CBlockIndex* pindexLastBest; - if (chainActive.Tip() != pindexLastBest) - { - pindexLastBest = chainActive.Tip(); - nLastUpdate = GetTime(); - } - return (GetTime() - nLastUpdate < 10 && - chainActive.Tip()->GetBlockTime() < GetTime() - 24 * 60 * 60); + if (fCheckpointsEnabled && chainActive.Height() < Checkpoints::GetTotalBlocksEstimate(chainParams.Checkpoints())) + return true; + static bool lockIBDState = false; + if (lockIBDState) + return false; + bool state = (chainActive.Height() < pindexBestHeader->nHeight - 24 * 6 || + pindexBestHeader->GetBlockTime() < GetTime() - 24 * 60 * 60); + if (!state) + lockIBDState = true; + return state; } bool fLargeWorkForkFound = false; @@ -1235,6 +1225,7 @@ CBlockIndex *pindexBestForkTip = NULL, *pindexBestForkBase = NULL; void CheckForkWarningConditions() { + AssertLockHeld(cs_main); // Before we get past initial download, we cannot reliably alert about forks // (we assume we don't get stuck on a fork before the last checkpoint) if (IsInitialBlockDownload()) @@ -1245,29 +1236,24 @@ void CheckForkWarningConditions() if (pindexBestForkTip && chainActive.Height() - pindexBestForkTip->nHeight >= 72) pindexBestForkTip = NULL; - if (pindexBestForkTip || (pindexBestInvalid && pindexBestInvalid->nChainWork > chainActive.Tip()->nChainWork + (chainActive.Tip()->GetBlockWork() * 6).getuint256())) + if (pindexBestForkTip || (pindexBestInvalid && pindexBestInvalid->nChainWork > chainActive.Tip()->nChainWork + (GetBlockProof(*chainActive.Tip()) * 6))) { - if (!fLargeWorkForkFound) + if (!fLargeWorkForkFound && pindexBestForkBase) { - std::string strCmd = GetArg("-alertnotify", ""); - if (!strCmd.empty()) - { - std::string warning = std::string("'Warning: Large-work fork detected, forking after block ") + - pindexBestForkBase->phashBlock->ToString() + std::string("'"); - boost::replace_all(strCmd, "%s", warning); - boost::thread t(runCommand, strCmd); // thread runs free - } + std::string warning = std::string("'Warning: Large-work fork detected, forking after block ") + + pindexBestForkBase->phashBlock->ToString() + std::string("'"); + CAlert::Notify(warning, true); } - if (pindexBestForkTip) + if (pindexBestForkTip && pindexBestForkBase) { - LogPrintf("CheckForkWarningConditions: Warning: Large valid fork found\n forking the chain at height %d (%s)\n lasting to height %d (%s).\nChain state database corruption likely.\n", + LogPrintf("%s: Warning: Large valid fork found\n forking the chain at height %d (%s)\n lasting to height %d (%s).\nChain state database corruption likely.\n", __func__, pindexBestForkBase->nHeight, pindexBestForkBase->phashBlock->ToString(), pindexBestForkTip->nHeight, pindexBestForkTip->phashBlock->ToString()); fLargeWorkForkFound = true; } else { - LogPrintf("CheckForkWarningConditions: Warning: Found invalid chain at least ~6 blocks longer than our best chain.\nChain state database corruption likely.\n"); + LogPrintf("%s: Warning: Found invalid chain at least ~6 blocks longer than our best chain.\nChain state database corruption likely.\n", __func__); fLargeWorkInvalidChainFound = true; } } @@ -1280,6 +1266,7 @@ void CheckForkWarningConditions() void CheckForkWarningConditionsOnNewFork(CBlockIndex* pindexNewForkTip) { + AssertLockHeld(cs_main); // If we are on a fork that is sufficiently large, set a warning flag CBlockIndex* pfork = pindexNewForkTip; CBlockIndex* plonger = chainActive.Tip(); @@ -1292,15 +1279,15 @@ void CheckForkWarningConditionsOnNewFork(CBlockIndex* pindexNewForkTip) pfork = pfork->pprev; } - // We define a condition which we should warn the user about as a fork of at least 7 blocks - // who's tip is within 72 blocks (+/- 12 hours if no one mines it) of ours + // We define a condition where we should warn the user about as a fork of at least 7 blocks + // with a tip within 72 blocks (+/- 12 hours if no one mines it) of ours // We use 7 blocks rather arbitrarily as it represents just under 10% of sustained network // hash rate operating on the fork. // or a chain that is entirely longer than ours and invalid (note that this should be detected by both) // We define it this way because it allows us to only store the highest fork tip (+ base) which meets // the 7-block condition and from this always have the most-likely-to-cause-warning fork if (pfork && (!pindexBestForkTip || (pindexBestForkTip && pindexNewForkTip->nHeight > pindexBestForkTip->nHeight)) && - pindexNewForkTip->nChainWork - pfork->nChainWork > (pfork->GetBlockWork() * 7).getuint256() && + pindexNewForkTip->nChainWork - pfork->nChainWork > (GetBlockProof(*pfork) * 7) && chainActive.Height() - pindexNewForkTip->nHeight < 72) { pindexBestForkTip = pindexNewForkTip; @@ -1310,6 +1297,7 @@ void CheckForkWarningConditionsOnNewFork(CBlockIndex* pindexNewForkTip) CheckForkWarningConditions(); } +// Requires cs_main. void Misbehaving(NodeId pnode, int howmuch) { if (howmuch == 0) @@ -1320,30 +1308,25 @@ void Misbehaving(NodeId pnode, int howmuch) return; state->nMisbehavior += howmuch; - if (state->nMisbehavior >= GetArg("-banscore", 100)) + int banscore = GetArg("-banscore", 100); + if (state->nMisbehavior >= banscore && state->nMisbehavior - howmuch < banscore) { - LogPrintf("Misbehaving: %s (%d -> %d) BAN THRESHOLD EXCEEDED\n", state->name, state->nMisbehavior-howmuch, state->nMisbehavior); + LogPrintf("%s: %s (%d -> %d) BAN THRESHOLD EXCEEDED\n", __func__, state->name, state->nMisbehavior-howmuch, state->nMisbehavior); state->fShouldBan = true; } else - LogPrintf("Misbehaving: %s (%d -> %d)\n", state->name, state->nMisbehavior-howmuch, state->nMisbehavior); + LogPrintf("%s: %s (%d -> %d)\n", __func__, state->name, state->nMisbehavior-howmuch, state->nMisbehavior); } void static InvalidChainFound(CBlockIndex* pindexNew) { if (!pindexBestInvalid || pindexNew->nChainWork > pindexBestInvalid->nChainWork) - { pindexBestInvalid = pindexNew; - // The current code doesn't actually read the BestInvalidWork entry in - // the block database anymore, as it is derived from the flags in block - // index entry. We only write it for backward compatibility. - pblocktree->WriteBestInvalidWork(CBigNum(pindexBestInvalid->nChainWork)); - uiInterface.NotifyBlocksChanged(); - } - LogPrintf("InvalidChainFound: invalid block=%s height=%d log2_work=%.8g date=%s\n", + + LogPrintf("%s: invalid block=%s height=%d log2_work=%.8g date=%s\n", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, log(pindexNew->nChainWork.getdouble())/log(2.0), DateTimeStrFormat("%Y-%m-%d %H:%M:%S", pindexNew->GetBlockTime())); - LogPrintf("InvalidChainFound: current best=%s height=%d log2_work=%.8g date=%s\n", + LogPrintf("%s: current best=%s height=%d log2_work=%.8g date=%s\n", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime())); CheckForkWarningConditions(); @@ -1354,7 +1337,7 @@ void static InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state if (state.IsInvalid(nDoS)) { std::map<uint256, NodeId>::iterator it = mapBlockSource.find(pindex->GetBlockHash()); if (it != mapBlockSource.end() && State(it->second)) { - CBlockReject reject = {state.GetRejectCode(), state.GetRejectReason(), pindex->GetBlockHash()}; + CBlockReject reject = {state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), pindex->GetBlockHash()}; State(it->second)->rejects.push_back(reject); if (nDoS > 0) Misbehaving(it->second, nDoS); @@ -1362,63 +1345,54 @@ void static InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state } if (!state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; - pblocktree->WriteBlockIndex(CDiskBlockIndex(pindex)); - setBlockIndexValid.erase(pindex); + setDirtyBlockIndex.insert(pindex); + setBlockIndexCandidates.erase(pindex); InvalidChainFound(pindex); } } -void UpdateTime(CBlockHeader& block, const CBlockIndex* pindexPrev) -{ - block.nTime = max(pindexPrev->GetMedianTimePast()+1, GetAdjustedTime()); - - // Updating time can change work required on testnet: - if (TestNet()) - block.nBits = GetNextWorkRequired(pindexPrev, &block); -} - - - - - - - - - - - -void UpdateCoins(const CTransaction& tx, CValidationState &state, CCoinsViewCache &inputs, CTxUndo &txundo, int nHeight, const uint256 &txhash) +void UpdateCoins(const CTransaction& tx, CValidationState &state, CCoinsViewCache &inputs, CTxUndo &txundo, int nHeight) { - bool ret; // mark inputs spent if (!tx.IsCoinBase()) { + txundo.vprevout.reserve(tx.vin.size()); BOOST_FOREACH(const CTxIn &txin, tx.vin) { - CCoins &coins = inputs.GetCoins(txin.prevout.hash); - CTxInUndo undo; - ret = coins.Spend(txin.prevout, undo); - assert(ret); - txundo.vprevout.push_back(undo); + CCoinsModifier coins = inputs.ModifyCoins(txin.prevout.hash); + unsigned nPos = txin.prevout.n; + + if (nPos >= coins->vout.size() || coins->vout[nPos].IsNull()) + assert(false); + // mark an outpoint spent, and construct undo information + txundo.vprevout.push_back(CTxInUndo(coins->vout[nPos])); + coins->Spend(nPos); + if (coins->vout.size() == 0) { + CTxInUndo& undo = txundo.vprevout.back(); + undo.nHeight = coins->nHeight; + undo.fCoinBase = coins->fCoinBase; + undo.nVersion = coins->nVersion; + } } } // add outputs - ret = inputs.SetCoins(txhash, CCoins(tx, nHeight)); - assert(ret); + inputs.ModifyCoins(tx.GetHash())->FromTx(tx, nHeight); } -bool CScriptCheck::operator()() const { - const CScript &scriptSig = ptxTo->vin[nIn].scriptSig; - if (!VerifyScript(scriptSig, scriptPubKey, *ptxTo, nIn, nFlags, nHashType)) - return error("CScriptCheck() : %s VerifySignature failed", ptxTo->GetHash().ToString()); - return true; +void UpdateCoins(const CTransaction& tx, CValidationState &state, CCoinsViewCache &inputs, int nHeight) +{ + CTxUndo txundo; + UpdateCoins(tx, state, inputs, txundo, nHeight); } -bool VerifySignature(const CCoins& txFrom, const CTransaction& txTo, unsigned int nIn, unsigned int flags, int nHashType) -{ - return CScriptCheck(txFrom, txTo, nIn, flags, nHashType)(); +bool CScriptCheck::operator()() { + const CScript &scriptSig = ptxTo->vin[nIn].scriptSig; + if (!VerifyScript(scriptSig, scriptPubKey, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, cacheStore), &error)) { + return ::error("CScriptCheck(): %s:%d VerifySignature failed: %s", ptxTo->GetHash().ToString(), nIn, ScriptErrorString(error)); + } + return true; } -bool CheckInputs(const CTransaction& tx, CValidationState &state, CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, std::vector<CScriptCheck> *pvChecks) +bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheStore, std::vector<CScriptCheck> *pvChecks) { if (!tx.IsCoinBase()) { @@ -1428,47 +1402,49 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, CCoinsViewCach // This doesn't trigger the DoS code on purpose; if it did, it would make it easier // for an attacker to attempt to split the network. if (!inputs.HaveInputs(tx)) - return state.Invalid(error("CheckInputs() : %s inputs unavailable", tx.GetHash().ToString())); + return state.Invalid(error("CheckInputs(): %s inputs unavailable", tx.GetHash().ToString())); // While checking, GetBestBlock() refers to the parent block. // This is also true for mempool checks. CBlockIndex *pindexPrev = mapBlockIndex.find(inputs.GetBestBlock())->second; int nSpendHeight = pindexPrev->nHeight + 1; - int64_t nValueIn = 0; - int64_t nFees = 0; + CAmount nValueIn = 0; + CAmount nFees = 0; for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; - const CCoins &coins = inputs.GetCoins(prevout.hash); + const CCoins *coins = inputs.AccessCoins(prevout.hash); + assert(coins); // If prev is coinbase, check that it's matured - if (coins.IsCoinBase()) { - if (nSpendHeight - coins.nHeight < COINBASE_MATURITY) + if (coins->IsCoinBase()) { + if (nSpendHeight - coins->nHeight < COINBASE_MATURITY) return state.Invalid( - error("CheckInputs() : tried to spend coinbase at depth %d", nSpendHeight - coins.nHeight), + error("CheckInputs(): tried to spend coinbase at depth %d", nSpendHeight - coins->nHeight), REJECT_INVALID, "bad-txns-premature-spend-of-coinbase"); } // Check for negative or overflow input values - nValueIn += coins.vout[prevout.n].nValue; - if (!MoneyRange(coins.vout[prevout.n].nValue) || !MoneyRange(nValueIn)) - return state.DoS(100, error("CheckInputs() : txin values out of range"), + nValueIn += coins->vout[prevout.n].nValue; + if (!MoneyRange(coins->vout[prevout.n].nValue) || !MoneyRange(nValueIn)) + return state.DoS(100, error("CheckInputs(): txin values out of range"), REJECT_INVALID, "bad-txns-inputvalues-outofrange"); } if (nValueIn < tx.GetValueOut()) - return state.DoS(100, error("CheckInputs() : %s value in < value out", tx.GetHash().ToString()), + return state.DoS(100, error("CheckInputs(): %s value in (%s) < value out (%s)", + tx.GetHash().ToString(), FormatMoney(nValueIn), FormatMoney(tx.GetValueOut())), REJECT_INVALID, "bad-txns-in-belowout"); // Tally transaction fees - int64_t nTxFee = nValueIn - tx.GetValueOut(); + CAmount nTxFee = nValueIn - tx.GetValueOut(); if (nTxFee < 0) - return state.DoS(100, error("CheckInputs() : %s nTxFee < 0", tx.GetHash().ToString()), + return state.DoS(100, error("CheckInputs(): %s nTxFee < 0", tx.GetHash().ToString()), REJECT_INVALID, "bad-txns-fee-negative"); nFees += nTxFee; if (!MoneyRange(nFees)) - return state.DoS(100, error("CheckInputs() : nFees out of range"), + return state.DoS(100, error("CheckInputs(): nFees out of range"), REJECT_INVALID, "bad-txns-fee-outofrange"); // The first loop above does all the inexpensive checks. @@ -1481,22 +1457,35 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, CCoinsViewCach if (fScriptChecks) { for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; - const CCoins &coins = inputs.GetCoins(prevout.hash); + const CCoins* coins = inputs.AccessCoins(prevout.hash); + assert(coins); // Verify signature - CScriptCheck check(coins, tx, i, flags, 0); + CScriptCheck check(*coins, tx, i, flags, cacheStore); if (pvChecks) { pvChecks->push_back(CScriptCheck()); check.swap(pvChecks->back()); } else if (!check()) { - if (flags & SCRIPT_VERIFY_STRICTENC) { - // For now, check whether the failure was caused by non-canonical - // encodings or not; if so, don't trigger DoS protection. - CScriptCheck check(coins, tx, i, flags & (~SCRIPT_VERIFY_STRICTENC), 0); + if (flags & STANDARD_NOT_MANDATORY_VERIFY_FLAGS) { + // Check whether the failure was caused by a + // non-mandatory script verification check, such as + // non-standard DER encodings or non-null dummy + // arguments; if so, don't trigger DoS protection to + // avoid splitting the network between upgraded and + // non-upgraded nodes. + CScriptCheck check(*coins, tx, i, + flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheStore); if (check()) - return state.Invalid(false, REJECT_NONSTANDARD, "non-canonical"); + return state.Invalid(false, REJECT_NONSTANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); } - return state.DoS(100,false, REJECT_NONSTANDARD, "non-canonical"); + // Failures of other flags indicate a transaction that is + // invalid in new blocks, e.g. a invalid P2SH. We DoS ban + // such nodes as they are not following the protocol. That + // said during an upgrade careful thought should be taken + // as to the correct behavior - we may want to continue + // peering with non-upgraded nodes even after a soft-fork + // super-majority vote has passed. + return state.DoS(100,false, REJECT_INVALID, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); } } } @@ -1505,7 +1494,114 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, CCoinsViewCach return true; } +namespace { + +bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +{ + // Open history file to append + CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) + return error("%s: OpenUndoFile failed", __func__); + + // Write index header + unsigned int nSize = fileout.GetSerializeSize(blockundo); + fileout << FLATDATA(messageStart) << nSize; + + // Write undo data + long fileOutPos = ftell(fileout.Get()); + if (fileOutPos < 0) + return error("%s: ftell failed", __func__); + pos.nPos = (unsigned int)fileOutPos; + fileout << blockundo; + // calculate & write checksum + CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); + hasher << hashBlock; + hasher << blockundo; + fileout << hasher.GetHash(); + + return true; +} + +bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uint256& hashBlock) +{ + // Open history file to read + CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) + return error("%s: OpenBlockFile failed", __func__); + + // Read block + uint256 hashChecksum; + try { + filein >> blockundo; + filein >> hashChecksum; + } + catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + + // Verify checksum + CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); + hasher << hashBlock; + hasher << blockundo; + if (hashChecksum != hasher.GetHash()) + return error("%s: Checksum mismatch", __func__); + + return true; +} + +/** Abort with a message */ +bool AbortNode(const std::string& strMessage, const std::string& userMessage="") +{ + strMiscWarning = strMessage; + LogPrintf("*** %s\n", strMessage); + uiInterface.ThreadSafeMessageBox( + userMessage.empty() ? _("Error: A fatal internal error occured, see debug.log for details") : userMessage, + "", CClientUIInterface::MSG_ERROR); + StartShutdown(); + return false; +} + +bool AbortNode(CValidationState& state, const std::string& strMessage, const std::string& userMessage="") +{ + AbortNode(strMessage, userMessage); + return state.Error(strMessage); +} + +} // anon namespace + +/** + * Apply the undo operation of a CTxInUndo to the given chain state. + * @param undo The undo object. + * @param view The coins view to which to apply the changes. + * @param out The out point that corresponds to the tx input. + * @return True on success. + */ +static bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out) +{ + bool fClean = true; + + CCoinsModifier coins = view.ModifyCoins(out.hash); + if (undo.nHeight != 0) { + // undo data contains height: this is the last output of the prevout tx being spent + if (!coins->IsPruned()) + fClean = fClean && error("%s: undo data overwriting existing transaction", __func__); + coins->Clear(); + coins->fCoinBase = undo.fCoinBase; + coins->nHeight = undo.nHeight; + coins->nVersion = undo.nVersion; + } else { + if (coins->IsPruned()) + fClean = fClean && error("%s: undo data adding output to missing transaction", __func__); + } + if (coins->IsAvailable(out.n)) + fClean = fClean && error("%s: undo data overwriting existing output", __func__); + if (coins->vout.size() < out.n+1) + coins->vout.resize(out.n+1); + coins->vout[out.n] = undo.txout; + + return fClean; +} bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean) { @@ -1519,12 +1615,12 @@ bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex CBlockUndo blockUndo; CDiskBlockPos pos = pindex->GetUndoPos(); if (pos.IsNull()) - return error("DisconnectBlock() : no undo data available"); - if (!blockUndo.ReadFromDisk(pos, pindex->pprev->GetBlockHash())) - return error("DisconnectBlock() : failure reading undo data"); + return error("DisconnectBlock(): no undo data available"); + if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) + return error("DisconnectBlock(): failure reading undo data"); if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) - return error("DisconnectBlock() : block and undo data inconsistent"); + return error("DisconnectBlock(): block and undo data inconsistent"); // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { @@ -1532,54 +1628,34 @@ bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex uint256 hash = tx.GetHash(); // Check that all outputs are available and match the outputs in the block itself - // exactly. Note that transactions with only provably unspendable outputs won't - // have outputs available even in the block itself, so we handle that case - // specially with outsEmpty. - CCoins outsEmpty; - CCoins &outs = view.HaveCoins(hash) ? view.GetCoins(hash) : outsEmpty; - outs.ClearUnspendable(); - - CCoins outsBlock = CCoins(tx, pindex->nHeight); + // exactly. + { + CCoinsModifier outs = view.ModifyCoins(hash); + outs->ClearUnspendable(); + + CCoins outsBlock(tx, pindex->nHeight); // The CCoins serialization does not serialize negative numbers. // No network rules currently depend on the version here, so an inconsistency is harmless // but it must be corrected before txout nversion ever influences a network rule. if (outsBlock.nVersion < 0) - outs.nVersion = outsBlock.nVersion; - if (outs != outsBlock) - fClean = fClean && error("DisconnectBlock() : added transaction mismatch? database corrupted"); + outs->nVersion = outsBlock.nVersion; + if (*outs != outsBlock) + fClean = fClean && error("DisconnectBlock(): added transaction mismatch? database corrupted"); // remove outputs - outs = CCoins(); + outs->Clear(); + } // restore inputs if (i > 0) { // not coinbases const CTxUndo &txundo = blockUndo.vtxundo[i-1]; if (txundo.vprevout.size() != tx.vin.size()) - return error("DisconnectBlock() : transaction and undo data inconsistent"); + return error("DisconnectBlock(): transaction and undo data inconsistent"); for (unsigned int j = tx.vin.size(); j-- > 0;) { const COutPoint &out = tx.vin[j].prevout; const CTxInUndo &undo = txundo.vprevout[j]; - CCoins coins; - view.GetCoins(out.hash, coins); // this can fail if the prevout was already entirely spent - if (undo.nHeight != 0) { - // undo data contains height: this is the last output of the prevout tx being spent - if (!coins.IsPruned()) - fClean = fClean && error("DisconnectBlock() : undo data overwriting existing transaction"); - coins = CCoins(); - coins.fCoinBase = undo.fCoinBase; - coins.nHeight = undo.nHeight; - coins.nVersion = undo.nVersion; - } else { - if (coins.IsPruned()) - fClean = fClean && error("DisconnectBlock() : undo data adding output to missing transaction"); - } - if (coins.IsAvailable(out.n)) - fClean = fClean && error("DisconnectBlock() : undo data overwriting existing output"); - if (coins.vout.size() < out.n+1) - coins.vout.resize(out.n+1); - coins.vout[out.n] = undo.txout; - if (!view.SetCoins(out.hash, coins)) - return error("DisconnectBlock() : cannot restore coin inputs"); + if (!ApplyTxInUndo(undo, view, out)) + fClean = false; } } } @@ -1590,9 +1666,9 @@ bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex if (pfClean) { *pfClean = fClean; return true; - } else { - return fClean; } + + return fClean; } void static FlushBlockFile(bool fFinalize = false) @@ -1604,7 +1680,7 @@ void static FlushBlockFile(bool fFinalize = false) FILE *fileOld = OpenBlockFile(posOld); if (fileOld) { if (fFinalize) - TruncateFile(fileOld, infoLastBlockFile.nSize); + TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nSize); FileCommit(fileOld); fclose(fileOld); } @@ -1612,7 +1688,7 @@ void static FlushBlockFile(bool fFinalize = false) fileOld = OpenUndoFile(posOld); if (fileOld) { if (fFinalize) - TruncateFile(fileOld, infoLastBlockFile.nUndoSize); + TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nUndoSize); FileCommit(fileOld); fclose(fileOld); } @@ -1627,24 +1703,91 @@ void ThreadScriptCheck() { scriptcheckqueue.Thread(); } -bool ConnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, bool fJustCheck) +// +// Called periodically asynchronously; alerts if it smells like +// we're being fed a bad chain (blocks being generated much +// too slowly or too quickly). +// +void PartitionCheck(bool (*initialDownloadCheck)(), CCriticalSection& cs, const CChain& chain, int64_t nPowTargetSpacing) +{ + if (initialDownloadCheck()) return; + + static int64_t lastAlertTime = 0; + int64_t now = GetAdjustedTime(); + if (lastAlertTime > now-60*60*24) return; // Alert at most once per day + + const int SPAN_HOURS=4; + const int SPAN_SECONDS=SPAN_HOURS*60*60; + int BLOCKS_EXPECTED = SPAN_SECONDS / nPowTargetSpacing; + + boost::math::poisson_distribution<double> poisson(BLOCKS_EXPECTED); + + std::string strWarning; + int64_t startTime = GetAdjustedTime()-SPAN_SECONDS; + + LOCK(cs); + int h = chain.Height(); + while (h > 0 && chain[h]->GetBlockTime() >= startTime) + --h; + int nBlocks = chain.Height()-h; + + // How likely is it to find that many by chance? + double p = boost::math::pdf(poisson, nBlocks); + + LogPrint("partitioncheck", "%s : Found %d blocks in the last %d hours\n", __func__, nBlocks, SPAN_HOURS); + LogPrint("partitioncheck", "%s : likelihood: %g\n", __func__, p); + + // Aim for one false-positive about every fifty years of normal running: + const int FIFTY_YEARS = 50*365*24*60*60; + double alertThreshold = 1.0 / (FIFTY_YEARS / SPAN_SECONDS); + + if (p <= alertThreshold && nBlocks < BLOCKS_EXPECTED) + { + // Many fewer blocks than expected: alert! + strWarning = strprintf(_("WARNING: check your network connection, %d blocks received in the last %d hours (%d expected)"), + nBlocks, SPAN_HOURS, BLOCKS_EXPECTED); + } + else if (p <= alertThreshold && nBlocks > BLOCKS_EXPECTED) + { + // Many more blocks than expected: alert! + strWarning = strprintf(_("WARNING: abnormally high number of blocks generated, %d blocks received in the last %d hours (%d expected)"), + nBlocks, SPAN_HOURS, BLOCKS_EXPECTED); + } + if (!strWarning.empty()) + { + strMiscWarning = strWarning; + CAlert::Notify(strWarning, true); + lastAlertTime = now; + } +} + +static int64_t nTimeVerify = 0; +static int64_t nTimeConnect = 0; +static int64_t nTimeIndex = 0; +static int64_t nTimeCallbacks = 0; +static int64_t nTimeTotal = 0; + +bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, bool fJustCheck) { + const CChainParams& chainparams = Params(); + AssertLockHeld(cs_main); // Check it again in case a previous version let a bad block in if (!CheckBlock(block, state, !fJustCheck, !fJustCheck)) return false; // verify that the view's current state corresponds to the previous block - uint256 hashPrevBlock = pindex->pprev == NULL ? uint256(0) : pindex->pprev->GetBlockHash(); + uint256 hashPrevBlock = pindex->pprev == NULL ? uint256() : pindex->pprev->GetBlockHash(); assert(hashPrevBlock == view.GetBestBlock()); // Special case for the genesis block, skipping connection of its transactions // (its coinbase is unspendable) - if (block.GetHash() == Params().HashGenesisBlock()) { - view.SetBestBlock(pindex->GetBlockHash()); + if (block.GetHash() == chainparams.GetConsensus().hashGenesisBlock) { + if (!fJustCheck) + view.SetBestBlock(pindex->GetBlockHash()); return true; } - bool fScriptChecks = pindex->nHeight >= Checkpoints::GetTotalBlocksEstimate(); + bool fScriptChecks = (!fCheckpointsEnabled || pindex->nHeight >= Checkpoints::GetTotalBlocksEstimate(chainparams.Checkpoints())); // Do not allow blocks that contain transactions which 'overwrite' older transactions, // unless those are already completely spent. @@ -1654,40 +1797,45 @@ bool ConnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, C // See BIP30 and http://r6.ca/blog/20120206T005236Z.html for more information. // This logic is not necessary for memory pool transactions, as AcceptToMemoryPool // already refuses previously-known transaction ids entirely. - // This rule was originally applied all blocks whose timestamp was after March 15, 2012, 0:00 UTC. + // This rule was originally applied to all blocks with a timestamp after March 15, 2012, 0:00 UTC. // Now that the whole chain is irreversibly beyond that time it is applied to all blocks except the - // two in the chain that violate it. This prevents exploiting the issue against nodes in their + // two in the chain that violate it. This prevents exploiting the issue against nodes during their // initial block download. bool fEnforceBIP30 = (!pindex->phashBlock) || // Enforce on CreateNewBlock invocations which don't have a hash. - !((pindex->nHeight==91842 && pindex->GetBlockHash() == uint256("0x00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec")) || - (pindex->nHeight==91880 && pindex->GetBlockHash() == uint256("0x00000000000743f190a18c5577a3c2d2a1f610ae9601ac046a38084ccb7cd721"))); + !((pindex->nHeight==91842 && pindex->GetBlockHash() == uint256S("0x00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec")) || + (pindex->nHeight==91880 && pindex->GetBlockHash() == uint256S("0x00000000000743f190a18c5577a3c2d2a1f610ae9601ac046a38084ccb7cd721"))); if (fEnforceBIP30) { - for (unsigned int i = 0; i < block.vtx.size(); i++) { - uint256 hash = block.GetTxHash(i); - if (view.HaveCoins(hash) && !view.GetCoins(hash).IsPruned()) - return state.DoS(100, error("ConnectBlock() : tried to overwrite transaction"), + BOOST_FOREACH(const CTransaction& tx, block.vtx) { + const CCoins* coins = view.AccessCoins(tx.GetHash()); + if (coins && !coins->IsPruned()) + return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), REJECT_INVALID, "bad-txns-BIP30"); } } // BIP16 didn't become active until Apr 1 2012 int64_t nBIP16SwitchTime = 1333238400; - bool fStrictPayToScriptHash = (pindex->nTime >= nBIP16SwitchTime); + bool fStrictPayToScriptHash = (pindex->GetBlockTime() >= nBIP16SwitchTime); - unsigned int flags = SCRIPT_VERIFY_NOCACHE | - (fStrictPayToScriptHash ? SCRIPT_VERIFY_P2SH : SCRIPT_VERIFY_NONE); + unsigned int flags = fStrictPayToScriptHash ? SCRIPT_VERIFY_P2SH : SCRIPT_VERIFY_NONE; + + // Start enforcing the DERSIG (BIP66) rules, for block.nVersion=3 blocks, when 75% of the network has upgraded: + if (block.nVersion >= 3 && IsSuperMajority(3, pindex->pprev, chainparams.GetConsensus().nMajorityEnforceBlockUpgrade, chainparams.GetConsensus())) { + flags |= SCRIPT_VERIFY_DERSIG; + } CBlockUndo blockundo; CCheckQueueControl<CScriptCheck> control(fScriptChecks && nScriptCheckThreads ? &scriptcheckqueue : NULL); - int64_t nStart = GetTimeMicros(); - int64_t nFees = 0; + int64_t nTimeStart = GetTimeMicros(); + CAmount nFees = 0; int nInputs = 0; unsigned int nSigOps = 0; CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); std::vector<std::pair<uint256, CDiskTxPos> > vPos; vPos.reserve(block.vtx.size()); + blockundo.vtxundo.reserve(block.vtx.size() - 1); for (unsigned int i = 0; i < block.vtx.size(); i++) { const CTransaction &tx = block.vtx[i]; @@ -1695,13 +1843,13 @@ bool ConnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, C nInputs += tx.vin.size(); nSigOps += GetLegacySigOpCount(tx); if (nSigOps > MAX_BLOCK_SIGOPS) - return state.DoS(100, error("ConnectBlock() : too many sigops"), + return state.DoS(100, error("ConnectBlock(): too many sigops"), REJECT_INVALID, "bad-blk-sigops"); if (!tx.IsCoinBase()) { if (!view.HaveInputs(tx)) - return state.DoS(100, error("ConnectBlock() : inputs missing/spent"), + return state.DoS(100, error("ConnectBlock(): inputs missing/spent"), REJECT_INVALID, "bad-txns-inputs-missingorspent"); if (fStrictPayToScriptHash) @@ -1711,122 +1859,224 @@ bool ConnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, C // an incredibly-expensive-to-validate block. nSigOps += GetP2SHSigOpCount(tx, view); if (nSigOps > MAX_BLOCK_SIGOPS) - return state.DoS(100, error("ConnectBlock() : too many sigops"), + return state.DoS(100, error("ConnectBlock(): too many sigops"), REJECT_INVALID, "bad-blk-sigops"); } nFees += view.GetValueIn(tx)-tx.GetValueOut(); std::vector<CScriptCheck> vChecks; - if (!CheckInputs(tx, state, view, fScriptChecks, flags, nScriptCheckThreads ? &vChecks : NULL)) + if (!CheckInputs(tx, state, view, fScriptChecks, flags, false, nScriptCheckThreads ? &vChecks : NULL)) return false; control.Add(vChecks); } - CTxUndo txundo; - UpdateCoins(tx, state, view, txundo, pindex->nHeight, block.GetTxHash(i)); - if (!tx.IsCoinBase()) - blockundo.vtxundo.push_back(txundo); + CTxUndo undoDummy; + if (i > 0) { + blockundo.vtxundo.push_back(CTxUndo()); + } + UpdateCoins(tx, state, view, i == 0 ? undoDummy : blockundo.vtxundo.back(), pindex->nHeight); - vPos.push_back(std::make_pair(block.GetTxHash(i), pos)); + vPos.push_back(std::make_pair(tx.GetHash(), pos)); pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION); } - int64_t nTime = GetTimeMicros() - nStart; - if (fBenchmark) - LogPrintf("- Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin)\n", (unsigned)block.vtx.size(), 0.001 * nTime, 0.001 * nTime / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * nTime / (nInputs-1)); + int64_t nTime1 = GetTimeMicros(); nTimeConnect += nTime1 - nTimeStart; + LogPrint("bench", " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs]\n", (unsigned)block.vtx.size(), 0.001 * (nTime1 - nTimeStart), 0.001 * (nTime1 - nTimeStart) / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * (nTime1 - nTimeStart) / (nInputs-1), nTimeConnect * 0.000001); - if (block.vtx[0].GetValueOut() > GetBlockValue(pindex->nHeight, nFees)) + CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, chainparams.GetConsensus()); + if (block.vtx[0].GetValueOut() > blockReward) return state.DoS(100, - error("ConnectBlock() : coinbase pays too much (actual=%d vs limit=%d)", - block.vtx[0].GetValueOut(), GetBlockValue(pindex->nHeight, nFees)), - REJECT_INVALID, "bad-cb-amount"); + error("ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)", + block.vtx[0].GetValueOut(), blockReward), + REJECT_INVALID, "bad-cb-amount"); if (!control.Wait()) return state.DoS(100, false); - int64_t nTime2 = GetTimeMicros() - nStart; - if (fBenchmark) - LogPrintf("- Verify %u txins: %.2fms (%.3fms/txin)\n", nInputs - 1, 0.001 * nTime2, nInputs <= 1 ? 0 : 0.001 * nTime2 / (nInputs-1)); + int64_t nTime2 = GetTimeMicros(); nTimeVerify += nTime2 - nTimeStart; + LogPrint("bench", " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs]\n", nInputs - 1, 0.001 * (nTime2 - nTimeStart), nInputs <= 1 ? 0 : 0.001 * (nTime2 - nTimeStart) / (nInputs-1), nTimeVerify * 0.000001); if (fJustCheck) return true; // Write undo information to disk - if (pindex->GetUndoPos().IsNull() || (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) + if (pindex->GetUndoPos().IsNull() || !pindex->IsValid(BLOCK_VALID_SCRIPTS)) { if (pindex->GetUndoPos().IsNull()) { CDiskBlockPos pos; if (!FindUndoPos(state, pindex->nFile, pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 40)) - return error("ConnectBlock() : FindUndoPos failed"); - if (!blockundo.WriteToDisk(pos, pindex->pprev->GetBlockHash())) - return state.Abort(_("Failed to write undo data")); + return error("ConnectBlock(): FindUndoPos failed"); + if (!UndoWriteToDisk(blockundo, pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) + return AbortNode(state, "Failed to write undo data"); // update nUndoPos in block index pindex->nUndoPos = pos.nPos; pindex->nStatus |= BLOCK_HAVE_UNDO; } - pindex->nStatus = (pindex->nStatus & ~BLOCK_VALID_MASK) | BLOCK_VALID_SCRIPTS; - - CDiskBlockIndex blockindex(pindex); - if (!pblocktree->WriteBlockIndex(blockindex)) - return state.Abort(_("Failed to write block index")); + pindex->RaiseValidity(BLOCK_VALID_SCRIPTS); + setDirtyBlockIndex.insert(pindex); } if (fTxIndex) if (!pblocktree->WriteTxIndex(vPos)) - return state.Abort(_("Failed to write transaction index")); + return AbortNode(state, "Failed to write transaction index"); // add this block to the view's block chain - bool ret; - ret = view.SetBestBlock(pindex->GetBlockHash()); - assert(ret); + view.SetBestBlock(pindex->GetBlockHash()); - // Watch for transactions paying to me - for (unsigned int i = 0; i < block.vtx.size(); i++) - g_signals.SyncTransaction(block.GetTxHash(i), block.vtx[i], &block); + int64_t nTime3 = GetTimeMicros(); nTimeIndex += nTime3 - nTime2; + LogPrint("bench", " - Index writing: %.2fms [%.2fs]\n", 0.001 * (nTime3 - nTime2), nTimeIndex * 0.000001); + + // Watch for changes to the previous coinbase transaction. + static uint256 hashPrevBestCoinBase; + GetMainSignals().UpdatedTransaction(hashPrevBestCoinBase); + hashPrevBestCoinBase = block.vtx[0].GetHash(); + + int64_t nTime4 = GetTimeMicros(); nTimeCallbacks += nTime4 - nTime3; + LogPrint("bench", " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime4 - nTime3), nTimeCallbacks * 0.000001); return true; } -// Update the on-disk chain state. -bool static WriteChainState(CValidationState &state) { +enum FlushStateMode { + FLUSH_STATE_NONE, + FLUSH_STATE_IF_NEEDED, + FLUSH_STATE_PERIODIC, + FLUSH_STATE_ALWAYS +}; + +/** + * Update the on-disk chain state. + * The caches and indexes are flushed depending on the mode we're called with + * if they're too large, if it's been a while since the last write, + * or always and in all cases if we're in prune mode and are deleting files. + */ +bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { + LOCK2(cs_main, cs_LastBlockFile); static int64_t nLastWrite = 0; - if (!IsInitialBlockDownload() || pcoinsTip->GetCacheSize() > nCoinCacheSize || GetTimeMicros() > nLastWrite + 600*1000000) { - // Typical CCoins structures on disk are around 100 bytes in size. + static int64_t nLastFlush = 0; + static int64_t nLastSetChain = 0; + std::set<int> setFilesToPrune; + bool fFlushForPrune = false; + try { + if (fPruneMode && fCheckForPruning) { + FindFilesToPrune(setFilesToPrune); + fCheckForPruning = false; + if (!setFilesToPrune.empty()) { + fFlushForPrune = true; + if (!fHavePruned) { + pblocktree->WriteFlag("prunedblockfiles", true); + fHavePruned = true; + } + } + } + int64_t nNow = GetTimeMicros(); + // Avoid writing/flushing immediately after startup. + if (nLastWrite == 0) { + nLastWrite = nNow; + } + if (nLastFlush == 0) { + nLastFlush = nNow; + } + if (nLastSetChain == 0) { + nLastSetChain = nNow; + } + size_t cacheSize = pcoinsTip->DynamicMemoryUsage(); + // The cache is large and close to the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize * (10.0/9) > nCoinCacheUsage; + // The cache is over the limit, we have to write now. + bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nCoinCacheUsage; + // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. + bool fPeriodicWrite = mode == FLUSH_STATE_PERIODIC && nNow > nLastWrite + (int64_t)DATABASE_WRITE_INTERVAL * 1000000; + // It's been very long since we flushed the cache. Do this infrequently, to optimize cache usage. + bool fPeriodicFlush = mode == FLUSH_STATE_PERIODIC && nNow > nLastFlush + (int64_t)DATABASE_FLUSH_INTERVAL * 1000000; + // Combine all conditions that result in a full cache flush. + bool fDoFullFlush = (mode == FLUSH_STATE_ALWAYS) || fCacheLarge || fCacheCritical || fPeriodicFlush || fFlushForPrune; + // Write blocks and block index to disk. + if (fDoFullFlush || fPeriodicWrite) { + // Depend on nMinDiskSpace to ensure we can write block index + if (!CheckDiskSpace(0)) + return state.Error("out of disk space"); + // First make sure all block and undo data is flushed to disk. + FlushBlockFile(); + // Then update all block file information (which may refer to block and undo files). + { + std::vector<std::pair<int, const CBlockFileInfo*> > vFiles; + vFiles.reserve(setDirtyFileInfo.size()); + for (set<int>::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end(); ) { + vFiles.push_back(make_pair(*it, &vinfoBlockFile[*it])); + setDirtyFileInfo.erase(it++); + } + std::vector<const CBlockIndex*> vBlocks; + vBlocks.reserve(setDirtyBlockIndex.size()); + for (set<CBlockIndex*>::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) { + vBlocks.push_back(*it); + setDirtyBlockIndex.erase(it++); + } + if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { + return AbortNode(state, "Files to write to block index database"); + } + } + // Finally remove any pruned files + if (fFlushForPrune) + UnlinkPrunedFiles(setFilesToPrune); + nLastWrite = nNow; + } + // Flush best chain related state. This can only be done if the blocks / block index write was also done. + if (fDoFullFlush) { + // Typical CCoins structures on disk are around 128 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(100 * 2 * 2 * pcoinsTip->GetCacheSize())) + if (!CheckDiskSpace(128 * 2 * 2 * pcoinsTip->GetCacheSize())) return state.Error("out of disk space"); - FlushBlockFile(); - pblocktree->Sync(); + // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) - return state.Abort(_("Failed to write to coin database")); - nLastWrite = GetTimeMicros(); + return AbortNode(state, "Failed to write to coin database"); + nLastFlush = nNow; + } + if ((mode == FLUSH_STATE_ALWAYS || mode == FLUSH_STATE_PERIODIC) && nNow > nLastSetChain + (int64_t)DATABASE_WRITE_INTERVAL * 1000000) { + // Update best block in wallet (so we can detect restored wallets). + GetMainSignals().SetBestChain(chainActive.GetLocator()); + nLastSetChain = nNow; + } + } catch (const std::runtime_error& e) { + return AbortNode(state, std::string("System error while flushing: ") + e.what()); } return true; } -// Update chainActive and related internal data structures. +void FlushStateToDisk() { + CValidationState state; + FlushStateToDisk(state, FLUSH_STATE_ALWAYS); +} + +void PruneAndFlush() { + CValidationState state; + fCheckForPruning = true; + FlushStateToDisk(state, FLUSH_STATE_NONE); +} + +/** Update chainActive and related internal data structures. */ void static UpdateTip(CBlockIndex *pindexNew) { + const CChainParams& chainParams = Params(); chainActive.SetTip(pindexNew); - // Update best block in wallet (so we can detect restored wallets) - bool fIsInitialDownload = IsInitialBlockDownload(); - if ((chainActive.Height() % 20160) == 0 || (!fIsInitialDownload && (chainActive.Height() % 144) == 0)) - g_signals.SetBestChain(chainActive.GetLocator()); - // New best block nTimeBestReceived = GetTime(); mempool.AddTransactionsUpdated(1); - LogPrintf("UpdateTip: new best=%s height=%d log2_work=%.8g tx=%lu date=%s progress=%f\n", + + LogPrintf("%s: new best=%s height=%d log2_work=%.8g tx=%lu date=%s progress=%f cache=%.1fMiB(%utx)\n", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), - Checkpoints::GuessVerificationProgress(chainActive.Tip())); + Checkpoints::GuessVerificationProgress(chainParams.Checkpoints(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); + + cvBlockChange.notify_all(); // Check the version of the last 100 blocks to see if we need to upgrade: - if (!fIsInitialDownload) + static bool fWarned = false; + if (!IsInitialBlockDownload() && !fWarned) { int nUpgraded = 0; const CBlockIndex* pindex = chainActive.Tip(); @@ -1837,14 +2087,18 @@ void static UpdateTip(CBlockIndex *pindexNew) { pindex = pindex->pprev; } if (nUpgraded > 0) - LogPrintf("SetBestChain: %d of last 100 blocks above version %d\n", nUpgraded, (int)CBlock::CURRENT_VERSION); + LogPrintf("%s: %d of last 100 blocks above version %d\n", __func__, nUpgraded, (int)CBlock::CURRENT_VERSION); if (nUpgraded > 100/2) + { // strMiscWarning is read by GetWarnings(), called by Qt and the JSON-RPC code to warn the user: - strMiscWarning = _("Warning: This version is obsolete, upgrade required!"); + strMiscWarning = _("Warning: This version is obsolete; upgrade required!"); + CAlert::Notify(strMiscWarning, true); + fWarned = true; + } } } -// Disconnect chainActive's tip. +/** Disconnect chainActive's tip. */ bool static DisconnectTip(CValidationState &state) { CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); @@ -1852,105 +2106,121 @@ bool static DisconnectTip(CValidationState &state) { // Read block from disk. CBlock block; if (!ReadBlockFromDisk(block, pindexDelete)) - return state.Abort(_("Failed to read block")); + return AbortNode(state, "Failed to read block"); // Apply the block atomically to the chain state. int64_t nStart = GetTimeMicros(); { - CCoinsViewCache view(*pcoinsTip, true); + CCoinsViewCache view(pcoinsTip); if (!DisconnectBlock(block, state, pindexDelete, view)) - return error("DisconnectTip() : DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); + return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); assert(view.Flush()); } - if (fBenchmark) - LogPrintf("- Disconnect: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); + LogPrint("bench", "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); // Write the chain state to disk, if necessary. - if (!WriteChainState(state)) + if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; // Resurrect mempool transactions from the disconnected block. BOOST_FOREACH(const CTransaction &tx, block.vtx) { // ignore validation errors in resurrected transactions list<CTransaction> removed; - CValidationState stateDummy; - if (!tx.IsCoinBase()) - if (!AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL)) - mempool.remove(tx, removed, true); + CValidationState stateDummy; + if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL)) + mempool.remove(tx, removed, true); } + mempool.removeCoinbaseSpends(pcoinsTip, pindexDelete->nHeight); mempool.check(pcoinsTip); // Update chainActive and related variables. UpdateTip(pindexDelete->pprev); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: BOOST_FOREACH(const CTransaction &tx, block.vtx) { - SyncWithWallets(tx.GetHash(), tx, NULL); + SyncWithWallets(tx, NULL); } return true; } -// Connect a new block to chainActive. -bool static ConnectTip(CValidationState &state, CBlockIndex *pindexNew) { +static int64_t nTimeReadFromDisk = 0; +static int64_t nTimeConnectTotal = 0; +static int64_t nTimeFlush = 0; +static int64_t nTimeChainState = 0; +static int64_t nTimePostConnect = 0; + +/** + * Connect a new block to chainActive. pblock is either NULL or a pointer to a CBlock + * corresponding to pindexNew, to bypass loading it again from disk. + */ +bool static ConnectTip(CValidationState &state, CBlockIndex *pindexNew, CBlock *pblock) { assert(pindexNew->pprev == chainActive.Tip()); mempool.check(pcoinsTip); // Read block from disk. + int64_t nTime1 = GetTimeMicros(); CBlock block; - if (!ReadBlockFromDisk(block, pindexNew)) - return state.Abort(_("Failed to read block")); + if (!pblock) { + if (!ReadBlockFromDisk(block, pindexNew)) + return AbortNode(state, "Failed to read block"); + pblock = █ + } // Apply the block atomically to the chain state. - int64_t nStart = GetTimeMicros(); + int64_t nTime2 = GetTimeMicros(); nTimeReadFromDisk += nTime2 - nTime1; + int64_t nTime3; + LogPrint("bench", " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); { - CCoinsViewCache view(*pcoinsTip, true); + CCoinsViewCache view(pcoinsTip); CInv inv(MSG_BLOCK, pindexNew->GetBlockHash()); - if (!ConnectBlock(block, state, pindexNew, view)) { + bool rv = ConnectBlock(*pblock, state, pindexNew, view); + GetMainSignals().BlockChecked(*pblock, state); + if (!rv) { if (state.IsInvalid()) InvalidBlockFound(pindexNew, state); - return error("ConnectTip() : ConnectBlock %s failed", pindexNew->GetBlockHash().ToString()); + return error("ConnectTip(): ConnectBlock %s failed", pindexNew->GetBlockHash().ToString()); } mapBlockSource.erase(inv.hash); + nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; + LogPrint("bench", " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); assert(view.Flush()); } - if (fBenchmark) - LogPrintf("- Connect: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); + int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; + LogPrint("bench", " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); // Write the chain state to disk, if necessary. - if (!WriteChainState(state)) + if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; + int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; + LogPrint("bench", " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool. list<CTransaction> txConflicted; - BOOST_FOREACH(const CTransaction &tx, block.vtx) { - list<CTransaction> unused; - mempool.remove(tx, unused); - mempool.removeConflicts(tx, txConflicted); - } + mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, txConflicted, !IsInitialBlockDownload()); mempool.check(pcoinsTip); // Update chainActive & related variables. UpdateTip(pindexNew); // Tell wallet about transactions that went from mempool // to conflicted: BOOST_FOREACH(const CTransaction &tx, txConflicted) { - SyncWithWallets(tx.GetHash(), tx, NULL); + SyncWithWallets(tx, NULL); } // ... and about transactions that got confirmed: - BOOST_FOREACH(const CTransaction &tx, block.vtx) { - SyncWithWallets(tx.GetHash(), tx, &block); + BOOST_FOREACH(const CTransaction &tx, pblock->vtx) { + SyncWithWallets(tx, pblock); } + + int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; + LogPrint("bench", " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); + LogPrint("bench", "- Connect block: %.2fms [%.2fs]\n", (nTime6 - nTime1) * 0.001, nTimeTotal * 0.000001); return true; } -// Make chainMostWork correspond to the chain with the most work in it, that isn't -// known to be invalid (it's however far from certain to be valid). -void static FindMostWorkChain() { - CBlockIndex *pindexNew = NULL; - - // In case the current best is invalid, do not consider it. - while (chainMostWork.Tip() && (chainMostWork.Tip()->nStatus & BLOCK_FAILED_MASK)) { - setBlockIndexValid.erase(chainMostWork.Tip()); - chainMostWork.SetTip(chainMostWork.Tip()->pprev); - } - +/** + * Return the tip of the chain with the most work in it, that isn't + * known to be invalid (it's however far from certain to be valid). + */ +static CBlockIndex* FindMostWorkChain() { do { + CBlockIndex *pindexNew = NULL; + // Find the best candidate header. { - std::set<CBlockIndex*, CBlockIndexWorkComparator>::reverse_iterator it = setBlockIndexValid.rbegin(); - if (it == setBlockIndexValid.rend()) - return; + std::set<CBlockIndex*, CBlockIndexWorkComparator>::reverse_iterator it = setBlockIndexCandidates.rbegin(); + if (it == setBlockIndexCandidates.rend()) + return NULL; pindexNew = *it; } @@ -1959,173 +2229,363 @@ void static FindMostWorkChain() { CBlockIndex *pindexTest = pindexNew; bool fInvalidAncestor = false; while (pindexTest && !chainActive.Contains(pindexTest)) { - if (pindexTest->nStatus & BLOCK_FAILED_MASK) { - // Candidate has an invalid ancestor, remove entire chain from the set. - if (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork) - pindexBestInvalid = pindexNew; CBlockIndex *pindexFailed = pindexNew; + assert(pindexTest->nChainTx || pindexTest->nHeight == 0); + + // Pruned nodes may have entries in setBlockIndexCandidates for + // which block files have been deleted. Remove those as candidates + // for the most work chain if we come across them; we can't switch + // to a chain unless we have all the non-active-chain parent blocks. + bool fFailedChain = pindexTest->nStatus & BLOCK_FAILED_MASK; + bool fMissingData = !(pindexTest->nStatus & BLOCK_HAVE_DATA); + if (fFailedChain || fMissingData) { + // Candidate chain is not usable (either invalid or missing data) + if (fFailedChain && (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork)) + pindexBestInvalid = pindexNew; + CBlockIndex *pindexFailed = pindexNew; + // Remove the entire chain from the set. while (pindexTest != pindexFailed) { - pindexFailed->nStatus |= BLOCK_FAILED_CHILD; - setBlockIndexValid.erase(pindexFailed); + if (fFailedChain) { + pindexFailed->nStatus |= BLOCK_FAILED_CHILD; + } else if (fMissingData) { + // If we're missing data, then add back to mapBlocksUnlinked, + // so that if the block arrives in the future we can try adding + // to setBlockIndexCandidates again. + mapBlocksUnlinked.insert(std::make_pair(pindexFailed->pprev, pindexFailed)); + } + setBlockIndexCandidates.erase(pindexFailed); pindexFailed = pindexFailed->pprev; } + setBlockIndexCandidates.erase(pindexTest); fInvalidAncestor = true; break; } pindexTest = pindexTest->pprev; } - if (fInvalidAncestor) - continue; - - break; + if (!fInvalidAncestor) + return pindexNew; } while(true); +} - // Check whether it's actually an improvement. - if (chainMostWork.Tip() && !CBlockIndexWorkComparator()(chainMostWork.Tip(), pindexNew)) - return; - - // We have a new best. - chainMostWork.SetTip(pindexNew); +/** Delete all entries in setBlockIndexCandidates that are worse than the current tip. */ +static void PruneBlockIndexCandidates() { + // Note that we can't delete the current block itself, as we may need to return to it later in case a + // reorganization to a better block fails. + std::set<CBlockIndex*, CBlockIndexWorkComparator>::iterator it = setBlockIndexCandidates.begin(); + while (it != setBlockIndexCandidates.end() && setBlockIndexCandidates.value_comp()(*it, chainActive.Tip())) { + setBlockIndexCandidates.erase(it++); + } + // Either the current tip or a successor of it we're working towards is left in setBlockIndexCandidates. + assert(!setBlockIndexCandidates.empty()); } -// Try to activate to the most-work chain (thereby connecting it). -bool ActivateBestChain(CValidationState &state) { - CBlockIndex *pindexOldTip = chainActive.Tip(); - bool fComplete = false; - while (!fComplete) { - FindMostWorkChain(); - fComplete = true; +/** + * Try to make some progress towards making pindexMostWork the active block. + * pblock is either NULL or a pointer to a CBlock corresponding to pindexMostWork. + */ +static bool ActivateBestChainStep(CValidationState &state, CBlockIndex *pindexMostWork, CBlock *pblock) { + AssertLockHeld(cs_main); + bool fInvalidFound = false; + const CBlockIndex *pindexOldTip = chainActive.Tip(); + const CBlockIndex *pindexFork = chainActive.FindFork(pindexMostWork); - // Check whether we have something to do. - if (chainMostWork.Tip() == NULL) break; + // Disconnect active blocks which are no longer in the best chain. + while (chainActive.Tip() && chainActive.Tip() != pindexFork) { + if (!DisconnectTip(state)) + return false; + } - // Disconnect active blocks which are no longer in the best chain. - while (chainActive.Tip() && !chainMostWork.Contains(chainActive.Tip())) { - if (!DisconnectTip(state)) + // Build list of new blocks to connect. + std::vector<CBlockIndex*> vpindexToConnect; + bool fContinue = true; + int nHeight = pindexFork ? pindexFork->nHeight : -1; + while (fContinue && nHeight != pindexMostWork->nHeight) { + // Don't iterate the entire list of potential improvements toward the best tip, as we likely only need + // a few blocks along the way. + int nTargetHeight = std::min(nHeight + 32, pindexMostWork->nHeight); + vpindexToConnect.clear(); + vpindexToConnect.reserve(nTargetHeight - nHeight); + CBlockIndex *pindexIter = pindexMostWork->GetAncestor(nTargetHeight); + while (pindexIter && pindexIter->nHeight != nHeight) { + vpindexToConnect.push_back(pindexIter); + pindexIter = pindexIter->pprev; + } + nHeight = nTargetHeight; + + // Connect new blocks. + BOOST_REVERSE_FOREACH(CBlockIndex *pindexConnect, vpindexToConnect) { + if (!ConnectTip(state, pindexConnect, pindexConnect == pindexMostWork ? pblock : NULL)) { + if (state.IsInvalid()) { + // The block violates a consensus rule. + if (!state.CorruptionPossible()) + InvalidChainFound(vpindexToConnect.back()); + state = CValidationState(); + fInvalidFound = true; + fContinue = false; + break; + } else { + // A system error occurred (disk space, database error, ...). return false; + } + } else { + PruneBlockIndexCandidates(); + if (!pindexOldTip || chainActive.Tip()->nChainWork > pindexOldTip->nChainWork) { + // We're in a better position than we were. Return temporarily to release the lock. + fContinue = false; + break; + } } + } + } - // Connect new blocks. - while (!chainActive.Contains(chainMostWork.Tip())) { - CBlockIndex *pindexConnect = chainMostWork[chainActive.Height() + 1]; - if (!ConnectTip(state, pindexConnect)) { - if (state.IsInvalid()) { - // The block violates a consensus rule. - if (!state.CorruptionPossible()) - InvalidChainFound(chainMostWork.Tip()); - fComplete = false; - state = CValidationState(); - break; - } else { - // A system error occurred (disk space, database error, ...). - return false; - } + // Callbacks/notifications for a new best chain. + if (fInvalidFound) + CheckForkWarningConditionsOnNewFork(vpindexToConnect.back()); + else + CheckForkWarningConditions(); + + return true; +} + +/** + * Make the best chain active, in multiple steps. The result is either failure + * or an activated best chain. pblock is either NULL or a pointer to a block + * that is already loaded (to avoid loading it again from disk). + */ +bool ActivateBestChain(CValidationState &state, CBlock *pblock) { + CBlockIndex *pindexNewTip = NULL; + CBlockIndex *pindexMostWork = NULL; + const CChainParams& chainParams = Params(); + do { + boost::this_thread::interruption_point(); + + bool fInitialDownload; + { + LOCK(cs_main); + pindexMostWork = FindMostWorkChain(); + + // Whether we have anything to do at all. + if (pindexMostWork == NULL || pindexMostWork == chainActive.Tip()) + return true; + + if (!ActivateBestChainStep(state, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : NULL)) + return false; + + pindexNewTip = chainActive.Tip(); + fInitialDownload = IsInitialBlockDownload(); + } + // When we reach this point, we switched to a new tip (stored in pindexNewTip). + + // Notifications/callbacks that can run without cs_main + if (!fInitialDownload) { + uint256 hashNewTip = pindexNewTip->GetBlockHash(); + // Relay inventory, but don't relay old inventory during initial block download. + int nBlockEstimate = 0; + if (fCheckpointsEnabled) + nBlockEstimate = Checkpoints::GetTotalBlocksEstimate(chainParams.Checkpoints()); + // Don't relay blocks if pruning -- could cause a peer to try to download, resulting + // in a stalled download if the block file is pruned before the request. + if (nLocalServices & NODE_NETWORK) { + LOCK(cs_vNodes); + BOOST_FOREACH(CNode* pnode, vNodes) + if (chainActive.Height() > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate)) + pnode->PushInventory(CInv(MSG_BLOCK, hashNewTip)); } + // Notify external listeners about the new tip. + uiInterface.NotifyBlockTip(hashNewTip); } + } while(pindexMostWork != chainActive.Tip()); + CheckBlockIndex(); + + // Write changes periodically to disk, after relay. + if (!FlushStateToDisk(state, FLUSH_STATE_PERIODIC)) { + return false; } - if (chainActive.Tip() != pindexOldTip) { - std::string strCmd = GetArg("-blocknotify", ""); - if (!IsInitialBlockDownload() && !strCmd.empty()) - { - boost::replace_all(strCmd, "%s", chainActive.Tip()->GetBlockHash().GetHex()); - boost::thread t(runCommand, strCmd); // thread runs free + return true; +} + +bool InvalidateBlock(CValidationState& state, CBlockIndex *pindex) { + AssertLockHeld(cs_main); + + // Mark the block itself as invalid. + pindex->nStatus |= BLOCK_FAILED_VALID; + setDirtyBlockIndex.insert(pindex); + setBlockIndexCandidates.erase(pindex); + + while (chainActive.Contains(pindex)) { + CBlockIndex *pindexWalk = chainActive.Tip(); + pindexWalk->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(pindexWalk); + setBlockIndexCandidates.erase(pindexWalk); + // ActivateBestChain considers blocks already in chainActive + // unconditionally valid already, so force disconnect away from it. + if (!DisconnectTip(state)) { + return false; + } + } + + // The resulting new best tip may not be in setBlockIndexCandidates anymore, so + // add it again. + BlockMap::iterator it = mapBlockIndex.begin(); + while (it != mapBlockIndex.end()) { + if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->nChainTx && !setBlockIndexCandidates.value_comp()(it->second, chainActive.Tip())) { + setBlockIndexCandidates.insert(it->second); + } + it++; + } + + InvalidChainFound(pindex); + return true; +} + +bool ReconsiderBlock(CValidationState& state, CBlockIndex *pindex) { + AssertLockHeld(cs_main); + + int nHeight = pindex->nHeight; + + // Remove the invalidity flag from this block and all its descendants. + BlockMap::iterator it = mapBlockIndex.begin(); + while (it != mapBlockIndex.end()) { + if (!it->second->IsValid() && it->second->GetAncestor(nHeight) == pindex) { + it->second->nStatus &= ~BLOCK_FAILED_MASK; + setDirtyBlockIndex.insert(it->second); + if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->nChainTx && setBlockIndexCandidates.value_comp()(chainActive.Tip(), it->second)) { + setBlockIndexCandidates.insert(it->second); + } + if (it->second == pindexBestInvalid) { + // Reset invalid block marker if it was pointing to one of those. + pindexBestInvalid = NULL; + } } + it++; } + // Remove the invalidity flag from all ancestors too. + while (pindex != NULL) { + if (pindex->nStatus & BLOCK_FAILED_MASK) { + pindex->nStatus &= ~BLOCK_FAILED_MASK; + setDirtyBlockIndex.insert(pindex); + } + pindex = pindex->pprev; + } return true; } -bool AddToBlockIndex(CBlock& block, CValidationState& state, const CDiskBlockPos& pos) +CBlockIndex* AddToBlockIndex(const CBlockHeader& block) { // Check for duplicate uint256 hash = block.GetHash(); - if (mapBlockIndex.count(hash)) - return state.Invalid(error("AddToBlockIndex() : %s already exists", hash.ToString()), 0, "duplicate"); + BlockMap::iterator it = mapBlockIndex.find(hash); + if (it != mapBlockIndex.end()) + return it->second; // Construct new block index object CBlockIndex* pindexNew = new CBlockIndex(block); - { - LOCK(cs_nBlockSequenceId); - pindexNew->nSequenceId = nBlockSequenceId++; - } assert(pindexNew); - mapAlreadyAskedFor.erase(CInv(MSG_BLOCK, hash)); - map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.insert(make_pair(hash, pindexNew)).first; + // We assign the sequence id to blocks only when the full data is available, + // to avoid miners withholding blocks but broadcasting headers, to get a + // competitive advantage. + pindexNew->nSequenceId = 0; + BlockMap::iterator mi = mapBlockIndex.insert(make_pair(hash, pindexNew)).first; pindexNew->phashBlock = &((*mi).first); - map<uint256, CBlockIndex*>::iterator miPrev = mapBlockIndex.find(block.hashPrevBlock); + BlockMap::iterator miPrev = mapBlockIndex.find(block.hashPrevBlock); if (miPrev != mapBlockIndex.end()) { pindexNew->pprev = (*miPrev).second; pindexNew->nHeight = pindexNew->pprev->nHeight + 1; + pindexNew->BuildSkip(); } + pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew); + pindexNew->RaiseValidity(BLOCK_VALID_TREE); + if (pindexBestHeader == NULL || pindexBestHeader->nChainWork < pindexNew->nChainWork) + pindexBestHeader = pindexNew; + + setDirtyBlockIndex.insert(pindexNew); + + return pindexNew; +} + +/** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ +bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos) +{ pindexNew->nTx = block.vtx.size(); - pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + pindexNew->GetBlockWork().getuint256(); - pindexNew->nChainTx = (pindexNew->pprev ? pindexNew->pprev->nChainTx : 0) + pindexNew->nTx; + pindexNew->nChainTx = 0; pindexNew->nFile = pos.nFile; pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; - pindexNew->nStatus = BLOCK_VALID_TRANSACTIONS | BLOCK_HAVE_DATA; - setBlockIndexValid.insert(pindexNew); - - if (!pblocktree->WriteBlockIndex(CDiskBlockIndex(pindexNew))) - return state.Abort(_("Failed to write block index")); - - // New best? - if (!ActivateBestChain(state)) - return false; - - if (pindexNew == chainActive.Tip()) - { - // Clear fork warning if its no longer applicable - CheckForkWarningConditions(); - // Notify UI to display prev block's coinbase if it was ours - static uint256 hashPrevBestCoinBase; - g_signals.UpdatedTransaction(hashPrevBestCoinBase); - hashPrevBestCoinBase = block.GetTxHash(0); - } else - CheckForkWarningConditionsOnNewFork(pindexNew); - - if (!pblocktree->Flush()) - return state.Abort(_("Failed to sync block index")); + pindexNew->nStatus |= BLOCK_HAVE_DATA; + pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); + setDirtyBlockIndex.insert(pindexNew); + + if (pindexNew->pprev == NULL || pindexNew->pprev->nChainTx) { + // If pindexNew is the genesis block or all parents are BLOCK_VALID_TRANSACTIONS. + deque<CBlockIndex*> queue; + queue.push_back(pindexNew); + + // Recursively process any descendant blocks that now may be eligible to be connected. + while (!queue.empty()) { + CBlockIndex *pindex = queue.front(); + queue.pop_front(); + pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; + { + LOCK(cs_nBlockSequenceId); + pindex->nSequenceId = nBlockSequenceId++; + } + if (chainActive.Tip() == NULL || !setBlockIndexCandidates.value_comp()(pindex, chainActive.Tip())) { + setBlockIndexCandidates.insert(pindex); + } + std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> range = mapBlocksUnlinked.equal_range(pindex); + while (range.first != range.second) { + std::multimap<CBlockIndex*, CBlockIndex*>::iterator it = range.first; + queue.push_back(it->second); + range.first++; + mapBlocksUnlinked.erase(it); + } + } + } else { + if (pindexNew->pprev && pindexNew->pprev->IsValid(BLOCK_VALID_TREE)) { + mapBlocksUnlinked.insert(std::make_pair(pindexNew->pprev, pindexNew)); + } + } - uiInterface.NotifyBlocksChanged(); return true; } - bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) { - bool fUpdatedLast = false; - LOCK(cs_LastBlockFile); - if (fKnown) { - if (nLastBlockFile != pos.nFile) { - nLastBlockFile = pos.nFile; - infoLastBlockFile.SetNull(); - pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); - fUpdatedLast = true; - } - } else { - while (infoLastBlockFile.nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { - LogPrintf("Leaving block file %i: %s\n", nLastBlockFile, infoLastBlockFile.ToString()); + unsigned int nFile = fKnown ? pos.nFile : nLastBlockFile; + if (vinfoBlockFile.size() <= nFile) { + vinfoBlockFile.resize(nFile + 1); + } + + if (!fKnown) { + while (vinfoBlockFile[nFile].nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { + LogPrintf("Leaving block file %i: %s\n", nFile, vinfoBlockFile[nFile].ToString()); FlushBlockFile(true); - nLastBlockFile++; - infoLastBlockFile.SetNull(); - pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine - fUpdatedLast = true; + nFile++; + if (vinfoBlockFile.size() <= nFile) { + vinfoBlockFile.resize(nFile + 1); + } } - pos.nFile = nLastBlockFile; - pos.nPos = infoLastBlockFile.nSize; + pos.nFile = nFile; + pos.nPos = vinfoBlockFile[nFile].nSize; } - infoLastBlockFile.nSize += nAddSize; - infoLastBlockFile.AddBlock(nHeight, nTime); + nLastBlockFile = nFile; + vinfoBlockFile[nFile].AddBlock(nHeight, nTime); + if (fKnown) + vinfoBlockFile[nFile].nSize = std::max(pos.nPos + nAddSize, vinfoBlockFile[nFile].nSize); + else + vinfoBlockFile[nFile].nSize += nAddSize; if (!fKnown) { unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - unsigned int nNewChunks = (infoLastBlockFile.nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; + unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; if (nNewChunks > nOldChunks) { + if (fPruneMode) + fCheckForPruning = true; if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos)) { FILE *file = OpenBlockFile(pos); if (file) { @@ -2139,11 +2599,7 @@ bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAdd } } - if (!pblocktree->WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) - return state.Abort(_("Failed to write file info")); - if (fUpdatedLast) - pblocktree->WriteLastBlockFile(nLastBlockFile); - + setDirtyFileInfo.insert(nFile); return true; } @@ -2154,24 +2610,15 @@ bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigne LOCK(cs_LastBlockFile); unsigned int nNewSize; - if (nFile == nLastBlockFile) { - pos.nPos = infoLastBlockFile.nUndoSize; - nNewSize = (infoLastBlockFile.nUndoSize += nAddSize); - if (!pblocktree->WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) - return state.Abort(_("Failed to write block info")); - } else { - CBlockFileInfo info; - if (!pblocktree->ReadBlockFileInfo(nFile, info)) - return state.Abort(_("Failed to read block info")); - pos.nPos = info.nUndoSize; - nNewSize = (info.nUndoSize += nAddSize); - if (!pblocktree->WriteBlockFileInfo(nFile, info)) - return state.Abort(_("Failed to write block info")); - } + pos.nPos = vinfoBlockFile[nFile].nUndoSize; + nNewSize = vinfoBlockFile[nFile].nUndoSize += nAddSize; + setDirtyFileInfo.insert(nFile); unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; if (nNewChunks > nOldChunks) { + if (fPruneMode) + fCheckForPruning = true; if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos)) { FILE *file = OpenUndoFile(pos); if (file) { @@ -2187,55 +2634,68 @@ bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigne return true; } - -bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bool fCheckMerkleRoot) +bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, bool fCheckPOW) { - // These are checks that are independent of context - // that can be verified before saving an orphan block. - - // Size limits - if (block.vtx.empty() || block.vtx.size() > MAX_BLOCK_SIZE || ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION) > MAX_BLOCK_SIZE) - return state.DoS(100, error("CheckBlock() : size limits failed"), - REJECT_INVALID, "bad-blk-length"); - // Check proof of work matches claimed amount - if (fCheckPOW && !CheckProofOfWork(block.GetHash(), block.nBits)) - return state.DoS(50, error("CheckBlock() : proof of work failed"), + if (fCheckPOW && !CheckProofOfWork(block.GetHash(), block.nBits, Params().GetConsensus())) + return state.DoS(50, error("CheckBlockHeader(): proof of work failed"), REJECT_INVALID, "high-hash"); // Check timestamp if (block.GetBlockTime() > GetAdjustedTime() + 2 * 60 * 60) - return state.Invalid(error("CheckBlock() : block timestamp too far in the future"), + return state.Invalid(error("CheckBlockHeader(): block timestamp too far in the future"), REJECT_INVALID, "time-too-new"); + return true; +} + +bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bool fCheckMerkleRoot) +{ + // These are checks that are independent of context. + + // Check that the header is valid (particularly PoW). This is mostly + // redundant with the call in AcceptBlockHeader. + if (!CheckBlockHeader(block, state, fCheckPOW)) + return false; + + // Check the merkle root. + if (fCheckMerkleRoot) { + bool mutated; + uint256 hashMerkleRoot2 = block.BuildMerkleTree(&mutated); + if (block.hashMerkleRoot != hashMerkleRoot2) + return state.DoS(100, error("CheckBlock(): hashMerkleRoot mismatch"), + REJECT_INVALID, "bad-txnmrklroot", true); + + // Check for merkle tree malleability (CVE-2012-2459): repeating sequences + // of transactions in a block without affecting the merkle root of a block, + // while still invalidating it. + if (mutated) + return state.DoS(100, error("CheckBlock(): duplicate transaction"), + REJECT_INVALID, "bad-txns-duplicate", true); + } + + // All potential-corruption validation must be done before we do any + // transaction validation, as otherwise we may mark the header as invalid + // because we receive the wrong transactions for it. + + // Size limits + if (block.vtx.empty() || block.vtx.size() > MAX_BLOCK_SIZE || ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION) > MAX_BLOCK_SIZE) + return state.DoS(100, error("CheckBlock(): size limits failed"), + REJECT_INVALID, "bad-blk-length"); + // First transaction must be coinbase, the rest must not be if (block.vtx.empty() || !block.vtx[0].IsCoinBase()) - return state.DoS(100, error("CheckBlock() : first tx is not coinbase"), + return state.DoS(100, error("CheckBlock(): first tx is not coinbase"), REJECT_INVALID, "bad-cb-missing"); for (unsigned int i = 1; i < block.vtx.size(); i++) if (block.vtx[i].IsCoinBase()) - return state.DoS(100, error("CheckBlock() : more than one coinbase"), + return state.DoS(100, error("CheckBlock(): more than one coinbase"), REJECT_INVALID, "bad-cb-multiple"); // Check transactions BOOST_FOREACH(const CTransaction& tx, block.vtx) if (!CheckTransaction(tx, state)) - return error("CheckBlock() : CheckTransaction failed"); - - // Build the merkle tree already. We need it anyway later, and it makes the - // block cache the transaction hashes, which means they don't need to be - // recalculated many times during this block's validation. - block.BuildMerkleTree(); - - // Check for duplicate txids. This is caught by ConnectInputs(), - // but catching it earlier avoids a potential DoS attack: - set<uint256> uniqueTx; - for (unsigned int i = 0; i < block.vtx.size(); i++) { - uniqueTx.insert(block.GetTxHash(i)); - } - if (uniqueTx.size() != block.vtx.size()) - return state.DoS(100, error("CheckBlock() : duplicate transaction"), - REJECT_INVALID, "bad-txns-duplicate", true); + return error("CheckBlock(): CheckTransaction failed"); unsigned int nSigOps = 0; BOOST_FOREACH(const CTransaction& tx, block.vtx) @@ -2243,120 +2703,185 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo nSigOps += GetLegacySigOpCount(tx); } if (nSigOps > MAX_BLOCK_SIGOPS) - return state.DoS(100, error("CheckBlock() : out-of-bounds SigOpCount"), + return state.DoS(100, error("CheckBlock(): out-of-bounds SigOpCount"), REJECT_INVALID, "bad-blk-sigops", true); - // Check merkle root - if (fCheckMerkleRoot && block.hashMerkleRoot != block.vMerkleTree.back()) - return state.DoS(100, error("CheckBlock() : hashMerkleRoot mismatch"), - REJECT_INVALID, "bad-txnmrklroot", true); + return true; +} + +bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex * const pindexPrev) +{ + const CChainParams& chainParams = Params(); + const Consensus::Params& consensusParams = chainParams.GetConsensus(); + uint256 hash = block.GetHash(); + if (hash == consensusParams.hashGenesisBlock) + return true; + + assert(pindexPrev); + + int nHeight = pindexPrev->nHeight+1; + + // Check proof of work + if (block.nBits != GetNextWorkRequired(pindexPrev, &block, consensusParams)) + return state.DoS(100, error("%s: incorrect proof of work", __func__), + REJECT_INVALID, "bad-diffbits"); + + // Check timestamp against prev + if (block.GetBlockTime() <= pindexPrev->GetMedianTimePast()) + return state.Invalid(error("%s: block's timestamp is too early", __func__), + REJECT_INVALID, "time-too-old"); + + if(fCheckpointsEnabled) + { + // Check that the block chain matches the known block chain up to a checkpoint + if (!Checkpoints::CheckBlock(chainParams.Checkpoints(), nHeight, hash)) + return state.DoS(100, error("%s: rejected by checkpoint lock-in at %d", __func__, nHeight), + REJECT_CHECKPOINT, "checkpoint mismatch"); + + // Don't accept any forks from the main chain prior to last checkpoint + CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(chainParams.Checkpoints()); + if (pcheckpoint && nHeight < pcheckpoint->nHeight) + return state.DoS(100, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight)); + } + + // Reject block.nVersion=1 blocks when 95% (75% on testnet) of the network has upgraded: + if (block.nVersion < 2 && IsSuperMajority(2, pindexPrev, consensusParams.nMajorityRejectBlockOutdated, consensusParams)) + return state.Invalid(error("%s: rejected nVersion=1 block", __func__), + REJECT_OBSOLETE, "bad-version"); + + // Reject block.nVersion=2 blocks when 95% (75% on testnet) of the network has upgraded: + if (block.nVersion < 3 && IsSuperMajority(3, pindexPrev, consensusParams.nMajorityRejectBlockOutdated, consensusParams)) + return state.Invalid(error("%s : rejected nVersion=2 block", __func__), + REJECT_OBSOLETE, "bad-version"); return true; } -bool AcceptBlock(CBlock& block, CValidationState& state, CDiskBlockPos* dbp) +bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIndex * const pindexPrev) { + const int nHeight = pindexPrev == NULL ? 0 : pindexPrev->nHeight + 1; + const Consensus::Params& consensusParams = Params().GetConsensus(); + + // Check that all transactions are finalized + BOOST_FOREACH(const CTransaction& tx, block.vtx) + if (!IsFinalTx(tx, nHeight, block.GetBlockTime())) { + return state.DoS(10, error("%s: contains a non-final transaction", __func__), REJECT_INVALID, "bad-txns-nonfinal"); + } + + // Enforce block.nVersion=2 rule that the coinbase starts with serialized block height + // if 750 of the last 1,000 blocks are version 2 or greater (51/100 if testnet): + if (block.nVersion >= 2 && IsSuperMajority(2, pindexPrev, consensusParams.nMajorityEnforceBlockUpgrade, consensusParams)) + { + CScript expect = CScript() << nHeight; + if (block.vtx[0].vin[0].scriptSig.size() < expect.size() || + !std::equal(expect.begin(), expect.end(), block.vtx[0].vin[0].scriptSig.begin())) { + return state.DoS(100, error("%s: block height mismatch in coinbase", __func__), REJECT_INVALID, "bad-cb-height"); + } + } + + return true; +} + +bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex** ppindex) +{ + const CChainParams& chainparams = Params(); + AssertLockHeld(cs_main); // Check for duplicate uint256 hash = block.GetHash(); - if (mapBlockIndex.count(hash)) - return state.Invalid(error("AcceptBlock() : block already in mapBlockIndex"), 0, "duplicate"); + BlockMap::iterator miSelf = mapBlockIndex.find(hash); + CBlockIndex *pindex = NULL; + if (miSelf != mapBlockIndex.end()) { + // Block header is already known. + pindex = miSelf->second; + if (ppindex) + *ppindex = pindex; + if (pindex->nStatus & BLOCK_FAILED_MASK) + return state.Invalid(error("%s: block is marked invalid", __func__), 0, "duplicate"); + return true; + } + + if (!CheckBlockHeader(block, state)) + return false; // Get prev block index CBlockIndex* pindexPrev = NULL; - int nHeight = 0; - if (hash != Params().HashGenesisBlock()) { - map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(block.hashPrevBlock); + if (hash != chainparams.GetConsensus().hashGenesisBlock) { + BlockMap::iterator mi = mapBlockIndex.find(block.hashPrevBlock); if (mi == mapBlockIndex.end()) - return state.DoS(10, error("AcceptBlock() : prev block not found"), 0, "bad-prevblk"); + return state.DoS(10, error("%s: prev block not found", __func__), 0, "bad-prevblk"); pindexPrev = (*mi).second; - nHeight = pindexPrev->nHeight+1; + if (pindexPrev->nStatus & BLOCK_FAILED_MASK) + return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); + } - // Check proof of work - if (block.nBits != GetNextWorkRequired(pindexPrev, &block)) - return state.DoS(100, error("AcceptBlock() : incorrect proof of work"), - REJECT_INVALID, "bad-diffbits"); + if (!ContextualCheckBlockHeader(block, state, pindexPrev)) + return false; - // Check timestamp against prev - if (block.GetBlockTime() <= pindexPrev->GetMedianTimePast()) - return state.Invalid(error("AcceptBlock() : block's timestamp is too early"), - REJECT_INVALID, "time-too-old"); + if (pindex == NULL) + pindex = AddToBlockIndex(block); - // Check that all transactions are finalized - BOOST_FOREACH(const CTransaction& tx, block.vtx) - if (!IsFinalTx(tx, nHeight, block.GetBlockTime())) - return state.DoS(10, error("AcceptBlock() : contains a non-final transaction"), - REJECT_INVALID, "bad-txns-nonfinal"); + if (ppindex) + *ppindex = pindex; - // Check that the block chain matches the known block chain up to a checkpoint - if (!Checkpoints::CheckBlock(nHeight, hash)) - return state.DoS(100, error("AcceptBlock() : rejected by checkpoint lock-in at %d", nHeight), - REJECT_CHECKPOINT, "checkpoint mismatch"); + return true; +} - // Don't accept any forks from the main chain prior to last checkpoint - CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(mapBlockIndex); - if (pcheckpoint && nHeight < pcheckpoint->nHeight) - return state.DoS(100, error("AcceptBlock() : forked chain older than last checkpoint (height %d)", nHeight)); +bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex, CDiskBlockPos* dbp) +{ + const CChainParams& chainparams = Params(); + AssertLockHeld(cs_main); - // Reject block.nVersion=1 blocks when 95% (75% on testnet) of the network has upgraded: - if (block.nVersion < 2) - { - if ((!TestNet() && CBlockIndex::IsSuperMajority(2, pindexPrev, 950, 1000)) || - (TestNet() && CBlockIndex::IsSuperMajority(2, pindexPrev, 75, 100))) - { - return state.Invalid(error("AcceptBlock() : rejected nVersion=1 block"), - REJECT_OBSOLETE, "bad-version"); - } - } - // Enforce block.nVersion=2 rule that the coinbase starts with serialized block height - if (block.nVersion >= 2) - { - // if 750 of the last 1,000 blocks are version 2 or greater (51/100 if testnet): - if ((!TestNet() && CBlockIndex::IsSuperMajority(2, pindexPrev, 750, 1000)) || - (TestNet() && CBlockIndex::IsSuperMajority(2, pindexPrev, 51, 100))) - { - CScript expect = CScript() << nHeight; - if (block.vtx[0].vin[0].scriptSig.size() < expect.size() || - !std::equal(expect.begin(), expect.end(), block.vtx[0].vin[0].scriptSig.begin())) - return state.DoS(100, error("AcceptBlock() : block height mismatch in coinbase"), - REJECT_INVALID, "bad-cb-height"); - } + CBlockIndex *&pindex = *ppindex; + + if (!AcceptBlockHeader(block, state, &pindex)) + return false; + + // If we're pruning, ensure that we don't allow a peer to dump a copy + // of old blocks. But we might need blocks that are not on the main chain + // to handle a reorg, even if we've processed once. + if (pindex->nStatus & BLOCK_HAVE_DATA || chainActive.Contains(pindex)) { + // TODO: deal better with duplicate blocks. + // return state.DoS(20, error("AcceptBlock(): already have block %d %s", pindex->nHeight, pindex->GetBlockHash().ToString()), REJECT_DUPLICATE, "duplicate"); + return true; + } + + if ((!CheckBlock(block, state)) || !ContextualCheckBlock(block, state, pindex->pprev)) { + if (state.IsInvalid() && !state.CorruptionPossible()) { + pindex->nStatus |= BLOCK_FAILED_VALID; + setDirtyBlockIndex.insert(pindex); } + return false; } + int nHeight = pindex->nHeight; + // Write block to history file try { unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); CDiskBlockPos blockPos; if (dbp != NULL) blockPos = *dbp; - if (!FindBlockPos(state, blockPos, nBlockSize+8, nHeight, block.nTime, dbp != NULL)) - return error("AcceptBlock() : FindBlockPos failed"); + if (!FindBlockPos(state, blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != NULL)) + return error("AcceptBlock(): FindBlockPos failed"); if (dbp == NULL) - if (!WriteBlockToDisk(block, blockPos)) - return state.Abort(_("Failed to write block")); - if (!AddToBlockIndex(block, state, blockPos)) - return error("AcceptBlock() : AddToBlockIndex failed"); - } catch(std::runtime_error &e) { - return state.Abort(_("System error: ") + e.what()); + if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) + AbortNode(state, "Failed to write block"); + if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) + return error("AcceptBlock(): ReceivedBlockTransactions failed"); + } catch (const std::runtime_error& e) { + return AbortNode(state, std::string("System error: ") + e.what()); } - // Relay inventory, but don't relay old inventory during initial block download - int nBlockEstimate = Checkpoints::GetTotalBlocksEstimate(); - if (chainActive.Tip()->GetBlockHash() == hash) - { - LOCK(cs_vNodes); - BOOST_FOREACH(CNode* pnode, vNodes) - if (chainActive.Height() > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate)) - pnode->PushInventory(CInv(MSG_BLOCK, hash)); - } + if (fCheckForPruning) + FlushStateToDisk(state, FLUSH_STATE_NONE); // we just allocated more disk space for block files return true; } -bool CBlockIndex::IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned int nRequired, unsigned int nToCheck) +static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned nRequired, const Consensus::Params& consensusParams) { unsigned int nFound = 0; - for (unsigned int i = 0; i < nToCheck && nFound < nRequired && pstart != NULL; i++) + for (int i = 0; i < consensusParams.nMajorityWindow && nFound < nRequired && pstart != NULL; i++) { if (pstart->nVersion >= minVersion) ++nFound; @@ -2365,303 +2890,172 @@ bool CBlockIndex::IsSuperMajority(int minVersion, const CBlockIndex* pstart, uns return (nFound >= nRequired); } -int64_t CBlockIndex::GetMedianTime() const -{ - const CBlockIndex* pindex = this; - for (int i = 0; i < nMedianTimeSpan/2; i++) - { - if (!chainActive.Next(pindex)) - return GetBlockTime(); - pindex = chainActive.Next(pindex); - } - return pindex->GetMedianTimePast(); -} -void PushGetBlocks(CNode* pnode, CBlockIndex* pindexBegin, uint256 hashEnd) +bool ProcessNewBlock(CValidationState &state, CNode* pfrom, CBlock* pblock, CDiskBlockPos *dbp) { - // Filter out duplicate requests - if (pindexBegin == pnode->pindexLastGetBlocksBegin && hashEnd == pnode->hashLastGetBlocksEnd) - return; - pnode->pindexLastGetBlocksBegin = pindexBegin; - pnode->hashLastGetBlocksEnd = hashEnd; - - pnode->PushMessage("getblocks", chainActive.GetLocator(pindexBegin), hashEnd); -} - -bool ProcessBlock(CValidationState &state, CNode* pfrom, CBlock* pblock, CDiskBlockPos *dbp) -{ - AssertLockHeld(cs_main); - - // Check for duplicate - uint256 hash = pblock->GetHash(); - if (mapBlockIndex.count(hash)) - return state.Invalid(error("ProcessBlock() : already have block %d %s", mapBlockIndex[hash]->nHeight, hash.ToString()), 0, "duplicate"); - if (mapOrphanBlocks.count(hash)) - return state.Invalid(error("ProcessBlock() : already have block (orphan) %s", hash.ToString()), 0, "duplicate"); - // Preliminary checks - if (!CheckBlock(*pblock, state)) { - if (state.CorruptionPossible()) - mapAlreadyAskedFor.erase(CInv(MSG_BLOCK, hash)); - return error("ProcessBlock() : CheckBlock FAILED"); - } + bool checked = CheckBlock(*pblock, state); - CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(mapBlockIndex); - if (pcheckpoint && pblock->hashPrevBlock != (chainActive.Tip() ? chainActive.Tip()->GetBlockHash() : uint256(0))) { - // Extra checks to prevent "fill up memory by spamming with bogus blocks" - int64_t deltaTime = pblock->GetBlockTime() - pcheckpoint->nTime; - if (deltaTime < 0) - { - return state.DoS(100, error("ProcessBlock() : block with timestamp before last checkpoint"), - REJECT_CHECKPOINT, "time-too-old"); - } - CBigNum bnNewBlock; - bnNewBlock.SetCompact(pblock->nBits); - CBigNum bnRequired; - bnRequired.SetCompact(ComputeMinWork(pcheckpoint->nBits, deltaTime)); - if (bnNewBlock > bnRequired) - { - return state.DoS(100, error("ProcessBlock() : block with too little proof-of-work"), - REJECT_INVALID, "bad-diffbits"); + LOCK(cs_main); + MarkBlockAsReceived(pblock->GetHash()); + if (!checked) { + return error("%s: CheckBlock FAILED", __func__); } - } - - // If we don't already have its previous block, shunt it off to holding area until we get it - if (pblock->hashPrevBlock != 0 && !mapBlockIndex.count(pblock->hashPrevBlock)) - { - LogPrintf("ProcessBlock: ORPHAN BLOCK %lu, prev=%s\n", (unsigned long)mapOrphanBlocks.size(), pblock->hashPrevBlock.ToString()); - - // Accept orphans as long as there is a node to request its parents from - if (pfrom) { - PruneOrphanBlocks(); - COrphanBlock* pblock2 = new COrphanBlock(); - { - CDataStream ss(SER_DISK, CLIENT_VERSION); - ss << *pblock; - pblock2->vchBlock = std::vector<unsigned char>(ss.begin(), ss.end()); - } - pblock2->hashBlock = hash; - pblock2->hashPrev = pblock->hashPrevBlock; - mapOrphanBlocks.insert(make_pair(hash, pblock2)); - mapOrphanBlocksByPrev.insert(make_pair(pblock2->hashPrev, pblock2)); - - // Ask this guy to fill in what we're missing - PushGetBlocks(pfrom, chainActive.Tip(), GetOrphanRoot(hash)); + // Store to disk + CBlockIndex *pindex = NULL; + bool ret = AcceptBlock(*pblock, state, &pindex, dbp); + if (pindex && pfrom) { + mapBlockSource[pindex->GetBlockHash()] = pfrom->GetId(); } - return true; + CheckBlockIndex(); + if (!ret) + return error("%s: AcceptBlock FAILED", __func__); } - // Store to disk - if (!AcceptBlock(*pblock, state, dbp)) - return error("ProcessBlock() : AcceptBlock FAILED"); + if (!ActivateBestChain(state, pblock)) + return error("%s: ActivateBestChain failed", __func__); - // Recursively process any orphan blocks that depended on this one - vector<uint256> vWorkQueue; - vWorkQueue.push_back(hash); - for (unsigned int i = 0; i < vWorkQueue.size(); i++) - { - uint256 hashPrev = vWorkQueue[i]; - for (multimap<uint256, COrphanBlock*>::iterator mi = mapOrphanBlocksByPrev.lower_bound(hashPrev); - mi != mapOrphanBlocksByPrev.upper_bound(hashPrev); - ++mi) - { - CBlock block; - { - CDataStream ss(mi->second->vchBlock, SER_DISK, CLIENT_VERSION); - ss >> block; - } - block.BuildMerkleTree(); - // Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan resolution (that is, feeding people an invalid block based on LegitBlockX in order to get anyone relaying LegitBlockX banned) - CValidationState stateDummy; - if (AcceptBlock(block, stateDummy)) - vWorkQueue.push_back(mi->second->hashBlock); - mapOrphanBlocks.erase(mi->second->hashBlock); - delete mi->second; - } - mapOrphanBlocksByPrev.erase(hashPrev); - } - - LogPrintf("ProcessBlock: ACCEPTED\n"); return true; } +bool TestBlockValidity(CValidationState &state, const CBlock& block, CBlockIndex * const pindexPrev, bool fCheckPOW, bool fCheckMerkleRoot) +{ + AssertLockHeld(cs_main); + assert(pindexPrev == chainActive.Tip()); + CCoinsViewCache viewNew(pcoinsTip); + CBlockIndex indexDummy(block); + indexDummy.pprev = pindexPrev; + indexDummy.nHeight = pindexPrev->nHeight + 1; + // NOTE: CheckBlockHeader is called by CheckBlock + if (!ContextualCheckBlockHeader(block, state, pindexPrev)) + return false; + if (!CheckBlock(block, state, fCheckPOW, fCheckMerkleRoot)) + return false; + if (!ContextualCheckBlock(block, state, pindexPrev)) + return false; + if (!ConnectBlock(block, state, &indexDummy, viewNew, true)) + return false; + assert(state.IsValid()); + return true; +} +/** + * BLOCK PRUNING CODE + */ - - -CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter& filter) +/* Calculate the amount of disk space the block & undo files currently use */ +uint64_t CalculateCurrentUsage() { - header = block.GetBlockHeader(); - - vector<bool> vMatch; - vector<uint256> vHashes; - - vMatch.reserve(block.vtx.size()); - vHashes.reserve(block.vtx.size()); + uint64_t retval = 0; + BOOST_FOREACH(const CBlockFileInfo &file, vinfoBlockFile) { + retval += file.nSize + file.nUndoSize; + } + return retval; +} - for (unsigned int i = 0; i < block.vtx.size(); i++) - { - uint256 hash = block.vtx[i].GetHash(); - if (filter.IsRelevantAndUpdate(block.vtx[i], hash)) - { - vMatch.push_back(true); - vMatchedTxn.push_back(make_pair(i, hash)); +/* Prune a block file (modify associated database entries)*/ +void PruneOneBlockFile(const int fileNumber) +{ + for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); ++it) { + CBlockIndex* pindex = it->second; + if (pindex->nFile == fileNumber) { + pindex->nStatus &= ~BLOCK_HAVE_DATA; + pindex->nStatus &= ~BLOCK_HAVE_UNDO; + pindex->nFile = 0; + pindex->nDataPos = 0; + pindex->nUndoPos = 0; + setDirtyBlockIndex.insert(pindex); + + // Prune from mapBlocksUnlinked -- any block we prune would have + // to be downloaded again in order to consider its chain, at which + // point it would be considered as a candidate for + // mapBlocksUnlinked or setBlockIndexCandidates. + std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> range = mapBlocksUnlinked.equal_range(pindex->pprev); + while (range.first != range.second) { + std::multimap<CBlockIndex *, CBlockIndex *>::iterator it = range.first; + range.first++; + if (it->second == pindex) { + mapBlocksUnlinked.erase(it); + } + } } - else - vMatch.push_back(false); - vHashes.push_back(hash); } - txn = CPartialMerkleTree(vHashes, vMatch); + vinfoBlockFile[fileNumber].SetNull(); + setDirtyFileInfo.insert(fileNumber); } - - - - - - -uint256 CPartialMerkleTree::CalcHash(int height, unsigned int pos, const std::vector<uint256> &vTxid) { - if (height == 0) { - // hash at height 0 is the txids themself - return vTxid[pos]; - } else { - // calculate left hash - uint256 left = CalcHash(height-1, pos*2, vTxid), right; - // calculate right hash if not beyong the end of the array - copy left hash otherwise1 - if (pos*2+1 < CalcTreeWidth(height-1)) - right = CalcHash(height-1, pos*2+1, vTxid); - else - right = left; - // combine subhashes - return Hash(BEGIN(left), END(left), BEGIN(right), END(right)); - } -} - -void CPartialMerkleTree::TraverseAndBuild(int height, unsigned int pos, const std::vector<uint256> &vTxid, const std::vector<bool> &vMatch) { - // determine whether this node is the parent of at least one matched txid - bool fParentOfMatch = false; - for (unsigned int p = pos << height; p < (pos+1) << height && p < nTransactions; p++) - fParentOfMatch |= vMatch[p]; - // store as flag bit - vBits.push_back(fParentOfMatch); - if (height==0 || !fParentOfMatch) { - // if at height 0, or nothing interesting below, store hash and stop - vHash.push_back(CalcHash(height, pos, vTxid)); - } else { - // otherwise, don't store any hash, but descend into the subtrees - TraverseAndBuild(height-1, pos*2, vTxid, vMatch); - if (pos*2+1 < CalcTreeWidth(height-1)) - TraverseAndBuild(height-1, pos*2+1, vTxid, vMatch); +void UnlinkPrunedFiles(std::set<int>& setFilesToPrune) +{ + for (set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { + CDiskBlockPos pos(*it, 0); + boost::filesystem::remove(GetBlockPosFilename(pos, "blk")); + boost::filesystem::remove(GetBlockPosFilename(pos, "rev")); + LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); } } -uint256 CPartialMerkleTree::TraverseAndExtract(int height, unsigned int pos, unsigned int &nBitsUsed, unsigned int &nHashUsed, std::vector<uint256> &vMatch) { - if (nBitsUsed >= vBits.size()) { - // overflowed the bits array - failure - fBad = true; - return 0; +/* Calculate the block/rev files that should be deleted to remain under target*/ +void FindFilesToPrune(std::set<int>& setFilesToPrune) +{ + LOCK2(cs_main, cs_LastBlockFile); + if (chainActive.Tip() == NULL || nPruneTarget == 0) { + return; } - bool fParentOfMatch = vBits[nBitsUsed++]; - if (height==0 || !fParentOfMatch) { - // if at height 0, or nothing interesting below, use stored hash and do not descend - if (nHashUsed >= vHash.size()) { - // overflowed the hash array - failure - fBad = true; - return 0; - } - const uint256 &hash = vHash[nHashUsed++]; - if (height==0 && fParentOfMatch) // in case of height 0, we have a matched txid - vMatch.push_back(hash); - return hash; - } else { - // otherwise, descend into the subtrees to extract matched txids and hashes - uint256 left = TraverseAndExtract(height-1, pos*2, nBitsUsed, nHashUsed, vMatch), right; - if (pos*2+1 < CalcTreeWidth(height-1)) - right = TraverseAndExtract(height-1, pos*2+1, nBitsUsed, nHashUsed, vMatch); - else - right = left; - // and combine them before returning - return Hash(BEGIN(left), END(left), BEGIN(right), END(right)); + if (chainActive.Tip()->nHeight <= Params().PruneAfterHeight()) { + return; } -} - -CPartialMerkleTree::CPartialMerkleTree(const std::vector<uint256> &vTxid, const std::vector<bool> &vMatch) : nTransactions(vTxid.size()), fBad(false) { - // reset state - vBits.clear(); - vHash.clear(); - - // calculate height of tree - int nHeight = 0; - while (CalcTreeWidth(nHeight) > 1) - nHeight++; - - // traverse the partial tree - TraverseAndBuild(nHeight, 0, vTxid, vMatch); -} - -CPartialMerkleTree::CPartialMerkleTree() : nTransactions(0), fBad(true) {} - -uint256 CPartialMerkleTree::ExtractMatches(std::vector<uint256> &vMatch) { - vMatch.clear(); - // An empty set will not work - if (nTransactions == 0) - return 0; - // check for excessively high numbers of transactions - if (nTransactions > MAX_BLOCK_SIZE / 60) // 60 is the lower bound for the size of a serialized CTransaction - return 0; - // there can never be more hashes provided than one for every txid - if (vHash.size() > nTransactions) - return 0; - // there must be at least one bit per node in the partial tree, and at least one node per hash - if (vBits.size() < vHash.size()) - return 0; - // calculate height of tree - int nHeight = 0; - while (CalcTreeWidth(nHeight) > 1) - nHeight++; - // traverse the partial tree - unsigned int nBitsUsed = 0, nHashUsed = 0; - uint256 hashMerkleRoot = TraverseAndExtract(nHeight, 0, nBitsUsed, nHashUsed, vMatch); - // verify that no problems occured during the tree traversal - if (fBad) - return 0; - // verify that all bits were consumed (except for the padding caused by serializing it as a byte sequence) - if ((nBitsUsed+7)/8 != (vBits.size()+7)/8) - return 0; - // verify that all hashes were consumed - if (nHashUsed != vHash.size()) - return 0; - return hashMerkleRoot; -} + unsigned int nLastBlockWeCanPrune = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP; + uint64_t nCurrentUsage = CalculateCurrentUsage(); + // We don't check to prune until after we've allocated new space for files + // So we should leave a buffer under our target to account for another allocation + // before the next pruning. + uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE; + uint64_t nBytesToPrune; + int count=0; + if (nCurrentUsage + nBuffer >= nPruneTarget) { + for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { + nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize; + if (vinfoBlockFile[fileNumber].nSize == 0) + continue; + if (nCurrentUsage + nBuffer < nPruneTarget) // are we below our target? + break; + // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip + if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) + break; + PruneOneBlockFile(fileNumber); + // Queue up the files for removal + setFilesToPrune.insert(fileNumber); + nCurrentUsage -= nBytesToPrune; + count++; + } + } -bool AbortNode(const std::string &strMessage) { - strMiscWarning = strMessage; - LogPrintf("*** %s\n", strMessage); - uiInterface.ThreadSafeMessageBox(strMessage, "", CClientUIInterface::MSG_ERROR); - StartShutdown(); - return false; + LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", + nPruneTarget/1024/1024, nCurrentUsage/1024/1024, + ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, + nLastBlockWeCanPrune, count); } bool CheckDiskSpace(uint64_t nAdditionalBytes) { - uint64_t nFreeBytesAvailable = filesystem::space(GetDataDir()).available; + uint64_t nFreeBytesAvailable = boost::filesystem::space(GetDataDir()).available; // Check for nMinDiskSpace bytes (currently 50MB) if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes) - return AbortNode(_("Error: Disk space is low!")); + return AbortNode("Disk space is low!", _("Error: Disk space is low!")); return true; } @@ -2670,7 +3064,7 @@ FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) { if (pos.IsNull()) return NULL; - boost::filesystem::path path = GetDataDir() / "blocks" / strprintf("%s%05u.dat", prefix, pos.nFile); + boost::filesystem::path path = GetBlockPosFilename(pos, prefix); boost::filesystem::create_directories(path.parent_path()); FILE* file = fopen(path.string().c_str(), "rb+"); if (!file && !fReadOnly) @@ -2697,20 +3091,25 @@ FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { return OpenDiskFile(pos, "rev", fReadOnly); } +boost::filesystem::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) +{ + return GetDataDir() / "blocks" / strprintf("%s%05u.dat", prefix, pos.nFile); +} + CBlockIndex * InsertBlockIndex(uint256 hash) { - if (hash == 0) + if (hash.IsNull()) return NULL; // Return existing - map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(hash); + BlockMap::iterator mi = mapBlockIndex.find(hash); if (mi != mapBlockIndex.end()) return (*mi).second; // Create new CBlockIndex* pindexNew = new CBlockIndex(); if (!pindexNew) - throw runtime_error("LoadBlockIndex() : new CBlockIndex failed"); + throw runtime_error("LoadBlockIndex(): new CBlockIndex failed"); mi = mapBlockIndex.insert(make_pair(hash, pindexNew)).first; pindexNew->phashBlock = &((*mi).first); @@ -2719,6 +3118,7 @@ CBlockIndex * InsertBlockIndex(uint256 hash) bool static LoadBlockIndexDB() { + const CChainParams& chainparams = Params(); if (!pblocktree->LoadBlockIndexGuts()) return false; @@ -2736,19 +3136,70 @@ bool static LoadBlockIndexDB() BOOST_FOREACH(const PAIRTYPE(int, CBlockIndex*)& item, vSortedByHeight) { CBlockIndex* pindex = item.second; - pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + pindex->GetBlockWork().getuint256(); - pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; - if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS && !(pindex->nStatus & BLOCK_FAILED_MASK)) - setBlockIndexValid.insert(pindex); + pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); + // We can link the chain of blocks for which we've received transactions at some point. + // Pruned nodes may have deleted the block. + if (pindex->nTx > 0) { + if (pindex->pprev) { + if (pindex->pprev->nChainTx) { + pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; + } else { + pindex->nChainTx = 0; + mapBlocksUnlinked.insert(std::make_pair(pindex->pprev, pindex)); + } + } else { + pindex->nChainTx = pindex->nTx; + } + } + if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->nChainTx || pindex->pprev == NULL)) + setBlockIndexCandidates.insert(pindex); if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork)) pindexBestInvalid = pindex; + if (pindex->pprev) + pindex->BuildSkip(); + if (pindex->IsValid(BLOCK_VALID_TREE) && (pindexBestHeader == NULL || CBlockIndexWorkComparator()(pindexBestHeader, pindex))) + pindexBestHeader = pindex; } // Load block file info pblocktree->ReadLastBlockFile(nLastBlockFile); - LogPrintf("LoadBlockIndexDB(): last block file = %i\n", nLastBlockFile); - if (pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile)) - LogPrintf("LoadBlockIndexDB(): last block file info: %s\n", infoLastBlockFile.ToString()); + vinfoBlockFile.resize(nLastBlockFile + 1); + LogPrintf("%s: last block file = %i\n", __func__, nLastBlockFile); + for (int nFile = 0; nFile <= nLastBlockFile; nFile++) { + pblocktree->ReadBlockFileInfo(nFile, vinfoBlockFile[nFile]); + } + LogPrintf("%s: last block file info: %s\n", __func__, vinfoBlockFile[nLastBlockFile].ToString()); + for (int nFile = nLastBlockFile + 1; true; nFile++) { + CBlockFileInfo info; + if (pblocktree->ReadBlockFileInfo(nFile, info)) { + vinfoBlockFile.push_back(info); + } else { + break; + } + } + + // Check presence of blk files + LogPrintf("Checking all blk files are present...\n"); + set<int> setBlkDataFiles; + BOOST_FOREACH(const PAIRTYPE(uint256, CBlockIndex*)& item, mapBlockIndex) + { + CBlockIndex* pindex = item.second; + if (pindex->nStatus & BLOCK_HAVE_DATA) { + setBlkDataFiles.insert(pindex->nFile); + } + } + for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) + { + CDiskBlockPos pos(*it, 0); + if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) { + return false; + } + } + + // Check whether we have ever pruned block & undo files + pblocktree->ReadFlag("prunedblockfiles", fHavePruned); + if (fHavePruned) + LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n"); // Check whether we need to continue reindexing bool fReindexing = false; @@ -2757,22 +3208,37 @@ bool static LoadBlockIndexDB() // Check whether we have a transaction index pblocktree->ReadFlag("txindex", fTxIndex); - LogPrintf("LoadBlockIndexDB(): transaction index %s\n", fTxIndex ? "enabled" : "disabled"); + LogPrintf("%s: transaction index %s\n", __func__, fTxIndex ? "enabled" : "disabled"); // Load pointer to end of best chain - std::map<uint256, CBlockIndex*>::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); + BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); if (it == mapBlockIndex.end()) return true; chainActive.SetTip(it->second); - LogPrintf("LoadBlockIndexDB(): hashBestChain=%s height=%d date=%s\n", + + PruneBlockIndexCandidates(); + + LogPrintf("%s: hashBestChain=%s height=%d date=%s progress=%f\n", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), - DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime())); + DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), + Checkpoints::GuessVerificationProgress(chainparams.Checkpoints(), chainActive.Tip())); return true; } -bool VerifyDB(int nCheckLevel, int nCheckDepth) +CVerifyDB::CVerifyDB() { + uiInterface.ShowProgress(_("Verifying blocks..."), 0); +} + +CVerifyDB::~CVerifyDB() +{ + uiInterface.ShowProgress("", 100); +} + +bool CVerifyDB::VerifyDB(CCoinsView *coinsview, int nCheckLevel, int nCheckDepth) +{ + LOCK(cs_main); if (chainActive.Tip() == NULL || chainActive.Tip()->pprev == NULL) return true; @@ -2783,7 +3249,7 @@ bool VerifyDB(int nCheckLevel, int nCheckDepth) nCheckDepth = chainActive.Height(); nCheckLevel = std::max(0, std::min(4, nCheckLevel)); LogPrintf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); - CCoinsViewCache coins(*pcoinsTip, true); + CCoinsViewCache coins(coinsview); CBlockIndex* pindexState = chainActive.Tip(); CBlockIndex* pindexFailure = NULL; int nGoodTransactions = 0; @@ -2791,29 +3257,30 @@ bool VerifyDB(int nCheckLevel, int nCheckDepth) for (CBlockIndex* pindex = chainActive.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { boost::this_thread::interruption_point(); + uiInterface.ShowProgress(_("Verifying blocks..."), std::max(1, std::min(99, (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100))))); if (pindex->nHeight < chainActive.Height()-nCheckDepth) break; CBlock block; // check level 0: read from disk if (!ReadBlockFromDisk(block, pindex)) - return error("VerifyDB() : *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); // check level 1: verify block validity if (nCheckLevel >= 1 && !CheckBlock(block, state)) - return error("VerifyDB() : *** found bad block at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + return error("VerifyDB(): *** found bad block at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); // check level 2: verify undo validity if (nCheckLevel >= 2 && pindex) { CBlockUndo undo; CDiskBlockPos pos = pindex->GetUndoPos(); if (!pos.IsNull()) { - if (!undo.ReadFromDisk(pos, pindex->pprev->GetBlockHash())) - return error("VerifyDB() : *** found bad undo data at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + if (!UndoReadFromDisk(undo, pos, pindex->pprev->GetBlockHash())) + return error("VerifyDB(): *** found bad undo data at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); } } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks - if (nCheckLevel >= 3 && pindex == pindexState && (coins.GetCacheSize() + pcoinsTip->GetCacheSize()) <= 2*nCoinCacheSize + 32000) { + if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { bool fClean = true; if (!DisconnectBlock(block, state, pindex, coins, &fClean)) - return error("VerifyDB() : *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); pindexState = pindex->pprev; if (!fClean) { nGoodTransactions = 0; @@ -2821,21 +3288,24 @@ bool VerifyDB(int nCheckLevel, int nCheckDepth) } else nGoodTransactions += block.vtx.size(); } + if (ShutdownRequested()) + return true; } if (pindexFailure) - return error("VerifyDB() : *** coin database inconsistencies found (last %i blocks, %i good transactions before that)\n", chainActive.Height() - pindexFailure->nHeight + 1, nGoodTransactions); + return error("VerifyDB(): *** coin database inconsistencies found (last %i blocks, %i good transactions before that)\n", chainActive.Height() - pindexFailure->nHeight + 1, nGoodTransactions); // check level 4: try reconnecting blocks if (nCheckLevel >= 4) { CBlockIndex *pindex = pindexState; while (pindex != chainActive.Tip()) { boost::this_thread::interruption_point(); + uiInterface.ShowProgress(_("Verifying blocks..."), std::max(1, std::min(99, 100 - (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * 50)))); pindex = chainActive.Next(pindex); CBlock block; if (!ReadBlockFromDisk(block, pindex)) - return error("VerifyDB() : *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); if (!ConnectBlock(block, state, pindex, coins)) - return error("VerifyDB() : *** found unconnectable block at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + return error("VerifyDB(): *** found unconnectable block at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } } @@ -2846,10 +3316,32 @@ bool VerifyDB(int nCheckLevel, int nCheckDepth) void UnloadBlockIndex() { - mapBlockIndex.clear(); - setBlockIndexValid.clear(); + LOCK(cs_main); + setBlockIndexCandidates.clear(); chainActive.SetTip(NULL); pindexBestInvalid = NULL; + pindexBestHeader = NULL; + mempool.clear(); + mapOrphanTransactions.clear(); + mapOrphanTransactionsByPrev.clear(); + nSyncStarted = 0; + mapBlocksUnlinked.clear(); + vinfoBlockFile.clear(); + nLastBlockFile = 0; + nBlockSequenceId = 1; + mapBlockSource.clear(); + mapBlocksInFlight.clear(); + nQueuedValidatedHeaders = 0; + nPreferredDownload = 0; + setDirtyBlockIndex.clear(); + setDirtyFileInfo.clear(); + mapNodeState.clear(); + + BOOST_FOREACH(BlockMap::value_type& entry, mapBlockIndex) { + delete entry.second; + } + mapBlockIndex.clear(); + fHavePruned = false; } bool LoadBlockIndex() @@ -2862,6 +3354,8 @@ bool LoadBlockIndex() bool InitBlockIndex() { + const CChainParams& chainparams = Params(); + LOCK(cs_main); // Check whether we're already initialized if (chainActive.Genesis() != NULL) return true; @@ -2879,14 +3373,19 @@ bool InitBlockIndex() { unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); CDiskBlockPos blockPos; CValidationState state; - if (!FindBlockPos(state, blockPos, nBlockSize+8, 0, block.nTime)) - return error("LoadBlockIndex() : FindBlockPos failed"); - if (!WriteBlockToDisk(block, blockPos)) - return error("LoadBlockIndex() : writing genesis block to disk failed"); - if (!AddToBlockIndex(block, state, blockPos)) - return error("LoadBlockIndex() : genesis block not accepted"); - } catch(std::runtime_error &e) { - return error("LoadBlockIndex() : failed to initialize block database: %s", e.what()); + if (!FindBlockPos(state, blockPos, nBlockSize+8, 0, block.GetBlockTime())) + return error("LoadBlockIndex(): FindBlockPos failed"); + if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) + return error("LoadBlockIndex(): writing genesis block to disk failed"); + CBlockIndex *pindex = AddToBlockIndex(block); + if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) + return error("LoadBlockIndex(): genesis block not accepted"); + if (!ActivateBestChain(state, &block)) + return error("LoadBlockIndex(): genesis block cannot be activated"); + // Force a chainstate write so that when we VerifyDB in a moment, it doesn't check stale data + return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); + } catch (const std::runtime_error& e) { + return error("LoadBlockIndex(): failed to initialize block database: %s", e.what()); } } @@ -2895,92 +3394,19 @@ bool InitBlockIndex() { -void PrintBlockTree() -{ - // pre-compute tree structure - map<CBlockIndex*, vector<CBlockIndex*> > mapNext; - for (map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.begin(); mi != mapBlockIndex.end(); ++mi) - { - CBlockIndex* pindex = (*mi).second; - mapNext[pindex->pprev].push_back(pindex); - // test - //while (rand() % 3 == 0) - // mapNext[pindex->pprev].push_back(pindex); - } - - vector<pair<int, CBlockIndex*> > vStack; - vStack.push_back(make_pair(0, chainActive.Genesis())); - - int nPrevCol = 0; - while (!vStack.empty()) - { - int nCol = vStack.back().first; - CBlockIndex* pindex = vStack.back().second; - vStack.pop_back(); - - // print split or gap - if (nCol > nPrevCol) - { - for (int i = 0; i < nCol-1; i++) - LogPrintf("| "); - LogPrintf("|\\\n"); - } - else if (nCol < nPrevCol) - { - for (int i = 0; i < nCol; i++) - LogPrintf("| "); - LogPrintf("|\n"); - } - nPrevCol = nCol; - - // print columns - for (int i = 0; i < nCol; i++) - LogPrintf("| "); - - // print item - CBlock block; - ReadBlockFromDisk(block, pindex); - LogPrintf("%d (blk%05u.dat:0x%x) %s tx %"PRIszu"", - pindex->nHeight, - pindex->GetBlockPos().nFile, pindex->GetBlockPos().nPos, - DateTimeStrFormat("%Y-%m-%d %H:%M:%S", block.GetBlockTime()), - block.vtx.size()); - - // put the main time-chain first - vector<CBlockIndex*>& vNext = mapNext[pindex]; - for (unsigned int i = 0; i < vNext.size(); i++) - { - if (chainActive.Next(vNext[i])) - { - swap(vNext[0], vNext[i]); - break; - } - } - - // iterate children - for (unsigned int i = 0; i < vNext.size(); i++) - vStack.push_back(make_pair(nCol+i, vNext[i])); - } -} - bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp) { + const CChainParams& chainparams = Params(); + // Map of disk positions for blocks with unknown parent (only used for reindex) + static std::multimap<uint256, CDiskBlockPos> mapBlocksUnknownParent; int64_t nStart = GetTimeMillis(); int nLoaded = 0; try { + // This takes over fileIn and calls fclose() on it in the CBufferedFile destructor CBufferedFile blkdat(fileIn, 2*MAX_BLOCK_SIZE, MAX_BLOCK_SIZE+8, SER_DISK, CLIENT_VERSION); - uint64_t nStartByte = 0; - if (dbp) { - // (try to) skip already indexed part - CBlockFileInfo info; - if (pblocktree->ReadBlockFileInfo(dbp->nFile, info)) { - nStartByte = info.nSize; - blkdat.Seek(info.nSize); - } - } uint64_t nRewind = blkdat.GetPos(); - while (blkdat.good() && !blkdat.eof()) { + while (!blkdat.eof()) { boost::this_thread::interruption_point(); blkdat.SetPos(nRewind); @@ -2989,60 +3415,272 @@ bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp) unsigned int nSize = 0; try { // locate a header - unsigned char buf[4]; + unsigned char buf[MESSAGE_START_SIZE]; blkdat.FindByte(Params().MessageStart()[0]); nRewind = blkdat.GetPos()+1; blkdat >> FLATDATA(buf); - if (memcmp(buf, Params().MessageStart(), 4)) + if (memcmp(buf, Params().MessageStart(), MESSAGE_START_SIZE)) continue; // read size blkdat >> nSize; if (nSize < 80 || nSize > MAX_BLOCK_SIZE) continue; - } catch (std::exception &e) { + } catch (const std::exception&) { // no valid block header found; don't complain break; } try { // read block uint64_t nBlockPos = blkdat.GetPos(); + if (dbp) + dbp->nPos = nBlockPos; blkdat.SetLimit(nBlockPos + nSize); + blkdat.SetPos(nBlockPos); CBlock block; blkdat >> block; nRewind = blkdat.GetPos(); - // process block - if (nBlockPos >= nStartByte) { - LOCK(cs_main); + // detect out of order blocks, and store them for later + uint256 hash = block.GetHash(); + if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex.find(block.hashPrevBlock) == mapBlockIndex.end()) { + LogPrint("reindex", "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), + block.hashPrevBlock.ToString()); if (dbp) - dbp->nPos = nBlockPos; + mapBlocksUnknownParent.insert(std::make_pair(block.hashPrevBlock, *dbp)); + continue; + } + + // process in case the block isn't known yet + if (mapBlockIndex.count(hash) == 0 || (mapBlockIndex[hash]->nStatus & BLOCK_HAVE_DATA) == 0) { CValidationState state; - if (ProcessBlock(state, NULL, &block, dbp)) + if (ProcessNewBlock(state, NULL, &block, dbp)) nLoaded++; if (state.IsError()) break; + } else if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex[hash]->nHeight % 1000 == 0) { + LogPrintf("Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight); + } + + // Recursively process earlier encountered successors of this block + deque<uint256> queue; + queue.push_back(hash); + while (!queue.empty()) { + uint256 head = queue.front(); + queue.pop_front(); + std::pair<std::multimap<uint256, CDiskBlockPos>::iterator, std::multimap<uint256, CDiskBlockPos>::iterator> range = mapBlocksUnknownParent.equal_range(head); + while (range.first != range.second) { + std::multimap<uint256, CDiskBlockPos>::iterator it = range.first; + if (ReadBlockFromDisk(block, it->second)) + { + LogPrintf("%s: Processing out of order child %s of %s\n", __func__, block.GetHash().ToString(), + head.ToString()); + CValidationState dummy; + if (ProcessNewBlock(dummy, NULL, &block, &it->second)) + { + nLoaded++; + queue.push_back(block.GetHash()); + } + } + range.first++; + mapBlocksUnknownParent.erase(it); + } } - } catch (std::exception &e) { - LogPrintf("%s : Deserialize or I/O error - %s", __PRETTY_FUNCTION__, e.what()); + } catch (const std::exception& e) { + LogPrintf("%s: Deserialize or I/O error - %s", __func__, e.what()); } } - fclose(fileIn); - } catch(std::runtime_error &e) { - AbortNode(_("Error: system error: ") + e.what()); + } catch (const std::runtime_error& e) { + AbortNode(std::string("System error: ") + e.what()); } if (nLoaded > 0) LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, GetTimeMillis() - nStart); return nLoaded > 0; } +void static CheckBlockIndex() +{ + const Consensus::Params& consensusParams = Params().GetConsensus(); + if (!fCheckBlockIndex) { + return; + } + LOCK(cs_main); + // During a reindex, we read the genesis block and call CheckBlockIndex before ActivateBestChain, + // so we have the genesis block in mapBlockIndex but no active chain. (A few of the tests when + // iterating the block tree require that chainActive has been initialized.) + if (chainActive.Height() < 0) { + assert(mapBlockIndex.size() <= 1); + return; + } + // Build forward-pointing map of the entire block tree. + std::multimap<CBlockIndex*,CBlockIndex*> forward; + for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { + forward.insert(std::make_pair(it->second->pprev, it->second)); + } + assert(forward.size() == mapBlockIndex.size()); + std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangeGenesis = forward.equal_range(NULL); + CBlockIndex *pindex = rangeGenesis.first->second; + rangeGenesis.first++; + assert(rangeGenesis.first == rangeGenesis.second); // There is only one index entry with parent NULL. + // Iterate over the entire block tree, using depth-first search. + // Along the way, remember whether there are blocks on the path from genesis + // block being explored which are the first to have certain properties. + size_t nNodes = 0; + int nHeight = 0; + CBlockIndex* pindexFirstInvalid = NULL; // Oldest ancestor of pindex which is invalid. + CBlockIndex* pindexFirstMissing = NULL; // Oldest ancestor of pindex which does not have BLOCK_HAVE_DATA. + CBlockIndex* pindexFirstNeverProcessed = NULL; // Oldest ancestor of pindex for which nTx == 0. + CBlockIndex* pindexFirstNotTreeValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TREE (regardless of being valid or not). + CBlockIndex* pindexFirstNotTransactionsValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not). + CBlockIndex* pindexFirstNotChainValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not). + CBlockIndex* pindexFirstNotScriptsValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_SCRIPTS (regardless of being valid or not). + while (pindex != NULL) { + nNodes++; + if (pindexFirstInvalid == NULL && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; + if (pindexFirstMissing == NULL && !(pindex->nStatus & BLOCK_HAVE_DATA)) pindexFirstMissing = pindex; + if (pindexFirstNeverProcessed == NULL && pindex->nTx == 0) pindexFirstNeverProcessed = pindex; + if (pindex->pprev != NULL && pindexFirstNotTreeValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex; + if (pindex->pprev != NULL && pindexFirstNotTransactionsValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) pindexFirstNotTransactionsValid = pindex; + if (pindex->pprev != NULL && pindexFirstNotChainValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) pindexFirstNotChainValid = pindex; + if (pindex->pprev != NULL && pindexFirstNotScriptsValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) pindexFirstNotScriptsValid = pindex; + + // Begin: actual consistency checks. + if (pindex->pprev == NULL) { + // Genesis block checks. + assert(pindex->GetBlockHash() == consensusParams.hashGenesisBlock); // Genesis block's hash must match. + assert(pindex == chainActive.Genesis()); // The current active chain's genesis block must be this block. + } + if (pindex->nChainTx == 0) assert(pindex->nSequenceId == 0); // nSequenceId can't be set for blocks that aren't linked + // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred). + // HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred. + if (!fHavePruned) { + // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0 + assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); + assert(pindexFirstMissing == pindexFirstNeverProcessed); + } else { + // If we have pruned, then we can only say that HAVE_DATA implies nTx > 0 + if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0); + } + if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA); + assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. + // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to nChainTx being set. + assert((pindexFirstNeverProcessed != NULL) == (pindex->nChainTx == 0)); // nChainTx != 0 is used to signal that all parent blocks have been processed (but may have been pruned). + assert((pindexFirstNotTransactionsValid != NULL) == (pindex->nChainTx == 0)); + assert(pindex->nHeight == nHeight); // nHeight must be consistent. + assert(pindex->pprev == NULL || pindex->nChainWork >= pindex->pprev->nChainWork); // For every block except the genesis block, the chainwork must be larger than the parent's. + assert(nHeight < 2 || (pindex->pskip && (pindex->pskip->nHeight < nHeight))); // The pskip pointer must point back for all but the first 2 blocks. + assert(pindexFirstNotTreeValid == NULL); // All mapBlockIndex entries must at least be TREE valid + if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TREE) assert(pindexFirstNotTreeValid == NULL); // TREE valid implies all parents are TREE valid + if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_CHAIN) assert(pindexFirstNotChainValid == NULL); // CHAIN valid implies all parents are CHAIN valid + if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_SCRIPTS) assert(pindexFirstNotScriptsValid == NULL); // SCRIPTS valid implies all parents are SCRIPTS valid + if (pindexFirstInvalid == NULL) { + // Checks for not-invalid blocks. + assert((pindex->nStatus & BLOCK_FAILED_MASK) == 0); // The failed mask cannot be set for blocks without invalid parents. + } + if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstNeverProcessed == NULL) { + if (pindexFirstInvalid == NULL) { + // If this block sorts at least as good as the current tip and + // is valid and we have all data for its parents, it must be in + // setBlockIndexCandidates. chainActive.Tip() must also be there + // even if some data has been pruned. + if (pindexFirstMissing == NULL || pindex == chainActive.Tip()) { + assert(setBlockIndexCandidates.count(pindex)); + } + // If some parent is missing, then it could be that this block was in + // setBlockIndexCandidates but had to be removed because of the missing data. + // In this case it must be in mapBlocksUnlinked -- see test below. + } + } else { // If this block sorts worse than the current tip or some ancestor's block has never been seen, it cannot be in setBlockIndexCandidates. + assert(setBlockIndexCandidates.count(pindex) == 0); + } + // Check whether this block is in mapBlocksUnlinked. + std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangeUnlinked = mapBlocksUnlinked.equal_range(pindex->pprev); + bool foundInUnlinked = false; + while (rangeUnlinked.first != rangeUnlinked.second) { + assert(rangeUnlinked.first->first == pindex->pprev); + if (rangeUnlinked.first->second == pindex) { + foundInUnlinked = true; + break; + } + rangeUnlinked.first++; + } + if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed != NULL && pindexFirstInvalid == NULL) { + // If this block has block data available, some parent was never received, and has no invalid parents, it must be in mapBlocksUnlinked. + assert(foundInUnlinked); + } + if (!(pindex->nStatus & BLOCK_HAVE_DATA)) assert(!foundInUnlinked); // Can't be in mapBlocksUnlinked if we don't HAVE_DATA + if (pindexFirstMissing == NULL) assert(!foundInUnlinked); // We aren't missing data for any parent -- cannot be in mapBlocksUnlinked. + if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == NULL && pindexFirstMissing != NULL) { + // We HAVE_DATA for this block, have received data for all parents at some point, but we're currently missing data for some parent. + assert(fHavePruned); // We must have pruned. + // This block may have entered mapBlocksUnlinked if: + // - it has a descendant that at some point had more work than the + // tip, and + // - we tried switching to that descendant but were missing + // data for some intermediate block between chainActive and the + // tip. + // So if this block is itself better than chainActive.Tip() and it wasn't in + // setBlockIndexCandidates, then it must be in mapBlocksUnlinked. + if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && setBlockIndexCandidates.count(pindex) == 0) { + if (pindexFirstInvalid == NULL) { + assert(foundInUnlinked); + } + } + } + // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow + // End: actual consistency checks. + + // Try descending into the first subnode. + std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> range = forward.equal_range(pindex); + if (range.first != range.second) { + // A subnode was found. + pindex = range.first->second; + nHeight++; + continue; + } + // This is a leaf node. + // Move upwards until we reach a node of which we have not yet visited the last child. + while (pindex) { + // We are going to either move to a parent or a sibling of pindex. + // If pindex was the first with a certain property, unset the corresponding variable. + if (pindex == pindexFirstInvalid) pindexFirstInvalid = NULL; + if (pindex == pindexFirstMissing) pindexFirstMissing = NULL; + if (pindex == pindexFirstNeverProcessed) pindexFirstNeverProcessed = NULL; + if (pindex == pindexFirstNotTreeValid) pindexFirstNotTreeValid = NULL; + if (pindex == pindexFirstNotTransactionsValid) pindexFirstNotTransactionsValid = NULL; + if (pindex == pindexFirstNotChainValid) pindexFirstNotChainValid = NULL; + if (pindex == pindexFirstNotScriptsValid) pindexFirstNotScriptsValid = NULL; + // Find our parent. + CBlockIndex* pindexPar = pindex->pprev; + // Find which child we just visited. + std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangePar = forward.equal_range(pindexPar); + while (rangePar.first->second != pindex) { + assert(rangePar.first != rangePar.second); // Our parent must have at least the node we're coming from as child. + rangePar.first++; + } + // Proceed to the next one. + rangePar.first++; + if (rangePar.first != rangePar.second) { + // Move to the sibling. + pindex = rangePar.first->second; + break; + } else { + // Move up further. + pindex = pindexPar; + nHeight--; + continue; + } + } + } - + // Check that we actually traversed the entire map. + assert(nNodes == forward.size()); +} ////////////////////////////////////////////////////////////////////////////// // @@ -3055,12 +3693,12 @@ string GetWarnings(string strFor) string strStatusBar; string strRPC; - if (GetBoolArg("-testsafemode", false)) - strRPC = "test"; - if (!CLIENT_VERSION_IS_RELEASE) strStatusBar = _("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"); + if (GetBoolArg("-testsafemode", false)) + strStatusBar = strRPC = "testsafemode enabled"; + // Misc warnings like out of disk space and clock is wrong if (strMiscWarning != "") { @@ -3097,7 +3735,7 @@ string GetWarnings(string strFor) return strStatusBar; else if (strFor == "rpc") return strRPC; - assert(!"GetWarnings() : invalid parameter"); + assert(!"GetWarnings(): invalid parameter"); return "error"; } @@ -3126,14 +3764,12 @@ bool static AlreadyHave(const CInv& inv) pcoinsTip->HaveCoins(inv.hash); } case MSG_BLOCK: - return mapBlockIndex.count(inv.hash) || - mapOrphanBlocks.count(inv.hash); + return mapBlockIndex.count(inv.hash); } // Don't know what it is, just say we already got one return true; } - void static ProcessGetData(CNode* pfrom) { std::deque<CInv>::iterator it = pfrom->vRecvGetData.begin(); @@ -3155,29 +3791,32 @@ void static ProcessGetData(CNode* pfrom) if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK) { bool send = false; - map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(inv.hash); + BlockMap::iterator mi = mapBlockIndex.find(inv.hash); if (mi != mapBlockIndex.end()) { - // If the requested block is at a height below our last - // checkpoint, only serve it if it's in the checkpointed chain - int nHeight = mi->second->nHeight; - CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(mapBlockIndex); - if (pcheckpoint && nHeight < pcheckpoint->nHeight) { - if (!chainActive.Contains(mi->second)) - { - LogPrintf("ProcessGetData(): ignoring request for old block that isn't in the main chain\n"); - } else { - send = true; - } + if (chainActive.Contains(mi->second)) { + send = true; } else { - send = true; + static const int nOneMonth = 30 * 24 * 60 * 60; + // To prevent fingerprinting attacks, only send blocks outside of the active + // chain if they are valid, and no more than a month older (both in time, and in + // best equivalent proof of work) than the best header chain we know about. + send = mi->second->IsValid(BLOCK_VALID_SCRIPTS) && (pindexBestHeader != NULL) && + (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() < nOneMonth) && + (GetBlockProofEquivalentTime(*pindexBestHeader, *mi->second, *pindexBestHeader, Params().GetConsensus()) < nOneMonth); + if (!send) { + LogPrintf("%s: ignoring request from peer=%i for old block that isn't in the main chain\n", __func__, pfrom->GetId()); + } } } - if (send) + // Pruned nodes may have deleted the block, so check whether + // it's available before trying to send. + if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) { // Send block from disk CBlock block; - ReadBlockFromDisk(block, (*mi).second); + if (!ReadBlockFromDisk(block, (*mi).second)) + assert(!"cannot load block from disk"); if (inv.type == MSG_BLOCK) pfrom->PushMessage("block", block); else // MSG_FILTERED_BLOCK) @@ -3189,7 +3828,7 @@ void static ProcessGetData(CNode* pfrom) pfrom->PushMessage("merkleblock", merkleBlock); // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see // This avoids hurting performance by pointlessly requiring a round-trip - // Note that there is currently no way for a node to request any single transactions we didnt send here - + // Note that there is currently no way for a node to request any single transactions we didn't send here - // they must either disconnect and retry or request the full block. // Thus, the protocol spec specified allows for us to provide duplicate txn here, // however we MUST always provide at least what the remote peer needs @@ -3202,7 +3841,7 @@ void static ProcessGetData(CNode* pfrom) // no response } - // Trigger them to send a getblocks request for the next batch of inventory + // Trigger the peer node to send a getblocks request for the next batch of inventory if (inv.hash == pfrom->hashContinue) { // Bypass PushInventory, this must send even if redundant, @@ -3211,7 +3850,7 @@ void static ProcessGetData(CNode* pfrom) vector<CInv> vInv; vInv.push_back(CInv(MSG_BLOCK, chainActive.Tip()->GetBlockHash())); pfrom->PushMessage("inv", vInv); - pfrom->hashContinue = 0; + pfrom->hashContinue.SetNull(); } } } @@ -3243,7 +3882,7 @@ void static ProcessGetData(CNode* pfrom) } // Track requests for our stuff. - g_signals.Inventory(inv.hash); + GetMainSignals().Inventory(inv.hash); if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK) break; @@ -3264,10 +3903,11 @@ void static ProcessGetData(CNode* pfrom) } } -bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) +bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, int64_t nTimeReceived) { + const CChainParams& chainparams = Params(); RandAddSeedPerfmon(); - LogPrint("net", "received: %s (%"PRIszu" bytes)\n", strCommand, vRecv.size()); + LogPrint("net", "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); if (mapArgs.count("-dropmessagestest") && GetRand(atoi(mapArgs["-dropmessagestest"])) == 0) { LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); @@ -3277,7 +3917,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) - if (strCommand == "version") { // Each connection can only send one version message @@ -3296,7 +3935,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) if (pfrom->nVersion < MIN_PEER_PROTO_VERSION) { // disconnect from peers older than this proto version - LogPrintf("partner %s using obsolete version %i; disconnecting\n", pfrom->addr.ToString(), pfrom->nVersion); + LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->id, pfrom->nVersion); pfrom->PushMessage("reject", strCommand, REJECT_OBSOLETE, strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION)); pfrom->fDisconnect = true; @@ -3308,7 +3947,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) if (!vRecv.empty()) vRecv >> addrFrom >> nNonce; if (!vRecv.empty()) { - vRecv >> pfrom->strSubVer; + vRecv >> LIMITED_STRING(pfrom->strSubVer, 256); pfrom->cleanSubVer = SanitizeString(pfrom->strSubVer); } if (!vRecv.empty()) @@ -3318,12 +3957,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) else pfrom->fRelayTxes = true; - if (pfrom->fInbound && addrMe.IsRoutable()) - { - pfrom->addrLocal = addrMe; - SeenLocal(addrMe); - } - // Disconnect if we connected to ourself if (nNonce == nLocalHostNonce && nNonce > 1) { @@ -3332,12 +3965,20 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) return true; } + pfrom->addrLocal = addrMe; + if (pfrom->fInbound && addrMe.IsRoutable()) + { + SeenLocal(addrMe); + } + // Be shy and don't send version until we hear if (pfrom->fInbound) pfrom->PushVersion(); pfrom->fClient = !(pfrom->nServices & NODE_NETWORK); + // Potentially mark this peer as a preferred download peer. + UpdatePreferredDownload(pfrom, State(pfrom->GetId())); // Change version pfrom->PushMessage("verack"); @@ -3346,11 +3987,16 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) if (!pfrom->fInbound) { // Advertise our address - if (!fNoListen && !IsInitialBlockDownload()) + if (fListen && !IsInitialBlockDownload()) { CAddress addr = GetLocalAddress(&pfrom->addr); if (addr.IsRoutable()) + { + pfrom->PushAddress(addr); + } else if (IsPeerAddrLocalGood(pfrom)) { + addr.SetIP(pfrom->addrLocal); pfrom->PushAddress(addr); + } } // Get recent addresses @@ -3377,12 +4023,18 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) pfrom->fSuccessfullyConnected = true; - LogPrintf("receive version message: %s: version %d, blocks=%d, us=%s, them=%s, peer=%s\n", pfrom->cleanSubVer, pfrom->nVersion, pfrom->nStartingHeight, addrMe.ToString(), addrFrom.ToString(), pfrom->addr.ToString()); + string remoteAddr; + if (fLogIPs) + remoteAddr = ", peeraddr=" + pfrom->addr.ToString(); - AddTimeData(pfrom->addr, nTime); + LogPrintf("receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", + pfrom->cleanSubVer, pfrom->nVersion, + pfrom->nStartingHeight, addrMe.ToString(), pfrom->id, + remoteAddr); - LOCK(cs_main); - cPeerBlockCounts.input(pfrom->nStartingHeight); + int64_t nTimeOffset = nTime - GetTime(); + pfrom->nTimeOffset = nTimeOffset; + AddTimeData(pfrom->addr, nTimeOffset); } @@ -3397,6 +4049,12 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) else if (strCommand == "verack") { pfrom->SetRecvVersion(min(pfrom->nVersion, PROTOCOL_VERSION)); + + // Mark this node as currently connected, so we update its timestamp later. + if (pfrom->fNetworkNode) { + LOCK(cs_main); + State(pfrom->GetId())->fCurrentlyConnected = true; + } } @@ -3411,7 +4069,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) if (vAddr.size() > 1000) { Misbehaving(pfrom->GetId(), 20); - return error("message addr size() = %"PRIszu"", vAddr.size()); + return error("message addr size() = %u", vAddr.size()); } // Store the new addresses @@ -3432,12 +4090,12 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) { LOCK(cs_vNodes); // Use deterministic randomness to send to the same nodes for 24 hours - // at a time so the setAddrKnowns of the chosen nodes prevent repeats + // at a time so the addrKnowns of the chosen nodes prevent repeats static uint256 hashSalt; - if (hashSalt == 0) + if (hashSalt.IsNull()) hashSalt = GetRandHash(); uint64_t hashAddr = addr.GetHash(); - uint256 hashRand = hashSalt ^ (hashAddr<<32) ^ ((GetTime()+hashAddr)/(24*60*60)); + uint256 hashRand = ArithToUint256(UintToArith256(hashSalt) ^ (hashAddr<<32) ^ ((GetTime()+hashAddr)/(24*60*60))); hashRand = Hash(BEGIN(hashRand), END(hashRand)); multimap<uint256, CNode*> mapMix; BOOST_FOREACH(CNode* pnode, vNodes) @@ -3446,7 +4104,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) continue; unsigned int nPointer; memcpy(&nPointer, &pnode, sizeof(nPointer)); - uint256 hashKey = hashRand ^ nPointer; + uint256 hashKey = ArithToUint256(UintToArith256(hashRand) ^ nPointer); hashKey = Hash(BEGIN(hashKey), END(hashKey)); mapMix.insert(make_pair(hashKey, pnode)); } @@ -3474,20 +4132,13 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) if (vInv.size() > MAX_INV_SZ) { Misbehaving(pfrom->GetId(), 20); - return error("message inv size() = %"PRIszu"", vInv.size()); - } - - // find last block in inv vector - unsigned int nLastBlock = (unsigned int)(-1); - for (unsigned int nInv = 0; nInv < vInv.size(); nInv++) { - if (vInv[vInv.size() - 1 - nInv].type == MSG_BLOCK) { - nLastBlock = vInv.size() - 1 - nInv; - break; - } + return error("message inv size() = %u", vInv.size()); } LOCK(cs_main); + std::vector<CInv> vToFetch; + for (unsigned int nInv = 0; nInv < vInv.size(); nInv++) { const CInv &inv = vInv[nInv]; @@ -3496,25 +4147,46 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) pfrom->AddInventoryKnown(inv); bool fAlreadyHave = AlreadyHave(inv); - LogPrint("net", " got inventory: %s %s\n", inv.ToString(), fAlreadyHave ? "have" : "new"); - - if (!fAlreadyHave) { - if (!fImporting && !fReindex) - pfrom->AskFor(inv); - } else if (inv.type == MSG_BLOCK && mapOrphanBlocks.count(inv.hash)) { - PushGetBlocks(pfrom, chainActive.Tip(), GetOrphanRoot(inv.hash)); - } else if (nInv == nLastBlock) { - // In case we are on a very long side-chain, it is possible that we already have - // the last block in an inv bundle sent in response to getblocks. Try to detect - // this situation and push another getblocks to continue. - PushGetBlocks(pfrom, mapBlockIndex[inv.hash], uint256(0)); - if (fDebug) - LogPrintf("force request: %s\n", inv.ToString()); + LogPrint("net", "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); + + if (!fAlreadyHave && !fImporting && !fReindex && inv.type != MSG_BLOCK) + pfrom->AskFor(inv); + + if (inv.type == MSG_BLOCK) { + UpdateBlockAvailability(pfrom->GetId(), inv.hash); + if (!fAlreadyHave && !fImporting && !fReindex && !mapBlocksInFlight.count(inv.hash)) { + // First request the headers preceding the announced block. In the normal fully-synced + // case where a new block is announced that succeeds the current tip (no reorganization), + // there are no such headers. + // Secondly, and only when we are close to being synced, we request the announced block directly, + // to avoid an extra round-trip. Note that we must *first* ask for the headers, so by the + // time the block arrives, the header chain leading up to it is already validated. Not + // doing this will result in the received block being rejected as an orphan in case it is + // not a direct successor. + pfrom->PushMessage("getheaders", chainActive.GetLocator(pindexBestHeader), inv.hash); + CNodeState *nodestate = State(pfrom->GetId()); + if (chainActive.Tip()->GetBlockTime() > GetAdjustedTime() - chainparams.GetConsensus().nPowTargetSpacing * 20 && + nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + vToFetch.push_back(inv); + // Mark block as in flight already, even though the actual "getdata" message only goes out + // later (within the same cs_main lock, though). + MarkBlockAsInFlight(pfrom->GetId(), inv.hash, chainparams.GetConsensus()); + } + LogPrint("net", "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->id); + } } // Track requests for our stuff - g_signals.Inventory(inv.hash); + GetMainSignals().Inventory(inv.hash); + + if (pfrom->nSendSize > (SendBufferSize() * 2)) { + Misbehaving(pfrom->GetId(), 50); + return error("send buffer size() = %u", pfrom->nSendSize); + } } + + if (!vToFetch.empty()) + pfrom->PushMessage("getdata", vToFetch); } @@ -3525,14 +4197,14 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) if (vInv.size() > MAX_INV_SZ) { Misbehaving(pfrom->GetId(), 20); - return error("message getdata size() = %"PRIszu"", vInv.size()); + return error("message getdata size() = %u", vInv.size()); } if (fDebug || (vInv.size() != 1)) - LogPrint("net", "received getdata (%"PRIszu" invsz)\n", vInv.size()); + LogPrint("net", "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom->id); if ((fDebug && vInv.size() > 0) || (vInv.size() == 1)) - LogPrint("net", "received getdata for: %s\n", vInv[0].ToString()); + LogPrint("net", "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->id); pfrom->vRecvGetData.insert(pfrom->vRecvGetData.end(), vInv.begin(), vInv.end()); ProcessGetData(pfrom); @@ -3548,13 +4220,13 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) LOCK(cs_main); // Find the last block the caller has in the main chain - CBlockIndex* pindex = chainActive.FindFork(locator); + CBlockIndex* pindex = FindForkInGlobalIndex(chainActive, locator); // Send the rest of the chain if (pindex) pindex = chainActive.Next(pindex); int nLimit = 500; - LogPrint("net", "getblocks %d to %s limit %d\n", (pindex ? pindex->nHeight : -1), hashStop.ToString(), nLimit); + LogPrint("net", "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->id); for (; pindex; pindex = chainActive.Next(pindex)) { if (pindex->GetBlockHash() == hashStop) @@ -3565,8 +4237,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) pfrom->PushInventory(CInv(MSG_BLOCK, pindex->GetBlockHash())); if (--nLimit <= 0) { - // When this block is requested, we'll send an inv that'll make them - // getblocks the next batch of inventory. + // When this block is requested, we'll send an inv that'll + // trigger the peer to getblocks the next batch of inventory. LogPrint("net", " getblocks stopping at limit %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); pfrom->hashContinue = pindex->GetBlockHash(); break; @@ -3583,11 +4255,14 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) LOCK(cs_main); + if (IsInitialBlockDownload()) + return true; + CBlockIndex* pindex = NULL; if (locator.IsNull()) { // If locator is null, return the hashStop block - map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(hashStop); + BlockMap::iterator mi = mapBlockIndex.find(hashStop); if (mi == mapBlockIndex.end()) return true; pindex = (*mi).second; @@ -3595,15 +4270,15 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) else { // Find the last block the caller has in the main chain - pindex = chainActive.FindFork(locator); + pindex = FindForkInGlobalIndex(chainActive, locator); if (pindex) pindex = chainActive.Next(pindex); } // we must use CBlocks, as CBlockHeaders won't include the 0x00 nTx count at the end vector<CBlock> vHeaders; - int nLimit = 2000; - LogPrint("net", "getheaders %d to %s\n", (pindex ? pindex->nHeight : -1), hashStop.ToString()); + int nLimit = MAX_HEADERS_RESULTS; + LogPrint("net", "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.ToString(), pfrom->id); for (; pindex; pindex = chainActive.Next(pindex)) { vHeaders.push_back(pindex->GetBlockHeader()); @@ -3628,48 +4303,62 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) bool fMissingInputs = false; CValidationState state; + + mapAlreadyAskedFor.erase(inv); + if (AcceptToMemoryPool(mempool, state, tx, true, &fMissingInputs)) { mempool.check(pcoinsTip); - RelayTransaction(tx, inv.hash); - mapAlreadyAskedFor.erase(inv); + RelayTransaction(tx); vWorkQueue.push_back(inv.hash); vEraseQueue.push_back(inv.hash); - - LogPrint("mempool", "AcceptToMemoryPool: %s %s : accepted %s (poolsz %"PRIszu")\n", - pfrom->addr.ToString(), pfrom->cleanSubVer, + LogPrint("mempool", "AcceptToMemoryPool: peer=%d %s: accepted %s (poolsz %u)\n", + pfrom->id, pfrom->cleanSubVer, tx.GetHash().ToString(), mempool.mapTx.size()); // Recursively process any orphan transactions that depended on this one + set<NodeId> setMisbehaving; for (unsigned int i = 0; i < vWorkQueue.size(); i++) { - uint256 hashPrev = vWorkQueue[i]; - for (set<uint256>::iterator mi = mapOrphanTransactionsByPrev[hashPrev].begin(); - mi != mapOrphanTransactionsByPrev[hashPrev].end(); + map<uint256, set<uint256> >::iterator itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue[i]); + if (itByPrev == mapOrphanTransactionsByPrev.end()) + continue; + for (set<uint256>::iterator mi = itByPrev->second.begin(); + mi != itByPrev->second.end(); ++mi) { const uint256& orphanHash = *mi; - const CTransaction& orphanTx = mapOrphanTransactions[orphanHash]; + const CTransaction& orphanTx = mapOrphanTransactions[orphanHash].tx; + NodeId fromPeer = mapOrphanTransactions[orphanHash].fromPeer; bool fMissingInputs2 = false; // Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan // resolution (that is, feeding people an invalid transaction based on LegitTxX in order to get // anyone relaying LegitTxX banned) CValidationState stateDummy; + vEraseQueue.push_back(orphanHash); + + if (setMisbehaving.count(fromPeer)) + continue; if (AcceptToMemoryPool(mempool, stateDummy, orphanTx, true, &fMissingInputs2)) { LogPrint("mempool", " accepted orphan tx %s\n", orphanHash.ToString()); - RelayTransaction(orphanTx, orphanHash); - mapAlreadyAskedFor.erase(CInv(MSG_TX, orphanHash)); + RelayTransaction(orphanTx); vWorkQueue.push_back(orphanHash); - vEraseQueue.push_back(orphanHash); } else if (!fMissingInputs2) { - // invalid or too-little-fee orphan - vEraseQueue.push_back(orphanHash); + int nDos = 0; + if (stateDummy.IsInvalid(nDos) && nDos > 0) + { + // Punish peer that gave us an invalid orphan tx + Misbehaving(fromPeer, nDos); + setMisbehaving.insert(fromPeer); + LogPrint("mempool", " invalid orphan tx %s\n", orphanHash.ToString()); + } + // too-little-fee orphan LogPrint("mempool", " removed orphan tx %s\n", orphanHash.ToString()); } mempool.check(pcoinsTip); @@ -3681,48 +4370,118 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) } else if (fMissingInputs) { - AddOrphanTx(tx); + AddOrphanTx(tx, pfrom->GetId()); // DoS prevention: do not allow mapOrphanTransactions to grow unbounded - unsigned int nEvicted = LimitOrphanTxSize(MAX_ORPHAN_TRANSACTIONS); + unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); + unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx); if (nEvicted > 0) LogPrint("mempool", "mapOrphan overflow, removed %u tx\n", nEvicted); + } else if (pfrom->fWhitelisted) { + // Always relay transactions received from whitelisted peers, even + // if they are already in the mempool (allowing the node to function + // as a gateway for nodes hidden behind it). + RelayTransaction(tx); } int nDoS = 0; if (state.IsInvalid(nDoS)) - { - LogPrint("mempool", "%s from %s %s was not accepted into the memory pool: %s\n", tx.GetHash().ToString(), - pfrom->addr.ToString(), pfrom->cleanSubVer, + { + LogPrint("mempool", "%s from peer=%d %s was not accepted into the memory pool: %s\n", tx.GetHash().ToString(), + pfrom->id, pfrom->cleanSubVer, state.GetRejectReason()); pfrom->PushMessage("reject", strCommand, state.GetRejectCode(), - state.GetRejectReason(), inv.hash); + state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash); if (nDoS > 0) Misbehaving(pfrom->GetId(), nDoS); } } + else if (strCommand == "headers" && !fImporting && !fReindex) // Ignore headers received while importing + { + std::vector<CBlockHeader> headers; + + // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. + unsigned int nCount = ReadCompactSize(vRecv); + if (nCount > MAX_HEADERS_RESULTS) { + Misbehaving(pfrom->GetId(), 20); + return error("headers message size = %u", nCount); + } + headers.resize(nCount); + for (unsigned int n = 0; n < nCount; n++) { + vRecv >> headers[n]; + ReadCompactSize(vRecv); // ignore tx count; assume it is 0. + } + + LOCK(cs_main); + + if (nCount == 0) { + // Nothing interesting. Stop asking this peers for more headers. + return true; + } + + CBlockIndex *pindexLast = NULL; + BOOST_FOREACH(const CBlockHeader& header, headers) { + CValidationState state; + if (pindexLast != NULL && header.hashPrevBlock != pindexLast->GetBlockHash()) { + Misbehaving(pfrom->GetId(), 20); + return error("non-continuous headers sequence"); + } + if (!AcceptBlockHeader(header, state, &pindexLast)) { + int nDoS; + if (state.IsInvalid(nDoS)) { + if (nDoS > 0) + Misbehaving(pfrom->GetId(), nDoS); + return error("invalid header received"); + } + } + } + + if (pindexLast) + UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); + + if (nCount == MAX_HEADERS_RESULTS && pindexLast) { + // Headers message had its maximum size; the peer may have more headers. + // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue + // from there instead. + LogPrint("net", "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); + pfrom->PushMessage("getheaders", chainActive.GetLocator(pindexLast), uint256()); + } + + CheckBlockIndex(); + } + else if (strCommand == "block" && !fImporting && !fReindex) // Ignore blocks received while importing { CBlock block; vRecv >> block; - LogPrint("net", "received block %s\n", block.GetHash().ToString()); - // block.print(); - CInv inv(MSG_BLOCK, block.GetHash()); - pfrom->AddInventoryKnown(inv); + LogPrint("net", "received block %s peer=%d\n", inv.hash.ToString(), pfrom->id); - LOCK(cs_main); - // Remember who we got this block from. - mapBlockSource[inv.hash] = pfrom->GetId(); + pfrom->AddInventoryKnown(inv); CValidationState state; - ProcessBlock(state, pfrom, &block); + ProcessNewBlock(state, pfrom, &block); + int nDoS; + if (state.IsInvalid(nDoS)) { + pfrom->PushMessage("reject", strCommand, state.GetRejectCode(), + state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash); + if (nDoS > 0) { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), nDoS); + } + } + } - else if (strCommand == "getaddr") + // This asymmetric behavior for inbound and outbound connections was introduced + // to prevent a fingerprinting attack: an attacker can send specific fake addresses + // to users' AddrMan and later request them by sending getaddr messages. + // Making nodes which are behind NAT and can only make outgoing connections ignore + // the getaddr message mitigates the attack. + else if ((strCommand == "getaddr") && (pfrom->fInbound)) { pfrom->vAddrToSend.clear(); vector<CAddress> vAddr = addrman.GetAddr(); @@ -3743,7 +4502,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) CTransaction tx; bool fInMemPool = mempool.lookup(hash, tx); if (!fInMemPool) continue; // another thread removed since queryHashes, maybe... - if ((pfrom->pfilter && pfrom->pfilter->IsRelevantAndUpdate(tx, hash)) || + if ((pfrom->pfilter && pfrom->pfilter->IsRelevantAndUpdate(tx)) || (!pfrom->pfilter)) vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { @@ -3780,7 +4539,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) else if (strCommand == "pong") { - int64_t pingUsecEnd = GetTimeMicros(); + int64_t pingUsecEnd = nTimeReceived; uint64_t nonce = 0; size_t nAvail = vRecv.in_avail(); bool bPingFinished = false; @@ -3806,7 +4565,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) // Nonce mismatches are normal when pings are overlapping sProblem = "Nonce mismatch"; if (nonce == 0) { - // This is most likely a bug in another implementation somewhere, cancel this ping + // This is most likely a bug in another implementation somewhere; cancel this ping bPingFinished = true; sProblem = "Nonce zero"; } @@ -3815,14 +4574,14 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) sProblem = "Unsolicited pong without ping"; } } else { - // This is most likely a bug in another implementation somewhere, cancel this ping + // This is most likely a bug in another implementation somewhere; cancel this ping bPingFinished = true; sProblem = "Short payload"; } if (!(sProblem.empty())) { - LogPrint("net", "pong %s %s: %s, %x expected, %x received, %"PRIszu" bytes\n", - pfrom->addr.ToString(), + LogPrint("net", "pong peer=%d %s: %s, %x expected, %x received, %u bytes\n", + pfrom->id, pfrom->cleanSubVer, sProblem, pfrom->nPingNonceSent, @@ -3843,7 +4602,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) uint256 alertHash = alert.GetHash(); if (pfrom->setKnown.count(alertHash) == 0) { - if (alert.ProcessAlert()) + if (alert.ProcessAlert(Params().AlertKey())) { // Relay pfrom->setKnown.insert(alertHash); @@ -3916,38 +4675,35 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) else if (strCommand == "reject") { - if (fDebug) - { - string strMsg; unsigned char ccode; string strReason; - vRecv >> strMsg >> ccode >> strReason; + if (fDebug) { + try { + string strMsg; unsigned char ccode; string strReason; + vRecv >> LIMITED_STRING(strMsg, CMessageHeader::COMMAND_SIZE) >> ccode >> LIMITED_STRING(strReason, MAX_REJECT_MESSAGE_LENGTH); - ostringstream ss; - ss << strMsg << " code " << itostr(ccode) << ": " << strReason; + ostringstream ss; + ss << strMsg << " code " << itostr(ccode) << ": " << strReason; - if (strMsg == "block" || strMsg == "tx") - { - uint256 hash; - vRecv >> hash; - ss << ": hash " << hash.ToString(); + if (strMsg == "block" || strMsg == "tx") + { + uint256 hash; + vRecv >> hash; + ss << ": hash " << hash.ToString(); + } + LogPrint("net", "Reject %s\n", SanitizeString(ss.str())); + } catch (const std::ios_base::failure&) { + // Avoid feedback loops by preventing reject messages from triggering a new reject message. + LogPrint("net", "Unparseable reject message received\n"); } - // Truncate to reasonable length and sanitize before printing: - string s = ss.str(); - if (s.size() > 111) s.erase(111, string::npos); - LogPrint("net", "Reject %s\n", SanitizeString(s)); } } else { // Ignore unknown commands for extensibility + LogPrint("net", "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->id); } - // Update the last seen time for this node's address - if (pfrom->fNetworkNode) - if (strCommand == "version" || strCommand == "addr" || strCommand == "inv" || strCommand == "getdata" || strCommand == "ping") - AddressCurrentlyConnected(pfrom->addr); - return true; } @@ -3956,7 +4712,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) bool ProcessMessages(CNode* pfrom) { //if (fDebug) - // LogPrintf("ProcessMessages(%"PRIszu" messages)\n", pfrom->vRecvMsg.size()); + // LogPrintf("%s(%u messages)\n", __func__, pfrom->vRecvMsg.size()); // // Message format @@ -3984,7 +4740,7 @@ bool ProcessMessages(CNode* pfrom) CNetMessage& msg = *it; //if (fDebug) - // LogPrintf("ProcessMessages(message %u msgsz, %"PRIszu" bytes, complete:%s)\n", + // LogPrintf("%s(message %u msgsz, %u bytes, complete:%s)\n", __func__, // msg.hdr.nMessageSize, msg.vRecv.size(), // msg.complete() ? "Y" : "N"); @@ -3997,16 +4753,16 @@ bool ProcessMessages(CNode* pfrom) // Scan for message start if (memcmp(msg.hdr.pchMessageStart, Params().MessageStart(), MESSAGE_START_SIZE) != 0) { - LogPrintf("\n\nPROCESSMESSAGE: INVALID MESSAGESTART\n\n"); + LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->id); fOk = false; break; } // Read header CMessageHeader& hdr = msg.hdr; - if (!hdr.IsValid()) + if (!hdr.IsValid(Params().MessageStart())) { - LogPrintf("\n\nPROCESSMESSAGE: ERRORS IN HEADER %s\n\n\n", hdr.GetCommand()); + LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->id); continue; } string strCommand = hdr.GetCommand(); @@ -4017,12 +4773,11 @@ bool ProcessMessages(CNode* pfrom) // Checksum CDataStream& vRecv = msg.vRecv; uint256 hash = Hash(vRecv.begin(), vRecv.begin() + nMessageSize); - unsigned int nChecksum = 0; - memcpy(&nChecksum, &hash, sizeof(nChecksum)); + unsigned int nChecksum = ReadLE32((unsigned char*)&hash); if (nChecksum != hdr.nChecksum) { - LogPrintf("ProcessMessages(%s, %u bytes) : CHECKSUM ERROR nChecksum=%08x hdr.nChecksum=%08x\n", - strCommand, nMessageSize, nChecksum, hdr.nChecksum); + LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR nChecksum=%08x hdr.nChecksum=%08x\n", __func__, + SanitizeString(strCommand), nMessageSize, nChecksum, hdr.nChecksum); continue; } @@ -4030,38 +4785,38 @@ bool ProcessMessages(CNode* pfrom) bool fRet = false; try { - fRet = ProcessMessage(pfrom, strCommand, vRecv); + fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime); boost::this_thread::interruption_point(); } - catch (std::ios_base::failure& e) + catch (const std::ios_base::failure& e) { pfrom->PushMessage("reject", strCommand, REJECT_MALFORMED, string("error parsing message")); if (strstr(e.what(), "end of data")) { // Allow exceptions from under-length message on vRecv - LogPrintf("ProcessMessages(%s, %u bytes) : Exception '%s' caught, normally caused by a message being shorter than its stated length\n", strCommand, nMessageSize, e.what()); + LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } else if (strstr(e.what(), "size too large")) { // Allow exceptions from over-long size - LogPrintf("ProcessMessages(%s, %u bytes) : Exception '%s' caught\n", strCommand, nMessageSize, e.what()); + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } else { PrintExceptionContinue(&e, "ProcessMessages()"); } } - catch (boost::thread_interrupted) { + catch (const boost::thread_interrupted&) { throw; } - catch (std::exception& e) { + catch (const std::exception& e) { PrintExceptionContinue(&e, "ProcessMessages()"); } catch (...) { PrintExceptionContinue(NULL, "ProcessMessages()"); } if (!fRet) - LogPrintf("ProcessMessage(%s, %u bytes) FAILED\n", strCommand, nMessageSize); + LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->id); break; } @@ -4076,8 +4831,9 @@ bool ProcessMessages(CNode* pfrom) bool SendMessages(CNode* pto, bool fSendTrickle) { + const Consensus::Params& consensusParams = Params().GetConsensus(); { - // Don't send anything until we get their version message + // Don't send anything until we get its version message if (pto->nVersion == 0) return true; @@ -4089,50 +4845,47 @@ bool SendMessages(CNode* pto, bool fSendTrickle) // RPC ping request by user pingSend = true; } - if (pto->nLastSend && GetTime() - pto->nLastSend > 30 * 60 && pto->vSendMsg.empty()) { - // Ping automatically sent as a keepalive + if (pto->nPingNonceSent == 0 && pto->nPingUsecStart + PING_INTERVAL * 1000000 < GetTimeMicros()) { + // Ping automatically sent as a latency probe & keepalive. pingSend = true; } if (pingSend) { uint64_t nonce = 0; while (nonce == 0) { - RAND_bytes((unsigned char*)&nonce, sizeof(nonce)); + GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); } - pto->nPingNonceSent = nonce; pto->fPingQueued = false; + pto->nPingUsecStart = GetTimeMicros(); if (pto->nVersion > BIP0031_VERSION) { - // Take timestamp as close as possible before transmitting ping - pto->nPingUsecStart = GetTimeMicros(); + pto->nPingNonceSent = nonce; pto->PushMessage("ping", nonce); } else { - // Peer is too old to support ping command with nonce, pong will never arrive, disable timing - pto->nPingUsecStart = 0; + // Peer is too old to support ping command with nonce, pong will never arrive. + pto->nPingNonceSent = 0; pto->PushMessage("ping"); } } + TRY_LOCK(cs_main, lockMain); // Acquire cs_main for IsInitialBlockDownload() and CNodeState() + if (!lockMain) + return true; + // Address refresh broadcast static int64_t nLastRebroadcast; if (!IsInitialBlockDownload() && (GetTime() - nLastRebroadcast > 24 * 60 * 60)) { + LOCK(cs_vNodes); + BOOST_FOREACH(CNode* pnode, vNodes) { - LOCK(cs_vNodes); - BOOST_FOREACH(CNode* pnode, vNodes) - { - // Periodically clear setAddrKnown to allow refresh broadcasts - if (nLastRebroadcast) - pnode->setAddrKnown.clear(); + // Periodically clear addrKnown to allow refresh broadcasts + if (nLastRebroadcast) + pnode->addrKnown.clear(); - // Rebroadcast our address - if (!fNoListen) - { - CAddress addr = GetLocalAddress(&pnode->addr); - if (addr.IsRoutable()) - pnode->PushAddress(addr); - } - } + // Rebroadcast our address + AdvertizeLocal(pnode); } - nLastRebroadcast = GetTime(); + if (!vNodes.empty()) + nLastRebroadcast = GetTime(); } // @@ -4144,9 +4897,9 @@ bool SendMessages(CNode* pto, bool fSendTrickle) vAddr.reserve(pto->vAddrToSend.size()); BOOST_FOREACH(const CAddress& addr, pto->vAddrToSend) { - // returns true if wasn't already contained in the set - if (pto->setAddrKnown.insert(addr).second) + if (!pto->addrKnown.contains(addr.GetKey())) { + pto->addrKnown.insert(addr.GetKey()); vAddr.push_back(addr); // receiver rejects addr messages larger than 1000 if (vAddr.size() >= 1000) @@ -4161,17 +4914,18 @@ bool SendMessages(CNode* pto, bool fSendTrickle) pto->PushMessage("addr", vAddr); } - TRY_LOCK(cs_main, lockMain); - if (!lockMain) - return true; - CNodeState &state = *State(pto->GetId()); if (state.fShouldBan) { - if (pto->addr.IsLocal()) - LogPrintf("Warning: not banning local node %s!\n", pto->addr.ToString()); + if (pto->fWhitelisted) + LogPrintf("Warning: not punishing whitelisted peer %s!\n", pto->addr.ToString()); else { pto->fDisconnect = true; - CNode::Ban(pto->addr); + if (pto->addr.IsLocal()) + LogPrintf("Warning: not banning local peer %s!\n", pto->addr.ToString()); + else + { + CNode::Ban(pto->addr); + } } state.fShouldBan = false; } @@ -4181,9 +4935,18 @@ bool SendMessages(CNode* pto, bool fSendTrickle) state.rejects.clear(); // Start block sync - if (pto->fStartSync && !fImporting && !fReindex) { - pto->fStartSync = false; - PushGetBlocks(pto, chainActive.Tip(), uint256(0)); + if (pindexBestHeader == NULL) + pindexBestHeader = chainActive.Tip(); + bool fFetch = state.fPreferredDownload || (nPreferredDownload == 0 && !pto->fClient && !pto->fOneShot); // Download if this is a nice peer, or we have no nice peers and this one might do. + if (!state.fSyncStarted && !pto->fClient && !fImporting && !fReindex) { + // Only actively request headers from a single peer, unless we're close to today. + if ((nSyncStarted == 0 && fFetch) || pindexBestHeader->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { + state.fSyncStarted = true; + nSyncStarted++; + CBlockIndex *pindexStart = pindexBestHeader->pprev ? pindexBestHeader->pprev : pindexBestHeader; + LogPrint("net", "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); + pto->PushMessage("getheaders", chainActive.GetLocator(pindexStart), uint256()); + } } // Resend wallet transactions that haven't gotten in a block yet @@ -4191,7 +4954,7 @@ bool SendMessages(CNode* pto, bool fSendTrickle) // transactions become unconfirmed and spams other nodes. if (!fReindex && !fImporting && !IsInitialBlockDownload()) { - g_signals.Broadcast(); + GetMainSignals().Broadcast(nTimeBestReceived); } // @@ -4213,11 +4976,11 @@ bool SendMessages(CNode* pto, bool fSendTrickle) { // 1/4 of tx invs blast to all immediately static uint256 hashSalt; - if (hashSalt == 0) + if (hashSalt.IsNull()) hashSalt = GetRandHash(); - uint256 hashRand = inv.hash ^ hashSalt; + uint256 hashRand = ArithToUint256(UintToArith256(inv.hash) ^ UintToArith256(hashSalt)); hashRand = Hash(BEGIN(hashRand), END(hashRand)); - bool fTrickleWait = ((hashRand & 3) != 0); + bool fTrickleWait = ((UintToArith256(hashRand) & 3) != 0); if (fTrickleWait) { @@ -4242,19 +5005,70 @@ bool SendMessages(CNode* pto, bool fSendTrickle) if (!vInv.empty()) pto->PushMessage("inv", vInv); + // Detect whether we're stalling + int64_t nNow = GetTimeMicros(); + if (!pto->fDisconnect && state.nStallingSince && state.nStallingSince < nNow - 1000000 * BLOCK_STALLING_TIMEOUT) { + // Stalling only triggers when the block download window cannot move. During normal steady state, + // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection + // should only happen during initial block download. + LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->id); + pto->fDisconnect = true; + } + // In case there is a block that has been in flight from this peer for (2 + 0.5 * N) times the block interval + // (with N the number of validated blocks that were in flight at the time it was requested), disconnect due to + // timeout. We compensate for in-flight blocks to prevent killing off peers due to our own downstream link + // being saturated. We only count validated in-flight blocks so peers can't advertise non-existing block hashes + // to unreasonably increase our timeout. + // We also compare the block download timeout originally calculated against the time at which we'd disconnect + // if we assumed the block were being requested now (ignoring blocks we've requested from this peer, since we're + // only looking at this peer's oldest request). This way a large queue in the past doesn't result in a + // permanently large window for this block to be delivered (ie if the number of blocks in flight is decreasing + // more quickly than once every 5 minutes, then we'll shorten the download window for this block). + if (!pto->fDisconnect && state.vBlocksInFlight.size() > 0) { + QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); + int64_t nTimeoutIfRequestedNow = GetBlockTimeout(nNow, nQueuedValidatedHeaders - state.nBlocksInFlightValidHeaders, consensusParams); + if (queuedBlock.nTimeDisconnect > nTimeoutIfRequestedNow) { + LogPrint("net", "Reducing block download timeout for peer=%d block=%s, orig=%d new=%d\n", pto->id, queuedBlock.hash.ToString(), queuedBlock.nTimeDisconnect, nTimeoutIfRequestedNow); + queuedBlock.nTimeDisconnect = nTimeoutIfRequestedNow; + } + if (queuedBlock.nTimeDisconnect < nNow) { + LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.hash.ToString(), pto->id); + pto->fDisconnect = true; + } + } // - // Message: getdata + // Message: getdata (blocks) // vector<CInv> vGetData; - int64_t nNow = GetTime() * 1000000; - while (!pto->mapAskFor.empty() && (*pto->mapAskFor.begin()).first <= nNow) + if (!pto->fDisconnect && !pto->fClient && (fFetch || !IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + vector<CBlockIndex*> vToDownload; + NodeId staller = -1; + FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); + BOOST_FOREACH(CBlockIndex *pindex, vToDownload) { + vGetData.push_back(CInv(MSG_BLOCK, pindex->GetBlockHash())); + MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), consensusParams, pindex); + LogPrint("net", "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), + pindex->nHeight, pto->id); + } + if (state.nBlocksInFlight == 0 && staller != -1) { + if (State(staller)->nStallingSince == 0) { + State(staller)->nStallingSince = nNow; + LogPrint("net", "Stall started peer=%d\n", staller); + } + } + } + + // + // Message: getdata (non-blocks) + // + while (!pto->fDisconnect && !pto->mapAskFor.empty() && (*pto->mapAskFor.begin()).first <= nNow) { const CInv& inv = (*pto->mapAskFor.begin()).second; if (!AlreadyHave(inv)) { if (fDebug) - LogPrint("net", "sending getdata: %s\n", inv.ToString()); + LogPrint("net", "Requesting %s peer=%d\n", inv.ToString(), pto->id); vGetData.push_back(inv); if (vGetData.size() >= 1000) { @@ -4271,8 +5085,9 @@ bool SendMessages(CNode* pto, bool fSendTrickle) return true; } - - + std::string CBlockFileInfo::ToString() const { + return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, DateTimeStrFormat("%Y-%m-%d", nTimeFirst), DateTimeStrFormat("%Y-%m-%d", nTimeLast)); + } @@ -4282,18 +5097,13 @@ public: CMainCleanup() {} ~CMainCleanup() { // block headers - std::map<uint256, CBlockIndex*>::iterator it1 = mapBlockIndex.begin(); + BlockMap::iterator it1 = mapBlockIndex.begin(); for (; it1 != mapBlockIndex.end(); it1++) delete (*it1).second; mapBlockIndex.clear(); - // orphan blocks - std::map<uint256, COrphanBlock*>::iterator it2 = mapOrphanBlocks.begin(); - for (; it2 != mapOrphanBlocks.end(); it2++) - delete (*it2).second; - mapOrphanBlocks.clear(); - // orphan transactions mapOrphanTransactions.clear(); + mapOrphanTransactionsByPrev.clear(); } } instance_of_cmaincleanup; |