From c7eb6b4f1fe5bd76388a023529977674534334a7 Mon Sep 17 00:00:00 2001 From: Amiti Uttarwar Date: Thu, 30 Apr 2020 18:20:01 -0700 Subject: Add wtxid to mempool unbroadcast tracking --- src/net_processing.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/net_processing.cpp') diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 7a58de35d..6663ece8d 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -830,14 +830,14 @@ void PeerLogicValidation::InitializeNode(CNode *pnode) { void PeerLogicValidation::ReattemptInitialBroadcast(CScheduler& scheduler) const { - std::set unbroadcast_txids = m_mempool.GetUnbroadcastTxs(); + std::map unbroadcast_txids = m_mempool.GetUnbroadcastTxs(); - for (const uint256& txid : unbroadcast_txids) { + for (const auto& elem : unbroadcast_txids) { // Sanity check: all unbroadcast txns should exist in the mempool - if (m_mempool.exists(txid)) { - RelayTransaction(txid, *connman); + if (m_mempool.exists(elem.first)) { + RelayTransaction(elem.first, *connman); } else { - m_mempool.RemoveUnbroadcastTx(txid, true); + m_mempool.RemoveUnbroadcastTx(elem.first, true); } } -- cgit v1.2.3 From 60f0acda713e7b9dc188aef54ef93981a93f4e44 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Thu, 30 Jan 2020 11:12:56 -0500 Subject: Just pass a hash to AddInventoryKnown Since it's only used for transactions, there's no need to pass in an inv type. --- src/net_processing.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'src/net_processing.cpp') diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 6663ece8d..1373338cf 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2593,7 +2593,7 @@ void ProcessMessage( best_block = &inv.hash; } } else { - pfrom.AddInventoryKnown(inv); + pfrom.AddInventoryKnown(inv.hash); if (fBlocksOnly) { LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); pfrom.fDisconnect = true; @@ -2832,26 +2832,26 @@ void ProcessMessage( vRecv >> ptx; const CTransaction& tx = *ptx; - CInv inv(MSG_TX, tx.GetHash()); - pfrom.AddInventoryKnown(inv); + const uint256& txid = ptx->GetHash(); + pfrom.AddInventoryKnown(txid); LOCK2(cs_main, g_cs_orphans); TxValidationState state; CNodeState* nodestate = State(pfrom.GetId()); - nodestate->m_tx_download.m_tx_announced.erase(inv.hash); - nodestate->m_tx_download.m_tx_in_flight.erase(inv.hash); - EraseTxRequest(inv.hash); + nodestate->m_tx_download.m_tx_announced.erase(txid); + nodestate->m_tx_download.m_tx_in_flight.erase(txid); + EraseTxRequest(txid); std::list lRemovedTxn; - if (!AlreadyHave(inv, mempool) && + if (!AlreadyHave(CInv(MSG_TX, txid), mempool) && AcceptToMemoryPool(mempool, state, ptx, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { mempool.check(&::ChainstateActive().CoinsTip()); RelayTransaction(tx.GetHash(), connman); for (unsigned int i = 0; i < tx.vout.size(); i++) { - auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(inv.hash, i)); + auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(txid, i)); if (it_by_prev != mapOrphanTransactionsByPrev.end()) { for (const auto& elem : it_by_prev->second) { pfrom.orphan_work_set.insert(elem->first); @@ -2884,7 +2884,7 @@ void ProcessMessage( for (const CTxIn& txin : tx.vin) { CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); - pfrom.AddInventoryKnown(_inv); + pfrom.AddInventoryKnown(txin.prevout.hash); if (!AlreadyHave(_inv, mempool)) RequestTx(State(pfrom.GetId()), _inv.hash, current_time); } AddOrphanTx(ptx, pfrom.GetId()); -- cgit v1.2.3 From 08b39955ec7f84e835ab0b1366f0dd28dfd6ce03 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Wed, 29 Jan 2020 10:40:54 -0500 Subject: Add a wtxid-index to mapRelay --- src/net_processing.cpp | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/net_processing.cpp') diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 1373338cf..7dd8da307 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -4246,6 +4246,11 @@ bool PeerLogicValidation::SendMessages(CNode* pto) if (ret.second) { vRelayExpiration.push_back(std::make_pair(nNow + std::chrono::microseconds{RELAY_TX_CACHE_TIME}.count(), ret.first)); } + // Add wtxid-based lookup into mapRelay as well, so that peers can request by wtxid + auto ret2 = mapRelay.emplace(ret.first->second->GetWitnessHash(), ret.first->second); + if (ret2.second) { + vRelayExpiration.emplace_back(nNow + std::chrono::microseconds{RELAY_TX_CACHE_TIME}.count(), ret2.first); + } } if (vInv.size() == MAX_INV_SZ) { connman->PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); -- cgit v1.2.3 From 85c78d54af462996a0bca6cf97f91e1aa8778ae8 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Wed, 29 Jan 2020 10:51:45 -0500 Subject: Add wtxid-index to orphan map --- src/net_processing.cpp | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/net_processing.cpp') diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 7dd8da307..d6fa777a4 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -151,6 +151,7 @@ struct COrphanTx { }; RecursiveMutex g_cs_orphans; std::map mapOrphanTransactions GUARDED_BY(g_cs_orphans); +std::map::iterator> g_orphans_by_wtxid GUARDED_BY(g_cs_orphans); void EraseOrphansFor(NodeId peer); @@ -936,6 +937,8 @@ bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRE auto ret = mapOrphanTransactions.emplace(hash, COrphanTx{tx, peer, GetTime() + ORPHAN_TX_EXPIRE_TIME, g_orphan_list.size()}); assert(ret.second); g_orphan_list.push_back(ret.first); + // Allow for lookups in the orphan pool by wtxid, as well as txid + g_orphans_by_wtxid.emplace(tx->GetWitnessHash(), ret.first); for (const CTxIn& txin : tx->vin) { mapOrphanTransactionsByPrev[txin.prevout].insert(ret.first); } @@ -972,6 +975,7 @@ int static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) it_last->second.list_pos = old_pos; } g_orphan_list.pop_back(); + g_orphans_by_wtxid.erase(it->second.tx->GetWitnessHash()); mapOrphanTransactions.erase(it); return 1; @@ -4464,6 +4468,7 @@ public: // orphan transactions mapOrphanTransactions.clear(); mapOrphanTransactionsByPrev.clear(); + g_orphans_by_wtxid.clear(); } }; static CNetProcessingCleanup instance_of_cnetprocessingcleanup; -- cgit v1.2.3 From 144c38582042c3b9ec53bb97ba0644fc0114f8fb Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Wed, 29 Jan 2020 10:57:08 -0500 Subject: Add wtxids of confirmed transactions to bloom filter This is in preparation for wtxid-based invs (we need to be able to tell whether we AlreadyHave() a transaction based on either txid or wtxid). This also double the size of the bloom filter, which is overkill, but still uses a manageable amount of memory. --- src/net_processing.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src/net_processing.cpp') diff --git a/src/net_processing.cpp b/src/net_processing.cpp index d6fa777a4..efc90f85e 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1185,14 +1185,15 @@ PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CS recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); // Blocks don't typically have more than 4000 transactions, so this should - // be at least six blocks (~1 hr) worth of transactions that we can store. + // be at least six blocks (~1 hr) worth of transactions that we can store, + // inserting both a txid and wtxid for every observed transaction. // If the number of transactions appearing in a block goes up, or if we are // seeing getdata requests more than an hour after initial announcement, we // can increase this number. // The false positive rate of 1/1M should come out to less than 1 // transaction per day that would be inadvertently ignored (which is the // same probability that we have in the reject filter). - g_recent_confirmed_transactions.reset(new CRollingBloomFilter(24000, 0.000001)); + g_recent_confirmed_transactions.reset(new CRollingBloomFilter(48000, 0.000001)); const Consensus::Params& consensusParams = Params().GetConsensus(); // Stale tip checking and peer eviction are on two different timers, but we @@ -1248,6 +1249,9 @@ void PeerLogicValidation::BlockConnected(const std::shared_ptr& pb LOCK(g_cs_recent_confirmed_transactions); for (const auto& ptx : pblock->vtx) { g_recent_confirmed_transactions->insert(ptx->GetHash()); + if (ptx->GetHash() != ptx->GetWitnessHash()) { + g_recent_confirmed_transactions->insert(ptx->GetWitnessHash()); + } } } } -- cgit v1.2.3 From 8e68fc246d09f1e6c6dfa8c676969d97c2eb4334 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Wed, 29 Jan 2020 14:09:08 -0500 Subject: Add wtxids to recentRejects instead of txids Previously, we only added txids to recentRejects if we were sure that the transaction couldn't have had the wrong witness (either because the witness was malleated or stripped). In preparation for wtxid-based relay, we can observe that txid == wtxid for transactions that have no witness, and add the wtxid of rejected transactions, provided the transaction wasn't a witness-stripped one. This means that we now add more data to the filter (as prior to this commit, any transaction with a witness that failed to be accepted was being skipped for inclusion in the filter) but witness malleation should still not interfere with relay of a valid segwit transaction, because the txid of a segwit transaction would not be added to the filter after failing validation. In the future, having wtxids in the recent rejects filter will allow us to skip downloading the same wtxid multiple times, once our peers use wtxids for transaction relay. --- src/net_processing.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src/net_processing.cpp') diff --git a/src/net_processing.cpp b/src/net_processing.cpp index efc90f85e..511d5b66e 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2005,12 +2005,12 @@ void static ProcessOrphanTx(CConnman& connman, CTxMemPool& mempool, std::setinsert(orphanHash); + recentRejects->insert(orphanTx.GetWitnessHash()); } EraseOrphanTx(orphanHash); done = true; @@ -2908,14 +2908,15 @@ void ProcessMessage( // We will continue to reject this tx since it has rejected // parents so avoid re-requesting it from other peers. recentRejects->insert(tx.GetHash()); + recentRejects->insert(tx.GetWitnessHash()); } } else { - if (!tx.HasWitness() && state.GetResult() != TxValidationResult::TX_WITNESS_MUTATED) { + if (tx.HasWitness() || state.GetResult() != TxValidationResult::TX_WITNESS_MUTATED) { // Do not use rejection cache for witness transactions or // witness-stripped transactions, as they can have been malleated. // See https://github.com/bitcoin/bitcoin/issues/8279 for details. assert(recentRejects); - recentRejects->insert(tx.GetHash()); + recentRejects->insert(tx.GetWitnessHash()); if (RecursiveDynamicUsage(*ptx) < 100000) { AddToCompactExtraTransactions(ptx); } -- cgit v1.2.3 From ac88e2eb619821ad7ae1d45d4b40be69051d3999 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Thu, 30 Jan 2020 09:35:00 -0500 Subject: Add support for tx-relay via wtxid This adds a field to CNodeState that tracks whether to relay transactions with that peer via wtxid, instead of txid. As of this commit the field will always be false, but in a later commit we will add a way to negotiate turning this on via p2p messages exchanged with the peer. --- src/net_processing.cpp | 138 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 98 insertions(+), 40 deletions(-) (limited to 'src/net_processing.cpp') diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 511d5b66e..55c67ed18 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -410,6 +410,9 @@ struct CNodeState { //! A rolling bloom filter of all announced tx CInvs to this peer. CRollingBloomFilter m_recently_announced_invs = CRollingBloomFilter{INVENTORY_MAX_RECENT_RELAY, 0.000001}; + //! Whether this peer relays txs via wtxid + bool m_wtxid_relay{false}; + CNodeState(CAddress addrIn, std::string addrNameIn, bool is_inbound, bool is_manual) : address(addrIn), name(std::move(addrNameIn)), m_is_inbound(is_inbound), m_is_manual_connection (is_manual) @@ -836,7 +839,8 @@ void PeerLogicValidation::ReattemptInitialBroadcast(CScheduler& scheduler) const for (const auto& elem : unbroadcast_txids) { // Sanity check: all unbroadcast txns should exist in the mempool if (m_mempool.exists(elem.first)) { - RelayTransaction(elem.first, *connman); + LOCK(cs_main); + RelayTransaction(elem.first, elem.second, *connman); } else { m_mempool.RemoveUnbroadcastTx(elem.first, true); } @@ -1405,6 +1409,7 @@ bool static AlreadyHave(const CInv& inv, const CTxMemPool& mempool) EXCLUSIVE_LO { case MSG_TX: case MSG_WITNESS_TX: + case MSG_WTX: { assert(recentRejects); if (::ChainActive().Tip()->GetBlockHash() != hashRecentRejectsChainTip) @@ -1419,7 +1424,11 @@ bool static AlreadyHave(const CInv& inv, const CTxMemPool& mempool) EXCLUSIVE_LO { LOCK(g_cs_orphans); - if (mapOrphanTransactions.count(inv.hash)) return true; + if (inv.type != MSG_WTX && mapOrphanTransactions.count(inv.hash)) { + return true; + } else if (inv.type == MSG_WTX && g_orphans_by_wtxid.count(inv.hash)) { + return true; + } } { @@ -1427,8 +1436,8 @@ bool static AlreadyHave(const CInv& inv, const CTxMemPool& mempool) EXCLUSIVE_LO if (g_recent_confirmed_transactions->contains(inv.hash)) return true; } - return recentRejects->contains(inv.hash) || - mempool.exists(inv.hash); + const bool by_wtxid = (inv.type == MSG_WTX); + return recentRejects->contains(inv.hash) || mempool.exists(inv.hash, by_wtxid); } case MSG_BLOCK: case MSG_WITNESS_BLOCK: @@ -1438,11 +1447,17 @@ bool static AlreadyHave(const CInv& inv, const CTxMemPool& mempool) EXCLUSIVE_LO return true; } -void RelayTransaction(const uint256& txid, const CConnman& connman) +void RelayTransaction(const uint256& txid, const uint256& wtxid, const CConnman& connman) { - connman.ForEachNode([&txid](CNode* pnode) + connman.ForEachNode([&txid, &wtxid](CNode* pnode) { - pnode->PushTxInventory(txid); + AssertLockHeld(cs_main); + CNodeState &state = *State(pnode->GetId()); + if (state.m_wtxid_relay) { + pnode->PushTxInventory(wtxid); + } else { + pnode->PushTxInventory(txid); + } }); } @@ -1640,9 +1655,9 @@ void static ProcessGetBlockData(CNode& pfrom, const CChainParams& chainparams, c } //! Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). -CTransactionRef static FindTxForGetData(const CNode& peer, const uint256& txid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) LOCKS_EXCLUDED(cs_main) +CTransactionRef static FindTxForGetData(const CNode& peer, const uint256& txid_or_wtxid, bool use_wtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) LOCKS_EXCLUDED(cs_main) { - auto txinfo = mempool.info(txid); + auto txinfo = mempool.info(txid_or_wtxid, use_wtxid); if (txinfo.tx) { // If a TX could have been INVed in reply to a MEMPOOL request, // or is older than UNCONDITIONAL_RELAY_DELAY, permit the request @@ -1654,13 +1669,12 @@ CTransactionRef static FindTxForGetData(const CNode& peer, const uint256& txid, { LOCK(cs_main); - // Otherwise, the transaction must have been announced recently. - if (State(peer.GetId())->m_recently_announced_invs.contains(txid)) { + if (State(peer.GetId())->m_recently_announced_invs.contains(txid_or_wtxid)) { // If it was, it can be relayed from either the mempool... if (txinfo.tx) return std::move(txinfo.tx); // ... or the relay pool. - auto mi = mapRelay.find(txid); + auto mi = mapRelay.find(txid_or_wtxid); if (mi != mapRelay.end()) return mi->second; } } @@ -1684,7 +1698,7 @@ void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnm // Process as many TX items from the front of the getdata queue as // possible, since they're common and it's efficient to batch process // them. - while (it != pfrom.vRecvGetData.end() && (it->type == MSG_TX || it->type == MSG_WITNESS_TX)) { + while (it != pfrom.vRecvGetData.end() && (it->type == MSG_TX || it->type == MSG_WITNESS_TX || it->type == MSG_WTX)) { if (interruptMsgProc) return; // The send buffer provides backpressure. If there's no space in // the buffer, pause processing until the next call. @@ -1697,11 +1711,12 @@ void static ProcessGetData(CNode& pfrom, const CChainParams& chainparams, CConnm continue; } - CTransactionRef tx = FindTxForGetData(pfrom, inv.hash, mempool_req, now); + CTransactionRef tx = FindTxForGetData(pfrom, inv.hash, inv.type == MSG_WTX, mempool_req, now); if (tx) { + // WTX and WITNESS_TX imply we serialize with witness int nSendFlags = (inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); - mempool.RemoveUnbroadcastTx(inv.hash); + mempool.RemoveUnbroadcastTx(tx->GetHash()); // As we're going to send tx, make sure its unconfirmed parents are made requestable. for (const auto& txin : tx->vin) { auto txinfo = mempool.info(txin.prevout.hash); @@ -1980,7 +1995,7 @@ void static ProcessOrphanTx(CConnman& connman, CTxMemPool& mempool, std::setGetWitnessHash(), connman); for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(orphanHash, i)); if (it_by_prev != mapOrphanTransactionsByPrev.end()) { @@ -2841,23 +2856,47 @@ void ProcessMessage( const CTransaction& tx = *ptx; const uint256& txid = ptx->GetHash(); - pfrom.AddInventoryKnown(txid); + const uint256& wtxid = ptx->GetWitnessHash(); LOCK2(cs_main, g_cs_orphans); + CNodeState* nodestate = State(pfrom.GetId()); + + const uint256& hash = nodestate->m_wtxid_relay ? wtxid : txid; + pfrom.AddInventoryKnown(hash); + if (nodestate->m_wtxid_relay && txid != wtxid) { + // Insert txid into filterInventoryKnown, even for + // wtxidrelay peers. This prevents re-adding of + // unconfirmed parents to the recently_announced + // filter, when a child tx is requested. See + // ProcessGetData(). + pfrom.AddInventoryKnown(txid); + } + TxValidationState state; - CNodeState* nodestate = State(pfrom.GetId()); - nodestate->m_tx_download.m_tx_announced.erase(txid); - nodestate->m_tx_download.m_tx_in_flight.erase(txid); - EraseTxRequest(txid); + nodestate->m_tx_download.m_tx_announced.erase(hash); + nodestate->m_tx_download.m_tx_in_flight.erase(hash); + EraseTxRequest(hash); std::list lRemovedTxn; - if (!AlreadyHave(CInv(MSG_TX, txid), mempool) && + // We do the AlreadyHave() check using wtxid, rather than txid - in the + // absence of witness malleation, this is strictly better, because the + // recent rejects filter may contain the wtxid but will never contain + // the txid of a segwit transaction that has been rejected. + // In the presence of witness malleation, it's possible that by only + // doing the check with wtxid, we could overlook a transaction which + // was confirmed with a different witness, or exists in our mempool + // with a different witness, but this has limited downside: + // mempool validation does its own lookup of whether we have the txid + // already; and an adversary can already relay us old transactions + // (older than our recency filter) if trying to DoS us, without any need + // for witness malleation. + if (!AlreadyHave(CInv(MSG_WTX, wtxid), mempool) && AcceptToMemoryPool(mempool, state, ptx, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { mempool.check(&::ChainstateActive().CoinsTip()); - RelayTransaction(tx.GetHash(), connman); + RelayTransaction(tx.GetHash(), tx.GetWitnessHash(), connman); for (unsigned int i = 0; i < tx.vout.size(); i++) { auto it_by_prev = mapOrphanTransactionsByPrev.find(COutPoint(txid, i)); if (it_by_prev != mapOrphanTransactionsByPrev.end()) { @@ -2890,10 +2929,17 @@ void ProcessMessage( uint32_t nFetchFlags = GetFetchFlags(pfrom); const auto current_time = GetTime(); - for (const CTxIn& txin : tx.vin) { - CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); - pfrom.AddInventoryKnown(txin.prevout.hash); - if (!AlreadyHave(_inv, mempool)) RequestTx(State(pfrom.GetId()), _inv.hash, current_time); + if (!State(pfrom.GetId())->m_wtxid_relay) { + for (const CTxIn& txin : tx.vin) { + // Here, we only have the txid (and not wtxid) of the + // inputs, so we only request parents from + // non-wtxid-relay peers. + // Eventually we should replace this with an improved + // protocol for getting all unconfirmed parents. + CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); + pfrom.AddInventoryKnown(txin.prevout.hash); + if (!AlreadyHave(_inv, mempool)) RequestTx(State(pfrom.GetId()), _inv.hash, current_time); + } } AddOrphanTx(ptx, pfrom.GetId()); @@ -2933,7 +2979,7 @@ void ProcessMessage( LogPrintf("Not relaying non-mempool transaction %s from forcerelay peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); } else { LogPrintf("Force relaying tx %s from peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); - RelayTransaction(tx.GetHash(), connman); + RelayTransaction(tx.GetHash(), tx.GetWitnessHash(), connman); } } } @@ -3573,7 +3619,7 @@ void ProcessMessage( vRecv >> vInv; if (vInv.size() <= MAX_PEER_TX_IN_FLIGHT + MAX_BLOCKS_IN_TRANSIT_PER_PEER) { for (CInv &inv : vInv) { - if (inv.type == MSG_TX || inv.type == MSG_WITNESS_TX) { + if (inv.type == MSG_TX || inv.type == MSG_WITNESS_TX || inv.type == MSG_WTX) { // If we receive a NOTFOUND message for a txid we requested, erase // it from our data structures for this peer. auto in_flight_it = state->m_tx_download.m_tx_in_flight.find(inv.hash); @@ -3858,17 +3904,19 @@ namespace { class CompareInvMempoolOrder { CTxMemPool *mp; + bool m_wtxid_relay; public: - explicit CompareInvMempoolOrder(CTxMemPool *_mempool) + explicit CompareInvMempoolOrder(CTxMemPool *_mempool, bool use_wtxid) { mp = _mempool; + m_wtxid_relay = use_wtxid; } bool operator()(std::set::iterator a, std::set::iterator b) { /* As std::make_heap produces a max-heap, we want the entries with the * fewest ancestors/highest fee to sort later. */ - return mp->CompareDepthAndScore(*b, *a); + return mp->CompareDepthAndScore(*b, *a, m_wtxid_relay); } }; } @@ -4175,8 +4223,8 @@ bool PeerLogicValidation::SendMessages(CNode* pto) LOCK(pto->m_tx_relay->cs_filter); for (const auto& txinfo : vtxinfo) { - const uint256& hash = txinfo.tx->GetHash(); - CInv inv(MSG_TX, hash); + const uint256& hash = state.m_wtxid_relay ? txinfo.tx->GetWitnessHash() : txinfo.tx->GetHash(); + CInv inv(state.m_wtxid_relay ? MSG_WTX : MSG_TX, hash); pto->m_tx_relay->setInventoryTxToSend.erase(hash); // Don't send transactions that peers will not put into their mempool if (txinfo.fee < filterrate.GetFee(txinfo.vsize)) { @@ -4211,7 +4259,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) } // Topologically and fee-rate sort the inventory we send for privacy and priority reasons. // A heap is used so that not all items need sorting if only a few are being sent. - CompareInvMempoolOrder compareInvMempoolOrder(&m_mempool); + CompareInvMempoolOrder compareInvMempoolOrder(&m_mempool, state.m_wtxid_relay); std::make_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); // No reason to drain out at many times the network's capacity, // especially since we have many peers and some will draw much shorter delays. @@ -4230,10 +4278,12 @@ bool PeerLogicValidation::SendMessages(CNode* pto) continue; } // Not in the mempool anymore? don't bother sending it. - auto txinfo = m_mempool.info(hash); + auto txinfo = m_mempool.info(hash, state.m_wtxid_relay); if (!txinfo.tx) { continue; } + auto txid = txinfo.tx->GetHash(); + auto wtxid = txinfo.tx->GetWitnessHash(); // Peer told you to not send transactions at that feerate? Don't bother sending it. if (txinfo.fee < filterrate.GetFee(txinfo.vsize)) { continue; @@ -4241,7 +4291,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) if (pto->m_tx_relay->pfilter && !pto->m_tx_relay->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; // Send State(pto->GetId())->m_recently_announced_invs.insert(hash); - vInv.push_back(CInv(MSG_TX, hash)); + vInv.push_back(CInv(state.m_wtxid_relay ? MSG_WTX : MSG_TX, hash)); nRelayedTransactions++; { // Expire old relay messages @@ -4251,12 +4301,12 @@ bool PeerLogicValidation::SendMessages(CNode* pto) vRelayExpiration.pop_front(); } - auto ret = mapRelay.insert(std::make_pair(hash, std::move(txinfo.tx))); + auto ret = mapRelay.emplace(txid, std::move(txinfo.tx)); if (ret.second) { - vRelayExpiration.push_back(std::make_pair(nNow + std::chrono::microseconds{RELAY_TX_CACHE_TIME}.count(), ret.first)); + vRelayExpiration.emplace_back(nNow + std::chrono::microseconds{RELAY_TX_CACHE_TIME}.count(), ret.first); } // Add wtxid-based lookup into mapRelay as well, so that peers can request by wtxid - auto ret2 = mapRelay.emplace(ret.first->second->GetWitnessHash(), ret.first->second); + auto ret2 = mapRelay.emplace(wtxid, ret.first->second); if (ret2.second) { vRelayExpiration.emplace_back(nNow + std::chrono::microseconds{RELAY_TX_CACHE_TIME}.count(), ret2.first); } @@ -4266,6 +4316,14 @@ bool PeerLogicValidation::SendMessages(CNode* pto) vInv.clear(); } pto->m_tx_relay->filterInventoryKnown.insert(hash); + if (hash != txid) { + // Insert txid into filterInventoryKnown, even for + // wtxidrelay peers. This prevents re-adding of + // unconfirmed parents to the recently_announced + // filter, when a child tx is requested. See + // ProcessGetData(). + pto->m_tx_relay->filterInventoryKnown.insert(txid); + } } } } @@ -4390,7 +4448,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // Erase this entry from tx_process_time (it may be added back for // processing at a later time, see below) tx_process_time.erase(tx_process_time.begin()); - CInv inv(MSG_TX | GetFetchFlags(*pto), txid); + CInv inv(state.m_wtxid_relay ? MSG_WTX : (MSG_TX | GetFetchFlags(*pto)), txid); if (!AlreadyHave(inv, m_mempool)) { // If this transaction was last requested more than 1 minute ago, // then request. -- cgit v1.2.3 From 2d282e0cba9761574b6b43d134ca95f3052d7fd2 Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Mon, 6 Apr 2020 19:09:05 +1000 Subject: ignore non-wtxidrelay compliant invs --- src/net_processing.cpp | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/net_processing.cpp') diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 55c67ed18..f3b34957b 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2598,6 +2598,13 @@ void ProcessMessage( if (interruptMsgProc) return; + // ignore INVs that don't match wtxidrelay setting + if (State(pfrom.GetId())->m_wtxid_relay) { + if (inv.type == MSG_TX) continue; + } else { + if (inv.type == MSG_WTX) continue; + } + bool fAlreadyHave = AlreadyHave(inv, mempool); LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); -- cgit v1.2.3 From 46d78d47dea345329ba094310eec56ab00a02ddc Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Thu, 30 Jan 2020 10:10:50 -0500 Subject: Add p2p message "wtxidrelay" When sent to and received from a given peer, enables using wtxid's for announcing and fetching transactions with that peer. --- src/net_processing.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'src/net_processing.cpp') diff --git a/src/net_processing.cpp b/src/net_processing.cpp index f3b34957b..ca94a1db1 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2342,6 +2342,10 @@ void ProcessMessage( if (pfrom.fInbound) PushNodeVersion(pfrom, connman, GetAdjustedTime()); + if (nVersion >= WTXID_RELAY_VERSION) { + connman.PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::WTXIDRELAY)); + } + connman.PushMessage(&pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERACK)); pfrom.nServices = nServices; @@ -2478,6 +2482,18 @@ void ProcessMessage( return; } + // Feature negotiation of wtxidrelay should happen between VERSION and + // VERACK, to avoid relay problems from switching after a connection is up + if (msg_type == NetMsgType::WTXIDRELAY) { + if (pfrom.nVersion >= WTXID_RELAY_VERSION) { + LOCK(cs_main); + if (!State(pfrom.GetId())->m_wtxid_relay) { + State(pfrom.GetId())->m_wtxid_relay = true; + } + } + return; + } + if (!pfrom.fSuccessfullyConnected) { // Must have a verack message before anything else LOCK(cs_main); -- cgit v1.2.3 From 97141ca442daea8fc9c307cf81a02b38dcc28fd8 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Fri, 31 Jan 2020 11:23:27 -0500 Subject: Delay getdata requests from peers using txid-based relay Using both txid and wtxid-based relay with peers means that we could sometimes download the same transaction twice, if announced via two different hashes from different peers. Use a heuristic of delaying txid-peer-getdata requests by 2 seconds, if we have at least one wtxid-based peer. --- src/net_processing.cpp | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) (limited to 'src/net_processing.cpp') diff --git a/src/net_processing.cpp b/src/net_processing.cpp index ca94a1db1..bc09e9409 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -75,6 +75,8 @@ static const unsigned int MAX_INV_SZ = 50000; static constexpr int32_t MAX_PEER_TX_IN_FLIGHT = 100; /** Maximum number of announced transactions from a peer */ static constexpr int32_t MAX_PEER_TX_ANNOUNCEMENTS = 2 * MAX_INV_SZ; +/** How many microseconds to delay requesting transactions via txids, if we have wtxid-relaying peers */ +static constexpr std::chrono::microseconds TXID_RELAY_DELAY{std::chrono::seconds{2}}; /** How many microseconds to delay requesting transactions from inbound peers */ static constexpr std::chrono::microseconds INBOUND_PEER_TX_DELAY{std::chrono::seconds{2}}; /** How long to wait (in microseconds) before downloading a transaction from an additional peer */ @@ -219,6 +221,9 @@ namespace { /** Number of peers from which we're downloading blocks. */ int nPeersWithValidatedDownloads GUARDED_BY(cs_main) = 0; + /** Number of peers with wtxid relay. */ + int g_wtxid_relay_peers GUARDED_BY(cs_main) = 0; + /** Number of outbound peers with m_chain_sync.m_protect. */ int g_outbound_peers_with_protect_from_disconnect GUARDED_BY(cs_main) = 0; @@ -764,7 +769,7 @@ void UpdateTxRequestTime(const uint256& txid, std::chrono::microseconds request_ } } -std::chrono::microseconds CalculateTxGetDataTime(const uint256& txid, std::chrono::microseconds current_time, bool use_inbound_delay) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +std::chrono::microseconds CalculateTxGetDataTime(const uint256& txid, std::chrono::microseconds current_time, bool use_inbound_delay, bool use_txid_delay) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { std::chrono::microseconds process_time; const auto last_request_time = GetTxRequestTime(txid); @@ -780,6 +785,9 @@ std::chrono::microseconds CalculateTxGetDataTime(const uint256& txid, std::chron // We delay processing announcements from inbound peers if (use_inbound_delay) process_time += INBOUND_PEER_TX_DELAY; + // We delay processing announcements from peers that use txid-relay (instead of wtxid) + if (use_txid_delay) process_time += TXID_RELAY_DELAY; + return process_time; } @@ -797,7 +805,7 @@ void RequestTx(CNodeState* state, const uint256& txid, std::chrono::microseconds // Calculate the time to try requesting this transaction. Use // fPreferredDownload as a proxy for outbound peers. - const auto process_time = CalculateTxGetDataTime(txid, current_time, !state->fPreferredDownload); + const auto process_time = CalculateTxGetDataTime(txid, current_time, !state->fPreferredDownload, !state->m_wtxid_relay && g_wtxid_relay_peers > 0); peer_download_state.m_tx_process_time.emplace(process_time, txid); } @@ -874,6 +882,8 @@ void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTim assert(nPeersWithValidatedDownloads >= 0); g_outbound_peers_with_protect_from_disconnect -= state->m_chain_sync.m_protect; assert(g_outbound_peers_with_protect_from_disconnect >= 0); + g_wtxid_relay_peers -= state->m_wtxid_relay; + assert(g_wtxid_relay_peers >= 0); mapNodeState.erase(nodeid); @@ -883,6 +893,7 @@ void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTim assert(nPreferredDownload == 0); assert(nPeersWithValidatedDownloads == 0); assert(g_outbound_peers_with_protect_from_disconnect == 0); + assert(g_wtxid_relay_peers == 0); } LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); } @@ -2489,6 +2500,7 @@ void ProcessMessage( LOCK(cs_main); if (!State(pfrom.GetId())->m_wtxid_relay) { State(pfrom.GetId())->m_wtxid_relay = true; + g_wtxid_relay_peers++; } } return; @@ -4490,7 +4502,15 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // up processing to happen after the download times out // (with a slight delay for inbound peers, to prefer // requests to outbound peers). - const auto next_process_time = CalculateTxGetDataTime(txid, current_time, !state.fPreferredDownload); + // Don't apply the txid-delay to re-requests of a + // transaction; the heuristic of delaying requests to + // txid-relay peers is to save bandwidth on initial + // announcement of a transaction, and doesn't make sense + // for a followup request if our first peer times out (and + // would open us up to an attacker using inbound + // wtxid-relay to prevent us from requesting transactions + // from outbound txid-relay peers). + const auto next_process_time = CalculateTxGetDataTime(txid, current_time, !state.fPreferredDownload, false); tx_process_time.emplace(next_process_time, txid); } } else { -- cgit v1.2.3 From 4eb515574e1012bc8ea5dafc3042dcdf4c766f26 Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Fri, 7 Feb 2020 04:30:41 -0500 Subject: Make TX_WITNESS_STRIPPED its own rejection reason Previously, TX_WITNESS_MUTATED could be returned during transaction validation for either transactions that had a witness that was non-standard, or for transactions that had no witness but were invalid due to segwit validation rules. However, for txid/wtxid-relay considerations, net_processing distinguishes the witness stripped case separately, because it affects whether a wtxid should be able to be added to the reject filter. It is safe to add the wtxid of a witness-mutated transaction to the filter (as that wtxid shouldn't collide with the txid, and hence it wouldn't interfere with transaction relay from txid-relay peers), but it is not safe to add the wtxid (== txid) of a witness-stripped transaction to the filter, because that would interfere with relay of another transaction with the same txid (but different wtxid) when relaying from txid-relay peers. Also updates the comment explaining this logic, and explaining that we can get rid of this complexity once there's a sufficient deployment of wtxid-relaying peers on the network. --- src/net_processing.cpp | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) (limited to 'src/net_processing.cpp') diff --git a/src/net_processing.cpp b/src/net_processing.cpp index bc09e9409..d2ac86ea4 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1160,6 +1160,7 @@ static bool MaybePunishNodeForTx(NodeId nodeid, const TxValidationState& state, case TxValidationResult::TX_MISSING_INPUTS: case TxValidationResult::TX_PREMATURE_SPEND: case TxValidationResult::TX_WITNESS_MUTATED: + case TxValidationResult::TX_WITNESS_STRIPPED: case TxValidationResult::TX_CONFLICT: case TxValidationResult::TX_MEMPOOL_POLICY: break; @@ -2031,10 +2032,19 @@ void static ProcessOrphanTx(CConnman& connman, CTxMemPool& mempool, std::setinsert(orphanTx.GetWitnessHash()); } @@ -2992,10 +3002,19 @@ void ProcessMessage( recentRejects->insert(tx.GetWitnessHash()); } } else { - if (tx.HasWitness() || state.GetResult() != TxValidationResult::TX_WITNESS_MUTATED) { - // Do not use rejection cache for witness transactions or - // witness-stripped transactions, as they can have been malleated. - // See https://github.com/bitcoin/bitcoin/issues/8279 for details. + if (state.GetResult() != TxValidationResult::TX_WITNESS_STRIPPED) { + // Do not add txids of witness transactions or witness-stripped + // transactions to the filter, as they can have been malleated; + // adding such txids to the reject filter would potentially + // interfere with relay of valid transactions from peers that + // do not support wtxid-based relay. See + // https://github.com/bitcoin/bitcoin/issues/8279 for details. + // We can remove this restriction (and always add wtxids to + // the filter even for witness stripped transactions) once + // wtxid-based relay is broadly deployed. + // See also comments in https://github.com/bitcoin/bitcoin/pull/18044#discussion_r443419034 + // for concerns around weakening security of unupgraded nodes + // if we start doing this too early. assert(recentRejects); recentRejects->insert(tx.GetWitnessHash()); if (RecursiveDynamicUsage(*ptx) < 100000) { -- cgit v1.2.3 From dd78d1d641178b473ab1156b71a837b9e686792b Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Wed, 26 Feb 2020 13:36:35 -0500 Subject: Rename AddInventoryKnown() to AddKnownTx() --- src/net_processing.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/net_processing.cpp') diff --git a/src/net_processing.cpp b/src/net_processing.cpp index d2ac86ea4..7822e9c0a 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2661,7 +2661,7 @@ void ProcessMessage( best_block = &inv.hash; } } else { - pfrom.AddInventoryKnown(inv.hash); + pfrom.AddKnownTx(inv.hash); if (fBlocksOnly) { LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); pfrom.fDisconnect = true; @@ -2908,14 +2908,14 @@ void ProcessMessage( CNodeState* nodestate = State(pfrom.GetId()); const uint256& hash = nodestate->m_wtxid_relay ? wtxid : txid; - pfrom.AddInventoryKnown(hash); + pfrom.AddKnownTx(hash); if (nodestate->m_wtxid_relay && txid != wtxid) { // Insert txid into filterInventoryKnown, even for // wtxidrelay peers. This prevents re-adding of // unconfirmed parents to the recently_announced // filter, when a child tx is requested. See // ProcessGetData(). - pfrom.AddInventoryKnown(txid); + pfrom.AddKnownTx(txid); } TxValidationState state; @@ -2982,7 +2982,7 @@ void ProcessMessage( // Eventually we should replace this with an improved // protocol for getting all unconfirmed parents. CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); - pfrom.AddInventoryKnown(txin.prevout.hash); + pfrom.AddKnownTx(txin.prevout.hash); if (!AlreadyHave(_inv, mempool)) RequestTx(State(pfrom.GetId()), _inv.hash, current_time); } } -- cgit v1.2.3 From 0e20cfedb704c1f76bb727e2009867d3d503a03d Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Mon, 29 Jun 2020 14:59:55 -0400 Subject: Disconnect peers sending wtxidrelay message after VERACK --- src/net_processing.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/net_processing.cpp') diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 7822e9c0a..c6eeeabbc 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2506,6 +2506,12 @@ void ProcessMessage( // Feature negotiation of wtxidrelay should happen between VERSION and // VERACK, to avoid relay problems from switching after a connection is up if (msg_type == NetMsgType::WTXIDRELAY) { + if (pfrom.fSuccessfullyConnected) { + // Disconnect peers that send wtxidrelay message after VERACK; this + // must be negotiated between VERSION and VERACK. + pfrom.fDisconnect = true; + return; + } if (pfrom.nVersion >= WTXID_RELAY_VERSION) { LOCK(cs_main); if (!State(pfrom.GetId())->m_wtxid_relay) { -- cgit v1.2.3 From 0a4f1422cd1c20e12a05d7ff1a2ef1d5e7c654bb Mon Sep 17 00:00:00 2001 From: Suhas Daftuar Date: Mon, 29 Jun 2020 17:14:40 -0400 Subject: Further improve comments around recentRejects --- src/net_processing.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'src/net_processing.cpp') diff --git a/src/net_processing.cpp b/src/net_processing.cpp index c6eeeabbc..5f1e7318f 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -190,6 +190,15 @@ namespace { * million to make it highly unlikely for users to have issues with this * filter. * + * We only need to add wtxids to this filter. For non-segwit + * transactions, the txid == wtxid, so this only prevents us from + * re-downloading non-segwit transactions when communicating with + * non-wtxidrelay peers -- which is important for avoiding malleation + * attacks that could otherwise interfere with transaction relay from + * non-wtxidrelay peers. For communicating with wtxidrelay peers, having + * the reject filter store wtxids is exactly what we want to avoid + * redownload of a rejected transaction. + * * Memory used: 1.3 MB */ std::unique_ptr recentRejects GUARDED_BY(cs_main); @@ -2033,6 +2042,7 @@ void static ProcessOrphanTx(CConnman& connman, CTxMemPool& mempool, std::setinsert(tx.GetHash()); recentRejects->insert(tx.GetWitnessHash()); } } else { if (state.GetResult() != TxValidationResult::TX_WITNESS_STRIPPED) { + // We can add the wtxid of this transaction to our reject filter. // Do not add txids of witness transactions or witness-stripped // transactions to the filter, as they can have been malleated; // adding such txids to the reject filter would potentially -- cgit v1.2.3