aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/Makefile.am2
-rw-r--r--src/Makefile.test.include1
-rw-r--r--src/dbwrapper.h3
-rw-r--r--src/index/txindex.cpp309
-rw-r--r--src/index/txindex.h94
-rw-r--r--src/init.cpp43
-rw-r--r--src/rest.cpp5
-rw-r--r--src/rpc/blockchain.cpp2
-rw-r--r--src/rpc/rawtransaction.cpp37
-rw-r--r--src/test/txindex_tests.cpp66
-rw-r--r--src/threadinterrupt.cpp2
-rw-r--r--src/threadinterrupt.h1
-rw-r--r--src/txdb.cpp171
-rw-r--r--src/txdb.h38
-rw-r--r--src/validation.cpp56
-rw-r--r--src/validation.h1
16 files changed, 745 insertions, 86 deletions
diff --git a/src/Makefile.am b/src/Makefile.am
index 521687eb4..0466d961c 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -103,6 +103,7 @@ BITCOIN_CORE_H = \
fs.h \
httprpc.h \
httpserver.h \
+ index/txindex.h \
indirectmap.h \
init.h \
interfaces/handler.h \
@@ -204,6 +205,7 @@ libbitcoin_server_a_SOURCES = \
consensus/tx_verify.cpp \
httprpc.cpp \
httpserver.cpp \
+ index/txindex.cpp \
init.cpp \
dbwrapper.cpp \
merkleblock.cpp \
diff --git a/src/Makefile.test.include b/src/Makefile.test.include
index f7eb71208..91d3a3d47 100644
--- a/src/Makefile.test.include
+++ b/src/Makefile.test.include
@@ -83,6 +83,7 @@ BITCOIN_TESTS =\
test/timedata_tests.cpp \
test/torcontrol_tests.cpp \
test/transaction_tests.cpp \
+ test/txindex_tests.cpp \
test/txvalidation_tests.cpp \
test/txvalidationcache_tests.cpp \
test/versionbits_tests.cpp \
diff --git a/src/dbwrapper.h b/src/dbwrapper.h
index 6f80eedc7..2a5e0cab0 100644
--- a/src/dbwrapper.h
+++ b/src/dbwrapper.h
@@ -224,6 +224,9 @@ public:
CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false);
~CDBWrapper();
+ CDBWrapper(const CDBWrapper&) = delete;
+ CDBWrapper& operator=(const CDBWrapper&) = delete;
+
template <typename K, typename V>
bool Read(const K& key, V& value) const
{
diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp
new file mode 100644
index 000000000..2a661f033
--- /dev/null
+++ b/src/index/txindex.cpp
@@ -0,0 +1,309 @@
+// Copyright (c) 2017-2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <chainparams.h>
+#include <index/txindex.h>
+#include <init.h>
+#include <tinyformat.h>
+#include <ui_interface.h>
+#include <util.h>
+#include <validation.h>
+#include <warnings.h>
+
+constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds
+constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds
+
+std::unique_ptr<TxIndex> g_txindex;
+
+template<typename... Args>
+static void FatalError(const char* fmt, const Args&... args)
+{
+ std::string strMessage = tfm::format(fmt, args...);
+ SetMiscWarning(strMessage);
+ LogPrintf("*** %s\n", strMessage);
+ uiInterface.ThreadSafeMessageBox(
+ "Error: A fatal internal error occurred, see debug.log for details",
+ "", CClientUIInterface::MSG_ERROR);
+ StartShutdown();
+}
+
+TxIndex::TxIndex(std::unique_ptr<TxIndexDB> db) :
+ m_db(std::move(db)), m_synced(false), m_best_block_index(nullptr)
+{}
+
+TxIndex::~TxIndex()
+{
+ Interrupt();
+ Stop();
+}
+
+bool TxIndex::Init()
+{
+ LOCK(cs_main);
+
+ // Attempt to migrate txindex from the old database to the new one. Even if
+ // chain_tip is null, the node could be reindexing and we still want to
+ // delete txindex records in the old database.
+ if (!m_db->MigrateData(*pblocktree, chainActive.GetLocator())) {
+ return false;
+ }
+
+ CBlockLocator locator;
+ if (!m_db->ReadBestBlock(locator)) {
+ locator.SetNull();
+ }
+
+ m_best_block_index = FindForkInGlobalIndex(chainActive, locator);
+ m_synced = m_best_block_index.load() == chainActive.Tip();
+ return true;
+}
+
+static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev)
+{
+ AssertLockHeld(cs_main);
+
+ if (!pindex_prev) {
+ return chainActive.Genesis();
+ }
+
+ const CBlockIndex* pindex = chainActive.Next(pindex_prev);
+ if (pindex) {
+ return pindex;
+ }
+
+ return chainActive.Next(chainActive.FindFork(pindex_prev));
+}
+
+void TxIndex::ThreadSync()
+{
+ const CBlockIndex* pindex = m_best_block_index.load();
+ if (!m_synced) {
+ auto& consensus_params = Params().GetConsensus();
+
+ int64_t last_log_time = 0;
+ int64_t last_locator_write_time = 0;
+ while (true) {
+ if (m_interrupt) {
+ WriteBestBlock(pindex);
+ return;
+ }
+
+ {
+ LOCK(cs_main);
+ const CBlockIndex* pindex_next = NextSyncBlock(pindex);
+ if (!pindex_next) {
+ WriteBestBlock(pindex);
+ m_best_block_index = pindex;
+ m_synced = true;
+ break;
+ }
+ pindex = pindex_next;
+ }
+
+ int64_t current_time = GetTime();
+ if (last_log_time + SYNC_LOG_INTERVAL < current_time) {
+ LogPrintf("Syncing txindex with block chain from height %d\n", pindex->nHeight);
+ last_log_time = current_time;
+ }
+
+ if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) {
+ WriteBestBlock(pindex);
+ last_locator_write_time = current_time;
+ }
+
+ CBlock block;
+ if (!ReadBlockFromDisk(block, pindex, consensus_params)) {
+ FatalError("%s: Failed to read block %s from disk",
+ __func__, pindex->GetBlockHash().ToString());
+ return;
+ }
+ if (!WriteBlock(block, pindex)) {
+ FatalError("%s: Failed to write block %s to tx index database",
+ __func__, pindex->GetBlockHash().ToString());
+ return;
+ }
+ }
+ }
+
+ if (pindex) {
+ LogPrintf("txindex is enabled at height %d\n", pindex->nHeight);
+ } else {
+ LogPrintf("txindex is enabled\n");
+ }
+}
+
+bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
+{
+ CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size()));
+ std::vector<std::pair<uint256, CDiskTxPos>> vPos;
+ vPos.reserve(block.vtx.size());
+ for (const auto& tx : block.vtx) {
+ vPos.emplace_back(tx->GetHash(), pos);
+ pos.nTxOffset += ::GetSerializeSize(*tx, SER_DISK, CLIENT_VERSION);
+ }
+ return m_db->WriteTxs(vPos);
+}
+
+bool TxIndex::WriteBestBlock(const CBlockIndex* block_index)
+{
+ LOCK(cs_main);
+ if (!m_db->WriteBestBlock(chainActive.GetLocator(block_index))) {
+ return error("%s: Failed to write locator to disk", __func__);
+ }
+ return true;
+}
+
+void TxIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex,
+ const std::vector<CTransactionRef>& txn_conflicted)
+{
+ if (!m_synced) {
+ return;
+ }
+
+ const CBlockIndex* best_block_index = m_best_block_index.load();
+ if (!best_block_index) {
+ if (pindex->nHeight != 0) {
+ FatalError("%s: First block connected is not the genesis block (height=%d)",
+ __func__, pindex->nHeight);
+ return;
+ }
+ } else {
+ // Ensure block connects to an ancestor of the current best block. This should be the case
+ // most of the time, but may not be immediately after the the sync thread catches up and sets
+ // m_synced. Consider the case where there is a reorg and the blocks on the stale branch are
+ // in the ValidationInterface queue backlog even after the sync thread has caught up to the
+ // new chain tip. In this unlikely event, log a warning and let the queue clear.
+ if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) {
+ LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " /* Continued */
+ "known best chain (tip=%s); not updating txindex\n",
+ __func__, pindex->GetBlockHash().ToString(),
+ best_block_index->GetBlockHash().ToString());
+ return;
+ }
+ }
+
+ if (WriteBlock(*block, pindex)) {
+ m_best_block_index = pindex;
+ } else {
+ FatalError("%s: Failed to write block %s to txindex",
+ __func__, pindex->GetBlockHash().ToString());
+ return;
+ }
+}
+
+void TxIndex::SetBestChain(const CBlockLocator& locator)
+{
+ if (!m_synced) {
+ return;
+ }
+
+ const uint256& locator_tip_hash = locator.vHave.front();
+ const CBlockIndex* locator_tip_index;
+ {
+ LOCK(cs_main);
+ locator_tip_index = LookupBlockIndex(locator_tip_hash);
+ }
+
+ if (!locator_tip_index) {
+ FatalError("%s: First block (hash=%s) in locator was not found",
+ __func__, locator_tip_hash.ToString());
+ return;
+ }
+
+ // This checks that SetBestChain callbacks are received after BlockConnected. The check may fail
+ // immediately after the the sync thread catches up and sets m_synced. Consider the case where
+ // there is a reorg and the blocks on the stale branch are in the ValidationInterface queue
+ // backlog even after the sync thread has caught up to the new chain tip. In this unlikely
+ // event, log a warning and let the queue clear.
+ const CBlockIndex* best_block_index = m_best_block_index.load();
+ if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) {
+ LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " /* Continued */
+ "chain (tip=%s); not writing txindex locator\n",
+ __func__, locator_tip_hash.ToString(),
+ best_block_index->GetBlockHash().ToString());
+ return;
+ }
+
+ if (!m_db->WriteBestBlock(locator)) {
+ error("%s: Failed to write locator to disk", __func__);
+ }
+}
+
+bool TxIndex::BlockUntilSyncedToCurrentChain()
+{
+ AssertLockNotHeld(cs_main);
+
+ if (!m_synced) {
+ return false;
+ }
+
+ {
+ // Skip the queue-draining stuff if we know we're caught up with
+ // chainActive.Tip().
+ LOCK(cs_main);
+ const CBlockIndex* chain_tip = chainActive.Tip();
+ const CBlockIndex* best_block_index = m_best_block_index.load();
+ if (best_block_index->GetAncestor(chain_tip->nHeight) == chain_tip) {
+ return true;
+ }
+ }
+
+ LogPrintf("%s: txindex is catching up on block notifications\n", __func__);
+ SyncWithValidationInterfaceQueue();
+ return true;
+}
+
+bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const
+{
+ CDiskTxPos postx;
+ if (!m_db->ReadTxPos(tx_hash, postx)) {
+ return false;
+ }
+
+ CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION);
+ if (file.IsNull()) {
+ return error("%s: OpenBlockFile failed", __func__);
+ }
+ CBlockHeader header;
+ try {
+ file >> header;
+ fseek(file.Get(), postx.nTxOffset, SEEK_CUR);
+ file >> tx;
+ } catch (const std::exception& e) {
+ return error("%s: Deserialize or I/O error - %s", __func__, e.what());
+ }
+ if (tx->GetHash() != tx_hash) {
+ return error("%s: txid mismatch", __func__);
+ }
+ block_hash = header.GetHash();
+ return true;
+}
+
+void TxIndex::Interrupt()
+{
+ m_interrupt();
+}
+
+void TxIndex::Start()
+{
+ // Need to register this ValidationInterface before running Init(), so that
+ // callbacks are not missed if Init sets m_synced to true.
+ RegisterValidationInterface(this);
+ if (!Init()) {
+ FatalError("%s: txindex failed to initialize", __func__);
+ return;
+ }
+
+ m_thread_sync = std::thread(&TraceThread<std::function<void()>>, "txindex",
+ std::bind(&TxIndex::ThreadSync, this));
+}
+
+void TxIndex::Stop()
+{
+ UnregisterValidationInterface(this);
+
+ if (m_thread_sync.joinable()) {
+ m_thread_sync.join();
+ }
+}
diff --git a/src/index/txindex.h b/src/index/txindex.h
new file mode 100644
index 000000000..ac746de05
--- /dev/null
+++ b/src/index/txindex.h
@@ -0,0 +1,94 @@
+// Copyright (c) 2017-2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_INDEX_TXINDEX_H
+#define BITCOIN_INDEX_TXINDEX_H
+
+#include <primitives/block.h>
+#include <primitives/transaction.h>
+#include <threadinterrupt.h>
+#include <txdb.h>
+#include <uint256.h>
+#include <validationinterface.h>
+
+class CBlockIndex;
+
+/**
+ * TxIndex is used to look up transactions included in the blockchain by hash.
+ * The index is written to a LevelDB database and records the filesystem
+ * location of each transaction by transaction hash.
+ */
+class TxIndex final : public CValidationInterface
+{
+private:
+ const std::unique_ptr<TxIndexDB> m_db;
+
+ /// Whether the index is in sync with the main chain. The flag is flipped
+ /// from false to true once, after which point this starts processing
+ /// ValidationInterface notifications to stay in sync.
+ std::atomic<bool> m_synced;
+
+ /// The last block in the chain that the TxIndex is in sync with.
+ std::atomic<const CBlockIndex*> m_best_block_index;
+
+ std::thread m_thread_sync;
+ CThreadInterrupt m_interrupt;
+
+ /// Initialize internal state from the database and block index.
+ bool Init();
+
+ /// Sync the tx index with the block index starting from the current best
+ /// block. Intended to be run in its own thread, m_thread_sync, and can be
+ /// interrupted with m_interrupt. Once the txindex gets in sync, the
+ /// m_synced flag is set and the BlockConnected ValidationInterface callback
+ /// takes over and the sync thread exits.
+ void ThreadSync();
+
+ /// Write update index entries for a newly connected block.
+ bool WriteBlock(const CBlock& block, const CBlockIndex* pindex);
+
+ /// Write the current chain block locator to the DB.
+ bool WriteBestBlock(const CBlockIndex* block_index);
+
+protected:
+ void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex,
+ const std::vector<CTransactionRef>& txn_conflicted) override;
+
+ void SetBestChain(const CBlockLocator& locator) override;
+
+public:
+ /// Constructs the TxIndex, which becomes available to be queried.
+ explicit TxIndex(std::unique_ptr<TxIndexDB> db);
+
+ /// Destructor interrupts sync thread if running and blocks until it exits.
+ ~TxIndex();
+
+ /// Blocks the current thread until the transaction index is caught up to
+ /// the current state of the block chain. This only blocks if the index has gotten in sync once
+ /// and only needs to process blocks in the ValidationInterface queue. If the index is catching
+ /// up from far behind, this method does not block and immediately returns false.
+ bool BlockUntilSyncedToCurrentChain();
+
+ /// Look up a transaction by hash.
+ ///
+ /// @param[in] tx_hash The hash of the transaction to be returned.
+ /// @param[out] block_hash The hash of the block the transaction is found in.
+ /// @param[out] tx The transaction itself.
+ /// @return true if transaction is found, false otherwise
+ bool FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const;
+
+ void Interrupt();
+
+ /// Start initializes the sync state and registers the instance as a
+ /// ValidationInterface so that it stays in sync with blockchain updates.
+ void Start();
+
+ /// Stops the instance from staying in sync with blockchain updates.
+ void Stop();
+};
+
+/// The global transaction index, used in GetTransaction. May be null.
+extern std::unique_ptr<TxIndex> g_txindex;
+
+#endif // BITCOIN_INDEX_TXINDEX_H
diff --git a/src/init.cpp b/src/init.cpp
index f403f90b0..eca3577f2 100644
--- a/src/init.cpp
+++ b/src/init.cpp
@@ -19,6 +19,7 @@
#include <fs.h>
#include <httpserver.h>
#include <httprpc.h>
+#include <index/txindex.h>
#include <key.h>
#include <validation.h>
#include <miner.h>
@@ -182,6 +183,9 @@ void Interrupt()
InterruptMapPort();
if (g_connman)
g_connman->Interrupt();
+ if (g_txindex) {
+ g_txindex->Interrupt();
+ }
}
void Shutdown()
@@ -212,6 +216,9 @@ void Shutdown()
if (g_connman) g_connman->Stop();
peerLogic.reset();
g_connman.reset();
+ if (g_txindex) {
+ g_txindex.reset();
+ }
StopTorControl();
@@ -1414,9 +1421,10 @@ bool AppInitMain()
int64_t nTotalCache = (gArgs.GetArg("-dbcache", nDefaultDbCache) << 20);
nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache
nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greater than nMaxDbcache
- int64_t nBlockTreeDBCache = nTotalCache / 8;
- nBlockTreeDBCache = std::min(nBlockTreeDBCache, (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxBlockDBAndTxIndexCache : nMaxBlockDBCache) << 20);
+ int64_t nBlockTreeDBCache = std::min(nTotalCache / 8, nMaxBlockDBCache << 20);
nTotalCache -= nBlockTreeDBCache;
+ int64_t nTxIndexCache = std::min(nTotalCache / 8, gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0);
+ nTotalCache -= nTxIndexCache;
int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache
nTotalCache -= nCoinDBCache;
@@ -1424,6 +1432,9 @@ bool AppInitMain()
int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000;
LogPrintf("Cache configuration:\n");
LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024));
+ if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
+ LogPrintf("* Using %.1fMiB for transaction index database\n", nTxIndexCache * (1.0 / 1024 / 1024));
+ }
LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
LogPrintf("* Using %.1fMiB for in-memory UTXO set (plus up to %.1fMiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024));
@@ -1457,9 +1468,8 @@ bool AppInitMain()
if (fRequestShutdown) break;
- // LoadBlockIndex will load fTxIndex from the db, or set it if
- // we're reindexing. It will also load fHavePruned if we've
- // ever removed a block file from disk.
+ // LoadBlockIndex will load fHavePruned if we've ever removed a
+ // block file from disk.
// Note that it also sets fReindex based on the disk flag!
// From here on out fReindex and fReset mean something different!
if (!LoadBlockIndex(chainparams)) {
@@ -1473,12 +1483,6 @@ bool AppInitMain()
return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?"));
}
- // Check for changed -txindex state
- if (fTxIndex != gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
- strLoadError = _("You need to rebuild the database using -reindex to change -txindex");
- break;
- }
-
// Check for changed -prune state. What we are concerned about is a user who has pruned blocks
// in the past, but is now trying to run unpruned.
if (fHavePruned && !fPruneMode) {
@@ -1608,10 +1612,17 @@ bool AppInitMain()
::feeEstimator.Read(est_filein);
fFeeEstimatesInitialized = true;
- // ********************************************************* Step 8: load wallet
+ // ********************************************************* Step 8: start indexers
+ if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
+ auto txindex_db = MakeUnique<TxIndexDB>(nTxIndexCache, false, fReindex);
+ g_txindex = MakeUnique<TxIndex>(std::move(txindex_db));
+ g_txindex->Start();
+ }
+
+ // ********************************************************* Step 9: load wallet
if (!g_wallet_init_interface.Open()) return false;
- // ********************************************************* Step 9: data directory maintenance
+ // ********************************************************* Step 10: data directory maintenance
// if pruning, unset the service bit and perform the initial blockstore prune
// after any wallet rescanning has taken place.
@@ -1633,7 +1644,7 @@ bool AppInitMain()
nLocalServices = ServiceFlags(nLocalServices | NODE_WITNESS);
}
- // ********************************************************* Step 10: import blocks
+ // ********************************************************* Step 11: import blocks
if (!CheckDiskSpace() && !CheckDiskSpace(0, true))
return false;
@@ -1672,7 +1683,7 @@ bool AppInitMain()
return false;
}
- // ********************************************************* Step 11: start node
+ // ********************************************************* Step 12: start node
int chain_active_height;
@@ -1750,7 +1761,7 @@ bool AppInitMain()
return false;
}
- // ********************************************************* Step 12: finished
+ // ********************************************************* Step 13: finished
SetRPCWarmupFinished();
uiInterface.InitMessage(_("Done loading"));
diff --git a/src/rest.cpp b/src/rest.cpp
index 5871b554a..095655b3a 100644
--- a/src/rest.cpp
+++ b/src/rest.cpp
@@ -6,6 +6,7 @@
#include <chain.h>
#include <chainparams.h>
#include <core_io.h>
+#include <index/txindex.h>
#include <primitives/block.h>
#include <primitives/transaction.h>
#include <validation.h>
@@ -350,6 +351,10 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart)
if (!ParseHashStr(hashStr, hash))
return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr);
+ if (g_txindex) {
+ g_txindex->BlockUntilSyncedToCurrentChain();
+ }
+
CTransactionRef tx;
uint256 hashBlock = uint256();
if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true))
diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
index 06c68ea27..19f964138 100644
--- a/src/rpc/blockchain.cpp
+++ b/src/rpc/blockchain.cpp
@@ -47,8 +47,6 @@ static std::mutex cs_blockchange;
static std::condition_variable cond_blockchange;
static CUpdatedBlock latestblock;
-extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry);
-
/* Calculate the difficulty for a given block index,
* or the block index of the given chain.
*/
diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp
index 7bdf09812..2659b0750 100644
--- a/src/rpc/rawtransaction.cpp
+++ b/src/rpc/rawtransaction.cpp
@@ -7,6 +7,7 @@
#include <coins.h>
#include <consensus/validation.h>
#include <core_io.h>
+#include <index/txindex.h>
#include <init.h>
#include <keystore.h>
#include <validation.h>
@@ -47,6 +48,8 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry)
TxToUniv(tx, uint256(), entry, true, RPCSerializationFlags());
if (!hashBlock.IsNull()) {
+ LOCK(cs_main);
+
entry.pushKV("blockhash", hashBlock.GetHex());
CBlockIndex* pindex = LookupBlockIndex(hashBlock);
if (pindex) {
@@ -141,8 +144,6 @@ UniValue getrawtransaction(const JSONRPCRequest& request)
+ HelpExampleCli("getrawtransaction", "\"mytxid\" true \"myblockhash\"")
);
- LOCK(cs_main);
-
bool in_active_chain = true;
uint256 hash = ParseHashV(request.params[0], "parameter 1");
CBlockIndex* blockindex = nullptr;
@@ -159,6 +160,8 @@ UniValue getrawtransaction(const JSONRPCRequest& request)
}
if (!request.params[2].isNull()) {
+ LOCK(cs_main);
+
uint256 blockhash = ParseHashV(request.params[2], "parameter 3");
blockindex = LookupBlockIndex(blockhash);
if (!blockindex) {
@@ -167,6 +170,11 @@ UniValue getrawtransaction(const JSONRPCRequest& request)
in_active_chain = chainActive.Contains(blockindex);
}
+ bool f_txindex_ready = false;
+ if (g_txindex && !blockindex) {
+ f_txindex_ready = g_txindex->BlockUntilSyncedToCurrentChain();
+ }
+
CTransactionRef tx;
uint256 hash_block;
if (!GetTransaction(hash, tx, Params().GetConsensus(), hash_block, true, blockindex)) {
@@ -176,10 +184,12 @@ UniValue getrawtransaction(const JSONRPCRequest& request)
throw JSONRPCError(RPC_MISC_ERROR, "Block not available");
}
errmsg = "No such transaction found in the provided block";
+ } else if (!g_txindex) {
+ errmsg = "No such mempool transaction. Use -txindex to enable blockchain transaction queries";
+ } else if (!f_txindex_ready) {
+ errmsg = "No such mempool transaction. Blockchain transactions are still in the process of being indexed";
} else {
- errmsg = fTxIndex
- ? "No such mempool or blockchain transaction"
- : "No such mempool transaction. Use -txindex to enable blockchain transaction queries";
+ errmsg = "No such mempool or blockchain transaction";
}
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errmsg + ". Use gettransaction for wallet transactions.");
}
@@ -229,19 +239,18 @@ UniValue gettxoutproof(const JSONRPCRequest& request)
oneTxid = hash;
}
- LOCK(cs_main);
-
CBlockIndex* pblockindex = nullptr;
-
uint256 hashBlock;
- if (!request.params[1].isNull())
- {
+ if (!request.params[1].isNull()) {
+ LOCK(cs_main);
hashBlock = uint256S(request.params[1].get_str());
pblockindex = LookupBlockIndex(hashBlock);
if (!pblockindex) {
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found");
}
} else {
+ LOCK(cs_main);
+
// Loop through txids and try to find which block they're in. Exit loop once a block is found.
for (const auto& tx : setTxids) {
const Coin& coin = AccessByTxid(*pcoinsTip, tx);
@@ -252,6 +261,14 @@ UniValue gettxoutproof(const JSONRPCRequest& request)
}
}
+
+ // Allow txindex to catch up if we need to query it and before we acquire cs_main.
+ if (g_txindex && !pblockindex) {
+ g_txindex->BlockUntilSyncedToCurrentChain();
+ }
+
+ LOCK(cs_main);
+
if (pblockindex == nullptr)
{
CTransactionRef tx;
diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp
new file mode 100644
index 000000000..14158f287
--- /dev/null
+++ b/src/test/txindex_tests.cpp
@@ -0,0 +1,66 @@
+// Copyright (c) 2017-2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <index/txindex.h>
+#include <script/standard.h>
+#include <test/test_bitcoin.h>
+#include <util.h>
+#include <utiltime.h>
+#include <validation.h>
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_AUTO_TEST_SUITE(txindex_tests)
+
+BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup)
+{
+ TxIndex txindex(MakeUnique<TxIndexDB>(1 << 20, true));
+
+ CTransactionRef tx_disk;
+ uint256 block_hash;
+
+ // Transaction should not be found in the index before it is started.
+ for (const auto& txn : m_coinbase_txns) {
+ BOOST_CHECK(!txindex.FindTx(txn->GetHash(), block_hash, tx_disk));
+ }
+
+ // BlockUntilSyncedToCurrentChain should return false before txindex is started.
+ BOOST_CHECK(!txindex.BlockUntilSyncedToCurrentChain());
+
+ txindex.Start();
+
+ // Allow tx index to catch up with the block index.
+ constexpr int64_t timeout_ms = 10 * 1000;
+ int64_t time_start = GetTimeMillis();
+ while (!txindex.BlockUntilSyncedToCurrentChain()) {
+ BOOST_REQUIRE(time_start + timeout_ms > GetTimeMillis());
+ MilliSleep(100);
+ }
+
+ // Check that txindex has all txs that were in the chain before it started.
+ for (const auto& txn : m_coinbase_txns) {
+ if (!txindex.FindTx(txn->GetHash(), block_hash, tx_disk)) {
+ BOOST_ERROR("FindTx failed");
+ } else if (tx_disk->GetHash() != txn->GetHash()) {
+ BOOST_ERROR("Read incorrect tx");
+ }
+ }
+
+ // Check that new transactions in new blocks make it into the index.
+ for (int i = 0; i < 10; i++) {
+ CScript coinbase_script_pub_key = GetScriptForDestination(coinbaseKey.GetPubKey().GetID());
+ std::vector<CMutableTransaction> no_txns;
+ const CBlock& block = CreateAndProcessBlock(no_txns, coinbase_script_pub_key);
+ const CTransaction& txn = *block.vtx[0];
+
+ BOOST_CHECK(txindex.BlockUntilSyncedToCurrentChain());
+ if (!txindex.FindTx(txn.GetHash(), block_hash, tx_disk)) {
+ BOOST_ERROR("FindTx failed");
+ } else if (tx_disk->GetHash() != txn.GetHash()) {
+ BOOST_ERROR("Read incorrect tx");
+ }
+ }
+}
+
+BOOST_AUTO_TEST_SUITE_END()
diff --git a/src/threadinterrupt.cpp b/src/threadinterrupt.cpp
index 5d932091c..7da4e136e 100644
--- a/src/threadinterrupt.cpp
+++ b/src/threadinterrupt.cpp
@@ -5,6 +5,8 @@
#include <threadinterrupt.h>
+CThreadInterrupt::CThreadInterrupt() : flag(false) {}
+
CThreadInterrupt::operator bool() const
{
return flag.load(std::memory_order_acquire);
diff --git a/src/threadinterrupt.h b/src/threadinterrupt.h
index 54e310280..d373e3c37 100644
--- a/src/threadinterrupt.h
+++ b/src/threadinterrupt.h
@@ -18,6 +18,7 @@
class CThreadInterrupt
{
public:
+ CThreadInterrupt();
explicit operator bool() const;
void operator()();
void reset();
diff --git a/src/txdb.cpp b/src/txdb.cpp
index 45ce94ae4..333d3596c 100644
--- a/src/txdb.cpp
+++ b/src/txdb.cpp
@@ -22,6 +22,7 @@ static const char DB_COIN = 'C';
static const char DB_COINS = 'c';
static const char DB_BLOCK_FILES = 'f';
static const char DB_TXINDEX = 't';
+static const char DB_TXINDEX_BLOCK = 'T';
static const char DB_BLOCK_INDEX = 'b';
static const char DB_BEST_BLOCK = 'B';
@@ -424,3 +425,173 @@ bool CCoinsViewDB::Upgrade() {
LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE");
return !ShutdownRequested();
}
+
+TxIndexDB::TxIndexDB(size_t n_cache_size, bool f_memory, bool f_wipe) :
+ CDBWrapper(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe)
+{}
+
+bool TxIndexDB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const
+{
+ return Read(std::make_pair(DB_TXINDEX, txid), pos);
+}
+
+bool TxIndexDB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos)
+{
+ CDBBatch batch(*this);
+ for (const auto& tuple : v_pos) {
+ batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second);
+ }
+ return WriteBatch(batch);
+}
+
+bool TxIndexDB::ReadBestBlock(CBlockLocator& locator) const
+{
+ bool success = Read(DB_BEST_BLOCK, locator);
+ if (!success) {
+ locator.SetNull();
+ }
+ return success;
+}
+
+bool TxIndexDB::WriteBestBlock(const CBlockLocator& locator)
+{
+ return Write(DB_BEST_BLOCK, locator);
+}
+
+/*
+ * Safely persist a transfer of data from the old txindex database to the new one, and compact the
+ * range of keys updated. This is used internally by MigrateData.
+ */
+static void WriteTxIndexMigrationBatches(TxIndexDB& newdb, CBlockTreeDB& olddb,
+ CDBBatch& batch_newdb, CDBBatch& batch_olddb,
+ const std::pair<unsigned char, uint256>& begin_key,
+ const std::pair<unsigned char, uint256>& end_key)
+{
+ // Sync new DB changes to disk before deleting from old DB.
+ newdb.WriteBatch(batch_newdb, /*fSync=*/ true);
+ olddb.WriteBatch(batch_olddb);
+ olddb.CompactRange(begin_key, end_key);
+
+ batch_newdb.Clear();
+ batch_olddb.Clear();
+}
+
+bool TxIndexDB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator)
+{
+ // The prior implementation of txindex was always in sync with block index
+ // and presence was indicated with a boolean DB flag. If the flag is set,
+ // this means the txindex from a previous version is valid and in sync with
+ // the chain tip. The first step of the migration is to unset the flag and
+ // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the
+ // index entries are copied over in batches to the new database. Finally,
+ // DB_TXINDEX_BLOCK is erased from the old database and the block hash is
+ // written to the new database.
+ //
+ // Unsetting the boolean flag ensures that if the node is downgraded to a
+ // previous version, it will not see a corrupted, partially migrated index
+ // -- it will see that the txindex is disabled. When the node is upgraded
+ // again, the migration will pick up where it left off and sync to the block
+ // with hash DB_TXINDEX_BLOCK.
+ bool f_legacy_flag = false;
+ block_tree_db.ReadFlag("txindex", f_legacy_flag);
+ if (f_legacy_flag) {
+ if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) {
+ return error("%s: cannot write block indicator", __func__);
+ }
+ if (!block_tree_db.WriteFlag("txindex", false)) {
+ return error("%s: cannot write block index db flag", __func__);
+ }
+ }
+
+ CBlockLocator locator;
+ if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) {
+ return true;
+ }
+
+ int64_t count = 0;
+ LogPrintf("Upgrading txindex database... [0%%]\n");
+ uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true);
+ int report_done = 0;
+ const size_t batch_size = 1 << 24; // 16 MiB
+
+ CDBBatch batch_newdb(*this);
+ CDBBatch batch_olddb(block_tree_db);
+
+ std::pair<unsigned char, uint256> key;
+ std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()};
+ std::pair<unsigned char, uint256> prev_key = begin_key;
+
+ bool interrupted = false;
+ std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator());
+ for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) {
+ boost::this_thread::interruption_point();
+ if (ShutdownRequested()) {
+ interrupted = true;
+ break;
+ }
+
+ if (!cursor->GetKey(key)) {
+ return error("%s: cannot get key from valid cursor", __func__);
+ }
+ if (key.first != DB_TXINDEX) {
+ break;
+ }
+
+ // Log progress every 10%.
+ if (++count % 256 == 0) {
+ // Since txids are uniformly random and traversed in increasing order, the high 16 bits
+ // of the hash can be used to estimate the current progress.
+ const uint256& txid = key.second;
+ uint32_t high_nibble =
+ (static_cast<uint32_t>(*(txid.begin() + 0)) << 8) +
+ (static_cast<uint32_t>(*(txid.begin() + 1)) << 0);
+ int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5);
+
+ uiInterface.ShowProgress(_("Upgrading txindex database"), percentage_done, true);
+ if (report_done < percentage_done/10) {
+ LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done);
+ report_done = percentage_done/10;
+ }
+ }
+
+ CDiskTxPos value;
+ if (!cursor->GetValue(value)) {
+ return error("%s: cannot parse txindex record", __func__);
+ }
+ batch_newdb.Write(key, value);
+ batch_olddb.Erase(key);
+
+ if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) {
+ // NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating
+ // because LevelDB iterators are guaranteed to provide a consistent view of the
+ // underlying data, like a lightweight snapshot.
+ WriteTxIndexMigrationBatches(*this, block_tree_db,
+ batch_newdb, batch_olddb,
+ prev_key, key);
+ prev_key = key;
+ }
+ }
+
+ // If these final DB batches complete the migration, write the best block
+ // hash marker to the new database and delete from the old one. This signals
+ // that the former is fully caught up to that point in the blockchain and
+ // that all txindex entries have been removed from the latter.
+ if (!interrupted) {
+ batch_olddb.Erase(DB_TXINDEX_BLOCK);
+ batch_newdb.Write(DB_BEST_BLOCK, locator);
+ }
+
+ WriteTxIndexMigrationBatches(*this, block_tree_db,
+ batch_newdb, batch_olddb,
+ begin_key, key);
+
+ if (interrupted) {
+ LogPrintf("[CANCELLED].\n");
+ return false;
+ }
+
+ uiInterface.ShowProgress("", 100, false);
+
+ LogPrintf("[DONE].\n");
+ return true;
+}
diff --git a/src/txdb.h b/src/txdb.h
index f3454e7d0..4193f98de 100644
--- a/src/txdb.h
+++ b/src/txdb.h
@@ -9,6 +9,7 @@
#include <coins.h>
#include <dbwrapper.h>
#include <chain.h>
+#include <primitives/block.h>
#include <map>
#include <memory>
@@ -35,7 +36,7 @@ static const int64_t nMaxBlockDBCache = 2;
//! Max memory allocated to block tree DB specific cache, if -txindex (MiB)
// Unlike for the UTXO database, for the txindex scenario the leveldb cache make
// a meaningful difference: https://github.com/bitcoin/bitcoin/pull/8273#issuecomment-229601991
-static const int64_t nMaxBlockDBAndTxIndexCache = 1024;
+static const int64_t nMaxTxIndexCache = 1024;
//! Max memory allocated to coin DB specific cache (MiB)
static const int64_t nMaxCoinsDBCache = 8;
@@ -112,9 +113,6 @@ class CBlockTreeDB : public CDBWrapper
public:
explicit CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false);
- CBlockTreeDB(const CBlockTreeDB&) = delete;
- CBlockTreeDB& operator=(const CBlockTreeDB&) = delete;
-
bool WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo);
bool ReadBlockFileInfo(int nFile, CBlockFileInfo &info);
bool ReadLastBlockFile(int &nFile);
@@ -127,4 +125,36 @@ public:
bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex);
};
+/**
+ * Access to the txindex database (indexes/txindex/)
+ *
+ * The database stores a block locator of the chain the database is synced to
+ * so that the TxIndex can efficiently determine the point it last stopped at.
+ * A locator is used instead of a simple hash of the chain tip because blocks
+ * and block index entries may not be flushed to disk until after this database
+ * is updated.
+ */
+class TxIndexDB : public CDBWrapper
+{
+public:
+ explicit TxIndexDB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
+
+ /// Read the disk location of the transaction data with the given hash. Returns false if the
+ /// transaction hash is not indexed.
+ bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const;
+
+ /// Write a batch of transaction positions to the DB.
+ bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos);
+
+ /// Read block locator of the chain that the txindex is in sync with.
+ bool ReadBestBlock(CBlockLocator& locator) const;
+
+ /// Write block locator of the chain that the txindex is in sync with.
+ bool WriteBestBlock(const CBlockLocator& locator);
+
+ /// Migrate txindex data from the block tree DB, where it may be for older nodes that have not
+ /// been upgraded yet to the new database.
+ bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator);
+};
+
#endif // BITCOIN_TXDB_H
diff --git a/src/validation.cpp b/src/validation.cpp
index bce8c4f9e..14257d78f 100644
--- a/src/validation.cpp
+++ b/src/validation.cpp
@@ -16,6 +16,7 @@
#include <consensus/validation.h>
#include <cuckoocache.h>
#include <hash.h>
+#include <index/txindex.h>
#include <init.h>
#include <policy/fees.h>
#include <policy/policy.h>
@@ -217,7 +218,6 @@ uint256 g_best_block;
int nScriptCheckThreads = 0;
std::atomic_bool fImporting(false);
std::atomic_bool fReindex(false);
-bool fTxIndex = false;
bool fHavePruned = false;
bool fPruneMode = false;
bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG;
@@ -1028,28 +1028,8 @@ bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus
return true;
}
- if (fTxIndex) {
- CDiskTxPos postx;
- if (pblocktree->ReadTxIndex(hash, postx)) {
- CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION);
- if (file.IsNull())
- return error("%s: OpenBlockFile failed", __func__);
- CBlockHeader header;
- try {
- file >> header;
- fseek(file.Get(), postx.nTxOffset, SEEK_CUR);
- file >> txOut;
- } catch (const std::exception& e) {
- return error("%s: Deserialize or I/O error - %s", __func__, e.what());
- }
- hashBlock = header.GetHash();
- if (txOut->GetHash() != hash)
- return error("%s: txid mismatch", __func__);
- return true;
- }
-
- // transaction not found in index, nothing more can be done
- return false;
+ if (g_txindex) {
+ return g_txindex->FindTx(hash, hashBlock, txOut);
}
if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it
@@ -1668,26 +1648,6 @@ static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState&
return true;
}
-static bool WriteTxIndexDataForBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex)
-{
- if (!fTxIndex) return true;
-
- CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size()));
- std::vector<std::pair<uint256, CDiskTxPos> > vPos;
- vPos.reserve(block.vtx.size());
- for (const CTransactionRef& tx : block.vtx)
- {
- vPos.push_back(std::make_pair(tx->GetHash(), pos));
- pos.nTxOffset += ::GetSerializeSize(*tx, SER_DISK, CLIENT_VERSION);
- }
-
- if (!pblocktree->WriteTxIndex(vPos)) {
- return AbortNode(state, "Failed to write transaction index");
- }
-
- return true;
-}
-
static CCheckQueue<CScriptCheck> scriptcheckqueue(128);
void ThreadScriptCheck() {
@@ -2079,9 +2039,6 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl
setDirtyBlockIndex.insert(pindex);
}
- if (!WriteTxIndexDataForBlock(block, state, pindex))
- return false;
-
assert(pindex->phashBlock);
// add this block to the view's block chain
view.SetBestBlock(pindex->GetBlockHash());
@@ -3903,10 +3860,6 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams)
pblocktree->ReadReindexing(fReindexing);
if(fReindexing) fReindex = true;
- // Check whether we have a transaction index
- pblocktree->ReadFlag("txindex", fTxIndex);
- LogPrintf("%s: transaction index %s\n", __func__, fTxIndex ? "enabled" : "disabled");
-
return true;
}
@@ -4300,9 +4253,6 @@ bool LoadBlockIndex(const CChainParams& chainparams)
// needs_init.
LogPrintf("Initializing databases...\n");
- // Use the provided setting for -txindex in the new database
- fTxIndex = gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX);
- pblocktree->WriteFlag("txindex", fTxIndex);
}
return true;
}
diff --git a/src/validation.h b/src/validation.h
index b415a8505..9b4010076 100644
--- a/src/validation.h
+++ b/src/validation.h
@@ -171,7 +171,6 @@ extern uint256 g_best_block;
extern std::atomic_bool fImporting;
extern std::atomic_bool fReindex;
extern int nScriptCheckThreads;
-extern bool fTxIndex;
extern bool fIsBareMultisigStd;
extern bool fRequireStandard;
extern bool fCheckBlockIndex;
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-19 09:13:45 +0000