From 450cbb0944cd20a06ce806e6679a1f4c83c50db2 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 1 Jul 2012 18:54:00 +0200 Subject: Ultraprune This switches bitcoin's transaction/block verification logic to use a "coin database", which contains all unredeemed transaction output scripts, amounts and heights. The name ultraprune comes from the fact that instead of a full transaction index, we only (need to) keep an index with unspent outputs. For now, the blocks themselves are kept as usual, although they are only necessary for serving, rescanning and reorganizing. The basic datastructures are CCoins (representing the coins of a single transaction), and CCoinsView (representing a state of the coins database). There are several implementations for CCoinsView. A dummy, one backed by the coins database (coins.dat), one backed by the memory pool, and one that adds a cache on top of it. FetchInputs, ConnectInputs, ConnectBlock, DisconnectBlock, ... now operate on a generic CCoinsView. The block switching logic now builds a single cached CCoinsView with changes to be committed to the database before any changes are made. This means no uncommitted changes are ever read from the database, and should ease the transition to another database layer which does not support transactions (but does support atomic writes), like LevelDB. For the getrawtransaction() RPC call, access to a txid-to-disk index would be preferable. As this index is not necessary or even useful for any other part of the implementation, it is not provided. Instead, getrawtransaction() uses the coin database to find the block height, and then scans that block to find the requested transaction. This is slow, but should suffice for debug purposes. --- src/init.cpp | 8 -------- 1 file changed, 8 deletions(-) (limited to 'src/init.cpp') diff --git a/src/init.cpp b/src/init.cpp index 92c752a8f..56108cece 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -638,14 +638,6 @@ bool AppInit2() return InitError(msg); } - if (GetBoolArg("-loadblockindextest")) - { - CTxDB txdb("r"); - txdb.LoadBlockIndex(); - PrintBlockTree(); - return false; - } - uiInterface.InitMessage(_("Loading block index...")); printf("Loading block index...\n"); nStart = GetTimeMillis(); -- cgit v1.2.3 From ae8bfd12daa802d20791e69d3477e799d2b99f45 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 6 Jul 2012 16:33:34 +0200 Subject: Batch block connection during IBD During the initial block download (or -loadblock), delay connection of new blocks a bit, and perform them in a single action. This reduces the load on the database engine, as subsequent blocks often update an earlier block's transaction already. --- src/init.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'src/init.cpp') diff --git a/src/init.cpp b/src/init.cpp index 56108cece..b05d57abf 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -50,6 +50,8 @@ void StartShutdown() #endif } +static CCoinsViewDB *pcoinsdbview; + void Shutdown(void* parg) { static CCriticalSection cs_Shutdown; @@ -74,6 +76,12 @@ void Shutdown(void* parg) nTransactionsUpdated++; bitdb.Flush(false); StopNode(); + { + LOCK(cs_main); + pcoinsTip->Flush(); + delete pcoinsTip; + delete pcoinsdbview; + } bitdb.Flush(true); boost::filesystem::remove(GetPidFile()); UnregisterWallet(pwalletMain); @@ -298,6 +306,7 @@ std::string HelpMessage() return strUsage; } + /** Initialize bitcoin. * @pre Parameters should be parsed and config file should be read. */ @@ -641,6 +650,9 @@ bool AppInit2() uiInterface.InitMessage(_("Loading block index...")); printf("Loading block index...\n"); nStart = GetTimeMillis(); + pcoinsdbview = new CCoinsViewDB(); + pcoinsTip = new CCoinsViewCache(*pcoinsdbview); + if (!LoadBlockIndex()) return InitError(_("Error loading blkindex.dat")); -- cgit v1.2.3 From 4fea06db25108e7f72710bf22c3d1896707eeb74 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 10 Aug 2012 15:13:57 +0200 Subject: Automatically reorganize at startup to best known block Given that the block tree database (chain.dat) and the active chain database (coins.dat) are entirely separate now, it becomes legal to swap one with another instance without affecting the other. This commit introduces a check in the startup code that detects the presence of a better chain in chain.dat that has not been activated yet, and does so efficiently (in batch, while reusing the blk???.dat files). --- src/init.cpp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'src/init.cpp') diff --git a/src/init.cpp b/src/init.cpp index b05d57abf..85aa4f600 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -780,6 +780,40 @@ bool AppInit2() // ********************************************************* Step 9: import blocks + // scan for better chains in the block chain database, that are not yet connected in the active best chain + CBlockIndex *pindexFoundBest = pindexBest; + for (std::map::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { + CBlockIndex *pindex = it->second; + if (pindexFoundBest==NULL || pindex->bnChainWork > pindexFoundBest->bnChainWork) + pindexFoundBest = pindex; + } + if (pindexFoundBest != pindexBest) { + uiInterface.InitMessage(_("Importing blocks from block database...")); + uint64 nTxs = 0; + uint64 nBlocks = 0; + std::vector vAttach; + vAttach.reserve(pindexFoundBest->nHeight - (pindexBest==NULL ? 0 : pindexBest->nHeight)); + while (pindexFoundBest && pindexFoundBest->bnChainWork > (pindexBest==NULL ? 0 : pindexBest->bnChainWork)) { + vAttach.push_back(pindexFoundBest); + pindexFoundBest = pindexFoundBest->pprev; + } + for (std::vector::reverse_iterator it = vAttach.rbegin(); it != vAttach.rend(); it++) { + CBlockIndex *pindex = *it; + CBlock block; + if (!block.ReadFromDisk(pindex)) + break; + nTxs += block.vtx.size(); + nBlocks++; + if (pindex->nHeight == 0 || nTxs + nBlocks*3 > 500) { + nTxs=0; + nBlocks=0; + block.SetBestChain(pindex); + } + if (fRequestShutdown) + break; + } + } + std::vector *vPath = new std::vector(); if (mapArgs.count("-loadblock")) { -- cgit v1.2.3 From 857c61df0b71c8a0482b1bf8fc55849f8ad831b8 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 19 Aug 2012 00:33:01 +0200 Subject: Prepare database format for multi-stage block processing This commit adds a status field and a transaction counter to the block indexes. --- src/init.cpp | 35 +++-------------------------------- 1 file changed, 3 insertions(+), 32 deletions(-) (limited to 'src/init.cpp') diff --git a/src/init.cpp b/src/init.cpp index 85aa4f600..99e2bba37 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -781,38 +781,9 @@ bool AppInit2() // ********************************************************* Step 9: import blocks // scan for better chains in the block chain database, that are not yet connected in the active best chain - CBlockIndex *pindexFoundBest = pindexBest; - for (std::map::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { - CBlockIndex *pindex = it->second; - if (pindexFoundBest==NULL || pindex->bnChainWork > pindexFoundBest->bnChainWork) - pindexFoundBest = pindex; - } - if (pindexFoundBest != pindexBest) { - uiInterface.InitMessage(_("Importing blocks from block database...")); - uint64 nTxs = 0; - uint64 nBlocks = 0; - std::vector vAttach; - vAttach.reserve(pindexFoundBest->nHeight - (pindexBest==NULL ? 0 : pindexBest->nHeight)); - while (pindexFoundBest && pindexFoundBest->bnChainWork > (pindexBest==NULL ? 0 : pindexBest->bnChainWork)) { - vAttach.push_back(pindexFoundBest); - pindexFoundBest = pindexFoundBest->pprev; - } - for (std::vector::reverse_iterator it = vAttach.rbegin(); it != vAttach.rend(); it++) { - CBlockIndex *pindex = *it; - CBlock block; - if (!block.ReadFromDisk(pindex)) - break; - nTxs += block.vtx.size(); - nBlocks++; - if (pindex->nHeight == 0 || nTxs + nBlocks*3 > 500) { - nTxs=0; - nBlocks=0; - block.SetBestChain(pindex); - } - if (fRequestShutdown) - break; - } - } + uiInterface.InitMessage(_("Importing blocks from block database...")); + if (!ConnectBestBlock()) + strErrors << "Failed to connect best block"; std::vector *vPath = new std::vector(); if (mapArgs.count("-loadblock")) -- cgit v1.2.3 From d979e6e36ac6be0d40b2a6bb70c668f9e6989ff9 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 3 Sep 2012 15:26:57 +0200 Subject: Use singleton block tree database instance --- src/init.cpp | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/init.cpp') diff --git a/src/init.cpp b/src/init.cpp index 99e2bba37..07e5d13d6 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -79,8 +79,10 @@ void Shutdown(void* parg) { LOCK(cs_main); pcoinsTip->Flush(); + pblocktree->Flush(); delete pcoinsTip; delete pcoinsdbview; + delete pblocktree; } bitdb.Flush(true); boost::filesystem::remove(GetPidFile()); @@ -650,6 +652,7 @@ bool AppInit2() uiInterface.InitMessage(_("Loading block index...")); printf("Loading block index...\n"); nStart = GetTimeMillis(); + pblocktree = new CBlockTreeDB("cr+"); pcoinsdbview = new CCoinsViewDB(); pcoinsTip = new CCoinsViewCache(*pcoinsdbview); -- cgit v1.2.3 From 2d8a48292b0da96cda8d7b45a24a22adfb4667b2 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 3 Sep 2012 21:14:03 +0200 Subject: LevelDB block and coin databases Split off CBlockTreeDB and CCoinsViewDB into txdb-*.{cpp,h} files, implemented by either LevelDB or BDB. Based on code from earlier commits by Mike Hearn in his leveldb branch. --- src/init.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/init.cpp') diff --git a/src/init.cpp b/src/init.cpp index 07e5d13d6..30f0d0049 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -2,7 +2,7 @@ // Copyright (c) 2009-2012 The Bitcoin developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "db.h" +#include "txdb.h" #include "walletdb.h" #include "bitcoinrpc.h" #include "net.h" -- cgit v1.2.3 From e1bfbab8029c33102889d7d970da5b6cc32ff75b Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 4 Sep 2012 18:12:00 +0200 Subject: Add LevelDB MemEnv support Support LevelDB memory-backed environments, and use them in unit tests. --- src/init.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/init.cpp') diff --git a/src/init.cpp b/src/init.cpp index 30f0d0049..4d5720306 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -652,7 +652,7 @@ bool AppInit2() uiInterface.InitMessage(_("Loading block index...")); printf("Loading block index...\n"); nStart = GetTimeMillis(); - pblocktree = new CBlockTreeDB("cr+"); + pblocktree = new CBlockTreeDB(); pcoinsdbview = new CCoinsViewDB(); pcoinsTip = new CCoinsViewCache(*pcoinsdbview); -- cgit v1.2.3