From 630fd8dcb6662ed167e967625f2715f97ddd2db5 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 19 Jun 2012 01:36:43 +0200 Subject: One file per block Refactor of the block storage code, which now stores one file per block. This will allow easier pruning, as blocks can be removed individually. --- src/db.cpp | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) (limited to 'src/db.cpp') diff --git a/src/db.cpp b/src/db.cpp index 9ad67892f..cef395c44 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -637,12 +637,12 @@ bool CTxDB::LoadBlockIndex() nCheckDepth = nBestHeight; printf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); CBlockIndex* pindexFork = NULL; - map, CBlockIndex*> mapBlockPos; for (CBlockIndex* pindex = pindexBest; pindex && pindex->pprev; pindex = pindex->pprev) { if (fRequestShutdown || pindex->nHeight < nBestHeight-nCheckDepth) break; CBlock block; + CDiskBlockPos blockPos = pindex->GetBlockPos(); if (!block.ReadFromDisk(pindex)) return error("LoadBlockIndex() : block.ReadFromDisk failed"); // check level 1: verify block validity @@ -654,8 +654,6 @@ bool CTxDB::LoadBlockIndex() // check level 2: verify transaction index validity if (nCheckLevel>1) { - pair pos = make_pair(pindex->nFile, pindex->nBlockPos); - mapBlockPos[pos] = pindex; BOOST_FOREACH(const CTransaction &tx, block.vtx) { uint256 hashTx = tx.GetHash(); @@ -663,7 +661,7 @@ bool CTxDB::LoadBlockIndex() if (ReadTxIndex(hashTx, txindex)) { // check level 3: checker transaction hashes - if (nCheckLevel>2 || pindex->nFile != txindex.pos.nFile || pindex->nBlockPos != txindex.pos.nBlockPos) + if (nCheckLevel>2 || blockPos != txindex.pos.blockPos) { // either an error or a duplicate transaction CTransaction txFound; @@ -687,12 +685,6 @@ bool CTxDB::LoadBlockIndex() { if (!txpos.IsNull()) { - pair posFind = make_pair(txpos.nFile, txpos.nBlockPos); - if (!mapBlockPos.count(posFind)) - { - printf("LoadBlockIndex(): *** found bad spend at %d, hashBlock=%s, hashTx=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString().c_str(), hashTx.ToString().c_str()); - pindexFork = pindex->pprev; - } // check level 6: check whether spent txouts were spent by a valid transaction that consume them if (nCheckLevel>5) { @@ -795,9 +787,8 @@ bool CTxDB::LoadBlockIndexGuts() CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); pindexNew->pnext = InsertBlockIndex(diskindex.hashNext); - pindexNew->nFile = diskindex.nFile; - pindexNew->nBlockPos = diskindex.nBlockPos; pindexNew->nHeight = diskindex.nHeight; + pindexNew->nAlternative = diskindex.nAlternative; pindexNew->nVersion = diskindex.nVersion; pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; pindexNew->nTime = diskindex.nTime; -- cgit v1.2.3 From 5382bcf8cd23c36a435c29080770a79b5e28af42 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 13 Aug 2012 19:11:05 +0200 Subject: Multiple blocks per file Change the block storage layer again, this time with multiple files per block, but tracked by txindex.dat database entries. The file format is exactly the same as the earlier blk00001.dat, but with smaller files (128 MiB for now). The database entries track how many bytes each block file already uses, how many blocks are in it, which range of heights is present and which range of dates. --- src/db.cpp | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) (limited to 'src/db.cpp') diff --git a/src/db.cpp b/src/db.cpp index cef395c44..53be48cb0 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -546,6 +546,22 @@ bool CTxDB::WriteBlockIndex(const CDiskBlockIndex& blockindex) return Write(make_pair(string("blockindex"), blockindex.GetBlockHash()), blockindex); } +bool CTxDB::WriteBlockFileInfo(int nFile, const CBlockFileInfo &info) { + return Write(make_pair(string("blockfile"), nFile), info); +} + +bool CTxDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { + return Read(make_pair(string("blockfile"), nFile), info); +} + +bool CTxDB::WriteLastBlockFile(int nFile) { + return Write(string("lastblockfile"), nFile); +} + +bool CTxDB::ReadLastBlockFile(int &nFile) { + return Read(string("lastblockfile"), nFile); +} + bool CTxDB::ReadHashBestChain(uint256& hashBestChain) { return Read(string("hashBestChain"), hashBestChain); @@ -609,6 +625,12 @@ bool CTxDB::LoadBlockIndex() pindex->bnChainWork = (pindex->pprev ? pindex->pprev->bnChainWork : 0) + pindex->GetBlockWork(); } + // Load block file info + ReadLastBlockFile(nLastBlockFile); + printf("LoadBlockIndex(): last block file = %i\n", nLastBlockFile); + if (ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + printf("LoadBlockIndex(): last block file: %s\n", infoLastBlockFile.ToString().c_str()); + // Load hashBestChain pointer to end of best chain if (!ReadHashBestChain(hashBestChain)) { @@ -788,7 +810,8 @@ bool CTxDB::LoadBlockIndexGuts() pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); pindexNew->pnext = InsertBlockIndex(diskindex.hashNext); pindexNew->nHeight = diskindex.nHeight; - pindexNew->nAlternative = diskindex.nAlternative; + pindexNew->pos = diskindex.pos; + pindexNew->nUndoPos = diskindex.nUndoPos; pindexNew->nVersion = diskindex.nVersion; pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; pindexNew->nTime = diskindex.nTime; -- cgit v1.2.3 From 450cbb0944cd20a06ce806e6679a1f4c83c50db2 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 1 Jul 2012 18:54:00 +0200 Subject: Ultraprune This switches bitcoin's transaction/block verification logic to use a "coin database", which contains all unredeemed transaction output scripts, amounts and heights. The name ultraprune comes from the fact that instead of a full transaction index, we only (need to) keep an index with unspent outputs. For now, the blocks themselves are kept as usual, although they are only necessary for serving, rescanning and reorganizing. The basic datastructures are CCoins (representing the coins of a single transaction), and CCoinsView (representing a state of the coins database). There are several implementations for CCoinsView. A dummy, one backed by the coins database (coins.dat), one backed by the memory pool, and one that adds a cache on top of it. FetchInputs, ConnectInputs, ConnectBlock, DisconnectBlock, ... now operate on a generic CCoinsView. The block switching logic now builds a single cached CCoinsView with changes to be committed to the database before any changes are made. This means no uncommitted changes are ever read from the database, and should ease the transition to another database layer which does not support transactions (but does support atomic writes), like LevelDB. For the getrawtransaction() RPC call, access to a txid-to-disk index would be preferable. As this index is not necessary or even useful for any other part of the implementation, it is not provided. Instead, getrawtransaction() uses the coin database to find the block height, and then scans that block to find the requested transaction. This is slow, but should suffice for debug purposes. --- src/db.cpp | 245 +++++++++++++++---------------------------------------------- 1 file changed, 61 insertions(+), 184 deletions(-) (limited to 'src/db.cpp') diff --git a/src/db.cpp b/src/db.cpp index 53be48cb0..0a6ba3fb3 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -244,7 +244,7 @@ CDB::CDB(const char *pszFile, const char* pszMode) : ret = pdb->open(NULL, // Txn pointer fMockDb ? NULL : pszFile, // Filename - "main", // Logical db name + fMockDb ? pszFile : "main", // Logical db name DB_BTREE, // Database type nFlags, // Flags 0); @@ -273,7 +273,7 @@ CDB::CDB(const char *pszFile, const char* pszMode) : static bool IsChainFile(std::string strFile) { - if (strFile == "blkindex.dat") + if (strFile == "coins.dat" || strFile == "chain.dat") return true; return false; @@ -475,111 +475,66 @@ void CDBEnv::Flush(bool fShutdown) // -// CTxDB +// CChainDB and CCoinsDB // -bool CTxDB::ReadTxIndex(uint256 hash, CTxIndex& txindex) -{ +bool CCoinsDB::HaveCoins(uint256 hash) { assert(!fClient); - txindex.SetNull(); - return Read(make_pair(string("tx"), hash), txindex); + return Exists(make_pair('c', hash)); } -bool CTxDB::UpdateTxIndex(uint256 hash, const CTxIndex& txindex) -{ +bool CCoinsDB::ReadCoins(uint256 hash, CCoins &coins) { assert(!fClient); - return Write(make_pair(string("tx"), hash), txindex); -} - -bool CTxDB::AddTxIndex(const CTransaction& tx, const CDiskTxPos& pos, int nHeight) -{ - assert(!fClient); - - // Add to tx index - uint256 hash = tx.GetHash(); - CTxIndex txindex(pos, tx.vout.size()); - return Write(make_pair(string("tx"), hash), txindex); + return Read(make_pair('c', hash), coins); } -bool CTxDB::EraseTxIndex(const CTransaction& tx) -{ - assert(!fClient); - uint256 hash = tx.GetHash(); - - return Erase(make_pair(string("tx"), hash)); -} - -bool CTxDB::ContainsTx(uint256 hash) -{ +bool CCoinsDB::WriteCoins(uint256 hash, const CCoins &coins) { assert(!fClient); - return Exists(make_pair(string("tx"), hash)); + if (coins.IsPruned()) + return Erase(make_pair('c', hash)); + else + return Write(make_pair('c', hash), coins); } -bool CTxDB::ReadDiskTx(uint256 hash, CTransaction& tx, CTxIndex& txindex) +bool CChainDB::WriteBlockIndex(const CDiskBlockIndex& blockindex) { - assert(!fClient); - tx.SetNull(); - if (!ReadTxIndex(hash, txindex)) - return false; - return (tx.ReadFromDisk(txindex.pos)); + return Write(make_pair('b', blockindex.GetBlockHash()), blockindex); } -bool CTxDB::ReadDiskTx(uint256 hash, CTransaction& tx) +bool CCoinsDB::ReadHashBestChain(uint256& hashBestChain) { - CTxIndex txindex; - return ReadDiskTx(hash, tx, txindex); + return Read('B', hashBestChain); } -bool CTxDB::ReadDiskTx(COutPoint outpoint, CTransaction& tx, CTxIndex& txindex) +bool CCoinsDB::WriteHashBestChain(uint256 hashBestChain) { - return ReadDiskTx(outpoint.hash, tx, txindex); + return Write('B', hashBestChain); } -bool CTxDB::ReadDiskTx(COutPoint outpoint, CTransaction& tx) +bool CChainDB::ReadBestInvalidWork(CBigNum& bnBestInvalidWork) { - CTxIndex txindex; - return ReadDiskTx(outpoint.hash, tx, txindex); + return Read('I', bnBestInvalidWork); } -bool CTxDB::WriteBlockIndex(const CDiskBlockIndex& blockindex) +bool CChainDB::WriteBestInvalidWork(CBigNum bnBestInvalidWork) { - return Write(make_pair(string("blockindex"), blockindex.GetBlockHash()), blockindex); + return Write('I', bnBestInvalidWork); } -bool CTxDB::WriteBlockFileInfo(int nFile, const CBlockFileInfo &info) { - return Write(make_pair(string("blockfile"), nFile), info); +bool CChainDB::WriteBlockFileInfo(int nFile, const CBlockFileInfo &info) { + return Write(make_pair('f', nFile), info); } -bool CTxDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { - return Read(make_pair(string("blockfile"), nFile), info); +bool CChainDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { + return Read(make_pair('f', nFile), info); } -bool CTxDB::WriteLastBlockFile(int nFile) { - return Write(string("lastblockfile"), nFile); +bool CChainDB::WriteLastBlockFile(int nFile) { + return Write('l', nFile); } -bool CTxDB::ReadLastBlockFile(int &nFile) { - return Read(string("lastblockfile"), nFile); -} - -bool CTxDB::ReadHashBestChain(uint256& hashBestChain) -{ - return Read(string("hashBestChain"), hashBestChain); -} - -bool CTxDB::WriteHashBestChain(uint256 hashBestChain) -{ - return Write(string("hashBestChain"), hashBestChain); -} - -bool CTxDB::ReadBestInvalidWork(CBigNum& bnBestInvalidWork) -{ - return Read(string("bnBestInvalidWork"), bnBestInvalidWork); -} - -bool CTxDB::WriteBestInvalidWork(CBigNum bnBestInvalidWork) -{ - return Write(string("bnBestInvalidWork"), bnBestInvalidWork); +bool CChainDB::ReadLastBlockFile(int &nFile) { + return Read('l', nFile); } CBlockIndex static * InsertBlockIndex(uint256 hash) @@ -602,9 +557,9 @@ CBlockIndex static * InsertBlockIndex(uint256 hash) return pindexNew; } -bool CTxDB::LoadBlockIndex() +bool LoadBlockIndex(CCoinsDB &coindb, CChainDB &chaindb) { - if (!LoadBlockIndexGuts()) + if (!chaindb.LoadBlockIndexGuts()) return false; if (fRequestShutdown) @@ -626,29 +581,39 @@ bool CTxDB::LoadBlockIndex() } // Load block file info - ReadLastBlockFile(nLastBlockFile); + chaindb.ReadLastBlockFile(nLastBlockFile); printf("LoadBlockIndex(): last block file = %i\n", nLastBlockFile); - if (ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + if (chaindb.ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile)) printf("LoadBlockIndex(): last block file: %s\n", infoLastBlockFile.ToString().c_str()); // Load hashBestChain pointer to end of best chain - if (!ReadHashBestChain(hashBestChain)) + if (!coindb.ReadHashBestChain(hashBestChain)) { if (pindexGenesisBlock == NULL) return true; return error("CTxDB::LoadBlockIndex() : hashBestChain not loaded"); } - if (!mapBlockIndex.count(hashBestChain)) + std::map::iterator it = mapBlockIndex.find(hashBestChain); + if (it == mapBlockIndex.end()) { return error("CTxDB::LoadBlockIndex() : hashBestChain not found in the block index"); - pindexBest = mapBlockIndex[hashBestChain]; - nBestHeight = pindexBest->nHeight; - bnBestChainWork = pindexBest->bnChainWork; - printf("LoadBlockIndex(): hashBestChain=%s height=%d date=%s\n", - hashBestChain.ToString().substr(0,20).c_str(), nBestHeight, - DateTimeStrFormat("%x %H:%M:%S", pindexBest->GetBlockTime()).c_str()); + } else { + // set 'next' pointers in best chain + CBlockIndex *pindex = it->second; + while(pindex != NULL && pindex->pprev != NULL) { + CBlockIndex *pindexPrev = pindex->pprev; + pindexPrev->pnext = pindex; + pindex = pindexPrev; + } + pindexBest = it->second; + nBestHeight = pindexBest->nHeight; + bnBestChainWork = pindexBest->bnChainWork; + } + printf("LoadBlockIndex(): hashBestChain=%s height=%d date=%s\n", + hashBestChain.ToString().substr(0,20).c_str(), nBestHeight, + DateTimeStrFormat("%x %H:%M:%S", pindexBest->GetBlockTime()).c_str()); // Load bnBestInvalidWork, OK if it doesn't exist - ReadBestInvalidWork(bnBestInvalidWork); + chaindb.ReadBestInvalidWork(bnBestInvalidWork); // Verify blocks in the best chain int nCheckLevel = GetArg("-checklevel", 1); @@ -664,7 +629,6 @@ bool CTxDB::LoadBlockIndex() if (fRequestShutdown || pindex->nHeight < nBestHeight-nCheckDepth) break; CBlock block; - CDiskBlockPos blockPos = pindex->GetBlockPos(); if (!block.ReadFromDisk(pindex)) return error("LoadBlockIndex() : block.ReadFromDisk failed"); // check level 1: verify block validity @@ -673,98 +637,12 @@ bool CTxDB::LoadBlockIndex() printf("LoadBlockIndex() : *** found bad block at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString().c_str()); pindexFork = pindex->pprev; } - // check level 2: verify transaction index validity - if (nCheckLevel>1) - { - BOOST_FOREACH(const CTransaction &tx, block.vtx) - { - uint256 hashTx = tx.GetHash(); - CTxIndex txindex; - if (ReadTxIndex(hashTx, txindex)) - { - // check level 3: checker transaction hashes - if (nCheckLevel>2 || blockPos != txindex.pos.blockPos) - { - // either an error or a duplicate transaction - CTransaction txFound; - if (!txFound.ReadFromDisk(txindex.pos)) - { - printf("LoadBlockIndex() : *** cannot read mislocated transaction %s\n", hashTx.ToString().c_str()); - pindexFork = pindex->pprev; - } - else - if (txFound.GetHash() != hashTx) // not a duplicate tx - { - printf("LoadBlockIndex(): *** invalid tx position for %s\n", hashTx.ToString().c_str()); - pindexFork = pindex->pprev; - } - } - // check level 4: check whether spent txouts were spent within the main chain - unsigned int nOutput = 0; - if (nCheckLevel>3) - { - BOOST_FOREACH(const CDiskTxPos &txpos, txindex.vSpent) - { - if (!txpos.IsNull()) - { - // check level 6: check whether spent txouts were spent by a valid transaction that consume them - if (nCheckLevel>5) - { - CTransaction txSpend; - if (!txSpend.ReadFromDisk(txpos)) - { - printf("LoadBlockIndex(): *** cannot read spending transaction of %s:%i from disk\n", hashTx.ToString().c_str(), nOutput); - pindexFork = pindex->pprev; - } - else if (!txSpend.CheckTransaction()) - { - printf("LoadBlockIndex(): *** spending transaction of %s:%i is invalid\n", hashTx.ToString().c_str(), nOutput); - pindexFork = pindex->pprev; - } - else - { - bool fFound = false; - BOOST_FOREACH(const CTxIn &txin, txSpend.vin) - if (txin.prevout.hash == hashTx && txin.prevout.n == nOutput) - fFound = true; - if (!fFound) - { - printf("LoadBlockIndex(): *** spending transaction of %s:%i does not spend it\n", hashTx.ToString().c_str(), nOutput); - pindexFork = pindex->pprev; - } - } - } - } - nOutput++; - } - } - } - // check level 5: check whether all prevouts are marked spent - if (nCheckLevel>4) - { - BOOST_FOREACH(const CTxIn &txin, tx.vin) - { - CTxIndex txindex; - if (ReadTxIndex(txin.prevout.hash, txindex)) - if (txindex.vSpent.size()-1 < txin.prevout.n || txindex.vSpent[txin.prevout.n].IsNull()) - { - printf("LoadBlockIndex(): *** found unspent prevout %s:%i in %s\n", txin.prevout.hash.ToString().c_str(), txin.prevout.n, hashTx.ToString().c_str()); - pindexFork = pindex->pprev; - } - } - } - } - } + // TODO: stronger verifications } if (pindexFork && !fRequestShutdown) { - // Reorg back to the fork - printf("LoadBlockIndex() : *** moving best chain pointer back to block %d\n", pindexFork->nHeight); - CBlock block; - if (!block.ReadFromDisk(pindexFork)) - return error("LoadBlockIndex() : block.ReadFromDisk failed"); - CTxDB txdb; - block.SetBestChain(txdb, pindexFork); + // TODO: reorg back + return error("LoadBlockIndex(): chain database corrupted"); } return true; @@ -772,7 +650,7 @@ bool CTxDB::LoadBlockIndex() -bool CTxDB::LoadBlockIndexGuts() +bool CChainDB::LoadBlockIndexGuts() { // Get database cursor Dbc* pcursor = GetCursor(); @@ -786,7 +664,7 @@ bool CTxDB::LoadBlockIndexGuts() // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); if (fFlags == DB_SET_RANGE) - ssKey << make_pair(string("blockindex"), uint256(0)); + ssKey << make_pair('b', uint256(0)); CDataStream ssValue(SER_DISK, CLIENT_VERSION); int ret = ReadAtCursor(pcursor, ssKey, ssValue, fFlags); fFlags = DB_NEXT; @@ -798,9 +676,9 @@ bool CTxDB::LoadBlockIndexGuts() // Unserialize try { - string strType; - ssKey >> strType; - if (strType == "blockindex" && !fRequestShutdown) + char chType; + ssKey >> chType; + if (chType == 'b' && !fRequestShutdown) { CDiskBlockIndex diskindex; ssValue >> diskindex; @@ -808,7 +686,6 @@ bool CTxDB::LoadBlockIndexGuts() // Construct block index object CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); - pindexNew->pnext = InsertBlockIndex(diskindex.hashNext); pindexNew->nHeight = diskindex.nHeight; pindexNew->pos = diskindex.pos; pindexNew->nUndoPos = diskindex.nUndoPos; -- cgit v1.2.3 From ae8bfd12daa802d20791e69d3477e799d2b99f45 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 6 Jul 2012 16:33:34 +0200 Subject: Batch block connection during IBD During the initial block download (or -loadblock), delay connection of new blocks a bit, and perform them in a single action. This reduces the load on the database engine, as subsequent blocks often update an earlier block's transaction already. --- src/db.cpp | 89 +++++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 24 deletions(-) (limited to 'src/db.cpp') diff --git a/src/db.cpp b/src/db.cpp index 0a6ba3fb3..06e5543b2 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -79,8 +79,8 @@ bool CDBEnv::Open(boost::filesystem::path pathEnv_) dbenv.set_cachesize(nDbCache / 1024, (nDbCache % 1024)*1048576, 1); dbenv.set_lg_bsize(1048576); dbenv.set_lg_max(10485760); - dbenv.set_lk_max_locks(10000); - dbenv.set_lk_max_objects(10000); + dbenv.set_lk_max_locks(40000); + dbenv.set_lk_max_objects(40000); dbenv.set_errfile(fopen(pathErrorFile.string().c_str(), "a")); /// debug dbenv.set_flags(DB_AUTO_COMMIT, 1); dbenv.set_flags(DB_TXN_WRITE_NOSYNC, 1); @@ -279,14 +279,10 @@ static bool IsChainFile(std::string strFile) return false; } -void CDB::Close() +void CDB::Flush() { - if (!pdb) - return; if (activeTxn) - activeTxn->abort(); - activeTxn = NULL; - pdb = NULL; + return; // Flush database activity from memory pool to disk log unsigned int nMinutes = 0; @@ -298,6 +294,18 @@ void CDB::Close() nMinutes = 5; bitdb.dbenv.txn_checkpoint(nMinutes ? GetArg("-dblogsize", 100)*1024 : 0, nMinutes, 0); +} + +void CDB::Close() +{ + if (!pdb) + return; + if (activeTxn) + activeTxn->abort(); + activeTxn = NULL; + pdb = NULL; + + Flush(); { LOCK(bitdb.cs_db); @@ -537,6 +545,42 @@ bool CChainDB::ReadLastBlockFile(int &nFile) { return Read('l', nFile); } +CCoinsViewDB::CCoinsViewDB() : db("cr+") {} +bool CCoinsViewDB::GetCoins(uint256 txid, CCoins &coins) { return db.ReadCoins(txid, coins); } +bool CCoinsViewDB::SetCoins(uint256 txid, const CCoins &coins) { return db.WriteCoins(txid, coins); } +bool CCoinsViewDB::HaveCoins(uint256 txid) { return db.HaveCoins(txid); } +CBlockIndex *CCoinsViewDB::GetBestBlock() { + uint256 hashBestChain; + if (!db.ReadHashBestChain(hashBestChain)) + return NULL; + std::map::iterator it = mapBlockIndex.find(hashBestChain); + if (it == mapBlockIndex.end()) + return NULL; + return it->second; +} +bool CCoinsViewDB::SetBestBlock(CBlockIndex *pindex) { return db.WriteHashBestChain(pindex->GetBlockHash()); } +bool CCoinsViewDB::BatchWrite(const std::map &mapCoins, CBlockIndex *pindex) { + printf("Committing %u changed transactions to coin database...\n", (unsigned int)mapCoins.size()); + + if (!db.TxnBegin()) + return false; + bool fOk = true; + for (std::map::const_iterator it = mapCoins.begin(); it != mapCoins.end(); it++) { + fOk = db.WriteCoins(it->first, it->second); + if (!fOk) + break; + } + if (fOk) + fOk = db.WriteHashBestChain(pindex->GetBlockHash()); + + if (!fOk) + db.TxnAbort(); + else + fOk = db.TxnCommit(); + + return fOk; +} + CBlockIndex static * InsertBlockIndex(uint256 hash) { if (hash == 0) @@ -557,7 +601,7 @@ CBlockIndex static * InsertBlockIndex(uint256 hash) return pindexNew; } -bool LoadBlockIndex(CCoinsDB &coindb, CChainDB &chaindb) +bool LoadBlockIndex(CChainDB &chaindb) { if (!chaindb.LoadBlockIndexGuts()) return false; @@ -587,26 +631,23 @@ bool LoadBlockIndex(CCoinsDB &coindb, CChainDB &chaindb) printf("LoadBlockIndex(): last block file: %s\n", infoLastBlockFile.ToString().c_str()); // Load hashBestChain pointer to end of best chain - if (!coindb.ReadHashBestChain(hashBestChain)) + pindexBest = pcoinsTip->GetBestBlock(); + if (pindexBest == NULL) { if (pindexGenesisBlock == NULL) return true; return error("CTxDB::LoadBlockIndex() : hashBestChain not loaded"); } - std::map::iterator it = mapBlockIndex.find(hashBestChain); - if (it == mapBlockIndex.end()) { - return error("CTxDB::LoadBlockIndex() : hashBestChain not found in the block index"); - } else { - // set 'next' pointers in best chain - CBlockIndex *pindex = it->second; - while(pindex != NULL && pindex->pprev != NULL) { - CBlockIndex *pindexPrev = pindex->pprev; - pindexPrev->pnext = pindex; - pindex = pindexPrev; - } - pindexBest = it->second; - nBestHeight = pindexBest->nHeight; - bnBestChainWork = pindexBest->bnChainWork; + hashBestChain = pindexBest->GetBlockHash(); + nBestHeight = pindexBest->nHeight; + bnBestChainWork = pindexBest->bnChainWork; + + // set 'next' pointers in best chain + CBlockIndex *pindex = pindexBest; + while(pindex != NULL && pindex->pprev != NULL) { + CBlockIndex *pindexPrev = pindex->pprev; + pindexPrev->pnext = pindex; + pindex = pindexPrev; } printf("LoadBlockIndex(): hashBestChain=%s height=%d date=%s\n", hashBestChain.ToString().substr(0,20).c_str(), nBestHeight, -- cgit v1.2.3 From 4fea06db25108e7f72710bf22c3d1896707eeb74 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 10 Aug 2012 15:13:57 +0200 Subject: Automatically reorganize at startup to best known block Given that the block tree database (chain.dat) and the active chain database (coins.dat) are entirely separate now, it becomes legal to swap one with another instance without affecting the other. This commit introduces a check in the startup code that detects the presence of a better chain in chain.dat that has not been activated yet, and does so efficiently (in batch, while reusing the blk???.dat files). --- src/db.cpp | 1 - 1 file changed, 1 deletion(-) (limited to 'src/db.cpp') diff --git a/src/db.cpp b/src/db.cpp index 06e5543b2..5ca9ea2c3 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -636,7 +636,6 @@ bool LoadBlockIndex(CChainDB &chaindb) { if (pindexGenesisBlock == NULL) return true; - return error("CTxDB::LoadBlockIndex() : hashBestChain not loaded"); } hashBestChain = pindexBest->GetBlockHash(); nBestHeight = pindexBest->nHeight; -- cgit v1.2.3 From 857c61df0b71c8a0482b1bf8fc55849f8ad831b8 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 19 Aug 2012 00:33:01 +0200 Subject: Prepare database format for multi-stage block processing This commit adds a status field and a transaction counter to the block indexes. --- src/db.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'src/db.cpp') diff --git a/src/db.cpp b/src/db.cpp index 5ca9ea2c3..e77ddd77e 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -622,6 +622,9 @@ bool LoadBlockIndex(CChainDB &chaindb) { CBlockIndex* pindex = item.second; pindex->bnChainWork = (pindex->pprev ? pindex->pprev->bnChainWork : 0) + pindex->GetBlockWork(); + pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; + if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS && !(pindex->nStatus & BLOCK_FAILED_MASK)) + setBlockIndexValid.insert(pindex); } // Load block file info @@ -727,20 +730,23 @@ bool CChainDB::LoadBlockIndexGuts() CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); pindexNew->nHeight = diskindex.nHeight; - pindexNew->pos = diskindex.pos; + pindexNew->nFile = diskindex.nFile; + pindexNew->nDataPos = diskindex.nDataPos; pindexNew->nUndoPos = diskindex.nUndoPos; pindexNew->nVersion = diskindex.nVersion; pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; pindexNew->nTime = diskindex.nTime; pindexNew->nBits = diskindex.nBits; pindexNew->nNonce = diskindex.nNonce; + pindexNew->nStatus = diskindex.nStatus; + pindexNew->nTx = diskindex.nTx; // Watch for genesis block if (pindexGenesisBlock == NULL && diskindex.GetBlockHash() == hashGenesisBlock) pindexGenesisBlock = pindexNew; if (!pindexNew->CheckIndex()) - return error("LoadBlockIndex() : CheckIndex failed at %d", pindexNew->nHeight); + return error("LoadBlockIndex() : CheckIndex failed: %s", pindexNew->ToString().c_str()); } else { -- cgit v1.2.3 From d979e6e36ac6be0d40b2a6bb70c668f9e6989ff9 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 3 Sep 2012 15:26:57 +0200 Subject: Use singleton block tree database instance --- src/db.cpp | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'src/db.cpp') diff --git a/src/db.cpp b/src/db.cpp index e77ddd77e..5fe7e0585 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -273,7 +273,7 @@ CDB::CDB(const char *pszFile, const char* pszMode) : static bool IsChainFile(std::string strFile) { - if (strFile == "coins.dat" || strFile == "chain.dat") + if (strFile == "coins.dat" || strFile == "blktree.dat") return true; return false; @@ -483,7 +483,7 @@ void CDBEnv::Flush(bool fShutdown) // -// CChainDB and CCoinsDB +// CBlockTreeDB and CCoinsDB // bool CCoinsDB::HaveCoins(uint256 hash) { @@ -504,7 +504,7 @@ bool CCoinsDB::WriteCoins(uint256 hash, const CCoins &coins) { return Write(make_pair('c', hash), coins); } -bool CChainDB::WriteBlockIndex(const CDiskBlockIndex& blockindex) +bool CBlockTreeDB::WriteBlockIndex(const CDiskBlockIndex& blockindex) { return Write(make_pair('b', blockindex.GetBlockHash()), blockindex); } @@ -519,29 +519,29 @@ bool CCoinsDB::WriteHashBestChain(uint256 hashBestChain) return Write('B', hashBestChain); } -bool CChainDB::ReadBestInvalidWork(CBigNum& bnBestInvalidWork) +bool CBlockTreeDB::ReadBestInvalidWork(CBigNum& bnBestInvalidWork) { return Read('I', bnBestInvalidWork); } -bool CChainDB::WriteBestInvalidWork(CBigNum bnBestInvalidWork) +bool CBlockTreeDB::WriteBestInvalidWork(CBigNum bnBestInvalidWork) { return Write('I', bnBestInvalidWork); } -bool CChainDB::WriteBlockFileInfo(int nFile, const CBlockFileInfo &info) { +bool CBlockTreeDB::WriteBlockFileInfo(int nFile, const CBlockFileInfo &info) { return Write(make_pair('f', nFile), info); } -bool CChainDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { +bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { return Read(make_pair('f', nFile), info); } -bool CChainDB::WriteLastBlockFile(int nFile) { +bool CBlockTreeDB::WriteLastBlockFile(int nFile) { return Write('l', nFile); } -bool CChainDB::ReadLastBlockFile(int &nFile) { +bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { return Read('l', nFile); } @@ -601,9 +601,9 @@ CBlockIndex static * InsertBlockIndex(uint256 hash) return pindexNew; } -bool LoadBlockIndex(CChainDB &chaindb) +bool LoadBlockIndexDB() { - if (!chaindb.LoadBlockIndexGuts()) + if (!pblocktree->LoadBlockIndexGuts()) return false; if (fRequestShutdown) @@ -628,9 +628,9 @@ bool LoadBlockIndex(CChainDB &chaindb) } // Load block file info - chaindb.ReadLastBlockFile(nLastBlockFile); + pblocktree->ReadLastBlockFile(nLastBlockFile); printf("LoadBlockIndex(): last block file = %i\n", nLastBlockFile); - if (chaindb.ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + if (pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile)) printf("LoadBlockIndex(): last block file: %s\n", infoLastBlockFile.ToString().c_str()); // Load hashBestChain pointer to end of best chain @@ -656,7 +656,7 @@ bool LoadBlockIndex(CChainDB &chaindb) DateTimeStrFormat("%x %H:%M:%S", pindexBest->GetBlockTime()).c_str()); // Load bnBestInvalidWork, OK if it doesn't exist - chaindb.ReadBestInvalidWork(bnBestInvalidWork); + pblocktree->ReadBestInvalidWork(bnBestInvalidWork); // Verify blocks in the best chain int nCheckLevel = GetArg("-checklevel", 1); @@ -693,7 +693,7 @@ bool LoadBlockIndex(CChainDB &chaindb) -bool CChainDB::LoadBlockIndexGuts() +bool CBlockTreeDB::LoadBlockIndexGuts() { // Get database cursor Dbc* pcursor = GetCursor(); -- cgit v1.2.3 From 2d8a48292b0da96cda8d7b45a24a22adfb4667b2 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 3 Sep 2012 21:14:03 +0200 Subject: LevelDB block and coin databases Split off CBlockTreeDB and CCoinsViewDB into txdb-*.{cpp,h} files, implemented by either LevelDB or BDB. Based on code from earlier commits by Mike Hearn in his leveldb branch. --- src/db.cpp | 279 ------------------------------------------------------------- 1 file changed, 279 deletions(-) (limited to 'src/db.cpp') diff --git a/src/db.cpp b/src/db.cpp index 5fe7e0585..8738a0af8 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -482,285 +482,6 @@ void CDBEnv::Flush(bool fShutdown) -// -// CBlockTreeDB and CCoinsDB -// - -bool CCoinsDB::HaveCoins(uint256 hash) { - assert(!fClient); - return Exists(make_pair('c', hash)); -} - -bool CCoinsDB::ReadCoins(uint256 hash, CCoins &coins) { - assert(!fClient); - return Read(make_pair('c', hash), coins); -} - -bool CCoinsDB::WriteCoins(uint256 hash, const CCoins &coins) { - assert(!fClient); - if (coins.IsPruned()) - return Erase(make_pair('c', hash)); - else - return Write(make_pair('c', hash), coins); -} - -bool CBlockTreeDB::WriteBlockIndex(const CDiskBlockIndex& blockindex) -{ - return Write(make_pair('b', blockindex.GetBlockHash()), blockindex); -} - -bool CCoinsDB::ReadHashBestChain(uint256& hashBestChain) -{ - return Read('B', hashBestChain); -} - -bool CCoinsDB::WriteHashBestChain(uint256 hashBestChain) -{ - return Write('B', hashBestChain); -} - -bool CBlockTreeDB::ReadBestInvalidWork(CBigNum& bnBestInvalidWork) -{ - return Read('I', bnBestInvalidWork); -} - -bool CBlockTreeDB::WriteBestInvalidWork(CBigNum bnBestInvalidWork) -{ - return Write('I', bnBestInvalidWork); -} - -bool CBlockTreeDB::WriteBlockFileInfo(int nFile, const CBlockFileInfo &info) { - return Write(make_pair('f', nFile), info); -} - -bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { - return Read(make_pair('f', nFile), info); -} - -bool CBlockTreeDB::WriteLastBlockFile(int nFile) { - return Write('l', nFile); -} - -bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { - return Read('l', nFile); -} - -CCoinsViewDB::CCoinsViewDB() : db("cr+") {} -bool CCoinsViewDB::GetCoins(uint256 txid, CCoins &coins) { return db.ReadCoins(txid, coins); } -bool CCoinsViewDB::SetCoins(uint256 txid, const CCoins &coins) { return db.WriteCoins(txid, coins); } -bool CCoinsViewDB::HaveCoins(uint256 txid) { return db.HaveCoins(txid); } -CBlockIndex *CCoinsViewDB::GetBestBlock() { - uint256 hashBestChain; - if (!db.ReadHashBestChain(hashBestChain)) - return NULL; - std::map::iterator it = mapBlockIndex.find(hashBestChain); - if (it == mapBlockIndex.end()) - return NULL; - return it->second; -} -bool CCoinsViewDB::SetBestBlock(CBlockIndex *pindex) { return db.WriteHashBestChain(pindex->GetBlockHash()); } -bool CCoinsViewDB::BatchWrite(const std::map &mapCoins, CBlockIndex *pindex) { - printf("Committing %u changed transactions to coin database...\n", (unsigned int)mapCoins.size()); - - if (!db.TxnBegin()) - return false; - bool fOk = true; - for (std::map::const_iterator it = mapCoins.begin(); it != mapCoins.end(); it++) { - fOk = db.WriteCoins(it->first, it->second); - if (!fOk) - break; - } - if (fOk) - fOk = db.WriteHashBestChain(pindex->GetBlockHash()); - - if (!fOk) - db.TxnAbort(); - else - fOk = db.TxnCommit(); - - return fOk; -} - -CBlockIndex static * InsertBlockIndex(uint256 hash) -{ - if (hash == 0) - return NULL; - - // Return existing - map::iterator mi = mapBlockIndex.find(hash); - if (mi != mapBlockIndex.end()) - return (*mi).second; - - // Create new - CBlockIndex* pindexNew = new CBlockIndex(); - if (!pindexNew) - throw runtime_error("LoadBlockIndex() : new CBlockIndex failed"); - mi = mapBlockIndex.insert(make_pair(hash, pindexNew)).first; - pindexNew->phashBlock = &((*mi).first); - - return pindexNew; -} - -bool LoadBlockIndexDB() -{ - if (!pblocktree->LoadBlockIndexGuts()) - return false; - - if (fRequestShutdown) - return true; - - // Calculate bnChainWork - vector > vSortedByHeight; - vSortedByHeight.reserve(mapBlockIndex.size()); - BOOST_FOREACH(const PAIRTYPE(uint256, CBlockIndex*)& item, mapBlockIndex) - { - CBlockIndex* pindex = item.second; - vSortedByHeight.push_back(make_pair(pindex->nHeight, pindex)); - } - sort(vSortedByHeight.begin(), vSortedByHeight.end()); - BOOST_FOREACH(const PAIRTYPE(int, CBlockIndex*)& item, vSortedByHeight) - { - CBlockIndex* pindex = item.second; - pindex->bnChainWork = (pindex->pprev ? pindex->pprev->bnChainWork : 0) + pindex->GetBlockWork(); - pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; - if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS && !(pindex->nStatus & BLOCK_FAILED_MASK)) - setBlockIndexValid.insert(pindex); - } - - // Load block file info - pblocktree->ReadLastBlockFile(nLastBlockFile); - printf("LoadBlockIndex(): last block file = %i\n", nLastBlockFile); - if (pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile)) - printf("LoadBlockIndex(): last block file: %s\n", infoLastBlockFile.ToString().c_str()); - - // Load hashBestChain pointer to end of best chain - pindexBest = pcoinsTip->GetBestBlock(); - if (pindexBest == NULL) - { - if (pindexGenesisBlock == NULL) - return true; - } - hashBestChain = pindexBest->GetBlockHash(); - nBestHeight = pindexBest->nHeight; - bnBestChainWork = pindexBest->bnChainWork; - - // set 'next' pointers in best chain - CBlockIndex *pindex = pindexBest; - while(pindex != NULL && pindex->pprev != NULL) { - CBlockIndex *pindexPrev = pindex->pprev; - pindexPrev->pnext = pindex; - pindex = pindexPrev; - } - printf("LoadBlockIndex(): hashBestChain=%s height=%d date=%s\n", - hashBestChain.ToString().substr(0,20).c_str(), nBestHeight, - DateTimeStrFormat("%x %H:%M:%S", pindexBest->GetBlockTime()).c_str()); - - // Load bnBestInvalidWork, OK if it doesn't exist - pblocktree->ReadBestInvalidWork(bnBestInvalidWork); - - // Verify blocks in the best chain - int nCheckLevel = GetArg("-checklevel", 1); - int nCheckDepth = GetArg( "-checkblocks", 2500); - if (nCheckDepth == 0) - nCheckDepth = 1000000000; // suffices until the year 19000 - if (nCheckDepth > nBestHeight) - nCheckDepth = nBestHeight; - printf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); - CBlockIndex* pindexFork = NULL; - for (CBlockIndex* pindex = pindexBest; pindex && pindex->pprev; pindex = pindex->pprev) - { - if (fRequestShutdown || pindex->nHeight < nBestHeight-nCheckDepth) - break; - CBlock block; - if (!block.ReadFromDisk(pindex)) - return error("LoadBlockIndex() : block.ReadFromDisk failed"); - // check level 1: verify block validity - if (nCheckLevel>0 && !block.CheckBlock()) - { - printf("LoadBlockIndex() : *** found bad block at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString().c_str()); - pindexFork = pindex->pprev; - } - // TODO: stronger verifications - } - if (pindexFork && !fRequestShutdown) - { - // TODO: reorg back - return error("LoadBlockIndex(): chain database corrupted"); - } - - return true; -} - - - -bool CBlockTreeDB::LoadBlockIndexGuts() -{ - // Get database cursor - Dbc* pcursor = GetCursor(); - if (!pcursor) - return false; - - // Load mapBlockIndex - unsigned int fFlags = DB_SET_RANGE; - loop - { - // Read next record - CDataStream ssKey(SER_DISK, CLIENT_VERSION); - if (fFlags == DB_SET_RANGE) - ssKey << make_pair('b', uint256(0)); - CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = ReadAtCursor(pcursor, ssKey, ssValue, fFlags); - fFlags = DB_NEXT; - if (ret == DB_NOTFOUND) - break; - else if (ret != 0) - return false; - - // Unserialize - - try { - char chType; - ssKey >> chType; - if (chType == 'b' && !fRequestShutdown) - { - CDiskBlockIndex diskindex; - ssValue >> diskindex; - - // Construct block index object - CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); - pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); - pindexNew->nHeight = diskindex.nHeight; - pindexNew->nFile = diskindex.nFile; - pindexNew->nDataPos = diskindex.nDataPos; - pindexNew->nUndoPos = diskindex.nUndoPos; - pindexNew->nVersion = diskindex.nVersion; - pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; - pindexNew->nTime = diskindex.nTime; - pindexNew->nBits = diskindex.nBits; - pindexNew->nNonce = diskindex.nNonce; - pindexNew->nStatus = diskindex.nStatus; - pindexNew->nTx = diskindex.nTx; - - // Watch for genesis block - if (pindexGenesisBlock == NULL && diskindex.GetBlockHash() == hashGenesisBlock) - pindexGenesisBlock = pindexNew; - - if (!pindexNew->CheckIndex()) - return error("LoadBlockIndex() : CheckIndex failed: %s", pindexNew->ToString().c_str()); - } - else - { - break; // if shutdown requested or finished loading block index - } - } // try - catch (std::exception &e) { - return error("%s() : deserialize error", __PRETTY_FUNCTION__); - } - } - pcursor->close(); - - return true; -} -- cgit v1.2.3