From 0fa593d0fb3a6f731a31610e98ce6bfd6a50a96c Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sat, 16 Jun 2012 13:36:00 +0200 Subject: Compact serialization for amounts Special serializer/deserializer for amount values. It is optimized for values which have few non-zero digits in decimal representation. Most amounts currently in the txout set take only 1 or 2 bytes to represent. --- src/main.cpp | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index 43bb4b179..ed677f31c 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4058,3 +4058,57 @@ void GenerateBitcoins(bool fGenerate, CWallet* pwallet) } } } + +// Amount compression: +// * If the amount is 0, output 0 +// * first, divide the amount (in base units) by the largest power of 10 possible; call the exponent e (e is max 9) +// * if e<9, the last digit of the resulting number cannot be 0; store it as d, and drop it (divide by 10) +// * call the result n +// * output 1 + 10*(9*n + d - 1) + e +// * if e==9, we only know the resulting number is not zero, so output 1 + 10*(n - 1) + 9 +// (this is decodable, as d is in [1-9] and e is in [0-9]) + +uint64 CTxOutCompressor::CompressAmount(uint64 n) +{ + if (n == 0) + return 0; + int e = 0; + while (((n % 10) == 0) && e < 9) { + n /= 10; + e++; + } + if (e < 9) { + int d = (n % 10); + assert(d >= 1 && d <= 9); + n /= 10; + return 1 + (n*9 + d - 1)*10 + e; + } else { + return 1 + (n - 1)*10 + 9; + } +} + +uint64 CTxOutCompressor::DecompressAmount(uint64 x) +{ + // x = 0 OR x = 1+10*(9*n + d - 1) + e OR x = 1+10*(n - 1) + 9 + if (x == 0) + return 0; + x--; + // x = 10*(9*n + d - 1) + e + int e = x % 10; + x /= 10; + uint64 n = 0; + if (e < 9) { + // x = 9*n + d - 1 + int d = (x % 9) + 1; + x /= 9; + // x = n + n = x*10 + d; + } else { + n = x+1; + } + while (e) { + n *= 10; + e--; + } + return n; +} -- cgit v1.2.3 From 630fd8dcb6662ed167e967625f2715f97ddd2db5 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 19 Jun 2012 01:36:43 +0200 Subject: One file per block Refactor of the block storage code, which now stores one file per block. This will allow easier pruning, as blocks can be removed individually. --- src/main.cpp | 103 ++++++++++++++++++++--------------------------------------- 1 file changed, 34 insertions(+), 69 deletions(-) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index ed677f31c..fa818bd05 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -389,7 +389,7 @@ int CMerkleTx::SetMerkleBranch(const CBlock* pblock) CTxIndex txindex; if (!CTxDB("r").ReadTxIndex(GetHash(), txindex)) return 0; - if (!blockTmp.ReadFromDisk(txindex.pos.nFile, txindex.pos.nBlockPos)) + if (!blockTmp.ReadFromDisk(txindex.pos.blockPos)) return 0; pblock = &blockTmp; } @@ -646,7 +646,7 @@ bool CTxMemPool::accept(CTxDB& txdb, CTransaction &tx, bool fCheckInputs, // Check against previous transactions // This is done last to help prevent CPU exhaustion denial-of-service attacks. - if (!tx.ConnectInputs(mapInputs, mapUnused, CDiskTxPos(1,1,1), pindexBest, false, false)) + if (!tx.ConnectInputs(mapInputs, mapUnused, CDiskTxPos(true), pindexBest, false, false)) { return error("CTxMemPool::accept() : ConnectInputs failed %s", hash.ToString().substr(0,10).c_str()); } @@ -817,7 +817,7 @@ int CTxIndex::GetDepthInMainChain() const { // Read block header CBlock block; - if (!block.ReadFromDisk(pos.nFile, pos.nBlockPos, false)) + if (!block.ReadFromDisk(pos.blockPos, false)) return 0; // Find the block in the index map::iterator mi = mapBlockIndex.find(block.GetHash()); @@ -847,7 +847,7 @@ bool GetTransaction(const uint256 &hash, CTransaction &tx, uint256 &hashBlock) if (tx.ReadFromDisk(txdb, COutPoint(hash, 0), txindex)) { CBlock block; - if (block.ReadFromDisk(txindex.pos.nFile, txindex.pos.nBlockPos, false)) + if (block.ReadFromDisk(txindex.pos.blockPos, false)) hashBlock = block.GetHash(); return true; } @@ -892,7 +892,7 @@ bool CBlock::ReadFromDisk(const CBlockIndex* pindex, bool fReadTransactions) *this = pindex->GetBlockHeader(); return true; } - if (!ReadFromDisk(pindex->nFile, pindex->nBlockPos, fReadTransactions)) + if (!ReadFromDisk(pindex->GetBlockPos(), fReadTransactions)) return false; if (GetHash() != pindex->GetBlockHash()) return error("CBlock::ReadFromDisk() : GetHash() doesn't match index"); @@ -1156,7 +1156,7 @@ bool CTransaction::FetchInputs(CTxDB& txdb, const map& mapTes // Read txPrev CTransaction& txPrev = inputsRet[prevout.hash].second; - if (!fFound || txindex.pos == CDiskTxPos(1,1,1)) + if (!fFound || txindex.pos.IsMemPool()) { // Get prev tx from single transactions in memory { @@ -1262,7 +1262,7 @@ bool CTransaction::ConnectInputs(MapPrevTx inputs, // If prev is coinbase, check that it's matured if (txPrev.IsCoinBase()) for (const CBlockIndex* pindex = pindexBlock; pindex && pindexBlock->nHeight - pindex->nHeight < COINBASE_MATURITY; pindex = pindex->pprev) - if (pindex->nBlockPos == txindex.pos.nBlockPos && pindex->nFile == txindex.pos.nFile) + if (pindex->GetBlockPos() == txindex.pos.blockPos) return error("ConnectInputs() : tried to spend coinbase at depth %d", pindexBlock->nHeight - pindex->nHeight); // Check for negative or overflow input values @@ -1427,11 +1427,10 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) //// issue here: it doesn't know the version unsigned int nTxPos; if (fJustCheck) - // FetchInputs treats CDiskTxPos(1,1,1) as a special "refer to memorypool" indicator // Since we're just checking the block and not actually connecting it, it might not (and probably shouldn't) be on the disk to get the transaction from nTxPos = 1; else - nTxPos = pindex->nBlockPos + ::GetSerializeSize(CBlock(), SER_DISK, CLIENT_VERSION) - 1 + GetSizeOfCompactSize(vtx.size()); + nTxPos = ::GetSerializeSize(CBlock(), SER_DISK, CLIENT_VERSION) - 1 + GetSizeOfCompactSize(vtx.size()); map mapQueuedChanges; int64 nFees = 0; @@ -1453,9 +1452,11 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) if (nSigOps > MAX_BLOCK_SIGOPS) return DoS(100, error("ConnectBlock() : too many sigops")); - CDiskTxPos posThisTx(pindex->nFile, pindex->nBlockPos, nTxPos); + CDiskTxPos posThisTx(pindex->GetBlockPos(), nTxPos); if (!fJustCheck) nTxPos += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION); + else + posThisTx = CDiskTxPos(true); MapPrevTx mapInputs; if (!tx.IsCoinBase()) @@ -1750,7 +1751,7 @@ bool CBlock::SetBestChain(CTxDB& txdb, CBlockIndex* pindexNew) } -bool CBlock::AddToBlockIndex(unsigned int nFile, unsigned int nBlockPos) +bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) { // Check for duplicate uint256 hash = GetHash(); @@ -1758,7 +1759,7 @@ bool CBlock::AddToBlockIndex(unsigned int nFile, unsigned int nBlockPos) return error("AddToBlockIndex() : %s already exists", hash.ToString().substr(0,20).c_str()); // Construct new block index object - CBlockIndex* pindexNew = new CBlockIndex(nFile, nBlockPos, *this); + CBlockIndex* pindexNew = new CBlockIndex(*this); if (!pindexNew) return error("AddToBlockIndex() : new CBlockIndex failed"); map::iterator mi = mapBlockIndex.insert(make_pair(hash, pindexNew)).first; @@ -1770,6 +1771,8 @@ bool CBlock::AddToBlockIndex(unsigned int nFile, unsigned int nBlockPos) pindexNew->nHeight = pindexNew->pprev->nHeight + 1; } pindexNew->bnChainWork = (pindexNew->pprev ? pindexNew->pprev->bnChainWork : 0) + pindexNew->GetBlockWork(); + assert(pos.nHeight == pindexNew->nHeight); + pindexNew->nAlternative = pos.nAlternative; CTxDB txdb; if (!txdb.TxnBegin()) @@ -1908,13 +1911,12 @@ bool CBlock::AcceptBlock() } // Write block to history file + CDiskBlockPos blockPos = CDiskBlockPos(nHeight); if (!CheckDiskSpace(::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION))) return error("AcceptBlock() : out of disk space"); - unsigned int nFile = -1; - unsigned int nBlockPos = 0; - if (!WriteToDisk(nFile, nBlockPos)) + if (!WriteToDisk(blockPos)) return error("AcceptBlock() : WriteToDisk failed"); - if (!AddToBlockIndex(nFile, nBlockPos)) + if (!AddToBlockIndex(blockPos)) return error("AcceptBlock() : AddToBlockIndex failed"); // Relay inventory, but don't relay old inventory during initial block download @@ -2048,53 +2050,18 @@ bool CheckDiskSpace(uint64 nAdditionalBytes) return true; } -static filesystem::path BlockFilePath(unsigned int nFile) +FILE* OpenBlockFile(const CDiskBlockPos &pos, const char* pszMode) { - string strBlockFn = strprintf("blk%04u.dat", nFile); - return GetDataDir() / strBlockFn; -} - -FILE* OpenBlockFile(unsigned int nFile, unsigned int nBlockPos, const char* pszMode) -{ - if ((nFile < 1) || (nFile == (unsigned int) -1)) + boost::filesystem::path path = pos.GetFileName(GetDataDir()); + boost::filesystem::create_directories(path.parent_path()); + if (pos.IsNull() || pos.IsMemPool()) return NULL; - FILE* file = fopen(BlockFilePath(nFile).string().c_str(), pszMode); + FILE* file = fopen(path.string().c_str(), pszMode); if (!file) return NULL; - if (nBlockPos != 0 && !strchr(pszMode, 'a') && !strchr(pszMode, 'w')) - { - if (fseek(file, nBlockPos, SEEK_SET) != 0) - { - fclose(file); - return NULL; - } - } return file; } -static unsigned int nCurrentBlockFile = 1; - -FILE* AppendBlockFile(unsigned int& nFileRet) -{ - nFileRet = 0; - loop - { - FILE* file = OpenBlockFile(nCurrentBlockFile, 0, "ab"); - if (!file) - return NULL; - if (fseek(file, 0, SEEK_END) != 0) - return NULL; - // FAT32 file size max 4GB, fseek and ftell max 2GB, so we must stay under 2GB - if (ftell(file) < (long)(0x7F000000 - MAX_SIZE)) - { - nFileRet = nCurrentBlockFile; - return file; - } - fclose(file); - nCurrentBlockFile++; - } -} - bool LoadBlockIndex(bool fAllowNew) { if (fTestNet) @@ -2153,19 +2120,19 @@ bool LoadBlockIndex(bool fAllowNew) } //// debug print - printf("%s\n", block.GetHash().ToString().c_str()); + uint256 hash = block.GetHash(); + printf("%s\n", hash.ToString().c_str()); printf("%s\n", hashGenesisBlock.ToString().c_str()); printf("%s\n", block.hashMerkleRoot.ToString().c_str()); assert(block.hashMerkleRoot == uint256("0x4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b")); block.print(); - assert(block.GetHash() == hashGenesisBlock); + assert(hash == hashGenesisBlock); // Start new block file - unsigned int nFile; - unsigned int nBlockPos; - if (!block.WriteToDisk(nFile, nBlockPos)) + CDiskBlockPos blockPos(0); + if (!block.WriteToDisk(blockPos)) return error("LoadBlockIndex() : writing genesis block to disk failed"); - if (!block.AddToBlockIndex(nFile, nBlockPos)) + if (!block.AddToBlockIndex(blockPos)) return error("LoadBlockIndex() : genesis block not accepted"); } @@ -2219,11 +2186,9 @@ void PrintBlockTree() // print item CBlock block; block.ReadFromDisk(pindex); - printf("%d (%u,%u) %s %s tx %"PRIszu"", + printf("%d (%s) %s tx %"PRIszu"", pindex->nHeight, - pindex->nFile, - pindex->nBlockPos, - block.GetHash().ToString().substr(0,20).c_str(), + pindex->GetBlockPos().GetFileName("").string().c_str(), DateTimeStrFormat("%x %H:%M:%S", block.GetBlockTime()).c_str(), block.vtx.size()); @@ -3693,9 +3658,9 @@ CBlock* CreateNewBlock(CReserveKey& reservekey) if (nBlockSigOps + nTxSigOps >= MAX_BLOCK_SIGOPS) continue; - if (!tx.ConnectInputs(mapInputs, mapTestPoolTmp, CDiskTxPos(1,1,1), pindexPrev, false, true)) + if (!tx.ConnectInputs(mapInputs, mapTestPoolTmp, CDiskTxPos(true), pindexPrev, false, true)) continue; - mapTestPoolTmp[tx.GetHash()] = CTxIndex(CDiskTxPos(1,1,1), tx.vout.size()); + mapTestPoolTmp[tx.GetHash()] = CTxIndex(CDiskTxPos(true), tx.vout.size()); swap(mapTestPool, mapTestPoolTmp); // Added @@ -3743,7 +3708,7 @@ CBlock* CreateNewBlock(CReserveKey& reservekey) pblock->nNonce = 0; pblock->vtx[0].vin[0].scriptSig = scriptDummy; - CBlockIndex indexDummy(1, 1, *pblock); + CBlockIndex indexDummy(*pblock); indexDummy.pprev = pindexPrev; indexDummy.nHeight = pindexPrev->nHeight + 1; if (!pblock->ConnectBlock(txdb, &indexDummy, true)) -- cgit v1.2.3 From 8adf48dc9b45816793c7b98e2f4fa625c2e09f2c Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sat, 23 Jun 2012 14:17:13 +0200 Subject: Preliminary undo file creation Create files (one per block) with undo information for the transactions in it. --- src/main.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index fa818bd05..8b1532e48 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1432,6 +1432,8 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) else nTxPos = ::GetSerializeSize(CBlock(), SER_DISK, CLIENT_VERSION) - 1 + GetSizeOfCompactSize(vtx.size()); + CBlockUndo blockundo; + map mapQueuedChanges; int64 nFees = 0; unsigned int nSigOps = 0; @@ -1461,6 +1463,8 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) MapPrevTx mapInputs; if (!tx.IsCoinBase()) { + CTxUndo undo; + bool fInvalid; if (!tx.FetchInputs(txdb, mapQueuedChanges, true, false, mapInputs, fInvalid)) return false; @@ -1477,8 +1481,14 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) nFees += tx.GetValueIn(mapInputs)-tx.GetValueOut(); + BOOST_FOREACH(const CTxIn &in, tx.vin) { + undo.vprevout.push_back(CTxInUndo(mapInputs[in.prevout.hash].second.vout[in.prevout.n], pindex->nHeight)); + } + if (!tx.ConnectInputs(mapInputs, mapQueuedChanges, posThisTx, pindex, true, false, fStrictPayToScriptHash)) return false; + + blockundo.vtxundo.push_back(undo); } mapQueuedChanges[hashTx] = CTxIndex(posThisTx, tx.vout.size()); @@ -1507,6 +1517,13 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) return error("ConnectBlock() : WriteBlockIndex failed"); } + // Write undo information to disk + if (pindex->nHeight > Checkpoints::GetTotalBlocksEstimate()) + { + CAutoFile fileUndo(fopen(pindex->GetBlockPos().GetUndoFile(GetDataDir()).string().c_str(), "wb"), SER_DISK, CLIENT_VERSION); + fileUndo << blockundo; + } + // Watch for transactions paying to me BOOST_FOREACH(CTransaction& tx, vtx) SyncWithWallets(tx, this, true); -- cgit v1.2.3 From 5382bcf8cd23c36a435c29080770a79b5e28af42 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 13 Aug 2012 19:11:05 +0200 Subject: Multiple blocks per file Change the block storage layer again, this time with multiple files per block, but tracked by txindex.dat database entries. The file format is exactly the same as the earlier blk00001.dat, but with smaller files (128 MiB for now). The database entries track how many bytes each block file already uses, how many blocks are in it, which range of heights is present and which range of dates. --- src/main.cpp | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 108 insertions(+), 18 deletions(-) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index 8b1532e48..15a233137 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1399,6 +1399,8 @@ bool CBlock::DisconnectBlock(CTxDB& txdb, CBlockIndex* pindex) return true; } +bool FindUndoPos(CTxDB &txdb, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); + bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) { // Check it again in case a previous version let a bad block in @@ -1430,7 +1432,7 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) // Since we're just checking the block and not actually connecting it, it might not (and probably shouldn't) be on the disk to get the transaction from nTxPos = 1; else - nTxPos = ::GetSerializeSize(CBlock(), SER_DISK, CLIENT_VERSION) - 1 + GetSizeOfCompactSize(vtx.size()); + nTxPos = pindex->pos.nPos + ::GetSerializeSize(CBlock(), SER_DISK, CLIENT_VERSION) - 1 + GetSizeOfCompactSize(vtx.size()); CBlockUndo blockundo; @@ -1507,6 +1509,17 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) return error("ConnectBlock() : UpdateTxIndex failed"); } + // Write undo information to disk + if (pindex->GetUndoPos().IsNull() && pindex->nHeight > Checkpoints::GetTotalBlocksEstimate()) + { + CDiskBlockPos pos; + if (!FindUndoPos(txdb, pindex->pos.nFile, pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 8)) + return error("ConnectBlock() : FindUndoPos failed"); + if (!blockundo.WriteToDisk(pos)) + return error("ConnectBlock() : CBlockUndo::WriteToDisk failed"); + pindex->nUndoPos = pos.nPos; + } + // Update block index on disk without changing it in memory. // The memory index structure will be changed after the db commits. if (pindex->pprev) @@ -1517,13 +1530,6 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) return error("ConnectBlock() : WriteBlockIndex failed"); } - // Write undo information to disk - if (pindex->nHeight > Checkpoints::GetTotalBlocksEstimate()) - { - CAutoFile fileUndo(fopen(pindex->GetBlockPos().GetUndoFile(GetDataDir()).string().c_str(), "wb"), SER_DISK, CLIENT_VERSION); - fileUndo << blockundo; - } - // Watch for transactions paying to me BOOST_FOREACH(CTransaction& tx, vtx) SyncWithWallets(tx, this, true); @@ -1788,8 +1794,8 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) pindexNew->nHeight = pindexNew->pprev->nHeight + 1; } pindexNew->bnChainWork = (pindexNew->pprev ? pindexNew->pprev->bnChainWork : 0) + pindexNew->GetBlockWork(); - assert(pos.nHeight == pindexNew->nHeight); - pindexNew->nAlternative = pos.nAlternative; + pindexNew->pos = pos; + pindexNew->nUndoPos = 0; CTxDB txdb; if (!txdb.TxnBegin()) @@ -1819,6 +1825,57 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) +bool FindBlockPos(CTxDB &txdb, CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64 nTime) +{ + bool fUpdatedLast = false; + + LOCK(cs_LastBlockFile); + + while (infoLastBlockFile.nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { + printf("Leaving block file %i: %s\n", nLastBlockFile, infoLastBlockFile.ToString().c_str()); + nLastBlockFile++; + infoLastBlockFile.SetNull(); + txdb.ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine + fUpdatedLast = true; + } + + pos.nFile = nLastBlockFile; + pos.nPos = infoLastBlockFile.nSize; + infoLastBlockFile.nSize += nAddSize; + infoLastBlockFile.AddBlock(nHeight, nTime); + + if (!txdb.WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + return error("FindBlockPos() : cannot write updated block info"); + if (fUpdatedLast) + txdb.WriteLastBlockFile(nLastBlockFile); + + return true; +} + +bool FindUndoPos(CTxDB &txdb, int nFile, CDiskBlockPos &pos, unsigned int nAddSize) +{ + pos.nFile = nFile; + + LOCK(cs_LastBlockFile); + + if (nFile == nLastBlockFile) { + pos.nPos = infoLastBlockFile.nUndoSize; + infoLastBlockFile.nUndoSize += nAddSize; + if (!txdb.WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + return error("FindUndoPos() : cannot write updated block info"); + return true; + } + + CBlockFileInfo info; + if (!txdb.ReadBlockFileInfo(nFile, info)) + return error("FindUndoPos() : cannot read block info"); + pos.nPos = info.nUndoSize; + info.nUndoSize += nAddSize; + if (!txdb.WriteBlockFileInfo(nFile, info)) + return error("FindUndoPos() : cannot write updated block info"); + return true; +} + bool CBlock::CheckBlock(bool fCheckPOW, bool fCheckMerkleRoot) const { @@ -1928,9 +1985,15 @@ bool CBlock::AcceptBlock() } // Write block to history file - CDiskBlockPos blockPos = CDiskBlockPos(nHeight); + unsigned int nBlockSize = ::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION); if (!CheckDiskSpace(::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION))) return error("AcceptBlock() : out of disk space"); + CDiskBlockPos blockPos; + { + CTxDB txdb; + if (!FindBlockPos(txdb, blockPos, nBlockSize+8, nHeight, nTime)) + return error("AcceptBlock() : FindBlockPos failed"); + } if (!WriteToDisk(blockPos)) return error("AcceptBlock() : WriteToDisk failed"); if (!AddToBlockIndex(blockPos)) @@ -2067,18 +2130,39 @@ bool CheckDiskSpace(uint64 nAdditionalBytes) return true; } -FILE* OpenBlockFile(const CDiskBlockPos &pos, const char* pszMode) +CCriticalSection cs_LastBlockFile; +CBlockFileInfo infoLastBlockFile; +int nLastBlockFile = 0; + +FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) { - boost::filesystem::path path = pos.GetFileName(GetDataDir()); - boost::filesystem::create_directories(path.parent_path()); if (pos.IsNull() || pos.IsMemPool()) return NULL; - FILE* file = fopen(path.string().c_str(), pszMode); + boost::filesystem::path path = GetDataDir() / "blocks" / strprintf("%s%05u.dat", prefix, pos.nFile); + boost::filesystem::create_directories(path.parent_path()); + FILE* file = fopen(path.string().c_str(), "rb+"); + if (!file && !fReadOnly) + file = fopen(path.string().c_str(), "wb+"); if (!file) return NULL; + if (pos.nPos) { + if (fseek(file, pos.nPos, SEEK_SET)) { + printf("Unable to seek to position %u of %s\n", pos.nPos, path.string().c_str()); + fclose(file); + return NULL; + } + } return file; } +FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) { + return OpenDiskFile(pos, "blk", fReadOnly); +} + +FILE *OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { + return OpenDiskFile(pos, "rev", fReadOnly); +} + bool LoadBlockIndex(bool fAllowNew) { if (fTestNet) @@ -2146,7 +2230,13 @@ bool LoadBlockIndex(bool fAllowNew) assert(hash == hashGenesisBlock); // Start new block file - CDiskBlockPos blockPos(0); + unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); + CDiskBlockPos blockPos; + { + CTxDB txdb; + if (!FindBlockPos(txdb, blockPos, nBlockSize+8, 0, block.nTime)) + return error("AcceptBlock() : FindBlockPos failed"); + } if (!block.WriteToDisk(blockPos)) return error("LoadBlockIndex() : writing genesis block to disk failed"); if (!block.AddToBlockIndex(blockPos)) @@ -2203,9 +2293,9 @@ void PrintBlockTree() // print item CBlock block; block.ReadFromDisk(pindex); - printf("%d (%s) %s tx %"PRIszu"", + printf("%d (blk%05u.dat:0x%lx) %s tx %"PRIszu"", pindex->nHeight, - pindex->GetBlockPos().GetFileName("").string().c_str(), + pindex->GetBlockPos().nFile, pindex->GetBlockPos().nPos, DateTimeStrFormat("%x %H:%M:%S", block.GetBlockTime()).c_str(), block.vtx.size()); -- cgit v1.2.3 From bba89aa82a80f0373dcb7288d96d5b0fcb453d73 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 16 Aug 2012 02:21:28 +0200 Subject: Pre-allocate block and undo files in chunks Introduce a AllocateFileRange() function in util, which wipes or at least allocates a given range of a file. It can be overriden by more efficient OS-dependent versions if necessary. Block and undo files are now allocated in chunks of 16 and 1 MiB, respectively. --- src/main.cpp | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index 15a233137..616845e92 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1844,6 +1844,17 @@ bool FindBlockPos(CTxDB &txdb, CDiskBlockPos &pos, unsigned int nAddSize, unsign infoLastBlockFile.nSize += nAddSize; infoLastBlockFile.AddBlock(nHeight, nTime); + unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; + unsigned int nNewChunks = (infoLastBlockFile.nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; + if (nNewChunks > nOldChunks) { + FILE *file = OpenBlockFile(pos); + if (file) { + printf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); + AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos); + } + fclose(file); + } + if (!txdb.WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) return error("FindBlockPos() : cannot write updated block info"); if (fUpdatedLast) @@ -1858,21 +1869,33 @@ bool FindUndoPos(CTxDB &txdb, int nFile, CDiskBlockPos &pos, unsigned int nAddSi LOCK(cs_LastBlockFile); + unsigned int nNewSize; if (nFile == nLastBlockFile) { pos.nPos = infoLastBlockFile.nUndoSize; - infoLastBlockFile.nUndoSize += nAddSize; + nNewSize = (infoLastBlockFile.nUndoSize += nAddSize); if (!txdb.WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) return error("FindUndoPos() : cannot write updated block info"); - return true; + } else { + CBlockFileInfo info; + if (!txdb.ReadBlockFileInfo(nFile, info)) + return error("FindUndoPos() : cannot read block info"); + pos.nPos = info.nUndoSize; + nNewSize = (info.nUndoSize += nAddSize); + if (!txdb.WriteBlockFileInfo(nFile, info)) + return error("FindUndoPos() : cannot write updated block info"); + } + + unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; + unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; + if (nNewChunks > nOldChunks) { + FILE *file = OpenUndoFile(pos); + if (file) { + printf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile); + AllocateFileRange(file, pos.nPos, nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos); + } + fclose(file); } - CBlockFileInfo info; - if (!txdb.ReadBlockFileInfo(nFile, info)) - return error("FindUndoPos() : cannot read block info"); - pos.nPos = info.nUndoSize; - info.nUndoSize += nAddSize; - if (!txdb.WriteBlockFileInfo(nFile, info)) - return error("FindUndoPos() : cannot write updated block info"); return true; } -- cgit v1.2.3 From 450cbb0944cd20a06ce806e6679a1f4c83c50db2 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 1 Jul 2012 18:54:00 +0200 Subject: Ultraprune This switches bitcoin's transaction/block verification logic to use a "coin database", which contains all unredeemed transaction output scripts, amounts and heights. The name ultraprune comes from the fact that instead of a full transaction index, we only (need to) keep an index with unspent outputs. For now, the blocks themselves are kept as usual, although they are only necessary for serving, rescanning and reorganizing. The basic datastructures are CCoins (representing the coins of a single transaction), and CCoinsView (representing a state of the coins database). There are several implementations for CCoinsView. A dummy, one backed by the coins database (coins.dat), one backed by the memory pool, and one that adds a cache on top of it. FetchInputs, ConnectInputs, ConnectBlock, DisconnectBlock, ... now operate on a generic CCoinsView. The block switching logic now builds a single cached CCoinsView with changes to be committed to the database before any changes are made. This means no uncommitted changes are ever read from the database, and should ease the transition to another database layer which does not support transactions (but does support atomic writes), like LevelDB. For the getrawtransaction() RPC call, access to a txid-to-disk index would be preferable. As this index is not necessary or even useful for any other part of the implementation, it is not provided. Instead, getrawtransaction() uses the coin database to find the block height, and then scans that block to find the requested transaction. This is slow, but should suffice for debug purposes. --- src/main.cpp | 1091 ++++++++++++++++++++++++++++------------------------------ 1 file changed, 531 insertions(+), 560 deletions(-) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index 616845e92..ef5f627d9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -158,6 +158,99 @@ void static ResendWalletTransactions() +////////////////////////////////////////////////////////////////////////////// +// +// CCoinsView implementations +// + +bool CCoinsView::GetCoins(uint256 txid, CCoins &coins) { return false; } +bool CCoinsView::SetCoins(uint256 txid, const CCoins &coins) { return false; } +bool CCoinsView::HaveCoins(uint256 txid) { return false; } +CBlockIndex *CCoinsView::GetBestBlock() { return NULL; } +bool CCoinsView::SetBestBlock(CBlockIndex *pindex) { return false; } + +CCoinsViewBacked::CCoinsViewBacked(CCoinsView &viewIn) : base(&viewIn) { } +bool CCoinsViewBacked::GetCoins(uint256 txid, CCoins &coins) { return base->GetCoins(txid, coins); } +bool CCoinsViewBacked::SetCoins(uint256 txid, const CCoins &coins) { return base->SetCoins(txid, coins); } +bool CCoinsViewBacked::HaveCoins(uint256 txid) { return base->HaveCoins(txid); } +CBlockIndex *CCoinsViewBacked::GetBestBlock() { return base->GetBestBlock(); } +bool CCoinsViewBacked::SetBestBlock(CBlockIndex *pindex) { return base->SetBestBlock(pindex); } +void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } + +CCoinsViewDB::CCoinsViewDB(CCoinsDB &dbIn) : db(dbIn) {} +bool CCoinsViewDB::GetCoins(uint256 txid, CCoins &coins) { return db.ReadCoins(txid, coins); } +bool CCoinsViewDB::SetCoins(uint256 txid, const CCoins &coins) { return db.WriteCoins(txid, coins); } +bool CCoinsViewDB::HaveCoins(uint256 txid) { return db.HaveCoins(txid); } +CBlockIndex *CCoinsViewDB::GetBestBlock() { return pindexBest; } +bool CCoinsViewDB::SetBestBlock(CBlockIndex *pindex) { return db.WriteHashBestChain(pindex->GetBlockHash()); } + +CCoinsViewCache::CCoinsViewCache(CCoinsView &baseIn, bool fDummy) : CCoinsViewBacked(baseIn), pindexTip(NULL) { } + +bool CCoinsViewCache::GetCoins(uint256 txid, CCoins &coins) { + if (cacheCoins.count(txid)) { + coins = cacheCoins[txid]; + return true; + } + if (base->GetCoins(txid, coins)) { + cacheCoins[txid] = coins; + return true; + } + return false; +} + +bool CCoinsViewCache::SetCoins(uint256 txid, const CCoins &coins) { + cacheCoins[txid] = coins; + return true; +} + +bool CCoinsViewCache::HaveCoins(uint256 txid) { + return cacheCoins.count(txid) || base->HaveCoins(txid); +} + +CBlockIndex *CCoinsViewCache::GetBestBlock() { + if (pindexTip == NULL) + pindexTip = base->GetBestBlock(); + return pindexTip; +} + +bool CCoinsViewCache::SetBestBlock(CBlockIndex *pindex) { + pindexTip = pindex; + return true; +} + +bool CCoinsViewCache::Flush() { + for (std::map::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) { + if (!base->SetCoins(it->first, it->second)) + return false; + } + if (!base->SetBestBlock(pindexTip)) + return false; + cacheCoins.clear(); + pindexTip = NULL; + return true; +} + +/** CCoinsView that brings transactions from a memorypool into view. + It does not check for spendings by memory pool transactions. */ +CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView &baseIn, CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } + +bool CCoinsViewMemPool::GetCoins(uint256 txid, CCoins &coins) { + if (base->GetCoins(txid, coins)) + return true; + if (mempool.exists(txid)) { + const CTransaction &tx = mempool.lookup(txid); + coins = CCoins(tx, MEMPOOL_HEIGHT); + return true; + } + return false; +} + +bool CCoinsViewMemPool::HaveCoins(uint256 txid) { + return mempool.exists(txid) || base->HaveCoins(txid); +} + + + ////////////////////////////////////////////////////////////////////////////// // // mapOrphanTransactions @@ -237,37 +330,9 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) ////////////////////////////////////////////////////////////////////////////// // -// CTransaction and CTxIndex +// CTransaction // -bool CTransaction::ReadFromDisk(CTxDB& txdb, COutPoint prevout, CTxIndex& txindexRet) -{ - SetNull(); - if (!txdb.ReadTxIndex(prevout.hash, txindexRet)) - return false; - if (!ReadFromDisk(txindexRet.pos)) - return false; - if (prevout.n >= vout.size()) - { - SetNull(); - return false; - } - return true; -} - -bool CTransaction::ReadFromDisk(CTxDB& txdb, COutPoint prevout) -{ - CTxIndex txindex; - return ReadFromDisk(txdb, prevout, txindex); -} - -bool CTransaction::ReadFromDisk(COutPoint prevout) -{ - CTxDB txdb("r"); - CTxIndex txindex; - return ReadFromDisk(txdb, prevout, txindex); -} - bool CTransaction::IsStandard() const { if (nVersion > CTransaction::CURRENT_VERSION) @@ -303,7 +368,7 @@ bool CTransaction::IsStandard() const // expensive-to-check-upon-redemption script like: // DUP CHECKSIG DROP ... repeated 100 times... OP_1 // -bool CTransaction::AreInputsStandard(const MapPrevTx& mapInputs) const +bool CTransaction::AreInputsStandard(CCoinsView& mapInputs) const { if (IsCoinBase()) return true; // Coinbases don't use vin normally @@ -383,17 +448,21 @@ int CMerkleTx::SetMerkleBranch(const CBlock* pblock) else { CBlock blockTmp; - if (pblock == NULL) - { - // Load the block this tx is in - CTxIndex txindex; - if (!CTxDB("r").ReadTxIndex(GetHash(), txindex)) - return 0; - if (!blockTmp.ReadFromDisk(txindex.pos.blockPos)) - return 0; - pblock = &blockTmp; + + if (pblock == NULL) { + CCoinsDB coinsdb("r"); + CCoins coins; + if (coinsdb.ReadCoins(GetHash(), coins)) { + CBlockIndex *pindex = FindBlockByHeight(coins.nHeight); + if (pindex) { + if (!blockTmp.ReadFromDisk(pindex)) + return 0; + pblock = &blockTmp; + } + } } + if (pblock) { // Update the tx's hashBlock hashBlock = pblock->GetHash(); @@ -411,6 +480,7 @@ int CMerkleTx::SetMerkleBranch(const CBlock* pblock) // Fill in merkle branch vMerkleBranch = pblock->GetMerkleBranch(nIndex); + } } // Is the tx in a block that's in the main chain @@ -526,8 +596,20 @@ int64 CTransaction::GetMinFee(unsigned int nBlockSize, bool fAllowFree, return nMinFee; } +void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins) +{ + LOCK(cs); + + std::map::iterator it = mapNextTx.lower_bound(COutPoint(hashTx, 0)); -bool CTxMemPool::accept(CTxDB& txdb, CTransaction &tx, bool fCheckInputs, + // iterate over all COutPoints in mapNextTx whose hash equals the provided hashTx + while (it != mapNextTx.end() && it->first.hash == hashTx) { + coins.Spend(it->first.n); // and remove those outputs from coins + it++; + } +} + +bool CTxMemPool::accept(CCoinsDB& coinsdb, CTransaction &tx, bool fCheckInputs, bool* pfMissingInputs) { if (pfMissingInputs) @@ -548,16 +630,13 @@ bool CTxMemPool::accept(CTxDB& txdb, CTransaction &tx, bool fCheckInputs, if (!fTestNet && !tx.IsStandard()) return error("CTxMemPool::accept() : nonstandard transaction type"); - // Do we already have it? + // is it already in the memory pool? uint256 hash = tx.GetHash(); { LOCK(cs); if (mapTx.count(hash)) return false; } - if (fCheckInputs) - if (txdb.ContainsTx(hash)) - return false; // Check for conflicts with in-memory transactions CTransaction* ptxOld = NULL; @@ -589,27 +668,32 @@ bool CTxMemPool::accept(CTxDB& txdb, CTransaction &tx, bool fCheckInputs, if (fCheckInputs) { - MapPrevTx mapInputs; - map mapUnused; - bool fInvalid = false; - if (!tx.FetchInputs(txdb, mapUnused, false, false, mapInputs, fInvalid)) - { - if (fInvalid) - return error("CTxMemPool::accept() : FetchInputs found invalid tx %s", hash.ToString().substr(0,10).c_str()); - if (pfMissingInputs) - *pfMissingInputs = true; + CCoinsViewDB viewDB(coinsdb); + CCoinsViewMemPool viewMemPool(viewDB, mempool); + CCoinsViewCache view(viewMemPool); + + // do we already have it? + if (view.HaveCoins(hash)) return false; + + // do all inputs exist? + BOOST_FOREACH(const CTxIn txin, tx.vin) { + if (!view.HaveCoins(txin.prevout.hash)) { + if (pfMissingInputs) + *pfMissingInputs = true; + return false; + } } // Check for non-standard pay-to-script-hash in inputs - if (!tx.AreInputsStandard(mapInputs) && !fTestNet) + if (!tx.AreInputsStandard(view) && !fTestNet) return error("CTxMemPool::accept() : nonstandard transaction input"); // Note: if you modify this code to accept non-standard transactions, then // you should add code here to check that the transaction does a // reasonable number of ECDSA signature verifications. - int64 nFees = tx.GetValueIn(mapInputs)-tx.GetValueOut(); + int64 nFees = tx.GetValueIn(view)-tx.GetValueOut(); unsigned int nSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION); // Don't accept it if it can't get into a block @@ -646,7 +730,7 @@ bool CTxMemPool::accept(CTxDB& txdb, CTransaction &tx, bool fCheckInputs, // Check against previous transactions // This is done last to help prevent CPU exhaustion denial-of-service attacks. - if (!tx.ConnectInputs(mapInputs, mapUnused, CDiskTxPos(true), pindexBest, false, false)) + if (!tx.CheckInputs(view, CS_ALWAYS, true, false)) { return error("CTxMemPool::accept() : ConnectInputs failed %s", hash.ToString().substr(0,10).c_str()); } @@ -674,9 +758,9 @@ bool CTxMemPool::accept(CTxDB& txdb, CTransaction &tx, bool fCheckInputs, return true; } -bool CTransaction::AcceptToMemoryPool(CTxDB& txdb, bool fCheckInputs, bool* pfMissingInputs) +bool CTransaction::AcceptToMemoryPool(CCoinsDB& coinsdb, bool fCheckInputs, bool* pfMissingInputs) { - return mempool.accept(txdb, *this, fCheckInputs, pfMissingInputs); + return mempool.accept(coinsdb, *this, fCheckInputs, pfMissingInputs); } bool CTxMemPool::addUnchecked(const uint256& hash, CTransaction &tx) @@ -765,29 +849,29 @@ int CMerkleTx::GetBlocksToMaturity() const } -bool CMerkleTx::AcceptToMemoryPool(CTxDB& txdb, bool fCheckInputs) +bool CMerkleTx::AcceptToMemoryPool(CCoinsDB& coinsdb, bool fCheckInputs) { if (fClient) { - if (!IsInMainChain() && !ClientConnectInputs()) + if (!IsInMainChain() && !ClientCheckInputs()) return false; - return CTransaction::AcceptToMemoryPool(txdb, false); + return CTransaction::AcceptToMemoryPool(coinsdb, false); } else { - return CTransaction::AcceptToMemoryPool(txdb, fCheckInputs); + return CTransaction::AcceptToMemoryPool(coinsdb, fCheckInputs); } } bool CMerkleTx::AcceptToMemoryPool() { - CTxDB txdb("r"); - return AcceptToMemoryPool(txdb); + CCoinsDB coinsdb("r"); + return AcceptToMemoryPool(coinsdb); } -bool CWalletTx::AcceptWalletTransaction(CTxDB& txdb, bool fCheckInputs) +bool CWalletTx::AcceptWalletTransaction(CCoinsDB& coinsdb, bool fCheckInputs) { { @@ -798,64 +882,65 @@ bool CWalletTx::AcceptWalletTransaction(CTxDB& txdb, bool fCheckInputs) if (!tx.IsCoinBase()) { uint256 hash = tx.GetHash(); - if (!mempool.exists(hash) && !txdb.ContainsTx(hash)) - tx.AcceptToMemoryPool(txdb, fCheckInputs); + if (!mempool.exists(hash) && !coinsdb.HaveCoins(hash)) + tx.AcceptToMemoryPool(coinsdb, fCheckInputs); } } - return AcceptToMemoryPool(txdb, fCheckInputs); + return AcceptToMemoryPool(coinsdb, fCheckInputs); } return false; } bool CWalletTx::AcceptWalletTransaction() { - CTxDB txdb("r"); - return AcceptWalletTransaction(txdb); -} - -int CTxIndex::GetDepthInMainChain() const -{ - // Read block header - CBlock block; - if (!block.ReadFromDisk(pos.blockPos, false)) - return 0; - // Find the block in the index - map::iterator mi = mapBlockIndex.find(block.GetHash()); - if (mi == mapBlockIndex.end()) - return 0; - CBlockIndex* pindex = (*mi).second; - if (!pindex || !pindex->IsInMainChain()) - return 0; - return 1 + nBestHeight - pindex->nHeight; + CCoinsDB coinsdb("r"); + return AcceptWalletTransaction(coinsdb); } // Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock -bool GetTransaction(const uint256 &hash, CTransaction &tx, uint256 &hashBlock) +bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock, bool fAllowSlow) { + CBlockIndex *pindexSlow = NULL; { LOCK(cs_main); { LOCK(mempool.cs); if (mempool.exists(hash)) { - tx = mempool.lookup(hash); + txOut = mempool.lookup(hash); return true; } } - CTxDB txdb("r"); - CTxIndex txindex; - if (tx.ReadFromDisk(txdb, COutPoint(hash, 0), txindex)) - { - CBlock block; - if (block.ReadFromDisk(txindex.pos.blockPos, false)) - hashBlock = block.GetHash(); - return true; + + if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it + int nHeight = -1; + { + CCoinsDB coindb("r"); + CCoinsViewDB view(coindb); + CCoins coins; + if (view.GetCoins(hash, coins)) + nHeight = coins.nHeight; + } + if (nHeight > 0) + pindexSlow = FindBlockByHeight(nHeight); } } - return false; -} + if (pindexSlow) { + CBlock block; + if (block.ReadFromDisk(pindexSlow)) { + BOOST_FOREACH(const CTransaction &tx, block.vtx) { + if (tx.GetHash() == hash) { + txOut = tx; + hashBlock = pindexSlow->GetBlockHash(); + return true; + } + } + } + } + return false; +} @@ -1051,7 +1136,7 @@ void static InvalidChainFound(CBlockIndex* pindexNew) if (pindexNew->bnChainWork > bnBestInvalidWork) { bnBestInvalidWork = pindexNew->bnChainWork; - CTxDB().WriteBestInvalidWork(bnBestInvalidWork); + CChainDB().WriteBestInvalidWork(bnBestInvalidWork); uiInterface.NotifyBlocksChanged(); } printf("InvalidChainFound: invalid block=%s height=%d work=%s date=%s\n", @@ -1084,145 +1169,35 @@ void CBlock::UpdateTime(const CBlockIndex* pindexPrev) -bool CTransaction::DisconnectInputs(CTxDB& txdb) -{ - // Relinquish previous transactions' spent pointers - if (!IsCoinBase()) - { - BOOST_FOREACH(const CTxIn& txin, vin) - { - COutPoint prevout = txin.prevout; - - // Get prev txindex from disk - CTxIndex txindex; - if (!txdb.ReadTxIndex(prevout.hash, txindex)) - return error("DisconnectInputs() : ReadTxIndex failed"); - - if (prevout.n >= txindex.vSpent.size()) - return error("DisconnectInputs() : prevout.n out of range"); - - // Mark outpoint as not spent - txindex.vSpent[prevout.n].SetNull(); - - // Write back - if (!txdb.UpdateTxIndex(prevout.hash, txindex)) - return error("DisconnectInputs() : UpdateTxIndex failed"); - } - } - - // Remove transaction from index - // This can fail if a duplicate of this transaction was in a chain that got - // reorganized away. This is only possible if this transaction was completely - // spent, so erasing it would be a no-op anyway. - txdb.EraseTxIndex(*this); - - return true; -} - - -bool CTransaction::FetchInputs(CTxDB& txdb, const map& mapTestPool, - bool fBlock, bool fMiner, MapPrevTx& inputsRet, bool& fInvalid) +CTxOut CTransaction::GetOutputFor(const CTxIn& input, CCoinsView& view) { - // FetchInputs can return false either because we just haven't seen some inputs - // (in which case the transaction should be stored as an orphan) - // or because the transaction is malformed (in which case the transaction should - // be dropped). If tx is definitely invalid, fInvalid will be set to true. - fInvalid = false; - - if (IsCoinBase()) - return true; // Coinbase transactions have no inputs to fetch. - - for (unsigned int i = 0; i < vin.size(); i++) - { - COutPoint prevout = vin[i].prevout; - if (inputsRet.count(prevout.hash)) - continue; // Got it already - - // Read txindex - CTxIndex& txindex = inputsRet[prevout.hash].first; - bool fFound = true; - if ((fBlock || fMiner) && mapTestPool.count(prevout.hash)) - { - // Get txindex from current proposed changes - txindex = mapTestPool.find(prevout.hash)->second; - } - else - { - // Read txindex from txdb - fFound = txdb.ReadTxIndex(prevout.hash, txindex); - } - if (!fFound && (fBlock || fMiner)) - return fMiner ? false : error("FetchInputs() : %s prev tx %s index entry not found", GetHash().ToString().substr(0,10).c_str(), prevout.hash.ToString().substr(0,10).c_str()); - - // Read txPrev - CTransaction& txPrev = inputsRet[prevout.hash].second; - if (!fFound || txindex.pos.IsMemPool()) - { - // Get prev tx from single transactions in memory - { - LOCK(mempool.cs); - if (!mempool.exists(prevout.hash)) - return error("FetchInputs() : %s mempool Tx prev not found %s", GetHash().ToString().substr(0,10).c_str(), prevout.hash.ToString().substr(0,10).c_str()); - txPrev = mempool.lookup(prevout.hash); - } - if (!fFound) - txindex.vSpent.resize(txPrev.vout.size()); - } - else - { - // Get prev tx from disk - if (!txPrev.ReadFromDisk(txindex.pos)) - return error("FetchInputs() : %s ReadFromDisk prev tx %s failed", GetHash().ToString().substr(0,10).c_str(), prevout.hash.ToString().substr(0,10).c_str()); - } - } - - // Make sure all prevout.n indexes are valid: - for (unsigned int i = 0; i < vin.size(); i++) - { - const COutPoint prevout = vin[i].prevout; - assert(inputsRet.count(prevout.hash) != 0); - const CTxIndex& txindex = inputsRet[prevout.hash].first; - const CTransaction& txPrev = inputsRet[prevout.hash].second; - if (prevout.n >= txPrev.vout.size() || prevout.n >= txindex.vSpent.size()) - { - // Revisit this if/when transaction replacement is implemented and allows - // adding inputs: - fInvalid = true; - return DoS(100, error("FetchInputs() : %s prevout.n out of range %d %"PRIszu" %"PRIszu" prev tx %s\n%s", GetHash().ToString().substr(0,10).c_str(), prevout.n, txPrev.vout.size(), txindex.vSpent.size(), prevout.hash.ToString().substr(0,10).c_str(), txPrev.ToString().c_str())); - } - } - - return true; -} - -const CTxOut& CTransaction::GetOutputFor(const CTxIn& input, const MapPrevTx& inputs) const -{ - MapPrevTx::const_iterator mi = inputs.find(input.prevout.hash); - if (mi == inputs.end()) + CCoins coins; + if (!view.GetCoins(input.prevout.hash, coins)) throw std::runtime_error("CTransaction::GetOutputFor() : prevout.hash not found"); - const CTransaction& txPrev = (mi->second).second; - if (input.prevout.n >= txPrev.vout.size()) - throw std::runtime_error("CTransaction::GetOutputFor() : prevout.n out of range"); + if (input.prevout.n >= coins.vout.size()) + throw std::runtime_error("CTransaction::GetOutputFor() : prevout.n out of range or already spent"); + + const CTxOut &out = coins.vout[input.prevout.n]; + if (out.IsNull()) + throw std::runtime_error("CTransaction::GetOutputFor() : already spent"); - return txPrev.vout[input.prevout.n]; + return out; } -int64 CTransaction::GetValueIn(const MapPrevTx& inputs) const +int64 CTransaction::GetValueIn(CCoinsView& inputs) const { if (IsCoinBase()) return 0; int64 nResult = 0; for (unsigned int i = 0; i < vin.size(); i++) - { nResult += GetOutputFor(vin[i], inputs).nValue; - } - return nResult; + return nResult; } -unsigned int CTransaction::GetP2SHSigOpCount(const MapPrevTx& inputs) const +unsigned int CTransaction::GetP2SHSigOpCount(CCoinsView& inputs) const { if (IsCoinBase()) return 0; @@ -1230,107 +1205,137 @@ unsigned int CTransaction::GetP2SHSigOpCount(const MapPrevTx& inputs) const unsigned int nSigOps = 0; for (unsigned int i = 0; i < vin.size(); i++) { - const CTxOut& prevout = GetOutputFor(vin[i], inputs); + CTxOut prevout = GetOutputFor(vin[i], inputs); if (prevout.scriptPubKey.IsPayToScriptHash()) nSigOps += prevout.scriptPubKey.GetSigOpCount(vin[i].scriptSig); } return nSigOps; } -bool CTransaction::ConnectInputs(MapPrevTx inputs, - map& mapTestPool, const CDiskTxPos& posThisTx, - const CBlockIndex* pindexBlock, bool fBlock, bool fMiner, bool fStrictPayToScriptHash) +bool CTransaction::UpdateCoins(CCoinsView &inputs, CTxUndo &txundo, int nHeight) const +{ + uint256 hash = GetHash(); + + // mark inputs spent + if (!IsCoinBase()) { + BOOST_FOREACH(const CTxIn &txin, vin) { + CCoins coins; + if (!inputs.GetCoins(txin.prevout.hash, coins)) + return error("UpdateCoins() : cannot find prevtx"); + CTxInUndo undo; + if (!coins.Spend(txin.prevout, undo)) + return error("UpdateCoins() : cannot spend input"); + txundo.vprevout.push_back(undo); + if (!inputs.SetCoins(txin.prevout.hash, coins)) + return error("UpdateCoins() : cannot update input"); + } + } + + // add outputs + if (!inputs.SetCoins(hash, CCoins(*this, nHeight))) + return error("UpdateCoins() : cannot update output"); + + return true; +} + +bool CTransaction::HaveInputs(CCoinsView &inputs) const +{ + if (!IsCoinBase()) { + // first check whether information about the prevout hash is available + for (unsigned int i = 0; i < vin.size(); i++) { + const COutPoint &prevout = vin[i].prevout; + if (!inputs.HaveCoins(prevout.hash)) + return false; + } + + // then check whether the actual outputs are available + for (unsigned int i = 0; i < vin.size(); i++) { + const COutPoint &prevout = vin[i].prevout; + CCoins coins; + inputs.GetCoins(prevout.hash, coins); + if (!coins.IsAvailable(prevout.n)) + return false; + } + } + return true; +} + +bool CTransaction::CheckInputs(CCoinsView &inputs, enum CheckSig_mode csmode, bool fStrictPayToScriptHash, bool fStrictEncodings) const { - // Take over previous transactions' spent pointers - // fBlock is true when this is called from AcceptBlock when a new best-block is added to the blockchain - // fMiner is true when called from the internal bitcoin miner - // ... both are false when called from CTransaction::AcceptToMemoryPool if (!IsCoinBase()) { int64 nValueIn = 0; int64 nFees = 0; for (unsigned int i = 0; i < vin.size(); i++) { - COutPoint prevout = vin[i].prevout; - assert(inputs.count(prevout.hash) > 0); - CTxIndex& txindex = inputs[prevout.hash].first; - CTransaction& txPrev = inputs[prevout.hash].second; + const COutPoint &prevout = vin[i].prevout; + CCoins coins; + if (!inputs.GetCoins(prevout.hash, coins)) + return error("CheckInputs() : cannot find prevout tx"); - if (prevout.n >= txPrev.vout.size() || prevout.n >= txindex.vSpent.size()) - return DoS(100, error("ConnectInputs() : %s prevout.n out of range %d %"PRIszu" %"PRIszu" prev tx %s\n%s", GetHash().ToString().substr(0,10).c_str(), prevout.n, txPrev.vout.size(), txindex.vSpent.size(), prevout.hash.ToString().substr(0,10).c_str(), txPrev.ToString().c_str())); + // Check for conflicts (double-spend) + // This doesn't trigger the DoS code on purpose; if it did, it would make it easier + // for an attacker to attempt to split the network. + if (!coins.IsAvailable(prevout.n)) + return error("CheckInputs() : %s prev tx already used", GetHash().ToString().substr(0,10).c_str()); // If prev is coinbase, check that it's matured - if (txPrev.IsCoinBase()) - for (const CBlockIndex* pindex = pindexBlock; pindex && pindexBlock->nHeight - pindex->nHeight < COINBASE_MATURITY; pindex = pindex->pprev) - if (pindex->GetBlockPos() == txindex.pos.blockPos) - return error("ConnectInputs() : tried to spend coinbase at depth %d", pindexBlock->nHeight - pindex->nHeight); + if (coins.IsCoinBase()) { + CBlockIndex *pindexBlock = inputs.GetBestBlock(); + if (pindexBlock->nHeight - coins.nHeight < COINBASE_MATURITY) + return error("CheckInputs() : tried to spend coinbase at depth %d", pindexBlock->nHeight - coins.nHeight); + } // Check for negative or overflow input values - nValueIn += txPrev.vout[prevout.n].nValue; - if (!MoneyRange(txPrev.vout[prevout.n].nValue) || !MoneyRange(nValueIn)) - return DoS(100, error("ConnectInputs() : txin values out of range")); + nValueIn += coins.vout[prevout.n].nValue; + if (!MoneyRange(coins.vout[prevout.n].nValue) || !MoneyRange(nValueIn)) + return DoS(100, error("CheckInputs() : txin values out of range")); } + + if (nValueIn < GetValueOut()) + return DoS(100, error("ChecktInputs() : %s value in < value out", GetHash().ToString().substr(0,10).c_str())); + + // Tally transaction fees + int64 nTxFee = nValueIn - GetValueOut(); + if (nTxFee < 0) + return DoS(100, error("CheckInputs() : %s nTxFee < 0", GetHash().ToString().substr(0,10).c_str())); + nFees += nTxFee; + if (!MoneyRange(nFees)) + return DoS(100, error("CheckInputs() : nFees out of range")); + // The first loop above does all the inexpensive checks. // Only if ALL inputs pass do we perform expensive ECDSA signature checks. // Helps prevent CPU exhaustion attacks. - for (unsigned int i = 0; i < vin.size(); i++) - { - COutPoint prevout = vin[i].prevout; - assert(inputs.count(prevout.hash) > 0); - CTxIndex& txindex = inputs[prevout.hash].first; - CTransaction& txPrev = inputs[prevout.hash].second; - // Check for conflicts (double-spend) - // This doesn't trigger the DoS code on purpose; if it did, it would make it easier - // for an attacker to attempt to split the network. - if (!txindex.vSpent[prevout.n].IsNull()) - return fMiner ? false : error("ConnectInputs() : %s prev tx already used at %s", GetHash().ToString().substr(0,10).c_str(), txindex.vSpent[prevout.n].ToString().c_str()); + // Skip ECDSA signature verification when connecting blocks + // before the last blockchain checkpoint. This is safe because block merkle hashes are + // still computed and checked, and any change will be caught at the next checkpoint. + if (csmode == CS_ALWAYS || + (csmode == CS_AFTER_CHECKPOINT && inputs.GetBestBlock()->nHeight >= Checkpoints::GetTotalBlocksEstimate())) { + for (unsigned int i = 0; i < vin.size(); i++) { + const COutPoint &prevout = vin[i].prevout; + CCoins coins; + inputs.GetCoins(prevout.hash, coins); - // Skip ECDSA signature verification when connecting blocks (fBlock=true) - // before the last blockchain checkpoint. This is safe because block merkle hashes are - // still computed and checked, and any change will be caught at the next checkpoint. - if (!(fBlock && (nBestHeight < Checkpoints::GetTotalBlocksEstimate()))) - { // Verify signature - if (!VerifySignature(txPrev, *this, i, fStrictPayToScriptHash, false, 0)) - { + if (!VerifySignature(coins, *this, i, fStrictPayToScriptHash, fStrictEncodings, 0)) { // only during transition phase for P2SH: do not invoke anti-DoS code for // potentially old clients relaying bad P2SH transactions - if (fStrictPayToScriptHash && VerifySignature(txPrev, *this, i, false, false, 0)) - return error("ConnectInputs() : %s P2SH VerifySignature failed", GetHash().ToString().substr(0,10).c_str()); + if (fStrictPayToScriptHash && VerifySignature(coins, *this, i, false, fStrictEncodings, 0)) + return error("CheckInputs() : %s P2SH VerifySignature failed", GetHash().ToString().substr(0,10).c_str()); - return DoS(100,error("ConnectInputs() : %s VerifySignature failed", GetHash().ToString().substr(0,10).c_str())); + return DoS(100,error("CheckInputs() : %s VerifySignature failed", GetHash().ToString().substr(0,10).c_str())); } } - - // Mark outpoints as spent - txindex.vSpent[prevout.n] = posThisTx; - - // Write back - if (fBlock || fMiner) - { - mapTestPool[prevout.hash] = txindex; - } } - - if (nValueIn < GetValueOut()) - return DoS(100, error("ConnectInputs() : %s value in < value out", GetHash().ToString().substr(0,10).c_str())); - - // Tally transaction fees - int64 nTxFee = nValueIn - GetValueOut(); - if (nTxFee < 0) - return DoS(100, error("ConnectInputs() : %s nTxFee < 0", GetHash().ToString().substr(0,10).c_str())); - nFees += nTxFee; - if (!MoneyRange(nFees)) - return DoS(100, error("ConnectInputs() : nFees out of range")); } return true; } -bool CTransaction::ClientConnectInputs() +bool CTransaction::ClientCheckInputs() const { if (IsCoinBase()) return false; @@ -1351,7 +1356,7 @@ bool CTransaction::ClientConnectInputs() return false; // Verify signature - if (!VerifySignature(txPrev, *this, i, true, false, 0)) + if (!VerifySignature(CCoins(txPrev, -1), *this, i, true, false, 0)) return error("ConnectInputs() : VerifySignature failed"); ///// this is redundant with the mempool.mapNextTx stuff, @@ -1379,34 +1384,89 @@ bool CTransaction::ClientConnectInputs() -bool CBlock::DisconnectBlock(CTxDB& txdb, CBlockIndex* pindex) +bool CBlock::DisconnectBlock(CBlockIndex *pindex, CCoinsView &view) { - // Disconnect in reverse order - for (int i = vtx.size()-1; i >= 0; i--) - if (!vtx[i].DisconnectInputs(txdb)) - return false; + assert(pindex == view.GetBestBlock()); - // Update block index on disk without changing it in memory. - // The memory index structure will be changed after the db commits. - if (pindex->pprev) + CBlockUndo blockUndo; { - CDiskBlockIndex blockindexPrev(pindex->pprev); - blockindexPrev.hashNext = 0; - if (!txdb.WriteBlockIndex(blockindexPrev)) - return error("DisconnectBlock() : WriteBlockIndex failed"); + CDiskBlockPos pos = pindex->GetUndoPos(); + if (pos.IsNull()) + return error("DisconnectBlock() : no undo data available"); + FILE *file = OpenUndoFile(pos, true); + if (file == NULL) + return error("DisconnectBlock() : undo file not available"); + CAutoFile fileUndo(file, SER_DISK, CLIENT_VERSION); + fileUndo >> blockUndo; } + assert(blockUndo.vtxundo.size() + 1 == vtx.size()); + + // undo transactions in reverse order + for (int i = vtx.size() - 1; i >= 0; i--) { + const CTransaction &tx = vtx[i]; + uint256 hash = tx.GetHash(); + + // check that all outputs are available + CCoins outs; + if (!view.GetCoins(hash, outs)) + return error("DisconnectBlock() : outputs still spent? database corrupted"); + + CCoins outsBlock = CCoins(tx, pindex->nHeight); + if (outs != outsBlock) + return error("DisconnectBlock() : added transaction mismatch? database corrupted"); + + // remove outputs + if (!view.SetCoins(hash, CCoins())) + return error("DisconnectBlock() : cannot delete coin outputs"); + + // restore inputs + if (i > 0) { // not coinbases + const CTxUndo &txundo = blockUndo.vtxundo[i-1]; + assert(txundo.vprevout.size() == tx.vin.size()); + for (unsigned int j = tx.vin.size(); j-- > 0;) { + const COutPoint &out = tx.vin[j].prevout; + const CTxInUndo &undo = txundo.vprevout[j]; + CCoins coins; + view.GetCoins(out.hash, coins); // this can fail if the prevout was already entirely spent + if (coins.IsPruned()) { + if (undo.nHeight == 0) + return error("DisconnectBlock() : undo data doesn't contain tx metadata? database corrupted"); + coins.fCoinBase = undo.fCoinBase; + coins.nHeight = undo.nHeight; + coins.nVersion = undo.nVersion; + } else { + if (undo.nHeight != 0) + return error("DisconnectBlock() : undo data contains unneeded tx metadata? database corrupted"); + } + if (coins.IsAvailable(out.n)) + return error("DisconnectBlock() : prevout output not spent? database corrupted"); + if (coins.vout.size() < out.n+1) + coins.vout.resize(out.n+1); + coins.vout[out.n] = undo.txout; + if (!view.SetCoins(out.hash, coins)) + return error("DisconnectBlock() : cannot restore coin inputs"); + } + } + } + + // move best block pointer to prevout block + view.SetBestBlock(pindex->pprev); + return true; } -bool FindUndoPos(CTxDB &txdb, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); +bool FindUndoPos(CChainDB &chaindb, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); -bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) +bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsView &view, bool fJustCheck) { // Check it again in case a previous version let a bad block in if (!CheckBlock(!fJustCheck, !fJustCheck)) return false; + // verify that the view's current state corresponds to the previous block + assert(pindex->pprev == view.GetBestBlock()); + // Do not allow blocks that contain transactions which 'overwrite' older transactions, // unless those are already completely spent. // If such overwrites are allowed, coinbases and transactions depending upon those @@ -1421,115 +1481,81 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) // initial block download. bool fEnforceBIP30 = !((pindex->nHeight==91842 && pindex->GetBlockHash() == uint256("0x00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec")) || (pindex->nHeight==91880 && pindex->GetBlockHash() == uint256("0x00000000000743f190a18c5577a3c2d2a1f610ae9601ac046a38084ccb7cd721"))); + if (fEnforceBIP30) { + BOOST_FOREACH(CTransaction& tx, vtx) { + uint256 hash = tx.GetHash(); + CCoins coins; + if (view.GetCoins(hash, coins) && !coins.IsPruned()) + return error("ConnectBlock() : tried to overwrite transaction"); + } + } // BIP16 didn't become active until Apr 1 2012 int64 nBIP16SwitchTime = 1333238400; bool fStrictPayToScriptHash = (pindex->nTime >= nBIP16SwitchTime); - //// issue here: it doesn't know the version - unsigned int nTxPos; - if (fJustCheck) - // Since we're just checking the block and not actually connecting it, it might not (and probably shouldn't) be on the disk to get the transaction from - nTxPos = 1; - else - nTxPos = pindex->pos.nPos + ::GetSerializeSize(CBlock(), SER_DISK, CLIENT_VERSION) - 1 + GetSizeOfCompactSize(vtx.size()); - CBlockUndo blockundo; - map mapQueuedChanges; int64 nFees = 0; unsigned int nSigOps = 0; BOOST_FOREACH(CTransaction& tx, vtx) { - uint256 hashTx = tx.GetHash(); - - if (fEnforceBIP30) { - CTxIndex txindexOld; - if (txdb.ReadTxIndex(hashTx, txindexOld)) { - BOOST_FOREACH(CDiskTxPos &pos, txindexOld.vSpent) - if (pos.IsNull()) - return false; - } - } - nSigOps += tx.GetLegacySigOpCount(); if (nSigOps > MAX_BLOCK_SIGOPS) return DoS(100, error("ConnectBlock() : too many sigops")); - CDiskTxPos posThisTx(pindex->GetBlockPos(), nTxPos); - if (!fJustCheck) - nTxPos += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION); - else - posThisTx = CDiskTxPos(true); - - MapPrevTx mapInputs; if (!tx.IsCoinBase()) { - CTxUndo undo; - - bool fInvalid; - if (!tx.FetchInputs(txdb, mapQueuedChanges, true, false, mapInputs, fInvalid)) - return false; + if (!tx.HaveInputs(view)) + return DoS(100, error("ConnectBlock() : inputs missing/spent")); if (fStrictPayToScriptHash) { // Add in sigops done by pay-to-script-hash inputs; // this is to prevent a "rogue miner" from creating // an incredibly-expensive-to-validate block. - nSigOps += tx.GetP2SHSigOpCount(mapInputs); + nSigOps += tx.GetP2SHSigOpCount(view); if (nSigOps > MAX_BLOCK_SIGOPS) - return DoS(100, error("ConnectBlock() : too many sigops")); + return DoS(100, error("ConnectBlock() : too many sigops")); } - nFees += tx.GetValueIn(mapInputs)-tx.GetValueOut(); + nFees += tx.GetValueIn(view)-tx.GetValueOut(); - BOOST_FOREACH(const CTxIn &in, tx.vin) { - undo.vprevout.push_back(CTxInUndo(mapInputs[in.prevout.hash].second.vout[in.prevout.n], pindex->nHeight)); - } - - if (!tx.ConnectInputs(mapInputs, mapQueuedChanges, posThisTx, pindex, true, false, fStrictPayToScriptHash)) + if (!tx.CheckInputs(view, CS_AFTER_CHECKPOINT, fStrictPayToScriptHash, false)) return false; - - blockundo.vtxundo.push_back(undo); } - mapQueuedChanges[hashTx] = CTxIndex(posThisTx, tx.vout.size()); + CTxUndo txundo; + if (!tx.UpdateCoins(view, txundo, pindex->nHeight)) + return error("ConnectBlock() : UpdateInputs failed"); + if (!tx.IsCoinBase()) + blockundo.vtxundo.push_back(txundo); } - if (vtx[0].GetValueOut() > GetBlockValue(pindex->nHeight, nFees)) - return false; - if (fJustCheck) return true; - // Write queued txindex changes - for (map::iterator mi = mapQueuedChanges.begin(); mi != mapQueuedChanges.end(); ++mi) - { - if (!txdb.UpdateTxIndex((*mi).first, (*mi).second)) - return error("ConnectBlock() : UpdateTxIndex failed"); - } - // Write undo information to disk - if (pindex->GetUndoPos().IsNull() && pindex->nHeight > Checkpoints::GetTotalBlocksEstimate()) + if (pindex->GetUndoPos().IsNull()) { + CChainDB chaindb; CDiskBlockPos pos; - if (!FindUndoPos(txdb, pindex->pos.nFile, pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 8)) + if (!FindUndoPos(chaindb, pindex->pos.nFile, pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 8)) return error("ConnectBlock() : FindUndoPos failed"); if (!blockundo.WriteToDisk(pos)) return error("ConnectBlock() : CBlockUndo::WriteToDisk failed"); - pindex->nUndoPos = pos.nPos; - } - // Update block index on disk without changing it in memory. - // The memory index structure will be changed after the db commits. - if (pindex->pprev) - { - CDiskBlockIndex blockindexPrev(pindex->pprev); - blockindexPrev.hashNext = pindex->GetBlockHash(); - if (!txdb.WriteBlockIndex(blockindexPrev)) + // update nUndoPos in block index + pindex->nUndoPos = pos.nPos + 1; + CDiskBlockIndex blockindex(pindex); + if (!chaindb.WriteBlockIndex(blockindex)) return error("ConnectBlock() : WriteBlockIndex failed"); } + // add this block to the view's blockchain + if (!view.SetBestBlock(pindex)) + return false; + // Watch for transactions paying to me BOOST_FOREACH(CTransaction& tx, vtx) SyncWithWallets(tx, this, true); @@ -1537,47 +1563,70 @@ bool CBlock::ConnectBlock(CTxDB& txdb, CBlockIndex* pindex, bool fJustCheck) return true; } -bool static Reorganize(CTxDB& txdb, CBlockIndex* pindexNew) +bool CBlock::SetBestChain(CBlockIndex* pindexNew) { - printf("REORGANIZE\n"); + // if this functions exits prematurely, the transaction is aborted + CCoinsDB coinsdb; + if (!coinsdb.TxnBegin()) + return error("SetBestChain() : TxnBegin failed"); + + // special case for attaching the genesis block + // note that no ConnectBlock is called, so its coinbase output is non-spendable + if (pindexGenesisBlock == NULL && pindexNew->GetBlockHash() == hashGenesisBlock) + { + coinsdb.WriteHashBestChain(pindexNew->GetBlockHash()); + if (!coinsdb.TxnCommit()) + return error("SetBestChain() : TxnCommit failed"); + pindexGenesisBlock = pindexNew; + pindexBest = pindexNew; + hashBestChain = pindexNew->GetBlockHash(); + nBestHeight = pindexBest->nHeight; + bnBestChainWork = pindexNew->bnChainWork; + return true; + } - // Find the fork - CBlockIndex* pfork = pindexBest; + // create cached view to the coins database + CCoinsViewDB viewDB(coinsdb); + CCoinsViewCache view(viewDB); + + // Find the fork (typically, there is none) + CBlockIndex* pfork = view.GetBestBlock(); CBlockIndex* plonger = pindexNew; while (pfork != plonger) { while (plonger->nHeight > pfork->nHeight) if (!(plonger = plonger->pprev)) - return error("Reorganize() : plonger->pprev is null"); + return error("SetBestChain() : plonger->pprev is null"); if (pfork == plonger) break; if (!(pfork = pfork->pprev)) - return error("Reorganize() : pfork->pprev is null"); + return error("SetBestChain() : pfork->pprev is null"); } - // List of what to disconnect + // List of what to disconnect (typically nothing) vector vDisconnect; - for (CBlockIndex* pindex = pindexBest; pindex != pfork; pindex = pindex->pprev) + for (CBlockIndex* pindex = view.GetBestBlock(); pindex != pfork; pindex = pindex->pprev) vDisconnect.push_back(pindex); - // List of what to connect + // List of what to connect (typically only pindexNew) vector vConnect; for (CBlockIndex* pindex = pindexNew; pindex != pfork; pindex = pindex->pprev) vConnect.push_back(pindex); reverse(vConnect.begin(), vConnect.end()); - printf("REORGANIZE: Disconnect %"PRIszu" blocks; %s..%s\n", vDisconnect.size(), pfork->GetBlockHash().ToString().substr(0,20).c_str(), pindexBest->GetBlockHash().ToString().substr(0,20).c_str()); - printf("REORGANIZE: Connect %"PRIszu" blocks; %s..%s\n", vConnect.size(), pfork->GetBlockHash().ToString().substr(0,20).c_str(), pindexNew->GetBlockHash().ToString().substr(0,20).c_str()); + if (vDisconnect.size() > 0) { + printf("REORGANIZE: Disconnect %"PRIszu" blocks; %s..%s\n", vDisconnect.size(), pfork->GetBlockHash().ToString().substr(0,20).c_str(), pindexBest->GetBlockHash().ToString().substr(0,20).c_str()); + printf("REORGANIZE: Connect %"PRIszu" blocks; %s..%s\n", vConnect.size(), pfork->GetBlockHash().ToString().substr(0,20).c_str(), pindexNew->GetBlockHash().ToString().substr(0,20).c_str()); + } // Disconnect shorter branch vector vResurrect; - BOOST_FOREACH(CBlockIndex* pindex, vDisconnect) - { + BOOST_FOREACH(CBlockIndex* pindex, vDisconnect) { CBlock block; if (!block.ReadFromDisk(pindex)) - return error("Reorganize() : ReadFromDisk for disconnect failed"); - if (!block.DisconnectBlock(txdb, pindex)) - return error("Reorganize() : DisconnectBlock %s failed", pindex->GetBlockHash().ToString().substr(0,20).c_str()); + return error("SetBestBlock() : ReadFromDisk for disconnect failed"); + if (!block.DisconnectBlock(pindex, view)) + return error("SetBestBlock() : DisconnectBlock %s failed", pindex->GetBlockHash().ToString().substr(0,20).c_str()); // Queue memory transactions to resurrect BOOST_FOREACH(const CTransaction& tx, block.vtx) @@ -1587,28 +1636,35 @@ bool static Reorganize(CTxDB& txdb, CBlockIndex* pindexNew) // Connect longer branch vector vDelete; - for (unsigned int i = 0; i < vConnect.size(); i++) - { - CBlockIndex* pindex = vConnect[i]; + BOOST_FOREACH(CBlockIndex *pindex, vConnect) { CBlock block; - if (!block.ReadFromDisk(pindex)) - return error("Reorganize() : ReadFromDisk for connect failed"); - if (!block.ConnectBlock(txdb, pindex)) - { - // Invalid block - return error("Reorganize() : ConnectBlock %s failed", pindex->GetBlockHash().ToString().substr(0,20).c_str()); + CBlock *pblock; + if (pindex == pindexNew) // connecting *this block + pblock = this; + else { // other block; read it from disk + if (!block.ReadFromDisk(pindex)) + return error("SetBestBlock() : ReadFromDisk for connect failed"); + pblock = █ + } + if (!pblock->ConnectBlock(pindex, view)) { + InvalidChainFound(pindexNew); + return error("SetBestBlock() : ConnectBlock %s failed", pindex->GetBlockHash().ToString().substr(0,20).c_str()); } // Queue memory transactions to delete - BOOST_FOREACH(const CTransaction& tx, block.vtx) + BOOST_FOREACH(const CTransaction& tx, pblock->vtx) vDelete.push_back(tx); } - if (!txdb.WriteHashBestChain(pindexNew->GetBlockHash())) - return error("Reorganize() : WriteHashBestChain failed"); // Make sure it's successfully written to disk before changing memory structure - if (!txdb.TxnCommit()) - return error("Reorganize() : TxnCommit failed"); + if (!view.Flush()) + return error("SetBestBlock() : failed to write coin changes"); + if (!coinsdb.TxnCommit()) + return error("SetBestBlock() : TxnCommit failed"); + coinsdb.Close(); + + // At this point, all changes have been done to the database. + // Proceed by updating the memory structures. // Disconnect shorter branch BOOST_FOREACH(CBlockIndex* pindex, vDisconnect) @@ -1622,108 +1678,12 @@ bool static Reorganize(CTxDB& txdb, CBlockIndex* pindexNew) // Resurrect memory transactions that were in the disconnected branch BOOST_FOREACH(CTransaction& tx, vResurrect) - tx.AcceptToMemoryPool(txdb, false); + tx.AcceptToMemoryPool(coinsdb, false); // Delete redundant memory transactions that are in the connected branch BOOST_FOREACH(CTransaction& tx, vDelete) mempool.remove(tx); - printf("REORGANIZE: done\n"); - - return true; -} - - -// Called from inside SetBestChain: attaches a block to the new best chain being built -bool CBlock::SetBestChainInner(CTxDB& txdb, CBlockIndex *pindexNew) -{ - uint256 hash = GetHash(); - - // Adding to current best branch - if (!ConnectBlock(txdb, pindexNew) || !txdb.WriteHashBestChain(hash)) - { - txdb.TxnAbort(); - InvalidChainFound(pindexNew); - return false; - } - if (!txdb.TxnCommit()) - return error("SetBestChain() : TxnCommit failed"); - - // Add to current best branch - pindexNew->pprev->pnext = pindexNew; - - // Delete redundant memory transactions - BOOST_FOREACH(CTransaction& tx, vtx) - mempool.remove(tx); - - return true; -} - -bool CBlock::SetBestChain(CTxDB& txdb, CBlockIndex* pindexNew) -{ - uint256 hash = GetHash(); - - if (!txdb.TxnBegin()) - return error("SetBestChain() : TxnBegin failed"); - - if (pindexGenesisBlock == NULL && hash == hashGenesisBlock) - { - txdb.WriteHashBestChain(hash); - if (!txdb.TxnCommit()) - return error("SetBestChain() : TxnCommit failed"); - pindexGenesisBlock = pindexNew; - } - else if (hashPrevBlock == hashBestChain) - { - if (!SetBestChainInner(txdb, pindexNew)) - return error("SetBestChain() : SetBestChainInner failed"); - } - else - { - // the first block in the new chain that will cause it to become the new best chain - CBlockIndex *pindexIntermediate = pindexNew; - - // list of blocks that need to be connected afterwards - std::vector vpindexSecondary; - - // Reorganize is costly in terms of db load, as it works in a single db transaction. - // Try to limit how much needs to be done inside - while (pindexIntermediate->pprev && pindexIntermediate->pprev->bnChainWork > pindexBest->bnChainWork) - { - vpindexSecondary.push_back(pindexIntermediate); - pindexIntermediate = pindexIntermediate->pprev; - } - - if (!vpindexSecondary.empty()) - printf("Postponing %"PRIszu" reconnects\n", vpindexSecondary.size()); - - // Switch to new best branch - if (!Reorganize(txdb, pindexIntermediate)) - { - txdb.TxnAbort(); - InvalidChainFound(pindexNew); - return error("SetBestChain() : Reorganize failed"); - } - - // Connect further blocks - BOOST_REVERSE_FOREACH(CBlockIndex *pindex, vpindexSecondary) - { - CBlock block; - if (!block.ReadFromDisk(pindex)) - { - printf("SetBestChain() : ReadFromDisk failed\n"); - break; - } - if (!txdb.TxnBegin()) { - printf("SetBestChain() : TxnBegin 2 failed\n"); - break; - } - // errors now are not fatal, we still did a reorganisation to a new chain in a valid way - if (!block.SetBestChainInner(txdb, pindex)) - break; - } - } - // Update best block in wallet (so we can detect restored wallets) bool fIsInitialDownload = IsInitialBlockDownload(); if (!fIsInitialDownload) @@ -1733,7 +1693,7 @@ bool CBlock::SetBestChain(CTxDB& txdb, CBlockIndex* pindexNew) } // New best block - hashBestChain = hash; + hashBestChain = pindexNew->GetBlockHash(); pindexBest = pindexNew; pblockindexFBBHLast = NULL; nBestHeight = pindexBest->nHeight; @@ -1797,19 +1757,19 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) pindexNew->pos = pos; pindexNew->nUndoPos = 0; - CTxDB txdb; - if (!txdb.TxnBegin()) + CChainDB chaindb; + if (!chaindb.TxnBegin()) return false; - txdb.WriteBlockIndex(CDiskBlockIndex(pindexNew)); - if (!txdb.TxnCommit()) + chaindb.WriteBlockIndex(CDiskBlockIndex(pindexNew)); + if (!chaindb.TxnCommit()) return false; // New best - if (pindexNew->bnChainWork > bnBestChainWork) - if (!SetBestChain(txdb, pindexNew)) - return false; - - txdb.Close(); + if (pindexNew->bnChainWork > bnBestChainWork) { + if (!IsInitialBlockDownload() || (pindexNew->nHeight % 1) == 0) + if (!SetBestChain(pindexNew)) + return false; + } if (pindexNew == pindexBest) { @@ -1825,7 +1785,7 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) -bool FindBlockPos(CTxDB &txdb, CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64 nTime) +bool FindBlockPos(CChainDB &chaindb, CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64 nTime) { bool fUpdatedLast = false; @@ -1833,9 +1793,15 @@ bool FindBlockPos(CTxDB &txdb, CDiskBlockPos &pos, unsigned int nAddSize, unsign while (infoLastBlockFile.nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { printf("Leaving block file %i: %s\n", nLastBlockFile, infoLastBlockFile.ToString().c_str()); + FILE *file = OpenBlockFile(pos); + FileCommit(file); + fclose(file); + file = OpenUndoFile(pos); + FileCommit(file); + fclose(file); nLastBlockFile++; infoLastBlockFile.SetNull(); - txdb.ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine + chaindb.ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine fUpdatedLast = true; } @@ -1855,15 +1821,15 @@ bool FindBlockPos(CTxDB &txdb, CDiskBlockPos &pos, unsigned int nAddSize, unsign fclose(file); } - if (!txdb.WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + if (!chaindb.WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) return error("FindBlockPos() : cannot write updated block info"); if (fUpdatedLast) - txdb.WriteLastBlockFile(nLastBlockFile); + chaindb.WriteLastBlockFile(nLastBlockFile); return true; } -bool FindUndoPos(CTxDB &txdb, int nFile, CDiskBlockPos &pos, unsigned int nAddSize) +bool FindUndoPos(CChainDB &chaindb, int nFile, CDiskBlockPos &pos, unsigned int nAddSize) { pos.nFile = nFile; @@ -1873,15 +1839,15 @@ bool FindUndoPos(CTxDB &txdb, int nFile, CDiskBlockPos &pos, unsigned int nAddSi if (nFile == nLastBlockFile) { pos.nPos = infoLastBlockFile.nUndoSize; nNewSize = (infoLastBlockFile.nUndoSize += nAddSize); - if (!txdb.WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + if (!chaindb.WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) return error("FindUndoPos() : cannot write updated block info"); } else { CBlockFileInfo info; - if (!txdb.ReadBlockFileInfo(nFile, info)) + if (!chaindb.ReadBlockFileInfo(nFile, info)) return error("FindUndoPos() : cannot read block info"); pos.nPos = info.nUndoSize; nNewSize = (info.nUndoSize += nAddSize); - if (!txdb.WriteBlockFileInfo(nFile, info)) + if (!chaindb.WriteBlockFileInfo(nFile, info)) return error("FindUndoPos() : cannot write updated block info"); } @@ -2013,8 +1979,8 @@ bool CBlock::AcceptBlock() return error("AcceptBlock() : out of disk space"); CDiskBlockPos blockPos; { - CTxDB txdb; - if (!FindBlockPos(txdb, blockPos, nBlockSize+8, nHeight, nTime)) + CChainDB chaindb; + if (!FindBlockPos(chaindb, blockPos, nBlockSize+8, nHeight, nTime)) return error("AcceptBlock() : FindBlockPos failed"); } if (!WriteToDisk(blockPos)) @@ -2159,15 +2125,17 @@ int nLastBlockFile = 0; FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) { - if (pos.IsNull() || pos.IsMemPool()) + if (pos.IsNull()) return NULL; boost::filesystem::path path = GetDataDir() / "blocks" / strprintf("%s%05u.dat", prefix, pos.nFile); boost::filesystem::create_directories(path.parent_path()); FILE* file = fopen(path.string().c_str(), "rb+"); if (!file && !fReadOnly) file = fopen(path.string().c_str(), "wb+"); - if (!file) + if (!file) { + printf("Unable to open file %s\n", path.string().c_str()); return NULL; + } if (pos.nPos) { if (fseek(file, pos.nPos, SEEK_SET)) { printf("Unable to seek to position %u of %s\n", pos.nPos, path.string().c_str()); @@ -2200,10 +2168,12 @@ bool LoadBlockIndex(bool fAllowNew) // // Load block index // - CTxDB txdb("cr"); - if (!txdb.LoadBlockIndex()) + CChainDB chaindb("cr"); + CCoinsDB coinsdb("cr"); + if (!LoadBlockIndex(coinsdb, chaindb)) return false; - txdb.Close(); + chaindb.Close(); + coinsdb.Close(); // // Init with genesis block @@ -2256,8 +2226,8 @@ bool LoadBlockIndex(bool fAllowNew) unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); CDiskBlockPos blockPos; { - CTxDB txdb; - if (!FindBlockPos(txdb, blockPos, nBlockSize+8, 0, block.nTime)) + CChainDB chaindb; + if (!FindBlockPos(chaindb, blockPos, nBlockSize+8, 0, block.nTime)) return error("AcceptBlock() : FindBlockPos failed"); } if (!block.WriteToDisk(blockPos)) @@ -2316,7 +2286,7 @@ void PrintBlockTree() // print item CBlock block; block.ReadFromDisk(pindex); - printf("%d (blk%05u.dat:0x%lx) %s tx %"PRIszu"", + printf("%d (blk%05u.dat:0x%x) %s tx %"PRIszu"", pindex->nHeight, pindex->GetBlockPos().nFile, pindex->GetBlockPos().nPos, DateTimeStrFormat("%x %H:%M:%S", block.GetBlockTime()).c_str(), @@ -2522,22 +2492,20 @@ string GetWarnings(string strFor) // -bool static AlreadyHave(CTxDB& txdb, const CInv& inv) +bool static AlreadyHave(CCoinsDB &coinsdb, const CInv& inv) { switch (inv.type) { case MSG_TX: { - bool txInMap = false; + bool txInMap = false; { - LOCK(mempool.cs); - txInMap = (mempool.exists(inv.hash)); + LOCK(mempool.cs); + txInMap = mempool.exists(inv.hash); } - return txInMap || - mapOrphanTransactions.count(inv.hash) || - txdb.ContainsTx(inv.hash); + return txInMap || mapOrphanTransactions.count(inv.hash) || + coinsdb.HaveCoins(inv.hash); } - case MSG_BLOCK: return mapBlockIndex.count(inv.hash) || mapOrphanBlocks.count(inv.hash); @@ -2780,7 +2748,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) break; } } - CTxDB txdb("r"); + CCoinsDB coinsdb("r"); for (unsigned int nInv = 0; nInv < vInv.size(); nInv++) { const CInv &inv = vInv[nInv]; @@ -2789,7 +2757,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) return true; pfrom->AddInventoryKnown(inv); - bool fAlreadyHave = AlreadyHave(txdb, inv); + bool fAlreadyHave = AlreadyHave(coinsdb, inv); if (fDebug) printf(" got inventory: %s %s\n", inv.ToString().c_str(), fAlreadyHave ? "have" : "new"); @@ -2961,7 +2929,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) vector vWorkQueue; vector vEraseQueue; CDataStream vMsg(vRecv); - CTxDB txdb("r"); + CCoinsDB coinsdb("r"); CTransaction tx; vRecv >> tx; @@ -2969,7 +2937,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) pfrom->AddInventoryKnown(inv); bool fMissingInputs = false; - if (tx.AcceptToMemoryPool(txdb, true, &fMissingInputs)) + if (tx.AcceptToMemoryPool(coinsdb, true, &fMissingInputs)) { SyncWithWallets(tx, NULL, true); RelayMessage(inv, vMsg); @@ -2991,7 +2959,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) CInv inv(MSG_TX, tx.GetHash()); bool fMissingInputs2 = false; - if (tx.AcceptToMemoryPool(txdb, true, &fMissingInputs2)) + if (tx.AcceptToMemoryPool(coinsdb, true, &fMissingInputs2)) { printf(" accepted orphan tx %s\n", inv.hash.ToString().substr(0,10).c_str()); SyncWithWallets(tx, NULL, true); @@ -3439,11 +3407,11 @@ bool SendMessages(CNode* pto, bool fSendTrickle) // vector vGetData; int64 nNow = GetTime() * 1000000; - CTxDB txdb("r"); + CCoinsDB coinsdb("r"); while (!pto->mapAskFor.empty() && (*pto->mapAskFor.begin()).first <= nNow) { const CInv& inv = (*pto->mapAskFor.begin()).second; - if (!AlreadyHave(txdb, inv)) + if (!AlreadyHave(coinsdb, inv)) { if (fDebugNet) printf("sending getdata: %s\n", inv.ToString().c_str()); @@ -3653,7 +3621,9 @@ CBlock* CreateNewBlock(CReserveKey& reservekey) int64 nFees = 0; { LOCK2(cs_main, mempool.cs); - CTxDB txdb("r"); + CCoinsDB coinsdb("r"); + CCoinsViewDB viewdb(coinsdb); + CCoinsViewCache view(viewdb); // Priority order to process transactions list vOrphan; // list memory doesn't move @@ -3675,9 +3645,8 @@ CBlock* CreateNewBlock(CReserveKey& reservekey) BOOST_FOREACH(const CTxIn& txin, tx.vin) { // Read prev transaction - CTransaction txPrev; - CTxIndex txindex; - if (!txPrev.ReadFromDisk(txdb, txin.prevout, txindex)) + CCoins coins; + if (!view.GetCoins(txin.prevout.hash, coins)) { // This should never happen; all transactions in the memory // pool should connect to either transactions in the chain @@ -3704,10 +3673,12 @@ CBlock* CreateNewBlock(CReserveKey& reservekey) nTotalIn += mempool.mapTx[txin.prevout.hash].vout[txin.prevout.n].nValue; continue; } - int64 nValueIn = txPrev.vout[txin.prevout.n].nValue; + + int64 nValueIn = coins.vout[txin.prevout.n].nValue; nTotalIn += nValueIn; - int nConf = txindex.GetDepthInMainChain(); + int nConf = pindexPrev->nHeight - coins.nHeight; + dPriority += (double)nValueIn * nConf; } if (fMissingInputs) continue; @@ -3731,7 +3702,6 @@ CBlock* CreateNewBlock(CReserveKey& reservekey) } // Collect transactions into block - map mapTestPool; uint64 nBlockSize = 1000; uint64 nBlockTx = 0; int nBlockSigOps = 100; @@ -3750,6 +3720,9 @@ CBlock* CreateNewBlock(CReserveKey& reservekey) std::pop_heap(vecPriority.begin(), vecPriority.end(), comparer); vecPriority.pop_back(); + // second layer cached modifications just for this transaction + CCoinsViewCache viewTemp(view, true); + // Size limits unsigned int nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION); if (nBlockSize + nTxSize >= nBlockMaxSize) @@ -3774,24 +3747,21 @@ CBlock* CreateNewBlock(CReserveKey& reservekey) std::make_heap(vecPriority.begin(), vecPriority.end(), comparer); } - // Connecting shouldn't fail due to dependency on other memory pool transactions - // because we're already processing them in order of dependency - map mapTestPoolTmp(mapTestPool); - MapPrevTx mapInputs; - bool fInvalid; - if (!tx.FetchInputs(txdb, mapTestPoolTmp, false, true, mapInputs, fInvalid)) + if (!tx.CheckInputs(viewTemp, CS_ALWAYS, true, false)) continue; - int64 nTxFees = tx.GetValueIn(mapInputs)-tx.GetValueOut(); + int64 nTxFees = tx.GetValueIn(viewTemp)-tx.GetValueOut(); - nTxSigOps += tx.GetP2SHSigOpCount(mapInputs); + nTxSigOps += tx.GetP2SHSigOpCount(viewTemp); if (nBlockSigOps + nTxSigOps >= MAX_BLOCK_SIGOPS) continue; - if (!tx.ConnectInputs(mapInputs, mapTestPoolTmp, CDiskTxPos(true), pindexPrev, false, true)) + CTxUndo txundo; + if (!tx.UpdateCoins(viewTemp, txundo, pindexPrev->nHeight+1)) continue; - mapTestPoolTmp[tx.GetHash()] = CTxIndex(CDiskTxPos(true), tx.vout.size()); - swap(mapTestPool, mapTestPoolTmp); + + // push changes from the second layer cache to the first one + viewTemp.Flush(); // Added pblock->vtx.push_back(tx); @@ -3829,19 +3799,20 @@ CBlock* CreateNewBlock(CReserveKey& reservekey) nLastBlockSize = nBlockSize; printf("CreateNewBlock(): total size %"PRI64u"\n", nBlockSize); - pblock->vtx[0].vout[0].nValue = GetBlockValue(pindexPrev->nHeight+1, nFees); - - // Fill in header - pblock->hashPrevBlock = pindexPrev->GetBlockHash(); - pblock->UpdateTime(pindexPrev); - pblock->nBits = GetNextWorkRequired(pindexPrev, pblock.get()); - pblock->nNonce = 0; + pblock->vtx[0].vout[0].nValue = GetBlockValue(pindexPrev->nHeight+1, nFees); + // Fill in header + pblock->hashPrevBlock = pindexPrev->GetBlockHash(); + pblock->UpdateTime(pindexPrev); + pblock->nBits = GetNextWorkRequired(pindexPrev, pblock.get()); + pblock->nNonce = 0; pblock->vtx[0].vin[0].scriptSig = scriptDummy; + CBlockIndex indexDummy(*pblock); indexDummy.pprev = pindexPrev; indexDummy.nHeight = pindexPrev->nHeight + 1; - if (!pblock->ConnectBlock(txdb, &indexDummy, true)) + CCoinsViewCache viewNew(viewdb); + if (!pblock->ConnectBlock(&indexDummy, viewNew, true)) throw std::runtime_error("CreateNewBlock() : ConnectBlock failed"); } -- cgit v1.2.3 From ae8bfd12daa802d20791e69d3477e799d2b99f45 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 6 Jul 2012 16:33:34 +0200 Subject: Batch block connection during IBD During the initial block download (or -loadblock), delay connection of new blocks a bit, and perform them in a single action. This reduces the load on the database engine, as subsequent blocks often update an earlier block's transaction already. --- src/main.cpp | 146 ++++++++++++++++++++++++----------------------------------- 1 file changed, 59 insertions(+), 87 deletions(-) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index ef5f627d9..c23aae320 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -168,6 +168,7 @@ bool CCoinsView::SetCoins(uint256 txid, const CCoins &coins) { return false; } bool CCoinsView::HaveCoins(uint256 txid) { return false; } CBlockIndex *CCoinsView::GetBestBlock() { return NULL; } bool CCoinsView::SetBestBlock(CBlockIndex *pindex) { return false; } +bool CCoinsView::BatchWrite(const std::map &mapCoins, CBlockIndex *pindex) { return false; } CCoinsViewBacked::CCoinsViewBacked(CCoinsView &viewIn) : base(&viewIn) { } bool CCoinsViewBacked::GetCoins(uint256 txid, CCoins &coins) { return base->GetCoins(txid, coins); } @@ -176,13 +177,7 @@ bool CCoinsViewBacked::HaveCoins(uint256 txid) { return base->HaveCoins(txid); } CBlockIndex *CCoinsViewBacked::GetBestBlock() { return base->GetBestBlock(); } bool CCoinsViewBacked::SetBestBlock(CBlockIndex *pindex) { return base->SetBestBlock(pindex); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } - -CCoinsViewDB::CCoinsViewDB(CCoinsDB &dbIn) : db(dbIn) {} -bool CCoinsViewDB::GetCoins(uint256 txid, CCoins &coins) { return db.ReadCoins(txid, coins); } -bool CCoinsViewDB::SetCoins(uint256 txid, const CCoins &coins) { return db.WriteCoins(txid, coins); } -bool CCoinsViewDB::HaveCoins(uint256 txid) { return db.HaveCoins(txid); } -CBlockIndex *CCoinsViewDB::GetBestBlock() { return pindexBest; } -bool CCoinsViewDB::SetBestBlock(CBlockIndex *pindex) { return db.WriteHashBestChain(pindex->GetBlockHash()); } +bool CCoinsViewBacked::BatchWrite(const std::map &mapCoins, CBlockIndex *pindex) { return base->BatchWrite(mapCoins, pindex); } CCoinsViewCache::CCoinsViewCache(CCoinsView &baseIn, bool fDummy) : CCoinsViewBacked(baseIn), pindexTip(NULL) { } @@ -218,18 +213,24 @@ bool CCoinsViewCache::SetBestBlock(CBlockIndex *pindex) { return true; } -bool CCoinsViewCache::Flush() { - for (std::map::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) { - if (!base->SetCoins(it->first, it->second)) - return false; - } - if (!base->SetBestBlock(pindexTip)) - return false; - cacheCoins.clear(); - pindexTip = NULL; +bool CCoinsViewCache::BatchWrite(const std::map &mapCoins, CBlockIndex *pindex) { + for (std::map::const_iterator it = mapCoins.begin(); it != mapCoins.end(); it++) + cacheCoins[it->first] = it->second; + pindexTip = pindex; return true; } +bool CCoinsViewCache::Flush() { + bool fOk = base->BatchWrite(cacheCoins, pindexTip); + if (fOk) + cacheCoins.clear(); + return fOk; +} + +unsigned int CCoinsViewCache::GetCacheSize() { + return cacheCoins.size(); +} + /** CCoinsView that brings transactions from a memorypool into view. It does not check for spendings by memory pool transactions. */ CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView &baseIn, CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } @@ -249,7 +250,7 @@ bool CCoinsViewMemPool::HaveCoins(uint256 txid) { return mempool.exists(txid) || base->HaveCoins(txid); } - +CCoinsViewCache *pcoinsTip = NULL; ////////////////////////////////////////////////////////////////////////////// // @@ -450,9 +451,8 @@ int CMerkleTx::SetMerkleBranch(const CBlock* pblock) CBlock blockTmp; if (pblock == NULL) { - CCoinsDB coinsdb("r"); CCoins coins; - if (coinsdb.ReadCoins(GetHash(), coins)) { + if (pcoinsTip->GetCoins(GetHash(), coins)) { CBlockIndex *pindex = FindBlockByHeight(coins.nHeight); if (pindex) { if (!blockTmp.ReadFromDisk(pindex)) @@ -609,7 +609,7 @@ void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins) } } -bool CTxMemPool::accept(CCoinsDB& coinsdb, CTransaction &tx, bool fCheckInputs, +bool CTxMemPool::accept(CTransaction &tx, bool fCheckInputs, bool* pfMissingInputs) { if (pfMissingInputs) @@ -668,9 +668,7 @@ bool CTxMemPool::accept(CCoinsDB& coinsdb, CTransaction &tx, bool fCheckInputs, if (fCheckInputs) { - CCoinsViewDB viewDB(coinsdb); - CCoinsViewMemPool viewMemPool(viewDB, mempool); - CCoinsViewCache view(viewMemPool); + CCoinsViewCache &view = *pcoinsTip; // do we already have it? if (view.HaveCoins(hash)) @@ -758,9 +756,9 @@ bool CTxMemPool::accept(CCoinsDB& coinsdb, CTransaction &tx, bool fCheckInputs, return true; } -bool CTransaction::AcceptToMemoryPool(CCoinsDB& coinsdb, bool fCheckInputs, bool* pfMissingInputs) +bool CTransaction::AcceptToMemoryPool(bool fCheckInputs, bool* pfMissingInputs) { - return mempool.accept(coinsdb, *this, fCheckInputs, pfMissingInputs); + return mempool.accept(*this, fCheckInputs, pfMissingInputs); } bool CTxMemPool::addUnchecked(const uint256& hash, CTransaction &tx) @@ -849,31 +847,24 @@ int CMerkleTx::GetBlocksToMaturity() const } -bool CMerkleTx::AcceptToMemoryPool(CCoinsDB& coinsdb, bool fCheckInputs) +bool CMerkleTx::AcceptToMemoryPool(bool fCheckInputs) { if (fClient) { if (!IsInMainChain() && !ClientCheckInputs()) return false; - return CTransaction::AcceptToMemoryPool(coinsdb, false); + return CTransaction::AcceptToMemoryPool(false); } else { - return CTransaction::AcceptToMemoryPool(coinsdb, fCheckInputs); + return CTransaction::AcceptToMemoryPool(fCheckInputs); } } -bool CMerkleTx::AcceptToMemoryPool() -{ - CCoinsDB coinsdb("r"); - return AcceptToMemoryPool(coinsdb); -} - -bool CWalletTx::AcceptWalletTransaction(CCoinsDB& coinsdb, bool fCheckInputs) +bool CWalletTx::AcceptWalletTransaction(bool fCheckInputs) { - { LOCK(mempool.cs); // Add previous supporting transactions first @@ -882,20 +873,15 @@ bool CWalletTx::AcceptWalletTransaction(CCoinsDB& coinsdb, bool fCheckInputs) if (!tx.IsCoinBase()) { uint256 hash = tx.GetHash(); - if (!mempool.exists(hash) && !coinsdb.HaveCoins(hash)) - tx.AcceptToMemoryPool(coinsdb, fCheckInputs); + if (!mempool.exists(hash) && pcoinsTip->HaveCoins(hash)) + tx.AcceptToMemoryPool(fCheckInputs); } } - return AcceptToMemoryPool(coinsdb, fCheckInputs); + return AcceptToMemoryPool(fCheckInputs); } return false; } -bool CWalletTx::AcceptWalletTransaction() -{ - CCoinsDB coinsdb("r"); - return AcceptWalletTransaction(coinsdb); -} // Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock, bool fAllowSlow) @@ -915,8 +901,7 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it int nHeight = -1; { - CCoinsDB coindb("r"); - CCoinsViewDB view(coindb); + CCoinsViewCache &view = *pcoinsTip; CCoins coins; if (view.GetCoins(hash, coins)) nHeight = coins.nHeight; @@ -1565,18 +1550,15 @@ bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsView &view, bool fJustCheck bool CBlock::SetBestChain(CBlockIndex* pindexNew) { - // if this functions exits prematurely, the transaction is aborted - CCoinsDB coinsdb; - if (!coinsdb.TxnBegin()) - return error("SetBestChain() : TxnBegin failed"); + CCoinsViewCache &view = *pcoinsTip; // special case for attaching the genesis block // note that no ConnectBlock is called, so its coinbase output is non-spendable if (pindexGenesisBlock == NULL && pindexNew->GetBlockHash() == hashGenesisBlock) { - coinsdb.WriteHashBestChain(pindexNew->GetBlockHash()); - if (!coinsdb.TxnCommit()) - return error("SetBestChain() : TxnCommit failed"); + view.SetBestBlock(pindexNew); + if (!view.Flush()) + return false; pindexGenesisBlock = pindexNew; pindexBest = pindexNew; hashBestChain = pindexNew->GetBlockHash(); @@ -1585,10 +1567,6 @@ bool CBlock::SetBestChain(CBlockIndex* pindexNew) return true; } - // create cached view to the coins database - CCoinsViewDB viewDB(coinsdb); - CCoinsViewCache view(viewDB); - // Find the fork (typically, there is none) CBlockIndex* pfork = view.GetBestBlock(); CBlockIndex* plonger = pindexNew; @@ -1625,8 +1603,11 @@ bool CBlock::SetBestChain(CBlockIndex* pindexNew) CBlock block; if (!block.ReadFromDisk(pindex)) return error("SetBestBlock() : ReadFromDisk for disconnect failed"); - if (!block.DisconnectBlock(pindex, view)) + CCoinsViewCache viewTemp(view, true); + if (!block.DisconnectBlock(pindex, viewTemp)) return error("SetBestBlock() : DisconnectBlock %s failed", pindex->GetBlockHash().ToString().substr(0,20).c_str()); + if (!viewTemp.Flush()) + return error("SetBestBlock() : Cache flush failed after disconnect"); // Queue memory transactions to resurrect BOOST_FOREACH(const CTransaction& tx, block.vtx) @@ -1646,10 +1627,13 @@ bool CBlock::SetBestChain(CBlockIndex* pindexNew) return error("SetBestBlock() : ReadFromDisk for connect failed"); pblock = █ } - if (!pblock->ConnectBlock(pindex, view)) { + CCoinsViewCache viewTemp(view, true); + if (!pblock->ConnectBlock(pindex, viewTemp)) { InvalidChainFound(pindexNew); return error("SetBestBlock() : ConnectBlock %s failed", pindex->GetBlockHash().ToString().substr(0,20).c_str()); } + if (!viewTemp.Flush()) + return error("SetBestBlock() : Cache flush failed after connect"); // Queue memory transactions to delete BOOST_FOREACH(const CTransaction& tx, pblock->vtx) @@ -1657,11 +1641,10 @@ bool CBlock::SetBestChain(CBlockIndex* pindexNew) } // Make sure it's successfully written to disk before changing memory structure - if (!view.Flush()) - return error("SetBestBlock() : failed to write coin changes"); - if (!coinsdb.TxnCommit()) - return error("SetBestBlock() : TxnCommit failed"); - coinsdb.Close(); + bool fIsInitialDownload = IsInitialBlockDownload(); + if (!fIsInitialDownload || view.GetCacheSize()>5000) + if (!view.Flush()) + return false; // At this point, all changes have been done to the database. // Proceed by updating the memory structures. @@ -1678,14 +1661,13 @@ bool CBlock::SetBestChain(CBlockIndex* pindexNew) // Resurrect memory transactions that were in the disconnected branch BOOST_FOREACH(CTransaction& tx, vResurrect) - tx.AcceptToMemoryPool(coinsdb, false); + tx.AcceptToMemoryPool(false); // Delete redundant memory transactions that are in the connected branch BOOST_FOREACH(CTransaction& tx, vDelete) mempool.remove(tx); // Update best block in wallet (so we can detect restored wallets) - bool fIsInitialDownload = IsInitialBlockDownload(); if (!fIsInitialDownload) { const CBlockLocator locator(pindexNew); @@ -1765,11 +1747,8 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) return false; // New best - if (pindexNew->bnChainWork > bnBestChainWork) { - if (!IsInitialBlockDownload() || (pindexNew->nHeight % 1) == 0) - if (!SetBestChain(pindexNew)) - return false; - } + if (!SetBestChain(pindexNew)) + return false; if (pindexNew == pindexBest) { @@ -2169,11 +2148,9 @@ bool LoadBlockIndex(bool fAllowNew) // Load block index // CChainDB chaindb("cr"); - CCoinsDB coinsdb("cr"); - if (!LoadBlockIndex(coinsdb, chaindb)) + if (!LoadBlockIndex(chaindb)) return false; chaindb.Close(); - coinsdb.Close(); // // Init with genesis block @@ -2492,7 +2469,7 @@ string GetWarnings(string strFor) // -bool static AlreadyHave(CCoinsDB &coinsdb, const CInv& inv) +bool static AlreadyHave(const CInv& inv) { switch (inv.type) { @@ -2504,7 +2481,7 @@ bool static AlreadyHave(CCoinsDB &coinsdb, const CInv& inv) txInMap = mempool.exists(inv.hash); } return txInMap || mapOrphanTransactions.count(inv.hash) || - coinsdb.HaveCoins(inv.hash); + pcoinsTip->HaveCoins(inv.hash); } case MSG_BLOCK: return mapBlockIndex.count(inv.hash) || @@ -2748,7 +2725,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) break; } } - CCoinsDB coinsdb("r"); for (unsigned int nInv = 0; nInv < vInv.size(); nInv++) { const CInv &inv = vInv[nInv]; @@ -2757,7 +2733,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) return true; pfrom->AddInventoryKnown(inv); - bool fAlreadyHave = AlreadyHave(coinsdb, inv); + bool fAlreadyHave = AlreadyHave(inv); if (fDebug) printf(" got inventory: %s %s\n", inv.ToString().c_str(), fAlreadyHave ? "have" : "new"); @@ -2929,7 +2905,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) vector vWorkQueue; vector vEraseQueue; CDataStream vMsg(vRecv); - CCoinsDB coinsdb("r"); CTransaction tx; vRecv >> tx; @@ -2937,7 +2912,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) pfrom->AddInventoryKnown(inv); bool fMissingInputs = false; - if (tx.AcceptToMemoryPool(coinsdb, true, &fMissingInputs)) + if (tx.AcceptToMemoryPool(true, &fMissingInputs)) { SyncWithWallets(tx, NULL, true); RelayMessage(inv, vMsg); @@ -2959,7 +2934,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) CInv inv(MSG_TX, tx.GetHash()); bool fMissingInputs2 = false; - if (tx.AcceptToMemoryPool(coinsdb, true, &fMissingInputs2)) + if (tx.AcceptToMemoryPool(true, &fMissingInputs2)) { printf(" accepted orphan tx %s\n", inv.hash.ToString().substr(0,10).c_str()); SyncWithWallets(tx, NULL, true); @@ -3407,11 +3382,10 @@ bool SendMessages(CNode* pto, bool fSendTrickle) // vector vGetData; int64 nNow = GetTime() * 1000000; - CCoinsDB coinsdb("r"); while (!pto->mapAskFor.empty() && (*pto->mapAskFor.begin()).first <= nNow) { const CInv& inv = (*pto->mapAskFor.begin()).second; - if (!AlreadyHave(coinsdb, inv)) + if (!AlreadyHave(inv)) { if (fDebugNet) printf("sending getdata: %s\n", inv.ToString().c_str()); @@ -3621,9 +3595,7 @@ CBlock* CreateNewBlock(CReserveKey& reservekey) int64 nFees = 0; { LOCK2(cs_main, mempool.cs); - CCoinsDB coinsdb("r"); - CCoinsViewDB viewdb(coinsdb); - CCoinsViewCache view(viewdb); + CCoinsViewCache view(*pcoinsTip, true); // Priority order to process transactions list vOrphan; // list memory doesn't move @@ -3811,7 +3783,7 @@ CBlock* CreateNewBlock(CReserveKey& reservekey) CBlockIndex indexDummy(*pblock); indexDummy.pprev = pindexPrev; indexDummy.nHeight = pindexPrev->nHeight + 1; - CCoinsViewCache viewNew(viewdb); + CCoinsViewCache viewNew(*pcoinsTip, true); if (!pblock->ConnectBlock(&indexDummy, viewNew, true)) throw std::runtime_error("CreateNewBlock() : ConnectBlock failed"); } -- cgit v1.2.3 From 64dd46fd05de1dd8ff5066e192e1345f733c6a1f Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 8 Jul 2012 00:06:34 +0200 Subject: Transaction hash caching Use CBlock's vMerkleTree to cache transaction hashes, and pass them along as argument in more function calls. During initial block download, this results in every transaction's hash to be only computed once. --- src/main.cpp | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index c23aae320..a4f90dda7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -111,10 +111,10 @@ void static EraseFromWallets(uint256 hash) } // make sure all wallets know about the given transaction, in the given block -void SyncWithWallets(const CTransaction& tx, const CBlock* pblock, bool fUpdate) +void SyncWithWallets(const uint256 &hash, const CTransaction& tx, const CBlock* pblock, bool fUpdate) { BOOST_FOREACH(CWallet* pwallet, setpwalletRegistered) - pwallet->AddToWalletIfInvolvingMe(tx, pblock, fUpdate); + pwallet->AddToWalletIfInvolvingMe(hash, tx, pblock, fUpdate); } // notify wallets about a new best chain @@ -1197,10 +1197,8 @@ unsigned int CTransaction::GetP2SHSigOpCount(CCoinsView& inputs) const return nSigOps; } -bool CTransaction::UpdateCoins(CCoinsView &inputs, CTxUndo &txundo, int nHeight) const +bool CTransaction::UpdateCoins(CCoinsView &inputs, CTxUndo &txundo, int nHeight, const uint256 &txhash) const { - uint256 hash = GetHash(); - // mark inputs spent if (!IsCoinBase()) { BOOST_FOREACH(const CTxIn &txin, vin) { @@ -1217,7 +1215,7 @@ bool CTransaction::UpdateCoins(CCoinsView &inputs, CTxUndo &txundo, int nHeight) } // add outputs - if (!inputs.SetCoins(hash, CCoins(*this, nHeight))) + if (!inputs.SetCoins(txhash, CCoins(*this, nHeight))) return error("UpdateCoins() : cannot update output"); return true; @@ -1467,8 +1465,8 @@ bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsView &view, bool fJustCheck bool fEnforceBIP30 = !((pindex->nHeight==91842 && pindex->GetBlockHash() == uint256("0x00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec")) || (pindex->nHeight==91880 && pindex->GetBlockHash() == uint256("0x00000000000743f190a18c5577a3c2d2a1f610ae9601ac046a38084ccb7cd721"))); if (fEnforceBIP30) { - BOOST_FOREACH(CTransaction& tx, vtx) { - uint256 hash = tx.GetHash(); + for (unsigned int i=0; i MAX_BLOCK_SIGOPS) return DoS(100, error("ConnectBlock() : too many sigops")); @@ -1511,7 +1511,7 @@ bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsView &view, bool fJustCheck } CTxUndo txundo; - if (!tx.UpdateCoins(view, txundo, pindex->nHeight)) + if (!tx.UpdateCoins(view, txundo, pindex->nHeight, GetTxHash(i))) return error("ConnectBlock() : UpdateInputs failed"); if (!tx.IsCoinBase()) blockundo.vtxundo.push_back(txundo); @@ -1542,8 +1542,8 @@ bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsView &view, bool fJustCheck return false; // Watch for transactions paying to me - BOOST_FOREACH(CTransaction& tx, vtx) - SyncWithWallets(tx, this, true); + for (unsigned int i=0; i uniqueTx; - BOOST_FOREACH(const CTransaction& tx, vtx) - { - uniqueTx.insert(tx.GetHash()); + for (unsigned int i=0; inHeight+1)) + uint256 hash = tx.GetHash(); + if (!tx.UpdateCoins(viewTemp, txundo, pindexPrev->nHeight+1, hash)) continue; // push changes from the second layer cache to the first one @@ -3749,7 +3750,6 @@ CBlock* CreateNewBlock(CReserveKey& reservekey) } // Add transactions that depend on this one to the priority queue - uint256 hash = tx.GetHash(); if (mapDependers.count(hash)) { BOOST_FOREACH(COrphan* porphan, mapDependers[hash]) -- cgit v1.2.3 From 13c51f20f619b9001bb6caf225b8cd6c5c2fbb31 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 8 Jul 2012 19:04:05 +0200 Subject: Direct CCoins references To prevent excessive copying of CCoins in and out of the CCoinsView implementations, introduce a GetCoins() function in CCoinsViewCache with returns a direct reference. The block validation and connection logic is updated to require caching CCoinsViews, and exploits the GetCoins() function heavily. --- src/main.cpp | 102 +++++++++++++++++++++++++++++++---------------------------- 1 file changed, 53 insertions(+), 49 deletions(-) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index a4f90dda7..42f70c023 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -170,6 +170,7 @@ CBlockIndex *CCoinsView::GetBestBlock() { return NULL; } bool CCoinsView::SetBestBlock(CBlockIndex *pindex) { return false; } bool CCoinsView::BatchWrite(const std::map &mapCoins, CBlockIndex *pindex) { return false; } + CCoinsViewBacked::CCoinsViewBacked(CCoinsView &viewIn) : base(&viewIn) { } bool CCoinsViewBacked::GetCoins(uint256 txid, CCoins &coins) { return base->GetCoins(txid, coins); } bool CCoinsViewBacked::SetCoins(uint256 txid, const CCoins &coins) { return base->SetCoins(txid, coins); } @@ -193,13 +194,30 @@ bool CCoinsViewCache::GetCoins(uint256 txid, CCoins &coins) { return false; } +std::map::iterator CCoinsViewCache::FetchCoins(uint256 txid) { + std::map::iterator it = cacheCoins.find(txid); + if (it != cacheCoins.end()) + return it; + CCoins tmp; + if (!base->GetCoins(txid,tmp)) + return it; + std::pair::iterator,bool> ret = cacheCoins.insert(std::make_pair(txid, tmp)); + return ret.first; +} + +CCoins &CCoinsViewCache::GetCoins(uint256 txid) { + std::map::iterator it = FetchCoins(txid); + assert(it != cacheCoins.end()); + return it->second; +} + bool CCoinsViewCache::SetCoins(uint256 txid, const CCoins &coins) { cacheCoins[txid] = coins; return true; } bool CCoinsViewCache::HaveCoins(uint256 txid) { - return cacheCoins.count(txid) || base->HaveCoins(txid); + return FetchCoins(txid) != cacheCoins.end(); } CBlockIndex *CCoinsViewCache::GetBestBlock() { @@ -369,7 +387,7 @@ bool CTransaction::IsStandard() const // expensive-to-check-upon-redemption script like: // DUP CHECKSIG DROP ... repeated 100 times... OP_1 // -bool CTransaction::AreInputsStandard(CCoinsView& mapInputs) const +bool CTransaction::AreInputsStandard(CCoinsViewCache& mapInputs) const { if (IsCoinBase()) return true; // Coinbases don't use vin normally @@ -683,6 +701,9 @@ bool CTxMemPool::accept(CTransaction &tx, bool fCheckInputs, } } + if (!tx.HaveInputs(view)) + return error("CTxMemPool::accept() : inputs already spent"); + // Check for non-standard pay-to-script-hash in inputs if (!tx.AreInputsStandard(view) && !fTestNet) return error("CTxMemPool::accept() : nonstandard transaction input"); @@ -1154,23 +1175,14 @@ void CBlock::UpdateTime(const CBlockIndex* pindexPrev) -CTxOut CTransaction::GetOutputFor(const CTxIn& input, CCoinsView& view) +const CTxOut &CTransaction::GetOutputFor(const CTxIn& input, CCoinsViewCache& view) { - CCoins coins; - if (!view.GetCoins(input.prevout.hash, coins)) - throw std::runtime_error("CTransaction::GetOutputFor() : prevout.hash not found"); - - if (input.prevout.n >= coins.vout.size()) - throw std::runtime_error("CTransaction::GetOutputFor() : prevout.n out of range or already spent"); - - const CTxOut &out = coins.vout[input.prevout.n]; - if (out.IsNull()) - throw std::runtime_error("CTransaction::GetOutputFor() : already spent"); - - return out; + const CCoins &coins = view.GetCoins(input.prevout.hash); + assert(coins.IsAvailable(input.prevout.n)); + return coins.vout[input.prevout.n]; } -int64 CTransaction::GetValueIn(CCoinsView& inputs) const +int64 CTransaction::GetValueIn(CCoinsViewCache& inputs) const { if (IsCoinBase()) return 0; @@ -1182,7 +1194,7 @@ int64 CTransaction::GetValueIn(CCoinsView& inputs) const return nResult; } -unsigned int CTransaction::GetP2SHSigOpCount(CCoinsView& inputs) const +unsigned int CTransaction::GetP2SHSigOpCount(CCoinsViewCache& inputs) const { if (IsCoinBase()) return 0; @@ -1190,27 +1202,23 @@ unsigned int CTransaction::GetP2SHSigOpCount(CCoinsView& inputs) const unsigned int nSigOps = 0; for (unsigned int i = 0; i < vin.size(); i++) { - CTxOut prevout = GetOutputFor(vin[i], inputs); + const CTxOut &prevout = GetOutputFor(vin[i], inputs); if (prevout.scriptPubKey.IsPayToScriptHash()) nSigOps += prevout.scriptPubKey.GetSigOpCount(vin[i].scriptSig); } return nSigOps; } -bool CTransaction::UpdateCoins(CCoinsView &inputs, CTxUndo &txundo, int nHeight, const uint256 &txhash) const +bool CTransaction::UpdateCoins(CCoinsViewCache &inputs, CTxUndo &txundo, int nHeight, const uint256 &txhash) const { // mark inputs spent if (!IsCoinBase()) { BOOST_FOREACH(const CTxIn &txin, vin) { - CCoins coins; - if (!inputs.GetCoins(txin.prevout.hash, coins)) - return error("UpdateCoins() : cannot find prevtx"); + CCoins &coins = inputs.GetCoins(txin.prevout.hash); CTxInUndo undo; if (!coins.Spend(txin.prevout, undo)) return error("UpdateCoins() : cannot spend input"); txundo.vprevout.push_back(undo); - if (!inputs.SetCoins(txin.prevout.hash, coins)) - return error("UpdateCoins() : cannot update input"); } } @@ -1221,7 +1229,7 @@ bool CTransaction::UpdateCoins(CCoinsView &inputs, CTxUndo &txundo, int nHeight, return true; } -bool CTransaction::HaveInputs(CCoinsView &inputs) const +bool CTransaction::HaveInputs(CCoinsViewCache &inputs) const { if (!IsCoinBase()) { // first check whether information about the prevout hash is available @@ -1234,8 +1242,7 @@ bool CTransaction::HaveInputs(CCoinsView &inputs) const // then check whether the actual outputs are available for (unsigned int i = 0; i < vin.size(); i++) { const COutPoint &prevout = vin[i].prevout; - CCoins coins; - inputs.GetCoins(prevout.hash, coins); + const CCoins &coins = inputs.GetCoins(prevout.hash); if (!coins.IsAvailable(prevout.n)) return false; } @@ -1243,28 +1250,25 @@ bool CTransaction::HaveInputs(CCoinsView &inputs) const return true; } -bool CTransaction::CheckInputs(CCoinsView &inputs, enum CheckSig_mode csmode, bool fStrictPayToScriptHash, bool fStrictEncodings) const +bool CTransaction::CheckInputs(CCoinsViewCache &inputs, enum CheckSig_mode csmode, bool fStrictPayToScriptHash, bool fStrictEncodings) const { if (!IsCoinBase()) { + // This doesn't trigger the DoS code on purpose; if it did, it would make it easier + // for an attacker to attempt to split the network. + if (!HaveInputs(inputs)) + return error("CheckInputs() : %s inputs unavailable", GetHash().ToString().substr(0,10).c_str()); + + CBlockIndex *pindexBlock = inputs.GetBestBlock(); int64 nValueIn = 0; int64 nFees = 0; for (unsigned int i = 0; i < vin.size(); i++) { const COutPoint &prevout = vin[i].prevout; - CCoins coins; - if (!inputs.GetCoins(prevout.hash, coins)) - return error("CheckInputs() : cannot find prevout tx"); - - // Check for conflicts (double-spend) - // This doesn't trigger the DoS code on purpose; if it did, it would make it easier - // for an attacker to attempt to split the network. - if (!coins.IsAvailable(prevout.n)) - return error("CheckInputs() : %s prev tx already used", GetHash().ToString().substr(0,10).c_str()); + const CCoins &coins = inputs.GetCoins(prevout.hash); // If prev is coinbase, check that it's matured if (coins.IsCoinBase()) { - CBlockIndex *pindexBlock = inputs.GetBestBlock(); if (pindexBlock->nHeight - coins.nHeight < COINBASE_MATURITY) return error("CheckInputs() : tried to spend coinbase at depth %d", pindexBlock->nHeight - coins.nHeight); } @@ -1298,8 +1302,7 @@ bool CTransaction::CheckInputs(CCoinsView &inputs, enum CheckSig_mode csmode, bo (csmode == CS_AFTER_CHECKPOINT && inputs.GetBestBlock()->nHeight >= Checkpoints::GetTotalBlocksEstimate())) { for (unsigned int i = 0; i < vin.size(); i++) { const COutPoint &prevout = vin[i].prevout; - CCoins coins; - inputs.GetCoins(prevout.hash, coins); + const CCoins &coins = inputs.GetCoins(prevout.hash); // Verify signature if (!VerifySignature(coins, *this, i, fStrictPayToScriptHash, fStrictEncodings, 0)) { @@ -1367,7 +1370,7 @@ bool CTransaction::ClientCheckInputs() const -bool CBlock::DisconnectBlock(CBlockIndex *pindex, CCoinsView &view) +bool CBlock::DisconnectBlock(CBlockIndex *pindex, CCoinsViewCache &view) { assert(pindex == view.GetBestBlock()); @@ -1391,17 +1394,16 @@ bool CBlock::DisconnectBlock(CBlockIndex *pindex, CCoinsView &view) uint256 hash = tx.GetHash(); // check that all outputs are available - CCoins outs; - if (!view.GetCoins(hash, outs)) + if (!view.HaveCoins(hash)) return error("DisconnectBlock() : outputs still spent? database corrupted"); + CCoins &outs = view.GetCoins(hash); CCoins outsBlock = CCoins(tx, pindex->nHeight); if (outs != outsBlock) return error("DisconnectBlock() : added transaction mismatch? database corrupted"); // remove outputs - if (!view.SetCoins(hash, CCoins())) - return error("DisconnectBlock() : cannot delete coin outputs"); + outs = CCoins(); // restore inputs if (i > 0) { // not coinbases @@ -1441,7 +1443,7 @@ bool CBlock::DisconnectBlock(CBlockIndex *pindex, CCoinsView &view) bool FindUndoPos(CChainDB &chaindb, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); -bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsView &view, bool fJustCheck) +bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsViewCache &view, bool fJustCheck) { // Check it again in case a previous version let a bad block in if (!CheckBlock(!fJustCheck, !fJustCheck)) @@ -1467,8 +1469,7 @@ bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsView &view, bool fJustCheck if (fEnforceBIP30) { for (unsigned int i=0; i= MAX_BLOCK_SIGOPS) continue; + if (!tx.CheckInputs(viewTemp, CS_ALWAYS, true, false)) + continue; + CTxUndo txundo; uint256 hash = tx.GetHash(); if (!tx.UpdateCoins(viewTemp, txundo, pindexPrev->nHeight+1, hash)) -- cgit v1.2.3 From 857c61df0b71c8a0482b1bf8fc55849f8ad831b8 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 19 Aug 2012 00:33:01 +0200 Subject: Prepare database format for multi-stage block processing This commit adds a status field and a transaction counter to the block indexes. --- src/main.cpp | 114 ++++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 89 insertions(+), 25 deletions(-) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index 42f70c023..b077eb9d6 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -37,6 +37,7 @@ CBigNum bnBestChainWork = 0; CBigNum bnBestInvalidWork = 0; uint256 hashBestChain = 0; CBlockIndex* pindexBest = NULL; +set setBlockIndexValid; // may contain all CBlockIndex*'s that have validness >=BLOCK_VALID_TRANSACTIONS, and must contain those who aren't failed int64 nTimeBestReceived = 0; bool fImporting = false; @@ -1156,6 +1157,62 @@ void static InvalidChainFound(CBlockIndex* pindexNew) printf("InvalidChainFound: Warning: Displayed transactions may not be correct! You may need to upgrade, or other nodes may need to upgrade.\n"); } +void static InvalidBlockFound(CBlockIndex *pindex) { + pindex->nStatus |= BLOCK_FAILED_VALID; + CChainDB().WriteBlockIndex(CDiskBlockIndex(pindex)); + setBlockIndexValid.erase(pindex); + InvalidChainFound(pindex); + if (pindex->pnext) + ConnectBestBlock(); // reorganise away from the failed block +} + +bool ConnectBestBlock() { + do { + CBlockIndex *pindexNewBest; + + { + std::set::reverse_iterator it = setBlockIndexValid.rbegin(); + if (it == setBlockIndexValid.rend()) + return true; + pindexNewBest = *it; + } + + if (pindexNewBest == pindexBest) + return true; // nothing to do + + // check ancestry + CBlockIndex *pindexTest = pindexNewBest; + std::vector vAttach; + do { + if (pindexTest->nStatus & BLOCK_FAILED_MASK) { + // mark descendants failed + CChainDB chaindb; + CBlockIndex *pindexFailed = pindexNewBest; + while (pindexTest != pindexFailed) { + pindexFailed->nStatus |= BLOCK_FAILED_CHILD; + setBlockIndexValid.erase(pindexFailed); + chaindb.WriteBlockIndex(CDiskBlockIndex(pindexFailed)); + pindexFailed = pindexFailed->pprev; + } + InvalidChainFound(pindexNewBest); + break; + } + + if (pindexBest == NULL || pindexTest->bnChainWork > pindexBest->bnChainWork) + vAttach.push_back(pindexTest); + + if (pindexTest->pprev == NULL || pindexTest->pnext != NULL) { + reverse(vAttach.begin(), vAttach.end()); + BOOST_FOREACH(CBlockIndex *pindexSwitch, vAttach) + if (!SetBestChain(pindexSwitch)) + return false; + return true; + } + pindexTest = pindexTest->pprev; + } while(true); + } while(true); +} + void CBlock::UpdateTime(const CBlockIndex* pindexPrev) { nTime = max(pindexPrev->GetMedianTimePast()+1, GetAdjustedTime()); @@ -1522,17 +1579,24 @@ bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsViewCache &view, bool fJust return true; // Write undo information to disk - if (pindex->GetUndoPos().IsNull()) + if (pindex->GetUndoPos().IsNull() || (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) { CChainDB chaindb; - CDiskBlockPos pos; - if (!FindUndoPos(chaindb, pindex->pos.nFile, pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 8)) - return error("ConnectBlock() : FindUndoPos failed"); - if (!blockundo.WriteToDisk(pos)) - return error("ConnectBlock() : CBlockUndo::WriteToDisk failed"); - - // update nUndoPos in block index - pindex->nUndoPos = pos.nPos + 1; + + if (pindex->GetUndoPos().IsNull()) { + CDiskBlockPos pos; + if (!FindUndoPos(chaindb, pindex->nFile, pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 8)) + return error("ConnectBlock() : FindUndoPos failed"); + if (!blockundo.WriteToDisk(pos)) + return error("ConnectBlock() : CBlockUndo::WriteToDisk failed"); + + // update nUndoPos in block index + pindex->nUndoPos = pos.nPos; + pindex->nStatus |= BLOCK_HAVE_UNDO; + } + + pindex->nStatus = (pindex->nStatus & ~BLOCK_VALID_MASK) | BLOCK_VALID_SCRIPTS; + CDiskBlockIndex blockindex(pindex); if (!chaindb.WriteBlockIndex(blockindex)) return error("ConnectBlock() : WriteBlockIndex failed"); @@ -1549,7 +1613,7 @@ bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsViewCache &view, bool fJust return true; } -bool CBlock::SetBestChain(CBlockIndex* pindexNew) +bool SetBestChain(CBlockIndex* pindexNew) { CCoinsViewCache &view = *pcoinsTip; @@ -1620,24 +1684,19 @@ bool CBlock::SetBestChain(CBlockIndex* pindexNew) vector vDelete; BOOST_FOREACH(CBlockIndex *pindex, vConnect) { CBlock block; - CBlock *pblock; - if (pindex == pindexNew) // connecting *this block - pblock = this; - else { // other block; read it from disk - if (!block.ReadFromDisk(pindex)) - return error("SetBestBlock() : ReadFromDisk for connect failed"); - pblock = █ - } + if (!block.ReadFromDisk(pindex)) + return error("SetBestBlock() : ReadFromDisk for connect failed"); CCoinsViewCache viewTemp(view, true); - if (!pblock->ConnectBlock(pindex, viewTemp)) { + if (!block.ConnectBlock(pindex, viewTemp)) { InvalidChainFound(pindexNew); + InvalidBlockFound(pindex); return error("SetBestBlock() : ConnectBlock %s failed", pindex->GetBlockHash().ToString().substr(0,20).c_str()); } if (!viewTemp.Flush()) return error("SetBestBlock() : Cache flush failed after connect"); // Queue memory transactions to delete - BOOST_FOREACH(const CTransaction& tx, pblock->vtx) + BOOST_FOREACH(const CTransaction& tx, block.vtx) vDelete.push_back(tx); } @@ -1683,8 +1742,8 @@ bool CBlock::SetBestChain(CBlockIndex* pindexNew) bnBestChainWork = pindexNew->bnChainWork; nTimeBestReceived = GetTime(); nTransactionsUpdated++; - printf("SetBestChain: new best=%s height=%d work=%s date=%s\n", - hashBestChain.ToString().substr(0,20).c_str(), nBestHeight, bnBestChainWork.ToString().c_str(), + printf("SetBestChain: new best=%s height=%d work=%s tx=%lu date=%s\n", + hashBestChain.ToString().substr(0,20).c_str(), nBestHeight, bnBestChainWork.ToString().c_str(), (unsigned long)pindexNew->nChainTx, DateTimeStrFormat("%x %H:%M:%S", pindexBest->GetBlockTime()).c_str()); // Check the version of the last 100 blocks to see if we need to upgrade: @@ -1736,9 +1795,14 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) pindexNew->pprev = (*miPrev).second; pindexNew->nHeight = pindexNew->pprev->nHeight + 1; } + pindexNew->nTx = vtx.size(); pindexNew->bnChainWork = (pindexNew->pprev ? pindexNew->pprev->bnChainWork : 0) + pindexNew->GetBlockWork(); - pindexNew->pos = pos; + pindexNew->nChainTx = (pindexNew->pprev ? pindexNew->pprev->nChainTx : 0) + pindexNew->nTx; + pindexNew->nFile = pos.nFile; + pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; + pindexNew->nStatus = BLOCK_VALID_TRANSACTIONS | BLOCK_HAVE_DATA; + setBlockIndexValid.insert(pindexNew); CChainDB chaindb; if (!chaindb.TxnBegin()) @@ -1747,8 +1811,8 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) if (!chaindb.TxnCommit()) return false; - // New best - if (!SetBestChain(pindexNew)) + // New best? + if (!ConnectBestBlock()) return false; if (pindexNew == pindexBest) -- cgit v1.2.3 From d979e6e36ac6be0d40b2a6bb70c668f9e6989ff9 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 3 Sep 2012 15:26:57 +0200 Subject: Use singleton block tree database instance --- src/main.cpp | 61 ++++++++++++++++++++++++------------------------------------ 1 file changed, 24 insertions(+), 37 deletions(-) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index b077eb9d6..6395a2541 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -270,6 +270,7 @@ bool CCoinsViewMemPool::HaveCoins(uint256 txid) { } CCoinsViewCache *pcoinsTip = NULL; +CBlockTreeDB *pblocktree = NULL; ////////////////////////////////////////////////////////////////////////////// // @@ -1143,7 +1144,7 @@ void static InvalidChainFound(CBlockIndex* pindexNew) if (pindexNew->bnChainWork > bnBestInvalidWork) { bnBestInvalidWork = pindexNew->bnChainWork; - CChainDB().WriteBestInvalidWork(bnBestInvalidWork); + pblocktree->WriteBestInvalidWork(bnBestInvalidWork); uiInterface.NotifyBlocksChanged(); } printf("InvalidChainFound: invalid block=%s height=%d work=%s date=%s\n", @@ -1159,7 +1160,7 @@ void static InvalidChainFound(CBlockIndex* pindexNew) void static InvalidBlockFound(CBlockIndex *pindex) { pindex->nStatus |= BLOCK_FAILED_VALID; - CChainDB().WriteBlockIndex(CDiskBlockIndex(pindex)); + pblocktree->WriteBlockIndex(CDiskBlockIndex(pindex)); setBlockIndexValid.erase(pindex); InvalidChainFound(pindex); if (pindex->pnext) @@ -1186,12 +1187,11 @@ bool ConnectBestBlock() { do { if (pindexTest->nStatus & BLOCK_FAILED_MASK) { // mark descendants failed - CChainDB chaindb; CBlockIndex *pindexFailed = pindexNewBest; while (pindexTest != pindexFailed) { pindexFailed->nStatus |= BLOCK_FAILED_CHILD; setBlockIndexValid.erase(pindexFailed); - chaindb.WriteBlockIndex(CDiskBlockIndex(pindexFailed)); + pblocktree->WriteBlockIndex(CDiskBlockIndex(pindexFailed)); pindexFailed = pindexFailed->pprev; } InvalidChainFound(pindexNewBest); @@ -1498,7 +1498,7 @@ bool CBlock::DisconnectBlock(CBlockIndex *pindex, CCoinsViewCache &view) return true; } -bool FindUndoPos(CChainDB &chaindb, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); +bool FindUndoPos(int nFile, CDiskBlockPos &pos, unsigned int nAddSize); bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsViewCache &view, bool fJustCheck) { @@ -1581,11 +1581,9 @@ bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsViewCache &view, bool fJust // Write undo information to disk if (pindex->GetUndoPos().IsNull() || (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) { - CChainDB chaindb; - if (pindex->GetUndoPos().IsNull()) { CDiskBlockPos pos; - if (!FindUndoPos(chaindb, pindex->nFile, pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 8)) + if (!FindUndoPos(pindex->nFile, pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 8)) return error("ConnectBlock() : FindUndoPos failed"); if (!blockundo.WriteToDisk(pos)) return error("ConnectBlock() : CBlockUndo::WriteToDisk failed"); @@ -1598,7 +1596,7 @@ bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsViewCache &view, bool fJust pindex->nStatus = (pindex->nStatus & ~BLOCK_VALID_MASK) | BLOCK_VALID_SCRIPTS; CDiskBlockIndex blockindex(pindex); - if (!chaindb.WriteBlockIndex(blockindex)) + if (!pblocktree->WriteBlockIndex(blockindex)) return error("ConnectBlock() : WriteBlockIndex failed"); } @@ -1804,12 +1802,7 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) pindexNew->nStatus = BLOCK_VALID_TRANSACTIONS | BLOCK_HAVE_DATA; setBlockIndexValid.insert(pindexNew); - CChainDB chaindb; - if (!chaindb.TxnBegin()) - return false; - chaindb.WriteBlockIndex(CDiskBlockIndex(pindexNew)); - if (!chaindb.TxnCommit()) - return false; + pblocktree->WriteBlockIndex(CDiskBlockIndex(pindexNew)); // New best? if (!ConnectBestBlock()) @@ -1823,13 +1816,15 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) hashPrevBestCoinBase = GetTxHash(0); } + pblocktree->Flush(); + uiInterface.NotifyBlocksChanged(); return true; } -bool FindBlockPos(CChainDB &chaindb, CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64 nTime) +bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64 nTime) { bool fUpdatedLast = false; @@ -1845,7 +1840,7 @@ bool FindBlockPos(CChainDB &chaindb, CDiskBlockPos &pos, unsigned int nAddSize, fclose(file); nLastBlockFile++; infoLastBlockFile.SetNull(); - chaindb.ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine + pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine fUpdatedLast = true; } @@ -1865,15 +1860,15 @@ bool FindBlockPos(CChainDB &chaindb, CDiskBlockPos &pos, unsigned int nAddSize, fclose(file); } - if (!chaindb.WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + if (!pblocktree->WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) return error("FindBlockPos() : cannot write updated block info"); if (fUpdatedLast) - chaindb.WriteLastBlockFile(nLastBlockFile); + pblocktree->WriteLastBlockFile(nLastBlockFile); return true; } -bool FindUndoPos(CChainDB &chaindb, int nFile, CDiskBlockPos &pos, unsigned int nAddSize) +bool FindUndoPos(int nFile, CDiskBlockPos &pos, unsigned int nAddSize) { pos.nFile = nFile; @@ -1883,15 +1878,15 @@ bool FindUndoPos(CChainDB &chaindb, int nFile, CDiskBlockPos &pos, unsigned int if (nFile == nLastBlockFile) { pos.nPos = infoLastBlockFile.nUndoSize; nNewSize = (infoLastBlockFile.nUndoSize += nAddSize); - if (!chaindb.WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + if (!pblocktree->WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) return error("FindUndoPos() : cannot write updated block info"); } else { CBlockFileInfo info; - if (!chaindb.ReadBlockFileInfo(nFile, info)) + if (!pblocktree->ReadBlockFileInfo(nFile, info)) return error("FindUndoPos() : cannot read block info"); pos.nPos = info.nUndoSize; nNewSize = (info.nUndoSize += nAddSize); - if (!chaindb.WriteBlockFileInfo(nFile, info)) + if (!pblocktree->WriteBlockFileInfo(nFile, info)) return error("FindUndoPos() : cannot write updated block info"); } @@ -2022,11 +2017,8 @@ bool CBlock::AcceptBlock() if (!CheckDiskSpace(::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION))) return error("AcceptBlock() : out of disk space"); CDiskBlockPos blockPos; - { - CChainDB chaindb; - if (!FindBlockPos(chaindb, blockPos, nBlockSize+8, nHeight, nTime)) - return error("AcceptBlock() : FindBlockPos failed"); - } + if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, nTime)) + return error("AcceptBlock() : FindBlockPos failed"); if (!WriteToDisk(blockPos)) return error("AcceptBlock() : WriteToDisk failed"); if (!AddToBlockIndex(blockPos)) @@ -2210,12 +2202,10 @@ bool LoadBlockIndex(bool fAllowNew) } // - // Load block index + // Load block index from databases // - CChainDB chaindb("cr"); - if (!LoadBlockIndex(chaindb)) + if (!LoadBlockIndexDB()) return false; - chaindb.Close(); // // Init with genesis block @@ -2267,11 +2257,8 @@ bool LoadBlockIndex(bool fAllowNew) // Start new block file unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); CDiskBlockPos blockPos; - { - CChainDB chaindb; - if (!FindBlockPos(chaindb, blockPos, nBlockSize+8, 0, block.nTime)) - return error("AcceptBlock() : FindBlockPos failed"); - } + if (!FindBlockPos(blockPos, nBlockSize+8, 0, block.nTime)) + return error("AcceptBlock() : FindBlockPos failed"); if (!block.WriteToDisk(blockPos)) return error("LoadBlockIndex() : writing genesis block to disk failed"); if (!block.AddToBlockIndex(blockPos)) -- cgit v1.2.3 From 44d40f26dcc73469bfbfda5f981c2c528d592ac7 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 6 Sep 2012 03:21:18 +0200 Subject: Flush and sync block data --- src/main.cpp | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index 6395a2541..44e690b95 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1498,6 +1498,23 @@ bool CBlock::DisconnectBlock(CBlockIndex *pindex, CCoinsViewCache &view) return true; } +void static FlushBlockFile() +{ + LOCK(cs_LastBlockFile); + + CDiskBlockPos posOld; + posOld.nFile = nLastBlockFile; + posOld.nPos = 0; + + FILE *fileOld = OpenBlockFile(posOld); + FileCommit(fileOld); + fclose(fileOld); + + fileOld = OpenUndoFile(posOld); + FileCommit(fileOld); + fclose(fileOld); +} + bool FindUndoPos(int nFile, CDiskBlockPos &pos, unsigned int nAddSize); bool CBlock::ConnectBlock(CBlockIndex* pindex, CCoinsViewCache &view, bool fJustCheck) @@ -1700,9 +1717,11 @@ bool SetBestChain(CBlockIndex* pindexNew) // Make sure it's successfully written to disk before changing memory structure bool fIsInitialDownload = IsInitialBlockDownload(); - if (!fIsInitialDownload || view.GetCacheSize()>5000) + if (!fIsInitialDownload || view.GetCacheSize()>5000) { + FlushBlockFile(); if (!view.Flush()) return false; + } // At this point, all changes have been done to the database. // Proceed by updating the memory structures. @@ -1823,7 +1842,6 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) } - bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64 nTime) { bool fUpdatedLast = false; @@ -1832,12 +1850,7 @@ bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeigh while (infoLastBlockFile.nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { printf("Leaving block file %i: %s\n", nLastBlockFile, infoLastBlockFile.ToString().c_str()); - FILE *file = OpenBlockFile(pos); - FileCommit(file); - fclose(file); - file = OpenUndoFile(pos); - FileCommit(file); - fclose(file); + FlushBlockFile(); nLastBlockFile++; infoLastBlockFile.SetNull(); pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine -- cgit v1.2.3 From 2d8a48292b0da96cda8d7b45a24a22adfb4667b2 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 3 Sep 2012 21:14:03 +0200 Subject: LevelDB block and coin databases Split off CBlockTreeDB and CCoinsViewDB into txdb-*.{cpp,h} files, implemented by either LevelDB or BDB. Based on code from earlier commits by Mike Hearn in his leveldb branch. --- src/main.cpp | 112 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index 44e690b95..94f1a9315 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -6,6 +6,7 @@ #include "alert.h" #include "checkpoints.h" #include "db.h" +#include "txdb.h" #include "net.h" #include "init.h" #include "ui_interface.h" @@ -1719,6 +1720,7 @@ bool SetBestChain(CBlockIndex* pindexNew) bool fIsInitialDownload = IsInitialBlockDownload(); if (!fIsInitialDownload || view.GetCacheSize()>5000) { FlushBlockFile(); + pblocktree->Sync(); if (!view.Flush()) return false; } @@ -2203,6 +2205,116 @@ FILE *OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { return OpenDiskFile(pos, "rev", fReadOnly); } +CBlockIndex * InsertBlockIndex(uint256 hash) +{ + if (hash == 0) + return NULL; + + // Return existing + map::iterator mi = mapBlockIndex.find(hash); + if (mi != mapBlockIndex.end()) + return (*mi).second; + + // Create new + CBlockIndex* pindexNew = new CBlockIndex(); + if (!pindexNew) + throw runtime_error("LoadBlockIndex() : new CBlockIndex failed"); + mi = mapBlockIndex.insert(make_pair(hash, pindexNew)).first; + pindexNew->phashBlock = &((*mi).first); + + return pindexNew; +} + +bool static LoadBlockIndexDB() +{ + if (!pblocktree->LoadBlockIndexGuts()) + return false; + + if (fRequestShutdown) + return true; + + // Calculate bnChainWork + vector > vSortedByHeight; + vSortedByHeight.reserve(mapBlockIndex.size()); + BOOST_FOREACH(const PAIRTYPE(uint256, CBlockIndex*)& item, mapBlockIndex) + { + CBlockIndex* pindex = item.second; + vSortedByHeight.push_back(make_pair(pindex->nHeight, pindex)); + } + sort(vSortedByHeight.begin(), vSortedByHeight.end()); + BOOST_FOREACH(const PAIRTYPE(int, CBlockIndex*)& item, vSortedByHeight) + { + CBlockIndex* pindex = item.second; + pindex->bnChainWork = (pindex->pprev ? pindex->pprev->bnChainWork : 0) + pindex->GetBlockWork(); + pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; + if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS && !(pindex->nStatus & BLOCK_FAILED_MASK)) + setBlockIndexValid.insert(pindex); + } + + // Load block file info + pblocktree->ReadLastBlockFile(nLastBlockFile); + printf("LoadBlockIndex(): last block file = %i\n", nLastBlockFile); + if (pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile)) + printf("LoadBlockIndex(): last block file: %s\n", infoLastBlockFile.ToString().c_str()); + + // Load hashBestChain pointer to end of best chain + pindexBest = pcoinsTip->GetBestBlock(); + if (pindexBest == NULL) + { + if (pindexGenesisBlock == NULL) + return true; + } + hashBestChain = pindexBest->GetBlockHash(); + nBestHeight = pindexBest->nHeight; + bnBestChainWork = pindexBest->bnChainWork; + + // set 'next' pointers in best chain + CBlockIndex *pindex = pindexBest; + while(pindex != NULL && pindex->pprev != NULL) { + CBlockIndex *pindexPrev = pindex->pprev; + pindexPrev->pnext = pindex; + pindex = pindexPrev; + } + printf("LoadBlockIndex(): hashBestChain=%s height=%d date=%s\n", + hashBestChain.ToString().substr(0,20).c_str(), nBestHeight, + DateTimeStrFormat("%x %H:%M:%S", pindexBest->GetBlockTime()).c_str()); + + // Load bnBestInvalidWork, OK if it doesn't exist + pblocktree->ReadBestInvalidWork(bnBestInvalidWork); + + // Verify blocks in the best chain + int nCheckLevel = GetArg("-checklevel", 1); + int nCheckDepth = GetArg( "-checkblocks", 2500); + if (nCheckDepth == 0) + nCheckDepth = 1000000000; // suffices until the year 19000 + if (nCheckDepth > nBestHeight) + nCheckDepth = nBestHeight; + printf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); + CBlockIndex* pindexFork = NULL; + for (CBlockIndex* pindex = pindexBest; pindex && pindex->pprev; pindex = pindex->pprev) + { + if (fRequestShutdown || pindex->nHeight < nBestHeight-nCheckDepth) + break; + CBlock block; + if (!block.ReadFromDisk(pindex)) + return error("LoadBlockIndex() : block.ReadFromDisk failed"); + // check level 1: verify block validity + if (nCheckLevel>0 && !block.CheckBlock()) + { + printf("LoadBlockIndex() : *** found bad block at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString().c_str()); + pindexFork = pindex->pprev; + } + // TODO: stronger verifications + } + if (pindexFork && !fRequestShutdown) + { + // TODO: reorg back + return error("LoadBlockIndex(): chain database corrupted"); + } + + return true; +} + bool LoadBlockIndex(bool fAllowNew) { if (fTestNet) -- cgit v1.2.3 From beeb57610cf57c1d802be993ac0e98211bfc5f0c Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Sep 2012 23:04:54 +0200 Subject: Add gettxout and gettxoutsetinfo RPCs --- src/main.cpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index 94f1a9315..064ac7e10 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -171,6 +171,7 @@ bool CCoinsView::HaveCoins(uint256 txid) { return false; } CBlockIndex *CCoinsView::GetBestBlock() { return NULL; } bool CCoinsView::SetBestBlock(CBlockIndex *pindex) { return false; } bool CCoinsView::BatchWrite(const std::map &mapCoins, CBlockIndex *pindex) { return false; } +bool CCoinsView::GetStats(CCoinsStats &stats) { return false; } CCoinsViewBacked::CCoinsViewBacked(CCoinsView &viewIn) : base(&viewIn) { } @@ -181,6 +182,7 @@ CBlockIndex *CCoinsViewBacked::GetBestBlock() { return base->GetBestBlock(); } bool CCoinsViewBacked::SetBestBlock(CBlockIndex *pindex) { return base->SetBestBlock(pindex); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } bool CCoinsViewBacked::BatchWrite(const std::map &mapCoins, CBlockIndex *pindex) { return base->BatchWrite(mapCoins, pindex); } +bool CCoinsViewBacked::GetStats(CCoinsStats &stats) { return base->GetStats(stats); } CCoinsViewCache::CCoinsViewCache(CCoinsView &baseIn, bool fDummy) : CCoinsViewBacked(baseIn), pindexTip(NULL) { } -- cgit v1.2.3