From 42cb388167ef78f47a3a440eb651b6938c10f508 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Mon, 7 Sep 2015 15:22:23 -0700 Subject: Add chainstate obfuscation to avoid spurious antivirus detection Adds an `obfuscate` parameter to `CLevelDBWrapper` and makes use of it for all new chainstate stores built via `CCoinsViewDB`. Also adds an `Xor` method to `CDataStream`. Thanks to @sipa @laanwj @pstratem @dexX7 @KyrosKrane @gmaxwell. --- src/txdb.cpp | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) (limited to 'src/txdb.cpp') diff --git a/src/txdb.cpp b/src/txdb.cpp index 21ecd6523..9738dea03 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -29,18 +29,8 @@ static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; -void static BatchWriteCoins(CLevelDBBatch &batch, const uint256 &hash, const CCoins &coins) { - if (coins.IsPruned()) - batch.Erase(make_pair(DB_COINS, hash)); - else - batch.Write(make_pair(DB_COINS, hash), coins); -} - -void static BatchWriteHashBestChain(CLevelDBBatch &batch, const uint256 &hash) { - batch.Write(DB_BEST_BLOCK, hash); -} - -CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe) { +CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) +{ } bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const { @@ -59,12 +49,15 @@ uint256 CCoinsViewDB::GetBestBlock() const { } bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { - CLevelDBBatch batch; + CLevelDBBatch batch(db.GetObfuscateKey()); size_t count = 0; size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { - BatchWriteCoins(batch, it->first, it->second.coins); + if (it->second.coins.IsPruned()) + batch.Erase(make_pair(DB_COINS, it->first)); + else + batch.Write(make_pair(DB_COINS, it->first), it->second.coins); changed++; } count++; @@ -72,7 +65,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { mapCoins.erase(itOld); } if (!hashBlock.IsNull()) - BatchWriteHashBestChain(batch, hashBlock); + batch.Write(DB_BEST_BLOCK, hashBlock); LogPrint("coindb", "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); return db.WriteBatch(batch); @@ -158,7 +151,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { } bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { - CLevelDBBatch batch; + CLevelDBBatch batch(GetObfuscateKey()); for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); } @@ -174,7 +167,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { } bool CBlockTreeDB::WriteTxIndex(const std::vector >&vect) { - CLevelDBBatch batch; + CLevelDBBatch batch(GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) batch.Write(make_pair(DB_TXINDEX, it->first), it->second); return WriteBatch(batch); -- cgit v1.2.3 From 3499ce1e1ad87a86598d00b7124072c91ddad833 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 7 Oct 2015 17:12:24 -0700 Subject: Encapsulate CLevelDB iterators cleanly Conflicts: src/leveldb.cpp src/leveldb.h src/txdb.cpp --- src/txdb.cpp | 59 ++++++++++++++++++++--------------------------------------- 1 file changed, 20 insertions(+), 39 deletions(-) (limited to 'src/txdb.cpp') diff --git a/src/txdb.cpp b/src/txdb.cpp index 9738dea03..3d2050891 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -98,8 +98,8 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); - pcursor->SeekToFirst(); + boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); + pcursor->Seek('c'); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); stats.hashBlock = GetBestBlock(); @@ -107,22 +107,10 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { CAmount nTotalAmount = 0; while (pcursor->Valid()) { boost::this_thread::interruption_point(); - try { - leveldb::Slice slKey = pcursor->key(); - CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION); - char chType; - ssKey >> chType; - if (chType == DB_COINS) { - leveldb::Slice slValue = pcursor->value(); - CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION); - CCoins coins; - ssValue >> coins; - uint256 txhash; - ssKey >> txhash; - ss << txhash; - ss << VARINT(coins.nVersion); - ss << (coins.fCoinBase ? 'c' : 'n'); - ss << VARINT(coins.nHeight); + std::pair key; + CCoins coins; + if (pcursor->GetKey(key) && key.first == 'c') { + if (pcursor->GetValue(coins)) { stats.nTransactions++; for (unsigned int i=0; iGetKeySize(); ss << VARINT(0); + } else { + return error("CCoinsViewDB::GetStats() : unable to read value"); } - pcursor->Next(); - } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } else { + break; } + pcursor->Next(); } { LOCK(cs_main); @@ -189,24 +179,15 @@ bool CBlockTreeDB::LoadBlockIndexGuts() { boost::scoped_ptr pcursor(NewIterator()); - CDataStream ssKeySet(SER_DISK, CLIENT_VERSION); - ssKeySet << make_pair(DB_BLOCK_INDEX, uint256()); - pcursor->Seek(ssKeySet.str()); + pcursor->Seek(make_pair('b', uint256(0))); // Load mapBlockIndex while (pcursor->Valid()) { boost::this_thread::interruption_point(); - try { - leveldb::Slice slKey = pcursor->key(); - CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION); - char chType; - ssKey >> chType; - if (chType == DB_BLOCK_INDEX) { - leveldb::Slice slValue = pcursor->value(); - CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION); - CDiskBlockIndex diskindex; - ssValue >> diskindex; - + std::pair key; + if (pcursor->GetKey(key) && key.first == 'b') { + CDiskBlockIndex diskindex; + if (pcursor->GetValue(diskindex)) { // Construct block index object CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); @@ -227,10 +208,10 @@ bool CBlockTreeDB::LoadBlockIndexGuts() pcursor->Next(); } else { - break; // if shutdown requested or finished loading block index + return error("LoadBlockIndex() : failed to read value"); } - } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } else { + break; } } -- cgit v1.2.3 From 0fdf8c80ee322ab747321d61faf9c72af4a51445 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 8 Oct 2015 00:44:10 -0700 Subject: Handle obfuscation in CLevelDBIterator --- src/txdb.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/txdb.cpp') diff --git a/src/txdb.cpp b/src/txdb.cpp index 3d2050891..4d7ec27aa 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -98,7 +98,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); + boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); pcursor->Seek('c'); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); @@ -177,9 +177,9 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { bool CBlockTreeDB::LoadBlockIndexGuts() { - boost::scoped_ptr pcursor(NewIterator()); + boost::scoped_ptr pcursor(NewIterator()); - pcursor->Seek(make_pair('b', uint256(0))); + pcursor->Seek(make_pair('b', uint256())); // Load mapBlockIndex while (pcursor->Valid()) { -- cgit v1.2.3 From dcd8e27c65de0d2cb972588a6e811ca7ccd1b3bd Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Fri, 9 Oct 2015 10:55:27 -0700 Subject: Refer to obfuscate_key via pointer in peripheral CLevelDB classes cc @sipa --- src/txdb.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/txdb.cpp') diff --git a/src/txdb.cpp b/src/txdb.cpp index 4d7ec27aa..5723c9244 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -49,7 +49,7 @@ uint256 CCoinsViewDB::GetBestBlock() const { } bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { - CLevelDBBatch batch(db.GetObfuscateKey()); + CLevelDBBatch batch(&db.GetObfuscateKey()); size_t count = 0; size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { @@ -141,7 +141,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { } bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { - CLevelDBBatch batch(GetObfuscateKey()); + CLevelDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); } @@ -157,7 +157,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { } bool CBlockTreeDB::WriteTxIndex(const std::vector >&vect) { - CLevelDBBatch batch(GetObfuscateKey()); + CLevelDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) batch.Write(make_pair(DB_TXINDEX, it->first), it->second); return WriteBatch(batch); -- cgit v1.2.3 From f8f2aceadd5f406f8bc6f6029b032d00828e6256 Mon Sep 17 00:00:00 2001 From: Daniel Kraft Date: Tue, 13 Oct 2015 20:25:57 +0200 Subject: trivial: use constants for db keys Replace literal occurances of the key "prefixes" 'c' and 'b' in txdb.cpp by the respective constants. --- src/txdb.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/txdb.cpp') diff --git a/src/txdb.cpp b/src/txdb.cpp index 5723c9244..a441aea68 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -99,7 +99,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { only need read operations on it, use a const-cast to get around that restriction. */ boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); - pcursor->Seek('c'); + pcursor->Seek(DB_COINS); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); stats.hashBlock = GetBestBlock(); @@ -109,7 +109,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { boost::this_thread::interruption_point(); std::pair key; CCoins coins; - if (pcursor->GetKey(key) && key.first == 'c') { + if (pcursor->GetKey(key) && key.first == DB_COINS) { if (pcursor->GetValue(coins)) { stats.nTransactions++; for (unsigned int i=0; i pcursor(NewIterator()); - pcursor->Seek(make_pair('b', uint256())); + pcursor->Seek(make_pair(DB_BLOCK_INDEX, uint256())); // Load mapBlockIndex while (pcursor->Valid()) { boost::this_thread::interruption_point(); std::pair key; - if (pcursor->GetKey(key) && key.first == 'b') { + if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) { CDiskBlockIndex diskindex; if (pcursor->GetValue(diskindex)) { // Construct block index object -- cgit v1.2.3 From 8587b23038b88340ec64253ea4282afe90187a69 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Thu, 22 Oct 2015 21:02:20 -0400 Subject: leveldbwrapper symbol rename: Remove "Level" from class, etc. names --- src/txdb.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src/txdb.cpp') diff --git a/src/txdb.cpp b/src/txdb.cpp index a441aea68..f0868a1eb 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -49,7 +49,7 @@ uint256 CCoinsViewDB::GetBestBlock() const { } bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { - CLevelDBBatch batch(&db.GetObfuscateKey()); + CDBBatch batch(&db.GetObfuscateKey()); size_t count = 0; size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { @@ -71,7 +71,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return db.WriteBatch(batch); } -CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CLevelDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { +CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { } bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { @@ -98,7 +98,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); + boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); pcursor->Seek(DB_COINS); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); @@ -141,7 +141,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { } bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { - CLevelDBBatch batch(&GetObfuscateKey()); + CDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); } @@ -157,7 +157,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { } bool CBlockTreeDB::WriteTxIndex(const std::vector >&vect) { - CLevelDBBatch batch(&GetObfuscateKey()); + CDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) batch.Write(make_pair(DB_TXINDEX, it->first), it->second); return WriteBatch(batch); @@ -177,7 +177,7 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { bool CBlockTreeDB::LoadBlockIndexGuts() { - boost::scoped_ptr pcursor(NewIterator()); + boost::scoped_ptr pcursor(NewIterator()); pcursor->Seek(make_pair(DB_BLOCK_INDEX, uint256())); -- cgit v1.2.3 From 298e040bca0aad62b616a1d3e66e1e44d2b9bd3a Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 21 Oct 2015 23:23:59 +0200 Subject: Fix chainstate serialized_size computation --- src/txdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/txdb.cpp') diff --git a/src/txdb.cpp b/src/txdb.cpp index f0868a1eb..cd76c0155 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -121,7 +121,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { nTotalAmount += out.nValue; } } - stats.nSerializedSize += 32 + pcursor->GetKeySize(); + stats.nSerializedSize += 32 + pcursor->GetValueSize(); ss << VARINT(0); } else { return error("CCoinsViewDB::GetStats() : unable to read value"); -- cgit v1.2.3 From fa24439ff3d8ab5b9efaf66ef4dae6713b88cb35 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sun, 13 Dec 2015 17:58:29 +0100 Subject: Bump copyright headers to 2015 --- src/txdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/txdb.cpp') diff --git a/src/txdb.cpp b/src/txdb.cpp index cd76c0155..f99e11f26 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2009-2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -- cgit v1.2.3