Merge AuxPoW support from Namecore

Changes are as below:

Wrap CBlockHeader::nVersion into a new class (CBlockVersion).  This allows to take care of interpreting the field into a base version, auxpow flag and the chain ID.

Update getauxblock.py for new 'generate' RPC call.

Add 'auxpow' to block JSON.

Accept auxpow as PoW verification.

Add unit tests for auxpow verification.

Add check for memory-layout of CBlockVersion.

Weaken auxpow chain ID checks for the testnet.

Allow Params() to overrule when to check the auxpow chain ID and for legacy blocks.  Use this to disable the checks on testnet.

Introduce CPureBlockHeader.

Split the block header part that is used by auxpow and the "real" block header (that uses auxpow) to resolve the cyclic dependency between the two.

Differentiate between uint256 and arith_uint256.

This change was done upstream, modify the auxpow code.

Add missing lock in auxpow_tests.

Fix REST header check for auxpow headers.

Those can be longer, thus take that into account.  Also perform the check actually on an auxpow header.

Correctly set the coinbase for getauxblock results.

Call IncrementExtraNonce in getauxblock so that the coinbase is actually initialised with the stuff it should be.  (BIP30 block height and COINBASE_FLAGS.)

Implement getauxblock plus regression test.

Turn auxpow test into FIXTURE test.

This allows using of the Params() calls.

Move CMerkleTx code to auxpow.cpp.

Otherwise we get linker errors when building without wallet.

Fix rebase with BIP66.

Update the code to handle BIP66's nVersion=3.

Enforce that auxpow parent blocks have no auxpow block version.

This is for compatibility with namecoind.  See also https://github.com/namecoin/namecoin/pull/199.

Move auxpow-related parameters to Consensus::Params.

4 files changed, 389 insertions, 5 deletions

diff --git a/test/functional/auxpow_mining.py b/test/functional/auxpow_mining.py
new file mode 100755
index 000000000..bb4568772
--- /dev/null
+++ b/test/functional/auxpow_mining.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+# Copyright (c) 2014-2018 Daniel Kraft
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+# Test the merge-mining RPC interface:
+# getauxblock, createauxblock, submitauxblock
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import *
+
+from test_framework.auxpow import reverseHex
+from test_framework.auxpow_testing import (
+  computeAuxpow,
+  getCoinbaseAddr,
+  mineAuxpowBlockWithMethods,
+)
+
+class AuxpowMiningTest (BitcoinTestFramework):
+
+  def set_test_params (self):
+    self.num_nodes = 2
+
+  def add_options (self, parser):
+    parser.add_argument ("--segwit", dest="segwit", default=False,
+                         action="store_true",
+                         help="Test behaviour with SegWit active")
+
+  def run_test (self):
+    # Enable mock time to be out of IBD.
+    self.enable_mocktime ()
+
+    # Activate segwit if requested.
+    if self.options.segwit:
+      self.nodes[0].generate (500)
+      self.sync_all ()
+
+    # Test with getauxblock and createauxblock/submitauxblock.
+    self.test_getauxblock ()
+    self.test_create_submit_auxblock ()
+
+  def test_common (self, create, submit):
+    """
+    Common test code that is shared between the tests for getauxblock and the
+    createauxblock / submitauxblock method pair.
+    """
+
+    # Verify data that can be found in another way.
+    auxblock = create ()
+    assert_equal (auxblock['chainid'], 1)
+    assert_equal (auxblock['height'], self.nodes[0].getblockcount () + 1)
+    assert_equal (auxblock['previousblockhash'],
+                  self.nodes[0].getblockhash (auxblock['height'] - 1))
+
+    # Calling again should give the same block.
+    auxblock2 = create ()
+    assert_equal (auxblock2, auxblock)
+
+    # If we receive a new block, the old hash will be replaced.
+    self.sync_all ()
+    self.nodes[1].generate (1)
+    self.sync_all ()
+    auxblock2 = create ()
+    assert auxblock['hash'] != auxblock2['hash']
+    assert_raises_rpc_error (-8, 'block hash unknown', submit,
+                             auxblock['hash'], "x")
+
+    # Invalid format for auxpow.
+    assert_raises_rpc_error (-1, None, submit,
+                             auxblock2['hash'], "x")
+
+    # Invalidate the block again, send a transaction and query for the
+    # auxblock to solve that contains the transaction.
+    self.nodes[0].generate (1)
+    addr = self.nodes[1].getnewaddress ()
+    txid = self.nodes[0].sendtoaddress (addr, 1)
+    self.sync_all ()
+    assert_equal (self.nodes[1].getrawmempool (), [txid])
+    auxblock = create ()
+    target = reverseHex (auxblock['_target'])
+
+    # Compute invalid auxpow.
+    apow = computeAuxpow (auxblock['hash'], target, False)
+    res = submit (auxblock['hash'], apow)
+    assert not res
+
+    # Compute and submit valid auxpow.
+    apow = computeAuxpow (auxblock['hash'], target, True)
+    res = submit (auxblock['hash'], apow)
+    assert res
+
+    # Make sure that the block is indeed accepted.
+    self.sync_all ()
+    assert_equal (self.nodes[1].getrawmempool (), [])
+    height = self.nodes[1].getblockcount ()
+    assert_equal (height, auxblock['height'])
+    assert_equal (self.nodes[1].getblockhash (height), auxblock['hash'])
+
+    # Call getblock and verify the auxpow field.
+    data = self.nodes[1].getblock (auxblock['hash'])
+    assert 'auxpow' in data
+    auxJson = data['auxpow']
+    assert_equal (auxJson['index'], 0)
+    assert_equal (auxJson['chainindex'], 0)
+    assert_equal (auxJson['merklebranch'], [])
+    assert_equal (auxJson['chainmerklebranch'], [])
+    assert_equal (auxJson['parentblock'], apow[-160:])
+
+    # Also previous blocks should have 'auxpow', since all blocks (also
+    # those generated by "generate") are merge-mined.
+    oldHash = self.nodes[1].getblockhash (100)
+    data = self.nodes[1].getblock (oldHash)
+    assert 'auxpow' in data
+
+    # Check that it paid correctly to the first node.
+    t = self.nodes[0].listtransactions ("*", 1)
+    assert_equal (len (t), 1)
+    t = t[0]
+    assert_equal (t['category'], "immature")
+    assert_equal (t['blockhash'], auxblock['hash'])
+    assert t['generated']
+    assert_greater_than_or_equal (t['amount'], Decimal ("1"))
+    assert_equal (t['confirmations'], 1)
+
+    # Verify the coinbase script.  Ensure that it includes the block height
+    # to make the coinbase tx unique.  The expected block height is around
+    # 200, so that the serialisation of the CScriptNum ends in an extra 00.
+    # The vector has length 2, which makes up for 02XX00 as the serialised
+    # height.  Check this.  (With segwit, the height is different, so we skip
+    # this for simplicity.)
+    if not self.options.segwit:
+      blk = self.nodes[1].getblock (auxblock['hash'])
+      tx = self.nodes[1].getrawtransaction (blk['tx'][0], 1)
+      coinbase = tx['vin'][0]['coinbase']
+      assert_equal ("02%02x00" % auxblock['height'], coinbase[0 : 6])
+
+  def test_getauxblock (self):
+    """
+    Test the getauxblock method.
+    """
+
+    create = self.nodes[0].getauxblock
+    submit = self.nodes[0].getauxblock
+    self.test_common (create, submit)
+
+    # Ensure that the payout address is changed from one block to the next.
+    hash1 = mineAuxpowBlockWithMethods (create, submit)
+    hash2 = mineAuxpowBlockWithMethods (create, submit)
+    self.sync_all ()
+    addr1 = getCoinbaseAddr (self.nodes[1], hash1)
+    addr2 = getCoinbaseAddr (self.nodes[1], hash2)
+    assert addr1 != addr2
+    info = self.nodes[0].getaddressinfo (addr1)
+    assert info['ismine']
+    info = self.nodes[0].getaddressinfo (addr2)
+    assert info['ismine']
+
+  def test_create_submit_auxblock (self):
+    """
+    Test the createauxblock / submitauxblock method pair.
+    """
+
+    # Check for errors with wrong parameters.
+    assert_raises_rpc_error (-1, None, self.nodes[0].createauxblock)
+    assert_raises_rpc_error (-5, "Invalid coinbase payout address",
+                             self.nodes[0].createauxblock,
+                             "this_an_invalid_address")
+
+    # Fix a coinbase address and construct methods for it.
+    coinbaseAddr = self.nodes[0].getnewaddress ()
+    def create ():
+      return self.nodes[0].createauxblock (coinbaseAddr)
+    submit = self.nodes[0].submitauxblock
+
+    # Run common tests.
+    self.test_common (create, submit)
+
+    # Ensure that the payout address is the one which we specify
+    hash1 = mineAuxpowBlockWithMethods (create, submit)
+    hash2 = mineAuxpowBlockWithMethods (create, submit)
+    self.sync_all ()
+    addr1 = getCoinbaseAddr (self.nodes[1], hash1)
+    addr2 = getCoinbaseAddr (self.nodes[1], hash2)
+    assert_equal (addr1, coinbaseAddr)
+    assert_equal (addr2, coinbaseAddr)
+
+if __name__ == '__main__':
+  AuxpowMiningTest ().main ()
diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py
index afa9de580..2ae7b7c94 100755
--- a/test/functional/interface_rest.py
+++ b/test/functional/interface_rest.py
@@ -14,6 +14,7 @@ from struct import pack, unpack
 import http.client
 import urllib.parse
 
+from test_framework import auxpow
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
@@ -175,7 +176,13 @@ class RESTTest (BitcoinTestFramework):
         assert_equal(len(json_obj['utxos']), 0)
 
         self.nodes[0].generate(1)
+
+        json_request = json_request.rstrip("/")
+        response = http_post_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True)
+        assert_equal(response.status, 200) #must be a 200 because we are within the limits
+
         self.sync_all()
+        bb_hash = self.nodes[0].getbestblockhash()
 
         json_obj = self.test_rest_request("/getutxos/{}-{}".format(*spending))
         assert_equal(len(json_obj['utxos']), 1)
@@ -195,7 +202,7 @@ class RESTTest (BitcoinTestFramework):
         long_uri = '/'.join(['{}-{}'.format(txid, n_) for n_ in range(15)])
         self.test_rest_request("/getutxos/checkmempool/{}".format(long_uri), http_method='POST', status=200)
 
-        self.nodes[0].generate(1)  # generate block to not affect upcoming tests
+        mineAuxpowBlock(self.nodes[0])  # generate block to not affect upcoming tests
         self.sync_all()
 
         self.log.info("Test the /block and /headers URIs")
@@ -208,9 +215,10 @@ class RESTTest (BitcoinTestFramework):
 
         # Compare with block header
         response_header = self.test_rest_request("/headers/1/{}".format(bb_hash), req_type=ReqType.BIN, ret_type=RetType.OBJ)
-        assert_equal(int(response_header.getheader('content-length')), 80)
+        headerLen = int(response_header.getheader('content-length'))
+        assert_greater_than(headerLen, 80)
         response_header_bytes = response_header.read()
-        assert_equal(response_bytes[:80], response_header_bytes)
+        assert_equal(response_bytes[:headerLen], response_header_bytes)
 
         # Check block hex format
         response_hex = self.test_rest_request("/block/{}".format(bb_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ)
@@ -221,8 +229,10 @@ class RESTTest (BitcoinTestFramework):
         # Compare with hex block header
         response_header_hex = self.test_rest_request("/headers/1/{}".format(bb_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ)
         assert_greater_than(int(response_header_hex.getheader('content-length')), 160)
-        response_header_hex_bytes = response_header_hex.read(160)
-        assert_equal(binascii.hexlify(response_bytes[:80]), response_header_hex_bytes)
+        response_header_hex_bytes = response_header_hex.read().strip()
+        headerLen = len (response_header_hex_bytes)
+        assert_equal(binascii.hexlify(response_bytes[:headerLen]), response_header_hex_bytes)
+        assert_equal(encode(response_header_bytes, "hex_codec"), response_header_hex_bytes)
 
         # Check json format
         block_json_obj = self.test_rest_request("/block/{}".format(bb_hash))
diff --git a/test/functional/test_framework/auxpow.py b/test/functional/test_framework/auxpow.py
new file mode 100644
index 000000000..e145e7ce7
--- /dev/null
+++ b/test/functional/test_framework/auxpow.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+# Copyright (c) 2014-2018 Daniel Kraft
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+# Basic code for working with auxpow.  This is used for the regtests (e.g. from
+# auxpow_testing.py), but also for contrib/auxpow/getwork-wrapper.py.
+
+import binascii
+import codecs
+import hashlib
+
+def constructAuxpow (block):
+  """
+  Starts to construct a minimal auxpow, ready to be mined.  Returns the fake
+  coinbase tx and the unmined parent block header as hex strings.
+  """
+
+  block = codecs.encode (block, 'ascii')
+
+  # Start by building the merge-mining coinbase.  The merkle tree
+  # consists only of the block hash as root.
+  coinbase = b"fabe" + binascii.hexlify (b"m" * 2)
+  coinbase += block
+  coinbase += b"01000000" + (b"00" * 4)
+
+  # Construct "vector" of transaction inputs.
+  vin = b"01"
+  vin += (b"00" * 32) + (b"ff" * 4)
+  vin += codecs.encode ("%02x" % (len (coinbase) // 2), "ascii") + coinbase
+  vin += (b"ff" * 4)
+
+  # Build up the full coinbase transaction.  It consists only
+  # of the input and has no outputs.
+  tx = b"01000000" + vin + b"00" + (b"00" * 4)
+  txHash = doubleHashHex (tx)
+
+  # Construct the parent block header.  It need not be valid, just good
+  # enough for auxpow purposes.
+  header = b"01000000"
+  header += b"00" * 32
+  header += reverseHex (txHash)
+  header += b"00" * 4
+  header += b"00" * 4
+  header += b"00" * 4
+
+  return (tx.decode ('ascii'), header.decode ('ascii'))
+
+def finishAuxpow (tx, header):
+  """
+  Constructs the finished auxpow hex string based on the mined header.
+  """
+
+  blockhash = doubleHashHex (header)
+
+  # Build the MerkleTx part of the auxpow.
+  auxpow = codecs.encode (tx, 'ascii')
+  auxpow += blockhash
+  auxpow += b"00"
+  auxpow += b"00" * 4
+
+  # Extend to full auxpow.
+  auxpow += b"00"
+  auxpow += b"00" * 4
+  auxpow += header
+
+  return auxpow.decode ("ascii")
+
+def doubleHashHex (data):
+  """
+  Perform Bitcoin's Double-SHA256 hash on the given hex string.
+  """
+
+  hasher = hashlib.sha256 ()
+  hasher.update (binascii.unhexlify (data))
+  data = hasher.digest ()
+
+  hasher = hashlib.sha256 ()
+  hasher.update (data)
+
+  return reverseHex (hasher.hexdigest ())
+
+def reverseHex (data):
+  """
+  Flip byte order in the given data (hex string).
+  """
+
+  b = bytearray (binascii.unhexlify (data))
+  b.reverse ()
+
+  return binascii.hexlify (b)
+
+def getworkByteswap (data):
+  """
+  Run the byte-order swapping step necessary for working with getwork.
+  """
+
+  data = bytearray (data)
+  assert len (data) % 4 == 0
+  for i in range (0, len (data), 4):
+    data[i], data[i + 3] = data[i + 3], data[i]
+    data[i + 1], data[i + 2] = data[i + 2], data[i + 1]
+
+  return data
diff --git a/test/functional/test_framework/auxpow_testing.py b/test/functional/test_framework/auxpow_testing.py
new file mode 100644
index 000000000..f1ef056b8
--- /dev/null
+++ b/test/functional/test_framework/auxpow_testing.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+# Copyright (c) 2014-2018 Daniel Kraft
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+# Utility routines for auxpow that are needed specifically by the regtests.
+# This is mostly about actually *solving* an auxpow block (with regtest
+# difficulty) or inspecting the information for verification.
+
+import binascii
+
+from test_framework import auxpow
+
+def computeAuxpow (block, target, ok):
+  """
+  Build an auxpow object (serialised as hex string) that solves
+  (ok = True) or doesn't solve (ok = False) the block.
+  """
+
+  (tx, header) = auxpow.constructAuxpow (block)
+  (header, _) = mineBlock (header, target, ok)
+  return auxpow.finishAuxpow (tx, header)
+
+def mineAuxpowBlock (node):
+  """
+  Mine an auxpow block on the given RPC connection.  This uses the
+  createauxblock and submitauxblock command pair.
+  """
+
+  def create ():
+    addr = node.getnewaddress ()
+    return node.createauxblock (addr)
+
+  return mineAuxpowBlockWithMethods (create, node.submitauxblock)
+
+def mineAuxpowBlockWithMethods (create, submit):
+  """
+  Mine an auxpow block, using the given methods for creation and submission.
+  """
+
+  auxblock = create ()
+  target = auxpow.reverseHex (auxblock['_target'])
+  apow = computeAuxpow (auxblock['hash'], target, True)
+  res = submit (auxblock['hash'], apow)
+  assert res
+
+  return auxblock['hash']
+
+def getCoinbaseAddr (node, blockHash):
+    """
+    Extract the coinbase tx' payout address for the given block.
+    """
+
+    blockData = node.getblock (blockHash)
+    txn = blockData['tx']
+    assert len (txn) >= 1
+
+    txData = node.getrawtransaction (txn[0], 1)
+    assert len (txData['vout']) >= 1 and len (txData['vin']) == 1
+    assert 'coinbase' in txData['vin'][0]
+
+    addr = txData['vout'][0]['scriptPubKey']['addresses']
+    assert len (addr) == 1
+    return addr[0]
+
+def mineBlock (header, target, ok):
+  """
+  Given a block header, update the nonce until it is ok (or not)
+  for the given target.
+  """
+
+  data = bytearray (binascii.unhexlify (header))
+  while True:
+    assert data[79] < 255
+    data[79] += 1
+    hexData = binascii.hexlify (data)
+
+    blockhash = auxpow.doubleHashHex (hexData)
+    if (ok and blockhash < target) or ((not ok) and blockhash > target):
+      break
+
+  return (hexData, blockhash)