Merge branch 'master' into 20150703_banlist_updates

8 files changed, 447 insertions, 76 deletions

diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md
index f90afa7f2..240ab6d9e 100644
--- a/contrib/devtools/README.md
+++ b/contrib/devtools/README.md
@@ -1,9 +1,45 @@
 Contents
-===========
+========
 This directory contains tools for developers working on this repository.
 
+clang-format.py
+===============
+
+A script to format cpp source code according to [.clang-format](../../src/.clang-format). This should only be applied to new files or files which are currently not actively developed on. Also, git subtrees are not subject to formatting.
+
+fix-copyright-headers.py
+========================
+
+Every year newly updated files need to have its copyright headers updated to reflect the current year.
+If you run this script from the root folder it will automatically update the year on the copyright header for all
+source files if these have a git commit from the current year.
+
+For example a file changed in 2015 (with 2015 being the current year):
+
+```// Copyright (c) 2009-2013 The Bitcoin Core developers```
+
+would be changed to:
+
+```// Copyright (c) 2009-2015 The Bitcoin Core developers```
+
+git-subtree-check.sh
+====================
+
+Run this script from the root of the repository to verify that a subtree matches the contents of
+the commit it claims to have been updated to.
+
+To use, make sure that you have fetched the upstream repository branch in which the subtree is
+maintained:
+* for `src/secp256k1`: https://github.com/bitcoin/secp256k1.git (branch master)
+* for `src/leveldb`: https://github.com/bitcoin/leveldb.git (branch bitcoin-fork)
+* for `src/univalue`: https://github.com/bitcoin/univalue.git (branch master)
+
+Usage: `git-subtree-check.sh DIR COMMIT`
+
+`COMMIT` may be omitted, in which case `HEAD` is used.
+
 github-merge.sh
-==================
+===============
 
 A small script to automate merging pull-requests securely and sign them with GPG.
 
@@ -36,24 +72,22 @@ Configuring the github-merge tool for the bitcoin repository is done in the foll
     git config githubmerge.testcmd "make -j4 check" (adapt to whatever you want to use for testing)
     git config --global user.signingkey mykeyid (if you want to GPG sign)
 
-fix-copyright-headers.py
-===========================
+optimize-pngs.py
+================
 
-Every year newly updated files need to have its copyright headers updated to reflect the current year.
-If you run this script from src/ it will automatically update the year on the copyright header for all
-.cpp and .h files if these have a git commit from the current year.
+A script to optimize png files in the bitcoin
+repository (requires pngcrush).
 
-For example a file changed in 2014 (with 2014 being the current year):
-```// Copyright (c) 2009-2013 The Bitcoin Core developers```
+security-check.py and test-security-check.py
+============================================
 
-would be changed to:
-```// Copyright (c) 2009-2014 The Bitcoin Core developers```
+Perform basic ELF security checks on a series of executables.
 
 symbol-check.py
-==================
+===============
 
 A script to check that the (Linux) executables produced by gitian only contain
-allowed gcc, glibc and libstdc++ version symbols.  This makes sure they are
+allowed gcc, glibc and libstdc++ version symbols. This makes sure they are
 still compatible with the minimum supported Linux distribution versions.
 
 Example usage after a gitian build:
@@ -70,7 +104,7 @@ If there are 'unsupported' symbols, the return value will be 1 a list like this
     .../64/test_bitcoin: symbol _ZNSt8__detail15_List_nod from unsupported version GLIBCXX_3.4.15
 
 update-translations.py
-=======================
+======================
 
 Run this script from the root of the repository to update all translations from transifex.
 It will do the following automatically:
@@ -80,17 +114,3 @@ It will do the following automatically:
 - add missing translations to the build system (TODO)
 
 See doc/translation-process.md for more information.
-
-git-subtree-check.sh
-====================
-
-Run this script from the root of the repository to verify that a subtree matches the contents of
-the commit it claims to have been updated to.
-
-To use, make sure that you have fetched the upstream repository branch in which the subtree is
-maintained:
-* for src/secp256k1: https://github.com/bitcoin/secp256k1.git (branch master)
-* for sec/leveldb: https://github.com/bitcoin/leveldb.git (branch bitcoin-fork)
-
-Usage: git-subtree-check.sh DIR COMMIT
-COMMIT may be omitted, in which case HEAD is used.
diff --git a/contrib/devtools/clang-format.py b/contrib/devtools/clang-format.py
new file mode 100755
index 000000000..cee99047a
--- /dev/null
+++ b/contrib/devtools/clang-format.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+'''
+Wrapper script for clang-format
+
+Copyright (c) 2015 MarcoFalke
+Copyright (c) 2015 The Bitcoin Core developers
+Distributed under the MIT software license, see the accompanying
+file COPYING or http://www.opensource.org/licenses/mit-license.php.
+'''
+
+import os
+import sys
+import subprocess
+
+tested_versions = ['3.6.0', '3.6.1', '3.6.2'] # A set of versions known to produce the same output
+accepted_file_extensions = ('.h', '.cpp') # Files to format
+
+def check_clang_format_version(clang_format_exe):
+    try:
+        output = subprocess.check_output([clang_format_exe, '-version'])
+        for ver in tested_versions:
+            if ver in output:
+                print "Detected clang-format version " + ver
+                return
+        raise RuntimeError("Untested version: " + output)
+    except Exception as e:
+        print 'Could not verify version of ' + clang_format_exe + '.'
+        raise e
+
+def check_command_line_args(argv):
+    required_args = ['{clang-format-exe}', '{files}']
+    example_args = ['clang-format-3.x', 'src/main.cpp', 'src/wallet/*']
+
+    if(len(argv) < len(required_args) + 1):
+        for word in (['Usage:', argv[0]] + required_args):
+            print word,
+        print ''
+        for word in (['E.g:', argv[0]] + example_args):
+            print word,
+        print ''
+        sys.exit(1)
+
+def run_clang_format(clang_format_exe, files):
+    for target in files:
+        if os.path.isdir(target):
+            for path, dirs, files in os.walk(target):
+                run_clang_format(clang_format_exe, (os.path.join(path, f) for f in files))
+        elif target.endswith(accepted_file_extensions):
+            print "Format " + target
+            subprocess.check_call([clang_format_exe, '-i', '-style=file', target], stdout=open(os.devnull, 'wb'), stderr=subprocess.STDOUT)
+        else:
+            print "Skip " + target
+
+def main(argv):
+    check_command_line_args(argv)
+    clang_format_exe = argv[1]
+    files = argv[2:]
+    check_clang_format_version(clang_format_exe)
+    run_clang_format(clang_format_exe, files)
+
+if __name__ == "__main__":
+    main(sys.argv)
diff --git a/contrib/devtools/fix-copyright-headers.py b/contrib/devtools/fix-copyright-headers.py
index 5e8495254..b6414a551 100755
--- a/contrib/devtools/fix-copyright-headers.py
+++ b/contrib/devtools/fix-copyright-headers.py
@@ -1,53 +1,46 @@
 #!/usr/bin/env python
 '''
-Run this script inside of src/ and it will look for all the files
-that were changed this year that still have the last year in the
-copyright headers, and it will fix the headers on that file using
-a perl regex one liner.
+Run this script to update all the copyright headers of files
+that were changed this year.
 
-For example: if it finds something like this and we're in 2014
+For example:
 
-// Copyright (c) 2009-2013 The Bitcoin Core developers
+// Copyright (c) 2009-2012 The Bitcoin Core developers
 
 it will change it to
 
-// Copyright (c) 2009-2014 The Bitcoin Core developers
-
-It will do this for all the files in the folder and its children.
-
-Author: @gubatron
+// Copyright (c) 2009-2015 The Bitcoin Core developers
 '''
 import os
 import time
+import re
 
 year = time.gmtime()[0]
-last_year = year - 1
-command = "perl -pi -e 's/%s The Bitcoin/%s The Bitcoin/' %s"
-listFilesCommand = "find . | grep %s"
-
-extensions = [".cpp",".h"]
-
-def getLastGitModifiedDate(filePath):
-  gitGetLastCommitDateCommand = "git log " + filePath +" | grep Date | head -n 1"
-  p = os.popen(gitGetLastCommitDateCommand)
-  result = ""
-  for l in p:
-    result = l
-    break
-  result = result.replace("\n","")
-  return result
+CMD_GIT_DATE = 'git log --format=@%%at -1 %s | date +"%%Y" -u -f -'
+CMD_REGEX= "perl -pi -e 's/(20\d\d)(?:-20\d\d)? The Bitcoin/$1-%s The Bitcoin/' %s"
+REGEX_CURRENT= re.compile("%s The Bitcoin" % year)
+CMD_LIST_FILES= "find %s | grep %s"
 
-n=1
-for extension in extensions:
-  foundFiles = os.popen(listFilesCommand % extension)
-  for filePath in foundFiles:
-    filePath = filePath[1:-1]
-    if filePath.endswith(extension):
-      filePath = os.getcwd() + filePath
-      modifiedTime = getLastGitModifiedDate(filePath)
-      if len(modifiedTime) > 0 and str(year) in modifiedTime:
-        print n,"Last Git Modified: ", modifiedTime, " - ", filePath
-        os.popen(command % (last_year,year,filePath))
-        n = n + 1
+FOLDERS = ["./qa", "./src"]
+EXTENSIONS = [".cpp",".h", ".py"]
 
+def get_git_date(file_path):
+  r = os.popen(CMD_GIT_DATE % file_path)
+  for l in r:
+    # Result is one line, so just return
+    return l.replace("\n","")
+  return ""
 
+n=1
+for folder in FOLDERS:
+  for extension in EXTENSIONS:
+    for file_path in os.popen(CMD_LIST_FILES % (folder, extension)):
+      file_path = os.getcwd() + file_path[1:-1]
+      if file_path.endswith(extension):
+        git_date = get_git_date(file_path)
+        if str(year) == git_date:
+          # Only update if current year is not found
+          if REGEX_CURRENT.search(open(file_path, "r").read()) is None:
+            print n,"Last git edit", git_date, "-", file_path
+            os.popen(CMD_REGEX % (year,file_path))
+            n = n + 1
diff --git a/contrib/devtools/optimize-pngs.py b/contrib/devtools/optimize-pngs.py
index b6d6a097d..799e0cc7d 100755
--- a/contrib/devtools/optimize-pngs.py
+++ b/contrib/devtools/optimize-pngs.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 '''
-Run this scrip every time you change one of the png files. Using pngcrush, it will optimize the png files, remove various color profiles, remove ancillary chunks (alla) and text chunks (text).
+Run this script every time you change one of the png files. Using pngcrush, it will optimize the png files, remove various color profiles, remove ancillary chunks (alla) and text chunks (text).
 #pngcrush -brute -ow -rem gAMA -rem cHRM -rem iCCP -rem sRGB -rem alla -rem text
 '''
 import os
@@ -18,12 +18,12 @@ def content_hash(filename):
     '''Return hash of RGBA contents of image'''
     i = Image.open(filename)
     i = i.convert('RGBA')
-    data = i.tostring()
+    data = i.tobytes()
     return hashlib.sha256(data).hexdigest()
 
 pngcrush = 'pngcrush'
 git = 'git'
-folders = ["src/qt/res/movies", "src/qt/res/icons"]
+folders = ["src/qt/res/movies", "src/qt/res/icons", "share/pixmaps"]
 basePath = subprocess.check_output([git, 'rev-parse', '--show-toplevel']).rstrip('\n')
 totalSaveBytes = 0
 noHashChange = True
diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py
new file mode 100755
index 000000000..e96eaa9c3
--- /dev/null
+++ b/contrib/devtools/security-check.py
@@ -0,0 +1,181 @@
+#!/usr/bin/python2
+'''
+Perform basic ELF security checks on a series of executables.
+Exit status will be 0 if succesful, and the program will be silent.
+Otherwise the exit status will be 1 and it will log which executables failed which checks.
+Needs `readelf` (for ELF) and `objdump` (for PE).
+'''
+from __future__ import division,print_function
+import subprocess
+import sys
+import os
+
+READELF_CMD = os.getenv('READELF', '/usr/bin/readelf')
+OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump')
+
+def check_ELF_PIE(executable):
+    '''
+    Check for position independent executable (PIE), allowing for address space randomization.
+    '''
+    p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+    (stdout, stderr) = p.communicate()
+    if p.returncode:
+        raise IOError('Error opening file')
+
+    ok = False
+    for line in stdout.split('\n'):
+        line = line.split()
+        if len(line)>=2 and line[0] == 'Type:' and line[1] == 'DYN':
+            ok = True
+    return ok
+
+def get_ELF_program_headers(executable):
+    '''Return type and flags for ELF program headers'''
+    p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+    (stdout, stderr) = p.communicate()
+    if p.returncode:
+        raise IOError('Error opening file')
+    in_headers = False
+    count = 0
+    headers = []
+    for line in stdout.split('\n'):
+        if line.startswith('Program Headers:'):
+            in_headers = True
+        if line == '':
+            in_headers = False
+        if in_headers:
+            if count == 1: # header line
+                ofs_typ = line.find('Type')
+                ofs_offset = line.find('Offset')
+                ofs_flags = line.find('Flg')
+                ofs_align = line.find('Align')
+                if ofs_typ == -1 or ofs_offset == -1 or ofs_flags == -1 or ofs_align  == -1:
+                    raise ValueError('Cannot parse elfread -lW output')
+            elif count > 1:
+                typ = line[ofs_typ:ofs_offset].rstrip()
+                flags = line[ofs_flags:ofs_align].rstrip()
+                headers.append((typ, flags))
+            count += 1
+    return headers
+
+def check_ELF_NX(executable):
+    '''
+    Check that no sections are writable and executable (including the stack)
+    '''
+    have_wx = False
+    have_gnu_stack = False
+    for (typ, flags) in get_ELF_program_headers(executable):
+        if typ == 'GNU_STACK':
+            have_gnu_stack = True
+        if 'W' in flags and 'E' in flags: # section is both writable and executable
+            have_wx = True
+    return have_gnu_stack and not have_wx
+
+def check_ELF_RELRO(executable):
+    '''
+    Check for read-only relocations.
+    GNU_RELRO program header must exist
+    Dynamic section must have BIND_NOW flag
+    '''
+    have_gnu_relro = False
+    for (typ, flags) in get_ELF_program_headers(executable):
+        # Note: not checking flags == 'R': here as linkers set the permission differently
+        # This does not affect security: the permission flags of the GNU_RELRO program header are ignored, the PT_LOAD header determines the effective permissions.
+        # However, the dynamic linker need to write to this area so these are RW.
+        # Glibc itself takes care of mprotecting this area R after relocations are finished.
+        # See also http://permalink.gmane.org/gmane.comp.gnu.binutils/71347
+        if typ == 'GNU_RELRO':
+            have_gnu_relro = True
+
+    have_bindnow = False
+    p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+    (stdout, stderr) = p.communicate()
+    if p.returncode:
+        raise IOError('Error opening file')
+    for line in stdout.split('\n'):
+        tokens = line.split()
+        if len(tokens)>1 and tokens[1] == '(BIND_NOW)':
+            have_bindnow = True
+    return have_gnu_relro and have_bindnow
+
+def check_ELF_Canary(executable):
+    '''
+    Check for use of stack canary
+    '''
+    p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+    (stdout, stderr) = p.communicate()
+    if p.returncode:
+        raise IOError('Error opening file')
+    ok = False
+    for line in stdout.split('\n'):
+        if '__stack_chk_fail' in line:
+            ok = True
+    return ok
+
+def get_PE_dll_characteristics(executable):
+    '''
+    Get PE DllCharacteristics bits
+    '''
+    p = subprocess.Popen([OBJDUMP_CMD, '-x',  executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+    (stdout, stderr) = p.communicate()
+    if p.returncode:
+        raise IOError('Error opening file')
+    for line in stdout.split('\n'):
+        tokens = line.split()
+        if len(tokens)>=2 and tokens[0] == 'DllCharacteristics':
+            return int(tokens[1],16)
+    return 0
+
+
+def check_PE_PIE(executable):
+    '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)'''
+    return bool(get_PE_dll_characteristics(executable) & 0x40)
+
+def check_PE_NX(executable):
+    '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)'''
+    return bool(get_PE_dll_characteristics(executable) & 0x100)
+
+CHECKS = {
+'ELF': [
+    ('PIE', check_ELF_PIE),
+    ('NX', check_ELF_NX),
+    ('RELRO', check_ELF_RELRO),
+    ('Canary', check_ELF_Canary)
+],
+'PE': [
+    ('PIE', check_PE_PIE),
+    ('NX', check_PE_NX)
+]
+}
+
+def identify_executable(executable):
+    with open(filename, 'rb') as f:
+        magic = f.read(4)
+    if magic.startswith(b'MZ'):
+        return 'PE'
+    elif magic.startswith(b'\x7fELF'):
+        return 'ELF'
+    return None
+
+if __name__ == '__main__':
+    retval = 0
+    for filename in sys.argv[1:]:
+        try:
+            etype = identify_executable(filename)
+            if etype is None:
+                print('%s: unknown format' % filename)
+                retval = 1
+                continue
+
+            failed = []
+            for (name, func) in CHECKS[etype]:
+                if not func(filename):
+                    failed.append(name)
+            if failed:
+                print('%s: failed %s' % (filename, ' '.join(failed)))
+                retval = 1
+        except IOError:
+            print('%s: cannot open' % filename)
+            retval = 1
+    exit(retval)
+
diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py
index fad891f80..93acfcdda 100755
--- a/contrib/devtools/symbol-check.py
+++ b/contrib/devtools/symbol-check.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 # Copyright (c) 2014 Wladimir J. van der Laan
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
@@ -15,6 +15,7 @@ from __future__ import division, print_function
 import subprocess
 import re
 import sys
+import os
 
 # Debian 6.0.9 (Squeeze) has:
 #
@@ -45,8 +46,27 @@ MAX_VERSIONS = {
 IGNORE_EXPORTS = {
 '_edata', '_end', '_init', '__bss_start', '_fini'
 }
-READELF_CMD = '/usr/bin/readelf'
-CPPFILT_CMD = '/usr/bin/c++filt'
+READELF_CMD = os.getenv('READELF', '/usr/bin/readelf')
+CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt')
+# Allowed NEEDED libraries
+ALLOWED_LIBRARIES = {
+# bitcoind and bitcoin-qt
+'libgcc_s.so.1', # GCC base support
+'libc.so.6', # C library
+'libpthread.so.0', # threading
+'libanl.so.1', # DNS resolve
+'libm.so.6', # math library
+'librt.so.1', # real-time (clock)
+'ld-linux-x86-64.so.2', # 64-bit dynamic linker
+'ld-linux.so.2', # 32-bit dynamic linker
+# bitcoin-qt only
+'libX11-xcb.so.1', # part of X11
+'libX11.so.6', # part of X11
+'libxcb.so.1', # part of X11
+'libfontconfig.so.1', # font support
+'libfreetype.so.6', # font parsing
+'libdl.so.2' # programming interface to dynamic linker
+}
 
 class CPPFilt(object):
     '''
@@ -98,6 +118,22 @@ def check_version(max_versions, version):
         return False
     return ver <= max_versions[lib]
 
+def read_libraries(filename):
+    p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+    (stdout, stderr) = p.communicate()
+    if p.returncode:
+        raise IOError('Error opening file')
+    libraries = []
+    for line in stdout.split('\n'):
+        tokens = line.split()
+        if len(tokens)>2 and tokens[1] == '(NEEDED)':
+            match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:]))
+            if match:
+                libraries.append(match.group(1))
+            else:
+                raise ValueError('Unparseable (NEEDED) specification')
+    return libraries
+
 if __name__ == '__main__':
     cppfilt = CPPFilt()
     retval = 0
@@ -113,6 +149,11 @@ if __name__ == '__main__':
                 continue
             print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym)))
             retval = 1
+        # Check dependency libraries
+        for library_name in read_libraries(filename):
+            if library_name not in ALLOWED_LIBRARIES:
+                print('%s: NEEDED library %s is not allowed' % (filename, library_name))
+                retval = 1
 
     exit(retval)
 
diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py
new file mode 100755
index 000000000..fed7626aa
--- /dev/null
+++ b/contrib/devtools/test-security-check.py
@@ -0,0 +1,60 @@
+#!/usr/bin/python2
+'''
+Test script for security-check.py
+'''
+from __future__ import division,print_function
+import subprocess
+import sys
+import unittest
+
+def write_testcode(filename):
+    with open(filename, 'w') as f:
+        f.write('''
+    #include <stdio.h>
+    int main()
+    {
+        printf("the quick brown fox jumps over the lazy god\\n");
+        return 0;
+    }
+    ''')
+
+def call_security_check(cc, source, executable, options):
+    subprocess.check_call([cc,source,'-o',executable] + options)
+    p = subprocess.Popen(['./security-check.py',executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+    (stdout, stderr) = p.communicate()
+    return (p.returncode, stdout.rstrip())
+
+class TestSecurityChecks(unittest.TestCase):
+    def test_ELF(self):
+        source = 'test1.c'
+        executable = 'test1'
+        cc = 'gcc'
+        write_testcode(source)
+
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro']), 
+                (1, executable+': failed PIE NX RELRO Canary'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro']), 
+                (1, executable+': failed PIE RELRO Canary'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro']), 
+                (1, executable+': failed PIE RELRO'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE']), 
+                (1, executable+': failed RELRO'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE']), 
+                (0, ''))
+
+    def test_PE(self):
+        source = 'test1.c'
+        executable = 'test1.exe'
+        cc = 'i686-w64-mingw32-gcc'
+        write_testcode(source)
+
+        self.assertEqual(call_security_check(cc, source, executable, []), 
+                (1, executable+': failed PIE NX'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat']), 
+                (1, executable+': failed PIE'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase']), 
+                (0, ''))
+
+if __name__ == '__main__':
+    unittest.main()
+
diff --git a/contrib/devtools/update-translations.py b/contrib/devtools/update-translations.py
index f955e4a1f..2b6e807b4 100755
--- a/contrib/devtools/update-translations.py
+++ b/contrib/devtools/update-translations.py
@@ -29,6 +29,8 @@ TX = 'tx'
 SOURCE_LANG = 'bitcoin_en.ts'
 # Directory with locale files
 LOCALE_DIR = 'src/qt/locale'
+# Minimum number of messages for translation to be considered at all
+MIN_NUM_MESSAGES = 10
 
 def check_at_repository_root():
     if not os.path.exists('.git'):
@@ -37,7 +39,7 @@ def check_at_repository_root():
         exit(1)
 
 def fetch_all_translations():
-    if subprocess.call([TX, 'pull', '-f']):
+    if subprocess.call([TX, 'pull', '-f', '-a']):
         print('Error while fetching translations', file=sys.stderr)
         exit(1)
 
@@ -70,7 +72,7 @@ def sanitize_string(s):
     '''Sanitize string for printing'''
     return s.replace('\n',' ')
 
-def check_format_specifiers(source, translation, errors):
+def check_format_specifiers(source, translation, errors, numerus):
     source_f = split_format_specifiers(find_format_specifiers(source))
     # assert that no source messages contain both Qt and strprintf format specifiers
     # if this fails, go change the source as this is hacky and confusing!
@@ -78,10 +80,13 @@ def check_format_specifiers(source, translation, errors):
     try:
         translation_f = split_format_specifiers(find_format_specifiers(translation))
     except IndexError:
-        errors.append("Parse error in translation '%s'" % sanitize_string(translation))
+        errors.append("Parse error in translation for '%s': '%s'" % (sanitize_string(source), sanitize_string(translation)))
         return False
     else:
         if source_f != translation_f:
+            if numerus and source_f == (set(), ['n']) and translation_f == (set(), []) and translation.find('%') == -1:
+                # Allow numerus translations to omit %n specifier (usually when it only has one possible value)
+                return True
             errors.append("Mismatch between '%s' and '%s'" % (sanitize_string(source), sanitize_string(translation)))
             return False
     return True
@@ -148,7 +153,7 @@ def postprocess_translations(reduce_diff_hacks=False):
                     if translation is None:
                         continue
                     errors = []
-                    valid = check_format_specifiers(source, translation, errors)
+                    valid = check_format_specifiers(source, translation, errors, numerus)
 
                     for error in errors:
                         print('%s: %s' % (filename, error))
@@ -166,6 +171,15 @@ def postprocess_translations(reduce_diff_hacks=False):
                 if translation_node.get('type') == 'unfinished':
                     context.remove(message)
 
+        # check if document is (virtually) empty, and remove it if so
+        num_messages = 0
+        for context in root.findall('context'):
+            for message in context.findall('message'):
+                num_messages += 1
+        if num_messages < MIN_NUM_MESSAGES:
+            print('Removing %s, as it contains only %i messages' % (filepath, num_messages))
+            continue
+
         # write fixed-up tree
         # if diff reduction requested, replace some XML to 'sanitize' to qt formatting
         if reduce_diff_hacks: