Add functionality for blacklisting compression

This feature was needed because Wizard101 was unable to read compressed
mp3 files from the archive. Of course, if you're not a fool, you'd use
ogg instead of mp3. But KI developers do lots of stupid things such
as leave unused data in Root.wad.

2 files changed, 27 insertions, 4 deletions

diff --git a/crcdiag.py b/crcdiag.py
new file mode 100644
index 0000000..e7eaae7
--- /dev/null
+++ b/crcdiag.py
@@ -0,0 +1,15 @@
+import sys

+import zlib

+

+h = open(sys.argv[1], mode="rb")

+contents = h.read(-1)

+compressedContents = zlib.compress(contents)

+goal = int(sys.argv[2])

+

+for i in range(0, 0x100000000):

+    if zlib.crc32(contents, i) == goal:

+        print("CRC32: %d matches" % i)

+    if zlib.adler32(contents, i) == goal:

+        print("Adler32: %d matches" % i)

+    if i & 0xFFFFF == 0:

+        print("Progress: %d" % i)

diff --git a/repack.py b/repack.py
index 4ca8e15..4e97b26 100644
--- a/repack.py
+++ b/repack.py
@@ -3,6 +3,7 @@
 import argparse

 import io

 import pathlib

+import re

 import zlib

 

 # The MIT License (MIT)

@@ -35,6 +36,8 @@ parser.add_argument('sourcePath', type=str,
     help='the root directory of the files to be archived')

 parser.add_argument('archivePath', type=str,

     help='the path of the archive file to be made')

+parser.add_argument('blacklist', type=str, nargs='?', default='',

+    help='a file containing patterns to avoid compressing')

 parser.add_argument('--verbose', '-v', type=bool,

     help='if enabled, use diagnostics')

 

@@ -44,6 +47,12 @@ args = parser.parse_args()
 

 root = pathlib.Path(args.sourcePath)

 files = root.glob("**/*");

+blacklist = set()

+if args.blacklist != '':

+    blacklistPatterns = re.split('\s', open(args.blacklist).read(-1))

+    for p in blacklistPatterns:

+        if p != '':

+            blacklist.update(set(root.glob(p)))

 entries = []

 offset = 14 # Precalculate offset of data section in loop below

 

@@ -57,11 +66,10 @@ for f in files:
             uSize = len(contents)

             zSize = len(compressedContents)

             name = str(f.relative_to(root))

-            isCompressed = zSize < uSize

+            isCompressed = zSize < uSize and not f in blacklist

             crc = zlib.crc32(contents)

             entry = {

-                "contents": contents,

-                "compressedContents": compressedContents,

+                "contents": compressedContents if isCompressed else contents,

                 "uSize": uSize,

                 "zSize": zSize,

                 "name": name,

@@ -112,6 +120,6 @@ for e in entries:
         print("> Wrote ENTRY for %s" % e["name"])

 # Write file DATA

 for e in entries:

-    out.write(e["compressedContents"] if e["isCompressed"] else e["contents"])

+    out.write(e["contents"])

     if args.verbose:

         print("> Wrote DATA for %s" % e["name"])