Match iQue compression (#2389)

* Match iQue compression

* Use a dict

* Use Makefile variables for compression settings

* Put COMPRESS_ARGS last

* Pad iQue ROMs to 16 KiB

* Replace --pad-rom with --pad-to and --fill-padding-bytes

* Clarify --fill-padding-bytes

Makefile

@@ -706,9 +706,17 @@ endif
 $(ROM): $(ELF)
 	$(ELF2ROM) -cic 6105 $< $@
 
+ifeq ($(PLATFORM),IQUE)
+  COMPRESS_ARGS := --format gzip --pad-to 0x4000
+  CIC = 6102
+else
+  COMPRESS_ARGS := --format yaz0 --pad-to 0x800000 --fill-padding-bytes
+  CIC = 6105
+endif
+
 $(ROMC): $(ROM) $(ELF) $(BUILD_DIR)/compress_ranges.txt
-	$(PYTHON) tools/compress.py --in $(ROM) --out $@ --dmadata-start `./tools/dmadata_start.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS)
+	$(PYTHON) tools/compress.py --in $(ROM) --out $@ --dmadata-start `./tools/dmadata_start.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS) $(COMPRESS_ARGS)
-	$(PYTHON) -m ipl3checksum sum --cic 6105 --update $@
+	$(PYTHON) -m ipl3checksum sum --cic $(CIC) --update $@
 
 $(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) $(BUILD_DIR)/undefined_syms.txt \
         $(SAMPLEBANK_O_FILES) $(SOUNDFONT_O_FILES) $(SEQUENCE_O_FILES) \

requirements.txt

@@ -1,5 +1,5 @@
 # Setup and compression
-crunch64>=0.3.1,<1.0.0
+crunch64>=0.5.1,<1.0.0
 ipl3checksum>=1.2.0,<2.0.0
 pyyaml>=6.0.1,<7.0.0
 

tools/compress.py

@@ -17,6 +17,12 @@ import crunch64
 import dmadata
 
 
+COMPRESSION_METHODS = {
+    "yaz0": crunch64.yaz0.compress,
+    "gzip": crunch64.gzip.compress,
+}
+
+
 def align(v: int):
     v += 0xF
     return v // 0x10 * 0x10
@@ -48,15 +54,24 @@ def compress_rom(
     rom_data: memoryview,
     dmadata_start: int,
     compress_entries_indices: set[int],
+    compression_format: str,
+    pad_to_multiple_of: int,
+    fill_padding_bytes: bool,
     n_threads: int = None,
 ):
     """
     rom_data: the uncompressed rom data
     dmadata_start: the offset in the rom where the dmadata starts
     compress_entries_indices: the indices in the dmadata of the segments that should be compressed
+    compression_format: the compression format to use
+    pad_to_multiple_of: pad the compressed rom to a multiple of this size, in bytes
+    fill_padding_bytes: fill the padding bytes with a 0x00 0x01 0x02 ... pattern instead of zeros
     n_threads: how many cores to use for compression
     """
 
+    # Compression function
+    compress = COMPRESSION_METHODS[compression_format]
+
     # Segments of the compressed rom (not all are compressed)
     compressed_rom_segments: list[RomSegment] = []
 
@@ -80,7 +95,7 @@ def compress_rom(
             if is_compressed:
                 segment_data = None
                 segment_data_async = p.apply_async(
-                    crunch64.yaz0.compress,
+                    compress,
                     (bytes(segment_data_uncompressed),),
                 )
             else:
@@ -149,7 +164,6 @@ def compress_rom(
     compressed_rom_size = sum(
         align(len(segment.data)) for segment in compressed_rom_segments
     )
-    pad_to_multiple_of = 8 * 2**20  # 8 MiB
     compressed_rom_size_padded = (
         (compressed_rom_size + pad_to_multiple_of - 1)
         // pad_to_multiple_of
@@ -186,9 +200,10 @@ def compress_rom(
         )
 
     assert rom_offset == compressed_rom_size
-    # Pad the compressed rom with the pattern matching the baseroms
+    if fill_padding_bytes:
-    for i in range(compressed_rom_size, compressed_rom_size_padded):
+        # Pad the compressed rom with the pattern matching the baseroms
-        compressed_rom_data[i] = i % 256
+        for i in range(compressed_rom_size, compressed_rom_size_padded):
+            compressed_rom_data[i] = i % 256
 
     # Write the new dmadata
     offset = dmadata_start
@@ -233,6 +248,25 @@ def main():
             " e.g. '0-1,3,5,6-9' is all indices from 0 to 9 (included) except 2 and 4."
         ),
     )
+    parser.add_argument(
+        "--format",
+        dest="format",
+        choices=COMPRESSION_METHODS.keys(),
+        default="yaz0",
+        help="compression format to use (default: yaz0)",
+    )
+    parser.add_argument(
+        "--pad-to",
+        dest="pad_to",
+        type=lambda s: int(s, 16),
+        help="pad the compressed rom to a multiple of this size, in hex (e.g. 0x800000 for 8 MiB)",
+    )
+    parser.add_argument(
+        "--fill-padding-bytes",
+        dest="fill_padding_bytes",
+        action="store_true",
+        help="fill the padding bytes with a 0x00 0x01 0x02 ... pattern instead of zeros",
+    )
     parser.add_argument(
         "--threads",
         dest="n_threads",
@@ -269,6 +303,9 @@ def main():
                 range(compress_range_first, compress_range_last + 1)
             )
 
+    compression_format = args.format
+    pad_to_multiple_of = args.pad_to
+    fill_padding_bytes = args.fill_padding_bytes
     n_threads = args.n_threads
 
     in_rom_data = in_rom_p.read_bytes()
@@ -276,6 +313,9 @@ def main():
         memoryview(in_rom_data),
         dmadata_start,
         compress_entries_indices,
+        compression_format,
+        pad_to_multiple_of,
+        fill_padding_bytes,
         n_threads,
     )
     out_rom_p.write_bytes(out_rom_data)