From ccfb3594ac6a34ee7d6539aa8322d14d2b9cecc2 Mon Sep 17 00:00:00 2001 From: cadmic Date: Fri, 27 Dec 2024 07:34:45 -0800 Subject: [PATCH] Match iQue compression (#2389) * Match iQue compression * Use a dict * Use Makefile variables for compression settings * Put COMPRESS_ARGS last * Pad iQue ROMs to 16 KiB * Replace --pad-rom with --pad-to and --fill-padding-bytes * Clarify --fill-padding-bytes --- Makefile | 12 ++++++++++-- requirements.txt | 2 +- tools/compress.py | 50 ++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 78ed03ead9..b233f84882 100644 --- a/Makefile +++ b/Makefile @@ -706,9 +706,17 @@ endif $(ROM): $(ELF) $(ELF2ROM) -cic 6105 $< $@ +ifeq ($(PLATFORM),IQUE) + COMPRESS_ARGS := --format gzip --pad-to 0x4000 + CIC = 6102 +else + COMPRESS_ARGS := --format yaz0 --pad-to 0x800000 --fill-padding-bytes + CIC = 6105 +endif + $(ROMC): $(ROM) $(ELF) $(BUILD_DIR)/compress_ranges.txt - $(PYTHON) tools/compress.py --in $(ROM) --out $@ --dmadata-start `./tools/dmadata_start.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS) - $(PYTHON) -m ipl3checksum sum --cic 6105 --update $@ + $(PYTHON) tools/compress.py --in $(ROM) --out $@ --dmadata-start `./tools/dmadata_start.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS) $(COMPRESS_ARGS) + $(PYTHON) -m ipl3checksum sum --cic $(CIC) --update $@ $(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) $(BUILD_DIR)/undefined_syms.txt \ $(SAMPLEBANK_O_FILES) $(SOUNDFONT_O_FILES) $(SEQUENCE_O_FILES) \ diff --git a/requirements.txt b/requirements.txt index 45c122f0ec..c9c4afed64 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Setup and compression -crunch64>=0.3.1,<1.0.0 +crunch64>=0.5.1,<1.0.0 ipl3checksum>=1.2.0,<2.0.0 pyyaml>=6.0.1,<7.0.0 diff --git a/tools/compress.py b/tools/compress.py index 8abb8b09c6..f4d6a7e95e 100755 --- a/tools/compress.py +++ b/tools/compress.py @@ -17,6 +17,12 @@ import crunch64 import dmadata +COMPRESSION_METHODS = { + "yaz0": crunch64.yaz0.compress, + "gzip": crunch64.gzip.compress, +} + + def align(v: int): v += 0xF return v // 0x10 * 0x10 @@ -48,15 +54,24 @@ def compress_rom( rom_data: memoryview, dmadata_start: int, compress_entries_indices: set[int], + compression_format: str, + pad_to_multiple_of: int, + fill_padding_bytes: bool, n_threads: int = None, ): """ rom_data: the uncompressed rom data dmadata_start: the offset in the rom where the dmadata starts compress_entries_indices: the indices in the dmadata of the segments that should be compressed + compression_format: the compression format to use + pad_to_multiple_of: pad the compressed rom to a multiple of this size, in bytes + fill_padding_bytes: fill the padding bytes with a 0x00 0x01 0x02 ... pattern instead of zeros n_threads: how many cores to use for compression """ + # Compression function + compress = COMPRESSION_METHODS[compression_format] + # Segments of the compressed rom (not all are compressed) compressed_rom_segments: list[RomSegment] = [] @@ -80,7 +95,7 @@ def compress_rom( if is_compressed: segment_data = None segment_data_async = p.apply_async( - crunch64.yaz0.compress, + compress, (bytes(segment_data_uncompressed),), ) else: @@ -149,7 +164,6 @@ def compress_rom( compressed_rom_size = sum( align(len(segment.data)) for segment in compressed_rom_segments ) - pad_to_multiple_of = 8 * 2**20 # 8 MiB compressed_rom_size_padded = ( (compressed_rom_size + pad_to_multiple_of - 1) // pad_to_multiple_of @@ -186,9 +200,10 @@ def compress_rom( ) assert rom_offset == compressed_rom_size - # Pad the compressed rom with the pattern matching the baseroms - for i in range(compressed_rom_size, compressed_rom_size_padded): - compressed_rom_data[i] = i % 256 + if fill_padding_bytes: + # Pad the compressed rom with the pattern matching the baseroms + for i in range(compressed_rom_size, compressed_rom_size_padded): + compressed_rom_data[i] = i % 256 # Write the new dmadata offset = dmadata_start @@ -233,6 +248,25 @@ def main(): " e.g. '0-1,3,5,6-9' is all indices from 0 to 9 (included) except 2 and 4." ), ) + parser.add_argument( + "--format", + dest="format", + choices=COMPRESSION_METHODS.keys(), + default="yaz0", + help="compression format to use (default: yaz0)", + ) + parser.add_argument( + "--pad-to", + dest="pad_to", + type=lambda s: int(s, 16), + help="pad the compressed rom to a multiple of this size, in hex (e.g. 0x800000 for 8 MiB)", + ) + parser.add_argument( + "--fill-padding-bytes", + dest="fill_padding_bytes", + action="store_true", + help="fill the padding bytes with a 0x00 0x01 0x02 ... pattern instead of zeros", + ) parser.add_argument( "--threads", dest="n_threads", @@ -269,6 +303,9 @@ def main(): range(compress_range_first, compress_range_last + 1) ) + compression_format = args.format + pad_to_multiple_of = args.pad_to + fill_padding_bytes = args.fill_padding_bytes n_threads = args.n_threads in_rom_data = in_rom_p.read_bytes() @@ -276,6 +313,9 @@ def main(): memoryview(in_rom_data), dmadata_start, compress_entries_indices, + compression_format, + pad_to_multiple_of, + fill_padding_bytes, n_threads, ) out_rom_p.write_bytes(out_rom_data)