From 0ac4448d99d2598f8b39602b9f385b03f7f6e98f Mon Sep 17 00:00:00 2001 From: Derek Hensley Date: Mon, 12 Feb 2024 17:10:20 -0800 Subject: [PATCH] Syms DMA Entries Extraction Support (#1708) * Archive compression support + small cleanups * UNSET spec and PR comment * UNSET -> SYMS * Syms comment * PR review * remove stderr * Format * format2 * Remove trailing ,s --- tools/compress.py | 15 +++++++++++---- tools/decompress_baserom.py | 14 +++++++------- tools/dmadata.py | 8 ++++++++ tools/extract_baserom.py | 9 ++++++--- tools/spec.c | 2 ++ tools/spec.h | 1 + 6 files changed, 35 insertions(+), 14 deletions(-) diff --git a/tools/compress.py b/tools/compress.py index d934ecb523..8abb8b09c6 100755 --- a/tools/compress.py +++ b/tools/compress.py @@ -8,7 +8,6 @@ from __future__ import annotations import argparse from pathlib import Path import dataclasses -import struct import time import multiprocessing import multiprocessing.pool @@ -28,6 +27,7 @@ class RomSegment: vrom_start: int vrom_end: int is_compressed: bool + is_syms: bool data: memoryview | None data_async: multiprocessing.pool.AsyncResult | None @@ -92,6 +92,7 @@ def compress_rom( dma_entry.vrom_start, dma_entry.vrom_end, is_compressed, + dma_entry.is_syms(), segment_data, segment_data_async, ) @@ -167,17 +168,23 @@ def compress_rom( assert i <= len(compressed_rom_data) compressed_rom_data[segment_rom_start:i] = segment.data + rom_offset = segment_rom_end + + if segment.is_syms: + segment_rom_start = 0xFFFFFFFF + segment_rom_end = 0xFFFFFFFF + elif not segment.is_compressed: + segment_rom_end = 0 + compressed_rom_dma_entries.append( dmadata.DmaEntry( segment.vrom_start, segment.vrom_end, segment_rom_start, - segment_rom_end if segment.is_compressed else 0, + segment_rom_end, ) ) - rom_offset = segment_rom_end - assert rom_offset == compressed_rom_size # Pad the compressed rom with the pattern matching the baseroms for i in range(compressed_rom_size, compressed_rom_size_padded): diff --git a/tools/decompress_baserom.py b/tools/decompress_baserom.py index dad9887a83..799c66a1e0 100755 --- a/tools/decompress_baserom.py +++ b/tools/decompress_baserom.py @@ -5,11 +5,11 @@ from __future__ import annotations +import argparse import hashlib import io -import struct from pathlib import Path -import argparse +import struct import crunch64 import ipl3checksum @@ -67,7 +67,7 @@ def decompress_rom( v_end = dma_entry.vrom_end p_start = dma_entry.rom_start p_end = dma_entry.rom_end - if p_start == 0xFFFFFFFF and p_end == 0xFFFFFFFF: + if dma_entry.is_syms(): new_dmadata.append(dma_entry) continue if dma_entry.is_compressed(): @@ -160,9 +160,9 @@ def find_baserom(version: str) -> Path | None: def main(): - description = "Convert a rom that uses dmadata to an uncompressed one." - - parser = argparse.ArgumentParser(description=description) + parser = argparse.ArgumentParser( + description="Convert a rom that uses dmadata to an uncompressed one." + ) parser.add_argument( "version", help="Version of the game to decompress.", @@ -245,7 +245,7 @@ def main(): exit(1) # Write out our new ROM - print(f"Writing new ROM {uncompressed_path}.") + print(f"Writing new ROM {uncompressed_path}...") uncompressed_path.write_bytes(file_content) print("Done!") diff --git a/tools/dmadata.py b/tools/dmadata.py index 68eb1e7107..06b9ae7557 100644 --- a/tools/dmadata.py +++ b/tools/dmadata.py @@ -58,6 +58,14 @@ class DmaEntry: def is_compressed(self) -> bool: return self.rom_end != 0 + def is_syms(self) -> bool: + """ + "SYMS" DMA entries describe segments that are always filled with 0's in the ROM. + The DMA entry has both rom_start and rom_end set to 0xFFFFFFFF, where the actual rom start and end is given by vrom_start and vrom_end instead. + These zeroed segments are used to record the offsets/sizes of subfiles in compressed yaz0 archive files but are not used by the game directly. + """ + return self.rom_start == 0xFFFFFFFF and self.rom_end == 0xFFFFFFFF + DMA_ENTRY_END = DmaEntry(0, 0, 0, 0) diff --git a/tools/extract_baserom.py b/tools/extract_baserom.py index bc4feb55a8..1d62f8c06e 100755 --- a/tools/extract_baserom.py +++ b/tools/extract_baserom.py @@ -56,12 +56,15 @@ def main(): args.output_dir.mkdir(parents=True, exist_ok=True) for dma_name, dma_entry in zip(dma_names, dma_entries): - if dma_entry.is_compressed(): + if dma_entry.is_syms(): + segment_rom_start = dma_entry.vrom_start + elif not dma_entry.is_compressed(): + segment_rom_start = dma_entry.rom_start + else: # Segment compressed print(f"Error: segment {dma_name} is compressed", file=sys.stderr) exit(1) - segment_rom_start = dma_entry.rom_start - segment_rom_end = dma_entry.rom_start + ( + segment_rom_end = segment_rom_start + ( dma_entry.vrom_end - dma_entry.vrom_start ) diff --git a/tools/spec.c b/tools/spec.c index 7d07caa13d..39dd088ec8 100644 --- a/tools/spec.c +++ b/tools/spec.c @@ -85,6 +85,8 @@ static bool parse_flags(char *str, unsigned int *flags) f |= FLAG_RAW; else if (strcmp(str, "NOLOAD") == 0) f |= FLAG_NOLOAD; + else if (strcmp(str, "SYMS") == 0) + f |= FLAG_SYMS; else return false; diff --git a/tools/spec.h b/tools/spec.h index aeabd07d1f..f0a16eeb4f 100644 --- a/tools/spec.h +++ b/tools/spec.h @@ -28,6 +28,7 @@ enum { FLAG_OBJECT = (1 << 1), FLAG_RAW = (1 << 2), FLAG_NOLOAD = (1 << 3), + FLAG_SYMS = (1 << 4) }; struct Include {