Syms DMA Entries Extraction Support (#1708)

* Archive compression support + small cleanups

* UNSET spec and PR comment

* UNSET -> SYMS

* Syms comment

* PR review

* remove stderr

* Format

* format2

* Remove trailing ,s

tools/compress.py

@@ -8,7 +8,6 @@ from __future__ import annotations
 import argparse
 from pathlib import Path
 import dataclasses
-import struct
 import time
 import multiprocessing
 import multiprocessing.pool
@@ -28,6 +27,7 @@ class RomSegment:
     vrom_start: int
     vrom_end: int
     is_compressed: bool
+    is_syms: bool
     data: memoryview | None
     data_async: multiprocessing.pool.AsyncResult | None
 
@@ -92,6 +92,7 @@ def compress_rom(
                     dma_entry.vrom_start,
                     dma_entry.vrom_end,
                     is_compressed,
+                    dma_entry.is_syms(),
                     segment_data,
                     segment_data_async,
                 )
@@ -167,17 +168,23 @@ def compress_rom(
         assert i <= len(compressed_rom_data)
         compressed_rom_data[segment_rom_start:i] = segment.data
 
+        rom_offset = segment_rom_end
+
+        if segment.is_syms:
+            segment_rom_start = 0xFFFFFFFF
+            segment_rom_end = 0xFFFFFFFF
+        elif not segment.is_compressed:
+            segment_rom_end = 0
+
         compressed_rom_dma_entries.append(
             dmadata.DmaEntry(
                 segment.vrom_start,
                 segment.vrom_end,
                 segment_rom_start,
-                segment_rom_end if segment.is_compressed else 0,
+                segment_rom_end,
             )
         )
 
-        rom_offset = segment_rom_end
-
     assert rom_offset == compressed_rom_size
     # Pad the compressed rom with the pattern matching the baseroms
     for i in range(compressed_rom_size, compressed_rom_size_padded):

tools/decompress_baserom.py

@@ -5,11 +5,11 @@
 
 from __future__ import annotations
 
+import argparse
 import hashlib
 import io
-import struct
 from pathlib import Path
-import argparse
+import struct
 
 import crunch64
 import ipl3checksum
@@ -67,7 +67,7 @@ def decompress_rom(
         v_end = dma_entry.vrom_end
         p_start = dma_entry.rom_start
         p_end = dma_entry.rom_end
-        if p_start == 0xFFFFFFFF and p_end == 0xFFFFFFFF:
+        if dma_entry.is_syms():
             new_dmadata.append(dma_entry)
             continue
         if dma_entry.is_compressed():
@@ -160,9 +160,9 @@ def find_baserom(version: str) -> Path | None:
 
 
 def main():
-    description = "Convert a rom that uses dmadata to an uncompressed one."
+    parser = argparse.ArgumentParser(
-
+        description="Convert a rom that uses dmadata to an uncompressed one."
-    parser = argparse.ArgumentParser(description=description)
+    )
     parser.add_argument(
         "version",
         help="Version of the game to decompress.",
@@ -245,7 +245,7 @@ def main():
         exit(1)
 
     # Write out our new ROM
-    print(f"Writing new ROM {uncompressed_path}.")
+    print(f"Writing new ROM {uncompressed_path}...")
     uncompressed_path.write_bytes(file_content)
 
     print("Done!")

tools/dmadata.py

@@ -58,6 +58,14 @@ class DmaEntry:
     def is_compressed(self) -> bool:
         return self.rom_end != 0
 
+    def is_syms(self) -> bool:
+        """
+        "SYMS" DMA entries describe segments that are always filled with 0's in the ROM.
+        The DMA entry has both rom_start and rom_end set to 0xFFFFFFFF, where the actual rom start and end is given by vrom_start and vrom_end instead.
+        These zeroed segments are used to record the offsets/sizes of subfiles in compressed yaz0 archive files but are not used by the game directly.
+        """
+        return self.rom_start == 0xFFFFFFFF and self.rom_end == 0xFFFFFFFF
+
 
 DMA_ENTRY_END = DmaEntry(0, 0, 0, 0)
 

tools/extract_baserom.py

@@ -56,12 +56,15 @@ def main():
 
     args.output_dir.mkdir(parents=True, exist_ok=True)
     for dma_name, dma_entry in zip(dma_names, dma_entries):
-        if dma_entry.is_compressed():
+        if dma_entry.is_syms():
+            segment_rom_start = dma_entry.vrom_start
+        elif not dma_entry.is_compressed():
+            segment_rom_start = dma_entry.rom_start
+        else: # Segment compressed
             print(f"Error: segment {dma_name} is compressed", file=sys.stderr)
             exit(1)
 
-        segment_rom_start = dma_entry.rom_start
+        segment_rom_end = segment_rom_start + (
-        segment_rom_end = dma_entry.rom_start + (
             dma_entry.vrom_end - dma_entry.vrom_start
         )
 

tools/spec.c

@@ -85,6 +85,8 @@ static bool parse_flags(char *str, unsigned int *flags)
             f |= FLAG_RAW;
         else if (strcmp(str, "NOLOAD") == 0)
             f |= FLAG_NOLOAD;
+        else if (strcmp(str, "SYMS") == 0)
+            f |= FLAG_SYMS;
         else
             return false;
 

tools/spec.h

@@ -28,6 +28,7 @@ enum {
     FLAG_OBJECT = (1 << 1),
     FLAG_RAW = (1 << 2),
     FLAG_NOLOAD = (1 << 3),
+    FLAG_SYMS = (1 << 4)
 };
 
 struct Include {