Match compression for gc-eu-mq (#1704)

* Improve compression

* Format

* Typo

* Use Python assignment expression in tools/dmadata.py

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

* Review decompress_baserom.py

* Replace DEFINE_DEBUG_SCENE with CPP defines

* Pass is_zlib_compressed instead of version

* Reword NOLOAD comment in write_compress_ranges

* Remove redundant comment

---------

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

Makefile

@@ -390,8 +390,7 @@ $(ROM): $(ELF)
 	$(ELF2ROM) -cic 6105 $< $@
 
 $(ROMC): $(ROM) $(ELF) $(BUILD_DIR)/compress_ranges.txt
-# note: $(BUILD_DIR)/compress_ranges.txt should only be used for nonmatching builds. it works by chance for matching builds too though
+	$(PYTHON) tools/compress.py --in $(ROM) --out $@ --dma-start `./tools/dmadata_start.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS)
-	$(PYTHON) tools/compress.py --in $(ROM) --out $@ --dma-range `./tools/dmadata_range.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS)
 	$(PYTHON) -m ipl3checksum sum --cic 6105 --update $@
 
 $(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) $(BUILD_DIR)/undefined_syms.txt

baseroms/gc-eu-mq-dbg/checksum-compressed.md5

@@ -1 +1 @@
-5831385a7f216370cdbea55616b12fed  build/gc-eu-mq-dbg/oot-gc-eu-mq-dbg-compressed.z64
+9704bc98c20c20319cc1633fe02b6fc7  build/gc-eu-mq-dbg/oot-gc-eu-mq-dbg-compressed.z64

include/tables/dmadata_table.h

@@ -1,8 +1,8 @@
 /**
  * Select dmadata table for version
  */
-#ifdef NON_MATCHING
+#if !OOT_DEBUG || NON_MATCHING
-// For non-matching builds, dmadata is generated from the specfile segments
+// For retail versions and non-matching builds, dmadata is generated from the specfile segments
 #include "dmadata_table_spec.h"
 #else
 #include "dmadata_table_mqdbg.h"

include/tables/scene_table.h

@@ -110,7 +110,7 @@
 /* 0x62 */ DEFINE_SCENE(spot18_scene, g_pn_41, SCENE_GORON_CITY, SDC_GORON_CITY, 0, 0)
 /* 0x63 */ DEFINE_SCENE(spot20_scene, g_pn_42, SCENE_LON_LON_RANCH, SDC_LON_LON_RANCH, 0, 0)
 /* 0x64 */ DEFINE_SCENE(ganon_tou_scene, g_pn_43, SCENE_OUTSIDE_GANONS_CASTLE, SDC_OUTSIDE_GANONS_CASTLE, 0, 0)
-// Debug-only scenes
+#if OOT_DEBUG
 /* 0x65 */ DEFINE_SCENE(test01_scene, none, SCENE_TEST01, SDC_CALM_WATER, 0, 0)
 /* 0x66 */ DEFINE_SCENE(besitu_scene, none, SCENE_BESITU, SDC_BESITU, 0, 0)
 /* 0x67 */ DEFINE_SCENE(depth_test_scene, none, SCENE_DEPTH_TEST, SDC_DEFAULT, 0, 0)
@@ -120,3 +120,4 @@
 /* 0x6B */ DEFINE_SCENE(hairal_niwa2_scene, g_pn_12, SCENE_HAIRAL_NIWA2, SDC_CASTLE_COURTYARD_GUARDS, 0, 0)
 /* 0x6C */ DEFINE_SCENE(sasatest_scene, none, SCENE_SASATEST, SDC_DEFAULT, 0, 0)
 /* 0x6D */ DEFINE_SCENE(testroom_scene, none, SCENE_TESTROOM, SDC_DEFAULT, 0, 0)
+#endif

include/z64scene.h

@@ -355,9 +355,21 @@ typedef enum {
 
 #undef DEFINE_SCENE
 
-// this define exists to preserve shiftability for an unused scene that is
+// Fake enum values for scenes that are still referenced in the entrance table
-// listed in the entrance table
+#if !OOT_DEBUG
-#define SCENE_UNUSED_6E SCENE_ID_MAX
+// Debug-only scenes
+#define SCENE_TEST01        0x65
+#define SCENE_BESITU        0x66
+#define SCENE_DEPTH_TEST    0x67
+#define SCENE_SYOTES        0x68
+#define SCENE_SYOTES2       0x69
+#define SCENE_SUTARU        0x6A
+#define SCENE_HAIRAL_NIWA2  0x6B
+#define SCENE_SASATEST      0x6C
+#define SCENE_TESTROOM      0x6D
+#endif
+// Deleted scene
+#define SCENE_UNUSED_6E     0x6E
 
 // Entrance Index Enum
 #define DEFINE_ENTRANCE(enum, _1, _2, _3, _4, _5, _6) enum,

spec

@@ -538,6 +538,7 @@ endseg
 
 beginseg
     name "buffers"
+    flags NOLOAD
     align 0x40
     include "$(BUILD_DIR)/src/buffers/zbuffer.o"
     include "$(BUILD_DIR)/src/buffers/gfxbuffers.o"
@@ -9627,6 +9628,7 @@ beginseg
     number 3
 endseg
 
+#if OOT_DEBUG
 beginseg
     name "syotes_scene"
     romalign 0x1000
@@ -9668,6 +9670,7 @@ beginseg
     include "$(BUILD_DIR)/assets/scenes/test_levels/depth_test/depth_test_room_0.o"
     number 3
 endseg
+#endif
 
 beginseg
     name "spot00_scene"
@@ -10149,6 +10152,7 @@ beginseg
     number 3
 endseg
 
+#if OOT_DEBUG
 beginseg
     name "testroom_scene"
     romalign 0x1000
@@ -10190,6 +10194,7 @@ beginseg
     include "$(BUILD_DIR)/assets/scenes/test_levels/testroom/testroom_room_4.o"
     number 3
 endseg
+#endif
 
 beginseg
     name "kenjyanoma_scene"
@@ -10231,6 +10236,7 @@ beginseg
     number 3
 endseg
 
+#if OOT_DEBUG
 beginseg
     name "sutaru_scene"
     romalign 0x1000
@@ -10244,6 +10250,7 @@ beginseg
     include "$(BUILD_DIR)/assets/scenes/test_levels/sutaru/sutaru_room_0.o"
     number 3
 endseg
+#endif
 
 beginseg
     name "link_home_scene"
@@ -10517,6 +10524,7 @@ beginseg
     number 3
 endseg
 
+#if OOT_DEBUG
 beginseg
     name "sasatest_scene"
     romalign 0x1000
@@ -10530,6 +10538,7 @@ beginseg
     include "$(BUILD_DIR)/assets/scenes/test_levels/sasatest/sasatest_room_0.o"
     number 3
 endseg
+#endif
 
 beginseg
     name "market_alley_scene"
@@ -11267,6 +11276,7 @@ beginseg
     number 3
 endseg
 
+#if OOT_DEBUG
 beginseg
     name "hairal_niwa2_scene"
     romalign 0x1000
@@ -11280,6 +11290,7 @@ beginseg
     include "$(BUILD_DIR)/assets/scenes/indoors/hairal_niwa2/hairal_niwa2_room_0.o"
     number 3
 endseg
+#endif
 
 beginseg
     name "hakasitarelay_scene"
@@ -11753,6 +11764,7 @@ beginseg
     number 3
 endseg
 
+#if OOT_DEBUG
 beginseg
     name "besitu_scene"
     romalign 0x1000
@@ -11766,6 +11778,7 @@ beginseg
     include "$(BUILD_DIR)/assets/scenes/test_levels/besitu/besitu_room_0.o"
     number 3
 endseg
+#endif
 
 beginseg
     name "face_shop_scene"
@@ -11823,6 +11836,7 @@ beginseg
     number 3
 endseg
 
+#if OOT_DEBUG
 beginseg
     name "test01_scene"
     romalign 0x1000
@@ -11836,6 +11850,7 @@ beginseg
     include "$(BUILD_DIR)/assets/scenes/test_levels/test01/test01_room_0.o"
     number 3
 endseg
+#endif
 
 beginseg
     name "bump_texture_static"

tools/compress.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 # SPDX-FileCopyrightText: 2024 zeldaret
 # SPDX-License-Identifier: CC0-1.0
 
@@ -13,57 +15,7 @@ import multiprocessing.pool
 
 import crunch64
 
-
+import dmadata
-STRUCT_IIII = struct.Struct(">IIII")
-
-
-@dataclasses.dataclass
-class DmaEntry:
-    """
-    A Python counterpart to the dmadata entry struct:
-    ```c
-    typedef struct {
-        /* 0x00 */ uintptr_t vromStart;
-        /* 0x04 */ uintptr_t vromEnd;
-        /* 0x08 */ uintptr_t romStart;
-        /* 0x0C */ uintptr_t romEnd;
-    } DmaEntry;
-    ```
-    """
-
-    vromStart: int
-    vromEnd: int
-    romStart: int
-    romEnd: int
-
-    def __repr__(self):
-        return (
-            "DmaEntry("
-            f"vromStart=0x{self.vromStart:08X}, "
-            f"vromEnd=0x{self.vromEnd:08X}, "
-            f"romStart=0x{self.romStart:08X}, "
-            f"romEnd=0x{self.romEnd:08X}"
-            ")"
-        )
-
-    SIZE_BYTES = STRUCT_IIII.size
-
-    def to_bin(self, data: memoryview):
-        STRUCT_IIII.pack_into(
-            data,
-            0,
-            self.vromStart,
-            self.vromEnd,
-            self.romStart,
-            self.romEnd,
-        )
-
-    @staticmethod
-    def from_bin(data: memoryview):
-        return DmaEntry(*STRUCT_IIII.unpack_from(data))
-
-
-DMA_ENTRY_ZERO = DmaEntry(0, 0, 0, 0)
 
 
 def align(v: int):
@@ -73,15 +25,15 @@ def align(v: int):
 
 @dataclasses.dataclass
 class RomSegment:
-    vromStart: int
+    vrom_start: int
-    vromEnd: int
+    vrom_end: int
     is_compressed: bool
     data: memoryview | None
     data_async: multiprocessing.pool.AsyncResult | None
 
     @property
     def uncompressed_size(self):
-        return self.vromEnd - self.vromStart
+        return self.vrom_end - self.vrom_start
 
 
 # Make interrupting the compression with ^C less jank
@@ -94,15 +46,13 @@ def set_sigint_ignored():
 
 def compress_rom(
     rom_data: memoryview,
-    dmadata_offset_start: int,
+    dmadata_offset: int,
-    dmadata_offset_end: int,
     compress_entries_indices: set[int],
     n_threads: int = None,
 ):
     """
     rom_data: the uncompressed rom data
-    dmadata_offset_start: the offset in the rom where the dmadata starts (inclusive)
+    dmadata_offset: the offset in the rom where the dmadata starts
-    dmadata_offset_end: the offset in the rom where the dmadata ends (exclusive)
     compress_entries_indices: the indices in the dmadata of the segments that should be compressed
     n_threads: how many cores to use for compression
     """
@@ -110,18 +60,18 @@ def compress_rom(
     # Segments of the compressed rom (not all are compressed)
     compressed_rom_segments: list[RomSegment] = []
 
+    dma_entries = dmadata.read_dmadata(rom_data, dmadata_offset)
+    # We sort the DMA entries by ROM start because `compress_entries_indices`
+    # refers to indices in ROM order, but the uncompressed dmadata might not be
+    # in ROM order.
+    dma_entries.sort(key=lambda dma_entry: dma_entry.vrom_start)
+
     with multiprocessing.Pool(n_threads, initializer=set_sigint_ignored) as p:
         # Extract each segment from the input rom
-        for entry_index, dmadata_offset in enumerate(
+        for entry_index, dma_entry in enumerate(dma_entries):
-            range(dmadata_offset_start, dmadata_offset_end, DmaEntry.SIZE_BYTES)
+            segment_rom_start = dma_entry.rom_start
-        ):
+            segment_rom_end = dma_entry.rom_start + (
-            dma_entry = DmaEntry.from_bin(rom_data[dmadata_offset:])
+                dma_entry.vrom_end - dma_entry.vrom_start
-            if dma_entry == DMA_ENTRY_ZERO:
-                continue
-
-            segment_rom_start = dma_entry.romStart
-            segment_rom_end = dma_entry.romStart + (
-                dma_entry.vromEnd - dma_entry.vromStart
             )
             segment_data_uncompressed = rom_data[segment_rom_start:segment_rom_end]
 
@@ -139,17 +89,14 @@ def compress_rom(
 
             compressed_rom_segments.append(
                 RomSegment(
-                    dma_entry.vromStart,
+                    dma_entry.vrom_start,
-                    dma_entry.vromEnd,
+                    dma_entry.vrom_end,
                     is_compressed,
                     segment_data,
                     segment_data_async,
                 )
             )
 
-        # Technically optional but required for matching.
-        compressed_rom_segments.sort(key=lambda segment: segment.vromStart)
-
         # Wait on compression of all compressed segments
         waiting_on_segments = [
             segment for segment in compressed_rom_segments if segment.is_compressed
@@ -208,7 +155,7 @@ def compress_rom(
         * pad_to_multiple_of
     )
     compressed_rom_data = memoryview(bytearray(compressed_rom_size_padded))
-    compressed_rom_dma_entries: list[DmaEntry] = []
+    compressed_rom_dma_entries: list[dmadata.DmaEntry] = []
     rom_offset = 0
     for segment in compressed_rom_segments:
         assert segment.data is not None
@@ -221,9 +168,9 @@ def compress_rom(
         compressed_rom_data[segment_rom_start:i] = segment.data
 
         compressed_rom_dma_entries.append(
-            DmaEntry(
+            dmadata.DmaEntry(
-                segment.vromStart,
+                segment.vrom_start,
-                segment.vromEnd,
+                segment.vrom_end,
                 segment_rom_start,
                 segment_rom_end if segment.is_compressed else 0,
             )
@@ -237,13 +184,10 @@ def compress_rom(
         compressed_rom_data[i] = i % 256
 
     # Write the new dmadata
-    dmadata_offset = dmadata_offset_start
+    offset = dmadata_offset
     for dma_entry in compressed_rom_dma_entries:
-        assert dmadata_offset + DmaEntry.SIZE_BYTES <= dmadata_offset_end
+        dma_entry.to_bin(compressed_rom_data[offset:])
-
+        offset += dmadata.DmaEntry.SIZE_BYTES
-        dma_entry.to_bin(compressed_rom_data[dmadata_offset:])
-
-        dmadata_offset += DmaEntry.SIZE_BYTES
 
     return compressed_rom_data
 
@@ -263,13 +207,12 @@ def main():
         help="path of the compressed rom to write out",
     )
     parser.add_argument(
-        "--dma-range",
+        "--dma-start",
-        dest="dma_range",
+        dest="dma_start",
+        type=lambda s: int(s, 16),
         required=True,
         help=(
-            "The dmadata location in the rom, in format"
+            "The dmadata location in the rom, as a hexadecimal offset (e.g. 0x12f70)."
-            " 'start_inclusive-end_exclusive' and using hexadecimal offsets"
-            " (e.g. '0x12f70-0x19030')."
         ),
     )
     parser.add_argument(
@@ -298,13 +241,7 @@ def main():
 
     out_rom_p = Path(args.out_rom)
 
-    dma_range_str: str = args.dma_range
+    dmadata_offset = args.dma_start
-    dma_range_ends_str = dma_range_str.split("-")
-    assert len(dma_range_ends_str) == 2, dma_range_str
-    dmadata_offset_start, dmadata_offset_end = (
-        int(v_str, 16) for v_str in dma_range_ends_str
-    )
-    assert dmadata_offset_start < dmadata_offset_end, dma_range_str
 
     compress_ranges_str: str = args.compress_ranges
     compress_entries_indices = set()
@@ -330,8 +267,7 @@ def main():
     in_rom_data = in_rom_p.read_bytes()
     out_rom_data = compress_rom(
         memoryview(in_rom_data),
-        dmadata_offset_start,
+        dmadata_offset,
-        dmadata_offset_end,
         compress_entries_indices,
         n_threads,
     )

tools/decompress_baserom.py

@@ -15,6 +15,8 @@ import crunch64
 import ipl3checksum
 import zlib
 
+import dmadata
+
 
 def decompress_zlib(data: bytes) -> bytes:
     decomp = zlib.decompressobj(-zlib.MAX_WBITS)
@@ -48,27 +50,6 @@ def round_up(n, shift):
     return (n + mod - 1) >> shift << shift
 
 
-def as_word_list(b) -> list[int]:
-    return [i[0] for i in struct.iter_unpack(">I", b)]
-
-
-def read_dmadata_entry(file_content: bytearray, addr: int) -> list[int]:
-    return as_word_list(file_content[addr : addr + 0x10])
-
-
-def read_dmadata(file_content: bytearray, start) -> list[list[int]]:
-    dmadata = []
-    addr = start
-    entry = read_dmadata_entry(file_content, addr)
-    i = 0
-    while any([e != 0 for e in entry]):
-        dmadata.append(entry)
-        addr += 0x10
-        i += 1
-        entry = read_dmadata_entry(file_content, addr)
-    return dmadata
-
-
 def update_crc(decompressed: io.BytesIO) -> io.BytesIO:
     print("Recalculating crc...")
     calculated_checksum = ipl3checksum.CICKind.CIC_X105.calculateChecksum(
@@ -82,40 +63,43 @@ def update_crc(decompressed: io.BytesIO) -> io.BytesIO:
 
 
 def decompress_rom(
-    file_content: bytearray, dmadata_addr: int, dmadata: list[list[int]], version: str
+    file_content: bytearray,
+    dmadata_offset: int,
+    dma_entries: list[dmadata.DmaEntry],
+    is_zlib_compressed: bool,
 ) -> bytearray:
     rom_segments = {}  # vrom start : data s.t. len(data) == vrom_end - vrom_start
-    new_dmadata = bytearray()  # new dmadata: {vrom start , vrom end , vrom start , 0}
+    new_dmadata = []  # new dmadata: list[dmadata.Entry]
 
     decompressed = io.BytesIO(b"")
 
-    for v_start, v_end, p_start, p_end in dmadata:
+    for dma_entry in dma_entries:
+        v_start = dma_entry.vrom_start
+        v_end = dma_entry.vrom_end
+        p_start = dma_entry.rom_start
+        p_end = dma_entry.rom_end
         if p_start == 0xFFFFFFFF and p_end == 0xFFFFFFFF:
-            new_dmadata.extend(struct.pack(">IIII", v_start, v_end, p_start, p_end))
+            new_dmadata.append(dma_entry)
             continue
-        if p_end == 0:  # uncompressed
+        if dma_entry.is_compressed():
-            rom_segments.update(
+            new_contents = decompress(file_content[p_start:p_end], is_zlib_compressed)
-                {v_start: file_content[p_start : p_start + v_end - v_start]}
+            rom_segments[v_start] = new_contents
-            )
+        else:
-        else:  # compressed
+            rom_segments[v_start] = file_content[p_start : p_start + v_end - v_start]
-            rom_segments.update(
+        new_dmadata.append(dmadata.DmaEntry(v_start, v_end, v_start, 0))
-                {
-                    v_start: decompress(
-                        file_content[p_start:p_end], version in {"ique-cn", "ique-zh"}
-                    )
-                }
-            )
-        new_dmadata.extend(struct.pack(">IIII", v_start, v_end, v_start, 0))
 
     # write rom segments to vaddrs
     for vrom_st, data in rom_segments.items():
         decompressed.seek(vrom_st)
         decompressed.write(data)
     # write new dmadata
-    decompressed.seek(dmadata_addr)
+    decompressed.seek(dmadata_offset)
-    decompressed.write(new_dmadata)
+    for dma_entry in new_dmadata:
+        entry_data = bytearray(dmadata.DmaEntry.SIZE_BYTES)
+        dma_entry.to_bin(entry_data)
+        decompressed.write(entry_data)
     # pad to size
-    padding_end = round_up(dmadata[-1][1], 14)
+    padding_end = round_up(dma_entries[-1].vrom_end, 14)
     decompressed.seek(padding_end - 1)
     decompressed.write(bytearray([0]))
     # re-calculate crc
@@ -164,17 +148,19 @@ def per_version_fixes(file_content: bytearray, version: str) -> bytearray:
     return file_content
 
 
-def pad_rom(file_content: bytearray, dmadata: list[list[int]]) -> bytearray:
+def pad_rom(file_content: bytearray, dma_entries: list[dmadata.DmaEntry]) -> bytearray:
-    padding_start = round_up(dmadata[-1][1], 12)
+    padding_start = round_up(dma_entries[-1].vrom_end, 12)
-    padding_end = round_up(dmadata[-1][1], 14)
+    padding_end = round_up(dma_entries[-1].vrom_end, 14)
     print(f"Padding from {padding_start:X} to {padding_end:X}...")
     for i in range(padding_start, padding_end):
         file_content[i] = 0xFF
     return file_content
 
+
 # Determine if we have a ROM file
 ROM_FILE_EXTENSIONS = ["z64", "n64", "v64"]
 
+
 def find_baserom(version: str) -> Path | None:
     for rom_file_ext_lower in ROM_FILE_EXTENSIONS:
         for rom_file_ext in (rom_file_ext_lower, rom_file_ext_lower.upper()):
@@ -188,7 +174,11 @@ def main():
     description = "Convert a rom that uses dmadata to an uncompressed one."
 
     parser = argparse.ArgumentParser(description=description)
-    parser.add_argument("version", help="Version of the game to decompress.", choices=list(VERSIONS_MD5S.keys()))
+    parser.add_argument(
+        "version",
+        help="Version of the game to decompress.",
+        choices=list(VERSIONS_MD5S.keys()),
+    )
 
     args = parser.parse_args()
 
@@ -196,7 +186,7 @@ def main():
 
     uncompressed_path = Path(f"baseroms/{version}/baserom-decompressed.z64")
 
-    file_table_offset = FILE_TABLE_OFFSET[version]
+    dmadata_offset = FILE_TABLE_OFFSET[version]
     correct_str_hash = VERSIONS_MD5S[version]
 
     if check_existing_rom(uncompressed_path, correct_str_hash):
@@ -207,7 +197,8 @@ def main():
 
     if rom_file_name is None:
         path_list = [
-            f"baseroms/{version}/baserom.{rom_file_ext}" for rom_file_ext in ROM_FILE_EXTENSIONS
+            f"baseroms/{version}/baserom.{rom_file_ext}"
+            for rom_file_ext in ROM_FILE_EXTENSIONS
         ]
         print(f"Error: Could not find {','.join(path_list)}.")
         exit(1)
@@ -234,20 +225,16 @@ def main():
 
     file_content = per_version_fixes(file_content, version)
 
-    dmadata = read_dmadata(file_content, file_table_offset)
+    dma_entries = dmadata.read_dmadata(file_content, dmadata_offset)
     # Decompress
-    if any(
+    if any(dma_entry.is_compressed() for dma_entry in dma_entries):
-        [
-            b != 0
-            for b in file_content[
-                file_table_offset + 0xAC : file_table_offset + 0xAC + 0x4
-            ]
-        ]
-    ):
         print("Decompressing rom...")
-        file_content = decompress_rom(file_content, file_table_offset, dmadata, version)
+        is_zlib_compressed = version in {"ique-cn", "ique-zh"}
+        file_content = decompress_rom(
+            file_content, dmadata_offset, dma_entries, is_zlib_compressed
+        )
 
-    file_content = pad_rom(file_content, dmadata)
+    file_content = pad_rom(file_content, dma_entries)
 
     # Check to see if the ROM is a "vanilla" ROM
     str_hash = get_str_hash(file_content)

tools/dmadata.py

@@ -0,0 +1,75 @@
+# SPDX-FileCopyrightText: © 2024 ZeldaRET
+# SPDX-License-Identifier: CC0-1.0
+
+from __future__ import annotations
+
+import dataclasses
+import struct
+
+
+STRUCT_IIII = struct.Struct(">IIII")
+
+
+@dataclasses.dataclass
+class DmaEntry:
+    """
+    A Python counterpart to the dmadata entry struct:
+    ```c
+    typedef struct {
+        /* 0x00 */ uintptr_t vromStart;
+        /* 0x04 */ uintptr_t vromEnd;
+        /* 0x08 */ uintptr_t romStart;
+        /* 0x0C */ uintptr_t romEnd;
+    } DmaEntry;
+    ```
+    """
+
+    vrom_start: int
+    vrom_end: int
+    rom_start: int
+    rom_end: int
+
+    def __repr__(self):
+        return (
+            "DmaEntry("
+            f"vrom_start=0x{self.vrom_start:08X}, "
+            f"vrom_end=0x{self.vrom_end:08X}, "
+            f"rom_start=0x{self.rom_start:08X}, "
+            f"rom_end=0x{self.rom_end:08X}"
+            ")"
+        )
+
+    SIZE_BYTES = STRUCT_IIII.size
+
+    def to_bin(self, data: memoryview):
+        STRUCT_IIII.pack_into(
+            data,
+            0,
+            self.vrom_start,
+            self.vrom_end,
+            self.rom_start,
+            self.rom_end,
+        )
+
+    @staticmethod
+    def from_bin(data: memoryview):
+        return DmaEntry(*STRUCT_IIII.unpack_from(data))
+
+    def is_compressed(self) -> bool:
+        return self.rom_end != 0
+
+
+DMA_ENTRY_END = DmaEntry(0, 0, 0, 0)
+
+
+def read_dmadata(rom_data: memoryview, start_offset: int) -> list[DmaEntry]:
+    result = []
+
+    offset = start_offset
+    while (
+        entry := DmaEntry.from_bin(rom_data[offset : offset + DmaEntry.SIZE_BYTES])
+    ) != DMA_ENTRY_END:
+        result.append(entry)
+        offset += DmaEntry.SIZE_BYTES
+
+    return result

tools/dmadata_start.sh

@@ -11,6 +11,5 @@ ELF=$2
 dmadata_syms=`$NM $ELF --no-sort --radix=x --format=bsd | grep dmadata`
 
 _dmadataSegmentRomStart=`echo "$dmadata_syms" | grep '\b_dmadataSegmentRomStart\b' | cut -d' ' -f1`
-_dmadataSegmentRomEnd=`  echo "$dmadata_syms" | grep '\b_dmadataSegmentRomEnd\b'   | cut -d' ' -f1`
 
-echo 0x"$_dmadataSegmentRomStart"-0x"$_dmadataSegmentRomEnd"
+echo 0x"$_dmadataSegmentRomStart"

tools/mkdmadata.c

@@ -16,24 +16,38 @@ static void write_dmadata_table(FILE *fout)
 {
     int i;
 
-    for (i = 0; i < g_segmentsCount; i++)
+    for (i = 0; i < g_segmentsCount; i++) {
+        // Don't emit dma entry for segments set with NOLOAD
+        if (g_segments[i].flags & FLAG_NOLOAD) {
+            continue;
+        }
+
         fprintf(fout, "DEFINE_DMA_ENTRY(%s, \"%s\")\n", g_segments[i].name, g_segments[i].name);
     }
+}
 
 static void write_compress_ranges(FILE *fout)
 {
     int i;
+    int rom_index = 0;
     bool continue_list = false;
     int stride_first = -1;
 
     for (i = 0; i < g_segmentsCount; i++) {
-        if (g_segments[i].compress) {
+        bool compress = g_segments[i].compress;
-            if (stride_first == -1)
+
-                stride_first = i;
+        // Don't consider segments set with NOLOAD when calculating indices
+        if (g_segments[i].flags & FLAG_NOLOAD) {
+            continue;
         }
-        if (!g_segments[i].compress || i == g_segmentsCount - 1) {
+
+        if (compress) {
+            if (stride_first == -1)
+                stride_first = rom_index;
+        }
+        if (!compress || i == g_segmentsCount - 1) {
             if (stride_first != -1) {
-                int stride_last = i - 1;
+                int stride_last = compress ? rom_index : rom_index - 1;
                 if (continue_list) {
                     fprintf(fout, ",");
                 }
@@ -46,6 +60,8 @@ static void write_compress_ranges(FILE *fout)
                 stride_first = -1;
             }
         }
+
+        rom_index++;
     }
 }
 

tools/spec.c

@@ -83,6 +83,8 @@ static bool parse_flags(char *str, unsigned int *flags)
             f |= FLAG_OBJECT;
         else if (strcmp(str, "RAW") == 0)
             f |= FLAG_RAW;
+        else if (strcmp(str, "NOLOAD") == 0)
+            f |= FLAG_NOLOAD;
         else
             return false;
 

tools/spec.h

@@ -27,6 +27,7 @@ enum {
     FLAG_BOOT = (1 << 0),
     FLAG_OBJECT = (1 << 1),
     FLAG_RAW = (1 << 2),
+    FLAG_NOLOAD = (1 << 3),
 };
 
 struct Include {