Setup rom decompression and compression (#1614)

* decompress baserom

* cleanup

* specific hash check

* rename baserom

* git subrepo clone (merge) --branch=5da3132606e4fd427a8d72b8428e4f921cd6e56f git@github.com:z64tools/z64compress.git tools/z64compress

subrepo:
  subdir:   "tools/z64compress"
  merged:   "5da313260"
upstream:
  origin:   "git@github.com:z64tools/z64compress.git"
  branch:   "5da3132606e4fd427a8d72b8428e4f921cd6e56f"
  commit:   "5da313260"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo.git"
  commit:   "2f68596"

* setup compression

* Add all compressed segments to the spec

* Update md5 files

* readme instructions

* cleanup

* Setup python dependencies on Jenkinsfile

* Update Makefile

Co-authored-by: cadmic <cadmic24@gmail.com>

* review

* . .venv/bin/activate

* update readme

* whoops

* Yeet other versions from decompress_baserom.py

* my bad

* Move everything to baseroms/VERSION/

* Active venv on each command

* jenkinsfile: use multiline syntax

* Put the correct path on the jenkinsfile

* Forgot to call per_version_fixes

* CC0

* Update readme

* Change where baserom segments are put into

* Update Makefile

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

* Update crunch64

* Label compressed instead of uncompressed

* Update README.md

Co-authored-by: fig02 <fig02srl@gmail.com>

* Fix

* `make rom`

* baserom-uncompressed

* Update README.md

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

* Update README.md

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

* Update README.md

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

* Update README.md

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

* Update README.md

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

* review

* baserom-decompressed.z64

* ignore baseroms

* rm -rf tools/z64compress

* wip crunch64-based compress.py

* OK compress

* use ipl3checksum sum directly for cic update on compressed rom

* multithreading

* "docs"

* fix MT: move set_sigint_ignored to global level for pickling

* license compress.py

* rm junk

* Fix (or at least sort out) compress_ranges.txt dependencies

* Update tools/overlayhelpers/damage_table.py

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

---------

Co-authored-by: cadmic <cadmic24@gmail.com>
Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>
Co-authored-by: fig02 <fig02srl@gmail.com>

tools/assist.py

@@ -13,7 +13,7 @@ asm_dir = root_dir + "asm/non_matchings/overlays/actors"
 build_dir = root_dir + "build/gc-eu-mq-dbg/"
 
 def read_rom():
-    with open("baserom.z64", "rb") as f:
+    with open("baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", "rb") as f:
         return f.read()
 
 

tools/compress.py

@@ -0,0 +1,342 @@
+# SPDX-FileCopyrightText: 2024 zeldaret
+# SPDX-License-Identifier: CC0-1.0
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import dataclasses
+import struct
+import time
+import multiprocessing
+import multiprocessing.pool
+
+import crunch64
+
+
+STRUCT_IIII = struct.Struct(">IIII")
+
+
+@dataclasses.dataclass
+class DmaEntry:
+    """
+    A Python counterpart to the dmadata entry struct:
+    ```c
+    typedef struct {
+        /* 0x00 */ uintptr_t vromStart;
+        /* 0x04 */ uintptr_t vromEnd;
+        /* 0x08 */ uintptr_t romStart;
+        /* 0x0C */ uintptr_t romEnd;
+    } DmaEntry;
+    ```
+    """
+
+    vromStart: int
+    vromEnd: int
+    romStart: int
+    romEnd: int
+
+    def __repr__(self):
+        return (
+            "DmaEntry("
+            f"vromStart=0x{self.vromStart:08X}, "
+            f"vromEnd=0x{self.vromEnd:08X}, "
+            f"romStart=0x{self.romStart:08X}, "
+            f"romEnd=0x{self.romEnd:08X}"
+            ")"
+        )
+
+    SIZE_BYTES = STRUCT_IIII.size
+
+    def to_bin(self, data: memoryview):
+        STRUCT_IIII.pack_into(
+            data,
+            0,
+            self.vromStart,
+            self.vromEnd,
+            self.romStart,
+            self.romEnd,
+        )
+
+    @staticmethod
+    def from_bin(data: memoryview):
+        return DmaEntry(*STRUCT_IIII.unpack_from(data))
+
+
+DMA_ENTRY_ZERO = DmaEntry(0, 0, 0, 0)
+
+
+def align(v: int):
+    v += 0xF
+    return v // 0x10 * 0x10
+
+
+@dataclasses.dataclass
+class RomSegment:
+    vromStart: int
+    vromEnd: int
+    is_compressed: bool
+    data: memoryview | None
+    data_async: multiprocessing.pool.AsyncResult | None
+
+    @property
+    def uncompressed_size(self):
+        return self.vromEnd - self.vromStart
+
+
+# Make interrupting the compression with ^C less jank
+# https://stackoverflow.com/questions/72967793/keyboardinterrupt-with-python-multiprocessing-pool
+def set_sigint_ignored():
+    import signal
+
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+
+def compress_rom(
+    rom_data: memoryview,
+    dmadata_offset_start: int,
+    dmadata_offset_end: int,
+    compress_entries_indices: set[int],
+    n_threads: int = None,
+):
+    """
+    rom_data: the uncompressed rom data
+    dmadata_offset_start: the offset in the rom where the dmadata starts (inclusive)
+    dmadata_offset_end: the offset in the rom where the dmadata ends (exclusive)
+    compress_entries_indices: the indices in the dmadata of the segments that should be compressed
+    n_threads: how many cores to use for compression
+    """
+
+    # Segments of the compressed rom (not all are compressed)
+    compressed_rom_segments: list[RomSegment] = []
+
+    with multiprocessing.Pool(n_threads, initializer=set_sigint_ignored) as p:
+        # Extract each segment from the input rom
+        for entry_index, dmadata_offset in enumerate(
+            range(dmadata_offset_start, dmadata_offset_end, DmaEntry.SIZE_BYTES)
+        ):
+            dma_entry = DmaEntry.from_bin(rom_data[dmadata_offset:])
+            if dma_entry == DMA_ENTRY_ZERO:
+                continue
+
+            segment_rom_start = dma_entry.romStart
+            segment_rom_end = dma_entry.romStart + (
+                dma_entry.vromEnd - dma_entry.vromStart
+            )
+            segment_data_uncompressed = rom_data[segment_rom_start:segment_rom_end]
+
+            is_compressed = entry_index in compress_entries_indices
+
+            if is_compressed:
+                segment_data = None
+                segment_data_async = p.apply_async(
+                    crunch64.yaz0.compress,
+                    (bytes(segment_data_uncompressed),),
+                )
+            else:
+                segment_data = segment_data_uncompressed
+                segment_data_async = None
+
+            compressed_rom_segments.append(
+                RomSegment(
+                    dma_entry.vromStart,
+                    dma_entry.vromEnd,
+                    is_compressed,
+                    segment_data,
+                    segment_data_async,
+                )
+            )
+
+        # Technically optional but required for matching.
+        compressed_rom_segments.sort(key=lambda segment: segment.vromStart)
+
+        # Wait on compression of all compressed segments
+        waiting_on_segments = [
+            segment for segment in compressed_rom_segments if segment.is_compressed
+        ]
+        total_uncompressed_size_of_data_to_compress = sum(
+            segment.uncompressed_size for segment in waiting_on_segments
+        )
+        uncompressed_size_of_data_compressed_so_far = 0
+        while waiting_on_segments:
+            # Show progress
+            progress = (
+                uncompressed_size_of_data_compressed_so_far
+                / total_uncompressed_size_of_data_to_compress
+            )
+            print(f"Compressing... {progress * 100:.1f}%", end="\r")
+
+            # The segments for which the compression is not finished yet are
+            # added to this list
+            still_waiting_on_segments = []
+            got_some_results = False
+            for segment in waiting_on_segments:
+                assert segment.data is None
+                assert segment.data_async is not None
+
+                try:
+                    compressed_data = segment.data_async.get(0)
+                except multiprocessing.TimeoutError:
+                    # Compression not finished yet
+                    still_waiting_on_segments.append(segment)
+                else:
+                    # Compression finished!
+                    assert isinstance(compressed_data, bytes)
+                    segment.data = memoryview(compressed_data)
+                    uncompressed_size_of_data_compressed_so_far += (
+                        segment.uncompressed_size
+                    )
+                    got_some_results = True
+                    segment.data_async = None
+
+            if not got_some_results and still_waiting_on_segments:
+                # Nothing happened this wait iteration, idle a bit
+                time.sleep(0.010)
+
+            waiting_on_segments = still_waiting_on_segments
+
+    print("Putting together the compressed rom...")
+
+    # Put together the compressed rom
+    compressed_rom_size = sum(
+        align(len(segment.data)) for segment in compressed_rom_segments
+    )
+    pad_to_multiple_of = 8 * 2**20  # 8 MiB
+    compressed_rom_size_padded = (
+        (compressed_rom_size + pad_to_multiple_of - 1)
+        // pad_to_multiple_of
+        * pad_to_multiple_of
+    )
+    compressed_rom_data = memoryview(bytearray(compressed_rom_size_padded))
+    compressed_rom_dma_entries: list[DmaEntry] = []
+    rom_offset = 0
+    for segment in compressed_rom_segments:
+        assert segment.data is not None
+
+        segment_rom_start = rom_offset
+        segment_rom_end = align(segment_rom_start + len(segment.data))
+
+        i = segment_rom_start + len(segment.data)
+        assert i <= len(compressed_rom_data)
+        compressed_rom_data[segment_rom_start:i] = segment.data
+
+        compressed_rom_dma_entries.append(
+            DmaEntry(
+                segment.vromStart,
+                segment.vromEnd,
+                segment_rom_start,
+                segment_rom_end if segment.is_compressed else 0,
+            )
+        )
+
+        rom_offset = segment_rom_end
+
+    assert rom_offset == compressed_rom_size
+    # Pad the compressed rom with the pattern matching the baseroms
+    for i in range(compressed_rom_size, compressed_rom_size_padded):
+        compressed_rom_data[i] = i % 256
+
+    # Write the new dmadata
+    dmadata_offset = dmadata_offset_start
+    for dma_entry in compressed_rom_dma_entries:
+        assert dmadata_offset + DmaEntry.SIZE_BYTES <= dmadata_offset_end
+
+        dma_entry.to_bin(compressed_rom_data[dmadata_offset:])
+
+        dmadata_offset += DmaEntry.SIZE_BYTES
+
+    return compressed_rom_data
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--in",
+        dest="in_rom",
+        required=True,
+        help="path to an uncompressed rom to be compressed",
+    )
+    parser.add_argument(
+        "--out",
+        dest="out_rom",
+        required=True,
+        help="path of the compressed rom to write out",
+    )
+    parser.add_argument(
+        "--dma-range",
+        dest="dma_range",
+        required=True,
+        help=(
+            "The dmadata location in the rom, in format"
+            " 'start_inclusive-end_exclusive' and using hexadecimal offsets"
+            " (e.g. '0x12f70-0x19030')."
+        ),
+    )
+    parser.add_argument(
+        "--compress",
+        dest="compress_ranges",
+        required=True,
+        help=(
+            "The indices in the dmadata of the entries to be compressed,"
+            " where 0 is the first entry."
+            " It is a comma-separated list of individual indices and inclusive ranges."
+            " e.g. '0-1,3,5,6-9' is all indices from 0 to 9 (included) except 2 and 4."
+        ),
+    )
+    parser.add_argument(
+        "--threads",
+        dest="n_threads",
+        type=int,
+        default=1,
+        help="how many cores to use for parallel compression",
+    )
+    args = parser.parse_args()
+
+    in_rom_p = Path(args.in_rom)
+    if not in_rom_p.exists():
+        parser.error(f"Input rom file {in_rom_p} doesn't exist.")
+
+    out_rom_p = Path(args.out_rom)
+
+    dma_range_str: str = args.dma_range
+    dma_range_ends_str = dma_range_str.split("-")
+    assert len(dma_range_ends_str) == 2, dma_range_str
+    dmadata_offset_start, dmadata_offset_end = (
+        int(v_str, 16) for v_str in dma_range_ends_str
+    )
+    assert dmadata_offset_start < dmadata_offset_end, dma_range_str
+
+    compress_ranges_str: str = args.compress_ranges
+    compress_entries_indices = set()
+    for compress_range_str in compress_ranges_str.split(","):
+        compress_range_ends_str = compress_range_str.split("-")
+        assert len(compress_range_ends_str) <= 2, (
+            compress_range_ends_str,
+            compress_range_str,
+            compress_ranges_str,
+        )
+        compress_range_ends = [int(v_str) for v_str in compress_range_ends_str]
+        if len(compress_range_ends) == 1:
+            compress_entries_indices.add(compress_range_ends[0])
+        else:
+            assert len(compress_range_ends) == 2
+            compress_range_first, compress_range_last = compress_range_ends
+            compress_entries_indices.update(
+                range(compress_range_first, compress_range_last + 1)
+            )
+
+    n_threads = args.n_threads
+
+    in_rom_data = in_rom_p.read_bytes()
+    out_rom_data = compress_rom(
+        memoryview(in_rom_data),
+        dmadata_offset_start,
+        dmadata_offset_end,
+        compress_entries_indices,
+        n_threads,
+    )
+    out_rom_p.write_bytes(out_rom_data)
+
+
+if __name__ == "__main__":
+    main()

tools/csdis.py

@@ -496,7 +496,7 @@ def main():
 
     script_dir = os.path.dirname(os.path.realpath(__file__))
     cs_data = None
-    with open(script_dir + "/../baserom/" + file_result.name, "rb") as ovl_file:
+    with open(script_dir + "/../baseroms/gc-eu-mq-dbg/segments/" + file_result.name, "rb") as ovl_file:
         ovl_file.seek(file_result.offset)
         cs_data = [i[0] for i in struct.iter_unpack(">I",  bytearray(ovl_file.read()))]
     if cs_data is not None:

tools/decompress_baserom.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: © 2024 ZeldaRET
+# SPDX-License-Identifier: CC0-1.0
+
+from __future__ import annotations
+
+import hashlib
+import io
+import struct
+from pathlib import Path
+import argparse
+
+import crunch64
+import ipl3checksum
+import zlib
+
+
+def decompress_zlib(data: bytes) -> bytes:
+    decomp = zlib.decompressobj(-zlib.MAX_WBITS)
+    output = bytearray()
+    output.extend(decomp.decompress(data))
+    while decomp.unconsumed_tail:
+        output.extend(decomp.decompress(decomp.unconsumed_tail))
+    output.extend(decomp.flush())
+    return bytes(output)
+
+
+def decompress(data: bytes, is_zlib_compressed: bool) -> bytes:
+    if is_zlib_compressed:
+        return decompress_zlib(data)
+    return crunch64.yaz0.decompress(data)
+
+
+FILE_TABLE_OFFSET = {
+    "gc-eu-mq-dbg":     0x12F70,
+}
+
+VERSIONS_MD5S = {
+    "gc-eu-mq-dbg":     "f0b7f35375f9cc8ca1b2d59d78e35405",
+}
+
+
+def round_up(n, shift):
+    mod = 1 << shift
+    return (n + mod - 1) >> shift << shift
+
+
+def as_word_list(b) -> list[int]:
+    return [i[0] for i in struct.iter_unpack(">I", b)]
+
+
+def read_dmadata_entry(file_content: bytearray, addr: int) -> list[int]:
+    return as_word_list(file_content[addr : addr + 0x10])
+
+
+def read_dmadata(file_content: bytearray, start) -> list[list[int]]:
+    dmadata = []
+    addr = start
+    entry = read_dmadata_entry(file_content, addr)
+    i = 0
+    while any([e != 0 for e in entry]):
+        dmadata.append(entry)
+        addr += 0x10
+        i += 1
+        entry = read_dmadata_entry(file_content, addr)
+    return dmadata
+
+
+def update_crc(decompressed: io.BytesIO) -> io.BytesIO:
+    print("Recalculating crc...")
+    calculated_checksum = ipl3checksum.CICKind.CIC_X105.calculateChecksum(
+        bytes(decompressed.getbuffer())
+    )
+    new_crc = struct.pack(f">II", calculated_checksum[0], calculated_checksum[1])
+
+    decompressed.seek(0x10)
+    decompressed.write(new_crc)
+    return decompressed
+
+
+def decompress_rom(
+    file_content: bytearray, dmadata_addr: int, dmadata: list[list[int]], version: str
+) -> bytearray:
+    rom_segments = {}  # vrom start : data s.t. len(data) == vrom_end - vrom_start
+    new_dmadata = bytearray()  # new dmadata: {vrom start , vrom end , vrom start , 0}
+
+    decompressed = io.BytesIO(b"")
+
+    for v_start, v_end, p_start, p_end in dmadata:
+        if p_start == 0xFFFFFFFF and p_end == 0xFFFFFFFF:
+            new_dmadata.extend(struct.pack(">IIII", v_start, v_end, p_start, p_end))
+            continue
+        if p_end == 0:  # uncompressed
+            rom_segments.update(
+                {v_start: file_content[p_start : p_start + v_end - v_start]}
+            )
+        else:  # compressed
+            rom_segments.update(
+                {
+                    v_start: decompress(
+                        file_content[p_start:p_end], version in {"ique-cn", "ique-zh"}
+                    )
+                }
+            )
+        new_dmadata.extend(struct.pack(">IIII", v_start, v_end, v_start, 0))
+
+    # write rom segments to vaddrs
+    for vrom_st, data in rom_segments.items():
+        decompressed.seek(vrom_st)
+        decompressed.write(data)
+    # write new dmadata
+    decompressed.seek(dmadata_addr)
+    decompressed.write(new_dmadata)
+    # pad to size
+    padding_end = round_up(dmadata[-1][1], 14)
+    decompressed.seek(padding_end - 1)
+    decompressed.write(bytearray([0]))
+    # re-calculate crc
+    return bytearray(update_crc(decompressed).getbuffer())
+
+
+def get_str_hash(byte_array):
+    return str(hashlib.md5(byte_array).hexdigest())
+
+
+def check_existing_rom(rom_path: Path, correct_str_hash: str):
+    # If the baserom exists and is correct, we don't need to change anything
+    if rom_path.exists():
+        if get_str_hash(rom_path.read_bytes()) == correct_str_hash:
+            return True
+    return False
+
+
+def word_swap(file_content: bytearray) -> bytearray:
+    words = str(int(len(file_content) / 4))
+    little_byte_format = "<" + words + "I"
+    big_byte_format = ">" + words + "I"
+    tmp = struct.unpack_from(little_byte_format, file_content, 0)
+    struct.pack_into(big_byte_format, file_content, 0, *tmp)
+    return file_content
+
+
+def byte_swap(file_content: bytearray) -> bytearray:
+    halfwords = str(int(len(file_content) / 2))
+    little_byte_format = "<" + halfwords + "H"
+    big_byte_format = ">" + halfwords + "H"
+    tmp = struct.unpack_from(little_byte_format, file_content, 0)
+    struct.pack_into(big_byte_format, file_content, 0, *tmp)
+    return file_content
+
+
+def per_version_fixes(file_content: bytearray, version: str) -> bytearray:
+    if version == "gc-eu-mq-dbg":
+        # Strip the overdump
+        print("Stripping overdump...")
+        file_content = file_content[0:0x3600000]
+
+        # Patch the header
+        print("Patching header...")
+        file_content[0x3E] = 0x50
+    return file_content
+
+
+def pad_rom(file_content: bytearray, dmadata: list[list[int]]) -> bytearray:
+    padding_start = round_up(dmadata[-1][1], 12)
+    padding_end = round_up(dmadata[-1][1], 14)
+    print(f"Padding from {padding_start:X} to {padding_end:X}...")
+    for i in range(padding_start, padding_end):
+        file_content[i] = 0xFF
+    return file_content
+
+# Determine if we have a ROM file
+ROM_FILE_EXTENSIONS = ["z64", "n64", "v64"]
+
+def find_baserom(version: str) -> Path | None:
+    for rom_file_ext_lower in ROM_FILE_EXTENSIONS:
+        for rom_file_ext in (rom_file_ext_lower, rom_file_ext_lower.upper()):
+            rom_file_name_candidate =  Path(f"baseroms/{version}/baserom.{rom_file_ext}")
+            if rom_file_name_candidate.exists():
+                return rom_file_name_candidate
+    return None
+
+
+def main():
+    description = "Convert a rom that uses dmadata to an uncompressed one."
+
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument("version", help="Version of the game to decompress.", choices=list(VERSIONS_MD5S.keys()))
+
+    args = parser.parse_args()
+
+    version = args.version
+
+    uncompressed_path = Path(f"baseroms/{version}/baserom-decompressed.z64")
+
+    file_table_offset = FILE_TABLE_OFFSET[version]
+    correct_str_hash = VERSIONS_MD5S[version]
+
+    if check_existing_rom(uncompressed_path, correct_str_hash):
+        print("Found valid baserom - exiting early")
+        return
+
+    rom_file_name = find_baserom(version)
+
+    if rom_file_name is None:
+        path_list = [
+            f"baseroms/{version}/baserom.{rom_file_ext}" for rom_file_ext in ROM_FILE_EXTENSIONS
+        ]
+        print(f"Error: Could not find {','.join(path_list)}.")
+        exit(1)
+
+    # Read in the original ROM
+    print(f"File '{rom_file_name}' found.")
+
+    file_content = bytearray(rom_file_name.read_bytes())
+
+    # Check if ROM needs to be byte/word swapped
+    # Little-endian
+    if file_content[0] == 0x40:
+        # Word Swap ROM
+        print("ROM needs to be word swapped...")
+        file_content = word_swap(file_content)
+        print("Word swapping done.")
+
+    # Byte-swapped
+    elif file_content[0] == 0x37:
+        # Byte Swap ROM
+        print("ROM needs to be byte swapped...")
+        file_content = byte_swap(file_content)
+        print("Byte swapping done.")
+
+    file_content = per_version_fixes(file_content, version)
+
+    dmadata = read_dmadata(file_content, file_table_offset)
+    # Decompress
+    if any(
+        [
+            b != 0
+            for b in file_content[
+                file_table_offset + 0xAC : file_table_offset + 0xAC + 0x4
+            ]
+        ]
+    ):
+        print("Decompressing rom...")
+        file_content = decompress_rom(file_content, file_table_offset, dmadata, version)
+
+    file_content = pad_rom(file_content, dmadata)
+
+    # Check to see if the ROM is a "vanilla" ROM
+    str_hash = get_str_hash(file_content)
+    if str_hash != correct_str_hash:
+        print(
+            f"Error: Expected a hash of {correct_str_hash} but got {str_hash}. The baserom has probably been tampered, find a new one"
+        )
+
+        if version == "gc-eu-mq-dbg":
+            if str_hash == "32fe2770c0f9b1a9cd2a4d449348c1cb":
+                print(
+                    "The provided baserom is a rom which has been edited with ZeldaEdit and is not suitable for use with decomp. Find a new one."
+                )
+
+        exit(1)
+
+    # Write out our new ROM
+    print(f"Writing new ROM {uncompressed_path}.")
+    uncompressed_path.write_bytes(file_content)
+
+    print("Done!")
+
+
+if __name__ == "__main__":
+    main()

tools/disasm_elf_msg.py

@@ -166,7 +166,7 @@ item_ids = {
 
 def disas_elfmsgs(start):
     baserom = None
-    with open("baserom.z64", "rb") as infile:
+    with open("baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", "rb") as infile:
         baserom = bytearray(infile.read())
 
     branches = []

tools/dmadata_range.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+NM=$1
+ELF=$2
+
+# dmadata_syms contents look like:
+# ffffffff80015850 T _dmadataSegmentTextStart
+# 00000000 A _dmadataSegmentTextSize
+# 00011a40 A _dmadataSegmentRomStart
+# ffffffff8001b920 T _dmadataSegmentRoDataEnd
+dmadata_syms=`$NM $ELF --no-sort --radix=x --format=bsd | grep dmadata`
+
+_dmadataSegmentRomStart=`echo "$dmadata_syms" | grep '\b_dmadataSegmentRomStart\b' | cut -d' ' -f1`
+_dmadataSegmentRomEnd=`  echo "$dmadata_syms" | grep '\b_dmadataSegmentRomEnd\b'   | cut -d' ' -f1`
+
+echo 0x"$_dmadataSegmentRomStart"-0x"$_dmadataSegmentRomEnd"

tools/mkdmadata.c

@@ -20,22 +20,53 @@ static void write_dmadata_table(FILE *fout)
         fprintf(fout, "DEFINE_DMA_ENTRY(%s, \"%s\")\n", g_segments[i].name, g_segments[i].name);
 }
 
+static void write_compress_ranges(FILE *fout)
+{
+    int i;
+    bool continue_list = false;
+    int stride_first = -1;
+
+    for (i = 0; i < g_segmentsCount; i++) {
+        if (g_segments[i].compress) {
+            if (stride_first == -1)
+                stride_first = i;
+        }
+        if (!g_segments[i].compress || i == g_segmentsCount - 1) {
+            if (stride_first != -1) {
+                int stride_last = i - 1;
+                if (continue_list) {
+                    fprintf(fout, ",");
+                }
+                if (stride_first == stride_last) {
+                    fprintf(fout, "%d", stride_first);
+                } else {
+                    fprintf(fout, "%d-%d", stride_first, stride_last);
+                }
+                continue_list = true;
+                stride_first = -1;
+            }
+        }
+    }
+}
+
 static void usage(const char *execname)
 {
     fprintf(stderr, "zelda64 dmadata generation tool v0.01\n"
-                    "usage: %s SPEC_FILE DMADATA_TABLE\n"
+                    "usage: %s SPEC_FILE DMADATA_TABLE COMPRESS_RANGES\n"
                     "SPEC_FILE      file describing the organization of object files into segments\n"
                     "DMADATA_TABLE  filename of output dmadata table header\n",
+                    "COMPRESS_RANGES filename to write which files are compressed (e.g. 0-5,7,10-20)\n",
                     execname);
 }
 
 int main(int argc, char **argv)
 {
     FILE *dmaout;
+    FILE *compress_ranges_out;
     void *spec;
     size_t size;
 
-    if (argc != 3)
+    if (argc != 4)
     {
         usage(argv[0]);
         return 1;
@@ -49,6 +80,13 @@ int main(int argc, char **argv)
         util_fatal_error("failed to open file '%s' for writing", argv[2]);
     write_dmadata_table(dmaout);
     fclose(dmaout);
+
+    compress_ranges_out = fopen(argv[3], "w");
+    if (compress_ranges_out == NULL)
+        util_fatal_error("failed to open file '%s' for writing", argv[3]);
+    write_compress_ranges(compress_ranges_out);
+    fclose(compress_ranges_out);
+
     free_rom_spec(g_segments, g_segmentsCount);
     free(spec);
 

tools/msgdis.py

@@ -282,7 +282,7 @@ def read_tables():
     global staff_message_entry_table
 
     baserom = None
-    with open("baserom.z64","rb") as infile:
+    with open("baseroms/gc-eu-mq-dbg/baserom-decompressed.z64","rb") as infile:
         baserom = infile.read()
 
     nes_message_entry_table = as_message_table_entry(baserom[nes_message_entry_table_addr:ger_message_entry_table_addr])
@@ -326,7 +326,7 @@ def dump_all_text():
             nes_offset = segmented_to_physical(entry[3])
             nes_length = next_entry[3] - entry[3]
             nes_text = ""
-            with open("baserom/nes_message_data_static","rb") as infile:
+            with open("baseroms/gc-eu-mq-dbg/segments/nes_message_data_static","rb") as infile:
                 infile.seek(nes_offset)
                 nes_text = fixup_message(decode(infile.read(nes_length), entry[1]).replace("\x00","",-1))
 
@@ -337,13 +337,13 @@ def dump_all_text():
                     next_entry = combined_message_entry_table[i+2]
                 ger_offset = segmented_to_physical(entry[4])
                 ger_length = next_entry[4] - entry[4]
-                with open("baserom/ger_message_data_static","rb") as infile:
+                with open("baseroms/gc-eu-mq-dbg/segments/ger_message_data_static","rb") as infile:
                     infile.seek(ger_offset)
                     ger_text = fixup_message(decode(infile.read(ger_length), entry[1]).replace("\x00","",-1))
 
                 fra_offset = segmented_to_physical(entry[5])
                 fra_length = next_entry[5] - entry[5]
-                with open("baserom/fra_message_data_static","rb") as infile:
+                with open("baseroms/gc-eu-mq-dbg/segments/fra_message_data_static","rb") as infile:
                     infile.seek(fra_offset)
                     fra_text = fixup_message(decode(infile.read(fra_length), entry[1]).replace("\x00","",-1))
 
@@ -352,7 +352,7 @@ def dump_all_text():
     return messages
 
 def dump_staff_text():
-    staff_message_data_static_size = path.getsize("baserom/staff_message_data_static")
+    staff_message_data_static_size = path.getsize("baseroms/gc-eu-mq-dbg/segments/staff_message_data_static")
     # text id, ypos, type, staff
     messages = []
     for i,entry in enumerate(staff_message_entry_table,0):
@@ -361,7 +361,7 @@ def dump_staff_text():
             staff_offset = segmented_to_physical(entry[3])
             # hacky way to ensure the staff message entry table is read all the way to the end
             staff_length = (staff_message_data_static_size if entry[0] == 0x052F else segmented_to_physical(next_entry[3])) - segmented_to_physical(entry[3])
-            with open("baserom/staff_message_data_static","rb") as infile:
+            with open("baseroms/gc-eu-mq-dbg/segments/staff_message_data_static","rb") as infile:
                 infile.seek(staff_offset)
                 messages.append((entry[0], entry[1], entry[2], fixup_message(decode(infile.read(staff_length), entry[1]).replace("\x00","",-1))))
         else:

tools/overlayhelpers/colliderinit.py

@@ -387,7 +387,7 @@ def GetColliderInit(address, type, num, path):
     else:
         return("ItemInit type must specify number of elements")
 
-    ovlFile = open(path + "/baserom/" + fileResult.name, "rb")
+    ovlFile = open(path + "/baseroms/gc-eu-mq-dbg/segments/" + fileResult.name, "rb")
     ovlData = bytearray(ovlFile.read())
     ovlFile.close()
 
@@ -470,7 +470,7 @@ print(GetColliderInitFull(args.address, args.type, args.num, repo))
 
 # script_dir = os.path.dirname(os.path.realpath(__file__))
 
-# ovlFile = open(script_dir + "/../../baserom/" + fileResult.name, "rb")
+# ovlFile = open(script_dir + "/../../baseroms/" + fileResult.name, "rb")
 # ovlData = bytearray(ovlFile.read())
 # ovlFile.close()
 

tools/overlayhelpers/damage_table.py

@@ -132,7 +132,7 @@ def get_damage_bytes(address, repo):
 
     print(file_result)
 
-    with open(repo + os.sep + "baserom" + os.sep + file_result.name, "rb") as ovl_file:
+    with open(repo + os.sep + "baseroms/gc-eu-mq-dbg/segments" + os.sep + file_result.name, "rb") as ovl_file:
         ovl_data = bytearray(ovl_file.read())
 
     damage_data = ovl_data[file_result.offset:file_result.offset+0x20]

tools/overlayhelpers/lmapmark.py

@@ -72,7 +72,7 @@ repo = scriptDir + os.sep +  ".." + os.sep + ".."
 
 
 kaleido_scope_data = []
-with open(repo + "/baserom/ovl_kaleido_scope", "rb") as file:
+with open(repo + "/baseroms/gc-eu-mq-dbg/segments/ovl_kaleido_scope", "rb") as file:
     kaleido_scope_data = bytearray(file.read())
 
 scenemaps = []

tools/overlayhelpers/mapmark.py

@@ -97,7 +97,7 @@ repo = scriptDir + os.sep +  ".." + os.sep + ".."
 
 
 map_mark_data = []
-with open(repo + "/baserom/ovl_map_mark_data", "rb") as file:
+with open(repo + "/baseroms/gc-eu-mq-dbg/segments/ovl_map_mark_data", "rb") as file:
     map_mark_data = bytearray(file.read())
 
 scenemaps = []

tools/spec.c

@@ -127,6 +127,7 @@ static const char *const stmtNames[] =
     [STMT_after]     = "after",
     [STMT_align]     = "align",
     [STMT_beginseg]  = "beginseg",
+    [STMT_compress]  = "compress",
     [STMT_endseg]    = "endseg",
     [STMT_entry]     = "entry",
     [STMT_flags]     = "flags",
@@ -216,10 +217,13 @@ bool parse_segment_statement(struct Segment *currSeg, STMTId stmt, char* args, i
         currSeg->includes[currSeg->includesCount - 1].linkerPadding = 0;
         currSeg->includes[currSeg->includesCount - 1].dataWithRodata = (stmt == STMT_include_data_with_rodata);
         break;
-        case STMT_increment:
+    case STMT_increment:
         if (!parse_number(args, &currSeg->increment))
             util_fatal_error("line %i: expected number after 'increment'", lineNum);
         break;
+    case STMT_compress:
+        currSeg->compress = true;
+        break;
     case STMT_pad_text:
         currSeg->includes[currSeg->includesCount - 1].linkerPadding += 0x10;
         break;

tools/spec.h

@@ -9,6 +9,7 @@ typedef enum {
     STMT_after,
     STMT_align,
     STMT_beginseg,
+    STMT_compress,
     STMT_endseg,
     STMT_entry,
     STMT_flags,
@@ -48,6 +49,7 @@ typedef struct Segment {
     uint32_t number;
     struct Include* includes;
     int includesCount;
+    bool compress;
 } Segment;
 
 void parse_rom_spec(char* spec, struct Segment** segments, int* segment_count);