Introduce version-specific YAML config (#1957)

Makefile

@@ -368,7 +368,7 @@ venv:
 setup: venv
 	$(MAKE) -C tools
 	$(PYTHON) tools/decompress_baserom.py $(VERSION)
-	$(PYTHON) tools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 -o $(EXTRACTED_DIR)/baserom --dmadata-start `cat $(BASEROM_DIR)/dmadata_start.txt` --dmadata-names $(BASEROM_DIR)/dmadata_names.txt
+	$(PYTHON) tools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 --oot-version $(VERSION) -o $(EXTRACTED_DIR)/baserom
 	$(PYTHON) tools/msgdis.py --oot-version $(VERSION) --text-out $(EXTRACTED_DIR)/text/message_data.h --staff-text-out $(EXTRACTED_DIR)/text/message_data_staff.h
 # TODO: for now, we only extract assets from the Debug ROM
 ifeq ($(VERSION),gc-eu-mq-dbg)

baseroms/gc-eu-mq-dbg/config.yml

@@ -0,0 +1,8 @@
+dmadata_start: 0x12F70
+variables:
+  gMtxClear: 0x8012DB20
+  sNesMessageEntryTable: 0x8014B320
+  sGerMessageEntryTable: 0x8014F548
+  sFraMessageEntryTable: 0x80151658
+  sStaffMessageEntryTable: 0x80153768
+  sNesMessageEntryTablePtr: 0x801538F0

baseroms/gc-eu-mq-dbg/dmadata_start.txt

@@ -1 +0,0 @@
-0x12f70

baseroms/gc-eu-mq/config.yml

@@ -0,0 +1,8 @@
+dmadata_start: 0x7170
+variables:
+  gMtxClear: 0x800FBC00
+  sNesMessageEntryTable: 0x801077F0
+  sGerMessageEntryTable: 0x8010BA18
+  sFraMessageEntryTable: 0x8010DB28
+  sStaffMessageEntryTable: 0x8010FC38
+  sNesMessageEntryTablePtr: 0x8010FDC0

baseroms/gc-eu-mq/dmadata_start.txt

@@ -1 +0,0 @@
-0x07170

baseroms/gc-eu/config.yml

@@ -0,0 +1,8 @@
+dmadata_start: 0x7170
+variables:
+  gMtxClear: 0x800FBC20
+  sNesMessageEntryTable: 0x80107810
+  sGerMessageEntryTable: 0x8010BA38
+  sFraMessageEntryTable: 0x8010DB48
+  sStaffMessageEntryTable: 0x8010FC58
+  sNesMessageEntryTablePtr: 0x8010FDE0

baseroms/gc-eu/dmadata_start.txt

@@ -1 +0,0 @@
-0x07170

requirements.txt

@@ -1,6 +1,7 @@
 # Setup and compression
 crunch64>=0.3.1,<1.0.0
 ipl3checksum>=1.2.0,<2.0.0
+pyyaml>=6.0.1,<7.0.0
 
 # asm-differ
 argcomplete

tools/decompress_baserom.py

@@ -16,6 +16,7 @@ import ipl3checksum
 import zlib
 
 import dmadata
+import version_config
 
 
 def decompress_zlib(data: bytes) -> bytes:
@@ -172,8 +173,12 @@ def main():
 
     uncompressed_path = baserom_dir / "baserom-decompressed.z64"
 
-    dmadata_start = int((baserom_dir / "dmadata_start.txt").read_text(), 16)
-    compressed_str_hash = (baserom_dir / "checksum-compressed.md5").read_text().split()[0]
+    config = version_config.load_version_config(version)
+    dmadata_start = config.dmadata_start
+
+    compressed_str_hash = (
+        (baserom_dir / "checksum-compressed.md5").read_text().split()[0]
+    )
     decompressed_str_hash = (baserom_dir / "checksum.md5").read_text().split()[0]
 
     if check_existing_rom(uncompressed_path, decompressed_str_hash):

tools/extract_baserom.py

@@ -10,6 +10,7 @@ from pathlib import Path
 import sys
 
 import dmadata
+import version_config
 
 
 def main():
@@ -19,6 +20,12 @@ def main():
     parser.add_argument(
         "rom", metavar="ROM", type=Path, help="Path to uncompressed ROM"
     )
+    parser.add_argument(
+        "-v",
+        "--oot-version",
+        required=True,
+        help="OOT version",
+    )
     parser.add_argument(
         "-o",
         "--output-dir",
@@ -29,24 +36,20 @@ def main():
     parser.add_argument(
         "--dmadata-start",
         type=lambda s: int(s, 16),
-        required=True,
         help=(
-            "The dmadata location in the rom, as a hexadecimal offset (e.g. 0x12f70)"
+            "Override dmadata location for non-matching ROMs, as a hexadecimal offset (e.g. 0x12F70)"
         ),
     )
-    parser.add_argument(
-        "--dmadata-names",
-        type=Path,
-        required=True,
-        help="Path to file containing segment names",
-    )
 
     args = parser.parse_args()
 
     rom_data = memoryview(args.rom.read_bytes())
 
-    dma_names = args.dmadata_names.read_text().splitlines()
-    dma_entries = dmadata.read_dmadata(rom_data, args.dmadata_start)
+    config = version_config.load_version_config(args.oot_version)
+    dmadata_start = args.dmadata_start or config.dmadata_start
+    dma_names = config.dmadata_segments.keys()
+
+    dma_entries = dmadata.read_dmadata(rom_data, dmadata_start)
     if len(dma_names) != len(dma_entries):
         print(
             f"Error: expected {len(dma_names)} DMA entries but found {len(dma_entries)} in ROM",

tools/msgdis.py

@@ -7,6 +7,8 @@ import re, struct
 from os import path
 import argparse
 
+import version_config
+
 # ===================================================
 #   Util
 # ===================================================
@@ -285,7 +287,7 @@ def read_tables():
     global staff_message_entry_table
 
     baserom = None
-    with open(f"baseroms/{version}/baserom-decompressed.z64","rb") as infile:
+    with open(f"extracted/{version}/baserom/code","rb") as infile:
         baserom = infile.read()
 
     nes_message_entry_table = as_message_table_entry(baserom[nes_message_entry_table_addr:ger_message_entry_table_addr])
@@ -437,26 +439,15 @@ def main():
         parser.error("No output file requested")
 
     version = args.oot_version
-    if version == "gc-eu-mq-dbg":
-        nes_message_entry_table_addr = 0x00BC24C0
-        ger_message_entry_table_addr = 0x00BC66E8
-        fra_message_entry_table_addr = 0x00BC87F8
-        staff_message_entry_table_addr = 0x00BCA908
-        staff_message_entry_table_addr_end = 0x00BCAA90
-    elif version == "gc-eu-mq":
-        nes_message_entry_table_addr = 0x00B7E8F0
-        ger_message_entry_table_addr = 0x00B82B18
-        fra_message_entry_table_addr = 0x00B84C28
-        staff_message_entry_table_addr = 0x00B86D38
-        staff_message_entry_table_addr_end = 0x00B86EC0
-    elif version == "gc-eu":
-        nes_message_entry_table_addr = 0x00B7E910
-        ger_message_entry_table_addr = 0x00B82B38
-        fra_message_entry_table_addr = 0x00B84C48
-        staff_message_entry_table_addr = 0x00B86D58
-        staff_message_entry_table_addr_end = 0x00B86EE0
-    else:
-        parser.error("Unsupported OOT version")
+
+    config = version_config.load_version_config(version)
+    code_vram = config.dmadata_segments["code"].vram
+
+    nes_message_entry_table_addr = config.variables["sNesMessageEntryTable"] - code_vram
+    ger_message_entry_table_addr = config.variables["sGerMessageEntryTable"] - code_vram
+    fra_message_entry_table_addr = config.variables["sFraMessageEntryTable"] - code_vram
+    staff_message_entry_table_addr = config.variables["sStaffMessageEntryTable"] - code_vram
+    staff_message_entry_table_addr_end = config.variables["sNesMessageEntryTablePtr"] - code_vram
 
     extract_all_text(args.text_out, args.staff_text_out)
 

tools/version_config.py

@@ -0,0 +1,53 @@
+# Version-specific configuration for setup and assets extraction
+
+# SPDX-FileCopyrightText: © 2024 ZeldaRET
+# SPDX-License-Identifier: CC0-1.0
+
+from __future__ import annotations
+
+from collections import OrderedDict
+import csv
+import dataclasses
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+PROJECT_ROOT = Path(__file__).parent.parent
+
+
+@dataclasses.dataclass
+class VersionConfig:
+    # ROM offset to start of DMA table
+    dmadata_start: int
+    # DMA segment information, in ROM order
+    dmadata_segments: OrderedDict[str, SegmentInfo]
+    # Addresses of important variables needed for asset extraction
+    variables: Dict[str, int]
+
+
+@dataclasses.dataclass
+class SegmentInfo:
+    name: str
+    vram: int | None
+
+
+def load_dmadata_segments(version: str) -> OrderedDict[str, SegmentInfo]:
+    segments = OrderedDict()
+    with open(PROJECT_ROOT / f"baseroms/{version}/segments.csv", "r") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            name = row["Name"]
+            vram = int(row["VRAM start"], 16) if row["VRAM start"] else None
+            segments[name] = SegmentInfo(name, vram)
+    return segments
+
+
+def load_version_config(version: str) -> VersionConfig:
+    with open(PROJECT_ROOT / f"baseroms/{version}/config.yml", "r") as f:
+        config = yaml.load(f, Loader=yaml.Loader)
+    return VersionConfig(
+        dmadata_start=config["dmadata_start"],
+        dmadata_segments=load_dmadata_segments(version),
+        variables=config["variables"],
+    )