From 5515aa170567f1dd52b625a5adae7619ccdde291 Mon Sep 17 00:00:00 2001 From: cadmic Date: Sat, 27 Jul 2024 21:24:13 -0700 Subject: [PATCH] Configure incbins in version config instead of using the Debug ROM (#2005) * Run CPP for data/*.s * Configure incbins in version config instead of using the Debug ROM --- Makefile | 5 ++- baseroms/gc-eu-mq-dbg/config.yml | 65 ++++++++++++++++++++++++++++++++ baseroms/gc-eu-mq/config.yml | 57 ++++++++++++++++++++++++++++ baseroms/gc-eu/config.yml | 57 ++++++++++++++++++++++++++++ baseroms/gc-us/config.yml | 57 ++++++++++++++++++++++++++++ data/audio_tables.rodata.s | 16 ++++---- data/rsp.rodata.s | 18 ++++----- data/rsp.text.s | 14 +++---- data/rsp_boot.text.s | 10 ++--- data/unk_800093F0.data.s | 15 ++++---- data/unk_80009410.data.s | 10 ++--- data/unk_8012ABC0.data.s | 10 ++--- data/z_text.data.s | 16 ++++---- docs/tutorial/data.md | 8 ++-- src/makerom/ipl3.s | 2 +- tools/extract_incbins.py | 54 ++++++++++++++++++++++++++ tools/version_config.py | 19 ++++++++++ 17 files changed, 373 insertions(+), 60 deletions(-) create mode 100755 tools/extract_incbins.py diff --git a/Makefile b/Makefile index c73d6fddca..eedf572cf9 100644 --- a/Makefile +++ b/Makefile @@ -170,7 +170,7 @@ endif CFLAGS += $(GBI_DEFINES) -ASFLAGS := -march=vr4300 -32 -no-pad-sections -Iinclude +ASFLAGS := -march=vr4300 -32 -no-pad-sections -Iinclude -I$(EXTRACTED_DIR) ifeq ($(COMPILER),gcc) CFLAGS += -G 0 -nostdinc $(INC) -march=vr4300 -mfix4300 -mabi=32 -mno-abicalls -mdivide-breaks -fno-PIC -fno-common -ffreestanding -fbuiltin -fno-builtin-sinf -fno-builtin-cosf $(CHECK_WARNINGS) -funsigned-char @@ -394,6 +394,7 @@ setup: venv $(MAKE) -C tools $(PYTHON) tools/decompress_baserom.py $(VERSION) $(PYTHON) tools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 --oot-version $(VERSION) -o $(EXTRACTED_DIR)/baserom + $(PYTHON) tools/extract_incbins.py $(EXTRACTED_DIR)/baserom --oot-version $(VERSION) -o $(EXTRACTED_DIR)/incbin $(PYTHON) tools/msgdis.py $(VERSION) $(PYTHON) extract_assets.py -v $(VERSION) -j$(N_THREADS) @@ -448,7 +449,7 @@ $(BUILD_DIR)/baserom/%.o: $(EXTRACTED_DIR)/baserom/% $(OBJCOPY) -I binary -O elf32-big $< $@ $(BUILD_DIR)/data/%.o: data/%.s - $(AS) $(ASFLAGS) $< -o $@ + $(CPP) $(CPPFLAGS) -Iinclude $< | $(AS) $(ASFLAGS) -o $@ $(BUILD_DIR)/assets/text/%.enc.jpn.h: assets/text/%.h $(EXTRACTED_DIR)/text/%.h assets/text/charmap.txt $(CPP) $(CPPFLAGS) -I$(EXTRACTED_DIR) $< | $(PYTHON) tools/msgenc.py --encoding jpn --charmap assets/text/charmap.txt - $@ diff --git a/baseroms/gc-eu-mq-dbg/config.yml b/baseroms/gc-eu-mq-dbg/config.yml index bd6c9bf6b3..c5e301afad 100644 --- a/baseroms/gc-eu-mq-dbg/config.yml +++ b/baseroms/gc-eu-mq-dbg/config.yml @@ -1,5 +1,70 @@ dmadata_start: 0x12F70 text_lang_pal: true +incbins: + - name: ipl3 + segment: makerom + vram: 0x80000040 + size: 0xFC0 + - name: rspbootText + segment: boot + vram: 0x80009320 + size: 0xD0 + - name: aspMainText + segment: code + vram: 0x801120C0 + size: 0xFB0 + - name: gspS2DEX2d_fifoText + segment: code + vram: 0x80113070 + size: 0x18C0 + - name: njpgdspMainText + segment: code + vram: 0x80114930 + size: 0xAF0 + - name: gMojiFontTLUTs + segment: code + vram: 0x8012A740 + size: 0x80 + - name: gMojiFontTex + segment: code + vram: 0x8012A7C0 + size: 0x400 + - name: gSoundFontTable + segment: code + vram: 0x801550D0 + size: 0x270 + - name: gSequenceFontTable + segment: code + vram: 0x80155340 + size: 0x1C0 + - name: gSequenceTable + segment: code + vram: 0x80155500 + size: 0x6F0 + - name: gSampleBankTable + segment: code + vram: 0x80155BF0 + size: 0x80 + - name: aspMainData + segment: code + vram: 0x80155C70 + size: 0x2E0 + - name: gspF3DZEX2_NoN_PosLight_fifoText + segment: code + vram: 0x80155F50 + size: 0x1630 + - name: gspF3DZEX2_NoN_PosLight_fifoData + segment: code + vram: 0x80157580 + size: 0x420 + - name: gspS2DEX2d_fifoData + segment: code + vram: 0x801579A0 + size: 0x390 + - name: njpgdspMainData + segment: code + vram: 0x80157D30 + size: 0x60 variables: gMtxClear: 0x8012DB20 sNesMessageEntryTable: 0x8014B320 diff --git a/baseroms/gc-eu-mq/config.yml b/baseroms/gc-eu-mq/config.yml index e04429d0a2..34c5cbe9c0 100644 --- a/baseroms/gc-eu-mq/config.yml +++ b/baseroms/gc-eu-mq/config.yml @@ -1,5 +1,62 @@ dmadata_start: 0x7170 text_lang_pal: true +incbins: + - name: ipl3 + segment: makerom + vram: 0x80000040 + size: 0xFC0 + - name: rspbootText + segment: boot + vram: 0x80005FC0 + size: 0xD0 + - name: aspMainText + segment: code + vram: 0x800E11A0 + size: 0xFB0 + - name: gspS2DEX2d_fifoText + segment: code + vram: 0x800E2150 + size: 0x18C0 + - name: njpgdspMainText + segment: code + vram: 0x800E3A10 + size: 0xAF0 + - name: gSoundFontTable + segment: code + vram: 0x80110470 + size: 0x270 + - name: gSequenceFontTable + segment: code + vram: 0x801106E0 + size: 0x1C0 + - name: gSequenceTable + segment: code + vram: 0x801108A0 + size: 0x6F0 + - name: gSampleBankTable + segment: code + vram: 0x80110F90 + size: 0x80 + - name: aspMainData + segment: code + vram: 0x80111010 + size: 0x2E0 + - name: gspF3DZEX2_NoN_PosLight_fifoText + segment: code + vram: 0x801112F0 + size: 0x1630 + - name: gspF3DZEX2_NoN_PosLight_fifoData + segment: code + vram: 0x80112920 + size: 0x420 + - name: gspS2DEX2d_fifoData + segment: code + vram: 0x80112D40 + size: 0x390 + - name: njpgdspMainData + segment: code + vram: 0x801130D0 + size: 0x60 variables: gMtxClear: 0x800FBC00 sNesMessageEntryTable: 0x801077F0 diff --git a/baseroms/gc-eu/config.yml b/baseroms/gc-eu/config.yml index 6f48daec7a..90712dc3f1 100644 --- a/baseroms/gc-eu/config.yml +++ b/baseroms/gc-eu/config.yml @@ -1,5 +1,62 @@ dmadata_start: 0x7170 text_lang_pal: true +incbins: + - name: ipl3 + segment: makerom + vram: 0x80000040 + size: 0xFC0 + - name: rspbootText + segment: boot + vram: 0x80005FC0 + size: 0xD0 + - name: aspMainText + segment: code + vram: 0x800E11C0 + size: 0xFB0 + - name: gspS2DEX2d_fifoText + segment: code + vram: 0x800E2170 + size: 0x18C0 + - name: njpgdspMainText + segment: code + vram: 0x800E3A30 + size: 0xAF0 + - name: gSoundFontTable + segment: code + vram: 0x80110490 + size: 0x270 + - name: gSequenceFontTable + segment: code + vram: 0x80110700 + size: 0x1C0 + - name: gSequenceTable + segment: code + vram: 0x801108C0 + size: 0x6F0 + - name: gSampleBankTable + segment: code + vram: 0x80110FB0 + size: 0x80 + - name: aspMainData + segment: code + vram: 0x80111030 + size: 0x2E0 + - name: gspF3DZEX2_NoN_PosLight_fifoText + segment: code + vram: 0x80111310 + size: 0x1630 + - name: gspF3DZEX2_NoN_PosLight_fifoData + segment: code + vram: 0x80112940 + size: 0x420 + - name: gspS2DEX2d_fifoData + segment: code + vram: 0x80112D60 + size: 0x390 + - name: njpgdspMainData + segment: code + vram: 0x801130F0 + size: 0x60 variables: gMtxClear: 0x800FBC20 sNesMessageEntryTable: 0x80107810 diff --git a/baseroms/gc-us/config.yml b/baseroms/gc-us/config.yml index 0d6acafc34..54498f9a20 100644 --- a/baseroms/gc-us/config.yml +++ b/baseroms/gc-us/config.yml @@ -1,5 +1,62 @@ dmadata_start: 0x7170 text_lang_pal: false +incbins: + - name: ipl3 + segment: makerom + vram: 0x80000040 + size: 0xFC0 + - name: rspbootText + segment: boot + vram: 0x80005FC0 + size: 0xD0 + - name: aspMainText + segment: code + vram: 0x800E3840 + size: 0xFB0 + - name: gspS2DEX2d_fifoText + segment: code + vram: 0x800E47F0 + size: 0x18C0 + - name: njpgdspMainText + segment: code + vram: 0x800E60B0 + size: 0xAF0 + - name: gSoundFontTable + segment: code + vram: 0x80112C80 + size: 0x270 + - name: gSequenceFontTable + segment: code + vram: 0x80112EF0 + size: 0x1C0 + - name: gSequenceTable + segment: code + vram: 0x801130B0 + size: 0x6F0 + - name: gSampleBankTable + segment: code + vram: 0x801137A0 + size: 0x80 + - name: aspMainData + segment: code + vram: 0x80113820 + size: 0x2E0 + - name: gspF3DZEX2_NoN_PosLight_fifoText + segment: code + vram: 0x80113B00 + size: 0x1630 + - name: gspF3DZEX2_NoN_PosLight_fifoData + segment: code + vram: 0x80115130 + size: 0x420 + - name: gspS2DEX2d_fifoData + segment: code + vram: 0x80115550 + size: 0x390 + - name: njpgdspMainData + segment: code + vram: 0x801158E0 + size: 0x60 variables: gMtxClear: 0x800FE2A0 sJpnMessageEntryTable: 0x80109E8C diff --git a/data/audio_tables.rodata.s b/data/audio_tables.rodata.s index 5e76a127b8..4f6655c861 100644 --- a/data/audio_tables.rodata.s +++ b/data/audio_tables.rodata.s @@ -1,22 +1,22 @@ .include "macro.inc" -# assembler directives -.set noat # allow manual use of $at -.set noreorder # don't insert nops after branches -.set gp=64 # allow use of 64-bit general purpose registers +/* assembler directives */ +.set noat /* allow manual use of $at */ +.set noreorder /* don't insert nops after branches */ +.set gp=64 /* allow use of 64-bit general purpose registers */ .section .rodata .balign 16 glabel gSoundFontTable - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCC270, 0x270 + .incbin "incbin/gSoundFontTable" glabel gSequenceFontTable - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCC4E0, 0x1C0 + .incbin "incbin/gSequenceFontTable" glabel gSequenceTable - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCC6A0, 0x6F0 + .incbin "incbin/gSequenceTable" glabel gSampleBankTable - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCCD90, 0x80 + .incbin "incbin/gSampleBankTable" diff --git a/data/rsp.rodata.s b/data/rsp.rodata.s index f3df8983f1..bdb9fe9912 100644 --- a/data/rsp.rodata.s +++ b/data/rsp.rodata.s @@ -1,30 +1,30 @@ .include "macro.inc" -# assembler directives -.set noat # allow manual use of $at -.set noreorder # don't insert nops after branches -.set gp=64 # allow use of 64-bit general purpose registers +/* assembler directives */ +.set noat /* allow manual use of $at */ +.set noreorder /* don't insert nops after branches */ +.set gp=64 /* allow use of 64-bit general purpose registers */ .section .rodata .balign 16 glabel aspMainDataStart - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCCE10, 0x2E0 + .incbin "incbin/aspMainData" glabel aspMainDataEnd glabel gspF3DZEX2_NoN_PosLight_fifoTextStart - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCD0F0, 0x1630 + .incbin "incbin/gspF3DZEX2_NoN_PosLight_fifoText" glabel gspF3DZEX2_NoN_PosLight_fifoTextEnd glabel gspF3DZEX2_NoN_PosLight_fifoDataStart - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCE720, 0x420 + .incbin "incbin/gspF3DZEX2_NoN_PosLight_fifoData" glabel gspF3DZEX2_NoN_PosLight_fifoDataEnd glabel gspS2DEX2d_fifoDataStart - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCEB40, 0x390 + .incbin "incbin/gspS2DEX2d_fifoData" glabel gspS2DEX2d_fifoDataEnd glabel njpgdspMainDataStart - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCEED0, 0x60 + .incbin "incbin/njpgdspMainData" glabel njpgdspMainDataEnd diff --git a/data/rsp.text.s b/data/rsp.text.s index 045d8d6143..fcf48a2515 100644 --- a/data/rsp.text.s +++ b/data/rsp.text.s @@ -1,22 +1,22 @@ .include "macro.inc" -# assembler directives -.set noat # allow manual use of $at -.set noreorder # don't insert nops after branches -.set gp=64 # allow use of 64-bit general purpose registers +/* assembler directives */ +.set noat /* allow manual use of $at */ +.set noreorder /* don't insert nops after branches */ +.set gp=64 /* allow use of 64-bit general purpose registers */ .section .text .balign 16 glabel aspMainTextStart - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xB89260, 0xFB0 + .incbin "incbin/aspMainText" glabel aspMainTextEnd glabel gspS2DEX2d_fifoTextStart - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xB8A210, 0x18C0 + .incbin "incbin/gspS2DEX2d_fifoText" glabel gspS2DEX2d_fifoTextEnd glabel njpgdspMainTextStart - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xB8BAD0, 0xAF0 + .incbin "incbin/njpgdspMainText" glabel njpgdspMainTextEnd diff --git a/data/rsp_boot.text.s b/data/rsp_boot.text.s index 01fe3f0a76..59c9ffc27f 100644 --- a/data/rsp_boot.text.s +++ b/data/rsp_boot.text.s @@ -1,14 +1,14 @@ .include "macro.inc" -# assembler directives -.set noat # allow manual use of $at -.set noreorder # don't insert nops after branches -.set gp=64 # allow use of 64-bit general purpose registers +/* assembler directives */ +.set noat /* allow manual use of $at */ +.set noreorder /* don't insert nops after branches */ +.set gp=64 /* allow use of 64-bit general purpose registers */ .section .text .balign 16 glabel rspbootTextStart - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0x9F20, 0xD0 + .incbin "incbin/rspbootText" glabel rspbootTextEnd diff --git a/data/unk_800093F0.data.s b/data/unk_800093F0.data.s index 380dbe64b1..bf8a636c07 100644 --- a/data/unk_800093F0.data.s +++ b/data/unk_800093F0.data.s @@ -1,17 +1,18 @@ .include "macro.inc" -# assembler directives -.set noat # allow manual use of $at -.set noreorder # don't insert nops after branches -.set gp=64 # allow use of 64-bit general purpose registers +/* assembler directives */ +.set noat /* allow manual use of $at */ +.set noreorder /* don't insert nops after branches */ +.set gp=64 /* allow use of 64-bit general purpose registers */ .section .data .balign 16 -# Unused. -# This appears to be RSP code, however it is not part of -# rspboot as rspbootTextEnd is at this symbol +/* Unused. + * This appears to be RSP code, however it is not part of + * rspboot as rspbootTextEnd is at this symbol. + */ glabel D_800093F0 .word 0xE80C2001 # sqv $v12[0], 0x10($zero) .word 0x34014000 # li $1, 0x4000 diff --git a/data/unk_80009410.data.s b/data/unk_80009410.data.s index d2bded6c36..8ff273e224 100644 --- a/data/unk_80009410.data.s +++ b/data/unk_80009410.data.s @@ -1,15 +1,15 @@ .include "macro.inc" -# assembler directives -.set noat # allow manual use of $at -.set noreorder # don't insert nops after branches -.set gp=64 # allow use of 64-bit general purpose registers +/* assembler directives */ +.set noat /* allow manual use of $at */ +.set noreorder /* don't insert nops after branches */ +.set gp=64 /* allow use of 64-bit general purpose registers */ .section .data .balign 16 -# Unused +/* Unused */ glabel D_80009410 .word osStopThread .word __osSetHWIntrRoutine diff --git a/data/unk_8012ABC0.data.s b/data/unk_8012ABC0.data.s index 31e03580e8..7f99d09e0d 100644 --- a/data/unk_8012ABC0.data.s +++ b/data/unk_8012ABC0.data.s @@ -1,15 +1,15 @@ .include "macro.inc" -# assembler directives -.set noat # allow manual use of $at -.set noreorder # don't insert nops after branches -.set gp=64 # allow use of 64-bit general purpose registers +/* assembler directives */ +.set noat /* allow manual use of $at */ +.set noreorder /* don't insert nops after branches */ +.set gp=64 /* allow use of 64-bit general purpose registers */ .section .data .balign 16 -# Unused +/* Unused */ glabel D_8012ABC0 .word fmodf .word guScale diff --git a/data/z_text.data.s b/data/z_text.data.s index 3fe890cdb6..d2a76e0b8e 100644 --- a/data/z_text.data.s +++ b/data/z_text.data.s @@ -1,18 +1,20 @@ .include "macro.inc" -# assembler directives -.set noat # allow manual use of $at -.set noreorder # don't insert nops after branches -.set gp=64 # allow use of 64-bit general purpose registers +/* assembler directives */ +.set noat /* allow manual use of $at */ +.set noreorder /* don't insert nops after branches */ +.set gp=64 /* allow use of 64-bit general purpose registers */ .section .data .balign 16 -# temporary file name, rename to something more appropriate when decompiled +/* temporary file name, rename to something more appropriate when decompiled */ +#if OOT_DEBUG glabel gMojiFontTLUTs - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBA18E0, 0x80 + .incbin "incbin/gMojiFontTLUTs" glabel gMojiFontTex - .incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBA1960, 0x400 + .incbin "incbin/gMojiFontTex" +#endif diff --git a/docs/tutorial/data.md b/docs/tutorial/data.md index a96d26d84f..bc3bf07659 100644 --- a/docs/tutorial/data.md +++ b/docs/tutorial/data.md @@ -37,10 +37,10 @@ Large code block, click to show ``` .include "macro.inc" - # assembler directives - .set noat # allow manual use of $at - .set noreorder # don't insert nops after branches - .set gp=64 # allow use of 64-bit general purpose registers + /* assembler directives */ + .set noat /* allow manual use of $at */ + .set noreorder /* don't insert nops after branches */ + .set gp=64 /* allow use of 64-bit general purpose registers */ .section .data diff --git a/src/makerom/ipl3.s b/src/makerom/ipl3.s index be02f7e2fc..45f1c8e6f8 100644 --- a/src/makerom/ipl3.s +++ b/src/makerom/ipl3.s @@ -1,4 +1,4 @@ .section .text -.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0x40, 0xFC0 +.incbin "incbin/ipl3" diff --git a/tools/extract_incbins.py b/tools/extract_incbins.py new file mode 100755 index 0000000000..c76129ca46 --- /dev/null +++ b/tools/extract_incbins.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: © 2024 ZeldaRET +# SPDX-License-Identifier: CC0-1.0 + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +import dmadata +import version_config + + +def main(): + parser = argparse.ArgumentParser( + description="Extract incbin pieces from an uncompressed ROM." + ) + parser.add_argument( + "baserom_dir", metavar="BASEROM_DIR", type=Path, help="Directory of uncompressed ROM segments" + ) + parser.add_argument( + "-v", + "--oot-version", + required=True, + help="OOT version", + ) + parser.add_argument( + "-o", + "--output-dir", + type=Path, + required=True, + help="Output directory for incbin pieces", + ) + + args = parser.parse_args() + + config = version_config.load_version_config(args.oot_version) + + args.output_dir.mkdir(parents=True, exist_ok=True) + for incbin in config.incbins: + incbin_path = args.output_dir / incbin.name + with open(args.baserom_dir / incbin.segment, "rb") as f: + offset = incbin.vram - config.dmadata_segments[incbin.segment].vram + f.seek(offset) + incbin_data = f.read(incbin.size) + incbin_path.write_bytes(incbin_data) + + print(f"Extracted {len(config.incbins)} incbin pieces to {args.output_dir}") + + +if __name__ == "__main__": + main() diff --git a/tools/version_config.py b/tools/version_config.py index 6ca203fd59..f1666d1452 100644 --- a/tools/version_config.py +++ b/tools/version_config.py @@ -26,6 +26,8 @@ class VersionConfig: text_lang_pal: bool # DMA segment information, in ROM order dmadata_segments: OrderedDict[str, SegmentInfo] + # ROM pieces that are copied directly into the build with .incbin + incbins: list[IncbinConfig] # Addresses of important variables needed for asset extraction variables: dict[str, int] # Assets to extract @@ -38,6 +40,14 @@ class SegmentInfo: vram: Optional[int] +@dataclasses.dataclass +class IncbinConfig: + name: str + segment: str + vram: int + size: int + + @dataclasses.dataclass class AssetConfig: name: str @@ -61,6 +71,14 @@ def load_version_config(version: str) -> VersionConfig: with open(PROJECT_ROOT / f"baseroms/{version}/config.yml", "r") as f: config = yaml.load(f, Loader=yaml.Loader) + incbins = [] + for incbin in config["incbins"]: + incbins.append( + IncbinConfig( + incbin["name"], incbin["segment"], incbin["vram"], incbin["size"] + ) + ) + assets = [] for asset in config["assets"]: name = asset["name"] @@ -74,6 +92,7 @@ def load_version_config(version: str) -> VersionConfig: dmadata_start=config["dmadata_start"], text_lang_pal=config["text_lang_pal"], dmadata_segments=load_dmadata_segments(version), + incbins=incbins, variables=config["variables"], assets=assets, )