mirror of
https://github.com/zeldaret/oot.git
synced 2024-11-13 04:39:36 +00:00
Extract ROM segments from gc-eu-mq (take 2) (#1709)
* Extract ROM segments from gc-eu-mq (take 2) * Apply suggestions from code review Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com> * Typo * dma_{start,names} -> dmadata_{start,names} * Restore dest * Don't assume rom location == vrom location --------- Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>
This commit is contained in:
parent
454b1caa52
commit
54ffd50fa2
9 changed files with 3148 additions and 1635 deletions
16
Makefile
16
Makefile
|
@ -65,6 +65,7 @@ endif
|
|||
PROJECT_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
|
||||
BUILD_DIR := build/$(VERSION)
|
||||
EXPECTED_DIR := expected/$(BUILD_DIR)
|
||||
BASEROM_DIR := baseroms/$(VERSION)
|
||||
VENV := .venv
|
||||
|
||||
MAKE = make
|
||||
|
@ -210,8 +211,7 @@ ASSET_FILES_OUT := $(foreach f,$(ASSET_FILES_XML:.xml=.c),$f) \
|
|||
|
||||
UNDECOMPILED_DATA_DIRS := $(shell find data -type d)
|
||||
|
||||
# TODO: for now, ROM segments are still taken from the Debug ROM even when building other versions
|
||||
BASEROM_SEGMENTS_DIR := baseroms/gc-eu-mq-dbg/segments
|
||||
BASEROM_SEGMENTS_DIR := $(BASEROM_DIR)/segments
|
||||
BASEROM_BIN_FILES := $(wildcard $(BASEROM_SEGMENTS_DIR)/*)
|
||||
|
||||
# source files
|
||||
|
@ -223,7 +223,7 @@ O_FILES := $(foreach f,$(S_FILES:.s=.o),$(BUILD_DIR)/$f) \
|
|||
|
||||
OVL_RELOC_FILES := $(shell $(CPP) $(CPPFLAGS) $(SPEC) | $(SPEC_REPLACE_VARS) | grep -o '[^"]*_reloc.o' )
|
||||
|
||||
DISASM_BASEROM := baseroms/$(VERSION)/baserom-decompressed.z64
|
||||
DISASM_BASEROM := $(BASEROM_DIR)/baserom-decompressed.z64
|
||||
DISASM_DATA_FILES := $(wildcard $(DISASM_DATA_DIR)/*.csv) $(wildcard $(DISASM_DATA_DIR)/*.txt)
|
||||
DISASM_S_FILES := $(shell test -e $(PYTHON) && $(PYTHON) tools/disasm/list_generated_files.py -o $(EXPECTED_DIR) --config-dir $(DISASM_DATA_DIR))
|
||||
DISASM_O_FILES := $(DISASM_S_FILES:.s=.o)
|
||||
|
@ -334,13 +334,13 @@ all: rom compress
|
|||
rom: $(ROM)
|
||||
ifneq ($(COMPARE),0)
|
||||
@md5sum $(ROM)
|
||||
@md5sum -c baseroms/$(VERSION)/checksum.md5
|
||||
@md5sum -c $(BASEROM_DIR)/checksum.md5
|
||||
endif
|
||||
|
||||
compress: $(ROMC)
|
||||
ifneq ($(COMPARE),0)
|
||||
@md5sum $(ROMC)
|
||||
@md5sum -c baseroms/$(VERSION)/checksum-compressed.md5
|
||||
@md5sum -c $(BASEROM_DIR)/checksum-compressed.md5
|
||||
endif
|
||||
|
||||
clean:
|
||||
|
@ -366,9 +366,9 @@ venv:
|
|||
setup: venv
|
||||
$(MAKE) -C tools
|
||||
$(PYTHON) tools/decompress_baserom.py $(VERSION)
|
||||
# TODO: for now, we only extract ROM segments and assets from the Debug ROM
|
||||
$(PYTHON) tools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 -o $(BASEROM_SEGMENTS_DIR) --dmadata-start `cat $(BASEROM_DIR)/dmadata_start.txt` --dmadata-names $(BASEROM_DIR)/dmadata_names.txt
|
||||
# TODO: for now, we only extract assets from the Debug ROM
|
||||
ifeq ($(VERSION),gc-eu-mq-dbg)
|
||||
$(PYTHON) extract_baserom.py
|
||||
$(PYTHON) extract_assets.py -j$(N_THREADS)
|
||||
endif
|
||||
|
||||
|
@ -390,7 +390,7 @@ $(ROM): $(ELF)
|
|||
$(ELF2ROM) -cic 6105 $< $@
|
||||
|
||||
$(ROMC): $(ROM) $(ELF) $(BUILD_DIR)/compress_ranges.txt
|
||||
$(PYTHON) tools/compress.py --in $(ROM) --out $@ --dma-start `./tools/dmadata_start.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS)
|
||||
$(PYTHON) tools/compress.py --in $(ROM) --out $@ --dmadata-start `./tools/dmadata_start.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS)
|
||||
$(PYTHON) -m ipl3checksum sum --cic 6105 --update $@
|
||||
|
||||
$(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) $(BUILD_DIR)/undefined_syms.txt
|
||||
|
|
1532
baseroms/gc-eu-mq-dbg/dmadata_names.txt
Normal file
1532
baseroms/gc-eu-mq-dbg/dmadata_names.txt
Normal file
File diff suppressed because it is too large
Load diff
1
baseroms/gc-eu-mq-dbg/dmadata_start.txt
Normal file
1
baseroms/gc-eu-mq-dbg/dmadata_start.txt
Normal file
|
@ -0,0 +1 @@
|
|||
0x12f70
|
1510
baseroms/gc-eu-mq/dmadata_names.txt
Normal file
1510
baseroms/gc-eu-mq/dmadata_names.txt
Normal file
File diff suppressed because it is too large
Load diff
1
baseroms/gc-eu-mq/dmadata_start.txt
Normal file
1
baseroms/gc-eu-mq/dmadata_start.txt
Normal file
|
@ -0,0 +1 @@
|
|||
0x07170
|
1600
extract_baserom.py
1600
extract_baserom.py
File diff suppressed because it is too large
Load diff
|
@ -46,13 +46,13 @@ def set_sigint_ignored():
|
|||
|
||||
def compress_rom(
|
||||
rom_data: memoryview,
|
||||
dmadata_offset: int,
|
||||
dmadata_start: int,
|
||||
compress_entries_indices: set[int],
|
||||
n_threads: int = None,
|
||||
):
|
||||
"""
|
||||
rom_data: the uncompressed rom data
|
||||
dmadata_offset: the offset in the rom where the dmadata starts
|
||||
dmadata_start: the offset in the rom where the dmadata starts
|
||||
compress_entries_indices: the indices in the dmadata of the segments that should be compressed
|
||||
n_threads: how many cores to use for compression
|
||||
"""
|
||||
|
@ -60,7 +60,7 @@ def compress_rom(
|
|||
# Segments of the compressed rom (not all are compressed)
|
||||
compressed_rom_segments: list[RomSegment] = []
|
||||
|
||||
dma_entries = dmadata.read_dmadata(rom_data, dmadata_offset)
|
||||
dma_entries = dmadata.read_dmadata(rom_data, dmadata_start)
|
||||
# We sort the DMA entries by ROM start because `compress_entries_indices`
|
||||
# refers to indices in ROM order, but the uncompressed dmadata might not be
|
||||
# in ROM order.
|
||||
|
@ -184,7 +184,7 @@ def compress_rom(
|
|||
compressed_rom_data[i] = i % 256
|
||||
|
||||
# Write the new dmadata
|
||||
offset = dmadata_offset
|
||||
offset = dmadata_start
|
||||
for dma_entry in compressed_rom_dma_entries:
|
||||
dma_entry.to_bin(compressed_rom_data[offset:])
|
||||
offset += dmadata.DmaEntry.SIZE_BYTES
|
||||
|
@ -207,8 +207,8 @@ def main():
|
|||
help="path of the compressed rom to write out",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dma-start",
|
||||
dest="dma_start",
|
||||
"--dmadata-start",
|
||||
dest="dmadata_start",
|
||||
type=lambda s: int(s, 16),
|
||||
required=True,
|
||||
help=(
|
||||
|
@ -241,7 +241,7 @@ def main():
|
|||
|
||||
out_rom_p = Path(args.out_rom)
|
||||
|
||||
dmadata_offset = args.dma_start
|
||||
dmadata_start = args.dmadata_start
|
||||
|
||||
compress_ranges_str: str = args.compress_ranges
|
||||
compress_entries_indices = set()
|
||||
|
@ -267,7 +267,7 @@ def main():
|
|||
in_rom_data = in_rom_p.read_bytes()
|
||||
out_rom_data = compress_rom(
|
||||
memoryview(in_rom_data),
|
||||
dmadata_offset,
|
||||
dmadata_start,
|
||||
compress_entries_indices,
|
||||
n_threads,
|
||||
)
|
||||
|
|
|
@ -34,17 +34,6 @@ def decompress(data: bytes, is_zlib_compressed: bool) -> bytes:
|
|||
return crunch64.yaz0.decompress(data)
|
||||
|
||||
|
||||
FILE_TABLE_OFFSET = {
|
||||
"gc-eu-mq": 0x07170,
|
||||
"gc-eu-mq-dbg": 0x12F70,
|
||||
}
|
||||
|
||||
VERSIONS_MD5S = {
|
||||
"gc-eu-mq": "1a438f4235f8038856971c14a798122a",
|
||||
"gc-eu-mq-dbg": "f0b7f35375f9cc8ca1b2d59d78e35405",
|
||||
}
|
||||
|
||||
|
||||
def round_up(n, shift):
|
||||
mod = 1 << shift
|
||||
return (n + mod - 1) >> shift << shift
|
||||
|
@ -64,7 +53,7 @@ def update_crc(decompressed: io.BytesIO) -> io.BytesIO:
|
|||
|
||||
def decompress_rom(
|
||||
file_content: bytearray,
|
||||
dmadata_offset: int,
|
||||
dmadata_start: int,
|
||||
dma_entries: list[dmadata.DmaEntry],
|
||||
is_zlib_compressed: bool,
|
||||
) -> bytearray:
|
||||
|
@ -93,7 +82,7 @@ def decompress_rom(
|
|||
decompressed.seek(vrom_st)
|
||||
decompressed.write(data)
|
||||
# write new dmadata
|
||||
decompressed.seek(dmadata_offset)
|
||||
decompressed.seek(dmadata_start)
|
||||
for dma_entry in new_dmadata:
|
||||
entry_data = bytearray(dmadata.DmaEntry.SIZE_BYTES)
|
||||
dma_entry.to_bin(entry_data)
|
||||
|
@ -177,17 +166,21 @@ def main():
|
|||
parser.add_argument(
|
||||
"version",
|
||||
help="Version of the game to decompress.",
|
||||
choices=list(VERSIONS_MD5S.keys()),
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
version = args.version
|
||||
|
||||
uncompressed_path = Path(f"baseroms/{version}/baserom-decompressed.z64")
|
||||
baserom_dir = Path(f"baseroms/{version}")
|
||||
if not baserom_dir.exists():
|
||||
print(f"Error: Unknown version '{version}'.")
|
||||
exit(1)
|
||||
|
||||
dmadata_offset = FILE_TABLE_OFFSET[version]
|
||||
correct_str_hash = VERSIONS_MD5S[version]
|
||||
uncompressed_path = baserom_dir / "baserom-decompressed.z64"
|
||||
|
||||
dmadata_start = int((baserom_dir / "dmadata_start.txt").read_text(), 16)
|
||||
correct_str_hash = (baserom_dir / "checksum.md5").read_text().split()[0]
|
||||
|
||||
if check_existing_rom(uncompressed_path, correct_str_hash):
|
||||
print("Found valid baserom - exiting early")
|
||||
|
@ -225,13 +218,13 @@ def main():
|
|||
|
||||
file_content = per_version_fixes(file_content, version)
|
||||
|
||||
dma_entries = dmadata.read_dmadata(file_content, dmadata_offset)
|
||||
dma_entries = dmadata.read_dmadata(file_content, dmadata_start)
|
||||
# Decompress
|
||||
if any(dma_entry.is_compressed() for dma_entry in dma_entries):
|
||||
print("Decompressing rom...")
|
||||
is_zlib_compressed = version in {"ique-cn", "ique-zh"}
|
||||
file_content = decompress_rom(
|
||||
file_content, dmadata_offset, dma_entries, is_zlib_compressed
|
||||
file_content, dmadata_start, dma_entries, is_zlib_compressed
|
||||
)
|
||||
|
||||
file_content = pad_rom(file_content, dma_entries)
|
||||
|
|
76
tools/extract_baserom.py
Executable file
76
tools/extract_baserom.py
Executable file
|
@ -0,0 +1,76 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# SPDX-FileCopyrightText: © 2024 ZeldaRET
|
||||
# SPDX-License-Identifier: CC0-1.0
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
import dmadata
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Extract segments from an uncompressed ROM, based on its dmadata."
|
||||
)
|
||||
parser.add_argument(
|
||||
"rom", metavar="ROM", type=Path, help="Path to uncompressed ROM"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output-dir",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Output directory for segments",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dmadata-start",
|
||||
type=lambda s: int(s, 16),
|
||||
required=True,
|
||||
help=(
|
||||
"The dmadata location in the rom, as a hexadecimal offset (e.g. 0x12f70)"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dmadata-names",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Path to file containing segment names",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
rom_data = memoryview(args.rom.read_bytes())
|
||||
|
||||
dma_names = args.dmadata_names.read_text().splitlines()
|
||||
dma_entries = dmadata.read_dmadata(rom_data, args.dmadata_start)
|
||||
if len(dma_names) != len(dma_entries):
|
||||
print(
|
||||
f"Error: expected {len(dma_names)} DMA entries but found {len(dma_entries)} in ROM",
|
||||
file=sys.stderr,
|
||||
)
|
||||
exit(1)
|
||||
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
for dma_name, dma_entry in zip(dma_names, dma_entries):
|
||||
if dma_entry.is_compressed():
|
||||
print(f"Error: segment {dma_name} is compressed", file=sys.stderr)
|
||||
exit(1)
|
||||
|
||||
segment_rom_start = dma_entry.rom_start
|
||||
segment_rom_end = dma_entry.rom_start + (
|
||||
dma_entry.vrom_end - dma_entry.vrom_start
|
||||
)
|
||||
|
||||
segment_data = rom_data[segment_rom_start:segment_rom_end]
|
||||
segment_path = args.output_dir / dma_name
|
||||
segment_path.write_bytes(segment_data)
|
||||
|
||||
print(f"Extracted {len(dma_entries)} segments to {args.output_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
Loading…
Reference in a new issue