From d674dad3da766d5f9d704c1fa5c6d0cf80d49d72 Mon Sep 17 00:00:00 2001 From: cadmic Date: Sat, 3 Feb 2024 14:59:19 -0800 Subject: [PATCH] Match compression for gc-eu-mq (#1704) * Improve compression * Format * Typo * Use Python assignment expression in tools/dmadata.py Co-authored-by: Dragorn421 * Review decompress_baserom.py * Replace DEFINE_DEBUG_SCENE with CPP defines * Pass is_zlib_compressed instead of version * Reword NOLOAD comment in write_compress_ranges * Remove redundant comment --------- Co-authored-by: Dragorn421 --- Makefile | 3 +- baseroms/gc-eu-mq-dbg/checksum-compressed.md5 | 2 +- include/tables/dmadata_table.h | 4 +- include/tables/scene_table.h | 3 +- include/z64scene.h | 18 ++- spec | 15 ++ tools/compress.py | 130 +++++------------- tools/decompress_baserom.py | 113 +++++++-------- tools/dmadata.py | 75 ++++++++++ tools/{dmadata_range.sh => dmadata_start.sh} | 3 +- tools/mkdmadata.c | 28 +++- tools/spec.c | 2 + tools/spec.h | 1 + 13 files changed, 220 insertions(+), 177 deletions(-) mode change 100644 => 100755 tools/compress.py create mode 100644 tools/dmadata.py rename tools/{dmadata_range.sh => dmadata_start.sh} (71%) diff --git a/Makefile b/Makefile index b28d555d90..1a4c65876f 100644 --- a/Makefile +++ b/Makefile @@ -390,8 +390,7 @@ $(ROM): $(ELF) $(ELF2ROM) -cic 6105 $< $@ $(ROMC): $(ROM) $(ELF) $(BUILD_DIR)/compress_ranges.txt -# note: $(BUILD_DIR)/compress_ranges.txt should only be used for nonmatching builds. it works by chance for matching builds too though - $(PYTHON) tools/compress.py --in $(ROM) --out $@ --dma-range `./tools/dmadata_range.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS) + $(PYTHON) tools/compress.py --in $(ROM) --out $@ --dma-start `./tools/dmadata_start.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS) $(PYTHON) -m ipl3checksum sum --cic 6105 --update $@ $(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) $(BUILD_DIR)/undefined_syms.txt diff --git a/baseroms/gc-eu-mq-dbg/checksum-compressed.md5 b/baseroms/gc-eu-mq-dbg/checksum-compressed.md5 index 181bb40418..d902d5fa9f 100644 --- a/baseroms/gc-eu-mq-dbg/checksum-compressed.md5 +++ b/baseroms/gc-eu-mq-dbg/checksum-compressed.md5 @@ -1 +1 @@ -5831385a7f216370cdbea55616b12fed build/gc-eu-mq-dbg/oot-gc-eu-mq-dbg-compressed.z64 +9704bc98c20c20319cc1633fe02b6fc7 build/gc-eu-mq-dbg/oot-gc-eu-mq-dbg-compressed.z64 diff --git a/include/tables/dmadata_table.h b/include/tables/dmadata_table.h index e633e7f56b..13c3d4bb5b 100644 --- a/include/tables/dmadata_table.h +++ b/include/tables/dmadata_table.h @@ -1,8 +1,8 @@ /** * Select dmadata table for version */ -#ifdef NON_MATCHING -// For non-matching builds, dmadata is generated from the specfile segments +#if !OOT_DEBUG || NON_MATCHING +// For retail versions and non-matching builds, dmadata is generated from the specfile segments #include "dmadata_table_spec.h" #else #include "dmadata_table_mqdbg.h" diff --git a/include/tables/scene_table.h b/include/tables/scene_table.h index 0337e12521..5c6dd595a2 100644 --- a/include/tables/scene_table.h +++ b/include/tables/scene_table.h @@ -110,7 +110,7 @@ /* 0x62 */ DEFINE_SCENE(spot18_scene, g_pn_41, SCENE_GORON_CITY, SDC_GORON_CITY, 0, 0) /* 0x63 */ DEFINE_SCENE(spot20_scene, g_pn_42, SCENE_LON_LON_RANCH, SDC_LON_LON_RANCH, 0, 0) /* 0x64 */ DEFINE_SCENE(ganon_tou_scene, g_pn_43, SCENE_OUTSIDE_GANONS_CASTLE, SDC_OUTSIDE_GANONS_CASTLE, 0, 0) -// Debug-only scenes +#if OOT_DEBUG /* 0x65 */ DEFINE_SCENE(test01_scene, none, SCENE_TEST01, SDC_CALM_WATER, 0, 0) /* 0x66 */ DEFINE_SCENE(besitu_scene, none, SCENE_BESITU, SDC_BESITU, 0, 0) /* 0x67 */ DEFINE_SCENE(depth_test_scene, none, SCENE_DEPTH_TEST, SDC_DEFAULT, 0, 0) @@ -120,3 +120,4 @@ /* 0x6B */ DEFINE_SCENE(hairal_niwa2_scene, g_pn_12, SCENE_HAIRAL_NIWA2, SDC_CASTLE_COURTYARD_GUARDS, 0, 0) /* 0x6C */ DEFINE_SCENE(sasatest_scene, none, SCENE_SASATEST, SDC_DEFAULT, 0, 0) /* 0x6D */ DEFINE_SCENE(testroom_scene, none, SCENE_TESTROOM, SDC_DEFAULT, 0, 0) +#endif diff --git a/include/z64scene.h b/include/z64scene.h index 7cb48e0bb4..c966913f15 100644 --- a/include/z64scene.h +++ b/include/z64scene.h @@ -355,9 +355,21 @@ typedef enum { #undef DEFINE_SCENE -// this define exists to preserve shiftability for an unused scene that is -// listed in the entrance table -#define SCENE_UNUSED_6E SCENE_ID_MAX +// Fake enum values for scenes that are still referenced in the entrance table +#if !OOT_DEBUG +// Debug-only scenes +#define SCENE_TEST01 0x65 +#define SCENE_BESITU 0x66 +#define SCENE_DEPTH_TEST 0x67 +#define SCENE_SYOTES 0x68 +#define SCENE_SYOTES2 0x69 +#define SCENE_SUTARU 0x6A +#define SCENE_HAIRAL_NIWA2 0x6B +#define SCENE_SASATEST 0x6C +#define SCENE_TESTROOM 0x6D +#endif +// Deleted scene +#define SCENE_UNUSED_6E 0x6E // Entrance Index Enum #define DEFINE_ENTRANCE(enum, _1, _2, _3, _4, _5, _6) enum, diff --git a/spec b/spec index 8e08b4f89e..f571845445 100644 --- a/spec +++ b/spec @@ -538,6 +538,7 @@ endseg beginseg name "buffers" + flags NOLOAD align 0x40 include "$(BUILD_DIR)/src/buffers/zbuffer.o" include "$(BUILD_DIR)/src/buffers/gfxbuffers.o" @@ -9627,6 +9628,7 @@ beginseg number 3 endseg +#if OOT_DEBUG beginseg name "syotes_scene" romalign 0x1000 @@ -9668,6 +9670,7 @@ beginseg include "$(BUILD_DIR)/assets/scenes/test_levels/depth_test/depth_test_room_0.o" number 3 endseg +#endif beginseg name "spot00_scene" @@ -10149,6 +10152,7 @@ beginseg number 3 endseg +#if OOT_DEBUG beginseg name "testroom_scene" romalign 0x1000 @@ -10190,6 +10194,7 @@ beginseg include "$(BUILD_DIR)/assets/scenes/test_levels/testroom/testroom_room_4.o" number 3 endseg +#endif beginseg name "kenjyanoma_scene" @@ -10231,6 +10236,7 @@ beginseg number 3 endseg +#if OOT_DEBUG beginseg name "sutaru_scene" romalign 0x1000 @@ -10244,6 +10250,7 @@ beginseg include "$(BUILD_DIR)/assets/scenes/test_levels/sutaru/sutaru_room_0.o" number 3 endseg +#endif beginseg name "link_home_scene" @@ -10517,6 +10524,7 @@ beginseg number 3 endseg +#if OOT_DEBUG beginseg name "sasatest_scene" romalign 0x1000 @@ -10530,6 +10538,7 @@ beginseg include "$(BUILD_DIR)/assets/scenes/test_levels/sasatest/sasatest_room_0.o" number 3 endseg +#endif beginseg name "market_alley_scene" @@ -11267,6 +11276,7 @@ beginseg number 3 endseg +#if OOT_DEBUG beginseg name "hairal_niwa2_scene" romalign 0x1000 @@ -11280,6 +11290,7 @@ beginseg include "$(BUILD_DIR)/assets/scenes/indoors/hairal_niwa2/hairal_niwa2_room_0.o" number 3 endseg +#endif beginseg name "hakasitarelay_scene" @@ -11753,6 +11764,7 @@ beginseg number 3 endseg +#if OOT_DEBUG beginseg name "besitu_scene" romalign 0x1000 @@ -11766,6 +11778,7 @@ beginseg include "$(BUILD_DIR)/assets/scenes/test_levels/besitu/besitu_room_0.o" number 3 endseg +#endif beginseg name "face_shop_scene" @@ -11823,6 +11836,7 @@ beginseg number 3 endseg +#if OOT_DEBUG beginseg name "test01_scene" romalign 0x1000 @@ -11836,6 +11850,7 @@ beginseg include "$(BUILD_DIR)/assets/scenes/test_levels/test01/test01_room_0.o" number 3 endseg +#endif beginseg name "bump_texture_static" diff --git a/tools/compress.py b/tools/compress.py old mode 100644 new mode 100755 index d72b5f2cc4..45c29ceb4f --- a/tools/compress.py +++ b/tools/compress.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + # SPDX-FileCopyrightText: 2024 zeldaret # SPDX-License-Identifier: CC0-1.0 @@ -13,57 +15,7 @@ import multiprocessing.pool import crunch64 - -STRUCT_IIII = struct.Struct(">IIII") - - -@dataclasses.dataclass -class DmaEntry: - """ - A Python counterpart to the dmadata entry struct: - ```c - typedef struct { - /* 0x00 */ uintptr_t vromStart; - /* 0x04 */ uintptr_t vromEnd; - /* 0x08 */ uintptr_t romStart; - /* 0x0C */ uintptr_t romEnd; - } DmaEntry; - ``` - """ - - vromStart: int - vromEnd: int - romStart: int - romEnd: int - - def __repr__(self): - return ( - "DmaEntry(" - f"vromStart=0x{self.vromStart:08X}, " - f"vromEnd=0x{self.vromEnd:08X}, " - f"romStart=0x{self.romStart:08X}, " - f"romEnd=0x{self.romEnd:08X}" - ")" - ) - - SIZE_BYTES = STRUCT_IIII.size - - def to_bin(self, data: memoryview): - STRUCT_IIII.pack_into( - data, - 0, - self.vromStart, - self.vromEnd, - self.romStart, - self.romEnd, - ) - - @staticmethod - def from_bin(data: memoryview): - return DmaEntry(*STRUCT_IIII.unpack_from(data)) - - -DMA_ENTRY_ZERO = DmaEntry(0, 0, 0, 0) +import dmadata def align(v: int): @@ -73,15 +25,15 @@ def align(v: int): @dataclasses.dataclass class RomSegment: - vromStart: int - vromEnd: int + vrom_start: int + vrom_end: int is_compressed: bool data: memoryview | None data_async: multiprocessing.pool.AsyncResult | None @property def uncompressed_size(self): - return self.vromEnd - self.vromStart + return self.vrom_end - self.vrom_start # Make interrupting the compression with ^C less jank @@ -94,15 +46,13 @@ def set_sigint_ignored(): def compress_rom( rom_data: memoryview, - dmadata_offset_start: int, - dmadata_offset_end: int, + dmadata_offset: int, compress_entries_indices: set[int], n_threads: int = None, ): """ rom_data: the uncompressed rom data - dmadata_offset_start: the offset in the rom where the dmadata starts (inclusive) - dmadata_offset_end: the offset in the rom where the dmadata ends (exclusive) + dmadata_offset: the offset in the rom where the dmadata starts compress_entries_indices: the indices in the dmadata of the segments that should be compressed n_threads: how many cores to use for compression """ @@ -110,18 +60,18 @@ def compress_rom( # Segments of the compressed rom (not all are compressed) compressed_rom_segments: list[RomSegment] = [] + dma_entries = dmadata.read_dmadata(rom_data, dmadata_offset) + # We sort the DMA entries by ROM start because `compress_entries_indices` + # refers to indices in ROM order, but the uncompressed dmadata might not be + # in ROM order. + dma_entries.sort(key=lambda dma_entry: dma_entry.vrom_start) + with multiprocessing.Pool(n_threads, initializer=set_sigint_ignored) as p: # Extract each segment from the input rom - for entry_index, dmadata_offset in enumerate( - range(dmadata_offset_start, dmadata_offset_end, DmaEntry.SIZE_BYTES) - ): - dma_entry = DmaEntry.from_bin(rom_data[dmadata_offset:]) - if dma_entry == DMA_ENTRY_ZERO: - continue - - segment_rom_start = dma_entry.romStart - segment_rom_end = dma_entry.romStart + ( - dma_entry.vromEnd - dma_entry.vromStart + for entry_index, dma_entry in enumerate(dma_entries): + segment_rom_start = dma_entry.rom_start + segment_rom_end = dma_entry.rom_start + ( + dma_entry.vrom_end - dma_entry.vrom_start ) segment_data_uncompressed = rom_data[segment_rom_start:segment_rom_end] @@ -139,17 +89,14 @@ def compress_rom( compressed_rom_segments.append( RomSegment( - dma_entry.vromStart, - dma_entry.vromEnd, + dma_entry.vrom_start, + dma_entry.vrom_end, is_compressed, segment_data, segment_data_async, ) ) - # Technically optional but required for matching. - compressed_rom_segments.sort(key=lambda segment: segment.vromStart) - # Wait on compression of all compressed segments waiting_on_segments = [ segment for segment in compressed_rom_segments if segment.is_compressed @@ -208,7 +155,7 @@ def compress_rom( * pad_to_multiple_of ) compressed_rom_data = memoryview(bytearray(compressed_rom_size_padded)) - compressed_rom_dma_entries: list[DmaEntry] = [] + compressed_rom_dma_entries: list[dmadata.DmaEntry] = [] rom_offset = 0 for segment in compressed_rom_segments: assert segment.data is not None @@ -221,9 +168,9 @@ def compress_rom( compressed_rom_data[segment_rom_start:i] = segment.data compressed_rom_dma_entries.append( - DmaEntry( - segment.vromStart, - segment.vromEnd, + dmadata.DmaEntry( + segment.vrom_start, + segment.vrom_end, segment_rom_start, segment_rom_end if segment.is_compressed else 0, ) @@ -237,13 +184,10 @@ def compress_rom( compressed_rom_data[i] = i % 256 # Write the new dmadata - dmadata_offset = dmadata_offset_start + offset = dmadata_offset for dma_entry in compressed_rom_dma_entries: - assert dmadata_offset + DmaEntry.SIZE_BYTES <= dmadata_offset_end - - dma_entry.to_bin(compressed_rom_data[dmadata_offset:]) - - dmadata_offset += DmaEntry.SIZE_BYTES + dma_entry.to_bin(compressed_rom_data[offset:]) + offset += dmadata.DmaEntry.SIZE_BYTES return compressed_rom_data @@ -263,13 +207,12 @@ def main(): help="path of the compressed rom to write out", ) parser.add_argument( - "--dma-range", - dest="dma_range", + "--dma-start", + dest="dma_start", + type=lambda s: int(s, 16), required=True, help=( - "The dmadata location in the rom, in format" - " 'start_inclusive-end_exclusive' and using hexadecimal offsets" - " (e.g. '0x12f70-0x19030')." + "The dmadata location in the rom, as a hexadecimal offset (e.g. 0x12f70)." ), ) parser.add_argument( @@ -298,13 +241,7 @@ def main(): out_rom_p = Path(args.out_rom) - dma_range_str: str = args.dma_range - dma_range_ends_str = dma_range_str.split("-") - assert len(dma_range_ends_str) == 2, dma_range_str - dmadata_offset_start, dmadata_offset_end = ( - int(v_str, 16) for v_str in dma_range_ends_str - ) - assert dmadata_offset_start < dmadata_offset_end, dma_range_str + dmadata_offset = args.dma_start compress_ranges_str: str = args.compress_ranges compress_entries_indices = set() @@ -330,8 +267,7 @@ def main(): in_rom_data = in_rom_p.read_bytes() out_rom_data = compress_rom( memoryview(in_rom_data), - dmadata_offset_start, - dmadata_offset_end, + dmadata_offset, compress_entries_indices, n_threads, ) diff --git a/tools/decompress_baserom.py b/tools/decompress_baserom.py index 551133e0e6..c5e55ac843 100755 --- a/tools/decompress_baserom.py +++ b/tools/decompress_baserom.py @@ -15,6 +15,8 @@ import crunch64 import ipl3checksum import zlib +import dmadata + def decompress_zlib(data: bytes) -> bytes: decomp = zlib.decompressobj(-zlib.MAX_WBITS) @@ -33,13 +35,13 @@ def decompress(data: bytes, is_zlib_compressed: bool) -> bytes: FILE_TABLE_OFFSET = { - "gc-eu-mq": 0x07170, - "gc-eu-mq-dbg": 0x12F70, + "gc-eu-mq": 0x07170, + "gc-eu-mq-dbg": 0x12F70, } VERSIONS_MD5S = { - "gc-eu-mq": "1a438f4235f8038856971c14a798122a", - "gc-eu-mq-dbg": "f0b7f35375f9cc8ca1b2d59d78e35405", + "gc-eu-mq": "1a438f4235f8038856971c14a798122a", + "gc-eu-mq-dbg": "f0b7f35375f9cc8ca1b2d59d78e35405", } @@ -48,27 +50,6 @@ def round_up(n, shift): return (n + mod - 1) >> shift << shift -def as_word_list(b) -> list[int]: - return [i[0] for i in struct.iter_unpack(">I", b)] - - -def read_dmadata_entry(file_content: bytearray, addr: int) -> list[int]: - return as_word_list(file_content[addr : addr + 0x10]) - - -def read_dmadata(file_content: bytearray, start) -> list[list[int]]: - dmadata = [] - addr = start - entry = read_dmadata_entry(file_content, addr) - i = 0 - while any([e != 0 for e in entry]): - dmadata.append(entry) - addr += 0x10 - i += 1 - entry = read_dmadata_entry(file_content, addr) - return dmadata - - def update_crc(decompressed: io.BytesIO) -> io.BytesIO: print("Recalculating crc...") calculated_checksum = ipl3checksum.CICKind.CIC_X105.calculateChecksum( @@ -82,40 +63,43 @@ def update_crc(decompressed: io.BytesIO) -> io.BytesIO: def decompress_rom( - file_content: bytearray, dmadata_addr: int, dmadata: list[list[int]], version: str + file_content: bytearray, + dmadata_offset: int, + dma_entries: list[dmadata.DmaEntry], + is_zlib_compressed: bool, ) -> bytearray: rom_segments = {} # vrom start : data s.t. len(data) == vrom_end - vrom_start - new_dmadata = bytearray() # new dmadata: {vrom start , vrom end , vrom start , 0} + new_dmadata = [] # new dmadata: list[dmadata.Entry] decompressed = io.BytesIO(b"") - for v_start, v_end, p_start, p_end in dmadata: + for dma_entry in dma_entries: + v_start = dma_entry.vrom_start + v_end = dma_entry.vrom_end + p_start = dma_entry.rom_start + p_end = dma_entry.rom_end if p_start == 0xFFFFFFFF and p_end == 0xFFFFFFFF: - new_dmadata.extend(struct.pack(">IIII", v_start, v_end, p_start, p_end)) + new_dmadata.append(dma_entry) continue - if p_end == 0: # uncompressed - rom_segments.update( - {v_start: file_content[p_start : p_start + v_end - v_start]} - ) - else: # compressed - rom_segments.update( - { - v_start: decompress( - file_content[p_start:p_end], version in {"ique-cn", "ique-zh"} - ) - } - ) - new_dmadata.extend(struct.pack(">IIII", v_start, v_end, v_start, 0)) + if dma_entry.is_compressed(): + new_contents = decompress(file_content[p_start:p_end], is_zlib_compressed) + rom_segments[v_start] = new_contents + else: + rom_segments[v_start] = file_content[p_start : p_start + v_end - v_start] + new_dmadata.append(dmadata.DmaEntry(v_start, v_end, v_start, 0)) # write rom segments to vaddrs for vrom_st, data in rom_segments.items(): decompressed.seek(vrom_st) decompressed.write(data) # write new dmadata - decompressed.seek(dmadata_addr) - decompressed.write(new_dmadata) + decompressed.seek(dmadata_offset) + for dma_entry in new_dmadata: + entry_data = bytearray(dmadata.DmaEntry.SIZE_BYTES) + dma_entry.to_bin(entry_data) + decompressed.write(entry_data) # pad to size - padding_end = round_up(dmadata[-1][1], 14) + padding_end = round_up(dma_entries[-1].vrom_end, 14) decompressed.seek(padding_end - 1) decompressed.write(bytearray([0])) # re-calculate crc @@ -164,21 +148,23 @@ def per_version_fixes(file_content: bytearray, version: str) -> bytearray: return file_content -def pad_rom(file_content: bytearray, dmadata: list[list[int]]) -> bytearray: - padding_start = round_up(dmadata[-1][1], 12) - padding_end = round_up(dmadata[-1][1], 14) +def pad_rom(file_content: bytearray, dma_entries: list[dmadata.DmaEntry]) -> bytearray: + padding_start = round_up(dma_entries[-1].vrom_end, 12) + padding_end = round_up(dma_entries[-1].vrom_end, 14) print(f"Padding from {padding_start:X} to {padding_end:X}...") for i in range(padding_start, padding_end): file_content[i] = 0xFF return file_content + # Determine if we have a ROM file ROM_FILE_EXTENSIONS = ["z64", "n64", "v64"] + def find_baserom(version: str) -> Path | None: for rom_file_ext_lower in ROM_FILE_EXTENSIONS: for rom_file_ext in (rom_file_ext_lower, rom_file_ext_lower.upper()): - rom_file_name_candidate = Path(f"baseroms/{version}/baserom.{rom_file_ext}") + rom_file_name_candidate = Path(f"baseroms/{version}/baserom.{rom_file_ext}") if rom_file_name_candidate.exists(): return rom_file_name_candidate return None @@ -188,7 +174,11 @@ def main(): description = "Convert a rom that uses dmadata to an uncompressed one." parser = argparse.ArgumentParser(description=description) - parser.add_argument("version", help="Version of the game to decompress.", choices=list(VERSIONS_MD5S.keys())) + parser.add_argument( + "version", + help="Version of the game to decompress.", + choices=list(VERSIONS_MD5S.keys()), + ) args = parser.parse_args() @@ -196,7 +186,7 @@ def main(): uncompressed_path = Path(f"baseroms/{version}/baserom-decompressed.z64") - file_table_offset = FILE_TABLE_OFFSET[version] + dmadata_offset = FILE_TABLE_OFFSET[version] correct_str_hash = VERSIONS_MD5S[version] if check_existing_rom(uncompressed_path, correct_str_hash): @@ -207,7 +197,8 @@ def main(): if rom_file_name is None: path_list = [ - f"baseroms/{version}/baserom.{rom_file_ext}" for rom_file_ext in ROM_FILE_EXTENSIONS + f"baseroms/{version}/baserom.{rom_file_ext}" + for rom_file_ext in ROM_FILE_EXTENSIONS ] print(f"Error: Could not find {','.join(path_list)}.") exit(1) @@ -234,20 +225,16 @@ def main(): file_content = per_version_fixes(file_content, version) - dmadata = read_dmadata(file_content, file_table_offset) + dma_entries = dmadata.read_dmadata(file_content, dmadata_offset) # Decompress - if any( - [ - b != 0 - for b in file_content[ - file_table_offset + 0xAC : file_table_offset + 0xAC + 0x4 - ] - ] - ): + if any(dma_entry.is_compressed() for dma_entry in dma_entries): print("Decompressing rom...") - file_content = decompress_rom(file_content, file_table_offset, dmadata, version) + is_zlib_compressed = version in {"ique-cn", "ique-zh"} + file_content = decompress_rom( + file_content, dmadata_offset, dma_entries, is_zlib_compressed + ) - file_content = pad_rom(file_content, dmadata) + file_content = pad_rom(file_content, dma_entries) # Check to see if the ROM is a "vanilla" ROM str_hash = get_str_hash(file_content) diff --git a/tools/dmadata.py b/tools/dmadata.py new file mode 100644 index 0000000000..68eb1e7107 --- /dev/null +++ b/tools/dmadata.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: © 2024 ZeldaRET +# SPDX-License-Identifier: CC0-1.0 + +from __future__ import annotations + +import dataclasses +import struct + + +STRUCT_IIII = struct.Struct(">IIII") + + +@dataclasses.dataclass +class DmaEntry: + """ + A Python counterpart to the dmadata entry struct: + ```c + typedef struct { + /* 0x00 */ uintptr_t vromStart; + /* 0x04 */ uintptr_t vromEnd; + /* 0x08 */ uintptr_t romStart; + /* 0x0C */ uintptr_t romEnd; + } DmaEntry; + ``` + """ + + vrom_start: int + vrom_end: int + rom_start: int + rom_end: int + + def __repr__(self): + return ( + "DmaEntry(" + f"vrom_start=0x{self.vrom_start:08X}, " + f"vrom_end=0x{self.vrom_end:08X}, " + f"rom_start=0x{self.rom_start:08X}, " + f"rom_end=0x{self.rom_end:08X}" + ")" + ) + + SIZE_BYTES = STRUCT_IIII.size + + def to_bin(self, data: memoryview): + STRUCT_IIII.pack_into( + data, + 0, + self.vrom_start, + self.vrom_end, + self.rom_start, + self.rom_end, + ) + + @staticmethod + def from_bin(data: memoryview): + return DmaEntry(*STRUCT_IIII.unpack_from(data)) + + def is_compressed(self) -> bool: + return self.rom_end != 0 + + +DMA_ENTRY_END = DmaEntry(0, 0, 0, 0) + + +def read_dmadata(rom_data: memoryview, start_offset: int) -> list[DmaEntry]: + result = [] + + offset = start_offset + while ( + entry := DmaEntry.from_bin(rom_data[offset : offset + DmaEntry.SIZE_BYTES]) + ) != DMA_ENTRY_END: + result.append(entry) + offset += DmaEntry.SIZE_BYTES + + return result diff --git a/tools/dmadata_range.sh b/tools/dmadata_start.sh similarity index 71% rename from tools/dmadata_range.sh rename to tools/dmadata_start.sh index 905322c9cb..920d9fe0b2 100755 --- a/tools/dmadata_range.sh +++ b/tools/dmadata_start.sh @@ -11,6 +11,5 @@ ELF=$2 dmadata_syms=`$NM $ELF --no-sort --radix=x --format=bsd | grep dmadata` _dmadataSegmentRomStart=`echo "$dmadata_syms" | grep '\b_dmadataSegmentRomStart\b' | cut -d' ' -f1` -_dmadataSegmentRomEnd=` echo "$dmadata_syms" | grep '\b_dmadataSegmentRomEnd\b' | cut -d' ' -f1` -echo 0x"$_dmadataSegmentRomStart"-0x"$_dmadataSegmentRomEnd" +echo 0x"$_dmadataSegmentRomStart" diff --git a/tools/mkdmadata.c b/tools/mkdmadata.c index c83864b559..fbbd5da9e6 100644 --- a/tools/mkdmadata.c +++ b/tools/mkdmadata.c @@ -16,24 +16,38 @@ static void write_dmadata_table(FILE *fout) { int i; - for (i = 0; i < g_segmentsCount; i++) + for (i = 0; i < g_segmentsCount; i++) { + // Don't emit dma entry for segments set with NOLOAD + if (g_segments[i].flags & FLAG_NOLOAD) { + continue; + } + fprintf(fout, "DEFINE_DMA_ENTRY(%s, \"%s\")\n", g_segments[i].name, g_segments[i].name); + } } static void write_compress_ranges(FILE *fout) { int i; + int rom_index = 0; bool continue_list = false; int stride_first = -1; for (i = 0; i < g_segmentsCount; i++) { - if (g_segments[i].compress) { - if (stride_first == -1) - stride_first = i; + bool compress = g_segments[i].compress; + + // Don't consider segments set with NOLOAD when calculating indices + if (g_segments[i].flags & FLAG_NOLOAD) { + continue; } - if (!g_segments[i].compress || i == g_segmentsCount - 1) { + + if (compress) { + if (stride_first == -1) + stride_first = rom_index; + } + if (!compress || i == g_segmentsCount - 1) { if (stride_first != -1) { - int stride_last = i - 1; + int stride_last = compress ? rom_index : rom_index - 1; if (continue_list) { fprintf(fout, ","); } @@ -46,6 +60,8 @@ static void write_compress_ranges(FILE *fout) stride_first = -1; } } + + rom_index++; } } diff --git a/tools/spec.c b/tools/spec.c index 3f130b07a5..7d07caa13d 100644 --- a/tools/spec.c +++ b/tools/spec.c @@ -83,6 +83,8 @@ static bool parse_flags(char *str, unsigned int *flags) f |= FLAG_OBJECT; else if (strcmp(str, "RAW") == 0) f |= FLAG_RAW; + else if (strcmp(str, "NOLOAD") == 0) + f |= FLAG_NOLOAD; else return false; diff --git a/tools/spec.h b/tools/spec.h index 8591a6b372..aeabd07d1f 100644 --- a/tools/spec.h +++ b/tools/spec.h @@ -27,6 +27,7 @@ enum { FLAG_BOOT = (1 << 0), FLAG_OBJECT = (1 << 1), FLAG_RAW = (1 << 2), + FLAG_NOLOAD = (1 << 3), }; struct Include {