mirror of
https://github.com/zeldaret/oot.git
synced 2024-11-25 01:34:18 +00:00
Match compression for gc-eu-mq (#1704)
* Improve compression * Format * Typo * Use Python assignment expression in tools/dmadata.py Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com> * Review decompress_baserom.py * Replace DEFINE_DEBUG_SCENE with CPP defines * Pass is_zlib_compressed instead of version * Reword NOLOAD comment in write_compress_ranges * Remove redundant comment --------- Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>
This commit is contained in:
cadmic
GitHub
parent
0cbcebfded
commit
d674dad3da
13 changed files with 220 additions and 177 deletions
3
Makefile
3
Makefile
|
|
@ -390,8 +390,7 @@ $(ROM): $(ELF)
|
|||
$(ELF2ROM) -cic 6105 $< $@
|
||||
|
||||
$(ROMC): $(ROM) $(ELF) $(BUILD_DIR)/compress_ranges.txt
|
||||
# note: $(BUILD_DIR)/compress_ranges.txt should only be used for nonmatching builds. it works by chance for matching builds too though
|
||||
$(PYTHON) tools/compress.py --in $(ROM) --out $@ --dma-range `./tools/dmadata_range.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS)
|
||||
$(PYTHON) tools/compress.py --in $(ROM) --out $@ --dma-start `./tools/dmadata_start.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS)
|
||||
$(PYTHON) -m ipl3checksum sum --cic 6105 --update $@
|
||||
|
||||
$(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_RELOC_FILES) $(LDSCRIPT) $(BUILD_DIR)/undefined_syms.txt
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
5831385a7f216370cdbea55616b12fed build/gc-eu-mq-dbg/oot-gc-eu-mq-dbg-compressed.z64
|
||||
9704bc98c20c20319cc1633fe02b6fc7 build/gc-eu-mq-dbg/oot-gc-eu-mq-dbg-compressed.z64
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
/**
|
||||
* Select dmadata table for version
|
||||
*/
|
||||
#ifdef NON_MATCHING
|
||||
// For non-matching builds, dmadata is generated from the specfile segments
|
||||
#if !OOT_DEBUG || NON_MATCHING
|
||||
// For retail versions and non-matching builds, dmadata is generated from the specfile segments
|
||||
#include "dmadata_table_spec.h"
|
||||
#else
|
||||
#include "dmadata_table_mqdbg.h"
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@
|
|||
/* 0x62 */ DEFINE_SCENE(spot18_scene, g_pn_41, SCENE_GORON_CITY, SDC_GORON_CITY, 0, 0)
|
||||
/* 0x63 */ DEFINE_SCENE(spot20_scene, g_pn_42, SCENE_LON_LON_RANCH, SDC_LON_LON_RANCH, 0, 0)
|
||||
/* 0x64 */ DEFINE_SCENE(ganon_tou_scene, g_pn_43, SCENE_OUTSIDE_GANONS_CASTLE, SDC_OUTSIDE_GANONS_CASTLE, 0, 0)
|
||||
// Debug-only scenes
|
||||
#if OOT_DEBUG
|
||||
/* 0x65 */ DEFINE_SCENE(test01_scene, none, SCENE_TEST01, SDC_CALM_WATER, 0, 0)
|
||||
/* 0x66 */ DEFINE_SCENE(besitu_scene, none, SCENE_BESITU, SDC_BESITU, 0, 0)
|
||||
/* 0x67 */ DEFINE_SCENE(depth_test_scene, none, SCENE_DEPTH_TEST, SDC_DEFAULT, 0, 0)
|
||||
|
|
@ -120,3 +120,4 @@
|
|||
/* 0x6B */ DEFINE_SCENE(hairal_niwa2_scene, g_pn_12, SCENE_HAIRAL_NIWA2, SDC_CASTLE_COURTYARD_GUARDS, 0, 0)
|
||||
/* 0x6C */ DEFINE_SCENE(sasatest_scene, none, SCENE_SASATEST, SDC_DEFAULT, 0, 0)
|
||||
/* 0x6D */ DEFINE_SCENE(testroom_scene, none, SCENE_TESTROOM, SDC_DEFAULT, 0, 0)
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -355,9 +355,21 @@ typedef enum {
|
|||
|
||||
#undef DEFINE_SCENE
|
||||
|
||||
// this define exists to preserve shiftability for an unused scene that is
|
||||
// listed in the entrance table
|
||||
#define SCENE_UNUSED_6E SCENE_ID_MAX
|
||||
// Fake enum values for scenes that are still referenced in the entrance table
|
||||
#if !OOT_DEBUG
|
||||
// Debug-only scenes
|
||||
#define SCENE_TEST01 0x65
|
||||
#define SCENE_BESITU 0x66
|
||||
#define SCENE_DEPTH_TEST 0x67
|
||||
#define SCENE_SYOTES 0x68
|
||||
#define SCENE_SYOTES2 0x69
|
||||
#define SCENE_SUTARU 0x6A
|
||||
#define SCENE_HAIRAL_NIWA2 0x6B
|
||||
#define SCENE_SASATEST 0x6C
|
||||
#define SCENE_TESTROOM 0x6D
|
||||
#endif
|
||||
// Deleted scene
|
||||
#define SCENE_UNUSED_6E 0x6E
|
||||
|
||||
// Entrance Index Enum
|
||||
#define DEFINE_ENTRANCE(enum, _1, _2, _3, _4, _5, _6) enum,
|
||||
|
|
|
|||
15
spec
15
spec
|
|
@ -538,6 +538,7 @@ endseg
|
|||
|
||||
beginseg
|
||||
name "buffers"
|
||||
flags NOLOAD
|
||||
align 0x40
|
||||
include "$(BUILD_DIR)/src/buffers/zbuffer.o"
|
||||
include "$(BUILD_DIR)/src/buffers/gfxbuffers.o"
|
||||
|
|
@ -9627,6 +9628,7 @@ beginseg
|
|||
number 3
|
||||
endseg
|
||||
|
||||
#if OOT_DEBUG
|
||||
beginseg
|
||||
name "syotes_scene"
|
||||
romalign 0x1000
|
||||
|
|
@ -9668,6 +9670,7 @@ beginseg
|
|||
include "$(BUILD_DIR)/assets/scenes/test_levels/depth_test/depth_test_room_0.o"
|
||||
number 3
|
||||
endseg
|
||||
#endif
|
||||
|
||||
beginseg
|
||||
name "spot00_scene"
|
||||
|
|
@ -10149,6 +10152,7 @@ beginseg
|
|||
number 3
|
||||
endseg
|
||||
|
||||
#if OOT_DEBUG
|
||||
beginseg
|
||||
name "testroom_scene"
|
||||
romalign 0x1000
|
||||
|
|
@ -10190,6 +10194,7 @@ beginseg
|
|||
include "$(BUILD_DIR)/assets/scenes/test_levels/testroom/testroom_room_4.o"
|
||||
number 3
|
||||
endseg
|
||||
#endif
|
||||
|
||||
beginseg
|
||||
name "kenjyanoma_scene"
|
||||
|
|
@ -10231,6 +10236,7 @@ beginseg
|
|||
number 3
|
||||
endseg
|
||||
|
||||
#if OOT_DEBUG
|
||||
beginseg
|
||||
name "sutaru_scene"
|
||||
romalign 0x1000
|
||||
|
|
@ -10244,6 +10250,7 @@ beginseg
|
|||
include "$(BUILD_DIR)/assets/scenes/test_levels/sutaru/sutaru_room_0.o"
|
||||
number 3
|
||||
endseg
|
||||
#endif
|
||||
|
||||
beginseg
|
||||
name "link_home_scene"
|
||||
|
|
@ -10517,6 +10524,7 @@ beginseg
|
|||
number 3
|
||||
endseg
|
||||
|
||||
#if OOT_DEBUG
|
||||
beginseg
|
||||
name "sasatest_scene"
|
||||
romalign 0x1000
|
||||
|
|
@ -10530,6 +10538,7 @@ beginseg
|
|||
include "$(BUILD_DIR)/assets/scenes/test_levels/sasatest/sasatest_room_0.o"
|
||||
number 3
|
||||
endseg
|
||||
#endif
|
||||
|
||||
beginseg
|
||||
name "market_alley_scene"
|
||||
|
|
@ -11267,6 +11276,7 @@ beginseg
|
|||
number 3
|
||||
endseg
|
||||
|
||||
#if OOT_DEBUG
|
||||
beginseg
|
||||
name "hairal_niwa2_scene"
|
||||
romalign 0x1000
|
||||
|
|
@ -11280,6 +11290,7 @@ beginseg
|
|||
include "$(BUILD_DIR)/assets/scenes/indoors/hairal_niwa2/hairal_niwa2_room_0.o"
|
||||
number 3
|
||||
endseg
|
||||
#endif
|
||||
|
||||
beginseg
|
||||
name "hakasitarelay_scene"
|
||||
|
|
@ -11753,6 +11764,7 @@ beginseg
|
|||
number 3
|
||||
endseg
|
||||
|
||||
#if OOT_DEBUG
|
||||
beginseg
|
||||
name "besitu_scene"
|
||||
romalign 0x1000
|
||||
|
|
@ -11766,6 +11778,7 @@ beginseg
|
|||
include "$(BUILD_DIR)/assets/scenes/test_levels/besitu/besitu_room_0.o"
|
||||
number 3
|
||||
endseg
|
||||
#endif
|
||||
|
||||
beginseg
|
||||
name "face_shop_scene"
|
||||
|
|
@ -11823,6 +11836,7 @@ beginseg
|
|||
number 3
|
||||
endseg
|
||||
|
||||
#if OOT_DEBUG
|
||||
beginseg
|
||||
name "test01_scene"
|
||||
romalign 0x1000
|
||||
|
|
@ -11836,6 +11850,7 @@ beginseg
|
|||
include "$(BUILD_DIR)/assets/scenes/test_levels/test01/test01_room_0.o"
|
||||
number 3
|
||||
endseg
|
||||
#endif
|
||||
|
||||
beginseg
|
||||
name "bump_texture_static"
|
||||
|
|
|
|||
130
tools/compress.py
Normal file → Executable file
130
tools/compress.py
Normal file → Executable file
|
|
@ -1,3 +1,5 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# SPDX-FileCopyrightText: 2024 zeldaret
|
||||
# SPDX-License-Identifier: CC0-1.0
|
||||
|
||||
|
|
@ -13,57 +15,7 @@ import multiprocessing.pool
|
|||
|
||||
import crunch64
|
||||
|
||||
|
||||
STRUCT_IIII = struct.Struct(">IIII")
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DmaEntry:
|
||||
"""
|
||||
A Python counterpart to the dmadata entry struct:
|
||||
```c
|
||||
typedef struct {
|
||||
/* 0x00 */ uintptr_t vromStart;
|
||||
/* 0x04 */ uintptr_t vromEnd;
|
||||
/* 0x08 */ uintptr_t romStart;
|
||||
/* 0x0C */ uintptr_t romEnd;
|
||||
} DmaEntry;
|
||||
```
|
||||
"""
|
||||
|
||||
vromStart: int
|
||||
vromEnd: int
|
||||
romStart: int
|
||||
romEnd: int
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
"DmaEntry("
|
||||
f"vromStart=0x{self.vromStart:08X}, "
|
||||
f"vromEnd=0x{self.vromEnd:08X}, "
|
||||
f"romStart=0x{self.romStart:08X}, "
|
||||
f"romEnd=0x{self.romEnd:08X}"
|
||||
")"
|
||||
)
|
||||
|
||||
SIZE_BYTES = STRUCT_IIII.size
|
||||
|
||||
def to_bin(self, data: memoryview):
|
||||
STRUCT_IIII.pack_into(
|
||||
data,
|
||||
0,
|
||||
self.vromStart,
|
||||
self.vromEnd,
|
||||
self.romStart,
|
||||
self.romEnd,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def from_bin(data: memoryview):
|
||||
return DmaEntry(*STRUCT_IIII.unpack_from(data))
|
||||
|
||||
|
||||
DMA_ENTRY_ZERO = DmaEntry(0, 0, 0, 0)
|
||||
import dmadata
|
||||
|
||||
|
||||
def align(v: int):
|
||||
|
|
@ -73,15 +25,15 @@ def align(v: int):
|
|||
|
||||
@dataclasses.dataclass
|
||||
class RomSegment:
|
||||
vromStart: int
|
||||
vromEnd: int
|
||||
vrom_start: int
|
||||
vrom_end: int
|
||||
is_compressed: bool
|
||||
data: memoryview | None
|
||||
data_async: multiprocessing.pool.AsyncResult | None
|
||||
|
||||
@property
|
||||
def uncompressed_size(self):
|
||||
return self.vromEnd - self.vromStart
|
||||
return self.vrom_end - self.vrom_start
|
||||
|
||||
|
||||
# Make interrupting the compression with ^C less jank
|
||||
|
|
@ -94,15 +46,13 @@ def set_sigint_ignored():
|
|||
|
||||
def compress_rom(
|
||||
rom_data: memoryview,
|
||||
dmadata_offset_start: int,
|
||||
dmadata_offset_end: int,
|
||||
dmadata_offset: int,
|
||||
compress_entries_indices: set[int],
|
||||
n_threads: int = None,
|
||||
):
|
||||
"""
|
||||
rom_data: the uncompressed rom data
|
||||
dmadata_offset_start: the offset in the rom where the dmadata starts (inclusive)
|
||||
dmadata_offset_end: the offset in the rom where the dmadata ends (exclusive)
|
||||
dmadata_offset: the offset in the rom where the dmadata starts
|
||||
compress_entries_indices: the indices in the dmadata of the segments that should be compressed
|
||||
n_threads: how many cores to use for compression
|
||||
"""
|
||||
|
|
@ -110,18 +60,18 @@ def compress_rom(
|
|||
# Segments of the compressed rom (not all are compressed)
|
||||
compressed_rom_segments: list[RomSegment] = []
|
||||
|
||||
dma_entries = dmadata.read_dmadata(rom_data, dmadata_offset)
|
||||
# We sort the DMA entries by ROM start because `compress_entries_indices`
|
||||
# refers to indices in ROM order, but the uncompressed dmadata might not be
|
||||
# in ROM order.
|
||||
dma_entries.sort(key=lambda dma_entry: dma_entry.vrom_start)
|
||||
|
||||
with multiprocessing.Pool(n_threads, initializer=set_sigint_ignored) as p:
|
||||
# Extract each segment from the input rom
|
||||
for entry_index, dmadata_offset in enumerate(
|
||||
range(dmadata_offset_start, dmadata_offset_end, DmaEntry.SIZE_BYTES)
|
||||
):
|
||||
dma_entry = DmaEntry.from_bin(rom_data[dmadata_offset:])
|
||||
if dma_entry == DMA_ENTRY_ZERO:
|
||||
continue
|
||||
|
||||
segment_rom_start = dma_entry.romStart
|
||||
segment_rom_end = dma_entry.romStart + (
|
||||
dma_entry.vromEnd - dma_entry.vromStart
|
||||
for entry_index, dma_entry in enumerate(dma_entries):
|
||||
segment_rom_start = dma_entry.rom_start
|
||||
segment_rom_end = dma_entry.rom_start + (
|
||||
dma_entry.vrom_end - dma_entry.vrom_start
|
||||
)
|
||||
segment_data_uncompressed = rom_data[segment_rom_start:segment_rom_end]
|
||||
|
||||
|
|
@ -139,17 +89,14 @@ def compress_rom(
|
|||
|
||||
compressed_rom_segments.append(
|
||||
RomSegment(
|
||||
dma_entry.vromStart,
|
||||
dma_entry.vromEnd,
|
||||
dma_entry.vrom_start,
|
||||
dma_entry.vrom_end,
|
||||
is_compressed,
|
||||
segment_data,
|
||||
segment_data_async,
|
||||
)
|
||||
)
|
||||
|
||||
# Technically optional but required for matching.
|
||||
compressed_rom_segments.sort(key=lambda segment: segment.vromStart)
|
||||
|
||||
# Wait on compression of all compressed segments
|
||||
waiting_on_segments = [
|
||||
segment for segment in compressed_rom_segments if segment.is_compressed
|
||||
|
|
@ -208,7 +155,7 @@ def compress_rom(
|
|||
* pad_to_multiple_of
|
||||
)
|
||||
compressed_rom_data = memoryview(bytearray(compressed_rom_size_padded))
|
||||
compressed_rom_dma_entries: list[DmaEntry] = []
|
||||
compressed_rom_dma_entries: list[dmadata.DmaEntry] = []
|
||||
rom_offset = 0
|
||||
for segment in compressed_rom_segments:
|
||||
assert segment.data is not None
|
||||
|
|
@ -221,9 +168,9 @@ def compress_rom(
|
|||
compressed_rom_data[segment_rom_start:i] = segment.data
|
||||
|
||||
compressed_rom_dma_entries.append(
|
||||
DmaEntry(
|
||||
segment.vromStart,
|
||||
segment.vromEnd,
|
||||
dmadata.DmaEntry(
|
||||
segment.vrom_start,
|
||||
segment.vrom_end,
|
||||
segment_rom_start,
|
||||
segment_rom_end if segment.is_compressed else 0,
|
||||
)
|
||||
|
|
@ -237,13 +184,10 @@ def compress_rom(
|
|||
compressed_rom_data[i] = i % 256
|
||||
|
||||
# Write the new dmadata
|
||||
dmadata_offset = dmadata_offset_start
|
||||
offset = dmadata_offset
|
||||
for dma_entry in compressed_rom_dma_entries:
|
||||
assert dmadata_offset + DmaEntry.SIZE_BYTES <= dmadata_offset_end
|
||||
|
||||
dma_entry.to_bin(compressed_rom_data[dmadata_offset:])
|
||||
|
||||
dmadata_offset += DmaEntry.SIZE_BYTES
|
||||
dma_entry.to_bin(compressed_rom_data[offset:])
|
||||
offset += dmadata.DmaEntry.SIZE_BYTES
|
||||
|
||||
return compressed_rom_data
|
||||
|
||||
|
|
@ -263,13 +207,12 @@ def main():
|
|||
help="path of the compressed rom to write out",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dma-range",
|
||||
dest="dma_range",
|
||||
"--dma-start",
|
||||
dest="dma_start",
|
||||
type=lambda s: int(s, 16),
|
||||
required=True,
|
||||
help=(
|
||||
"The dmadata location in the rom, in format"
|
||||
" 'start_inclusive-end_exclusive' and using hexadecimal offsets"
|
||||
" (e.g. '0x12f70-0x19030')."
|
||||
"The dmadata location in the rom, as a hexadecimal offset (e.g. 0x12f70)."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
|
|
@ -298,13 +241,7 @@ def main():
|
|||
|
||||
out_rom_p = Path(args.out_rom)
|
||||
|
||||
dma_range_str: str = args.dma_range
|
||||
dma_range_ends_str = dma_range_str.split("-")
|
||||
assert len(dma_range_ends_str) == 2, dma_range_str
|
||||
dmadata_offset_start, dmadata_offset_end = (
|
||||
int(v_str, 16) for v_str in dma_range_ends_str
|
||||
)
|
||||
assert dmadata_offset_start < dmadata_offset_end, dma_range_str
|
||||
dmadata_offset = args.dma_start
|
||||
|
||||
compress_ranges_str: str = args.compress_ranges
|
||||
compress_entries_indices = set()
|
||||
|
|
@ -330,8 +267,7 @@ def main():
|
|||
in_rom_data = in_rom_p.read_bytes()
|
||||
out_rom_data = compress_rom(
|
||||
memoryview(in_rom_data),
|
||||
dmadata_offset_start,
|
||||
dmadata_offset_end,
|
||||
dmadata_offset,
|
||||
compress_entries_indices,
|
||||
n_threads,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ import crunch64
|
|||
import ipl3checksum
|
||||
import zlib
|
||||
|
||||
import dmadata
|
||||
|
||||
|
||||
def decompress_zlib(data: bytes) -> bytes:
|
||||
decomp = zlib.decompressobj(-zlib.MAX_WBITS)
|
||||
|
|
@ -33,13 +35,13 @@ def decompress(data: bytes, is_zlib_compressed: bool) -> bytes:
|
|||
|
||||
|
||||
FILE_TABLE_OFFSET = {
|
||||
"gc-eu-mq": 0x07170,
|
||||
"gc-eu-mq-dbg": 0x12F70,
|
||||
"gc-eu-mq": 0x07170,
|
||||
"gc-eu-mq-dbg": 0x12F70,
|
||||
}
|
||||
|
||||
VERSIONS_MD5S = {
|
||||
"gc-eu-mq": "1a438f4235f8038856971c14a798122a",
|
||||
"gc-eu-mq-dbg": "f0b7f35375f9cc8ca1b2d59d78e35405",
|
||||
"gc-eu-mq": "1a438f4235f8038856971c14a798122a",
|
||||
"gc-eu-mq-dbg": "f0b7f35375f9cc8ca1b2d59d78e35405",
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -48,27 +50,6 @@ def round_up(n, shift):
|
|||
return (n + mod - 1) >> shift << shift
|
||||
|
||||
|
||||
def as_word_list(b) -> list[int]:
|
||||
return [i[0] for i in struct.iter_unpack(">I", b)]
|
||||
|
||||
|
||||
def read_dmadata_entry(file_content: bytearray, addr: int) -> list[int]:
|
||||
return as_word_list(file_content[addr : addr + 0x10])
|
||||
|
||||
|
||||
def read_dmadata(file_content: bytearray, start) -> list[list[int]]:
|
||||
dmadata = []
|
||||
addr = start
|
||||
entry = read_dmadata_entry(file_content, addr)
|
||||
i = 0
|
||||
while any([e != 0 for e in entry]):
|
||||
dmadata.append(entry)
|
||||
addr += 0x10
|
||||
i += 1
|
||||
entry = read_dmadata_entry(file_content, addr)
|
||||
return dmadata
|
||||
|
||||
|
||||
def update_crc(decompressed: io.BytesIO) -> io.BytesIO:
|
||||
print("Recalculating crc...")
|
||||
calculated_checksum = ipl3checksum.CICKind.CIC_X105.calculateChecksum(
|
||||
|
|
@ -82,40 +63,43 @@ def update_crc(decompressed: io.BytesIO) -> io.BytesIO:
|
|||
|
||||
|
||||
def decompress_rom(
|
||||
file_content: bytearray, dmadata_addr: int, dmadata: list[list[int]], version: str
|
||||
file_content: bytearray,
|
||||
dmadata_offset: int,
|
||||
dma_entries: list[dmadata.DmaEntry],
|
||||
is_zlib_compressed: bool,
|
||||
) -> bytearray:
|
||||
rom_segments = {} # vrom start : data s.t. len(data) == vrom_end - vrom_start
|
||||
new_dmadata = bytearray() # new dmadata: {vrom start , vrom end , vrom start , 0}
|
||||
new_dmadata = [] # new dmadata: list[dmadata.Entry]
|
||||
|
||||
decompressed = io.BytesIO(b"")
|
||||
|
||||
for v_start, v_end, p_start, p_end in dmadata:
|
||||
for dma_entry in dma_entries:
|
||||
v_start = dma_entry.vrom_start
|
||||
v_end = dma_entry.vrom_end
|
||||
p_start = dma_entry.rom_start
|
||||
p_end = dma_entry.rom_end
|
||||
if p_start == 0xFFFFFFFF and p_end == 0xFFFFFFFF:
|
||||
new_dmadata.extend(struct.pack(">IIII", v_start, v_end, p_start, p_end))
|
||||
new_dmadata.append(dma_entry)
|
||||
continue
|
||||
if p_end == 0: # uncompressed
|
||||
rom_segments.update(
|
||||
{v_start: file_content[p_start : p_start + v_end - v_start]}
|
||||
)
|
||||
else: # compressed
|
||||
rom_segments.update(
|
||||
{
|
||||
v_start: decompress(
|
||||
file_content[p_start:p_end], version in {"ique-cn", "ique-zh"}
|
||||
)
|
||||
}
|
||||
)
|
||||
new_dmadata.extend(struct.pack(">IIII", v_start, v_end, v_start, 0))
|
||||
if dma_entry.is_compressed():
|
||||
new_contents = decompress(file_content[p_start:p_end], is_zlib_compressed)
|
||||
rom_segments[v_start] = new_contents
|
||||
else:
|
||||
rom_segments[v_start] = file_content[p_start : p_start + v_end - v_start]
|
||||
new_dmadata.append(dmadata.DmaEntry(v_start, v_end, v_start, 0))
|
||||
|
||||
# write rom segments to vaddrs
|
||||
for vrom_st, data in rom_segments.items():
|
||||
decompressed.seek(vrom_st)
|
||||
decompressed.write(data)
|
||||
# write new dmadata
|
||||
decompressed.seek(dmadata_addr)
|
||||
decompressed.write(new_dmadata)
|
||||
decompressed.seek(dmadata_offset)
|
||||
for dma_entry in new_dmadata:
|
||||
entry_data = bytearray(dmadata.DmaEntry.SIZE_BYTES)
|
||||
dma_entry.to_bin(entry_data)
|
||||
decompressed.write(entry_data)
|
||||
# pad to size
|
||||
padding_end = round_up(dmadata[-1][1], 14)
|
||||
padding_end = round_up(dma_entries[-1].vrom_end, 14)
|
||||
decompressed.seek(padding_end - 1)
|
||||
decompressed.write(bytearray([0]))
|
||||
# re-calculate crc
|
||||
|
|
@ -164,21 +148,23 @@ def per_version_fixes(file_content: bytearray, version: str) -> bytearray:
|
|||
return file_content
|
||||
|
||||
|
||||
def pad_rom(file_content: bytearray, dmadata: list[list[int]]) -> bytearray:
|
||||
padding_start = round_up(dmadata[-1][1], 12)
|
||||
padding_end = round_up(dmadata[-1][1], 14)
|
||||
def pad_rom(file_content: bytearray, dma_entries: list[dmadata.DmaEntry]) -> bytearray:
|
||||
padding_start = round_up(dma_entries[-1].vrom_end, 12)
|
||||
padding_end = round_up(dma_entries[-1].vrom_end, 14)
|
||||
print(f"Padding from {padding_start:X} to {padding_end:X}...")
|
||||
for i in range(padding_start, padding_end):
|
||||
file_content[i] = 0xFF
|
||||
return file_content
|
||||
|
||||
|
||||
# Determine if we have a ROM file
|
||||
ROM_FILE_EXTENSIONS = ["z64", "n64", "v64"]
|
||||
|
||||
|
||||
def find_baserom(version: str) -> Path | None:
|
||||
for rom_file_ext_lower in ROM_FILE_EXTENSIONS:
|
||||
for rom_file_ext in (rom_file_ext_lower, rom_file_ext_lower.upper()):
|
||||
rom_file_name_candidate = Path(f"baseroms/{version}/baserom.{rom_file_ext}")
|
||||
rom_file_name_candidate = Path(f"baseroms/{version}/baserom.{rom_file_ext}")
|
||||
if rom_file_name_candidate.exists():
|
||||
return rom_file_name_candidate
|
||||
return None
|
||||
|
|
@ -188,7 +174,11 @@ def main():
|
|||
description = "Convert a rom that uses dmadata to an uncompressed one."
|
||||
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
parser.add_argument("version", help="Version of the game to decompress.", choices=list(VERSIONS_MD5S.keys()))
|
||||
parser.add_argument(
|
||||
"version",
|
||||
help="Version of the game to decompress.",
|
||||
choices=list(VERSIONS_MD5S.keys()),
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
|
@ -196,7 +186,7 @@ def main():
|
|||
|
||||
uncompressed_path = Path(f"baseroms/{version}/baserom-decompressed.z64")
|
||||
|
||||
file_table_offset = FILE_TABLE_OFFSET[version]
|
||||
dmadata_offset = FILE_TABLE_OFFSET[version]
|
||||
correct_str_hash = VERSIONS_MD5S[version]
|
||||
|
||||
if check_existing_rom(uncompressed_path, correct_str_hash):
|
||||
|
|
@ -207,7 +197,8 @@ def main():
|
|||
|
||||
if rom_file_name is None:
|
||||
path_list = [
|
||||
f"baseroms/{version}/baserom.{rom_file_ext}" for rom_file_ext in ROM_FILE_EXTENSIONS
|
||||
f"baseroms/{version}/baserom.{rom_file_ext}"
|
||||
for rom_file_ext in ROM_FILE_EXTENSIONS
|
||||
]
|
||||
print(f"Error: Could not find {','.join(path_list)}.")
|
||||
exit(1)
|
||||
|
|
@ -234,20 +225,16 @@ def main():
|
|||
|
||||
file_content = per_version_fixes(file_content, version)
|
||||
|
||||
dmadata = read_dmadata(file_content, file_table_offset)
|
||||
dma_entries = dmadata.read_dmadata(file_content, dmadata_offset)
|
||||
# Decompress
|
||||
if any(
|
||||
[
|
||||
b != 0
|
||||
for b in file_content[
|
||||
file_table_offset + 0xAC : file_table_offset + 0xAC + 0x4
|
||||
]
|
||||
]
|
||||
):
|
||||
if any(dma_entry.is_compressed() for dma_entry in dma_entries):
|
||||
print("Decompressing rom...")
|
||||
file_content = decompress_rom(file_content, file_table_offset, dmadata, version)
|
||||
is_zlib_compressed = version in {"ique-cn", "ique-zh"}
|
||||
file_content = decompress_rom(
|
||||
file_content, dmadata_offset, dma_entries, is_zlib_compressed
|
||||
)
|
||||
|
||||
file_content = pad_rom(file_content, dmadata)
|
||||
file_content = pad_rom(file_content, dma_entries)
|
||||
|
||||
# Check to see if the ROM is a "vanilla" ROM
|
||||
str_hash = get_str_hash(file_content)
|
||||
|
|
|
|||
75
tools/dmadata.py
Normal file
75
tools/dmadata.py
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
# SPDX-FileCopyrightText: © 2024 ZeldaRET
|
||||
# SPDX-License-Identifier: CC0-1.0
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import struct
|
||||
|
||||
|
||||
STRUCT_IIII = struct.Struct(">IIII")
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class DmaEntry:
|
||||
"""
|
||||
A Python counterpart to the dmadata entry struct:
|
||||
```c
|
||||
typedef struct {
|
||||
/* 0x00 */ uintptr_t vromStart;
|
||||
/* 0x04 */ uintptr_t vromEnd;
|
||||
/* 0x08 */ uintptr_t romStart;
|
||||
/* 0x0C */ uintptr_t romEnd;
|
||||
} DmaEntry;
|
||||
```
|
||||
"""
|
||||
|
||||
vrom_start: int
|
||||
vrom_end: int
|
||||
rom_start: int
|
||||
rom_end: int
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
"DmaEntry("
|
||||
f"vrom_start=0x{self.vrom_start:08X}, "
|
||||
f"vrom_end=0x{self.vrom_end:08X}, "
|
||||
f"rom_start=0x{self.rom_start:08X}, "
|
||||
f"rom_end=0x{self.rom_end:08X}"
|
||||
")"
|
||||
)
|
||||
|
||||
SIZE_BYTES = STRUCT_IIII.size
|
||||
|
||||
def to_bin(self, data: memoryview):
|
||||
STRUCT_IIII.pack_into(
|
||||
data,
|
||||
0,
|
||||
self.vrom_start,
|
||||
self.vrom_end,
|
||||
self.rom_start,
|
||||
self.rom_end,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def from_bin(data: memoryview):
|
||||
return DmaEntry(*STRUCT_IIII.unpack_from(data))
|
||||
|
||||
def is_compressed(self) -> bool:
|
||||
return self.rom_end != 0
|
||||
|
||||
|
||||
DMA_ENTRY_END = DmaEntry(0, 0, 0, 0)
|
||||
|
||||
|
||||
def read_dmadata(rom_data: memoryview, start_offset: int) -> list[DmaEntry]:
|
||||
result = []
|
||||
|
||||
offset = start_offset
|
||||
while (
|
||||
entry := DmaEntry.from_bin(rom_data[offset : offset + DmaEntry.SIZE_BYTES])
|
||||
) != DMA_ENTRY_END:
|
||||
result.append(entry)
|
||||
offset += DmaEntry.SIZE_BYTES
|
||||
|
||||
return result
|
||||
|
|
@ -11,6 +11,5 @@ ELF=$2
|
|||
dmadata_syms=`$NM $ELF --no-sort --radix=x --format=bsd | grep dmadata`
|
||||
|
||||
_dmadataSegmentRomStart=`echo "$dmadata_syms" | grep '\b_dmadataSegmentRomStart\b' | cut -d' ' -f1`
|
||||
_dmadataSegmentRomEnd=` echo "$dmadata_syms" | grep '\b_dmadataSegmentRomEnd\b' | cut -d' ' -f1`
|
||||
|
||||
echo 0x"$_dmadataSegmentRomStart"-0x"$_dmadataSegmentRomEnd"
|
||||
echo 0x"$_dmadataSegmentRomStart"
|
||||
|
|
@ -16,24 +16,38 @@ static void write_dmadata_table(FILE *fout)
|
|||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < g_segmentsCount; i++)
|
||||
for (i = 0; i < g_segmentsCount; i++) {
|
||||
// Don't emit dma entry for segments set with NOLOAD
|
||||
if (g_segments[i].flags & FLAG_NOLOAD) {
|
||||
continue;
|
||||
}
|
||||
|
||||
fprintf(fout, "DEFINE_DMA_ENTRY(%s, \"%s\")\n", g_segments[i].name, g_segments[i].name);
|
||||
}
|
||||
}
|
||||
|
||||
static void write_compress_ranges(FILE *fout)
|
||||
{
|
||||
int i;
|
||||
int rom_index = 0;
|
||||
bool continue_list = false;
|
||||
int stride_first = -1;
|
||||
|
||||
for (i = 0; i < g_segmentsCount; i++) {
|
||||
if (g_segments[i].compress) {
|
||||
if (stride_first == -1)
|
||||
stride_first = i;
|
||||
bool compress = g_segments[i].compress;
|
||||
|
||||
// Don't consider segments set with NOLOAD when calculating indices
|
||||
if (g_segments[i].flags & FLAG_NOLOAD) {
|
||||
continue;
|
||||
}
|
||||
if (!g_segments[i].compress || i == g_segmentsCount - 1) {
|
||||
|
||||
if (compress) {
|
||||
if (stride_first == -1)
|
||||
stride_first = rom_index;
|
||||
}
|
||||
if (!compress || i == g_segmentsCount - 1) {
|
||||
if (stride_first != -1) {
|
||||
int stride_last = i - 1;
|
||||
int stride_last = compress ? rom_index : rom_index - 1;
|
||||
if (continue_list) {
|
||||
fprintf(fout, ",");
|
||||
}
|
||||
|
|
@ -46,6 +60,8 @@ static void write_compress_ranges(FILE *fout)
|
|||
stride_first = -1;
|
||||
}
|
||||
}
|
||||
|
||||
rom_index++;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -83,6 +83,8 @@ static bool parse_flags(char *str, unsigned int *flags)
|
|||
f |= FLAG_OBJECT;
|
||||
else if (strcmp(str, "RAW") == 0)
|
||||
f |= FLAG_RAW;
|
||||
else if (strcmp(str, "NOLOAD") == 0)
|
||||
f |= FLAG_NOLOAD;
|
||||
else
|
||||
return false;
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ enum {
|
|||
FLAG_BOOT = (1 << 0),
|
||||
FLAG_OBJECT = (1 << 1),
|
||||
FLAG_RAW = (1 << 2),
|
||||
FLAG_NOLOAD = (1 << 3),
|
||||
};
|
||||
|
||||
struct Include {
|
||||
|
|
|
|||
Loading…
Reference in a new issue