From 956b3b4a96971ec59e214159eb565166e2a9ad33 Mon Sep 17 00:00:00 2001 From: Dragorn421 Date: Wed, 5 Feb 2025 18:26:25 +0100 Subject: [PATCH] use variables from config.yml for gMtxClear and sShadowTex addresses --- tools/assets/extract/__main__.py | 2 +- tools/assets/extract/extase/memorymap.py | 58 ++++++++++++------------ tools/assets/extract/extract_xml_z64.py | 47 ++++++++++++++----- 3 files changed, 67 insertions(+), 40 deletions(-) diff --git a/tools/assets/extract/__main__.py b/tools/assets/extract/__main__.py index 7fccc73875..991e280533 100644 --- a/tools/assets/extract/__main__.py +++ b/tools/assets/extract/__main__.py @@ -1,7 +1,7 @@ from . import extract_xml_z64 if __name__ == "__main__": - profile = True + profile = False if profile: import cProfile diff --git a/tools/assets/extract/extase/memorymap.py b/tools/assets/extract/extase/memorymap.py index 34f9fe3e27..34e78920bd 100644 --- a/tools/assets/extract/extase/memorymap.py +++ b/tools/assets/extract/extase/memorymap.py @@ -1,4 +1,4 @@ -from __future__ import annotations +import abc from dataclasses import dataclass from typing import Callable, TypeVar, Generic @@ -38,7 +38,7 @@ class RangeMap(Generic[RangeMapValueT]): def get_all_by_predicate(self, predicate: Callable[[int, int], bool]): """Return all values associated to a range for which the predicate returns True""" values: dict[tuple[int, int], RangeMapValueT] = dict() - for ((range_start, range_end), value) in self.values_by_range.items(): + for (range_start, range_end), value in self.values_by_range.items(): if predicate(range_start, range_end): values[(range_start, range_end)] = value return values @@ -180,11 +180,11 @@ class AddressResolveResult: ) -def address_resolver(original_address, address_offset): - return AddressResolveResult(original_address) - - -AddressResolver = Callable[[int, int], AddressResolveResult] +class AddressResolver(abc.ABC): + @abc.abstractmethod + def resolve( + self, original_address: int, address_offset: int + ) -> AddressResolveResult: ... class MemoryMap: @@ -212,6 +212,25 @@ def get_segment_num(address: int): return (address & 0x0F00_0000) >> 24 +@dataclass +class FileDirectAddressResolver(AddressResolver): + direct_file_offset_start: int + target_file: File + + def resolve(self, original_address, address_offset): + file_offset = address_offset - self.direct_file_offset_start + return AddressResolveResult(original_address, self.target_file, file_offset) + + +@dataclass +class FileSegmentAddressResolver(AddressResolver): + target_file: File + + def resolve(self, original_address, address_offset): + file_offset = address_offset + return AddressResolveResult(original_address, self.target_file, file_offset) + + class MemoryContext: """ handles segmented addresses, pointers, external symbols (eg gMtxClear) @@ -222,15 +241,6 @@ class MemoryContext: def __init__(self): self.memory_map = MemoryMap() - # FIXME HACK - # TODO config for this - from ..extase_oot64.dlist_resources import MtxResource - - file_gMtxClear = File("sys_matrix__gMtxClear", size=0x40) - resource_gMtxClear = MtxResource(file_gMtxClear, 0, "gMtxClear") - file_gMtxClear.add_resource(resource_gMtxClear) - self.set_direct_file(0x12DB20, file_gMtxClear) - def copy(self): other = MemoryContext() other.memory_map = self.memory_map.copy() @@ -248,16 +258,12 @@ class MemoryContext: direct_file_offset_start = self._direct_address_to_offset(address) direct_file_offset_end = direct_file_offset_start + target_file.size - def file_direct_address_resolver(original_address, address_offset): - file_offset = address_offset - direct_file_offset_start - return AddressResolveResult(original_address, target_file, file_offset) - # TODO should memory_map members be directly accessed, # or should MemoryMap have functions (applies elsewhere in MemoryContext) self.memory_map.direct.set( direct_file_offset_start, direct_file_offset_end, - file_direct_address_resolver, + FileDirectAddressResolver(direct_file_offset_start, target_file), ) def set_segment_file(self, segment_num: int, target_file: File): @@ -266,12 +272,8 @@ class MemoryContext: "Segment number must be between 1 and 15 (inclusive)", segment_num ) - def file_segment_address_resolver(original_address, address_offset): - file_offset = address_offset - return AddressResolveResult(original_address, target_file, file_offset) - self.memory_map.segments[segment_num].set( - 0, 0x0100_0000, file_segment_address_resolver + 0, 0x0100_0000, FileSegmentAddressResolver(target_file) ) def resolve_direct(self, address: int): @@ -282,7 +284,7 @@ class MemoryContext: raise UnmappedAddressError( "direct address is not mapped", hex(address) ) from e - return address_resolver(address, offset) + return address_resolver.resolve(address, offset) def resolve_segmented(self, address: int): segment_num = get_segment_num(address) @@ -297,7 +299,7 @@ class MemoryContext: raise UnmappedAddressError( "segment address is not mapped", hex(address) ) from e - return address_resolver(address, offset) + return address_resolver.resolve(address, offset) def report_resource_at_segmented( self, diff --git a/tools/assets/extract/extract_xml_z64.py b/tools/assets/extract/extract_xml_z64.py index b28db097f2..bf0462f4ae 100644 --- a/tools/assets/extract/extract_xml_z64.py +++ b/tools/assets/extract/extract_xml_z64.py @@ -1,5 +1,6 @@ -from pathlib import Path +import dataclasses import functools +from pathlib import Path from pprint import pprint from ..descriptor.base import ( @@ -83,7 +84,9 @@ def create_file_resources(rescoll: ResourcesDescCollection, file: File): ) -def process_pool(pool_desc: ResourcesDescCollectionsPool): +def process_pool( + version_memctx_base: MemoryContext, pool_desc: ResourcesDescCollectionsPool +): if VERBOSE2: print("> process_pool") print( @@ -125,7 +128,7 @@ def process_pool(pool_desc: ResourcesDescCollectionsPool): # 2) Build a MemoryContext for each File - memctx_base = MemoryContext() + memctx_base = version_memctx_base.copy() files_by_segment: dict[int, list[File]] = dict() for rescoll, file in file_by_rescoll.items(): @@ -242,17 +245,20 @@ def process_pool(pool_desc: ResourcesDescCollectionsPool): file.write_source() -def process_pool_wrapped(pd): +def process_pool_wrapped(version_memctx_base, pd): try: - process_pool(pd) + process_pool(version_memctx_base, pd) except Exception as e: import traceback import sys # Some exceptions can't be pickled for passing back to the main process # so print them now as well - traceback.print_exc(sys.stdout) - raise Exception() from e + traceback.print_exc(file=sys.stdout) + raise Exception( + "ERROR with pool_desc collections:", + [str(rescoll.out_path) for rescoll in pd.collections], + ) from e def main(): @@ -271,9 +277,25 @@ def main(): pools_desc = get_resources_desc(vc) + version_memctx_base = MemoryContext() + + from .extase_oot64.dlist_resources import MtxResource, TextureResource + from ..n64 import G_IM_FMT, G_IM_SIZ + + file_gMtxClear = File("sys_matrix__gMtxClear", size=0x40) + file_gMtxClear.add_resource(MtxResource(file_gMtxClear, 0, "gMtxClear")) + version_memctx_base.set_direct_file(vc.variables["gMtxClear"], file_gMtxClear) + + file_sShadowTex = File("z_en_jsjutan__sShadowTex", size=0x800) + file_sShadowTex.add_resource( + TextureResource( + file_sShadowTex, 0, "sShadowTex", G_IM_FMT.I, G_IM_SIZ._8b, 32, 64 + ) + ) + version_memctx_base.set_direct_file(vc.variables["sShadowTex"], file_sShadowTex) + z64_resource_handlers.register_resource_handlers() - # TODO single pool extract cli arg # TODO extract only when a pool xml was modified since last extract try: if args.single is not None: @@ -285,7 +307,7 @@ def main(): if coll.backing_memory.name == args.single: do_process_pool = True if do_process_pool: - process_pool(pool_desc) + process_pool(version_memctx_base, pool_desc) any_do_process_pool = True if any_do_process_pool: print("OK") @@ -293,13 +315,16 @@ def main(): print("Not found:", args.single) elif not args.use_multiprocessing: # everything on one process for pool_desc in pools_desc: - process_pool(pool_desc) + process_pool(version_memctx_base, pool_desc) print("all OK!!!") else: # multiprocessing import multiprocessing with multiprocessing.Pool() as pool: - pool.map(process_pool_wrapped, pools_desc) + pool.starmap( + process_pool_wrapped, + zip([version_memctx_base] * len(pools_desc), pools_desc), + ) print("all OK!?") except Exception as e: import traceback