use variables from config.yml for gMtxClear and sShadowTex addresses

tools/assets/extract/__main__.py

@@ -1,7 +1,7 @@
 from . import extract_xml_z64
 
 if __name__ == "__main__":
-    profile = True
+    profile = False
     if profile:
         import cProfile
 

tools/assets/extract/extase/memorymap.py

@@ -1,4 +1,4 @@
-from __future__ import annotations
+import abc
 from dataclasses import dataclass
 
 from typing import Callable, TypeVar, Generic
@@ -38,7 +38,7 @@ class RangeMap(Generic[RangeMapValueT]):
     def get_all_by_predicate(self, predicate: Callable[[int, int], bool]):
         """Return all values associated to a range for which the predicate returns True"""
         values: dict[tuple[int, int], RangeMapValueT] = dict()
-        for ((range_start, range_end), value) in self.values_by_range.items():
+        for (range_start, range_end), value in self.values_by_range.items():
             if predicate(range_start, range_end):
                 values[(range_start, range_end)] = value
         return values
@@ -180,11 +180,11 @@ class AddressResolveResult:
         )
 
 
-def address_resolver(original_address, address_offset):
+class AddressResolver(abc.ABC):
-    return AddressResolveResult(original_address)
+    @abc.abstractmethod
-
+    def resolve(
-
+        self, original_address: int, address_offset: int
-AddressResolver = Callable[[int, int], AddressResolveResult]
+    ) -> AddressResolveResult: ...
 
 
 class MemoryMap:
@@ -212,6 +212,25 @@ def get_segment_num(address: int):
     return (address & 0x0F00_0000) >> 24
 
 
+@dataclass
+class FileDirectAddressResolver(AddressResolver):
+    direct_file_offset_start: int
+    target_file: File
+
+    def resolve(self, original_address, address_offset):
+        file_offset = address_offset - self.direct_file_offset_start
+        return AddressResolveResult(original_address, self.target_file, file_offset)
+
+
+@dataclass
+class FileSegmentAddressResolver(AddressResolver):
+    target_file: File
+
+    def resolve(self, original_address, address_offset):
+        file_offset = address_offset
+        return AddressResolveResult(original_address, self.target_file, file_offset)
+
+
 class MemoryContext:
     """
     handles segmented addresses, pointers, external symbols (eg gMtxClear)
@@ -222,15 +241,6 @@ class MemoryContext:
     def __init__(self):
         self.memory_map = MemoryMap()
 
-        # FIXME HACK
-        # TODO config for this
-        from ..extase_oot64.dlist_resources import MtxResource
-
-        file_gMtxClear = File("sys_matrix__gMtxClear", size=0x40)
-        resource_gMtxClear = MtxResource(file_gMtxClear, 0, "gMtxClear")
-        file_gMtxClear.add_resource(resource_gMtxClear)
-        self.set_direct_file(0x12DB20, file_gMtxClear)
-
     def copy(self):
         other = MemoryContext()
         other.memory_map = self.memory_map.copy()
@@ -248,16 +258,12 @@ class MemoryContext:
         direct_file_offset_start = self._direct_address_to_offset(address)
         direct_file_offset_end = direct_file_offset_start + target_file.size
 
-        def file_direct_address_resolver(original_address, address_offset):
-            file_offset = address_offset - direct_file_offset_start
-            return AddressResolveResult(original_address, target_file, file_offset)
-
         # TODO should memory_map members be directly accessed,
         # or should MemoryMap have functions (applies elsewhere in MemoryContext)
         self.memory_map.direct.set(
             direct_file_offset_start,
             direct_file_offset_end,
-            file_direct_address_resolver,
+            FileDirectAddressResolver(direct_file_offset_start, target_file),
         )
 
     def set_segment_file(self, segment_num: int, target_file: File):
@@ -266,12 +272,8 @@ class MemoryContext:
                 "Segment number must be between 1 and 15 (inclusive)", segment_num
             )
 
-        def file_segment_address_resolver(original_address, address_offset):
-            file_offset = address_offset
-            return AddressResolveResult(original_address, target_file, file_offset)
-
         self.memory_map.segments[segment_num].set(
-            0, 0x0100_0000, file_segment_address_resolver
+            0, 0x0100_0000, FileSegmentAddressResolver(target_file)
         )
 
     def resolve_direct(self, address: int):
@@ -282,7 +284,7 @@ class MemoryContext:
             raise UnmappedAddressError(
                 "direct address is not mapped", hex(address)
             ) from e
-        return address_resolver(address, offset)
+        return address_resolver.resolve(address, offset)
 
     def resolve_segmented(self, address: int):
         segment_num = get_segment_num(address)
@@ -297,7 +299,7 @@ class MemoryContext:
                 raise UnmappedAddressError(
                     "segment address is not mapped", hex(address)
                 ) from e
-            return address_resolver(address, offset)
+            return address_resolver.resolve(address, offset)
 
     def report_resource_at_segmented(
         self,

tools/assets/extract/extract_xml_z64.py

@@ -1,5 +1,6 @@
-from pathlib import Path
+import dataclasses
 import functools
+from pathlib import Path
 from pprint import pprint
 
 from ..descriptor.base import (
@@ -83,7 +84,9 @@ def create_file_resources(rescoll: ResourcesDescCollection, file: File):
     )
 
 
-def process_pool(pool_desc: ResourcesDescCollectionsPool):
+def process_pool(
+    version_memctx_base: MemoryContext, pool_desc: ResourcesDescCollectionsPool
+):
     if VERBOSE2:
         print("> process_pool")
         print(
@@ -125,7 +128,7 @@ def process_pool(pool_desc: ResourcesDescCollectionsPool):
 
     # 2) Build a MemoryContext for each File
 
-    memctx_base = MemoryContext()
+    memctx_base = version_memctx_base.copy()
     files_by_segment: dict[int, list[File]] = dict()
 
     for rescoll, file in file_by_rescoll.items():
@@ -242,17 +245,20 @@ def process_pool(pool_desc: ResourcesDescCollectionsPool):
             file.write_source()
 
 
-def process_pool_wrapped(pd):
+def process_pool_wrapped(version_memctx_base, pd):
     try:
-        process_pool(pd)
+        process_pool(version_memctx_base, pd)
     except Exception as e:
         import traceback
         import sys
 
         # Some exceptions can't be pickled for passing back to the main process
         # so print them now as well
-        traceback.print_exc(sys.stdout)
+        traceback.print_exc(file=sys.stdout)
-        raise Exception() from e
+        raise Exception(
+            "ERROR with pool_desc collections:",
+            [str(rescoll.out_path) for rescoll in pd.collections],
+        ) from e
 
 
 def main():
@@ -271,9 +277,25 @@ def main():
 
     pools_desc = get_resources_desc(vc)
 
+    version_memctx_base = MemoryContext()
+
+    from .extase_oot64.dlist_resources import MtxResource, TextureResource
+    from ..n64 import G_IM_FMT, G_IM_SIZ
+
+    file_gMtxClear = File("sys_matrix__gMtxClear", size=0x40)
+    file_gMtxClear.add_resource(MtxResource(file_gMtxClear, 0, "gMtxClear"))
+    version_memctx_base.set_direct_file(vc.variables["gMtxClear"], file_gMtxClear)
+
+    file_sShadowTex = File("z_en_jsjutan__sShadowTex", size=0x800)
+    file_sShadowTex.add_resource(
+        TextureResource(
+            file_sShadowTex, 0, "sShadowTex", G_IM_FMT.I, G_IM_SIZ._8b, 32, 64
+        )
+    )
+    version_memctx_base.set_direct_file(vc.variables["sShadowTex"], file_sShadowTex)
+
     z64_resource_handlers.register_resource_handlers()
 
-    # TODO single pool extract cli arg
     # TODO extract only when a pool xml was modified since last extract
     try:
         if args.single is not None:
@@ -285,7 +307,7 @@ def main():
                         if coll.backing_memory.name == args.single:
                             do_process_pool = True
                 if do_process_pool:
-                    process_pool(pool_desc)
+                    process_pool(version_memctx_base, pool_desc)
                     any_do_process_pool = True
             if any_do_process_pool:
                 print("OK")
@@ -293,13 +315,16 @@ def main():
                 print("Not found:", args.single)
         elif not args.use_multiprocessing:  # everything on one process
             for pool_desc in pools_desc:
-                process_pool(pool_desc)
+                process_pool(version_memctx_base, pool_desc)
             print("all OK!!!")
         else:  # multiprocessing
             import multiprocessing
 
             with multiprocessing.Pool() as pool:
-                pool.map(process_pool_wrapped, pools_desc)
+                pool.starmap(
+                    process_pool_wrapped,
+                    zip([version_memctx_base] * len(pools_desc), pools_desc),
+                )
             print("all OK!?")
     except Exception as e:
         import traceback