mirror/oot
1
0
Fork 0
mirror of https://github.com/zeldaret/oot.git synced 2025-07-18 13:54:46 +00:00
Code Issues Projects Releases Wiki Activity

New assets system (#2481)

Browse source 
* wip: New assets system tm

Builds gc-eu-mq-dbg OK from clean after
1) make setup
2) python3 -m tools.assets.extract -j
3) replace 0x80A8E610 with sShadowTex in extracted/gc-eu-mq-dbg/assets/overlays/ovl_En_Jsjutan/sShadowMaterialDL.inc.c
4) make various symbols in extracted data like sTex static

* use variables from config.yml for gMtxClear and sShadowTex addresses

* Write source with static for overlays using `HACK_IS_STATIC_ON` hack

* gc-eu-mq-dbg OK from clean with `make setup && make`

* implement more skeleton-related types, cleanups, fixups

* fix extracted data to no longer produce compilation warnings

* implement more of RoomShapeImage types

* yeet XmlPath from ExternalFile usage

* Implement PlayerAnimationDataResource (link_animetion data)

* fix csdis CS_TIME extra arg

* dmadata file names no longer hardcoded for gc-eu-mq-dbg

* ntsc-1.0 OK

* xml fixes

* slightly improve standard output

* rm extract_assets.py

* generate and use Limb enums (TODO: check Skin skels and implement for Curve skels)

* handle dependencies between xmls

* introduce RawPointers xml attribute to ignore specific pointers and keep them raw

* add tools/extract_assets.sh

* fixups

* only extract if xmls changed or if -f (force) is used

* fixups, gc-eu OK

* all versions OK

* check attributes of xml resources elements

* Implement legacy skelanime resources

* fix ASSET_FILES_BIN_EXTRACTED/COMMITTED: look for .u8.bin specifically instead of just .bin

* implement JFIFResource

* fix png/jpg wildcards: look specifically for .u64.png .u32.png .u64.jpg

* Makefile: Add rules to build .png, .bin and .jpg in assets/ too

* start writing actual docs

* extract sTransCircleDL and sTransWipeDL

* misc cleanup/fixes, pygfxd 1.0.3

* refactor CDataExt.set_write callback args to use a dataclass

* Move {} to in-source

* misc

* more progress on spec

* fix missing braces in n64dd_error_textures.c

* finish xml spec doc

* assets xmls fixes

* some cleanup, use `gNameTex_WIDTH/HEIGHT` macros in dlists

* handle hackmode_syotes_room, fix compile

* C build_from_png

* rm tools/assets/bin2c

* rm ZAPD

* format

* remove rule to generate dmadata_table.py

* CC0 license (and some import cleanup)

* dont try to build zapd (rmd)

* simplify palettes with single user (ci images with a non-shared palette)

* add docs on how images are handled

* bss

* allow -j N

* fix n64texconv python bindings memory management

* move -j at the end of calling extraction script

* with -j, update last_extracts.json as each job completes rather than only if all complete

* make interrupting less jank by making child processes ignore sigint

* use enum names in `SCENE_CMD_SKYBOX_SETTINGS`

* `multiprocessing.get_context("fork")`

* import rich, except ImportError s

* fix optional rich usage

* .bss

* .bss

* .bss

* assets extraction: -j -> -j$(N_THREADS)

* .bss

* change LIMB_NONE/MAX defaults to be FILE_OFFSET instead of SKELNAME

* 0XHEX -> 0xHEX

* fix bss

* Proper includes for assets

mostly proper, some includes like dlists resources always causing a sys_matrix.h include (when not every dlist references gIdentityMtx) could be done better

* rm z64.h

* rm z64.h take two

* bss

* Make .u64 suffix for pngs optional

* fixup: rm .u64 suffix from n64dd image paths

* Remove elemtype suffixes from .bin and .jpg files

* Update images.md

* some build_from_png cleanup, more error handling, comments

* Handle skybox textures

Introduce "sub-format" suffix for pngs, with sub-formats split_lo and split_hi being used for skybox textures

* fixup for older python

* improve collision output some

* fully use SURFACETYPE[01] macros in writing extracted surface types

* use WATERBOX_PROPERTIES in extracted waterboxes

* some SceneCommandsResource cleanup

* format EnvLightSettingsList output
This commit is contained in:
Dragorn421 2025-05-18 01:29:09 +02:00 • committed by GitHub
parent 0c6c112cb9
commit 1e556e3a3d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
460 changed files with 14342 additions and 48656 deletions
Download patch file Download diff file Expand all files Collapse all files

521
tools/assets/extract/extase/cdata_resources.py Normal file
View file

@ -0,0 +1,521 @@
# SPDX-FileCopyrightText: © 2025 ZeldaRET
# SPDX-License-Identifier: CC0-1.0
import abc
import dataclasses
import io
from typing import TYPE_CHECKING, Callable, Any, Sequence, Union
if TYPE_CHECKING:
from .memorymap import MemoryContext
from . import (
RESOURCE_PARSE_SUCCESS,
Resource,
File,
ResourceParseWaiting,
)
from .repr_c_struct import (
CData,
CData_Value,
CData_Struct,
CData_Array,
)
@dataclasses.dataclass
class CDataExtWriteContext:
f: io.TextIOBase
line_prefix: str
inhibit_top_braces: bool
class CDataExt(CData, abc.ABC):
report_f = None
write_f = None
def set_report(
self, report_f: Callable[["CDataResource", "MemoryContext", Any], None]
):
self.report_f = report_f
return self
def set_write(
self,
write_f: Callable[
["CDataResource", "MemoryContext", Any, CDataExtWriteContext],
bool,
],
):
"""
write_f should return True if it wrote anything
"""
self.write_f = write_f
return self
def freeze(self):
self.set_report = None
self.set_write = None
return self
@abc.abstractmethod
def write_default(
self,
resource: "CDataResource",
memory_context: "MemoryContext",
v: Any,
f: io.TextIOBase,
line_prefix: str,
*,
inhibit_top_braces: bool,
) -> bool: ...
def report(
self,
resource: "CDataResource",
memory_context: "MemoryContext",
v: Any,
):
if self.report_f:
try:
self.report_f(resource, memory_context, v)
except:
print("Error reporting data", self, self.report_f, resource, v)
raise
def write(
self,
resource: "CDataResource",
memory_context: "MemoryContext",
v: Any,
f: io.TextIOBase,
line_prefix: str,
*,
inhibit_top_braces: bool,
) -> bool:
"""
Returns True if something has been written
(typically, False will be returned if this data is struct padding)
"""
if self.write_f:
ret = self.write_f(
resource,
memory_context,
v,
CDataExtWriteContext(f, line_prefix, inhibit_top_braces),
)
# This assert is meant to ensure the function returns a value at all,
# since it's easy to forget to return a value (typically True)
assert isinstance(ret, bool), ("must return a bool", self.write_f)
else:
ret = self.write_default(
resource,
memory_context,
v,
f,
line_prefix,
inhibit_top_braces=inhibit_top_braces,
)
assert isinstance(ret, bool), self
return ret
class CDataExt_Value(CData_Value, CDataExt):
is_padding = False
def padding(self):
self.is_padding = True
return self
def freeze(self):
self.padding = None
return super().freeze()
def set_write_str_v(self, str_v: Callable[[Any], str]):
"""Utility wrapper for set_write, writes the value as stringified by str_v."""
def write_f(
resource: "CDataResource",
memory_context: "MemoryContext",
v: Any,
wctx: CDataExtWriteContext,
):
wctx.f.write(wctx.line_prefix)
wctx.f.write(str_v(v))
return True
self.set_write(write_f)
return self
def report(self, resource, memory_context, v):
super().report(resource, memory_context, v)
if self.is_padding:
if v != 0:
raise Exception("non-0 padding")
def write_default(
self, resource, memory_context, v, f, line_prefix, *, inhibit_top_braces
):
assert (
not inhibit_top_braces
), "CDataExt_Value can't inhibit top braces, it doesn't have any"
if not self.is_padding:
f.write(line_prefix)
f.write(str(v))
return True
else:
return False
CDataExt_Value.s8 = CDataExt_Value("b").freeze()
CDataExt_Value.u8 = CDataExt_Value("B").freeze()
CDataExt_Value.s16 = CDataExt_Value("h").freeze()
CDataExt_Value.u16 = CDataExt_Value("H").freeze()
CDataExt_Value.s32 = CDataExt_Value("i").freeze()
CDataExt_Value.u32 = CDataExt_Value("I").freeze()
CDataExt_Value.f32 = CDataExt_Value("f").freeze()
CDataExt_Value.f64 = CDataExt_Value("d").freeze()
CDataExt_Value.pointer = CDataExt_Value("I").freeze()
CDataExt_Value.pad8 = CDataExt_Value("b").padding().freeze()
CDataExt_Value.pad16 = CDataExt_Value("h").padding().freeze()
CDataExt_Value.pad32 = CDataExt_Value("i").padding().freeze()
INDENT = " " * 4
class CDataExt_Array(CData_Array, CDataExt):
def __init__(self, element_cdata_ext: CDataExt, length: int):
super().__init__(element_cdata_ext, length)
self.element_cdata_ext = element_cdata_ext
def report(self, resource, memory_context, v):
assert isinstance(v, list)
super().report(resource, memory_context, v)
for elem in v:
self.element_cdata_ext.report(resource, memory_context, elem)
def write_default(
self, resource, memory_context, v, f, line_prefix, *, inhibit_top_braces
):
assert isinstance(v, list)
if not inhibit_top_braces:
f.write(line_prefix)
f.write("{\n")
for i, elem in enumerate(v):
ret = self.element_cdata_ext.write(
resource,
memory_context,
elem,
f,
line_prefix + INDENT,
inhibit_top_braces=False,
)
assert ret
f.write(f", // {i}\n")
if not inhibit_top_braces:
f.write(line_prefix)
f.write("}")
return True
class CDataExt_Struct(CData_Struct, CDataExt):
def __init__(self, members: Sequence[tuple[str, CDataExt]]):
super().__init__(members)
self.members_ext = members
def report(self, resource, memory_context, v):
assert isinstance(v, dict)
super().report(resource, memory_context, v)
for member_name, member_cdata_ext in self.members_ext:
member_cdata_ext.report(resource, memory_context, v[member_name])
def write_default(
self, resource, memory_context, v, f, line_prefix, *, inhibit_top_braces
):
assert isinstance(v, dict)
if not inhibit_top_braces:
f.write(line_prefix)
f.write("{\n")
for member_name, member_cdata_ext in self.members_ext:
if member_cdata_ext.write(
resource,
memory_context,
v[member_name],
f,
line_prefix + INDENT,
inhibit_top_braces=False,
):
f.write(f", // {member_name}\n")
if not inhibit_top_braces:
f.write(line_prefix)
f.write("}")
return True
class CDataResource(Resource):
# Set by child classes
cdata_ext: CDataExt
# Resource implementation
def __init__(self, file: File, range_start: int, name: str):
if not self.can_size_be_unknown:
assert hasattr(self, "cdata_ext"), self.__class__
assert self.cdata_ext is not None
range_end = range_start + self.cdata_ext.size
else:
if hasattr(self, "cdata_ext") and self.cdata_ext is not None:
range_end = range_start + self.cdata_ext.size
else:
range_end = None
super().__init__(file, range_start, range_end, name)
self._is_cdata_processed = False
def try_parse_data(self, memory_context: "MemoryContext"):
if self.can_size_be_unknown:
assert hasattr(self, "cdata_ext") and self.cdata_ext is not None, (
"Subclasses with can_size_be_unknown=True should redefine try_parse_data"
" and call the superclass definition (CDataResource.try_parse_data)"
" only once cdata_ext has been set",
self.__class__,
)
assert (
self.range_end is not None
), "Subclasses with can_size_be_unknown=True should also set range_end once the size is known"
assert hasattr(self, "cdata_ext")
assert self.cdata_ext is not None
# In case the subclass does more involved processing, the self.is_data_parsed
# bool wouldn't necessarily reflect the state of the cdata.
# Use own bool self._is_cdata_processed to remember if cdata has been unpacked and
# reported already.
if not self._is_cdata_processed:
self.cdata_unpacked = self.cdata_ext.unpack_from(
self.file.data, self.range_start
)
self.cdata_ext.report(self, memory_context, self.cdata_unpacked)
self._is_cdata_processed = True
return RESOURCE_PARSE_SUCCESS
def write_extracted(self, memory_context):
with self.extract_to_path.open("w") as f:
self.cdata_ext.write(
self,
memory_context,
self.cdata_unpacked,
f,
"",
inhibit_top_braces=self.braces_in_source,
)
f.write("\n")
class CDataArrayResource(CDataResource):
"""Helper for variable-length array resources.
The length is unknown at object creation, and must be set eventually
with set_length (for example by another resource).
The length being set then allows this resource to be parsed.
For static-length array resources, just use CDataResource.
"""
def __init_subclass__(cls, /, **kwargs):
super().__init_subclass__(can_size_be_unknown=True, **kwargs)
elem_cdata_ext: CDataExt
def __init__(self, file: File, range_start: int, name: str):
super().__init__(file, range_start, name)
self._length: Union[None, int] = None
def set_length(self, length: int):
if self._length is not None:
if self._length != length:
raise Exception(
"length already set and is different", self._length, length
)
assert length > 0
self._length = length
def try_parse_data(self, memory_context: "MemoryContext"):
if self._length is None:
raise ResourceParseWaiting(waiting_for=["self._length"])
assert isinstance(self.elem_cdata_ext, CDataExt), (self.__class__, self)
self.cdata_ext = CDataExt_Array(self.elem_cdata_ext, self._length)
self.range_end = self.range_start + self.cdata_ext.size
return super().try_parse_data(memory_context)
def get_c_reference(self, resource_offset: int):
if resource_offset == 0:
return self.symbol_name
else:
raise ValueError
def get_c_expression_length(self, resource_offset: int):
if resource_offset == 0:
return f"ARRAY_COUNT({self.symbol_name})"
else:
raise ValueError
class CDataArrayNamedLengthResource(CDataArrayResource):
"""CDataArrayResource and with a macro (define) for its length.
This is useful for arrays that have a length that should be referenced somewhere,
but cannot due to the order the definitions are in.
This writes a macro to the .h for the length, along the symbol declaration,
to be used in the declaration base (! by the subclass, in get_c_declaration_base)
"""
def __init__(self, file: File, range_start: int, name: str):
super().__init__(file, range_start, name)
self.length_name = f"LENGTH_{self.symbol_name}"
def write_c_declaration(self, h: io.TextIOBase):
h.write(f"#define {self.length_name} {self._length}\n")
super().write_c_declaration(h)
cdata_ext_Vec3s = CDataExt_Struct(
(
("x", CDataExt_Value.s16),
("y", CDataExt_Value.s16),
("z", CDataExt_Value.s16),
)
).freeze()
def write_Vec3s_aligned(resource, memory_context, v, wctx: CDataExtWriteContext):
s = f'{v["x"]:6}, {v["y"]:6}, {v["z"]:6}'
if not wctx.inhibit_top_braces:
s = "{ " + s + " }"
wctx.f.write(wctx.line_prefix)
wctx.f.write(s)
return True
cdata_ext_Vec3s_aligned = (
CDataExt_Struct(
(
("x", CDataExt_Value.s16),
("y", CDataExt_Value.s16),
("z", CDataExt_Value.s16),
)
)
.set_write(write_Vec3s_aligned)
.freeze()
)
class Vec3sArrayResource(CDataResource):
elem_cdata_ext = cdata_ext_Vec3s
def __init__(self, file: File, range_start: int, name: str, length: int):
assert length > 0
self.cdata_ext = CDataExt_Array(self.elem_cdata_ext, length)
super().__init__(file, range_start, name)
def get_c_declaration_base(self):
return f"Vec3s {self.symbol_name}[]"
def get_c_reference(self, resource_offset: int):
if resource_offset == 0:
return self.symbol_name
else:
raise ValueError()
def get_c_expression_length(self, resource_offset: int):
if resource_offset == 0:
return f"ARRAY_COUNT({self.symbol_name})"
else:
raise ValueError()
def get_h_includes(self):
return ("z64math.h",)
class S16ArrayResource(CDataResource):
elem_cdata_ext = CDataExt_Value.s16
def __init__(self, file: File, range_start: int, name: str, length: int):
assert length > 0
self.cdata_ext = CDataExt_Array(self.elem_cdata_ext, length)
super().__init__(file, range_start, name)
def get_c_declaration_base(self):
if hasattr(self, "HACK_IS_STATIC_ON"):
return f"s16 {self.symbol_name}[{self.cdata_ext.size // self.elem_cdata_ext.size}]"
return f"s16 {self.symbol_name}[]"
def get_c_reference(self, resource_offset: int):
if resource_offset == 0:
return self.symbol_name
else:
raise ValueError()
def get_c_expression_length(self, resource_offset: int):
if resource_offset == 0:
return f"ARRAY_COUNT({self.symbol_name})"
else:
raise ValueError()
def get_h_includes(self):
return ("ultra64.h",)
cdata_ext_Vec3f = CDataExt_Struct(
(
("x", CDataExt_Value.f32),
("y", CDataExt_Value.f32),
("z", CDataExt_Value.f32),
)
)
def fmt_hex_s(v: int, nibbles: int = 0):
"""Format v to 0x-prefixed uppercase hexadecimal, using (at least) the specified amount of nibbles.
Meant for signed values (_s suffix),
adds a space in place of where the - sign would be for positive values.
Note compared to this,
- f"{v:#X}" would produce an uppercase 0X (1 -> 0X1)
- f"0x{v:X}" doesn't work with negative values (-1 -> 0x-1)
"""
v_str = f"{v:0{nibbles}X}"
if v < 0:
v_str = v_str.removeprefix("-")
return f"-0x{v_str}"
else:
return f" 0x{v_str}"
def fmt_hex_u(v: int, nibbles: int = 0):
"""Format v to 0x-prefixed uppercase hexadecimal, using (at least) the specified amount of nibbles.
Meant for unsigned values (_u suffix),
but won't fail for negative values.
See: fmt_hex_s
"""
v_str = f"{v:0{nibbles}X}"
if v < 0:
# Also handle v being negative just in case,
# it will only mean the output isn't aligned as expected
v_str = v_str.removeprefix("-")
return f"-0x{v_str}"
else:
return f"0x{v_str}"