mirror of
https://github.com/zeldaret/oot.git
synced 2025-08-09 08:20:17 +00:00
New assets system (#2481)
* wip: New assets system tm
Builds gc-eu-mq-dbg OK from clean after
1) make setup
2) python3 -m tools.assets.extract -j
3) replace 0x80A8E610 with sShadowTex in extracted/gc-eu-mq-dbg/assets/overlays/ovl_En_Jsjutan/sShadowMaterialDL.inc.c
4) make various symbols in extracted data like sTex static
* use variables from config.yml for gMtxClear and sShadowTex addresses
* Write source with static for overlays using `HACK_IS_STATIC_ON` hack
* gc-eu-mq-dbg OK from clean with `make setup && make`
* implement more skeleton-related types, cleanups, fixups
* fix extracted data to no longer produce compilation warnings
* implement more of RoomShapeImage types
* yeet XmlPath from ExternalFile usage
* Implement PlayerAnimationDataResource (link_animetion data)
* fix csdis CS_TIME extra arg
* dmadata file names no longer hardcoded for gc-eu-mq-dbg
* ntsc-1.0 OK
* xml fixes
* slightly improve standard output
* rm extract_assets.py
* generate and use Limb enums (TODO: check Skin skels and implement for Curve skels)
* handle dependencies between xmls
* introduce RawPointers xml attribute to ignore specific pointers and keep them raw
* add tools/extract_assets.sh
* fixups
* only extract if xmls changed or if -f (force) is used
* fixups, gc-eu OK
* all versions OK
* check attributes of xml resources elements
* Implement legacy skelanime resources
* fix ASSET_FILES_BIN_EXTRACTED/COMMITTED: look for .u8.bin specifically instead of just .bin
* implement JFIFResource
* fix png/jpg wildcards: look specifically for .u64.png .u32.png .u64.jpg
* Makefile: Add rules to build .png, .bin and .jpg in assets/ too
* start writing actual docs
* extract sTransCircleDL and sTransWipeDL
* misc cleanup/fixes, pygfxd 1.0.3
* refactor CDataExt.set_write callback args to use a dataclass
* Move {} to in-source
* misc
* more progress on spec
* fix missing braces in n64dd_error_textures.c
* finish xml spec doc
* assets xmls fixes
* some cleanup, use `gNameTex_WIDTH/HEIGHT` macros in dlists
* handle hackmode_syotes_room, fix compile
* C build_from_png
* rm tools/assets/bin2c
* rm ZAPD
* format
* remove rule to generate dmadata_table.py
* CC0 license (and some import cleanup)
* dont try to build zapd (rmd)
* simplify palettes with single user (ci images with a non-shared palette)
* add docs on how images are handled
* bss
* allow -j N
* fix n64texconv python bindings memory management
* move -j at the end of calling extraction script
* with -j, update last_extracts.json as each job completes rather than only if all complete
* make interrupting less jank by making child processes ignore sigint
* use enum names in `SCENE_CMD_SKYBOX_SETTINGS`
* `multiprocessing.get_context("fork")`
* import rich, except ImportError s
* fix optional rich usage
* .bss
* .bss
* .bss
* assets extraction: -j -> -j$(N_THREADS)
* .bss
* change LIMB_NONE/MAX defaults to be FILE_OFFSET instead of SKELNAME
* 0XHEX -> 0xHEX
* fix bss
* Proper includes for assets
mostly proper, some includes like dlists resources always causing a sys_matrix.h include (when not every dlist references gIdentityMtx) could be done better
* rm z64.h
* rm z64.h take two
* bss
* Make .u64 suffix for pngs optional
* fixup: rm .u64 suffix from n64dd image paths
* Remove elemtype suffixes from .bin and .jpg files
* Update images.md
* some build_from_png cleanup, more error handling, comments
* Handle skybox textures
Introduce "sub-format" suffix for pngs, with sub-formats split_lo and split_hi being used for skybox textures
* fixup for older python
* improve collision output some
* fully use SURFACETYPE[01] macros in writing extracted surface types
* use WATERBOX_PROPERTIES in extracted waterboxes
* some SceneCommandsResource cleanup
* format EnvLightSettingsList output
This commit is contained in:
Dragorn421
GitHub
parent
0c6c112cb9
commit
1e556e3a3d
460 changed files with 14342 additions and 48656 deletions
223
tools/assets/extract/extase/repr_c_struct.py
Normal file
223
tools/assets/extract/extase/repr_c_struct.py
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
# SPDX-FileCopyrightText: © 2025 ZeldaRET
|
||||
# SPDX-License-Identifier: CC0-1.0
|
||||
|
||||
import struct
|
||||
import abc
|
||||
from typing import Sequence, Any
|
||||
|
||||
|
||||
# NOTE: this system does NOT handle struct alignment/padding automatically, it should be made explicit
|
||||
|
||||
# this system voluntarily does not handle variable length arrays. which is not a valid "type" in C anyway (?)
|
||||
# having variable-sized data is too messy to handle, because it needs a size at some point anyway
|
||||
# This choice allows the root CData ABC to have a size as a guaranteed attribute
|
||||
|
||||
|
||||
# BOSA = "Byte Order, Size, and Alignment" for the struct module
|
||||
# Big Endian
|
||||
STRUCT_BOSA_CHAR = ">"
|
||||
|
||||
|
||||
class CData(abc.ABC):
|
||||
@abc.abstractmethod
|
||||
def __init__(self, size: int):
|
||||
self.size = size
|
||||
|
||||
# Unpack
|
||||
|
||||
@abc.abstractmethod
|
||||
def unpack_from(self, data: memoryview, offset: int = 0) -> Any: ...
|
||||
|
||||
|
||||
class CData_Value(CData):
|
||||
def __init__(self, format_char: str):
|
||||
assert format_char in set("bBhHiIfd")
|
||||
self.unpack_struct = struct.Struct(STRUCT_BOSA_CHAR + format_char)
|
||||
super().__init__(self.unpack_struct.size)
|
||||
|
||||
def unpack_from(self, data: memoryview, offset: int = 0):
|
||||
return self.unpack_struct.unpack_from(data, offset)[0]
|
||||
|
||||
|
||||
CData_Value.s8 = CData_Value("b")
|
||||
CData_Value.u8 = CData_Value("B")
|
||||
CData_Value.s16 = CData_Value("h")
|
||||
CData_Value.u16 = CData_Value("H")
|
||||
CData_Value.s32 = CData_Value("i")
|
||||
CData_Value.u32 = CData_Value("I")
|
||||
CData_Value.f32 = CData_Value("f")
|
||||
CData_Value.f64 = CData_Value("d")
|
||||
CData_Value.pointer = CData_Value("I")
|
||||
|
||||
|
||||
class CData_Array(CData):
|
||||
def __init__(self, element_cdata: CData, length: int):
|
||||
assert length > 0
|
||||
self.element_cdata = element_cdata
|
||||
self.length = length
|
||||
super().__init__(element_cdata.size * length)
|
||||
|
||||
def unpack_from(self, data: memoryview, offset: int = 0):
|
||||
array_unpacked = []
|
||||
|
||||
for i in range(self.length):
|
||||
unpacked = self.element_cdata.unpack_from(data, offset)
|
||||
array_unpacked.append(unpacked)
|
||||
offset += self.element_cdata.size
|
||||
|
||||
assert len(array_unpacked) == self.length
|
||||
|
||||
return array_unpacked
|
||||
|
||||
|
||||
class CData_Struct(CData):
|
||||
def __init__(self, members: Sequence[tuple[str, CData]]):
|
||||
# assert all members have different names
|
||||
assert len(members) == len(
|
||||
set(member_name for member_name, member_cdata in members)
|
||||
), members
|
||||
|
||||
self.members = members
|
||||
super().__init__(
|
||||
sum(member_cdata.size for member_name, member_cdata in members)
|
||||
)
|
||||
|
||||
if __debug__:
|
||||
# Check alignment
|
||||
|
||||
# This may mess up with CData instances other than CData_Value, Array and Struct
|
||||
def get_required_alignment(cdata: CData):
|
||||
if isinstance(cdata, CData_Struct):
|
||||
return max(
|
||||
get_required_alignment(cdata_member_cdata)
|
||||
for cdata_member_name, cdata_member_cdata in cdata.members
|
||||
)
|
||||
elif isinstance(cdata, CData_Array):
|
||||
return get_required_alignment(cdata.element_cdata)
|
||||
else:
|
||||
# Assume the alignment requirement corresponds to the size
|
||||
# (e.g. this is correct for CData_Value)
|
||||
return cdata.size
|
||||
|
||||
# Check alignment of the members of the struct
|
||||
offset = 0
|
||||
for member_name, member_cdata in members:
|
||||
alignment = get_required_alignment(member_cdata)
|
||||
assert offset % alignment == 0, (member_name, offset, alignment)
|
||||
offset += member_cdata.size
|
||||
|
||||
# Check alignment of the struct size
|
||||
alignment = get_required_alignment(self)
|
||||
assert self.size % alignment == 0, (self.size, alignment)
|
||||
|
||||
def unpack_from(self, data: memoryview, offset: int = 0):
|
||||
struct_unpacked = dict()
|
||||
|
||||
for member_name, member_cdata in self.members:
|
||||
member_unpacked = member_cdata.unpack_from(data, offset)
|
||||
struct_unpacked[member_name] = member_unpacked
|
||||
offset += member_cdata.size
|
||||
|
||||
return struct_unpacked
|
||||
|
||||
|
||||
def try_stuff():
|
||||
"""
|
||||
struct {
|
||||
s8 fun;
|
||||
// u8 pad;
|
||||
s16 games;
|
||||
} array[] = { { 1, 2 }, { 3, 4 } };
|
||||
|
||||
|
||||
u8 varLenArray[] = { 1, 2, 3 };
|
||||
|
||||
struct {
|
||||
u8* ptr;
|
||||
u16 len;
|
||||
struct {
|
||||
s32 secret1;
|
||||
u32 secret2;
|
||||
} mySubStruct;
|
||||
} data = { varLenArray, 3, { 421, 0x01020304 } };
|
||||
"""
|
||||
|
||||
array_bytes = bytes(
|
||||
[
|
||||
1,
|
||||
0,
|
||||
*(0, 2),
|
||||
3,
|
||||
0,
|
||||
*(0, 4),
|
||||
]
|
||||
)
|
||||
varLenArray_bytes = bytes([1, 2, 3])
|
||||
data_bytes = bytes(
|
||||
[
|
||||
*(0x12, 0x34, 0x56, 0x78),
|
||||
*(0, 3),
|
||||
0,
|
||||
0,
|
||||
*(0, 0, 421 >> 8, 421 & 0xFF),
|
||||
*(1, 2, 3, 4),
|
||||
]
|
||||
)
|
||||
|
||||
arrayElem_CData_Struct = CData_Struct(
|
||||
(
|
||||
("fun", CData_Value.s8),
|
||||
("pad1", CData_Value.s8),
|
||||
("games", CData_Value.s16),
|
||||
)
|
||||
)
|
||||
array_CData_Array = CData_Array(arrayElem_CData_Struct, 2)
|
||||
|
||||
print(array_CData_Array.unpack_from(array_bytes))
|
||||
|
||||
mySubStruct_CData_Struct = CData_Struct(
|
||||
(
|
||||
("secret1", CData_Value.s32),
|
||||
("secret2", CData_Value.u32),
|
||||
)
|
||||
)
|
||||
|
||||
data_CData_Struct = CData_Struct(
|
||||
(
|
||||
("ptr", CData_Value.pointer),
|
||||
("len", CData_Value.u16),
|
||||
("pad_6", CData_Value.s16),
|
||||
("mySubStruct", mySubStruct_CData_Struct),
|
||||
)
|
||||
)
|
||||
|
||||
data_unpacked = data_CData_Struct.unpack_from(data_bytes)
|
||||
print(data_unpacked)
|
||||
|
||||
varLenArray_CData_Array = CData_Array(CData_Value.u8, data_unpacked["len"])
|
||||
|
||||
print(varLenArray_CData_Array.unpack_from(varLenArray_bytes))
|
||||
|
||||
data_integratedSubStruct_CData_Struct = CData_Struct(
|
||||
(
|
||||
("ptr", CData_Value.pointer),
|
||||
("len", CData_Value.u16),
|
||||
("pad_6", CData_Value.s16),
|
||||
(
|
||||
"mySubStruct",
|
||||
CData_Struct(
|
||||
(
|
||||
("secret1", CData_Value.s32),
|
||||
("secret2", CData_Value.u32),
|
||||
)
|
||||
),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
data_unpacked = data_integratedSubStruct_CData_Struct.unpack_from(data_bytes)
|
||||
print(data_unpacked)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try_stuff()
|
||||
Loading…
Add table
Add a link
Reference in a new issue