mirror of
https://github.com/zeldaret/oot.git
synced 2025-08-08 15:30:14 +00:00
New assets system (#2481)
* wip: New assets system tm
Builds gc-eu-mq-dbg OK from clean after
1) make setup
2) python3 -m tools.assets.extract -j
3) replace 0x80A8E610 with sShadowTex in extracted/gc-eu-mq-dbg/assets/overlays/ovl_En_Jsjutan/sShadowMaterialDL.inc.c
4) make various symbols in extracted data like sTex static
* use variables from config.yml for gMtxClear and sShadowTex addresses
* Write source with static for overlays using `HACK_IS_STATIC_ON` hack
* gc-eu-mq-dbg OK from clean with `make setup && make`
* implement more skeleton-related types, cleanups, fixups
* fix extracted data to no longer produce compilation warnings
* implement more of RoomShapeImage types
* yeet XmlPath from ExternalFile usage
* Implement PlayerAnimationDataResource (link_animetion data)
* fix csdis CS_TIME extra arg
* dmadata file names no longer hardcoded for gc-eu-mq-dbg
* ntsc-1.0 OK
* xml fixes
* slightly improve standard output
* rm extract_assets.py
* generate and use Limb enums (TODO: check Skin skels and implement for Curve skels)
* handle dependencies between xmls
* introduce RawPointers xml attribute to ignore specific pointers and keep them raw
* add tools/extract_assets.sh
* fixups
* only extract if xmls changed or if -f (force) is used
* fixups, gc-eu OK
* all versions OK
* check attributes of xml resources elements
* Implement legacy skelanime resources
* fix ASSET_FILES_BIN_EXTRACTED/COMMITTED: look for .u8.bin specifically instead of just .bin
* implement JFIFResource
* fix png/jpg wildcards: look specifically for .u64.png .u32.png .u64.jpg
* Makefile: Add rules to build .png, .bin and .jpg in assets/ too
* start writing actual docs
* extract sTransCircleDL and sTransWipeDL
* misc cleanup/fixes, pygfxd 1.0.3
* refactor CDataExt.set_write callback args to use a dataclass
* Move {} to in-source
* misc
* more progress on spec
* fix missing braces in n64dd_error_textures.c
* finish xml spec doc
* assets xmls fixes
* some cleanup, use `gNameTex_WIDTH/HEIGHT` macros in dlists
* handle hackmode_syotes_room, fix compile
* C build_from_png
* rm tools/assets/bin2c
* rm ZAPD
* format
* remove rule to generate dmadata_table.py
* CC0 license (and some import cleanup)
* dont try to build zapd (rmd)
* simplify palettes with single user (ci images with a non-shared palette)
* add docs on how images are handled
* bss
* allow -j N
* fix n64texconv python bindings memory management
* move -j at the end of calling extraction script
* with -j, update last_extracts.json as each job completes rather than only if all complete
* make interrupting less jank by making child processes ignore sigint
* use enum names in `SCENE_CMD_SKYBOX_SETTINGS`
* `multiprocessing.get_context("fork")`
* import rich, except ImportError s
* fix optional rich usage
* .bss
* .bss
* .bss
* assets extraction: -j -> -j$(N_THREADS)
* .bss
* change LIMB_NONE/MAX defaults to be FILE_OFFSET instead of SKELNAME
* 0XHEX -> 0xHEX
* fix bss
* Proper includes for assets
mostly proper, some includes like dlists resources always causing a sys_matrix.h include (when not every dlist references gIdentityMtx) could be done better
* rm z64.h
* rm z64.h take two
* bss
* Make .u64 suffix for pngs optional
* fixup: rm .u64 suffix from n64dd image paths
* Remove elemtype suffixes from .bin and .jpg files
* Update images.md
* some build_from_png cleanup, more error handling, comments
* Handle skybox textures
Introduce "sub-format" suffix for pngs, with sub-formats split_lo and split_hi being used for skybox textures
* fixup for older python
* improve collision output some
* fully use SURFACETYPE[01] macros in writing extracted surface types
* use WATERBOX_PROPERTIES in extracted waterboxes
* some SceneCommandsResource cleanup
* format EnvLightSettingsList output
This commit is contained in:
Dragorn421
GitHub
parent
0c6c112cb9
commit
1e556e3a3d
460 changed files with 14342 additions and 48656 deletions
241
tools/assets/descriptor/n64resources.py
Normal file
241
tools/assets/descriptor/n64resources.py
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
# SPDX-FileCopyrightText: © 2025 ZeldaRET
|
||||
# SPDX-License-Identifier: CC0-1.0
|
||||
|
||||
import dataclasses
|
||||
import enum
|
||||
from xml.etree.ElementTree import Element
|
||||
|
||||
from ..n64 import G_IM_FMT, G_IM_SIZ
|
||||
|
||||
from .base import (
|
||||
ResourceDesc,
|
||||
ResourcesDescCollection,
|
||||
ResourcesDescCollectionsPool,
|
||||
ResourceHandlerNeedsPass2Exception,
|
||||
BaseromFileBackingMemory,
|
||||
)
|
||||
from . import xml_errors
|
||||
|
||||
# TODO remove
|
||||
STATIC_ATTRIB = {"Static"}
|
||||
|
||||
|
||||
class GfxMicroCode(enum.Enum):
|
||||
F3DEX = enum.auto()
|
||||
F3DEX2 = enum.auto()
|
||||
|
||||
|
||||
@dataclasses.dataclass(eq=False)
|
||||
class DListResourceDesc(ResourceDesc):
|
||||
ucode: GfxMicroCode
|
||||
raw_pointers: set[int] = dataclasses.field(default_factory=set)
|
||||
"""Pointers in the dlist that are fine to keep raw ("in hex") instead of using symbols"""
|
||||
|
||||
|
||||
def handler_DList(symbol_name, offset, collection, reselem: Element):
|
||||
xml_errors.check_attrib(
|
||||
reselem, {"Name", "Offset"}, {"Ucode", "RawPointers"} | STATIC_ATTRIB
|
||||
)
|
||||
if "Ucode" in reselem.attrib:
|
||||
ucode = GfxMicroCode[reselem.attrib["Ucode"].upper()]
|
||||
else:
|
||||
ucode = GfxMicroCode.F3DEX2
|
||||
res = DListResourceDesc(symbol_name, offset, collection, reselem, ucode)
|
||||
raw_pointers_str = reselem.attrib.get("RawPointers")
|
||||
if raw_pointers_str:
|
||||
for rp_str in raw_pointers_str.split(","):
|
||||
res.raw_pointers.add(int(rp_str, 16))
|
||||
return res
|
||||
|
||||
|
||||
@dataclasses.dataclass(eq=False)
|
||||
class BlobResourceDesc(ResourceDesc):
|
||||
size: int
|
||||
|
||||
|
||||
def handler_Blob(symbol_name, offset, collection, reselem: Element):
|
||||
xml_errors.check_attrib(reselem, {"Name", "Offset", "Size"}, STATIC_ATTRIB)
|
||||
size = int(reselem.attrib["Size"], 16)
|
||||
return BlobResourceDesc(symbol_name, offset, collection, reselem, size)
|
||||
|
||||
|
||||
@dataclasses.dataclass(eq=False)
|
||||
class MtxResourceDesc(ResourceDesc):
|
||||
pass
|
||||
|
||||
|
||||
def handler_Mtx(symbol_name, offset, collection, reselem: Element):
|
||||
xml_errors.check_attrib(reselem, {"Name", "Offset"}, STATIC_ATTRIB)
|
||||
return MtxResourceDesc(symbol_name, offset, collection, reselem)
|
||||
|
||||
|
||||
@dataclasses.dataclass(eq=False)
|
||||
class S16ArrayResourceDesc(ResourceDesc):
|
||||
count: int
|
||||
|
||||
|
||||
@dataclasses.dataclass(eq=False)
|
||||
class Vec3sArrayResourceDesc(ResourceDesc):
|
||||
count: int
|
||||
|
||||
|
||||
@dataclasses.dataclass(eq=False)
|
||||
class VtxArrayResourceDesc(ResourceDesc):
|
||||
count: int
|
||||
|
||||
|
||||
def handler_Array(symbol_name, offset, collection, reselem: Element):
|
||||
xml_errors.check_attrib(reselem, {"Name", "Offset", "Count"}, STATIC_ATTRIB)
|
||||
count = int(reselem.attrib["Count"])
|
||||
assert len(reselem) == 1, "Expected exactly one child of Array node"
|
||||
array_elem = reselem[0]
|
||||
if array_elem.tag == "Vtx":
|
||||
array_resource_type = VtxArrayResourceDesc
|
||||
elif (
|
||||
array_elem.tag == "Vector"
|
||||
and array_elem.attrib["Type"] == "s16"
|
||||
and int(array_elem.attrib["Dimensions"]) == 3
|
||||
):
|
||||
array_resource_type = Vec3sArrayResourceDesc
|
||||
elif array_elem.tag == "Scalar" and array_elem.attrib["Type"] == "s16":
|
||||
array_resource_type = S16ArrayResourceDesc
|
||||
else:
|
||||
raise NotImplementedError(f"Array of {array_elem.tag}")
|
||||
return array_resource_type(symbol_name, offset, collection, reselem, count)
|
||||
|
||||
|
||||
class TextureFormat(enum.Enum):
|
||||
RGBA16 = (G_IM_FMT.RGBA, G_IM_SIZ._16b)
|
||||
RGBA32 = (G_IM_FMT.RGBA, G_IM_SIZ._32b)
|
||||
CI4 = (G_IM_FMT.CI, G_IM_SIZ._4b)
|
||||
CI8 = (G_IM_FMT.CI, G_IM_SIZ._8b)
|
||||
I4 = (G_IM_FMT.I, G_IM_SIZ._4b)
|
||||
I8 = (G_IM_FMT.I, G_IM_SIZ._8b)
|
||||
IA4 = (G_IM_FMT.IA, G_IM_SIZ._4b)
|
||||
IA8 = (G_IM_FMT.IA, G_IM_SIZ._8b)
|
||||
IA16 = (G_IM_FMT.IA, G_IM_SIZ._16b)
|
||||
|
||||
def __init__(self, fmt: G_IM_FMT, siz: G_IM_SIZ):
|
||||
self.fmt = fmt
|
||||
self.siz = siz
|
||||
|
||||
|
||||
@dataclasses.dataclass(eq=False)
|
||||
class TextureResourceDesc(ResourceDesc):
|
||||
format: TextureFormat
|
||||
width: int
|
||||
height: int
|
||||
|
||||
|
||||
@dataclasses.dataclass(eq=False)
|
||||
class CITextureResourceDesc(TextureResourceDesc):
|
||||
tlut: TextureResourceDesc
|
||||
|
||||
|
||||
def handler_Texture(
|
||||
symbol_name, offset, collection: ResourcesDescCollection, reselem: Element
|
||||
):
|
||||
xml_errors.check_attrib(
|
||||
reselem,
|
||||
{"Name", "Offset", "Format", "Width", "Height"},
|
||||
# TODO remove OutName, SplitTlut
|
||||
{
|
||||
"OutName",
|
||||
"SplitTlut",
|
||||
"TlutOffset",
|
||||
"ExternalTlut",
|
||||
"ExternalTlutOffset",
|
||||
"HackMode",
|
||||
}
|
||||
| STATIC_ATTRIB,
|
||||
)
|
||||
format = TextureFormat[reselem.attrib["Format"].upper()]
|
||||
width = int(reselem.attrib["Width"])
|
||||
height = int(reselem.attrib["Height"])
|
||||
if format.fmt == G_IM_FMT.CI:
|
||||
res = CITextureResourceDesc(
|
||||
symbol_name, offset, collection, reselem, format, width, height, None
|
||||
)
|
||||
|
||||
if reselem.attrib.get("SplitTlut") == "true":
|
||||
res.hack_modes.add("hackmode_split_tlut_true")
|
||||
if reselem.attrib.get("SplitTlut") == "false":
|
||||
res.hack_modes.add("hackmode_split_tlut_false")
|
||||
|
||||
assert (
|
||||
"TlutOffset" in reselem.attrib or "ExternalTlutOffset" in reselem.attrib
|
||||
), f"CI texture {symbol_name} is missing a tlut offset"
|
||||
|
||||
if "TlutOffset" in reselem.attrib:
|
||||
xml_errors.check_attrib(
|
||||
reselem,
|
||||
{"Name", "Offset", "Format", "Width", "Height", "TlutOffset"},
|
||||
# TODO remove OutName, SplitTlut
|
||||
{"OutName", "SplitTlut", "HackMode"} | STATIC_ATTRIB,
|
||||
)
|
||||
tlut_offset = int(reselem.attrib["TlutOffset"], 16)
|
||||
|
||||
def pass2_callback(pool: ResourcesDescCollectionsPool):
|
||||
matching_tlut_resources = [
|
||||
res for res in collection.resources if res.offset == tlut_offset
|
||||
]
|
||||
assert len(matching_tlut_resources) == 1, (
|
||||
f"Found {len(matching_tlut_resources)} resources at TlutOffset "
|
||||
f"0x{tlut_offset:X} instead of exactly one"
|
||||
)
|
||||
assert isinstance(
|
||||
matching_tlut_resources[0], TextureResourceDesc
|
||||
), matching_tlut_resources[0]
|
||||
res.tlut = matching_tlut_resources[0]
|
||||
|
||||
else:
|
||||
xml_errors.check_attrib(
|
||||
reselem,
|
||||
{
|
||||
"Name",
|
||||
"Offset",
|
||||
"Format",
|
||||
"Width",
|
||||
"Height",
|
||||
"ExternalTlut",
|
||||
"ExternalTlutOffset",
|
||||
},
|
||||
# TODO remove OutName, SplitTlut
|
||||
{"OutName", "SplitTlut", "HackMode"} | STATIC_ATTRIB,
|
||||
)
|
||||
external_tlut_file = reselem.attrib["ExternalTlut"]
|
||||
external_tlut_offset = int(reselem.attrib["ExternalTlutOffset"], 16)
|
||||
|
||||
def pass2_callback(pool: ResourcesDescCollectionsPool):
|
||||
matching_collections = [
|
||||
coll
|
||||
for coll in pool.collections
|
||||
if isinstance(coll.backing_memory, BaseromFileBackingMemory)
|
||||
and coll.backing_memory.name == external_tlut_file
|
||||
]
|
||||
assert len(matching_collections) == 1
|
||||
matching_tlut_resources = [
|
||||
res
|
||||
for res in matching_collections[0].resources
|
||||
if res.offset == external_tlut_offset
|
||||
]
|
||||
assert len(matching_tlut_resources) == 1, matching_tlut_resources
|
||||
assert isinstance(
|
||||
matching_tlut_resources[0], TextureResourceDesc
|
||||
), matching_tlut_resources[0]
|
||||
res.tlut = matching_tlut_resources[0]
|
||||
|
||||
raise ResourceHandlerNeedsPass2Exception(res, pass2_callback)
|
||||
else:
|
||||
xml_errors.check_attrib(
|
||||
reselem,
|
||||
{"Name", "Offset", "Format", "Width", "Height"},
|
||||
# TODO remove OutName
|
||||
{"OutName", "HackMode"} | STATIC_ATTRIB,
|
||||
)
|
||||
res = TextureResourceDesc(
|
||||
symbol_name, offset, collection, reselem, format, width, height
|
||||
)
|
||||
if reselem.attrib.get("HackMode") == "ignore_orphaned_tlut":
|
||||
res.hack_modes.add("hackmode_ignore_orphaned_tlut")
|
||||
return res
|
||||
Loading…
Add table
Add a link
Reference in a new issue