mirror of
https://github.com/zeldaret/oot.git
synced 2025-07-04 15:04:31 +00:00
Set up multiversion assets with ZAPD and match gc-eu (#1967)
* Add ZAPD hack to deal with extracted/VERSION/ in include paths
* Extract assets to extracted/VERSION
* Add ZAPD flags to override virtual address / start offset / end offset
* Configure offsets for code and overlay assets
* Reorganize ZAPD configs
* Match gc-eu-mq
* Match gc-eu
* Remove old asset dirs during distclean
* Revert "Remove old asset dirs during distclean"
This reverts commit fc8027a75f.
* make zapd addresses globals int64_t so they can store uint32_t addresses and -1
* slight cleanup extract_assets.py
* git subrepo pull --force tools/ZAPD
subrepo:
subdir: "tools/ZAPD"
merged: "0285e11f0"
upstream:
origin: "https://github.com/zeldaret/ZAPD.git"
branch: "master"
commit: "0285e11f0"
git-subrepo:
version: "0.4.6"
origin: "git@github.com:ingydotnet/git-subrepo.git"
commit: "110b9eb"
---------
Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>
This commit is contained in:
cadmic
GitHub
parent
b2d80568b9
commit
9def6f4d0d
92 changed files with 4911 additions and 487 deletions
|
|
@ -8,8 +8,7 @@ import time
|
|||
import multiprocessing
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
EXTRACTED_ASSETS_NAMEFILE = ".extracted-assets.json"
|
||||
from tools import version_config
|
||||
|
||||
|
||||
def SignalHandler(sig, frame):
|
||||
|
|
@ -17,20 +16,33 @@ def SignalHandler(sig, frame):
|
|||
mainAbort.set()
|
||||
# Don't exit immediately to update the extracted assets file.
|
||||
|
||||
def ExtractFile(xmlPath, outputPath, outputSourcePath):
|
||||
def ExtractFile(assetConfig: version_config.AssetConfig, outputPath: Path, outputSourcePath: Path):
|
||||
xmlPath = assetConfig.xml_path
|
||||
version = globalVersionConfig.version
|
||||
if globalAbort.is_set():
|
||||
# Don't extract if another file wasn't extracted properly.
|
||||
return
|
||||
|
||||
zapdPath = Path("tools") / "ZAPD" / "ZAPD.out"
|
||||
configPath = Path("tools") / "ZAPDConfigs" / "MqDbg" / "Config.xml"
|
||||
configPath = Path("tools") / "ZAPDConfigs" / version / "Config.xml"
|
||||
|
||||
Path(outputPath).mkdir(parents=True, exist_ok=True)
|
||||
Path(outputSourcePath).mkdir(parents=True, exist_ok=True)
|
||||
outputPath.mkdir(parents=True, exist_ok=True)
|
||||
outputSourcePath.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
execStr = f"{zapdPath} e -eh -i {xmlPath} -b extracted/gc-eu-mq-dbg/baserom -o {outputPath} -osf {outputSourcePath} -gsf 1 -rconf {configPath} --cs-float both {ZAPDArgs}"
|
||||
execStr = f"{zapdPath} e -eh -i {xmlPath} -b extracted/{version}/baserom -o {outputPath} -osf {outputSourcePath} -gsf 1 -rconf {configPath} --cs-float both {ZAPDArgs}"
|
||||
|
||||
if "overlays" in xmlPath:
|
||||
if "code" in xmlPath.parts or "overlays" in xmlPath.parts:
|
||||
assert assetConfig.start_offset is not None
|
||||
assert assetConfig.end_offset is not None
|
||||
|
||||
execStr += f" --start-offset 0x{assetConfig.start_offset:X}"
|
||||
execStr += f" --end-offset 0x{assetConfig.end_offset:X}"
|
||||
|
||||
if "overlays" in xmlPath.parts:
|
||||
overlayName = xmlPath.stem
|
||||
baseAddress = globalVersionConfig.dmadata_segments[overlayName].vram + assetConfig.start_offset
|
||||
|
||||
execStr += f" --base-address 0x{baseAddress:X}"
|
||||
execStr += " --static"
|
||||
|
||||
if globalUnaccounted:
|
||||
|
|
@ -45,35 +57,39 @@ def ExtractFile(xmlPath, outputPath, outputSourcePath):
|
|||
print("Aborting...", file=os.sys.stderr)
|
||||
print("\n")
|
||||
|
||||
def ExtractFunc(fullPath):
|
||||
*pathList, xmlName = fullPath.split(os.sep)
|
||||
objectName = os.path.splitext(xmlName)[0]
|
||||
def ExtractFunc(assetConfig: version_config.AssetConfig):
|
||||
objectName = assetConfig.name
|
||||
xml_path = assetConfig.xml_path
|
||||
xml_path_str = str(xml_path)
|
||||
|
||||
outPath = os.path.join("assets", *pathList[2:], objectName)
|
||||
version = globalVersionConfig.version
|
||||
outPath = Path("extracted") / version / "assets" / objectName
|
||||
outSourcePath = outPath
|
||||
|
||||
if fullPath in globalExtractedAssetsTracker:
|
||||
timestamp = globalExtractedAssetsTracker[fullPath]["timestamp"]
|
||||
modificationTime = int(os.path.getmtime(fullPath))
|
||||
if xml_path_str in globalExtractedAssetsTracker:
|
||||
timestamp = globalExtractedAssetsTracker[xml_path_str]["timestamp"]
|
||||
modificationTime = int(os.path.getmtime(xml_path))
|
||||
if modificationTime < timestamp:
|
||||
# XML has not been modified since last extraction.
|
||||
return
|
||||
|
||||
currentTimeStamp = int(time.time())
|
||||
|
||||
ExtractFile(fullPath, outPath, outSourcePath)
|
||||
ExtractFile(assetConfig, outPath, outSourcePath)
|
||||
|
||||
if not globalAbort.is_set():
|
||||
# Only update timestamp on succesful extractions
|
||||
if fullPath not in globalExtractedAssetsTracker:
|
||||
globalExtractedAssetsTracker[fullPath] = globalManager.dict()
|
||||
globalExtractedAssetsTracker[fullPath]["timestamp"] = currentTimeStamp
|
||||
if xml_path_str not in globalExtractedAssetsTracker:
|
||||
globalExtractedAssetsTracker[xml_path_str] = globalManager.dict()
|
||||
globalExtractedAssetsTracker[xml_path_str]["timestamp"] = currentTimeStamp
|
||||
|
||||
def initializeWorker(abort, unaccounted: bool, extractedAssetsTracker: dict, manager):
|
||||
def initializeWorker(versionConfig: version_config.VersionConfig, abort, unaccounted: bool, extractedAssetsTracker: dict, manager):
|
||||
global globalVersionConfig
|
||||
global globalAbort
|
||||
global globalUnaccounted
|
||||
global globalExtractedAssetsTracker
|
||||
global globalManager
|
||||
globalVersionConfig = versionConfig
|
||||
globalAbort = abort
|
||||
globalUnaccounted = unaccounted
|
||||
globalExtractedAssetsTracker = extractedAssetsTracker
|
||||
|
|
@ -95,13 +111,17 @@ def processZAPDArgs(argsZ):
|
|||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="baserom asset extractor")
|
||||
parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep")
|
||||
parser.add_argument("-v", "--oot-version", dest="oot_version", help="OOT game version", default="gc-eu-mq-dbg")
|
||||
parser.add_argument("-s", "--single", help="Extract a single asset by name, e.g. objects/gameplay_keep")
|
||||
parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones (overwriting current files).", action="store_true")
|
||||
parser.add_argument("-j", "--jobs", help="Number of cpu cores to extract with.")
|
||||
parser.add_argument("-u", "--unaccounted", help="Enables ZAPD unaccounted detector warning system.", action="store_true")
|
||||
parser.add_argument("-Z", help="Pass the argument on to ZAPD, e.g. `-ZWunaccounted` to warn about unaccounted blocks in XMLs. Each argument should be passed separately, *without* the leading dash.", metavar="ZAPD_ARG", action="append")
|
||||
args = parser.parse_args()
|
||||
|
||||
version: str = args.oot_version
|
||||
versionConfig = version_config.load_version_config(version)
|
||||
|
||||
global ZAPDArgs
|
||||
ZAPDArgs = processZAPDArgs(args.Z) if args.Z else ""
|
||||
|
||||
|
|
@ -110,31 +130,30 @@ def main():
|
|||
manager = multiprocessing.Manager()
|
||||
signal.signal(signal.SIGINT, SignalHandler)
|
||||
|
||||
extraction_times_p = Path("extracted") / version / "assets_extraction_times.json"
|
||||
extractedAssetsTracker = manager.dict()
|
||||
if os.path.exists(EXTRACTED_ASSETS_NAMEFILE) and not args.force:
|
||||
with open(EXTRACTED_ASSETS_NAMEFILE, encoding='utf-8') as f:
|
||||
if extraction_times_p.exists() and not args.force:
|
||||
with extraction_times_p.open(encoding='utf-8') as f:
|
||||
extractedAssetsTracker.update(json.load(f, object_hook=manager.dict))
|
||||
|
||||
asset_path = args.single
|
||||
if asset_path is not None:
|
||||
fullPath = os.path.join("assets", "xml", asset_path + ".xml")
|
||||
if not os.path.exists(fullPath):
|
||||
print(f"Error. File {fullPath} does not exist.", file=os.sys.stderr)
|
||||
singleAssetName = args.single
|
||||
if singleAssetName is not None:
|
||||
assetConfig = None
|
||||
for asset in versionConfig.assets:
|
||||
if asset.name == singleAssetName:
|
||||
assetConfig = asset
|
||||
break
|
||||
else:
|
||||
print(f"Error. Asset {singleAssetName} not found in config.", file=os.sys.stderr)
|
||||
exit(1)
|
||||
|
||||
initializeWorker(mainAbort, args.unaccounted, extractedAssetsTracker, manager)
|
||||
initializeWorker(versionConfig, mainAbort, args.unaccounted, extractedAssetsTracker, manager)
|
||||
# Always extract if -s is used.
|
||||
fullPath = assetConfig.xml_path
|
||||
if fullPath in extractedAssetsTracker:
|
||||
del extractedAssetsTracker[fullPath]
|
||||
ExtractFunc(fullPath)
|
||||
ExtractFunc(assetConfig)
|
||||
else:
|
||||
xmlFiles = []
|
||||
for currentPath, _, files in os.walk(os.path.join("assets", "xml")):
|
||||
for file in files:
|
||||
fullPath = os.path.join(currentPath, file)
|
||||
if file.endswith(".xml"):
|
||||
xmlFiles.append(fullPath)
|
||||
|
||||
class CannotMultiprocessError(Exception):
|
||||
pass
|
||||
|
||||
|
|
@ -147,17 +166,17 @@ def main():
|
|||
mp_context = multiprocessing.get_context("fork")
|
||||
except ValueError as e:
|
||||
raise CannotMultiprocessError() from e
|
||||
with mp_context.Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, args.unaccounted, extractedAssetsTracker, manager)) as p:
|
||||
p.map(ExtractFunc, xmlFiles)
|
||||
with mp_context.Pool(numCores, initializer=initializeWorker, initargs=(versionConfig, mainAbort, args.unaccounted, extractedAssetsTracker, manager)) as p:
|
||||
p.map(ExtractFunc, versionConfig.assets)
|
||||
except (multiprocessing.ProcessError, TypeError, CannotMultiprocessError):
|
||||
print("Warning: Multiprocessing exception ocurred.", file=os.sys.stderr)
|
||||
print("Disabling mutliprocessing.", file=os.sys.stderr)
|
||||
|
||||
initializeWorker(mainAbort, args.unaccounted, extractedAssetsTracker, manager)
|
||||
for singlePath in xmlFiles:
|
||||
ExtractFunc(singlePath)
|
||||
initializeWorker(versionConfig, mainAbort, args.unaccounted, extractedAssetsTracker, manager)
|
||||
for assetConfig in versionConfig.assets:
|
||||
ExtractFunc(assetConfig)
|
||||
|
||||
with open(EXTRACTED_ASSETS_NAMEFILE, 'w', encoding='utf-8') as f:
|
||||
with extraction_times_p.open('w', encoding='utf-8') as f:
|
||||
serializableDict = dict()
|
||||
for xml, data in extractedAssetsTracker.items():
|
||||
serializableDict[xml] = dict(data)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue