diff --git a/Makefile b/Makefile index 6ed8f3e2c1..fd391f2450 100644 --- a/Makefile +++ b/Makefile @@ -203,11 +203,12 @@ else SRC_DIRS := $(shell find src -type d) endif -ASSET_BIN_DIRS := $(shell find assets/* -type d -not -path "assets/xml*" -not -path "assets/text") -ASSET_FILES_XML := $(foreach dir,$(ASSET_BIN_DIRS),$(wildcard $(dir)/*.xml)) +# create extracted directories +$(shell mkdir -p $(EXTRACTED_DIR) $(EXTRACTED_DIR)/assets $(EXTRACTED_DIR)/text) + +ASSET_BIN_DIRS := $(shell find $(EXTRACTED_DIR)/assets -type d) ASSET_FILES_BIN := $(foreach dir,$(ASSET_BIN_DIRS),$(wildcard $(dir)/*.bin)) -ASSET_FILES_OUT := $(foreach f,$(ASSET_FILES_XML:.xml=.c),$f) \ - $(foreach f,$(ASSET_FILES_BIN:.bin=.bin.inc.c),$(BUILD_DIR)/$f) \ +ASSET_FILES_OUT := $(foreach f,$(ASSET_FILES_BIN:.bin=.bin.inc.c),$(f:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%)) \ $(foreach f,$(wildcard assets/text/*.c),$(BUILD_DIR)/$(f:.c=.o)) UNDECOMPILED_DATA_DIRS := $(shell find data -type d) @@ -215,10 +216,12 @@ UNDECOMPILED_DATA_DIRS := $(shell find data -type d) BASEROM_BIN_FILES := $(wildcard $(EXTRACTED_DIR)/baserom/*) # source files -C_FILES := $(filter-out %.inc.c,$(foreach dir,$(SRC_DIRS) $(ASSET_BIN_DIRS),$(wildcard $(dir)/*.c))) +SRC_C_FILES := $(filter-out %.inc.c,$(foreach dir,$(SRC_DIRS),$(wildcard $(dir)/*.c))) +ASSET_C_FILES := $(filter-out %.inc.c,$(foreach dir,$(ASSET_BIN_DIRS),$(wildcard $(dir)/*.c))) S_FILES := $(foreach dir,$(SRC_DIRS) $(UNDECOMPILED_DATA_DIRS),$(wildcard $(dir)/*.s)) O_FILES := $(foreach f,$(S_FILES:.s=.o),$(BUILD_DIR)/$f) \ - $(foreach f,$(C_FILES:.c=.o),$(BUILD_DIR)/$f) \ + $(foreach f,$(SRC_C_FILES:.c=.o),$(BUILD_DIR)/$f) \ + $(foreach f,$(ASSET_C_FILES:.c=.o),$(f:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%)) \ $(foreach f,$(BASEROM_BIN_FILES),$(BUILD_DIR)/baserom/$(notdir $f).o) OVL_RELOC_FILES := $(shell $(CPP) $(CPPFLAGS) $(SPEC) | $(SPEC_REPLACE_VARS) | grep -o '[^"]*_reloc.o' ) @@ -230,11 +233,11 @@ DEP_FILES := $(O_FILES:.o=.asmproc.d) $(OVL_RELOC_FILES:.o=.d) TEXTURE_FILES_PNG := $(foreach dir,$(ASSET_BIN_DIRS),$(wildcard $(dir)/*.png)) TEXTURE_FILES_JPG := $(foreach dir,$(ASSET_BIN_DIRS),$(wildcard $(dir)/*.jpg)) -TEXTURE_FILES_OUT := $(foreach f,$(TEXTURE_FILES_PNG:.png=.inc.c),$(BUILD_DIR)/$f) \ - $(foreach f,$(TEXTURE_FILES_JPG:.jpg=.jpg.inc.c),$(BUILD_DIR)/$f) \ +TEXTURE_FILES_OUT := $(foreach f,$(TEXTURE_FILES_PNG:.png=.inc.c),$(f:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%)) \ + $(foreach f,$(TEXTURE_FILES_JPG:.jpg=.jpg.inc.c),$(f:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%)) # create build directories -$(shell mkdir -p $(BUILD_DIR)/baserom $(EXTRACTED_DIR)/text $(BUILD_DIR)/assets/text $(foreach dir,$(SRC_DIRS) $(UNDECOMPILED_DATA_DIRS) $(ASSET_BIN_DIRS),$(BUILD_DIR)/$(dir))) +$(shell mkdir -p $(BUILD_DIR)/baserom $(BUILD_DIR)/assets/text $(foreach dir,$(SRC_DIRS) $(UNDECOMPILED_DATA_DIRS),$(BUILD_DIR)/$(dir)) $(foreach dir,$(ASSET_BIN_DIRS),$(dir:$(EXTRACTED_DIR)/%=$(BUILD_DIR)/%))) ifeq ($(COMPILER),ido) $(BUILD_DIR)/src/boot/stackcheck.o: OPTFLAGS := -O2 @@ -348,12 +351,9 @@ clean: $(RM) -r $(BUILD_DIR) assetclean: - $(RM) -r $(ASSET_BIN_DIRS) $(RM) -r $(EXTRACTED_DIR) - $(RM) -r $(BUILD_DIR)/assets - $(RM) -r .extracted-assets.json -distclean: assetclean +distclean: $(RM) -r extracted/ $(RM) -r build/ $(MAKE) -C tools distclean @@ -370,10 +370,7 @@ setup: venv $(PYTHON) tools/decompress_baserom.py $(VERSION) $(PYTHON) tools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 --oot-version $(VERSION) -o $(EXTRACTED_DIR)/baserom $(PYTHON) tools/msgdis.py --oot-version $(VERSION) --text-out $(EXTRACTED_DIR)/text/message_data.h --staff-text-out $(EXTRACTED_DIR)/text/message_data_staff.h -# TODO: for now, we only extract assets from the Debug ROM -ifeq ($(VERSION),gc-eu-mq-dbg) - $(PYTHON) extract_assets.py -j$(N_THREADS) -endif + $(PYTHON) extract_assets.py -v $(VERSION) -j$(N_THREADS) disasm: $(RM) -r $(EXPECTED_DIR) @@ -439,7 +436,11 @@ $(BUILD_DIR)/assets/text/nes_message_data_static.o: $(BUILD_DIR)/assets/text/mes $(BUILD_DIR)/assets/text/staff_message_data_static.o: $(BUILD_DIR)/assets/text/message_data_staff.enc.h $(BUILD_DIR)/src/code/z_message_PAL.o: $(BUILD_DIR)/assets/text/message_data.enc.h $(BUILD_DIR)/assets/text/message_data_staff.enc.h -$(BUILD_DIR)/assets/%.o: assets/%.c +$(BUILD_DIR)/assets/text/%.o: assets/text/%.c + $(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $< + $(OBJCOPY) -O binary $@ $@.bin + +$(BUILD_DIR)/assets/%.o: $(EXTRACTED_DIR)/assets/%.c $(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $< $(OBJCOPY) -O binary $@ $@.bin @@ -490,13 +491,13 @@ $(BUILD_DIR)/src/overlays/%_reloc.o: $(BUILD_DIR)/$(SPEC) $(FADO) $$(tools/reloc_prereq $< $(notdir $*)) -n $(notdir $*) -o $(@:.o=.s) -M $(@:.o=.d) $(AS) $(ASFLAGS) $(@:.o=.s) -o $@ -$(BUILD_DIR)/%.inc.c: %.png +$(BUILD_DIR)/assets/%.inc.c: $(EXTRACTED_DIR)/assets/%.png $(ZAPD) btex -eh -tt $(subst .,,$(suffix $*)) -i $< -o $@ -$(BUILD_DIR)/assets/%.bin.inc.c: assets/%.bin +$(BUILD_DIR)/assets/%.bin.inc.c: $(EXTRACTED_DIR)/assets/%.bin $(ZAPD) bblb -eh -i $< -o $@ -$(BUILD_DIR)/assets/%.jpg.inc.c: assets/%.jpg +$(BUILD_DIR)/assets/%.jpg.inc.c: $(EXTRACTED_DIR)/assets/%.jpg $(ZAPD) bren -eh -i $< -o $@ -include $(DEP_FILES) diff --git a/assets/.gitignore b/assets/.gitignore deleted file mode 100644 index 0bece9e89f..0000000000 --- a/assets/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -*.bin -*.c -*.h -!text/*.[ch] -*.cfg -*.vtx.inc -*.dlist.inc diff --git a/extract_assets.py b/extract_assets.py index 9c83503ab8..dc5ba9df3b 100755 --- a/extract_assets.py +++ b/extract_assets.py @@ -9,7 +9,7 @@ import multiprocessing from pathlib import Path -EXTRACTED_ASSETS_NAMEFILE = ".extracted-assets.json" +EXTRACTED_ASSETS_FILENAME = ".extracted-assets.json" def SignalHandler(sig, frame): @@ -28,6 +28,7 @@ def ExtractFile(xmlPath, outputPath, outputSourcePath): Path(outputPath).mkdir(parents=True, exist_ok=True) Path(outputSourcePath).mkdir(parents=True, exist_ok=True) + # TODO: Extract assets from the correct version instead of the Debug ROM execStr = f"{zapdPath} e -eh -i {xmlPath} -b extracted/gc-eu-mq-dbg/baserom -o {outputPath} -osf {outputSourcePath} -gsf 1 -rconf {configPath} --cs-float both {ZAPDArgs}" if "overlays" in xmlPath: @@ -49,7 +50,7 @@ def ExtractFunc(fullPath): *pathList, xmlName = fullPath.split(os.sep) objectName = os.path.splitext(xmlName)[0] - outPath = os.path.join("assets", *pathList[2:], objectName) + outPath = os.path.join("extracted", globalVersion, "assets", *pathList[2:], objectName) outSourcePath = outPath if fullPath in globalExtractedAssetsTracker: @@ -69,11 +70,13 @@ def ExtractFunc(fullPath): globalExtractedAssetsTracker[fullPath] = globalManager.dict() globalExtractedAssetsTracker[fullPath]["timestamp"] = currentTimeStamp -def initializeWorker(abort, unaccounted: bool, extractedAssetsTracker: dict, manager): +def initializeWorker(version, abort, unaccounted: bool, extractedAssetsTracker: dict, manager): + global globalVersion global globalAbort global globalUnaccounted global globalExtractedAssetsTracker global globalManager + globalVersion = version globalAbort = abort globalUnaccounted = unaccounted globalExtractedAssetsTracker = extractedAssetsTracker @@ -95,7 +98,8 @@ def processZAPDArgs(argsZ): def main(): parser = argparse.ArgumentParser(description="baserom asset extractor") - parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep") + parser.add_argument("-v", "--oot-version", help="OOT game version", default="gc-eu-mq-dbg") + parser.add_argument("-s", "--single", help="Asset path relative to assets/, e.g. objects/gameplay_keep") parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones (overwriting current files).", action="store_true") parser.add_argument("-j", "--jobs", help="Number of cpu cores to extract with.") parser.add_argument("-u", "--unaccounted", help="Enables ZAPD unaccounted detector warning system.", action="store_true") @@ -110,9 +114,11 @@ def main(): manager = multiprocessing.Manager() signal.signal(signal.SIGINT, SignalHandler) + version = args.oot_version + extracted_assets_filename = os.path.join("extracted", version, EXTRACTED_ASSETS_FILENAME) extractedAssetsTracker = manager.dict() - if os.path.exists(EXTRACTED_ASSETS_NAMEFILE) and not args.force: - with open(EXTRACTED_ASSETS_NAMEFILE, encoding='utf-8') as f: + if os.path.exists(extracted_assets_filename) and not args.force: + with open(extracted_assets_filename, encoding='utf-8') as f: extractedAssetsTracker.update(json.load(f, object_hook=manager.dict)) asset_path = args.single @@ -122,7 +128,7 @@ def main(): print(f"Error. File {fullPath} does not exist.", file=os.sys.stderr) exit(1) - initializeWorker(mainAbort, args.unaccounted, extractedAssetsTracker, manager) + initializeWorker(version, mainAbort, args.unaccounted, extractedAssetsTracker, manager) # Always extract if -s is used. if fullPath in extractedAssetsTracker: del extractedAssetsTracker[fullPath] @@ -147,17 +153,17 @@ def main(): mp_context = multiprocessing.get_context("fork") except ValueError as e: raise CannotMultiprocessError() from e - with mp_context.Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, args.unaccounted, extractedAssetsTracker, manager)) as p: + with mp_context.Pool(numCores, initializer=initializeWorker, initargs=(version, mainAbort, args.unaccounted, extractedAssetsTracker, manager)) as p: p.map(ExtractFunc, xmlFiles) except (multiprocessing.ProcessError, TypeError, CannotMultiprocessError): print("Warning: Multiprocessing exception ocurred.", file=os.sys.stderr) print("Disabling mutliprocessing.", file=os.sys.stderr) - initializeWorker(mainAbort, args.unaccounted, extractedAssetsTracker, manager) + initializeWorker(version, mainAbort, args.unaccounted, extractedAssetsTracker, manager) for singlePath in xmlFiles: ExtractFunc(singlePath) - with open(EXTRACTED_ASSETS_NAMEFILE, 'w', encoding='utf-8') as f: + with open(extracted_assets_filename, 'w', encoding='utf-8') as f: serializableDict = dict() for xml, data in extractedAssetsTracker.items(): serializableDict[xml] = dict(data)