From 9b67778a000940e00609146f7510738527d54818 Mon Sep 17 00:00:00 2001 From: EllipticEllipsis <73679967+EllipticEllipsis@users.noreply.github.com> Date: Mon, 17 Jan 2022 00:43:07 +0000 Subject: [PATCH] Building on Macs (#1086) * git subrepo pull (merge) tools/ZAPD subrepo: subdir: "tools/ZAPD" merged: "945e6ca1a" upstream: origin: "https://github.com/zeldaret/ZAPD.git" branch: "master" commit: "50242eca9" git-subrepo: version: "0.4.3" origin: "https://github.com/ingydotnet/git-subrepo.git" commit: "2f68596" * Fix extract_assets.py multithreading * Update binutils doc a bit * Remove * import, add multiprocessing option and way to pass arguments to ZAPD * Update format.sh to be more platform-independent * git subrepo pull --force tools/ZAPD subrepo: subdir: "tools/ZAPD" merged: "fd5a7f434" upstream: origin: "https://github.com/zeldaret/ZAPD.git" branch: "master" commit: "fd5a7f434" git-subrepo: version: "0.4.3" origin: "https://github.com/ingydotnet/git-subrepo.git" commit: "2f68596" * Remove ; * Update formatting script to not just use 11 * Add Python requirements, move the Mac stuff in the README into its own doc * Fix readme link * Minor format thing * . * Move ZAPDArgs into its own function * Update readme and remove requirements.txt * Dragorn-inspired rewrite of processZAPDArgs * git subrepo pull --force tools/ZAPD subrepo: subdir: "tools/ZAPD" merged: "a0d3f7b68" upstream: origin: "https://github.com/zeldaret/ZAPD.git" branch: "master" commit: "a0d3f7b68" git-subrepo: version: "0.4.3" origin: "https://github.com/ingydotnet/git-subrepo.git" commit: "2f68596" * Fix function definition * Building docs overhaul * Add Python packages to Mac and Cygwin * Heading number * format format.sh (!) * Replace "currently" * Remove Debian * git subrepo pull (merge) --force tools/ZAPD subrepo: subdir: "tools/ZAPD" merged: "0ba781304" upstream: origin: "https://github.com/zeldaret/ZAPD.git" branch: "master" commit: "0ba781304" git-subrepo: version: "0.4.3" origin: "https://github.com/ingydotnet/git-subrepo.git" commit: "2f68596" --- Makefile | 4 +- README.md | 100 +++++------------- docs/BUILDING_BINUTILS_MACOS.md | 47 -------- ..._BINUTILS_CYGWIN.md => BUILDING_CYGWIN.md} | 46 +++++++- docs/BUILDING_DOCKER.md | 27 +++++ docs/BUILDING_MACOS.md | 89 ++++++++++++++++ extract_assets.py | 48 ++++++--- format.sh | 38 ++++++- tools/ZAPD/.gitrepo | 4 +- tools/ZAPD/ExporterTest/Makefile | 2 +- tools/ZAPD/Jenkinsfile | 58 +++++----- tools/ZAPD/Makefile | 24 ++++- tools/ZAPD/README.md | 9 ++ tools/ZAPD/ZAPD/Declaration.cpp | 2 +- tools/ZAPD/ZAPD/Declaration.h | 3 +- tools/ZAPD/ZAPD/Globals.cpp | 4 +- tools/ZAPD/ZAPD/Globals.h | 2 +- tools/ZAPD/ZAPD/Main.cpp | 10 +- .../ZAPD/OtherStructs/SkinLimbStructs.cpp | 6 +- tools/ZAPD/ZAPD/Overlays/ZOverlay.cpp | 3 +- tools/ZAPD/ZAPD/ZAnimation.cpp | 6 +- tools/ZAPD/ZAPD/ZArray.cpp | 13 ++- tools/ZAPD/ZAPD/ZCollision.cpp | 89 ++++++++++------ tools/ZAPD/ZAPD/ZCollision.h | 3 +- tools/ZAPD/ZAPD/ZDisplayList.cpp | 59 +++++++---- tools/ZAPD/ZAPD/ZFile.cpp | 80 ++++++-------- tools/ZAPD/ZAPD/ZFile.h | 2 +- tools/ZAPD/ZAPD/ZLimb.cpp | 4 +- tools/ZAPD/ZAPD/ZPointer.cpp | 57 ++++++++++ tools/ZAPD/ZAPD/ZPointer.h | 22 ++++ tools/ZAPD/ZAPD/ZResource.h | 1 + tools/ZAPD/ZAPD/ZRoom/Commands/SetMesh.cpp | 1 - .../ZAPD/ZRoom/Commands/SetSpecialObjects.cpp | 8 +- tools/ZAPD/ZAPD/ZRoom/ZRoom.cpp | 7 +- tools/ZAPD/ZAPD/ZSkeleton.cpp | 2 +- tools/ZAPD/ZAPD/ZTexture.cpp | 53 ++++++---- tools/ZAPD/ZAPD/ZTexture.h | 1 + tools/ZAPD/ZAPD/ZVtx.cpp | 2 +- tools/ZAPD/ZAPDUtils/Makefile | 2 +- tools/ZAPD/ZAPDUtils/Utils/StringHelper.h | 2 +- .../docs/zapd_extraction_xml_reference.md | 25 ++++- 41 files changed, 640 insertions(+), 325 deletions(-) delete mode 100644 docs/BUILDING_BINUTILS_MACOS.md rename docs/{BUILDING_BINUTILS_CYGWIN.md => BUILDING_CYGWIN.md} (52%) create mode 100644 docs/BUILDING_DOCKER.md create mode 100644 docs/BUILDING_MACOS.md create mode 100644 tools/ZAPD/ZAPD/ZPointer.cpp create mode 100644 tools/ZAPD/ZAPD/ZPointer.h diff --git a/Makefile b/Makefile index b07fbf2e64..276eda320e 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,8 @@ else endif endif +N_THREADS ?= $(shell nproc) + #### Tools #### ifeq ($(shell type mips-linux-gnu-ld >/dev/null 2>/dev/null; echo $$?), 0) MIPS_BINUTILS_PREFIX := mips-linux-gnu- @@ -203,7 +205,7 @@ setup: $(MAKE) -C tools python3 fixbaserom.py python3 extract_baserom.py - python3 extract_assets.py + python3 extract_assets.py -j$(N_THREADS) resources: $(ASSET_FILES_OUT) test: $(ROM) diff --git a/README.md b/README.md index d6ac1dc877..3b538cf0c8 100644 --- a/README.md +++ b/README.md @@ -39,37 +39,24 @@ It builds the following ROM: ## Installation +We recommend using WSL on Windows, or native Linux, which the rest of this readme describes. We currently have instructions for + +* [Windows](#Windows), with and without WSL +* [macOS](docs/BUILDING_MACOS.md) +* [Linux](#Linux-Native-or-under-WSL--VM), natively or using WSL / VM +* [Docker](docs/BUILDING_DOCKER.md) + +(These will also depend on the Linux instructions.) +Some of these may also be out of date or unmaintained; usually our contributors use WSL, Linux, and macOS, so these instructions should be up to date. + ### Windows For Windows 10 or 11, install WSL and a distribution by following this [WSL Installation Guide](https://docs.microsoft.com/en-us/windows/wsl/install). -We recommend using Debian or Ubuntu 20.04 Linux distributions. +We recommend using Ubuntu 20.04 as the Linux distribution. -For older versions of Windows, install a Linux VM or refer to either [Cygwin](#Cygwin) or [Docker](#Docker) instructions. +For older versions of Windows, install a Linux VM or refer to either [Cygwin](docs/BUILDING_CYGWIN.md) or [Docker](docs/BUILDING_DOCKER.md) instructions. -### macOS - -For macOS, use homebrew to install the following dependencies: - -* coreutils -* make -* python3 -* md5sha1sum -* libpng - -You can install them with the following commands: - -```bash -brew update -brew install coreutils make python3 md5sha1sum libpng -``` - -You'll also need to [build and install mips-linux-binutils](docs/BUILDING_BINUTILS_MACOS.md). - -Going forward in this guide, please use `gmake` whenever you encounter a `make` command. -The `make` that comes with macOS behaves differently than GNU make and is incompatible with this project. - -You should now be able to continue from [step 2](#2-clone-the-repository) of the Linux instructions. ### Linux (Native or under WSL / VM) @@ -90,6 +77,12 @@ sudo apt-get update sudo apt-get install git build-essential binutils-mips-linux-gnu python3 libpng-dev ``` +To install the Python dependencies simply run in a terminal: + +```bash +python3 -m pip install colorama +``` + #### 2. Clone the repository Clone `https://github.com/zeldaret/oot.git` where you wish to have the project, with a command such as: @@ -98,6 +91,12 @@ Clone `https://github.com/zeldaret/oot.git` where you wish to have the project, git clone https://github.com/zeldaret/oot.git ``` +This will copy the GitHub repository contents into a new folder in the current directory called `oot`. Change into this directory before doing anything else: + +```bash +cd oot +``` + #### 3. Prepare a base ROM Copy over your copy of the Master Quest (Debug) ROM inside the root of this new project directory. @@ -145,57 +144,6 @@ This means that the built ROM isn't the same as the base one, so something went Both of these have the disadvantage that the ordering of the terminal output is scrambled, so for debugging it is best to stick to one thread (i.e. not pass `-j` or `-jN`). -### Cygwin - -If you want to use Cygwin, you will need to: - -* Download and install [Git Bash](https://git-scm.com/download/win). -* Download and install [Cygwin](https://cygwin.com). -* [Build and install mips-linux-binutils](docs/BUILDING_BINUTILS_CYGWIN.md). - -Once mips-linux-binutils is installed you will need to install the following packages using Cygwin's installer: - -* libiconv -* dos2unix -* python3 -* libpng-devel - -Then you can continue from step [step 2](#2-clone-the-repository) of the Linux instructions. - -Note that, before building anything, you will need to run the following commands to fix line endings: - -```bash -dos2unix fixle.sh -./fixle.sh -``` - -### Docker - -#### 1. Setup requirements - -To use Docker, you'll need either Docker Desktop or Docker Toolbox installed and setup based on your system. - -You'll also need to prepare a local version of the project with a copied base ROM (see steps [2](#2-clone-the-repository) and [3](#3-prepare-a-base-rom) of the Linux instructions). - -#### 2. Create the Docker image - -From inside your local project, run the following command: - -```bash -docker build . -t oot -``` - -#### 3. Start the container - -To start the container, you can mount your local filesystem into the Docker container and run an interactive bash session. - -```bash -docker run -it --rm --mount type=bind,source="$(pwd)",destination=/oot oot /bin/bash -``` - -#### 4. Setup and Build the ROM - -Once inside the container, you can follow steps [4](#4-setup-the-rom-and-build-process) and [5](#5-build-the-rom) of the Linux instructions to setup and build the ROM, or run any other command you need. ## Contributing diff --git a/docs/BUILDING_BINUTILS_MACOS.md b/docs/BUILDING_BINUTILS_MACOS.md deleted file mode 100644 index c03a1b909b..0000000000 --- a/docs/BUILDING_BINUTILS_MACOS.md +++ /dev/null @@ -1,47 +0,0 @@ -# Building mips-linux-binutils on MacOS - -The following instructions are written for MacOS users but should apply to any unix-like system, with maybe some modifications at the end regarding the bash_profile. - -Create destination dir for binutils -```bash -sudo mkdir -p /opt/cross -``` - -Create and enter local working dir -```bash -mkdir ~/binutils-tmp -cd ~/binutils-tmp -``` - -Get and extract binutils source -```bash -wget https://ftp.gnu.org/gnu/binutils/binutils-2.35.tar.bz2 -tar xjf binutils-2.35.tar.bz2 -``` - -Create and enter build dir -```bash -mkdir build-binutils -cd build-binutils -``` - -Configure the build -```bash -../binutils-2.35/configure --target=mips-linux-gnu --prefix=/opt/cross --disable-gprof --disable-gdb --disable-werror -``` - -Make and install binutils -```bash -make -j -sudo make install -``` - -Edit your ~/.bash_profile to add the new binutils binaries to the system PATH -```bash -echo "export PATH=$PATH:/opt/cross/bin" >> ~/.bash_profile -``` - -Reload ~/.bash_profile (or just launch a new terminal tab) -```bash -source ~/.bash_profile -``` diff --git a/docs/BUILDING_BINUTILS_CYGWIN.md b/docs/BUILDING_CYGWIN.md similarity index 52% rename from docs/BUILDING_BINUTILS_CYGWIN.md rename to docs/BUILDING_CYGWIN.md index 31fbdfe6c5..a673bdd73b 100644 --- a/docs/BUILDING_BINUTILS_CYGWIN.md +++ b/docs/BUILDING_CYGWIN.md @@ -1,4 +1,17 @@ -# Building mips-linux-binutils on Windows using Cygwin +# Building with Cygwin + +**N.B.** These have not been tested recently, you may find the requirements have changed a bit. + +If you want to use Cygwin, you will need to: + + +## 1. Install Git Bash and Cygwin + +* Download and install [Git Bash](https://git-scm.com/download/win). +* Download and install [Cygwin](https://cygwin.com). + + +## 2. Build mips-linux-binutils on Windows using Cygwin First, you will need to install the following packages using the Cygwin installer: - make @@ -40,3 +53,34 @@ Add the new binutils binaries to your system PATH: You can do that by adding `PATH=$PATH:/opt/cross/bin` to `~/.bashrc` and then reloading `~/.bashrc`. Alternatively you can edit the `Path` variable in `Edit the system environment variables`>`Environment Variables` (in which case you will need to relaunch your terminal). + + +## 3. Install required Cygwin packages + +Once mips-linux-binutils is installed you will need to install the following packages using Cygwin's installer: + +* libiconv +* dos2unix +* python3 +* libpng-devel + + +## 4. Install required Python packages + +To install the Python dependencies simply run in a terminal: + +```bash +python3 -m pip install colorama +``` + + +## 5. Continue with Linux instructions + +You should be able to continue from step [step 2](../README.md#2-clone-the-repository) of the Linux instructions. + +**N.B.** Before building anything, you will need to run the following commands to fix line endings: + +```bash +dos2unix fixle.sh +./fixle.sh +``` diff --git a/docs/BUILDING_DOCKER.md b/docs/BUILDING_DOCKER.md new file mode 100644 index 0000000000..afc35b48bc --- /dev/null +++ b/docs/BUILDING_DOCKER.md @@ -0,0 +1,27 @@ +# Building using Docker + +## 1. Setup requirements + +To use Docker, you'll need either Docker Desktop or Docker Toolbox installed and setup based on your system. + +You'll also need to prepare a local version of the project with a copied base ROM (see steps [2](../README.md#2-clone-the-repository) and [3](../README.md#3-prepare-a-base-rom) of the Linux instructions). + +## 2. Create the Docker image + +From inside your local project, run the following command: + +```bash +docker build . -t oot +``` + +## 3. Start the container + +To start the container, you can mount your local filesystem into the Docker container and run an interactive bash session. + +```bash +docker run -it --rm --mount type=bind,source="$(pwd)",destination=/oot oot /bin/bash +``` + +## 4. Setup and Build the ROM + +Once inside the container, you can follow steps [4](../README.md#4-setup-the-rom-and-build-process) and [5](../README.md#5-build-the-rom) of the Linux instructions to setup and build the ROM, or run any other command you need. diff --git a/docs/BUILDING_MACOS.md b/docs/BUILDING_MACOS.md new file mode 100644 index 0000000000..a5668eeea7 --- /dev/null +++ b/docs/BUILDING_MACOS.md @@ -0,0 +1,89 @@ +# Building on macOS + +**N.B.** C++17 is required to build the asset processing program that we use (ZAPD), so check your OS version can support this before proceeding. + + +## 1. Dependencies + +For macOS, use Homebrew to install the following dependencies: + +* coreutils +* make +* python3 +* libpng +* bash +* clang-format + +You can install them with the following commands: + +```bash +brew update +brew install coreutils make python3 libpng bash clang-format +``` + +(The repository expects Homebrew-installed programs to be either linked correctly in `$PATH` etc. or in their default locations.) + +To install the Python dependencies simply run in a terminal: + +```bash +python3 -m pip install colorama +``` + + +## 2. Building mips-linux-binutils + +The following instructions are written for MacOS users but should apply to any Unix-like system, with maybe some modifications at the end regarding the bash_profile. + +Create destination dir for binutils +```bash +sudo mkdir -p /opt/cross +``` + +Create and enter local working dir +```bash +mkdir ~/binutils-tmp +cd ~/binutils-tmp +``` + +Get and extract binutils source +```bash +wget https://ftp.gnu.org/gnu/binutils/binutils-2.35.tar.bz2 +tar xjf binutils-2.35.tar.bz2 +``` +(You may find this command does not work: if so, just access the URL in a browser and save it to `~/binutils-tmp`.) + +Create and enter a build directory +```bash +mkdir build-binutils +cd build-binutils +``` + +Configure the build +```bash +../binutils-2.35/configure --target=mips-linux-gnu --prefix=/opt/cross --disable-gprof --disable-gdb --disable-werror +``` + +Make and install binutils +```bash +make -j +sudo make install +``` + +Edit your `~/.bash_profile`/`~/.zsh_profile` (or whichever shell you use) to add the new binutils binaries to the system PATH +```bash +echo "export PATH=$PATH:/opt/cross/bin" >> ~/.bash_profile +``` + +Reload `~/.bash_profile` (or just launch a new terminal tab) +```bash +source ~/.bash_profile +``` + +If this worked, you can now delete the temporary directory `~/binutils-tmp`. + + +## 3. Final note + +Apple's version of `make` is very out-of-date, so you should use the brew-installed `gmake` in place of `make` in this repo from now on. + +You should now be able to continue from [step 2](../README.md#2-clone-the-repository) of the Linux instructions. diff --git a/extract_assets.py b/extract_assets.py index 0364059508..10768b9dbf 100755 --- a/extract_assets.py +++ b/extract_assets.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 -import argparse, json, os, signal, time -from multiprocessing import Pool, cpu_count, Event, Manager, ProcessError +import argparse, json, os, signal, time, colorama, multiprocessing + +colorama.init() EXTRACTED_ASSETS_NAMEFILE = ".extracted-assets.json" @@ -16,13 +17,13 @@ def ExtractFile(xmlPath, outputPath, outputSourcePath): # Don't extract if another file wasn't extracted properly. return - execStr = "tools/ZAPD/ZAPD.out e -eh -i %s -b baserom/ -o %s -osf %s -gsf 1 -rconf tools/ZAPDConfigs/MqDbg/Config.xml" % (xmlPath, outputPath, outputSourcePath) - + execStr = f"tools/ZAPD/ZAPD.out e -eh -i {xmlPath} -b baserom/ -o {outputPath} -osf {outputSourcePath} -gsf 1 -rconf tools/ZAPDConfigs/MqDbg/Config.xml {ZAPDArgs}" + if "overlays" in xmlPath: execStr += " --static" if globalUnaccounted: - execStr += " -wu" + execStr += " -Wunaccounted" print(execStr) exitValue = os.system(execStr) @@ -67,16 +68,35 @@ def initializeWorker(abort, unaccounted: bool, extractedAssetsTracker: dict, man globalExtractedAssetsTracker = extractedAssetsTracker globalManager = manager +def processZAPDArgs(argsZ): + badZAPDArg = False + for z in argsZ: + if z[0] == '-': + print(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: argument \"{z}\" starts with \"-\", which is not supported.", file=os.sys.stderr) + badZAPDArg = True + + if badZAPDArg: + exit(1) + + ZAPDArgs = " ".join(f"-{z}" for z in argsZ) + print("Using extra ZAPD arguments: " + ZAPDArgs) + return ZAPDArgs + def main(): parser = argparse.ArgumentParser(description="baserom asset extractor") parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep") parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones.", action="store_true") + parser.add_argument("-j", "--jobs", help="Number of cpu cores to extract with.") parser.add_argument("-u", "--unaccounted", help="Enables ZAPD unaccounted detector warning system.", action="store_true") + parser.add_argument("-Z", help="Pass the argument on to ZAPD, e.g. `-ZWunaccounted` to warn about unaccounted blocks in XMLs. Each argument should be passed separately, *without* the leading dash.", metavar="ZAPD_ARG", action="append") args = parser.parse_args() + global ZAPDArgs + ZAPDArgs = processZAPDArgs(args.Z) if args.Z else "" + global mainAbort - mainAbort = Event() - manager = Manager() + mainAbort = multiprocessing.Event() + manager = multiprocessing.Manager() signal.signal(signal.SIGINT, SignalHandler) extractedAssetsTracker = manager.dict() @@ -88,7 +108,7 @@ def main(): if asset_path is not None: fullPath = os.path.join("assets", "xml", asset_path + ".xml") if not os.path.exists(fullPath): - print(f"Error. File {fullPath} doesn't exists.", file=os.sys.stderr) + print(f"Error. File {fullPath} does not exist.", file=os.sys.stderr) exit(1) initializeWorker(mainAbort, args.unaccounted, extractedAssetsTracker, manager) @@ -117,11 +137,13 @@ def main(): xmlFiles.append(fullPath) try: - numCores = cpu_count() - print("Extracting assets with " + str(numCores) + " CPU cores.") - with Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, args.unaccounted, extractedAssetsTracker, manager)) as p: + numCores = int(args.jobs or 0) + if numCores <= 0: + numCores = 1 + print("Extracting assets with " + str(numCores) + " CPU core" + ("s" if numCores > 1 else "") + ".") + with multiprocessing.get_context("fork").Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, args.unaccounted, extractedAssetsTracker, manager)) as p: p.map(ExtractFunc, xmlFiles) - except (ProcessError, TypeError): + except (multiprocessing.ProcessError, TypeError): print("Warning: Multiprocessing exception ocurred.", file=os.sys.stderr) print("Disabling mutliprocessing.", file=os.sys.stderr) @@ -139,4 +161,4 @@ def main(): exit(1) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/format.sh b/format.sh index 26f6f8a5b8..2065443996 100755 --- a/format.sh +++ b/format.sh @@ -1,29 +1,57 @@ #!/usr/bin/env bash +FORMAT_VER="11" FORMAT_OPTS="-i -style=file" TIDY_OPTS="-p . --fix --fix-errors" COMPILER_OPTS="-fno-builtin -std=gnu90 -Iinclude -Isrc -D_LANGUAGE_C -DNON_MATCHING" +# https://backreference.org/2010/05/23/sanitizing-files-with-no-trailing-newline/index.html +# "gets the last character of the file pipes it into read, which will exit with +# a nonzero exit code if it encounters EOF before newline (so, if the last +# character of the file isn't a newline). If read exits nonzero, then append a +# newline onto the file using echo (if read exits 0, that satisfies the ||, so +# the echo command isn't run)." (https://stackoverflow.com/a/34865616) +function add_final_newline () { + for file in "$@" + do + tail -c1 $file | read -r _ || echo >> $file + done +} +export -f add_final_newline + shopt -s globstar +if [ $(command -v clang-format) ] +then + CLANG_FORMAT="clang-format" +else + if [ $(command -v clang-format-${FORMAT_VER}) ] + then + CLANG_FORMAT="clang-format-${FORMAT_VER}" + else + echo "Neither clang-format nor clang-format-${FORMAT_VER} found. Exiting." + exit 1 + fi +fi + if (( $# > 0 )); then echo "Formatting file(s) $*" echo "Running clang-format..." - clang-format ${FORMAT_OPTS} "$@" + ${CLANG_FORMAT} ${FORMAT_OPTS} "$@" echo "Running clang-tidy..." clang-tidy ${TIDY_OPTS} "$@" -- ${COMPILER_OPTS} &> /dev/null echo "Adding missing final new lines..." - sed -i -e '$a\' "$@" + add_final_newline "$@" echo "Done formatting file(s) $*" exit fi echo "Formatting C files. This will take a bit" echo "Running clang-format..." -clang-format ${FORMAT_OPTS} src/**/*.c +${CLANG_FORMAT} ${FORMAT_OPTS} src/**/*.c echo "Running clang-tidy..." clang-tidy ${TIDY_OPTS} src/**/*.c -- ${COMPILER_OPTS} &> /dev/null echo "Adding missing final new lines..." -find src/ -type f -name "*.c" -exec sed -i -e '$a\' {} \; -find assets/xml/ -type f -name "*.xml" -exec sed -i -e '$a\' {} \; +find src/ -type f -name "*.c" -exec bash -c 'add_final_newline "$@"' bash {} + +find assets/xml/ -type f -name "*.xml" -exec bash -c 'add_final_newline "$@"' bash {} + echo "Done formatting all files." diff --git a/tools/ZAPD/.gitrepo b/tools/ZAPD/.gitrepo index be24b438e2..05be4d0b4a 100644 --- a/tools/ZAPD/.gitrepo +++ b/tools/ZAPD/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/zeldaret/ZAPD.git branch = master - commit = a3363333d809e8089a55efc3ea32eaea9ddb8d8c - parent = 1cf11907fa8d636babba2df854850035f04bd4fc + commit = 0ba78130478ee1272bc0e2f2fec2d162e7f7f995 + parent = 4afee2cfb8bc9cd5c7c217c38138174a9ce1fd99 method = merge cmdver = 0.4.3 diff --git a/tools/ZAPD/ExporterTest/Makefile b/tools/ZAPD/ExporterTest/Makefile index a2bfa9a814..42033b7c0c 100644 --- a/tools/ZAPD/ExporterTest/Makefile +++ b/tools/ZAPD/ExporterTest/Makefile @@ -1,7 +1,7 @@ # Only used for standalone compilation, usually inherits these from the main makefile CXXFLAGS ?= -Wall -Wextra -O2 -g -std=c++17 -SRC_DIRS := $(shell find -type d -not -path "*build*") +SRC_DIRS := $(shell find . -type d -not -path "*build*") CPP_FILES := $(foreach dir,$(SRC_DIRS),$(wildcard $(dir)/*.cpp)) H_FILES := $(foreach dir,$(SRC_DIRS),$(wildcard $(dir)/*.h)) diff --git a/tools/ZAPD/Jenkinsfile b/tools/ZAPD/Jenkinsfile index a12f30a253..9c4dde7122 100644 --- a/tools/ZAPD/Jenkinsfile +++ b/tools/ZAPD/Jenkinsfile @@ -7,7 +7,7 @@ pipeline { // Non-parallel ZAPD stage stage('Build ZAPD') { steps { - sh 'make -j' + sh 'make -j WERROR=1' } } @@ -22,13 +22,13 @@ pipeline { } } - // stage('Checkout mm') { - // steps{ - // dir('mm') { - // git url: 'https://github.com/zeldaret/mm.git' - // } - // } - // } + stage('Checkout mm') { + steps{ + dir('mm') { + git url: 'https://github.com/zeldaret/mm.git' + } + } + } } } @@ -51,20 +51,20 @@ pipeline { } } - // stage('Setup MM') { - // steps { - // dir('mm') { - // sh 'cp /usr/local/etc/roms/mm.us.rev1.z64 baserom.mm.us.rev1.z64' + stage('Setup MM') { + steps { + dir('mm') { + sh 'cp /usr/local/etc/roms/mm.us.rev1.z64 baserom.mm.us.rev1.z64' - // // Identical to `make setup` except for copying our newer ZAPD.out into mm - // sh 'make -C tools' - // sh 'cp ../ZAPD.out tools/ZAPD/' - // sh 'python3 tools/fixbaserom.py' - // sh 'python3 tools/extract_baserom.py' - // sh 'python3 extract_assets.py -t 4' - // } - // } - // } + // Identical to `make setup` except for copying our newer ZAPD.out into mm + sh 'make -C tools' + sh 'cp ../ZAPD.out tools/ZAPD/' + sh 'python3 tools/fixbaserom.py' + sh 'python3 tools/extract_baserom.py' + sh 'python3 extract_assets.py -j $(nproc)' + } + } + } } } @@ -78,14 +78,14 @@ pipeline { } } } - // stage('Build mm') { - // steps { - // dir('mm') { - // sh 'make -j disasm' - // sh 'make -j all' - // } - // } - // } + stage('Build mm') { + steps { + dir('mm') { + sh 'make -j disasm' + sh 'make -j all' + } + } + } } } } diff --git a/tools/ZAPD/Makefile b/tools/ZAPD/Makefile index 737b85ecf7..2b47a8039b 100644 --- a/tools/ZAPD/Makefile +++ b/tools/ZAPD/Makefile @@ -6,6 +6,7 @@ DEPRECATION_ON ?= 1 DEBUG ?= 0 COPYCHECK_ARGS ?= LLD ?= 0 +WERROR ?= 0 # Use clang++ if available, else use g++ ifeq ($(shell command -v clang++ >/dev/null 2>&1; echo $$?),0) @@ -23,14 +24,16 @@ ifneq ($(DEBUG),0) CXXFLAGS += -g3 -DDEVELOPMENT -D_DEBUG COPYCHECK_ARGS += --devel DEPRECATION_ON = 0 -else +endif + +ifneq ($(WERROR),0) CXXFLAGS += -Werror endif ifeq ($(OPTIMIZATION_ON),0) OPTFLAGS := -O0 else - OPTFLAGS := -O2 -march=native -mtune=native + OPTFLAGS := -O2 endif ifneq ($(ASAN),0) @@ -53,10 +56,23 @@ ifneq ($(LLD),0) endif UNAME := $(shell uname) +UNAMEM := $(shell uname -m) ifneq ($(UNAME), Darwin) - LDFLAGS += -Wl,-export-dynamic -lstdc++fs + LDFLAGS += -Wl,-export-dynamic -lstdc++fs + EXPORTERS := -Wl,--whole-archive ExporterTest/ExporterTest.a -Wl,--no-whole-archive +else + EXPORTERS := -Wl,-force_load ExporterTest/ExporterTest.a + ifeq ($(UNAMEM),arm64) + ifeq ($(shell brew list libpng > /dev/null 2>&1; echo $$?),0) + LDFLAGS += -L $(shell brew --prefix)/lib + INC += -I $(shell brew --prefix)/include + else + $(error Please install libpng via Homebrew) + endif + endif endif + ZAPD_SRC_DIRS := $(shell find ZAPD -type d) SRC_DIRS = $(ZAPD_SRC_DIRS) lib/tinyxml2 @@ -115,4 +131,4 @@ ZAPDUtils: # Linking ZAPD.out: $(O_FILES) lib/libgfxd/libgfxd.a ExporterTest ZAPDUtils - $(CXX) $(CXXFLAGS) $(O_FILES) lib/libgfxd/libgfxd.a ZAPDUtils/ZAPDUtils.a -Wl,--whole-archive ExporterTest/ExporterTest.a -Wl,--no-whole-archive $(LDFLAGS) $(OUTPUT_OPTION) + $(CXX) $(CXXFLAGS) $(O_FILES) lib/libgfxd/libgfxd.a ZAPDUtils/ZAPDUtils.a $(EXPORTERS) $(LDFLAGS) $(OUTPUT_OPTION) diff --git a/tools/ZAPD/README.md b/tools/ZAPD/README.md index e2821bc28a..448aea0339 100644 --- a/tools/ZAPD/README.md +++ b/tools/ZAPD/README.md @@ -16,6 +16,14 @@ In a Debian/Ubuntu based environment, those could be installed with the followin sudo apt install libpng-dev ``` +On a Mac, you will need to install libpng with Homebrew or MacPorts; we currently only support Homebrew. You can run + +```bash +brew install libpng +``` + +to install it via Homebrew. + ### Building #### Linux / *nix @@ -111,6 +119,7 @@ ZAPD also accepts the following list of extra parameters: - `-tm MODE`: Test Mode (enables certain experimental features). To enable it, set `MODE` to `1`. - `-se` / `--set-exporter` : Sets which exporter to use. - `--gcc-compat` : Enables GCC compatibly mode. Slower. +- `-us` / `--unaccounted-static` : Mark unaccounted data as `static` - `-s` / `--static` : Mark every asset as `static`. - This behaviour can be overridden per asset using `Static=` in the respective XML node. - `-W...`: warning flags, see below diff --git a/tools/ZAPD/ZAPD/Declaration.cpp b/tools/ZAPD/ZAPD/Declaration.cpp index be06b0bea7..34ab953517 100644 --- a/tools/ZAPD/ZAPD/Declaration.cpp +++ b/tools/ZAPD/ZAPD/Declaration.cpp @@ -171,7 +171,7 @@ std::string Declaration::GetExternalDeclarationStr() const std::string Declaration::GetExternStr() const { - if (IsStatic() || varType == "") + if (IsStatic() || varType == "" || isUnaccounted) { return ""; } diff --git a/tools/ZAPD/ZAPD/Declaration.h b/tools/ZAPD/ZAPD/Declaration.h index b7bb0d30d2..4a743b50fa 100644 --- a/tools/ZAPD/ZAPD/Declaration.h +++ b/tools/ZAPD/ZAPD/Declaration.h @@ -12,8 +12,7 @@ typedef uint32_t offset_t; enum class DeclarationAlignment { Align4, - Align8, - Align16 + Align8 }; enum class StaticConfig diff --git a/tools/ZAPD/ZAPD/Globals.cpp b/tools/ZAPD/ZAPD/Globals.cpp index 528a09d256..a10705e9af 100644 --- a/tools/ZAPD/ZAPD/Globals.cpp +++ b/tools/ZAPD/ZAPD/Globals.cpp @@ -152,8 +152,8 @@ bool Globals::GetSegmentedPtrName(segptr_t segAddress, ZFile* currentFile, } } - const auto& symbolFromMap = Globals::Instance->symbolMap.find(segAddress); - if (symbolFromMap != Globals::Instance->symbolMap.end()) + const auto& symbolFromMap = Globals::Instance->cfg.symbolMap.find(segAddress); + if (symbolFromMap != Globals::Instance->cfg.symbolMap.end()) { declName = "&" + symbolFromMap->second; return true; diff --git a/tools/ZAPD/ZAPD/Globals.h b/tools/ZAPD/ZAPD/Globals.h index 19e193f123..1ae753e204 100644 --- a/tools/ZAPD/ZAPD/Globals.h +++ b/tools/ZAPD/ZAPD/Globals.h @@ -58,11 +58,11 @@ public: bool verboseUnaccounted = false; bool gccCompat = false; bool forceStatic = false; + bool forceUnaccountedStatic = false; std::vector files; std::vector externalFiles; std::vector segments; - std::map symbolMap; std::string currentExporter; static std::map& GetExporterMap(); diff --git a/tools/ZAPD/ZAPD/Main.cpp b/tools/ZAPD/ZAPD/Main.cpp index fd2ed06dc0..531edc0946 100644 --- a/tools/ZAPD/ZAPD/Main.cpp +++ b/tools/ZAPD/ZAPD/Main.cpp @@ -48,7 +48,7 @@ void ErrorHandler(int sig) const char* crashEasterEgg[] = { "\tYou've met with a terrible fate, haven't you?", "\tSEA BEARS FOAM. SLEEP BEARS DREAMS. \n\tBOTH END IN THE SAME WAY: CRASSSH!", - "ZAPD has fallen and cannot get up." + "\tZAPD has fallen and cannot get up.", }; srand(time(nullptr)); @@ -216,6 +216,10 @@ int main(int argc, char* argv[]) { Globals::Instance->forceStatic = true; } + else if (arg == "-us" || arg == "--unaccounted-static") + { + Globals::Instance->forceUnaccountedStatic = true; + } } // Parse File Mode @@ -430,6 +434,10 @@ void BuildAssetTexture(const fs::path& pngFilePath, TextureType texType, const f std::string name = outPath.stem().string(); ZTexture tex(nullptr); + + if (name.find("u32") != std::string::npos) + tex.dWordAligned = false; + tex.FromPNG(pngFilePath.string(), texType); std::string cfgPath = StringHelper::Split(pngFilePath.string(), ".")[0] + ".cfg"; diff --git a/tools/ZAPD/ZAPD/OtherStructs/SkinLimbStructs.cpp b/tools/ZAPD/ZAPD/OtherStructs/SkinLimbStructs.cpp index 9d16de4389..0879861997 100644 --- a/tools/ZAPD/ZAPD/OtherStructs/SkinLimbStructs.cpp +++ b/tools/ZAPD/ZAPD/OtherStructs/SkinLimbStructs.cpp @@ -263,7 +263,7 @@ void Struct_800A5E28::DeclareReferences(const std::string& prefix) ZResource::DeclareReferences(varPrefix); - if (unk_4 != 0 && GETSEGNUM(unk_4) == parent->segment) + if (unk_4 != SEGMENTED_NULL && GETSEGNUM(unk_4) == parent->segment) { const auto& res = unk_4_arr.at(0); std::string unk_4_Str = res.GetDefaultName(varPrefix); @@ -293,7 +293,7 @@ void Struct_800A5E28::DeclareReferences(const std::string& prefix) decl->text = entryStr; } - if (unk_8 != 0 && GETSEGNUM(unk_8) == parent->segment) + if (unk_8 != SEGMENTED_NULL && GETSEGNUM(unk_8) == parent->segment) { uint32_t unk_8_Offset = Seg2Filespace(unk_8, parent->baseAddress); @@ -306,6 +306,8 @@ void Struct_800A5E28::DeclareReferences(const std::string& prefix) std::string dListStr = StringHelper::Sprintf("%sSkinLimbDL_%06X", varPrefix.c_str(), unk_8_Offset); unk_8_dlist->SetName(dListStr); + unk_8_dlist->DeclareVar(varPrefix, ""); + unk_8_dlist->DeclareReferences(varPrefix); parent->AddResource(unk_8_dlist); } } diff --git a/tools/ZAPD/ZAPD/Overlays/ZOverlay.cpp b/tools/ZAPD/ZAPD/Overlays/ZOverlay.cpp index 510de19aba..113c4ef339 100644 --- a/tools/ZAPD/ZAPD/Overlays/ZOverlay.cpp +++ b/tools/ZAPD/ZAPD/Overlays/ZOverlay.cpp @@ -90,7 +90,8 @@ ZOverlay* ZOverlay::FromBuild(fs::path buildPath, fs::path cfgFolderPath) std::vector readers; for (size_t i = 1; i < cfgLines.size(); i++) { - std::string elfPath = (buildPath / (cfgLines[i].substr(0, cfgLines[i].size() - 2) + ".o")).string(); + std::string elfPath = + (buildPath / (cfgLines[i].substr(0, cfgLines[i].size() - 2) + ".o")).string(); elfio* reader = new elfio(); if (!reader->load(elfPath)) diff --git a/tools/ZAPD/ZAPD/ZAnimation.cpp b/tools/ZAPD/ZAPD/ZAnimation.cpp index adb0ae7c9a..4d9266b091 100644 --- a/tools/ZAPD/ZAPD/ZAnimation.cpp +++ b/tools/ZAPD/ZAPD/ZAnimation.cpp @@ -104,7 +104,7 @@ void ZNormalAnimation::DeclareReferences(const std::string& prefix) valuesStr += "\n "; } - parent->AddDeclarationArray(rotationValuesOffset, DeclarationAlignment::Align16, + parent->AddDeclarationArray(rotationValuesOffset, DeclarationAlignment::Align4, rotationValues.size() * 2, "s16", StringHelper::Sprintf("%sFrameData", defaultPrefix.c_str()), rotationValues.size(), valuesStr); @@ -118,7 +118,7 @@ void ZNormalAnimation::DeclareReferences(const std::string& prefix) indicesStr += "\n"; } - parent->AddDeclarationArray(rotationIndicesOffset, DeclarationAlignment::Align16, + parent->AddDeclarationArray(rotationIndicesOffset, DeclarationAlignment::Align4, rotationIndices.size() * 6, "JointIndex", StringHelper::Sprintf("%sJointIndices", defaultPrefix.c_str()), rotationIndices.size(), indicesStr); @@ -385,7 +385,7 @@ size_t ZCurveAnimation::GetRawDataSize() const DeclarationAlignment ZCurveAnimation::GetDeclarationAlignment() const { - return DeclarationAlignment::Align16; + return DeclarationAlignment::Align4; } std::string ZCurveAnimation::GetSourceTypeName() const diff --git a/tools/ZAPD/ZAPD/ZArray.cpp b/tools/ZAPD/ZAPD/ZArray.cpp index ebfb13ee74..b6d9e50e50 100644 --- a/tools/ZAPD/ZAPD/ZArray.cpp +++ b/tools/ZAPD/ZAPD/ZArray.cpp @@ -101,11 +101,18 @@ std::string ZArray::GetBodySourceCode() const const auto& res = resList[i]; output += "\t"; - if (res->GetResourceType() == ZResourceType::Scalar || - res->GetResourceType() == ZResourceType::Vertex) + switch (res->GetResourceType()) + { + case ZResourceType::Pointer: + case ZResourceType::Scalar: + case ZResourceType::Vertex: output += resList.at(i)->GetBodySourceCode(); - else + break; + + default: output += StringHelper::Sprintf("{ %s }", resList.at(i)->GetBodySourceCode().c_str()); + break; + } if (i < arrayCnt - 1 || res->IsExternalResource()) output += ",\n"; diff --git a/tools/ZAPD/ZAPD/ZCollision.cpp b/tools/ZAPD/ZAPD/ZCollision.cpp index 1dfa46b1dd..6554005eca 100644 --- a/tools/ZAPD/ZAPD/ZCollision.cpp +++ b/tools/ZAPD/ZAPD/ZCollision.cpp @@ -1,5 +1,6 @@ #include "ZCollision.h" +#include #include #include @@ -76,9 +77,42 @@ void ZCollisionHeader::ParseRawData() polygonTypes.push_back( BitConverter::ToUInt64BE(rawData, polyTypeDefSegmentOffset + (i * 8))); - if (camDataAddress != 0) - camData = new CameraDataList(parent, name, rawData, camDataSegmentOffset, - polyTypeDefSegmentOffset, polygonTypes.size()); + if (camDataAddress != SEGMENTED_NULL) + { + // Try to guess how many elements the CamDataList array has. + // The "guessing algorithm" is basically a "best effort" one and it + // is error-prone. + // This is based mostly on observation of how CollisionHeader data is + // usually ordered. If for some reason the data was in some other funny + // order, this would probably break. + // The most common ordering is: + // - *CamData* + // - SurfaceType + // - CollisionPoly + // - Vertices + // - WaterBoxes + // - CollisionHeader + offset_t upperCameraBoundary = polyTypeDefSegmentOffset; + if (upperCameraBoundary == 0) + { + upperCameraBoundary = polySegmentOffset; + } + if (upperCameraBoundary == 0) + { + upperCameraBoundary = vtxSegmentOffset; + } + if (upperCameraBoundary == 0) + { + upperCameraBoundary = waterBoxSegmentOffset; + } + if (upperCameraBoundary == 0) + { + upperCameraBoundary = rawDataIndex; + } + + camData = + new CameraDataList(parent, name, rawData, camDataSegmentOffset, upperCameraBoundary); + } for (uint16_t i = 0; i < numWaterBoxes; i++) waterBoxes.push_back(WaterBoxHeader( @@ -106,8 +140,7 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix) parent->AddDeclarationArray( waterBoxSegmentOffset, DeclarationAlignment::Align4, 16 * waterBoxes.size(), "WaterBox", - StringHelper::Sprintf("%s_waterBoxes_%06X", auxName.c_str(), waterBoxSegmentOffset), - waterBoxes.size(), declaration); + StringHelper::Sprintf("%sWaterBoxes", auxName.c_str()), waterBoxes.size(), declaration); } if (polygons.size() > 0) @@ -126,8 +159,7 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix) parent->AddDeclarationArray( polySegmentOffset, DeclarationAlignment::Align4, polygons.size() * 16, "CollisionPoly", - StringHelper::Sprintf("%s_polygons_%08X", auxName.c_str(), polySegmentOffset), - polygons.size(), declaration); + StringHelper::Sprintf("%sPolygons", auxName.c_str()), polygons.size(), declaration); } declaration.clear(); @@ -141,11 +173,10 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix) } if (polyTypeDefAddress != 0) - parent->AddDeclarationArray( - polyTypeDefSegmentOffset, DeclarationAlignment::Align4, polygonTypes.size() * 8, - "SurfaceType", - StringHelper::Sprintf("%s_surfaceType_%08X", auxName.c_str(), polyTypeDefSegmentOffset), - polygonTypes.size(), declaration); + parent->AddDeclarationArray(polyTypeDefSegmentOffset, DeclarationAlignment::Align4, + polygonTypes.size() * 8, "SurfaceType", + StringHelper::Sprintf("%sSurfaceType", auxName.c_str()), + polygonTypes.size(), declaration); declaration.clear(); @@ -167,8 +198,7 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix) parent->AddDeclarationArray( vtxSegmentOffset, first.GetDeclarationAlignment(), vertices.size() * first.GetRawDataSize(), first.GetSourceTypeName(), - StringHelper::Sprintf("%s_vtx_%08X", auxName.c_str(), vtxSegmentOffset), - vertices.size(), declaration); + StringHelper::Sprintf("%sVertices", auxName.c_str()), vertices.size(), declaration); } } @@ -183,11 +213,11 @@ std::string ZCollisionHeader::GetBodySourceCode() const std::string vtxName; Globals::Instance->GetSegmentedPtrName(vtxAddress, parent, "Vec3s", vtxName); - declaration += StringHelper::Sprintf("\t%i,\n\t%s,\n", numVerts, vtxName.c_str()); + declaration += StringHelper::Sprintf("\t%i, %s,\n", numVerts, vtxName.c_str()); std::string polyName; Globals::Instance->GetSegmentedPtrName(polyAddress, parent, "CollisionPoly", polyName); - declaration += StringHelper::Sprintf("\t%i,\n\t%s,\n", numPolygons, polyName.c_str()); + declaration += StringHelper::Sprintf("\t%i, %s,\n", numPolygons, polyName.c_str()); std::string surfaceName; Globals::Instance->GetSegmentedPtrName(polyTypeDefAddress, parent, "SurfaceType", surfaceName); @@ -199,7 +229,7 @@ std::string ZCollisionHeader::GetBodySourceCode() const std::string waterBoxName; Globals::Instance->GetSegmentedPtrName(waterBoxAddress, parent, "WaterBox", waterBoxName); - declaration += StringHelper::Sprintf("\t%i,\n\t%s\n", numWaterBoxes, waterBoxName.c_str()); + declaration += StringHelper::Sprintf("\t%i, %s\n", numWaterBoxes, waterBoxName.c_str()); return declaration; } @@ -261,16 +291,17 @@ std::string WaterBoxHeader::GetBodySourceCode() const } CameraDataList::CameraDataList(ZFile* parent, const std::string& prefix, - const std::vector& rawData, uint32_t rawDataIndex, - uint32_t polyTypeDefSegmentOffset, - [[maybe_unused]] uint32_t polygonTypesCnt) + const std::vector& rawData, offset_t rawDataIndex, + offset_t upperCameraBoundary) { std::string declaration; // Parse CameraDataEntries - int32_t numElements = (polyTypeDefSegmentOffset - rawDataIndex) / 8; - uint32_t cameraPosDataSeg = rawDataIndex; - for (int32_t i = 0; i < numElements; i++) + size_t numElements = (upperCameraBoundary - rawDataIndex) / 8; + assert(numElements < 10000); + + offset_t cameraPosDataSeg = rawDataIndex; + for (size_t i = 0; i < numElements; i++) { CameraDataEntry* entry = new CameraDataEntry(); @@ -302,8 +333,7 @@ CameraDataList::CameraDataList(ZFile* parent, const std::string& prefix, { int32_t index = ((entries[i]->cameraPosDataSeg & 0x00FFFFFF) - cameraPosDataOffset) / 0x6; - sprintf(camSegLine, "&%s_camPosData_%08X[%i]", prefix.c_str(), cameraPosDataOffset, - index); + sprintf(camSegLine, "&%sCamPosData[%i]", prefix.c_str(), index); } else sprintf(camSegLine, "NULL"); @@ -318,7 +348,7 @@ CameraDataList::CameraDataList(ZFile* parent, const std::string& prefix, parent->AddDeclarationArray( rawDataIndex, DeclarationAlignment::Align4, entries.size() * 8, "CamData", - StringHelper::Sprintf("%s_camDataList_%08X", prefix.c_str(), rawDataIndex), entries.size(), + StringHelper::Sprintf("%sCamDataList", prefix.c_str(), rawDataIndex), entries.size(), declaration); uint32_t numDataTotal = (rawDataIndex - cameraPosDataOffset) / 0x6; @@ -339,10 +369,9 @@ CameraDataList::CameraDataList(ZFile* parent, const std::string& prefix, int32_t cameraPosDataIndex = GETSEGOFFSET(cameraPosDataSeg); uint32_t entrySize = numDataTotal * 0x6; - parent->AddDeclarationArray( - cameraPosDataIndex, DeclarationAlignment::Align4, entrySize, "Vec3s", - StringHelper::Sprintf("%s_camPosData_%08X", prefix.c_str(), cameraPosDataIndex), - numDataTotal, declaration); + parent->AddDeclarationArray(cameraPosDataIndex, DeclarationAlignment::Align4, entrySize, + "Vec3s", StringHelper::Sprintf("%sCamPosData", prefix.c_str()), + numDataTotal, declaration); } } diff --git a/tools/ZAPD/ZAPD/ZCollision.h b/tools/ZAPD/ZAPD/ZCollision.h index d0325bfb95..222fd25f9a 100644 --- a/tools/ZAPD/ZAPD/ZCollision.h +++ b/tools/ZAPD/ZAPD/ZCollision.h @@ -55,8 +55,7 @@ public: std::vector cameraPositionData; CameraDataList(ZFile* parent, const std::string& prefix, const std::vector& rawData, - uint32_t rawDataIndex, uint32_t polyTypeDefSegmentOffset, - uint32_t polygonTypesCnt); + offset_t rawDataIndex, offset_t upperCameraBoundary); ~CameraDataList(); }; diff --git a/tools/ZAPD/ZAPD/ZDisplayList.cpp b/tools/ZAPD/ZAPD/ZDisplayList.cpp index 869b563d96..5e9b1b205e 100644 --- a/tools/ZAPD/ZAPD/ZDisplayList.cpp +++ b/tools/ZAPD/ZAPD/ZDisplayList.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include "Globals.h" @@ -26,8 +27,8 @@ ZDisplayList::ZDisplayList(ZFile* nParent) : ZResource(nParent) lastTexSizTest = F3DZEXTexSizes::G_IM_SIZ_16b; lastTexLoaded = false; lastTexIsPalette = false; - name = ""; dListType = Globals::Instance->game == ZGame::OOT_SW97 ? DListType::F3DEX : DListType::F3DZEX; + RegisterOptionalAttribute("Ucode"); } ZDisplayList::~ZDisplayList() @@ -43,13 +44,29 @@ void ZDisplayList::ExtractFromXML(tinyxml2::XMLElement* reader, uint32_t nRawDat { rawDataIndex = nRawDataIndex; ParseXML(reader); + // TODO add error handling here + bool ucodeSet = registeredAttributes.at("Ucode").wasSet; + std::string ucodeValue = registeredAttributes.at("Ucode").value; + if ((Globals::Instance->game == ZGame::OOT_SW97) || (ucodeValue == "f3dex")) + { + dListType = DListType::F3DEX; + } + else if (!ucodeSet || ucodeValue == "f3dex2") + { + dListType = DListType::F3DZEX; + } + else + { + HANDLE_ERROR_RESOURCE( + WarningType::InvalidAttributeValue, parent, this, rawDataIndex, + StringHelper::Sprintf("Invalid ucode type in node: %s\n", reader->Name()), ""); + } // Don't parse raw data of external files if (parent->GetMode() != ZFileMode::ExternalFile) { - int32_t rawDataSize = ZDisplayList::GetDListLength( - parent->GetRawData(), rawDataIndex, - Globals::Instance->game == ZGame::OOT_SW97 ? DListType::F3DEX : DListType::F3DZEX); + int32_t rawDataSize = + ZDisplayList::GetDListLength(parent->GetRawData(), rawDataIndex, dListType); numInstructions = rawDataSize / 8; ParseRawData(); } @@ -106,7 +123,7 @@ void ZDisplayList::ParseF3DZEX(F3DZEXOpcode opcode, uint64_t data, int32_t i, switch (opcode) { case F3DZEXOpcode::G_NOOP: - sprintf(line, "gsDPNoOpTag(0x%08lX),", data & 0xFFFFFFFF); + sprintf(line, "gsDPNoOpTag(0x%08" PRIX64 "),", data & 0xFFFFFFFF); break; case F3DZEXOpcode::G_DL: Opcode_G_DL(data, prefix, line); @@ -221,7 +238,7 @@ void ZDisplayList::ParseF3DZEX(F3DZEXOpcode opcode, uint64_t data, int32_t i, break; case F3DZEXOpcode::G_POPMTX: { - sprintf(line, "gsSPPopMatrix(%li),", data); + sprintf(line, "gsSPPopMatrix(%" PRIi64 "),", data); } break; case F3DZEXOpcode::G_LOADTLUT: @@ -298,7 +315,7 @@ void ZDisplayList::ParseF3DEX(F3DEXOpcode opcode, uint64_t data, const std::stri switch (opcode) { case F3DEXOpcode::G_NOOP: - sprintf(line, "gsDPNoOpTag(0x%08lX),", data & 0xFFFFFFFF); + sprintf(line, "gsDPNoOpTag(0x%08" PRIX64 "),", data & 0xFFFFFFFF); break; case F3DEXOpcode::G_VTX: Opcode_G_VTX(data, line); @@ -688,20 +705,22 @@ void ZDisplayList::Opcode_G_DL(uint64_t data, const std::string& prefix, char* l if (pp != 0) { if (!Globals::Instance->HasSegment(segNum)) - sprintf(line, "gsSPBranchList(0x%08lX),", data & 0xFFFFFFFF); + sprintf(line, "gsSPBranchList(0x%08" PRIX64 "),", data & 0xFFFFFFFF); else if (dListDecl != nullptr) sprintf(line, "gsSPBranchList(%s),", dListDecl->varName.c_str()); else - sprintf(line, "gsSPBranchList(%sDlist0x%06lX),", prefix.c_str(), GETSEGOFFSET(data)); + sprintf(line, "gsSPBranchList(%sDlist0x%06" PRIX64 "),", prefix.c_str(), + GETSEGOFFSET(data)); } else { if (!Globals::Instance->HasSegment(segNum)) - sprintf(line, "gsSPDisplayList(0x%08lX),", data & 0xFFFFFFFF); + sprintf(line, "gsSPDisplayList(0x%08" PRIX64 "),", data & 0xFFFFFFFF); else if (dListDecl != nullptr) sprintf(line, "gsSPDisplayList(%s),", dListDecl->varName.c_str()); else - sprintf(line, "gsSPDisplayList(%sDlist0x%06lX),", prefix.c_str(), GETSEGOFFSET(data)); + sprintf(line, "gsSPDisplayList(%sDlist0x%06" PRIX64 "),", prefix.c_str(), + GETSEGOFFSET(data)); } // if (segNum == 8 || segNum == 9 || segNum == 10 || segNum == 11 || segNum == 12 || segNum == @@ -709,9 +728,9 @@ void ZDisplayList::Opcode_G_DL(uint64_t data, const std::string& prefix, char* l if (!Globals::Instance->HasSegment(segNum)) { if (pp != 0) - sprintf(line, "gsSPBranchList(0x%08lX),", data & 0xFFFFFFFF); + sprintf(line, "gsSPBranchList(0x%08" PRIX64 "),", data & 0xFFFFFFFF); else - sprintf(line, "gsSPDisplayList(0x%08lX),", data & 0xFFFFFFFF); + sprintf(line, "gsSPDisplayList(0x%08" PRIX64 "),", data & 0xFFFFFFFF); } else { @@ -831,7 +850,7 @@ void ZDisplayList::Opcode_G_VTX(uint64_t data, char* line) { segptr_t segmented = data & 0xFFFFFFFF; references.push_back(segmented); - parent->AddDeclaration(segmented, DeclarationAlignment::Align16, 16, "Vtx", + parent->AddDeclaration(segmented, DeclarationAlignment::Align8, 16, "Vtx", StringHelper::Sprintf("0x%08X", segmented), ""); return; } @@ -943,7 +962,7 @@ void ZDisplayList::Opcode_G_SETTIMG(uint64_t data, const std::string& prefix, ch sprintf(texStr, "%sTex_%06X", prefix.c_str(), texAddress); else { - sprintf(texStr, "0x%08lX", data & 0xFFFFFFFF); + sprintf(texStr, "0x%08" PRIX64, data & 0xFFFFFFFF); } sprintf(line, "gsDPSetTextureImage(%s, %s, %i, %s),", fmtTbl[fmt], sizTbl[siz], www + 1, @@ -1747,7 +1766,7 @@ void ZDisplayList::DeclareReferences(const std::string& prefix) if (vertices.size() > 0) { std::vector>> verticesSorted(vertices.begin(), - vertices.end()); + vertices.end()); for (size_t i = 0; i < verticesSorted.size() - 1; i++) { @@ -1795,7 +1814,7 @@ void ZDisplayList::DeclareReferences(const std::string& prefix) if (vertices.size() > 0) { std::vector>> verticesSorted(vertices.begin(), - vertices.end()); + vertices.end()); for (size_t i = 0; i < verticesSorted.size() - 1; i++) { @@ -1889,10 +1908,10 @@ std::string ZDisplayList::ProcessLegacy(const std::string& prefix) } auto end = std::chrono::steady_clock::now(); - auto diff = std::chrono::duration_cast(end - start).count(); + int64_t diff = std::chrono::duration_cast(end - start).count(); if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_DEBUG && diff > 5) - printf("F3DOP: 0x%02X, TIME: %lims\n", opcode, diff); + printf("F3DOP: 0x%02X, TIME: %" PRIi64 "ms\n", opcode, diff); sourceOutput += line; @@ -1973,7 +1992,7 @@ bool ZDisplayList::TextureGenCheck(int32_t texWidth, int32_t texHeight, uint32_t } else { - // Try to find a non-external file (ie, one we are actually extracting) + // Try to find a non-external file (i.e., one we are actually extracting) // which has the same segment number we are looking for. for (auto& otherFile : Globals::Instance->cfg.segmentRefFiles[segmentNumber]) { diff --git a/tools/ZAPD/ZAPD/ZFile.cpp b/tools/ZAPD/ZAPD/ZFile.cpp index 77387fc72c..22a6a45058 100644 --- a/tools/ZAPD/ZAPD/ZFile.cpp +++ b/tools/ZAPD/ZAPD/ZFile.cpp @@ -8,6 +8,7 @@ #include "Globals.h" #include "OutputFormatter.h" #include "Utils/BinaryWriter.h" +#include "Utils/BitConverter.h" #include "Utils/Directory.h" #include "Utils/File.h" #include "Utils/MemoryStream.h" @@ -192,7 +193,7 @@ void ZFile::ParseXML(tinyxml2::XMLElement* reader, const std::string& filename) rawData = File::ReadAllBytes((basePath / name).string()); if (reader->Attribute("RangeEnd") == nullptr) - rangeEnd = rawData.size(); + rangeEnd = rawData.size(); } std::unordered_set nameSet; @@ -378,7 +379,7 @@ void ZFile::ExtractResources() ZResourceExporter* exporter = Globals::Instance->GetExporter(res->GetResourceType()); if (exporter != nullptr) { - //exporter->Save(res, Globals::Instance->outputPath.string(), &writerFile); + // exporter->Save(res, Globals::Instance->outputPath.string(), &writerFile); exporter->Save(res, Globals::Instance->outputPath.string(), &writerRes); } @@ -391,7 +392,7 @@ void ZFile::ExtractResources() File::WriteAllBytes(StringHelper::Sprintf("%s%s.bin", Globals::Instance->outputPath.string().c_str(), GetName().c_str()), - memStreamFile->ToVector()); + memStreamFile->ToVector()); } writerFile.Close(); @@ -949,9 +950,6 @@ std::string ZFile::ProcessDeclarations() defines += ProcessTextureIntersections(name); - // Account for padding/alignment - uint32_t lastAddr = 0; - // printf("RANGE START: 0x%06X - RANGE END: 0x%06X\n", rangeStart, rangeEnd); // Optimization: See if there are any arrays side by side that can be merged... @@ -1002,43 +1000,6 @@ std::string ZFile::ProcessDeclarations() { while (item.second->size % 4 != 0) item.second->size++; - - if (lastAddr != 0) - { - if (item.second->alignment == DeclarationAlignment::Align16) - { - int32_t curPtr = lastAddr + declarations[lastAddr]->size; - - while (curPtr % 4 != 0) - { - declarations[lastAddr]->size++; - curPtr++; - } - } - else if (item.second->alignment == DeclarationAlignment::Align8) - { - size_t curPtr = lastAddr + declarations[lastAddr]->size; - - while (curPtr % 4 != 0) - { - declarations[lastAddr]->size++; - curPtr++; - } - - while (curPtr % 8 != 0) - { - char buffer[2048]; - - sprintf(buffer, "u32 %s_align%02zX = 0;\n", name.c_str(), curPtr); - item.second->preText = buffer + item.second->preText; - - declarations[lastAddr]->size += 4; - curPtr += 4; - } - } - } - - lastAddr = item.first; } HandleUnaccountedData(); @@ -1199,14 +1160,15 @@ void ZFile::HandleUnaccountedData() { uint32_t lastAddr = 0; uint32_t lastSize = 0; - std::vector declsAddresses; + std::vector declsAddresses; + for (const auto& item : declarations) { declsAddresses.push_back(item.first); } bool breakLoop = false; - for (uint32_t currentAddress : declsAddresses) + for (offset_t currentAddress : declsAddresses) { if (currentAddress >= rangeEnd) { @@ -1235,7 +1197,7 @@ void ZFile::HandleUnaccountedData() } } -bool ZFile::HandleUnaccountedAddress(uint32_t currentAddress, uint32_t lastAddr, uint32_t& lastSize) +bool ZFile::HandleUnaccountedAddress(offset_t currentAddress, offset_t lastAddr, uint32_t& lastSize) { if (currentAddress != lastAddr && declarations.find(lastAddr) != declarations.end()) { @@ -1275,6 +1237,29 @@ bool ZFile::HandleUnaccountedAddress(uint32_t currentAddress, uint32_t lastAddr, xmlFilePath.c_str(), currentAddress, name.c_str(), rawData.size())); } + // Handle Align8 + if (currentAddress % 8 == 0 && diff % 8 != 0) + { + Declaration* currentDecl = GetDeclaration(currentAddress); + + if (currentDecl != nullptr) + { + if (currentDecl->alignment == DeclarationAlignment::Align8) + { + // Check removed bytes are zeroes + if (BitConverter::ToUInt32BE(rawData, unaccountedAddress + diff - 4) == 0) + { + diff -= 4; + } + } + + if (diff == 0) + { + return false; + } + } + } + for (int i = 0; i < diff; i++) { uint8_t val = rawData.at(unaccountedAddress + i); @@ -1320,7 +1305,10 @@ bool ZFile::HandleUnaccountedAddress(uint32_t currentAddress, uint32_t lastAddr, StringHelper::Sprintf("%s_%s_%06X", name.c_str(), unaccountedPrefix.c_str(), unaccountedAddress), diff, src); + decl->isUnaccounted = true; + if (Globals::Instance->forceUnaccountedStatic) + decl->staticConf = StaticConfig::On; if (nonZeroUnaccounted) { diff --git a/tools/ZAPD/ZAPD/ZFile.h b/tools/ZAPD/ZAPD/ZFile.h index 7918d5f595..ac4062d5b1 100644 --- a/tools/ZAPD/ZAPD/ZFile.h +++ b/tools/ZAPD/ZAPD/ZFile.h @@ -133,5 +133,5 @@ protected: std::string ProcessTextureIntersections(const std::string& prefix); void HandleUnaccountedData(); - bool HandleUnaccountedAddress(uint32_t currentAddress, uint32_t lastAddr, uint32_t& lastSize); + bool HandleUnaccountedAddress(offset_t currentAddress, offset_t lastAddr, uint32_t& lastSize); }; diff --git a/tools/ZAPD/ZAPD/ZLimb.cpp b/tools/ZAPD/ZAPD/ZLimb.cpp index a47615d6c6..330fbaf7ce 100644 --- a/tools/ZAPD/ZAPD/ZLimb.cpp +++ b/tools/ZAPD/ZAPD/ZLimb.cpp @@ -228,8 +228,8 @@ std::string ZLimb::GetBodySourceCode() const { std::string childName; std::string siblingName; - Globals::Instance->GetSegmentedPtrName(childPtr, parent, "Gfx", childName); - Globals::Instance->GetSegmentedPtrName(siblingPtr, parent, "Gfx", siblingName); + Globals::Instance->GetSegmentedPtrName(childPtr, parent, "LegacyLimb", childName); + Globals::Instance->GetSegmentedPtrName(siblingPtr, parent, "LegacyLimb", siblingName); entryStr += StringHelper::Sprintf("%s,\n", dListStr.c_str()); entryStr += diff --git a/tools/ZAPD/ZAPD/ZPointer.cpp b/tools/ZAPD/ZAPD/ZPointer.cpp new file mode 100644 index 0000000000..6e6945cada --- /dev/null +++ b/tools/ZAPD/ZAPD/ZPointer.cpp @@ -0,0 +1,57 @@ +#include "ZPointer.h" + +#include "Globals.h" +#include "Utils/BitConverter.h" +#include "Utils/StringHelper.h" +#include "WarningHandler.h" +#include "ZFile.h" + +REGISTER_ZFILENODE(Pointer, ZPointer); + +ZPointer::ZPointer(ZFile* nParent) : ZResource(nParent) +{ + RegisterRequiredAttribute("Type"); +} + +void ZPointer::ParseXML(tinyxml2::XMLElement* reader) +{ + ZResource::ParseXML(reader); + + type = registeredAttributes.at("Type").value; +} + +void ZPointer::ParseRawData() +{ + auto& rawData = parent->GetRawData(); + + ptr = BitConverter::ToUInt32BE(rawData, rawDataIndex); +} + +std::string ZPointer::GetBodySourceCode() const +{ + std::string ptrName; + + Globals::Instance->GetSegmentedPtrName(ptr, parent, "", ptrName); + + return ptrName; +} + +bool ZPointer::DoesSupportArray() const +{ + return true; +} + +std::string ZPointer::GetSourceTypeName() const +{ + return type + "*"; +} + +ZResourceType ZPointer::GetResourceType() const +{ + return ZResourceType::Pointer; +} + +size_t ZPointer::GetRawDataSize() const +{ + return 0x04; +} diff --git a/tools/ZAPD/ZAPD/ZPointer.h b/tools/ZAPD/ZAPD/ZPointer.h new file mode 100644 index 0000000000..8fb5889339 --- /dev/null +++ b/tools/ZAPD/ZAPD/ZPointer.h @@ -0,0 +1,22 @@ +#pragma once + +#include "ZResource.h" + +class ZPointer : public ZResource +{ +public: + segptr_t ptr = SEGMENTED_NULL; + std::string type; + + ZPointer(ZFile* nParent); + + void ParseXML(tinyxml2::XMLElement* reader) override; + void ParseRawData() override; + std::string GetBodySourceCode() const override; + + bool DoesSupportArray() const override; + std::string GetSourceTypeName() const override; + ZResourceType GetResourceType() const override; + + size_t GetRawDataSize() const override; +}; diff --git a/tools/ZAPD/ZAPD/ZResource.h b/tools/ZAPD/ZAPD/ZResource.h index 4dad398955..c65c3c31b4 100644 --- a/tools/ZAPD/ZAPD/ZResource.h +++ b/tools/ZAPD/ZAPD/ZResource.h @@ -38,6 +38,7 @@ enum class ZResourceType Mtx, Path, PlayerAnimationData, + Pointer, Room, RoomCommand, Scalar, diff --git a/tools/ZAPD/ZAPD/ZRoom/Commands/SetMesh.cpp b/tools/ZAPD/ZAPD/ZRoom/Commands/SetMesh.cpp index 69668c49c1..ba0bbe2c24 100644 --- a/tools/ZAPD/ZAPD/ZRoom/Commands/SetMesh.cpp +++ b/tools/ZAPD/ZAPD/ZRoom/Commands/SetMesh.cpp @@ -152,7 +152,6 @@ void PolygonDlist::GetSourceOutputCode(const std::string& prefix) DeclareVar(prefix, bodyStr); else decl->text = bodyStr; - } std::string PolygonDlist::GetSourceTypeName() const diff --git a/tools/ZAPD/ZAPD/ZRoom/Commands/SetSpecialObjects.cpp b/tools/ZAPD/ZAPD/ZRoom/Commands/SetSpecialObjects.cpp index 0c24a2f328..696a3de014 100644 --- a/tools/ZAPD/ZAPD/ZRoom/Commands/SetSpecialObjects.cpp +++ b/tools/ZAPD/ZAPD/ZRoom/Commands/SetSpecialObjects.cpp @@ -1,6 +1,8 @@ #include "SetSpecialObjects.h" + #include "Utils/BitConverter.h" #include "Utils/StringHelper.h" +#include "ZRoom/ZNames.h" SetSpecialObjects::SetSpecialObjects(ZFile* nParent) : ZRoomCommand(nParent) { @@ -15,8 +17,10 @@ void SetSpecialObjects::ParseRawData() std::string SetSpecialObjects::GetBodySourceCode() const { - return StringHelper::Sprintf("SCENE_CMD_SPECIAL_FILES(0x%02X, 0x%04X)", elfMessage, - globalObject); + std::string objectName = ZNames::GetObjectName(globalObject); + + return StringHelper::Sprintf("SCENE_CMD_SPECIAL_FILES(0x%02X, %s)", elfMessage, + objectName.c_str()); } std::string SetSpecialObjects::GetCommandCName() const diff --git a/tools/ZAPD/ZAPD/ZRoom/ZRoom.cpp b/tools/ZAPD/ZAPD/ZRoom/ZRoom.cpp index c7e60d3592..5831eaa56b 100644 --- a/tools/ZAPD/ZAPD/ZRoom/ZRoom.cpp +++ b/tools/ZAPD/ZAPD/ZRoom/ZRoom.cpp @@ -2,7 +2,9 @@ #include #include #include +#include #include + #include "Commands/EndMarker.h" #include "Commands/SetActorCutsceneList.h" #include "Commands/SetActorList.h" @@ -259,9 +261,10 @@ void ZRoom::ParseRawData() if (Globals::Instance->profile) { auto end = std::chrono::steady_clock::now(); - auto diff = std::chrono::duration_cast(end - start).count(); + int64_t diff = + std::chrono::duration_cast(end - start).count(); if (diff > 50) - printf("OP: %s, TIME: %lims\n", cmd->GetCommandCName().c_str(), diff); + printf("OP: %s, TIME: %" PRIi64 "ms\n", cmd->GetCommandCName().c_str(), diff); } cmd->cmdIndex = currentIndex; diff --git a/tools/ZAPD/ZAPD/ZSkeleton.cpp b/tools/ZAPD/ZAPD/ZSkeleton.cpp index 1a2f93ff7b..4467c96320 100644 --- a/tools/ZAPD/ZAPD/ZSkeleton.cpp +++ b/tools/ZAPD/ZAPD/ZSkeleton.cpp @@ -139,7 +139,7 @@ ZResourceType ZSkeleton::GetResourceType() const DeclarationAlignment ZSkeleton::GetDeclarationAlignment() const { - return DeclarationAlignment::Align16; + return DeclarationAlignment::Align4; } uint8_t ZSkeleton::GetLimbCount() diff --git a/tools/ZAPD/ZAPD/ZTexture.cpp b/tools/ZAPD/ZAPD/ZTexture.cpp index 7bd31438b4..255d30886e 100644 --- a/tools/ZAPD/ZAPD/ZTexture.cpp +++ b/tools/ZAPD/ZAPD/ZTexture.cpp @@ -16,6 +16,7 @@ ZTexture::ZTexture(ZFile* nParent) : ZResource(nParent) { width = 0; height = 0; + dWordAligned = true; RegisterRequiredAttribute("Width"); RegisterRequiredAttribute("Height"); @@ -105,10 +106,7 @@ void ZTexture::ParseXML(tinyxml2::XMLElement* reader) void ZTexture::ParseRawData() { if (rawDataIndex % 8 != 0) - { - HANDLE_WARNING_RESOURCE(WarningType::NotImplemented, parent, this, rawDataIndex, - "this texture is not 64-bit aligned", ""); - } + dWordAligned = false; switch (format) { @@ -724,7 +722,12 @@ void ZTexture::Save(const fs::path& outFolder) if (!Directory::Exists(outPath.string())) Directory::CreateDirectory(outPath.string()); - auto outFileName = outPath / (outName + "." + GetExternalExtension() + ".png"); + fs::path outFileName; + + if (!dWordAligned) + outFileName = outPath / (outName + ".u32" + "." + GetExternalExtension() + ".png"); + else + outFileName = outPath / (outName + +"." + GetExternalExtension() + ".png"); #ifdef TEXTURE_DEBUG printf("Saving PNG: %s\n", outFileName.c_str()); @@ -745,7 +748,7 @@ Declaration* ZTexture::DeclareVar(const std::string& prefix, { std::string auxName = name; std::string auxOutName = outName; - + std::string incStr; if (auxName == "") auxName = GetDefaultName(prefix); @@ -754,8 +757,12 @@ Declaration* ZTexture::DeclareVar(const std::string& prefix, auto filepath = Globals::Instance->outputPath / fs::path(auxOutName).stem(); - std::string incStr = - StringHelper::Sprintf("%s.%s.inc.c", filepath.c_str(), GetExternalExtension().c_str()); + if (dWordAligned) + incStr = + StringHelper::Sprintf("%s.%s.inc.c", filepath.c_str(), GetExternalExtension().c_str()); + else + incStr = StringHelper::Sprintf("%s.u32.%s.inc.c", filepath.c_str(), + GetExternalExtension().c_str()); if (!Globals::Instance->cfg.texturePool.empty()) { @@ -765,13 +772,19 @@ Declaration* ZTexture::DeclareVar(const std::string& prefix, const auto& poolEntry = Globals::Instance->cfg.texturePool.find(hash); if (poolEntry != Globals::Instance->cfg.texturePool.end()) { - incStr = StringHelper::Sprintf("%s.%s.inc.c", poolEntry->second.path.c_str(), - GetExternalExtension().c_str()); + if (dWordAligned) + incStr = StringHelper::Sprintf("%s.%s.inc.c", poolEntry->second.path.c_str(), + GetExternalExtension().c_str()); + else + incStr = StringHelper::Sprintf("%s.u32.%s.inc.c", poolEntry->second.path.c_str(), + GetExternalExtension().c_str()); } } + size_t texSizeDivisor = (dWordAligned) ? 8 : 4; - Declaration* decl = parent->AddDeclarationIncludeArray( - rawDataIndex, incStr, GetRawDataSize(), GetSourceTypeName(), auxName, GetRawDataSize() / 8); + Declaration* decl = parent->AddDeclarationIncludeArray(rawDataIndex, incStr, GetRawDataSize(), + GetSourceTypeName(), auxName, + GetRawDataSize() / texSizeDivisor); decl->staticConf = staticConf; return decl; } @@ -779,15 +792,17 @@ Declaration* ZTexture::DeclareVar(const std::string& prefix, std::string ZTexture::GetBodySourceCode() const { std::string sourceOutput; - - for (size_t i = 0; i < textureDataRaw.size(); i += 8) + size_t texSizeInc = (dWordAligned) ? 8 : 4; + for (size_t i = 0; i < textureDataRaw.size(); i += texSizeInc) { if (i % 32 == 0) sourceOutput += " "; - - sourceOutput += - StringHelper::Sprintf("0x%016llX, ", BitConverter::ToUInt64BE(textureDataRaw, i)); - + if (dWordAligned) + sourceOutput += + StringHelper::Sprintf("0x%016llX, ", BitConverter::ToUInt64BE(textureDataRaw, i)); + else + sourceOutput += + StringHelper::Sprintf("0x%08llX, ", BitConverter::ToUInt32BE(textureDataRaw, i)); if (i % 32 == 24) sourceOutput += StringHelper::Sprintf(" // 0x%06X \n", rawDataIndex + ((i / 32) * 32)); } @@ -812,7 +827,7 @@ ZResourceType ZTexture::GetResourceType() const std::string ZTexture::GetSourceTypeName() const { - return "u64"; + return dWordAligned ? "u64" : "u32"; } void ZTexture::CalcHash() diff --git a/tools/ZAPD/ZAPD/ZTexture.h b/tools/ZAPD/ZAPD/ZTexture.h index ef67c3f6ef..29c3fbc0a1 100644 --- a/tools/ZAPD/ZAPD/ZTexture.h +++ b/tools/ZAPD/ZAPD/ZTexture.h @@ -54,6 +54,7 @@ public: ZTexture(ZFile* nParent); bool isPalette = false; + bool dWordAligned = true; void ExtractFromBinary(uint32_t nRawDataIndex, int32_t nWidth, int32_t nHeight, TextureType nType, bool nIsPalette); diff --git a/tools/ZAPD/ZAPD/ZVtx.cpp b/tools/ZAPD/ZAPD/ZVtx.cpp index d5214e840a..e4b3d97968 100644 --- a/tools/ZAPD/ZAPD/ZVtx.cpp +++ b/tools/ZAPD/ZAPD/ZVtx.cpp @@ -82,5 +82,5 @@ std::string ZVtx::GetExternalExtension() const DeclarationAlignment ZVtx::GetDeclarationAlignment() const { - return DeclarationAlignment::Align16; + return DeclarationAlignment::Align8; } diff --git a/tools/ZAPD/ZAPDUtils/Makefile b/tools/ZAPD/ZAPDUtils/Makefile index aef6780313..e8941ed773 100644 --- a/tools/ZAPD/ZAPDUtils/Makefile +++ b/tools/ZAPD/ZAPDUtils/Makefile @@ -1,7 +1,7 @@ # Only used for standalone compilation, usually inherits these from the main makefile CXXFLAGS ?= -Wall -Wextra -O2 -g -std=c++17 -SRC_DIRS := $(shell find -type d -not -path "*build*") +SRC_DIRS := $(shell find . -type d -not -path "*build*") CPP_FILES := $(foreach dir,$(SRC_DIRS),$(wildcard $(dir)/*.cpp)) H_FILES := $(foreach dir,$(SRC_DIRS),$(wildcard $(dir)/*.h)) diff --git a/tools/ZAPD/ZAPDUtils/Utils/StringHelper.h b/tools/ZAPD/ZAPDUtils/Utils/StringHelper.h index 0b0d676429..6c9e541190 100644 --- a/tools/ZAPD/ZAPDUtils/Utils/StringHelper.h +++ b/tools/ZAPD/ZAPDUtils/Utils/StringHelper.h @@ -109,6 +109,6 @@ public: static bool IEquals(const std::string& a, const std::string& b) { return std::equal(a.begin(), a.end(), b.begin(), b.end(), - [](char a, char b) { return tolower(a) == tolower(b); }); + [](char a, char b) { return tolower(a) == tolower(b); }); } }; diff --git a/tools/ZAPD/docs/zapd_extraction_xml_reference.md b/tools/ZAPD/docs/zapd_extraction_xml_reference.md index 06b95bb575..b3f5575c15 100644 --- a/tools/ZAPD/docs/zapd_extraction_xml_reference.md +++ b/tools/ZAPD/docs/zapd_extraction_xml_reference.md @@ -9,6 +9,7 @@ This document aims to be a small reference of how to create a compatible xml fil - [Basic XML](#basic-xml) - [Resources types](#resources-types) - [File](#file) + - [ExternalFile](#externalfile) - [Texture](#texture) - [Background](#background) - [Blob](#blob) @@ -33,6 +34,7 @@ This document aims to be a small reference of how to create a compatible xml fil - [Array](#array) - [Path](#path) - [PlayerAnimationData](#playeranimationdata) + - [Pointer](#pointer) ## Basic XML @@ -589,7 +591,7 @@ Vec3s D_04002040[24] = { The `Array` element is special, because it needs an inner element to work. It will declare an array of its inner element. -Currently, only [`Scalar`](#scalar), [`Vector`](#vector) and [`Vtx`](#vtx) support being wrapped in an array. +Currently, only [`Pointer`](#pointer), [`Scalar`](#scalar), [`Vector`](#vector) and [`Vtx`](#vtx) support being wrapped in an array. - Example: @@ -637,3 +639,24 @@ Allows the extraction of the specific data of the player animations which are fo - `FrameCount`: Required. The length of the animation in frames. It must be a positive integer. ------------------------- + +### Pointer + +Allows the extraction of a variable that contains a pointer + +- Example: + +```xml + + + +``` + +- Attributes: + + - `Name`: Required. + - `Type`: Required. The type of the extracted pointer. + +※ Can be wrapped in an [`Array`](#array) tag. + +-------------------------