From a6f646dc659eb1711901fc05d82704b34a9c23fd Mon Sep 17 00:00:00 2001 From: Dragorn421 Date: Sat, 2 Mar 2024 04:09:57 +0100 Subject: [PATCH] Introduce extracted/VERSION, with text extracted there (#1730) * Introduce assets/_extracted/VERSION, with text extracted there * move to `extracted/text/` * Update gitignore s * rework args for msgenc.py * put mkdir with others, until theyre all moved at once * move 0xFFFC back to being extracted, making it use specific macro `DEFINE_MESSAGE_NES` to handle its special behavior * prettier gitignore * Move messages 0xFFFC, 0xFFFD to committed message_data.h --- .gitignore | 1 + Makefile | 13 ++++++----- assets/.gitignore | 2 +- assets/text/fra_message_data_static.c | 2 ++ assets/text/ger_message_data_static.c | 2 ++ assets/text/message_data.h | 16 +++++++++++++ assets/text/message_data_staff.h | 1 + assets/text/nes_message_data_static.c | 4 +--- docs/vscode.md | 7 +++--- include/message_data_static.h | 5 +++- src/code/z_message_PAL.c | 8 +++++-- tools/msgdis.py | 28 ++++++++++++++--------- tools/msgenc.py | 33 +++++++++++++++++++++------ 13 files changed, 88 insertions(+), 34 deletions(-) create mode 100644 assets/text/message_data.h create mode 100644 assets/text/message_data_staff.h diff --git a/.gitignore b/.gitignore index 8585eeebbf..f10e23510e 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ graphs/ *.fbx !*_custom* .extracted-assets.json +extracted/ # Docs !docs/tutorial/ diff --git a/Makefile b/Makefile index 8c8a6f9f90..d50bb7ceed 100644 --- a/Makefile +++ b/Makefile @@ -66,6 +66,7 @@ PROJECT_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) BUILD_DIR := build/$(VERSION) EXPECTED_DIR := expected/$(BUILD_DIR) BASEROM_DIR := baseroms/$(VERSION) +EXTRACTED_DIR := extracted/$(VERSION) VENV := .venv MAKE = make @@ -130,7 +131,7 @@ NM := $(MIPS_BINUTILS_PREFIX)nm N64_EMULATOR ?= -INC := -Iinclude -Iinclude/libc -Isrc -I$(BUILD_DIR) -I. +INC := -Iinclude -Iinclude/libc -Isrc -I$(BUILD_DIR) -I. -I$(EXTRACTED_DIR) # Check code syntax with host compiler CHECK_WARNINGS := -Wall -Wextra -Wno-format-security -Wno-unknown-pragmas -Wno-unused-parameter -Wno-unused-variable -Wno-missing-braces @@ -230,7 +231,7 @@ TEXTURE_FILES_OUT := $(foreach f,$(TEXTURE_FILES_PNG:.png=.inc.c),$(BUILD_DIR)/$ $(foreach f,$(TEXTURE_FILES_JPG:.jpg=.jpg.inc.c),$(BUILD_DIR)/$f) \ # create build directories -$(shell mkdir -p $(BUILD_DIR)/baserom $(BUILD_DIR)/assets/text $(foreach dir,$(SRC_DIRS) $(UNDECOMPILED_DATA_DIRS) $(ASSET_BIN_DIRS),$(BUILD_DIR)/$(dir))) +$(shell mkdir -p $(BUILD_DIR)/baserom $(EXTRACTED_DIR)/text $(BUILD_DIR)/assets/text $(foreach dir,$(SRC_DIRS) $(UNDECOMPILED_DATA_DIRS) $(ASSET_BIN_DIRS),$(BUILD_DIR)/$(dir))) ifeq ($(COMPILER),ido) $(BUILD_DIR)/src/boot/stackcheck.o: OPTFLAGS := -O2 @@ -345,7 +346,7 @@ clean: assetclean: $(RM) -r $(ASSET_BIN_DIRS) - $(RM) -r assets/text/*.h + $(RM) -r $(EXTRACTED_DIR) $(RM) -r $(BUILD_DIR)/assets $(RM) -r .extracted-assets.json @@ -367,7 +368,7 @@ setup: venv # TODO: for now, we only extract assets from the Debug ROM ifeq ($(VERSION),gc-eu-mq-dbg) $(PYTHON) extract_assets.py -j$(N_THREADS) - $(PYTHON) tools/msgdis.py --text-out assets/text/message_data.h --staff-text-out assets/text/message_data_staff.h + $(PYTHON) tools/msgdis.py --text-out $(EXTRACTED_DIR)/text/message_data.h --staff-text-out $(EXTRACTED_DIR)/text/message_data_staff.h endif disasm: @@ -423,8 +424,8 @@ $(BUILD_DIR)/baserom/%.o: $(BASEROM_SEGMENTS_DIR)/% $(BUILD_DIR)/data/%.o: data/%.s $(AS) $(ASFLAGS) $< -o $@ -$(BUILD_DIR)/assets/text/%.enc.h: assets/text/%.h assets/text/charmap.txt - $(PYTHON) tools/msgenc.py assets/text/charmap.txt $< $@ +$(BUILD_DIR)/assets/text/%.enc.h: assets/text/%.h $(EXTRACTED_DIR)/text/%.h assets/text/charmap.txt + $(CPP) $(CPPFLAGS) -I$(EXTRACTED_DIR) $< | $(PYTHON) tools/msgenc.py - --output $@ --charmap assets/text/charmap.txt # Dependencies for files including message data headers # TODO remove when full header dependencies are used. diff --git a/assets/.gitignore b/assets/.gitignore index e5633a6496..0bece9e89f 100644 --- a/assets/.gitignore +++ b/assets/.gitignore @@ -1,7 +1,7 @@ *.bin *.c -!text/*.c *.h +!text/*.[ch] *.cfg *.vtx.inc *.dlist.inc diff --git a/assets/text/fra_message_data_static.c b/assets/text/fra_message_data_static.c index abad26c541..944d9f03f5 100644 --- a/assets/text/fra_message_data_static.c +++ b/assets/text/fra_message_data_static.c @@ -5,4 +5,6 @@ #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \ const char _message_##textId##_fra[sizeof(fraMessage)] = { fraMessage END }; +#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) + #include "assets/text/message_data.enc.h" diff --git a/assets/text/ger_message_data_static.c b/assets/text/ger_message_data_static.c index c4b8229a00..9d3e190bce 100644 --- a/assets/text/ger_message_data_static.c +++ b/assets/text/ger_message_data_static.c @@ -5,4 +5,6 @@ #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \ const char _message_##textId##_ger[sizeof(gerMessage)] = { gerMessage END }; +#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) + #include "assets/text/message_data.enc.h" diff --git a/assets/text/message_data.h b/assets/text/message_data.h new file mode 100644 index 0000000000..57a0859b56 --- /dev/null +++ b/assets/text/message_data.h @@ -0,0 +1,16 @@ +#include "text/message_data.h" + +/* + * The following two messages should be kept last and in this order. + * Message 0xFFFD must be last to not break the message debugger (see R_MESSAGE_DEBUGGER_TEXTID). + * Message 0xFFFC must be immediately before message 0xFFFD to not break Font_LoadOrderedFont. + */ +DEFINE_MESSAGE_NES(0xFFFC, TEXTBOX_TYPE_BLACK, TEXTBOX_POS_VARIABLE, +"0123456789\n" +"ABCDEFGHIJKLMN\n" +"OPQRSTUVWXYZ\n" +"abcdefghijklmn\n" +"opqrstuvwxyz\n" +" -.\n" +) +DEFINE_MESSAGE(0xFFFD, TEXTBOX_TYPE_BLACK, TEXTBOX_POS_VARIABLE, "", "", "") diff --git a/assets/text/message_data_staff.h b/assets/text/message_data_staff.h new file mode 100644 index 0000000000..c282d7f7d3 --- /dev/null +++ b/assets/text/message_data_staff.h @@ -0,0 +1 @@ +#include "text/message_data_staff.h" diff --git a/assets/text/nes_message_data_static.c b/assets/text/nes_message_data_static.c index 2c37692d26..ff14d66b86 100644 --- a/assets/text/nes_message_data_static.c +++ b/assets/text/nes_message_data_static.c @@ -5,8 +5,6 @@ #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \ const char _message_##textId##_nes[sizeof(nesMessage)] = { nesMessage END }; -#define DEFINE_MESSAGE_FFFC +#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) DEFINE_MESSAGE(textId, type, yPos, nesMessage, , ) #include "assets/text/message_data.enc.h" - -#undef DEFINE_MESSAGE_FFFC diff --git a/docs/vscode.md b/docs/vscode.md index 2bab5a578c..c88b85c072 100644 --- a/docs/vscode.md +++ b/docs/vscode.md @@ -48,11 +48,12 @@ You can create a `.vscode/c_cpp_properties.json` file with `C/C++: Edit Configur ], "intelliSenseMode": "${default}", // Shouldn't matter "includePath": [ // Matches makefile's includes - "${workspaceFolder}/**", + "include", + "include/libc", "src", "build/gc-eu-mq-dbg", - "include", - "include/libc" + ".", + "extracted/gc-eu-mq-dbg" ], "defines": [ "_LANGUAGE_C", // For gbi.h diff --git a/include/message_data_static.h b/include/message_data_static.h index b803203821..b888192d58 100644 --- a/include/message_data_static.h +++ b/include/message_data_static.h @@ -47,9 +47,12 @@ typedef struct { extern const char _message_##textId##_ger[]; \ extern const char _message_##textId##_fra[]; +#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) \ + extern const char _message_##textId##_nes[]; + #include "assets/text/message_data.h" -extern const char _message_0xFFFC_nes[]; #undef DEFINE_MESSAGE +#undef DEFINE_MESSAGE_NES #endif diff --git a/src/code/z_message_PAL.c b/src/code/z_message_PAL.c index 03db0dc188..f63b41052d 100644 --- a/src/code/z_message_PAL.c +++ b/src/code/z_message_PAL.c @@ -30,24 +30,28 @@ u16 sOcarinaSongBitFlags = 0; // ocarina bit flags MessageTableEntry sNesMessageEntryTable[] = { #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \ { textId, (_SHIFTL(type, 4, 8) | _SHIFTL(yPos, 0, 8)), _message_##textId##_nes }, -#define DEFINE_MESSAGE_FFFC +#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) DEFINE_MESSAGE(textId, type, yPos, nesMessage, , ) #include "assets/text/message_data.h" -#undef DEFINE_MESSAGE_FFFC #undef DEFINE_MESSAGE +#undef DEFINE_MESSAGE_NES { 0xFFFF, 0, NULL }, }; const char* sGerMessageEntryTable[] = { #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) _message_##textId##_ger, +#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) #include "assets/text/message_data.h" #undef DEFINE_MESSAGE +#undef DEFINE_MESSAGE_NES NULL, }; const char* sFraMessageEntryTable[] = { #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) _message_##textId##_fra, +#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) #include "assets/text/message_data.h" #undef DEFINE_MESSAGE +#undef DEFINE_MESSAGE_NES NULL, }; diff --git a/tools/msgdis.py b/tools/msgdis.py index 1da85713ec..53b8b5d02b 100755 --- a/tools/msgdis.py +++ b/tools/msgdis.py @@ -376,20 +376,26 @@ def extract_all_text(text_out, staff_text_out): if text_out is not None: out = "" for message in dump_all_text(): - if message[0] == 0xFFFF: + # Skip 0xFFFC and 0xFFFD because they are committed + # Skip 0xFFFF, the last entry + if message[0] in {0xFFFC, 0xFFFD, 0xFFFF}: continue - if message[0] == 0xFFFC: - out += "#ifdef DEFINE_MESSAGE_FFFC\n" - out += f"DEFINE_MESSAGE(0x{message[0]:04X}, {textbox_type[message[1]]}, {textbox_ypos[message[2]]}," + is_nes_message = message[0] == 0xFFFC + if not is_nes_message: + out += "DEFINE_MESSAGE" + else: + out += "DEFINE_MESSAGE_NES" + out += f"(0x{message[0]:04X}, {textbox_type[message[1]]}, {textbox_ypos[message[2]]}," out += "\n" - out += f"{message[3]}" + ("\n" if message[3] != "" else "") + "," - out += "\n" if message[3] != "" else "" - out += f"{message[4]}" + ("\n" if message[4] != "" else "") + "," - out += "\n" if message[4] != "" else "" - out += f"{message[5]}\n)" - if message[0] == 0xFFFC: - out += "\n#endif" + out += f"{message[3]}" + if not is_nes_message: + out += ("\n" if message[3] != "" else "") + "," + out += "\n" if message[3] != "" else "" + out += f"{message[4]}" + ("\n" if message[4] != "" else "") + "," + out += "\n" if message[4] != "" else "" + out += f"{message[5]}" + out += "\n)" out += "\n\n" with open(text_out, "w", encoding="utf8") as outfile: diff --git a/tools/msgenc.py b/tools/msgenc.py index b62b7383cc..9af3b80454 100644 --- a/tools/msgenc.py +++ b/tools/msgenc.py @@ -4,6 +4,7 @@ # import argparse, ast, re +import sys def read_charmap(path): with open(path) as infile: @@ -50,22 +51,40 @@ def convert_text(text, charmap): def main(): parser = argparse.ArgumentParser(description="Encode message_data_static text headers") - parser.add_argument("charmap", help="path to charmap file specifying custom encoding elements") - parser.add_argument("input", help="path to file to be encoded") - parser.add_argument("output", help="encoded file") + parser.add_argument( + "input", + help="path to file to be encoded, or - for stdin", + ) + parser.add_argument( + "--output", + "-o", + help="path to write encoded file, or - for stdout", + required=True, + ) + parser.add_argument( + "--charmap", + help="path to charmap file specifying custom encoding elements", + required=True, + ) args = parser.parse_args() charmap = read_charmap(args.charmap) text = "" - with open(args.input, "r") as infile: - text = infile.read() + if args.input == "-": + text = sys.stdin.read() + else: + with open(args.input, "r") as infile: + text = infile.read() text = remove_comments(text) text = convert_text(text, charmap) - with open(args.output, "w", encoding="raw_unicode_escape") as outfile: - outfile.write(text) + if args.output == "-": + sys.stdout.buffer.write(text.encode("raw_unicode_escape")) + else: + with open(args.output, "w", encoding="raw_unicode_escape") as outfile: + outfile.write(text) if __name__ == "__main__": main()