Introduce extracted/VERSION, with text extracted there (#1730)

* Introduce assets/_extracted/VERSION, with text extracted there * move to `extracted/text/` * Update gitignore s * rework args for msgenc.py * put mkdir with others, until theyre all moved at once * move 0xFFFC back to being extracted, making it use specific macro `DEFINE_MESSAGE_NES` to handle its special behavior * prettier gitignore * Move messages 0xFFFC, 0xFFFD to committed message_data.h
2024-12-25 22:26:12 +00:00 · 2024-03-02 04:09:57 +01:00 · 2024-03-02 04:09:57 +01:00 · a6f646dc65
commit a6f646dc65
parent ca45c543f6
13 changed files with 88 additions and 34 deletions
--- a/.gitignore
+++ b/.gitignore
@ -51,6 +51,7 @@ graphs/
 *.fbx
 !*_custom*
 .extracted-assets.json
+extracted/

 # Docs
 !docs/tutorial/
--- a/13
+++ b/13
@ -66,6 +66,7 @@ PROJECT_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
 BUILD_DIR := build/$(VERSION)
 EXPECTED_DIR := expected/$(BUILD_DIR)
 BASEROM_DIR := baseroms/$(VERSION)
+EXTRACTED_DIR := extracted/$(VERSION)
 VENV := .venv

 MAKE = make
@ -130,7 +131,7 @@ NM      := $(MIPS_BINUTILS_PREFIX)nm

 N64_EMULATOR ?= 

-INC := -Iinclude -Iinclude/libc -Isrc -I$(BUILD_DIR) -I.
+INC := -Iinclude -Iinclude/libc -Isrc -I$(BUILD_DIR) -I. -I$(EXTRACTED_DIR)

 # Check code syntax with host compiler
 CHECK_WARNINGS := -Wall -Wextra -Wno-format-security -Wno-unknown-pragmas -Wno-unused-parameter -Wno-unused-variable -Wno-missing-braces
@ -230,7 +231,7 @@ TEXTURE_FILES_OUT := $(foreach f,$(TEXTURE_FILES_PNG:.png=.inc.c),$(BUILD_DIR)/$
 					 $(foreach f,$(TEXTURE_FILES_JPG:.jpg=.jpg.inc.c),$(BUILD_DIR)/$f) \

 # create build directories
-$(shell mkdir -p $(BUILD_DIR)/baserom $(BUILD_DIR)/assets/text $(foreach dir,$(SRC_DIRS) $(UNDECOMPILED_DATA_DIRS) $(ASSET_BIN_DIRS),$(BUILD_DIR)/$(dir)))
+$(shell mkdir -p $(BUILD_DIR)/baserom $(EXTRACTED_DIR)/text $(BUILD_DIR)/assets/text $(foreach dir,$(SRC_DIRS) $(UNDECOMPILED_DATA_DIRS) $(ASSET_BIN_DIRS),$(BUILD_DIR)/$(dir)))

 ifeq ($(COMPILER),ido)
 $(BUILD_DIR)/src/boot/stackcheck.o: OPTFLAGS := -O2
@ -345,7 +346,7 @@ clean:

 assetclean:
 	$(RM) -r $(ASSET_BIN_DIRS)
-	$(RM) -r assets/text/*.h
+	$(RM) -r $(EXTRACTED_DIR)
 	$(RM) -r $(BUILD_DIR)/assets
 	$(RM) -r .extracted-assets.json

@ -367,7 +368,7 @@ setup: venv
 # TODO: for now, we only extract assets from the Debug ROM
 ifeq ($(VERSION),gc-eu-mq-dbg)
 	$(PYTHON) extract_assets.py -j$(N_THREADS)
-	$(PYTHON) tools/msgdis.py --text-out assets/text/message_data.h --staff-text-out assets/text/message_data_staff.h
+	$(PYTHON) tools/msgdis.py --text-out $(EXTRACTED_DIR)/text/message_data.h --staff-text-out $(EXTRACTED_DIR)/text/message_data_staff.h
 endif

 disasm:
@ -423,8 +424,8 @@ $(BUILD_DIR)/baserom/%.o: $(BASEROM_SEGMENTS_DIR)/%
 $(BUILD_DIR)/data/%.o: data/%.s
 	$(AS) $(ASFLAGS) $< -o $@

-$(BUILD_DIR)/assets/text/%.enc.h: assets/text/%.h assets/text/charmap.txt
-	$(PYTHON) tools/msgenc.py assets/text/charmap.txt $< $@
+$(BUILD_DIR)/assets/text/%.enc.h: assets/text/%.h $(EXTRACTED_DIR)/text/%.h assets/text/charmap.txt
+	$(CPP) $(CPPFLAGS) -I$(EXTRACTED_DIR) $< | $(PYTHON) tools/msgenc.py - --output $@ --charmap assets/text/charmap.txt

 # Dependencies for files including message data headers
 # TODO remove when full header dependencies are used.
--- a/assets/.gitignore
+++ b/assets/.gitignore
@ -1,7 +1,7 @@
 *.bin
 *.c
-!text/*.c
 *.h
+!text/*.[ch]
 *.cfg
 *.vtx.inc
 *.dlist.inc
--- a/assets/text/fra_message_data_static.c
+++ b/assets/text/fra_message_data_static.c
@ -5,4 +5,6 @@
 #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \
    const char _message_##textId##_fra[sizeof(fraMessage)] = { fraMessage END };

+#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage)
+
 #include "assets/text/message_data.enc.h"
--- a/assets/text/ger_message_data_static.c
+++ b/assets/text/ger_message_data_static.c
@ -5,4 +5,6 @@
 #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \
    const char _message_##textId##_ger[sizeof(gerMessage)] = { gerMessage END };

+#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage)
+
 #include "assets/text/message_data.enc.h"
--- a/assets/text/message_data.h
+++ b/assets/text/message_data.h
@ -0,0 +1,16 @@
+#include "text/message_data.h"
+
+/*
+ * The following two messages should be kept last and in this order.
+ * Message 0xFFFD must be last to not break the message debugger (see R_MESSAGE_DEBUGGER_TEXTID).
+ * Message 0xFFFC must be immediately before message 0xFFFD to not break Font_LoadOrderedFont.
+ */
+DEFINE_MESSAGE_NES(0xFFFC, TEXTBOX_TYPE_BLACK, TEXTBOX_POS_VARIABLE,
+"0123456789\n"
+"ABCDEFGHIJKLMN\n"
+"OPQRSTUVWXYZ\n"
+"abcdefghijklmn\n"
+"opqrstuvwxyz\n"
+" -.\n"
+)
+DEFINE_MESSAGE(0xFFFD, TEXTBOX_TYPE_BLACK, TEXTBOX_POS_VARIABLE, "", "", "")
--- a/assets/text/message_data_staff.h
+++ b/assets/text/message_data_staff.h
@ -0,0 +1 @@
+#include "text/message_data_staff.h"
--- a/assets/text/nes_message_data_static.c
+++ b/assets/text/nes_message_data_static.c
@ -5,8 +5,6 @@
 #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \
    const char _message_##textId##_nes[sizeof(nesMessage)] = { nesMessage END };

-#define DEFINE_MESSAGE_FFFC
+#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) DEFINE_MESSAGE(textId, type, yPos, nesMessage, , )

 #include "assets/text/message_data.enc.h"
-
-#undef DEFINE_MESSAGE_FFFC
--- a/docs/vscode.md
+++ b/docs/vscode.md
@ -48,11 +48,12 @@ You can create a `.vscode/c_cpp_properties.json` file with `C/C++: Edit Configur
            ],
            "intelliSenseMode": "${default}", // Shouldn't matter
            "includePath": [ // Matches makefile's includes
-                "${workspaceFolder}/**",
+                "include",
+                "include/libc",
                "src",
                "build/gc-eu-mq-dbg",
-                "include",
-                "include/libc"
+                ".",
+                "extracted/gc-eu-mq-dbg"
            ],
            "defines": [
                "_LANGUAGE_C", // For gbi.h
--- a/include/message_data_static.h
+++ b/include/message_data_static.h
@ -47,9 +47,12 @@ typedef struct {
    extern const char _message_##textId##_ger[]; \
    extern const char _message_##textId##_fra[];

+#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) \
+    extern const char _message_##textId##_nes[];
+
 #include "assets/text/message_data.h"
-extern const char _message_0xFFFC_nes[];

 #undef DEFINE_MESSAGE
+#undef DEFINE_MESSAGE_NES

 #endif
--- a/src/code/z_message_PAL.c
+++ b/src/code/z_message_PAL.c
@ -30,24 +30,28 @@ u16 sOcarinaSongBitFlags = 0; // ocarina bit flags
 MessageTableEntry sNesMessageEntryTable[] = {
 #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \
    { textId, (_SHIFTL(type, 4, 8) | _SHIFTL(yPos, 0, 8)), _message_##textId##_nes },
-#define DEFINE_MESSAGE_FFFC
+#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) DEFINE_MESSAGE(textId, type, yPos, nesMessage, , )
 #include "assets/text/message_data.h"
-#undef DEFINE_MESSAGE_FFFC
 #undef DEFINE_MESSAGE
+#undef DEFINE_MESSAGE_NES
    { 0xFFFF, 0, NULL },
 };

 const char* sGerMessageEntryTable[] = {
 #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) _message_##textId##_ger,
+#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage)
 #include "assets/text/message_data.h"
 #undef DEFINE_MESSAGE
+#undef DEFINE_MESSAGE_NES
    NULL,
 };

 const char* sFraMessageEntryTable[] = {
 #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) _message_##textId##_fra,
+#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage)
 #include "assets/text/message_data.h"
 #undef DEFINE_MESSAGE
+#undef DEFINE_MESSAGE_NES
    NULL,
 };

--- a/tools/msgdis.py
+++ b/tools/msgdis.py
@ -376,20 +376,26 @@ def extract_all_text(text_out, staff_text_out):
    if text_out is not None:
        out = ""
        for message in dump_all_text():
-            if message[0] == 0xFFFF:
+            # Skip 0xFFFC and 0xFFFD because they are committed
+            # Skip 0xFFFF, the last entry
+            if message[0] in {0xFFFC, 0xFFFD, 0xFFFF}:
                continue

-            if message[0] == 0xFFFC:
-                out += "#ifdef DEFINE_MESSAGE_FFFC\n"
-            out += f"DEFINE_MESSAGE(0x{message[0]:04X}, {textbox_type[message[1]]}, {textbox_ypos[message[2]]},"
+            is_nes_message = message[0] == 0xFFFC
+            if not is_nes_message:
+                out += "DEFINE_MESSAGE"
+            else:
+                out += "DEFINE_MESSAGE_NES"
+            out += f"(0x{message[0]:04X}, {textbox_type[message[1]]}, {textbox_ypos[message[2]]},"
            out += "\n"
-            out += f"{message[3]}" + ("\n" if message[3] != "" else "") + ","
-            out += "\n" if message[3] != "" else ""
-            out += f"{message[4]}" + ("\n" if message[4] != "" else "") + ","
-            out += "\n" if message[4] != "" else ""
-            out += f"{message[5]}\n)"
-            if message[0] == 0xFFFC:
-                out += "\n#endif"
+            out += f"{message[3]}"
+            if not is_nes_message:
+                out += ("\n" if message[3] != "" else "") + ","
+                out += "\n" if message[3] != "" else ""
+                out += f"{message[4]}" + ("\n" if message[4] != "" else "") + ","
+                out += "\n" if message[4] != "" else ""
+                out += f"{message[5]}"
+            out += "\n)"
            out += "\n\n"

        with open(text_out, "w", encoding="utf8") as outfile:
--- a/tools/msgenc.py
+++ b/tools/msgenc.py
@ -4,6 +4,7 @@
 #

 import argparse, ast, re
+import sys

 def read_charmap(path):
    with open(path) as infile:
@ -50,22 +51,40 @@ def convert_text(text, charmap):

 def main():
    parser = argparse.ArgumentParser(description="Encode message_data_static text headers")
-    parser.add_argument("charmap", help="path to charmap file specifying custom encoding elements")
-    parser.add_argument("input", help="path to file to be encoded")
-    parser.add_argument("output", help="encoded file")
+    parser.add_argument(
+        "input",
+        help="path to file to be encoded, or - for stdin",
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        help="path to write encoded file, or - for stdout",
+        required=True,
+    )
+    parser.add_argument(
+        "--charmap",
+        help="path to charmap file specifying custom encoding elements",
+        required=True,
+    )
    args = parser.parse_args()

    charmap = read_charmap(args.charmap)

    text = ""
-    with open(args.input, "r") as infile:
-        text = infile.read()
+    if args.input == "-":
+        text = sys.stdin.read()
+    else:
+        with open(args.input, "r") as infile:
+            text = infile.read()

    text = remove_comments(text)
    text = convert_text(text, charmap)

-    with open(args.output, "w", encoding="raw_unicode_escape") as outfile:
-        outfile.write(text)
+    if args.output == "-":
+        sys.stdout.buffer.write(text.encode("raw_unicode_escape"))
+    else:
+        with open(args.output, "w", encoding="raw_unicode_escape") as outfile:
+            outfile.write(text)

 if __name__ == "__main__":
    main()