Introduce extracted/VERSION, with text extracted there (#1730)

* Introduce assets/_extracted/VERSION, with text extracted there

* move to `extracted/text/`

* Update gitignore s

* rework args for msgenc.py

* put mkdir with others, until theyre all moved at once

* move 0xFFFC back to being extracted, making it use specific macro `DEFINE_MESSAGE_NES` to handle its special behavior

* prettier gitignore

* Move messages 0xFFFC, 0xFFFD to committed message_data.h

.gitignore

@@ -51,6 +51,7 @@ graphs/
 *.fbx
 !*_custom*
 .extracted-assets.json
+extracted/
 
 # Docs
 !docs/tutorial/

Makefile

@@ -66,6 +66,7 @@ PROJECT_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
 BUILD_DIR := build/$(VERSION)
 EXPECTED_DIR := expected/$(BUILD_DIR)
 BASEROM_DIR := baseroms/$(VERSION)
+EXTRACTED_DIR := extracted/$(VERSION)
 VENV := .venv
 
 MAKE = make
@@ -130,7 +131,7 @@ NM      := $(MIPS_BINUTILS_PREFIX)nm
 
 N64_EMULATOR ?= 
 
-INC := -Iinclude -Iinclude/libc -Isrc -I$(BUILD_DIR) -I.
+INC := -Iinclude -Iinclude/libc -Isrc -I$(BUILD_DIR) -I. -I$(EXTRACTED_DIR)
 
 # Check code syntax with host compiler
 CHECK_WARNINGS := -Wall -Wextra -Wno-format-security -Wno-unknown-pragmas -Wno-unused-parameter -Wno-unused-variable -Wno-missing-braces
@@ -230,7 +231,7 @@ TEXTURE_FILES_OUT := $(foreach f,$(TEXTURE_FILES_PNG:.png=.inc.c),$(BUILD_DIR)/$
 					 $(foreach f,$(TEXTURE_FILES_JPG:.jpg=.jpg.inc.c),$(BUILD_DIR)/$f) \
 
 # create build directories
-$(shell mkdir -p $(BUILD_DIR)/baserom $(BUILD_DIR)/assets/text $(foreach dir,$(SRC_DIRS) $(UNDECOMPILED_DATA_DIRS) $(ASSET_BIN_DIRS),$(BUILD_DIR)/$(dir)))
+$(shell mkdir -p $(BUILD_DIR)/baserom $(EXTRACTED_DIR)/text $(BUILD_DIR)/assets/text $(foreach dir,$(SRC_DIRS) $(UNDECOMPILED_DATA_DIRS) $(ASSET_BIN_DIRS),$(BUILD_DIR)/$(dir)))
 
 ifeq ($(COMPILER),ido)
 $(BUILD_DIR)/src/boot/stackcheck.o: OPTFLAGS := -O2
@@ -345,7 +346,7 @@ clean:
 
 assetclean:
 	$(RM) -r $(ASSET_BIN_DIRS)
-	$(RM) -r assets/text/*.h
+	$(RM) -r $(EXTRACTED_DIR)
 	$(RM) -r $(BUILD_DIR)/assets
 	$(RM) -r .extracted-assets.json
 
@@ -367,7 +368,7 @@ setup: venv
 # TODO: for now, we only extract assets from the Debug ROM
 ifeq ($(VERSION),gc-eu-mq-dbg)
 	$(PYTHON) extract_assets.py -j$(N_THREADS)
-	$(PYTHON) tools/msgdis.py --text-out assets/text/message_data.h --staff-text-out assets/text/message_data_staff.h
+	$(PYTHON) tools/msgdis.py --text-out $(EXTRACTED_DIR)/text/message_data.h --staff-text-out $(EXTRACTED_DIR)/text/message_data_staff.h
 endif
 
 disasm:
@@ -423,8 +424,8 @@ $(BUILD_DIR)/baserom/%.o: $(BASEROM_SEGMENTS_DIR)/%
 $(BUILD_DIR)/data/%.o: data/%.s
 	$(AS) $(ASFLAGS) $< -o $@
 
-$(BUILD_DIR)/assets/text/%.enc.h: assets/text/%.h assets/text/charmap.txt
+$(BUILD_DIR)/assets/text/%.enc.h: assets/text/%.h $(EXTRACTED_DIR)/text/%.h assets/text/charmap.txt
-	$(PYTHON) tools/msgenc.py assets/text/charmap.txt $< $@
+	$(CPP) $(CPPFLAGS) -I$(EXTRACTED_DIR) $< | $(PYTHON) tools/msgenc.py - --output $@ --charmap assets/text/charmap.txt
 
 # Dependencies for files including message data headers
 # TODO remove when full header dependencies are used.

assets/.gitignore

@@ -1,7 +1,7 @@
 *.bin
 *.c
-!text/*.c
 *.h
+!text/*.[ch]
 *.cfg
 *.vtx.inc
 *.dlist.inc

assets/text/fra_message_data_static.c

@@ -5,4 +5,6 @@
 #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \
     const char _message_##textId##_fra[sizeof(fraMessage)] = { fraMessage END };
 
+#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage)
+
 #include "assets/text/message_data.enc.h"

assets/text/ger_message_data_static.c

@@ -5,4 +5,6 @@
 #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \
     const char _message_##textId##_ger[sizeof(gerMessage)] = { gerMessage END };
 
+#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage)
+
 #include "assets/text/message_data.enc.h"

assets/text/message_data.h

@@ -0,0 +1,16 @@
+#include "text/message_data.h"
+
+/*
+ * The following two messages should be kept last and in this order.
+ * Message 0xFFFD must be last to not break the message debugger (see R_MESSAGE_DEBUGGER_TEXTID).
+ * Message 0xFFFC must be immediately before message 0xFFFD to not break Font_LoadOrderedFont.
+ */
+DEFINE_MESSAGE_NES(0xFFFC, TEXTBOX_TYPE_BLACK, TEXTBOX_POS_VARIABLE,
+"0123456789\n"
+"ABCDEFGHIJKLMN\n"
+"OPQRSTUVWXYZ\n"
+"abcdefghijklmn\n"
+"opqrstuvwxyz\n"
+" -.\n"
+)
+DEFINE_MESSAGE(0xFFFD, TEXTBOX_TYPE_BLACK, TEXTBOX_POS_VARIABLE, "", "", "")

assets/text/message_data_staff.h

@@ -0,0 +1 @@
+#include "text/message_data_staff.h"

assets/text/nes_message_data_static.c

@@ -5,8 +5,6 @@
 #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \
     const char _message_##textId##_nes[sizeof(nesMessage)] = { nesMessage END };
 
-#define DEFINE_MESSAGE_FFFC
+#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) DEFINE_MESSAGE(textId, type, yPos, nesMessage, , )
 
 #include "assets/text/message_data.enc.h"
-
-#undef DEFINE_MESSAGE_FFFC

docs/vscode.md

@@ -48,11 +48,12 @@ You can create a `.vscode/c_cpp_properties.json` file with `C/C++: Edit Configur
             ],
             "intelliSenseMode": "${default}", // Shouldn't matter
             "includePath": [ // Matches makefile's includes
-                "${workspaceFolder}/**",
+                "include",
+                "include/libc",
                 "src",
                 "build/gc-eu-mq-dbg",
-                "include",
+                ".",
-                "include/libc"
+                "extracted/gc-eu-mq-dbg"
             ],
             "defines": [
                 "_LANGUAGE_C", // For gbi.h

include/message_data_static.h

@@ -47,9 +47,12 @@ typedef struct {
     extern const char _message_##textId##_ger[]; \
     extern const char _message_##textId##_fra[];
 
+#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) \
+    extern const char _message_##textId##_nes[];
+
 #include "assets/text/message_data.h"
-extern const char _message_0xFFFC_nes[];
 
 #undef DEFINE_MESSAGE
+#undef DEFINE_MESSAGE_NES
 
 #endif

src/code/z_message_PAL.c

@@ -30,24 +30,28 @@ u16 sOcarinaSongBitFlags = 0; // ocarina bit flags
 MessageTableEntry sNesMessageEntryTable[] = {
 #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \
     { textId, (_SHIFTL(type, 4, 8) | _SHIFTL(yPos, 0, 8)), _message_##textId##_nes },
-#define DEFINE_MESSAGE_FFFC
+#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) DEFINE_MESSAGE(textId, type, yPos, nesMessage, , )
 #include "assets/text/message_data.h"
-#undef DEFINE_MESSAGE_FFFC
 #undef DEFINE_MESSAGE
+#undef DEFINE_MESSAGE_NES
     { 0xFFFF, 0, NULL },
 };
 
 const char* sGerMessageEntryTable[] = {
 #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) _message_##textId##_ger,
+#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage)
 #include "assets/text/message_data.h"
 #undef DEFINE_MESSAGE
+#undef DEFINE_MESSAGE_NES
     NULL,
 };
 
 const char* sFraMessageEntryTable[] = {
 #define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) _message_##textId##_fra,
+#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage)
 #include "assets/text/message_data.h"
 #undef DEFINE_MESSAGE
+#undef DEFINE_MESSAGE_NES
     NULL,
 };
 

tools/msgdis.py

@@ -376,20 +376,26 @@ def extract_all_text(text_out, staff_text_out):
     if text_out is not None:
         out = ""
         for message in dump_all_text():
-            if message[0] == 0xFFFF:
+            # Skip 0xFFFC and 0xFFFD because they are committed
+            # Skip 0xFFFF, the last entry
+            if message[0] in {0xFFFC, 0xFFFD, 0xFFFF}:
                 continue
 
-            if message[0] == 0xFFFC:
+            is_nes_message = message[0] == 0xFFFC
-                out += "#ifdef DEFINE_MESSAGE_FFFC\n"
+            if not is_nes_message:
-            out += f"DEFINE_MESSAGE(0x{message[0]:04X}, {textbox_type[message[1]]}, {textbox_ypos[message[2]]},"
+                out += "DEFINE_MESSAGE"
+            else:
+                out += "DEFINE_MESSAGE_NES"
+            out += f"(0x{message[0]:04X}, {textbox_type[message[1]]}, {textbox_ypos[message[2]]},"
             out += "\n"
-            out += f"{message[3]}" + ("\n" if message[3] != "" else "") + ","
+            out += f"{message[3]}"
-            out += "\n" if message[3] != "" else ""
+            if not is_nes_message:
-            out += f"{message[4]}" + ("\n" if message[4] != "" else "") + ","
+                out += ("\n" if message[3] != "" else "") + ","
-            out += "\n" if message[4] != "" else ""
+                out += "\n" if message[3] != "" else ""
-            out += f"{message[5]}\n)"
+                out += f"{message[4]}" + ("\n" if message[4] != "" else "") + ","
-            if message[0] == 0xFFFC:
+                out += "\n" if message[4] != "" else ""
-                out += "\n#endif"
+                out += f"{message[5]}"
+            out += "\n)"
             out += "\n\n"
 
         with open(text_out, "w", encoding="utf8") as outfile:

tools/msgenc.py

@@ -4,6 +4,7 @@
 #
 
 import argparse, ast, re
+import sys
 
 def read_charmap(path):
     with open(path) as infile:
@@ -50,22 +51,40 @@ def convert_text(text, charmap):
 
 def main():
     parser = argparse.ArgumentParser(description="Encode message_data_static text headers")
-    parser.add_argument("charmap", help="path to charmap file specifying custom encoding elements")
+    parser.add_argument(
-    parser.add_argument("input", help="path to file to be encoded")
+        "input",
-    parser.add_argument("output", help="encoded file")
+        help="path to file to be encoded, or - for stdin",
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        help="path to write encoded file, or - for stdout",
+        required=True,
+    )
+    parser.add_argument(
+        "--charmap",
+        help="path to charmap file specifying custom encoding elements",
+        required=True,
+    )
     args = parser.parse_args()
 
     charmap = read_charmap(args.charmap)
 
     text = ""
-    with open(args.input, "r") as infile:
+    if args.input == "-":
-        text = infile.read()
+        text = sys.stdin.read()
+    else:
+        with open(args.input, "r") as infile:
+            text = infile.read()
 
     text = remove_comments(text)
     text = convert_text(text, charmap)
 
-    with open(args.output, "w", encoding="raw_unicode_escape") as outfile:
+    if args.output == "-":
-        outfile.write(text)
+        sys.stdout.buffer.write(text.encode("raw_unicode_escape"))
+    else:
+        with open(args.output, "w", encoding="raw_unicode_escape") as outfile:
+            outfile.write(text)
 
 if __name__ == "__main__":
     main()