Reworked text extraction + add JP text extraction (#1980)

* Reworked text extraction + add JP text extraction * Format * Suggested changes * Correct address for gc-us sJpnMessageEntryTable Co-authored-by: cadmic <cadmic24@gmail.com> --------- Co-authored-by: cadmic <cadmic24@gmail.com>
2024-11-25 01:34:18 +00:00 · 2024-07-03 03:42:52 +01:00 · 2024-07-03 03:42:52 +01:00 · baf1e8c174
commit baf1e8c174
parent 264581ff3f
19 changed files with 1326 additions and 707 deletions
--- a/34
+++ b/34
@ -54,14 +54,16 @@ endif
 # Version-specific settings
 ifeq ($(VERSION),gc-us)
  DEBUG := 0
+  CPP_DEFINES += -DTEXT_LANGUAGE=TEXT_LANG_US_JP
 else ifeq ($(VERSION),gc-eu)
  DEBUG := 0
+  CPP_DEFINES += -DTEXT_LANGUAGE=TEXT_LANG_EU
 else ifeq ($(VERSION),gc-eu-mq)
  DEBUG := 0
-  CPP_DEFINES += -DOOT_MQ
+  CPP_DEFINES += -DTEXT_LANGUAGE=TEXT_LANG_EU -DOOT_MQ
 else ifeq ($(VERSION),gc-eu-mq-dbg)
  DEBUG := 1
-  CPP_DEFINES += -DOOT_MQ
+  CPP_DEFINES += -DTEXT_LANGUAGE=TEXT_LANG_EU -DOOT_MQ
 else
 $(error Unsupported version $(VERSION))
 endif
@ -380,7 +382,7 @@ setup: venv
 	$(MAKE) -C tools
 	$(PYTHON) tools/decompress_baserom.py $(VERSION)
 	$(PYTHON) tools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 --oot-version $(VERSION) -o $(EXTRACTED_DIR)/baserom
-	$(PYTHON) tools/msgdis.py --oot-version $(VERSION) --text-out $(EXTRACTED_DIR)/text/message_data.h --staff-text-out $(EXTRACTED_DIR)/text/message_data_staff.h
+	$(PYTHON) tools/msgdis.py $(VERSION)
 	$(PYTHON) extract_assets.py -v $(VERSION) -j$(N_THREADS)

 disasm:
@ -436,20 +438,30 @@ $(BUILD_DIR)/baserom/%.o: $(EXTRACTED_DIR)/baserom/%
 $(BUILD_DIR)/data/%.o: data/%.s
 	$(AS) $(ASFLAGS) $< -o $@

-$(BUILD_DIR)/assets/text/%.enc.h: assets/text/%.h $(EXTRACTED_DIR)/text/%.h assets/text/charmap.txt
-	$(CPP) $(CPPFLAGS) -I$(EXTRACTED_DIR) $< | $(PYTHON) tools/msgenc.py - --output $@ --charmap assets/text/charmap.txt
+$(BUILD_DIR)/assets/text/%.enc.jpn.h: assets/text/%.h $(EXTRACTED_DIR)/text/%.h assets/text/charmap.txt
+	$(CPP) $(CPPFLAGS) -I$(EXTRACTED_DIR) $< | $(PYTHON) tools/msgenc.py --encoding jpn --charmap assets/text/charmap.txt - $@
+
+$(BUILD_DIR)/assets/text/%.enc.nes.h: assets/text/%.h $(EXTRACTED_DIR)/text/%.h assets/text/charmap.txt
+	$(CPP) $(CPPFLAGS) -I$(EXTRACTED_DIR) $< | $(PYTHON) tools/msgenc.py --encoding nes --charmap assets/text/charmap.txt - $@

 # Dependencies for files including message data headers
 # TODO remove when full header dependencies are used.
-$(BUILD_DIR)/assets/text/fra_message_data_static.o: $(BUILD_DIR)/assets/text/message_data.enc.h
-$(BUILD_DIR)/assets/text/ger_message_data_static.o: $(BUILD_DIR)/assets/text/message_data.enc.h
-$(BUILD_DIR)/assets/text/nes_message_data_static.o: $(BUILD_DIR)/assets/text/message_data.enc.h
-$(BUILD_DIR)/assets/text/staff_message_data_static.o: $(BUILD_DIR)/assets/text/message_data_staff.enc.h
-$(BUILD_DIR)/src/code/z_message_PAL.o: $(BUILD_DIR)/assets/text/message_data.enc.h $(BUILD_DIR)/assets/text/message_data_staff.enc.h
+$(BUILD_DIR)/assets/text/jpn_message_data_static.o: $(BUILD_DIR)/assets/text/message_data.enc.jpn.h
+$(BUILD_DIR)/assets/text/nes_message_data_static.o: $(BUILD_DIR)/assets/text/message_data.enc.nes.h
+$(BUILD_DIR)/assets/text/ger_message_data_static.o: $(BUILD_DIR)/assets/text/message_data.enc.nes.h
+$(BUILD_DIR)/assets/text/fra_message_data_static.o: $(BUILD_DIR)/assets/text/message_data.enc.nes.h
+$(BUILD_DIR)/assets/text/staff_message_data_static.o: $(BUILD_DIR)/assets/text/message_data_staff.enc.nes.h
+$(BUILD_DIR)/src/code/z_message_PAL.o: assets/text/message_data.h assets/text/message_data_staff.h

 $(BUILD_DIR)/assets/text/%.o: assets/text/%.c
+ifneq ($(COMPILER),gcc)
+# Preprocess text with modern cpp for varargs macros
+	$(CPP) -undef -D_LANGUAGE_C -D__sgi $(CPPFLAGS) $(INC) $< -o $(@:.o=.c)
+	$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $(@:.o=.c)
+else
 	$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $<
-	$(OBJCOPY) -O binary $@ $@.bin
+endif
+	$(OBJCOPY) -O binary -j.rodata $@ $@.bin

 $(BUILD_DIR)/assets/%.o: $(EXTRACTED_DIR)/assets/%.c
 	$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $<
--- a/assets/text/charmap.txt
+++ b/assets/text/charmap.txt
@ -1,50 +1,58 @@
+# Determines how certain text sequences should be encoded. The text sequence is
+# converted to either the first or second tuple element based on whether the
+# target encoding is the "wide" encoding. The first element is for the non-wide
+# encoding, used for all languages besides JP, while the second element is for
+# the wide encoding, used for JP.
 {
-    '\n' : 0x01,
+    '\n'            : (0x01, 0x000A),

-    '‾' : 0x7F,
-    'À' : 0x80,
-    'î' : 0x81,
-    'Â' : 0x82,
-    'Ä' : 0x83,
-    'Ç' : 0x84,
-    'È' : 0x85,
-    'É' : 0x86,
-    'Ê' : 0x87,
-    'Ë' : 0x88,
-    'Ï' : 0x89,
-    'Ô' : 0x8A,
-    'Ö' : 0x8B,
-    'Ù' : 0x8C,
-    'Û' : 0x8D,
-    'Ü' : 0x8E,
-    'ß' : 0x8F,
-    'à' : 0x90,
-    'á' : 0x91,
-    'â' : 0x92,
-    'ä' : 0x93,
-    'ç' : 0x94,
-    'è' : 0x95,
-    'é' : 0x96,
-    'ê' : 0x97,
-    'ë' : 0x98,
-    'ï' : 0x99,
-    'ô' : 0x9A,
-    'ö' : 0x9B,
-    'ù' : 0x9C,
-    'û' : 0x9D,
-    'ü' : 0x9E,
+    '[A]'           : (0x9F, 0x839F),
+    '[B]'           : (0xA0, 0x83A0),
+    '[C]'           : (0xA1, 0x83A1),
+    '[L]'           : (0xA2, 0x83A2),
+    '[R]'           : (0xA3, 0x83A3),
+    '[Z]'           : (0xA4, 0x83A4),
+    '[C-Up]'        : (0xA5, 0x83A5),
+    '[C-Down]'      : (0xA6, 0x83A6),
+    '[C-Left]'      : (0xA7, 0x83A7),
+    '[C-Right]'     : (0xA8, 0x83A8),
+    '▼'             : (0xA9, 0x83A9),
+    '[Control-Pad]' : (0xAA, 0x83AA),
+    '[D-Pad]'       : (0xAB, None),

-    '[A]' : 0x9F,
-    '[B]' : 0xA0,
-    '[C]' : 0xA1,
-    '[L]' : 0xA2,
-    '[R]' : 0xA3,
-    '[Z]' : 0xA4,
-    '[C-Up]' : 0xA5,
-    '[C-Down]' : 0xA6,
-    '[C-Left]' : 0xA7,
-    '[C-Right]' : 0xA8,
-    '▼' : 0xA9,
-    '[Control-Pad]' : 0xAA,
-    '[D-Pad]' : 0xAB,
+    # Possibly from a SHIFT-JIS extension, python doesn't have builtin support
+    '┯'             : (None, 0x86D3),
+
+    '‾'             : (0x7F, None),
+    'À'             : (0x80, None),
+    'î'             : (0x81, None),
+    'Â'             : (0x82, None),
+    'Ä'             : (0x83, None),
+    'Ç'             : (0x84, None),
+    'È'             : (0x85, None),
+    'É'             : (0x86, None),
+    'Ê'             : (0x87, None),
+    'Ë'             : (0x88, None),
+    'Ï'             : (0x89, None),
+    'Ô'             : (0x8A, None),
+    'Ö'             : (0x8B, None),
+    'Ù'             : (0x8C, None),
+    'Û'             : (0x8D, None),
+    'Ü'             : (0x8E, None),
+    'ß'             : (0x8F, None),
+    'à'             : (0x90, None),
+    'á'             : (0x91, None),
+    'â'             : (0x92, None),
+    'ä'             : (0x93, None),
+    'ç'             : (0x94, None),
+    'è'             : (0x95, None),
+    'é'             : (0x96, None),
+    'ê'             : (0x97, None),
+    'ë'             : (0x98, None),
+    'ï'             : (0x99, None),
+    'ô'             : (0x9A, None),
+    'ö'             : (0x9B, None),
+    'ù'             : (0x9C, None),
+    'û'             : (0x9D, None),
+    'ü'             : (0x9E, None),
 }
--- a/assets/text/fra_message_data_static.c
+++ b/assets/text/fra_message_data_static.c
@ -2,9 +2,16 @@

 #include "message_data_fmt.h"

-#define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \
-    const char _message_##textId##_fra[sizeof(fraMessage)] = { fraMessage END };
+#define DEFINE_MESSAGE(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    const char _message_##textId##_fra[] = fraMessage;

-#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage)
+#define DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Present */ const char _message_##textId##_fra[] = fraMessage;
+#define DEFINE_MESSAGE_JPN(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Not Present */

-#include "assets/text/message_data.enc.h"
+// Font Message
+#define DEFINE_MESSAGE_FFFC(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Not Present */
+
+#include "assets/text/message_data.enc.nes.h"
--- a/assets/text/ger_message_data_static.c
+++ b/assets/text/ger_message_data_static.c
@ -2,9 +2,16 @@

 #include "message_data_fmt.h"

-#define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \
-    const char _message_##textId##_ger[sizeof(gerMessage)] = { gerMessage END };
+#define DEFINE_MESSAGE(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    const char _message_##textId##_ger[] = gerMessage;

-#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage)
+#define DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Present */ const char _message_##textId##_ger[] = gerMessage;
+#define DEFINE_MESSAGE_JPN(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Not Present */

-#include "assets/text/message_data.enc.h"
+// Font Message
+#define DEFINE_MESSAGE_FFFC(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Not Present */
+
+#include "assets/text/message_data.enc.nes.h"
--- a/assets/text/jpn_message_data_static.c
+++ b/assets/text/jpn_message_data_static.c
@ -0,0 +1,17 @@
+#define MESSAGE_DATA_STATIC
+#define MESSAGE_WCHAR
+#include "message_data_fmt.h"
+
+#define DEFINE_MESSAGE(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    const char _message_##textId##_jpn[] = jpnMessage;
+
+#define DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Not Present */
+#define DEFINE_MESSAGE_JPN(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Present */ const char _message_##textId##_jpn[] = jpnMessage;
+
+// Font Message
+#define DEFINE_MESSAGE_FFFC(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Present */ const char _message_##textId##_jpn[] = jpnMessage;
+
+#include "assets/text/message_data.enc.jpn.h"
--- a/assets/text/message_data.h
+++ b/assets/text/message_data.h
@ -5,7 +5,23 @@
 * Message 0xFFFD must be last to not break the message debugger (see R_MESSAGE_DEBUGGER_TEXTID).
 * Message 0xFFFC must be immediately before message 0xFFFD to not break Font_LoadOrderedFont.
 */
-DEFINE_MESSAGE_NES(0xFFFC, TEXTBOX_TYPE_BLACK, TEXTBOX_POS_VARIABLE,
+DEFINE_MESSAGE_FFFC(0xFFFC, TEXTBOX_TYPE_BLACK, TEXTBOX_POS_VARIABLE,
+MSG(
+"０１２３４５６７８９あいうえおかきくけこ\n"
+"さしすせそたちつてとなにぬねのはひふへほ\n"
+"まみむめもやゆよらりるれろわをんぁぃぅぇ\n"
+"ぉっゃゅょがぎぐげござじずぜぞだぢづでど\n"
+"ばびぶべぼぱぴぷぺぽアイウエオカキクケコ\n"
+"サシスセソタチツテトナニヌネノハヒフヘホ\n"
+"マミムメモヤユヨラリルレロワヲンァィゥェ\n"
+"ォッャュョガギグゲゴザジズゼゾダヂヅデド\n"
+"バビブベボパピプペポヴＡＢＣＤＥＦＧＨＩ\n"
+"ＪＫＬＭＮＯＰＱＲＳＴＵＶＷＸＹＺａｂｃ\n"
+"ｄｅｆｇｈｉｊｋｌｍｎｏｐｑｒｓｔｕｖｗ\n"
+"ｘｙｚ　┯？！：−（）゛゜，．／"
+)
+,
+MSG(
 "0123456789\n"
 "ABCDEFGHIJKLMN\n"
 "OPQRSTUVWXYZ\n"
@ -13,4 +29,9 @@ DEFINE_MESSAGE_NES(0xFFFC, TEXTBOX_TYPE_BLACK, TEXTBOX_POS_VARIABLE,
 "opqrstuvwxyz\n"
 " -.\n"
 )
-DEFINE_MESSAGE(0xFFFD, TEXTBOX_TYPE_BLACK, TEXTBOX_POS_VARIABLE, "", "", "")
+,
+MSG(/* UNUSED */)
+,
+MSG(/* UNUSED */)
+)
+DEFINE_MESSAGE(0xFFFD, TEXTBOX_TYPE_BLACK, TEXTBOX_POS_VARIABLE, MSG(), MSG(), MSG(), MSG())
--- a/assets/text/nes_message_data_static.c
+++ b/assets/text/nes_message_data_static.c
@ -2,9 +2,22 @@

 #include "message_data_fmt.h"

-#define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \
-    const char _message_##textId##_nes[sizeof(nesMessage)] = { nesMessage END };
+#define DEFINE_MESSAGE(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    const char _message_##textId##_nes[] = nesMessage;

-#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) DEFINE_MESSAGE(textId, type, yPos, nesMessage, , )
+#define DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Present */ const char _message_##textId##_nes[] = nesMessage;
+#define DEFINE_MESSAGE_JPN(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Not Present */

-#include "assets/text/message_data.enc.h"
+#if (TEXT_LANGUAGE == TEXT_LANG_US_JP)
+// On US/JP versions, the font message is with JPN
+#define DEFINE_MESSAGE_FFFC(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Not Present */
+#else
+// On EU versions, the font message is with NES
+#define DEFINE_MESSAGE_FFFC(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Present */ const char _message_##textId##_nes[] = nesMessage;
+#endif
+
+#include "assets/text/message_data.enc.nes.h"
--- a/assets/text/staff_message_data_static.c
+++ b/assets/text/staff_message_data_static.c
@ -3,6 +3,6 @@
 #include "message_data_fmt.h"

 #define DEFINE_MESSAGE(textId, type, yPos, staffMessage) \
-    const char _message_##textId##_staff[sizeof(staffMessage)] = { staffMessage END };
+    const char _message_##textId##_staff[] = staffMessage;

-#include "assets/text/message_data_staff.enc.h"
+#include "assets/text/message_data_staff.enc.nes.h"
--- a/baseroms/gc-eu-mq-dbg/config.yml
+++ b/baseroms/gc-eu-mq-dbg/config.yml
@ -1,4 +1,5 @@
 dmadata_start: 0x12F70
+text_lang_pal: true
 variables:
  gMtxClear: 0x8012DB20
  sNesMessageEntryTable: 0x8014B320
--- a/baseroms/gc-eu-mq/config.yml
+++ b/baseroms/gc-eu-mq/config.yml
@ -1,4 +1,5 @@
 dmadata_start: 0x7170
+text_lang_pal: true
 variables:
  gMtxClear: 0x800FBC00
  sNesMessageEntryTable: 0x801077F0
--- a/baseroms/gc-eu/config.yml
+++ b/baseroms/gc-eu/config.yml
@ -1,4 +1,5 @@
 dmadata_start: 0x7170
+text_lang_pal: true
 variables:
  gMtxClear: 0x800FBC20
  sNesMessageEntryTable: 0x80107810
--- a/baseroms/gc-us/config.yml
+++ b/baseroms/gc-us/config.yml
@ -1,7 +1,8 @@
 dmadata_start: 0x7170
+text_lang_pal: false
 variables:
  gMtxClear: 0x800FE2A0
-  sJpnMessageEntryTable: 0x80107810
+  sJpnMessageEntryTable: 0x80109E8C
  sNesMessageEntryTable: 0x8010DFCC
  sStaffMessageEntryTable: 0x801121EC
  sShadowTex: 0x80A74130
--- a/include/message_data_fmt.h
+++ b/include/message_data_fmt.h
@ -1,180 +1,189 @@
 #ifndef MESSAGE_DATA_FMT_H
 #define MESSAGE_DATA_FMT_H

-/*
- * Macros to create both a constant and a string literal from a magic value
- *   The constants are used in code files when parsing text for various purposes
- *   The strings are used in the message_data_static files themselves, as you can only concat strings with other strings
- */
+#include "z64save.h" // For highscores
+#include "z64item.h" // For item ids
+#include "sfx.h" // For sfx ids

-#ifndef GLUE
-#define GLUE(a, b) a##b
-#endif
-
-#define STRINGIFY(s) #s
-#define EXPAND_AND_STRINGIFY(s) STRINGIFY(s)
-
-#define HEX(N) GLUE(0x, N)
-#define STR(N) EXPAND_AND_STRINGIFY(GLUE(\x, N))
+#define TEXT_LANG_US_JP 0
+#define TEXT_LANG_EU    1

 /*
 * Text control characters
 */

-// Control character magic values, in 2-digit hex without prefix
+// Non-Wide (nes/ger/fra)

-#define CTRL_NEWLINE                 01
-#define CTRL_END                     02
-#define CTRL_BOX_BREAK               04
-#define CTRL_COLOR                   05
-#define CTRL_SHIFT                   06
-#define CTRL_TEXTID                  07
-#define CTRL_QUICKTEXT_ENABLE        08
-#define CTRL_QUICKTEXT_DISABLE       09
-#define CTRL_PERSISTENT              0A
-#define CTRL_EVENT                   0B
-#define CTRL_BOX_BREAK_DELAYED       0C
-#define CTRL_AWAIT_BUTTON_PRESS      0D
-#define CTRL_FADE                    0E
-#define CTRL_NAME                    0F
-#define CTRL_OCARINA                 10
-#define CTRL_FADE2                   11
-#define CTRL_SFX                     12
-#define CTRL_ITEM_ICON               13
-#define CTRL_TEXT_SPEED              14
-#define CTRL_BACKGROUND              15
-#define CTRL_MARATHON_TIME           16
-#define CTRL_RACE_TIME               17
-#define CTRL_POINTS                  18
-#define CTRL_TOKENS                  19
-#define CTRL_UNSKIPPABLE             1A
-#define CTRL_TWO_CHOICE              1B
-#define CTRL_THREE_CHOICE            1C
-#define CTRL_FISH_INFO               1D
-#define CTRL_HIGHSCORE               1E
-#define CTRL_TIME                    1F
+#define MESSAGE_NEWLINE                    0x01
+#define MESSAGE_END                        0x02
+#define MESSAGE_BOX_BREAK                  0x04
+#define MESSAGE_COLOR                      0x05
+#define MESSAGE_SHIFT                      0x06
+#define MESSAGE_TEXTID                     0x07
+#define MESSAGE_QUICKTEXT_ENABLE           0x08
+#define MESSAGE_QUICKTEXT_DISABLE          0x09
+#define MESSAGE_PERSISTENT                 0x0A
+#define MESSAGE_EVENT                      0x0B
+#define MESSAGE_BOX_BREAK_DELAYED          0x0C
+#define MESSAGE_AWAIT_BUTTON_PRESS         0x0D
+#define MESSAGE_FADE                       0x0E
+#define MESSAGE_NAME                       0x0F
+#define MESSAGE_OCARINA                    0x10
+#define MESSAGE_FADE2                      0x11
+#define MESSAGE_SFX                        0x12
+#define MESSAGE_ITEM_ICON                  0x13
+#define MESSAGE_TEXT_SPEED                 0x14
+#define MESSAGE_BACKGROUND                 0x15
+#define MESSAGE_MARATHON_TIME              0x16
+#define MESSAGE_RACE_TIME                  0x17
+#define MESSAGE_POINTS                     0x18
+#define MESSAGE_TOKENS                     0x19
+#define MESSAGE_UNSKIPPABLE                0x1A
+#define MESSAGE_TWO_CHOICE                 0x1B
+#define MESSAGE_THREE_CHOICE               0x1C
+#define MESSAGE_FISH_INFO                  0x1D
+#define MESSAGE_HIGHSCORE                  0x1E
+#define MESSAGE_TIME                       0x1F
+
+// Wide (jpn)
+
+#define MESSAGE_WIDE_NEWLINE               0x000A
+#define MESSAGE_WIDE_END                   0x8170
+#define MESSAGE_WIDE_BOX_BREAK             0x81A5
+#define MESSAGE_WIDE_COLOR                 0x000B
+#define MESSAGE_WIDE_SHIFT                 0x86C7
+#define MESSAGE_WIDE_TEXTID                0x81CB
+#define MESSAGE_WIDE_QUICKTEXT_ENABLE      0x8189
+#define MESSAGE_WIDE_QUICKTEXT_DISABLE     0x818A
+#define MESSAGE_WIDE_PERSISTENT            0x86C8
+#define MESSAGE_WIDE_EVENT                 0x819F
+#define MESSAGE_WIDE_BOX_BREAK_DELAYED     0x81A3
+#define MESSAGE_WIDE_AWAIT_BUTTON_PRESS    0x81A4
+#define MESSAGE_WIDE_FADE                  0x819E
+#define MESSAGE_WIDE_NAME                  0x874F
+#define MESSAGE_WIDE_OCARINA               0x81F0
+#define MESSAGE_WIDE_FADE2                 0x81F4
+#define MESSAGE_WIDE_SFX                   0x81F3
+#define MESSAGE_WIDE_ITEM_ICON             0x819A
+#define MESSAGE_WIDE_TEXT_SPEED            0x86C9
+#define MESSAGE_WIDE_BACKGROUND            0x86B3
+#define MESSAGE_WIDE_MARATHON_TIME         0x8791
+#define MESSAGE_WIDE_RACE_TIME             0x8792
+#define MESSAGE_WIDE_POINTS                0x879B
+#define MESSAGE_WIDE_TOKENS                0x86A3
+#define MESSAGE_WIDE_UNSKIPPABLE           0x8199
+#define MESSAGE_WIDE_TWO_CHOICE            0x81BC
+#define MESSAGE_WIDE_THREE_CHOICE          0x81B8
+#define MESSAGE_WIDE_FISH_INFO             0x86A4
+#define MESSAGE_WIDE_HIGHSCORE             0x869F
+#define MESSAGE_WIDE_TIME                  0x81A1

 /*
 *  Colors
 */

-#define COLOR_STR(N) EXPAND_AND_STRINGIFY(GLUE(\x4, N))
+typedef enum {
+    TEXT_COLOR_DEFAULT,
+    TEXT_COLOR_RED,
+    TEXT_COLOR_ADJUSTABLE,
+    TEXT_COLOR_BLUE,
+    TEXT_COLOR_LIGHTBLUE,
+    TEXT_COLOR_PURPLE,
+    TEXT_COLOR_YELLOW,
+    TEXT_COLOR_BLACK
+} TextColor;

-// Color magic values, in single-digit hex without prefix
+/*
+ * Background
+ */

-#define CTRL_DEFAULT    0
-#define CTRL_RED        1
-#define CTRL_ADJUSTABLE 2
-#define CTRL_BLUE       3
-#define CTRL_LIGHTBLUE  4
-#define CTRL_PURPLE     5
-#define CTRL_YELLOW     6
-#define CTRL_BLACK      7
+typedef enum {
+    TEXTBOX_BG_X_LEFT,
+    TEXTBOX_BG_X_RIGHT
+} TextboxBackgroundIndex;
+
+typedef enum {
+    TEXTBOX_BG_FGCOL_WHITE,
+    TEXTBOX_BG_FGCOL_DARK_RED,
+    TEXTBOX_BG_FGCOL_ORANGE,
+    TEXTBOX_BG_FGCOL_WHITE_3,
+    TEXTBOX_BG_FGCOL_WHITE_4,
+    TEXTBOX_BG_FGCOL_WHITE_5,
+    TEXTBOX_BG_FGCOL_WHITE_6,
+    TEXTBOX_BG_FGCOL_WHITE_7
+} TextboxBackgroundForegroundColor;
+
+typedef enum {
+    TEXTBOX_BG_BGCOL_BLACK,
+    TEXTBOX_BG_BGCOL_GOLD,
+    TEXTBOX_BG_BGCOL_BLACK_2,
+    TEXTBOX_BG_BGCOL_BLACK_3
+} TextboxBackgroundBackgroundColor;
+
+typedef enum {
+    TEXTBOX_BG_Y_OFFSET_1,
+    TEXTBOX_BG_Y_OFFSET_2
+} TextboxBackgroundYOffsetIndex;

 #ifdef MESSAGE_DATA_STATIC
 // For use in message_data_static files

-#define ARG(x) x
+// Encoded text consists of an array of bytes. Since it's in a macro it must be wrapped in a varargs macro so that each
+// byte is not treated as a separate macro argument to DEFINE_MESSAGE. IDO doesn't support varargs macros, however we
+// preprocess the message_data_static files with modern cpp instead. See the makefile rule for assets/text/
+# define MSG(...)  { __VA_ARGS__ END }
+
+// Encoding helpers
+# define ARG2(x) (((x) >> 8) & 0xFF), (((x) >> 0) & 0xFF),
+# ifdef MESSAGE_WCHAR
+#  define ARG1(x) 0x00, (x),
+#  define ARGC(x) 0x0C, (TEXT_COLOR_ ## x),
+#  define CTRL_BASE(name) ARG2(MESSAGE_WIDE_ ## name)
+# else
+#  define ARG1(x) (x),
+#  define ARGC(x) (0x40 | (TEXT_COLOR_ ## x)),
+#  define CTRL_BASE(name) ARG1(MESSAGE_ ## name)
+# endif
+# define ARGB1(x) ARG1(TEXTBOX_BG_ ## x)
+# define ARGB2(a,b,c,d) (((TEXTBOX_BG_FGCOL_ ## a) << 4) | ((TEXTBOX_BG_BGCOL_ ## b) << 0)), (((TEXTBOX_BG_Y_OFFSET_ ## c) << 4) | ((d) << 0)),
+
+/*
+ * Control characters
+ */

 // while a control character, newlines are handled in the charmap conversion
 // stage to allow normal newline \n usage in message_data_static files
-#define NEWLINE                 STR(CTRL_NEWLINE)
-#define END                     STR(CTRL_END)
-#define BOX_BREAK               STR(CTRL_BOX_BREAK)
-#define COLOR(x)                STR(CTRL_COLOR)                ARG(x) // 1
-#define SHIFT(x)                STR(CTRL_SHIFT)                ARG(x) // 1
-#define TEXTID(x)               STR(CTRL_TEXTID)               ARG(x) // 2
-#define QUICKTEXT_ENABLE        STR(CTRL_QUICKTEXT_ENABLE)
-#define QUICKTEXT_DISABLE       STR(CTRL_QUICKTEXT_DISABLE)
-#define PERSISTENT              STR(CTRL_PERSISTENT)
-#define EVENT                   STR(CTRL_EVENT)
-#define BOX_BREAK_DELAYED(x)    STR(CTRL_BOX_BREAK_DELAYED)    ARG(x) // 1
-#define AWAIT_BUTTON_PRESS      STR(CTRL_AWAIT_BUTTON_PRESS)
-#define FADE(x)                 STR(CTRL_FADE)                 ARG(x) // 1
-#define NAME                    STR(CTRL_NAME)
-#define OCARINA                 STR(CTRL_OCARINA)
-#define FADE2(x)                STR(CTRL_FADE2)                ARG(x) // 2
-#define SFX(x)                  STR(CTRL_SFX)                  ARG(x) // 2
-#define ITEM_ICON(x)            STR(CTRL_ITEM_ICON)            ARG(x) // 1
-#define TEXT_SPEED(x)           STR(CTRL_TEXT_SPEED)           ARG(x) // 1
-#define BACKGROUND(x,y,z)       STR(CTRL_BACKGROUND)           ARG(x) ARG(y) ARG(z)
-#define MARATHON_TIME           STR(CTRL_MARATHON_TIME)
-#define RACE_TIME               STR(CTRL_RACE_TIME)
-#define POINTS                  STR(CTRL_POINTS)
-#define TOKENS                  STR(CTRL_TOKENS)
-#define UNSKIPPABLE             STR(CTRL_UNSKIPPABLE)
-#define TWO_CHOICE              STR(CTRL_TWO_CHOICE)
-#define THREE_CHOICE            STR(CTRL_THREE_CHOICE)
-#define FISH_INFO               STR(CTRL_FISH_INFO)
-#define HIGHSCORE(x)            STR(CTRL_HIGHSCORE)            ARG(x) // 1
-#define TIME                    STR(CTRL_TIME)
+# define NEWLINE                                            CTRL_BASE(NEWLINE)
+# define END                                                CTRL_BASE(END)
+# define BOX_BREAK                                          CTRL_BASE(BOX_BREAK)
+# define COLOR(color)                                       CTRL_BASE(COLOR)                ARGC(color)
+# define SHIFT(amount)                                      CTRL_BASE(SHIFT)                ARG1(amount)
+# define TEXTID(textId)                                     CTRL_BASE(TEXTID)               ARG2(textId)
+# define QUICKTEXT_ENABLE                                   CTRL_BASE(QUICKTEXT_ENABLE)
+# define QUICKTEXT_DISABLE                                  CTRL_BASE(QUICKTEXT_DISABLE)
+# define PERSISTENT                                         CTRL_BASE(PERSISTENT)
+# define EVENT                                              CTRL_BASE(EVENT)
+# define BOX_BREAK_DELAYED(delay)                           CTRL_BASE(BOX_BREAK_DELAYED)    ARG1(delay)
+# define AWAIT_BUTTON_PRESS                                 CTRL_BASE(AWAIT_BUTTON_PRESS)
+# define FADE(delay)                                        CTRL_BASE(FADE)                 ARG1(delay)
+# define NAME                                               CTRL_BASE(NAME)
+# define OCARINA                                            CTRL_BASE(OCARINA)
+# define FADE2(delay)                                       CTRL_BASE(FADE2)                ARG2(delay)
+# define SFX(sfxId)                                         CTRL_BASE(SFX)                  ARG2(sfxId)
+# define ITEM_ICON(itemId)                                  CTRL_BASE(ITEM_ICON)            ARG1(itemId)
+# define TEXT_SPEED(amount)                                 CTRL_BASE(TEXT_SPEED)           ARG1(amount)
+# define BACKGROUND(bgIdx, fgColor, bgColor, yOffset, unk)  CTRL_BASE(BACKGROUND)           ARGB1(bgIdx) ARGB2(fgColor, bgColor, yOffset, unk)
+# define MARATHON_TIME                                      CTRL_BASE(MARATHON_TIME)
+# define RACE_TIME                                          CTRL_BASE(RACE_TIME)
+# define POINTS                                             CTRL_BASE(POINTS)
+# define TOKENS                                             CTRL_BASE(TOKENS)
+# define UNSKIPPABLE                                        CTRL_BASE(UNSKIPPABLE)
+# define TWO_CHOICE                                         CTRL_BASE(TWO_CHOICE)
+# define THREE_CHOICE                                       CTRL_BASE(THREE_CHOICE)
+# define FISH_INFO                                          CTRL_BASE(FISH_INFO)
+# define HIGHSCORE(highscore)                               CTRL_BASE(HIGHSCORE)            ARG1(highscore)
+# define TIME                                               CTRL_BASE(TIME)

-/*
- * Highscore values as strings, for code references the HighScores
- * enum should be used.
- */
-
-#define HS_HORSE_ARCHERY "\x00"
-#define HS_POE_POINTS    "\x01"
-#define HS_LARGEST_FISH  "\x02"
-#define HS_HORSE_RACE    "\x03"
-#define HS_MARATHON      "\x04"
-#define HS_DAMPE_RACE    "\x06"
-
-/*
- * Color values as strings
- */
-
-#define DEFAULT     COLOR_STR(CTRL_DEFAULT)
-#define RED         COLOR_STR(CTRL_RED)
-#define ADJUSTABLE  COLOR_STR(CTRL_ADJUSTABLE)
-#define BLUE        COLOR_STR(CTRL_BLUE)
-#define LIGHTBLUE   COLOR_STR(CTRL_LIGHTBLUE)
-#define PURPLE      COLOR_STR(CTRL_PURPLE)
-#define YELLOW      COLOR_STR(CTRL_YELLOW)
-#define BLACK       COLOR_STR(CTRL_BLACK)
-
-#else
-// For use in code files
-#define MSGCOL_DEFAULT      HEX(CTRL_DEFAULT)
-#define MSGCOL_RED          HEX(CTRL_RED)
-#define MSGCOL_ADJUSTABLE   HEX(CTRL_ADJUSTABLE)
-#define MSGCOL_BLUE         HEX(CTRL_BLUE)
-#define MSGCOL_LIGHTBLUE    HEX(CTRL_LIGHTBLUE)
-#define MSGCOL_PURPLE       HEX(CTRL_PURPLE)
-#define MSGCOL_YELLOW       HEX(CTRL_YELLOW)
-#define MSGCOL_BLACK        HEX(CTRL_BLACK)
-
-#define MESSAGE_NEWLINE                 HEX(CTRL_NEWLINE)
-#define MESSAGE_END                     HEX(CTRL_END)
-#define MESSAGE_BOX_BREAK               HEX(CTRL_BOX_BREAK)
-#define MESSAGE_COLOR                   HEX(CTRL_COLOR)
-#define MESSAGE_SHIFT                   HEX(CTRL_SHIFT)
-#define MESSAGE_TEXTID                  HEX(CTRL_TEXTID)
-#define MESSAGE_QUICKTEXT_ENABLE        HEX(CTRL_QUICKTEXT_ENABLE)
-#define MESSAGE_QUICKTEXT_DISABLE       HEX(CTRL_QUICKTEXT_DISABLE)
-#define MESSAGE_PERSISTENT              HEX(CTRL_PERSISTENT)
-#define MESSAGE_EVENT                   HEX(CTRL_EVENT)
-#define MESSAGE_BOX_BREAK_DELAYED       HEX(CTRL_BOX_BREAK_DELAYED)
-#define MESSAGE_AWAIT_BUTTON_PRESS      HEX(CTRL_AWAIT_BUTTON_PRESS)
-#define MESSAGE_FADE                    HEX(CTRL_FADE)
-#define MESSAGE_NAME                    HEX(CTRL_NAME)
-#define MESSAGE_OCARINA                 HEX(CTRL_OCARINA)
-#define MESSAGE_FADE2                   HEX(CTRL_FADE2)
-#define MESSAGE_SFX                     HEX(CTRL_SFX)
-#define MESSAGE_ITEM_ICON               HEX(CTRL_ITEM_ICON)
-#define MESSAGE_TEXT_SPEED              HEX(CTRL_TEXT_SPEED)
-#define MESSAGE_BACKGROUND              HEX(CTRL_BACKGROUND)
-#define MESSAGE_MARATHON_TIME           HEX(CTRL_MARATHON_TIME)
-#define MESSAGE_RACE_TIME               HEX(CTRL_RACE_TIME)
-#define MESSAGE_POINTS                  HEX(CTRL_POINTS)
-#define MESSAGE_TOKENS                  HEX(CTRL_TOKENS)
-#define MESSAGE_UNSKIPPABLE             HEX(CTRL_UNSKIPPABLE)
-#define MESSAGE_TWO_CHOICE              HEX(CTRL_TWO_CHOICE)
-#define MESSAGE_THREE_CHOICE            HEX(CTRL_THREE_CHOICE)
-#define MESSAGE_FISH_INFO               HEX(CTRL_FISH_INFO)
-#define MESSAGE_HIGHSCORE               HEX(CTRL_HIGHSCORE)
-#define MESSAGE_TIME                    HEX(CTRL_TIME)
 #endif

 #endif
--- a/include/message_data_static.h
+++ b/include/message_data_static.h
@ -1,7 +1,7 @@
 #ifndef MESSAGE_DATA_STATIC_H
 #define MESSAGE_DATA_STATIC_H

-#include "global.h"
+#include "ultra64.h"
 #include "message_data_fmt.h"

 typedef enum {
@ -35,6 +35,49 @@ typedef struct {
 *  Message Symbol Declarations
 */

+/* Non-Credits Messages */
+
+#if (TEXT_LANGUAGE == TEXT_LANG_US_JP)
+#define DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    extern const char _message_##textId##_nes[];
+
+#define DEFINE_MESSAGE_JPN(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    extern const char _message_##textId##_jpn[];
+
+#define DEFINE_MESSAGE_FFFC(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    DEFINE_MESSAGE_JPN(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage)
+
+#define FONT_MESSAGE_OFFSET (_message_0xFFFC_jpn - (const char*)_jpn_message_data_staticSegmentStart)
+#define FONT_MESSAGE_LENGTH (_message_0xFFFD_jpn - _message_0xFFFC_jpn)
+#else
+#define DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    extern const char _message_##textId##_nes[]; \
+    extern const char _message_##textId##_ger[]; \
+    extern const char _message_##textId##_fra[];
+
+#define DEFINE_MESSAGE_JPN(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Empty */
+
+#define DEFINE_MESSAGE_FFFC(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage)
+
+#define FONT_MESSAGE_OFFSET (_message_0xFFFC_nes - (const char*)_nes_message_data_staticSegmentStart)
+#define FONT_MESSAGE_LENGTH (_message_0xFFFD_nes - _message_0xFFFC_nes)
+#endif
+
+#define DEFINE_MESSAGE(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    DEFINE_MESSAGE_JPN(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage)
+
+#include "assets/text/message_data.h"
+
+#undef DEFINE_MESSAGE
+#undef DEFINE_MESSAGE_NES
+#undef DEFINE_MESSAGE_JPN
+#undef DEFINE_MESSAGE_FFFC
+
+/* Credits Messages */
+
 #define DEFINE_MESSAGE(textId, type, yPos, staffMessage) \
    extern const char _message_##textId##_staff[];

@ -42,17 +85,4 @@ typedef struct {

 #undef DEFINE_MESSAGE

-#define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \
-    extern const char _message_##textId##_nes[]; \
-    extern const char _message_##textId##_ger[]; \
-    extern const char _message_##textId##_fra[];
-
-#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) \
-    extern const char _message_##textId##_nes[];
-
-#include "assets/text/message_data.h"
-
-#undef DEFINE_MESSAGE
-#undef DEFINE_MESSAGE_NES
-
 #endif
--- a/src/code/z_kanfont.c
+++ b/src/code/z_kanfont.c
@ -38,8 +38,8 @@ void Font_LoadOrderedFont(Font* font) {
    s32 fontBufIndex;
    u32 offset;

-    font->msgOffset = _message_0xFFFC_nes - (const char*)_nes_message_data_staticSegmentStart;
-    len = font->msgLength = _message_0xFFFD_nes - _message_0xFFFC_nes;
+    font->msgOffset = FONT_MESSAGE_OFFSET;
+    len = font->msgLength = FONT_MESSAGE_LENGTH;

    DMA_REQUEST_SYNC(font->msgBuf, (uintptr_t)_nes_message_data_staticSegmentRomStart + font->msgOffset, len,
                     "../z_kanfont.c", 122);
--- a/src/code/z_message_PAL.c
+++ b/src/code/z_message_PAL.c
@ -28,30 +28,49 @@ s16 sMessageHasSetSfx = false;
 u16 sOcarinaSongBitFlags = 0; // ocarina bit flags

 MessageTableEntry sNesMessageEntryTable[] = {
-#define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) \
+#define DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
    { textId, (_SHIFTL(type, 4, 8) | _SHIFTL(yPos, 0, 8)), _message_##textId##_nes },
-#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage) DEFINE_MESSAGE(textId, type, yPos, nesMessage, , )
+#define DEFINE_MESSAGE_JPN(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) /* Not Present */
+#define DEFINE_MESSAGE(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    DEFINE_MESSAGE_JPN(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage)
+#define DEFINE_MESSAGE_FFFC(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    /* Present */ DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage)
 #include "assets/text/message_data.h"
 #undef DEFINE_MESSAGE
 #undef DEFINE_MESSAGE_NES
+#undef DEFINE_MESSAGE_JPN
+#undef DEFINE_MESSAGE_FFFC
    { 0xFFFF, 0, NULL },
 };

 const char* sGerMessageEntryTable[] = {
-#define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) _message_##textId##_ger,
-#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage)
+#define DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) _message_##textId##_ger,
+#define DEFINE_MESSAGE_JPN(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) /* Not Present */
+#define DEFINE_MESSAGE(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    DEFINE_MESSAGE_JPN(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage)
+#define DEFINE_MESSAGE_FFFC(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) /* Not Present */
 #include "assets/text/message_data.h"
 #undef DEFINE_MESSAGE
 #undef DEFINE_MESSAGE_NES
+#undef DEFINE_MESSAGE_JPN
+#undef DEFINE_MESSAGE_FFFC
    NULL,
 };

 const char* sFraMessageEntryTable[] = {
-#define DEFINE_MESSAGE(textId, type, yPos, nesMessage, gerMessage, fraMessage) _message_##textId##_fra,
-#define DEFINE_MESSAGE_NES(textId, type, yPos, nesMessage)
+#define DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) _message_##textId##_fra,
+#define DEFINE_MESSAGE_JPN(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) /* Not Present */
+#define DEFINE_MESSAGE(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    DEFINE_MESSAGE_NES(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) \
+    DEFINE_MESSAGE_JPN(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage)
+#define DEFINE_MESSAGE_FFFC(textId, type, yPos, jpnMessage, nesMessage, gerMessage, fraMessage) /* Not Present */
 #include "assets/text/message_data.h"
 #undef DEFINE_MESSAGE
 #undef DEFINE_MESSAGE_NES
+#undef DEFINE_MESSAGE_JPN
+#undef DEFINE_MESSAGE_FFFC
    NULL,
 };

@ -390,7 +409,7 @@ void Message_FindCreditsMessage(PlayState* play, u16 textId) {

 void Message_SetTextColor(MessageContext* msgCtx, u16 colorParameter) {
    switch (colorParameter) {
-        case MSGCOL_RED:
+        case TEXT_COLOR_RED:
            if (msgCtx->textBoxType == TEXTBOX_TYPE_WOODEN) {
                msgCtx->textColorR = 255;
                msgCtx->textColorG = 120;
@ -401,7 +420,7 @@ void Message_SetTextColor(MessageContext* msgCtx, u16 colorParameter) {
                msgCtx->textColorB = 60;
            }
            break;
-        case MSGCOL_ADJUSTABLE:
+        case TEXT_COLOR_ADJUSTABLE:
            if (msgCtx->textBoxType == TEXTBOX_TYPE_WOODEN) {
                msgCtx->textColorR = R_TEXT_ADJUST_COLOR_1_R;
                msgCtx->textColorG = R_TEXT_ADJUST_COLOR_1_G;
@ -412,7 +431,7 @@ void Message_SetTextColor(MessageContext* msgCtx, u16 colorParameter) {
                msgCtx->textColorB = R_TEXT_ADJUST_COLOR_2_B;
            }
            break;
-        case MSGCOL_BLUE:
+        case TEXT_COLOR_BLUE:
            if (msgCtx->textBoxType == TEXTBOX_TYPE_WOODEN) {
                msgCtx->textColorR = 80;
                msgCtx->textColorG = 110;
@ -423,7 +442,7 @@ void Message_SetTextColor(MessageContext* msgCtx, u16 colorParameter) {
                msgCtx->textColorB = 255;
            }
            break;
-        case MSGCOL_LIGHTBLUE:
+        case TEXT_COLOR_LIGHTBLUE:
            if (msgCtx->textBoxType == TEXTBOX_TYPE_WOODEN) {
                msgCtx->textColorR = 90;
                msgCtx->textColorG = 180;
@ -438,7 +457,7 @@ void Message_SetTextColor(MessageContext* msgCtx, u16 colorParameter) {
                msgCtx->textColorB = 255;
            }
            break;
-        case MSGCOL_PURPLE:
+        case TEXT_COLOR_PURPLE:
            if (msgCtx->textBoxType == TEXTBOX_TYPE_WOODEN) {
                msgCtx->textColorR = 210;
                msgCtx->textColorG = 100;
@ -449,7 +468,7 @@ void Message_SetTextColor(MessageContext* msgCtx, u16 colorParameter) {
                msgCtx->textColorB = 180;
            }
            break;
-        case MSGCOL_YELLOW:
+        case TEXT_COLOR_YELLOW:
            if (msgCtx->textBoxType == TEXTBOX_TYPE_WOODEN) {
                msgCtx->textColorR = 255;
                msgCtx->textColorG = 255;
@ -460,10 +479,10 @@ void Message_SetTextColor(MessageContext* msgCtx, u16 colorParameter) {
                msgCtx->textColorB = 50;
            }
            break;
-        case MSGCOL_BLACK:
+        case TEXT_COLOR_BLACK:
            msgCtx->textColorR = msgCtx->textColorG = msgCtx->textColorB = 0;
            break;
-        case MSGCOL_DEFAULT:
+        case TEXT_COLOR_DEFAULT:
        default:
            if (msgCtx->textBoxType == TEXTBOX_TYPE_NONE_NO_SHADOW) {
                msgCtx->textColorR = msgCtx->textColorG = msgCtx->textColorB = 0;
--- a/tools/msgdis.py
+++ b/tools/msgdis.py
--- a/tools/msgenc.py
+++ b/tools/msgenc.py
@ -3,41 +3,104 @@
 #   message_data_static text encoder
 #

-import argparse, ast, re
-import sys
+import argparse, ast, re, sys
+from typing import Dict, Optional

-def read_charmap(path):
+def read_charmap(path : str, wchar : bool) -> Dict[str,str]:
    with open(path) as infile:
        charmap = infile.read()
-
    charmap = ast.literal_eval(charmap)
-    charmap = { repr(k)[1:-1] : chr(v) for k,v in charmap.items() }

-    return charmap
+    out_charmap = {}
+    for k,v in charmap.items():
+        v = v[wchar]
+        if v is None:
+            v = 0
+        assert isinstance(k, str)
+        assert v in (range(0xFFFF + 1) if wchar else range(0xFF + 1))
+
+        k = repr(k)[1:-1]
+
+        if wchar:
+            u = (v >> 8) & 0xFF
+            l = (v >> 0) & 0xFF
+            out_charmap[k] = f"0x{u:02X}, 0x{l:02X},"
+        else:
+            out_charmap[k] = f"0x{v:02X},"
+
+    return out_charmap

 # From https://stackoverflow.com/questions/241327/remove-c-and-c-comments-using-python
-def remove_comments(text):
-    def replacer(match):
-        s = match.group(0)
-        if s.startswith('/'):
-            return " " # note: a space and not an empty string
+def remove_comments(text : str) -> str:
+    def replacer(match : re.Match) -> str:
+        string : str = match.group(0)
+        if string.startswith("/"):
+            return " "  # note: a space and not an empty string
        else:
-            return s
+            return string

    pattern = re.compile(
-        r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
-        re.DOTALL | re.MULTILINE
+        r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE
    )
    return re.sub(pattern, replacer, text)

-def convert_text(text, charmap):
-    def cvt_str(m):
-        string = m.group(0)
+def convert_text(text : str, encoding : str, charmap : Dict[str, str]) -> str:
+    def cvt_str(match : re.Match) -> str:
+        string : str = match.group(0)

-        for orig,char in charmap.items():
-            string = string.replace(orig, char)
+        # strip quotes
+        string = string[1:-1]

-        return string
+        def cvt_escape(s : str):
+            # Convert escape sequences such as "\\\"" to "\""
+            return s.encode("ascii").decode("unicode-escape")
+
+        run_start = 0
+
+        def emit(text : Optional[str], advance : int):
+            nonlocal out, string, i, run_start
+            # flush text
+            to_flush = string[run_start:i]
+            if len(string[run_start:i]) != 0:
+                out += ",".join(f"0x{b:02X}" for b in to_flush.encode(encoding))
+                out += ","
+            if text is None:
+                return
+            # emit + advance source pos
+            out += text
+            i += advance
+            # start new run
+            run_start = i
+
+        out = ""
+
+        i = 0
+        while i != len(string):
+            # check charmap
+            for k in charmap.keys():
+                if string.startswith(k, i):
+                    # is in charmap, emit the mapped sequence
+                    emit(charmap[k], len(k))
+                    break
+            else:
+                if string[i] == "\\" and string[i + 1] != "\\":
+                    # is already escaped, emit the escape sequence verbatim
+                    if string[i + 1] == "x":
+                        # \x**
+                        emit("0" + string[i + 1 : i + 4] + ",", 4)
+                    else:
+                        # \*
+                        e = cvt_escape(string[i : i + 2]).encode(encoding)
+                        assert len(e) == 1
+                        emit(f"0x{e[0]:02X},", 2)
+                else:
+                    # increment pos, accumulating text that requires encoding
+                    i += 1
+
+        # emit remaining accumulated text
+        emit(None, 0)
+
+        return out

    # Naive string matcher, assumes single line strings and no comments, handles escaped quotations
    string_regex = re.compile(r'"((?:[^\\"\n]|\\.)*)"')
@ -50,16 +113,23 @@ def convert_text(text, charmap):
    return text

 def main():
-    parser = argparse.ArgumentParser(description="Encode message_data_static text headers")
+    parser = argparse.ArgumentParser(
+        description="Encode message_data_static text headers"
+    )
    parser.add_argument(
        "input",
        help="path to file to be encoded, or - for stdin",
    )
    parser.add_argument(
-        "--output",
-        "-o",
+        "output",
        help="path to write encoded file, or - for stdout",
+    )
+    parser.add_argument(
+        "--encoding",
+        help="encoding (jpn or nes)",
        required=True,
+        type=str,
+        choices=("jpn", "nes"),
    )
    parser.add_argument(
        "--charmap",
@ -68,7 +138,12 @@ def main():
    )
    args = parser.parse_args()

-    charmap = read_charmap(args.charmap)
+    wchar,encoding = {
+        "jpn" : (True, "SHIFT-JIS"),
+        "nes" : (False, "raw-unicode-escape"),
+    }[args.encoding]
+
+    charmap = read_charmap(args.charmap, wchar)

    text = ""
    if args.input == "-":
@ -78,12 +153,12 @@ def main():
            text = infile.read()

    text = remove_comments(text)
-    text = convert_text(text, charmap)
+    text = convert_text(text, encoding, charmap)

    if args.output == "-":
-        sys.stdout.buffer.write(text.encode("raw_unicode_escape"))
+        sys.stdout.buffer.write(text.encode("utf-8"))
    else:
-        with open(args.output, "w", encoding="raw_unicode_escape") as outfile:
+        with open(args.output, "w") as outfile:
            outfile.write(text)

 if __name__ == "__main__":
--- a/tools/version_config.py
+++ b/tools/version_config.py
@ -22,6 +22,8 @@ class VersionConfig:
    version: str
    # ROM offset to start of DMA table
    dmadata_start: int
+    # Whether the languages are PAL (EN/DE/FR) or not (JP/EN)
+    text_lang_pal: bool
    # DMA segment information, in ROM order
    dmadata_segments: OrderedDict[str, SegmentInfo]
    # Addresses of important variables needed for asset extraction
@ -70,6 +72,7 @@ def load_version_config(version: str) -> VersionConfig:
    return VersionConfig(
        version=version,
        dmadata_start=config["dmadata_start"],
+        text_lang_pal=config["text_lang_pal"],
        dmadata_segments=load_dmadata_segments(version),
        variables=config["variables"],
        assets=assets,