1
0
Fork 0
mirror of https://github.com/zeldaret/oot.git synced 2024-12-05 01:06:37 +00:00
oot/tools/msgenc.py
Tharo baf1e8c174
Reworked text extraction + add JP text extraction (#1980)
* Reworked text extraction + add JP text extraction

* Format

* Suggested changes

* Correct address for gc-us sJpnMessageEntryTable

Co-authored-by: cadmic <cadmic24@gmail.com>

---------

Co-authored-by: cadmic <cadmic24@gmail.com>
2024-07-02 22:42:52 -04:00

165 lines
4.8 KiB
Python

#!/usr/bin/env python3
#
# message_data_static text encoder
#
import argparse, ast, re, sys
from typing import Dict, Optional
def read_charmap(path : str, wchar : bool) -> Dict[str,str]:
with open(path) as infile:
charmap = infile.read()
charmap = ast.literal_eval(charmap)
out_charmap = {}
for k,v in charmap.items():
v = v[wchar]
if v is None:
v = 0
assert isinstance(k, str)
assert v in (range(0xFFFF + 1) if wchar else range(0xFF + 1))
k = repr(k)[1:-1]
if wchar:
u = (v >> 8) & 0xFF
l = (v >> 0) & 0xFF
out_charmap[k] = f"0x{u:02X}, 0x{l:02X},"
else:
out_charmap[k] = f"0x{v:02X},"
return out_charmap
# From https://stackoverflow.com/questions/241327/remove-c-and-c-comments-using-python
def remove_comments(text : str) -> str:
def replacer(match : re.Match) -> str:
string : str = match.group(0)
if string.startswith("/"):
return " " # note: a space and not an empty string
else:
return string
pattern = re.compile(
r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE
)
return re.sub(pattern, replacer, text)
def convert_text(text : str, encoding : str, charmap : Dict[str, str]) -> str:
def cvt_str(match : re.Match) -> str:
string : str = match.group(0)
# strip quotes
string = string[1:-1]
def cvt_escape(s : str):
# Convert escape sequences such as "\\\"" to "\""
return s.encode("ascii").decode("unicode-escape")
run_start = 0
def emit(text : Optional[str], advance : int):
nonlocal out, string, i, run_start
# flush text
to_flush = string[run_start:i]
if len(string[run_start:i]) != 0:
out += ",".join(f"0x{b:02X}" for b in to_flush.encode(encoding))
out += ","
if text is None:
return
# emit + advance source pos
out += text
i += advance
# start new run
run_start = i
out = ""
i = 0
while i != len(string):
# check charmap
for k in charmap.keys():
if string.startswith(k, i):
# is in charmap, emit the mapped sequence
emit(charmap[k], len(k))
break
else:
if string[i] == "\\" and string[i + 1] != "\\":
# is already escaped, emit the escape sequence verbatim
if string[i + 1] == "x":
# \x**
emit("0" + string[i + 1 : i + 4] + ",", 4)
else:
# \*
e = cvt_escape(string[i : i + 2]).encode(encoding)
assert len(e) == 1
emit(f"0x{e[0]:02X},", 2)
else:
# increment pos, accumulating text that requires encoding
i += 1
# emit remaining accumulated text
emit(None, 0)
return out
# Naive string matcher, assumes single line strings and no comments, handles escaped quotations
string_regex = re.compile(r'"((?:[^\\"\n]|\\.)*)"')
# Collapse escaped newlines
text = text.replace("\\\n", "")
# Encode according to charmap
text = re.sub(string_regex, cvt_str, text)
return text
def main():
parser = argparse.ArgumentParser(
description="Encode message_data_static text headers"
)
parser.add_argument(
"input",
help="path to file to be encoded, or - for stdin",
)
parser.add_argument(
"output",
help="path to write encoded file, or - for stdout",
)
parser.add_argument(
"--encoding",
help="encoding (jpn or nes)",
required=True,
type=str,
choices=("jpn", "nes"),
)
parser.add_argument(
"--charmap",
help="path to charmap file specifying custom encoding elements",
required=True,
)
args = parser.parse_args()
wchar,encoding = {
"jpn" : (True, "SHIFT-JIS"),
"nes" : (False, "raw-unicode-escape"),
}[args.encoding]
charmap = read_charmap(args.charmap, wchar)
text = ""
if args.input == "-":
text = sys.stdin.read()
else:
with open(args.input, "r") as infile:
text = infile.read()
text = remove_comments(text)
text = convert_text(text, encoding, charmap)
if args.output == "-":
sys.stdout.buffer.write(text.encode("utf-8"))
else:
with open(args.output, "w") as outfile:
outfile.write(text)
if __name__ == "__main__":
main()