tools: teach msgenc to avoid comments (#1151)

Currently the C preprocessor is used to remove comments from the text
files before encoding them.

However the option used (`-fpreprocessed`) is only supported by gcc –
but not by clang, which breaks the macOS build.

By using Python to remove comments, we can remove the use of this
gcc-specific option, and fix the macOS build.

Makefile

@@ -297,8 +297,7 @@ build/data/%.o: data/%.s
 	$(AS) $(ASFLAGS) $< -o $@
 
 build/assets/text/%.enc.h: assets/text/%.h assets/text/charmap.txt
-	$(CPP) -P -dD -fpreprocessed $< > $(@:.enc.h=.h)
-	python3 tools/msgenc.py assets/text/charmap.txt $(@:.enc.h=.h) $@
+	python3 tools/msgenc.py assets/text/charmap.txt $< $@
 
 build/assets/text/fra_message_data_static.o: build/assets/text/message_data.enc.h
 build/assets/text/ger_message_data_static.o: build/assets/text/message_data.enc.h

tools/msgenc.py

@@ -14,6 +14,21 @@ def read_charmap(path):
 
     return charmap
 
+# From https://stackoverflow.com/questions/241327/remove-c-and-c-comments-using-python
+def remove_comments(text):
+    def replacer(match):
+        s = match.group(0)
+        if s.startswith('/'):
+            return " " # note: a space and not an empty string
+        else:
+            return s
+
+    pattern = re.compile(
+        r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
+        re.DOTALL | re.MULTILINE
+    )
+    return re.sub(pattern, replacer, text)
+
 def convert_text(text, charmap):
     def cvt_str(m):
         string = m.group(0)
@@ -46,6 +61,7 @@ def main():
     with open(args.input, "r") as infile:
         text = infile.read()
 
+    text = remove_comments(text)
     text = convert_text(text, charmap)
 
     with open(args.output, "w", encoding="raw_unicode_escape") as outfile: