Match retail BSS ordering (#1927)

* Match retail BSS ordering

* Revert moving some global variables to headers

* Adjust block numbers after header changes

* Fix debug build

* Overlay bss ordering

* Fix BSS ordering after header changes

* gc-eu-mq OK

* Implement preprocessor for #pragma increment_block_number

* Transfer usage comment from reencode.sh

* Use temporary directory instead of temporary file

* Move ColChkMassType back

tools/disasm/gc-eu-mq/variables.txt

@@ -41,7 +41,7 @@ sIdleThread = 0x80006590; // size:0x1B0
 sIdleThreadStack = 0x80006740; // size:0x400
 sIdleThreadInfo = 0x80006B40; // size:0x1C
 sBootThreadStack = 0x80006B60; // size:0x400
-gMainThread = 0x80006F60; // size:0x1B0
+sMainThread = 0x80006F60; // size:0x1B0
 sMainStack = 0x80007110; // size:0x900
 sMainStackInfo = 0x80007A10; // size:0x1C
 sPiMgrCmdBuff = 0x80007A30; // size:0xC8
@@ -570,9 +570,9 @@ sKaleidoScopeUpdateFunc = 0x8011B3C0; // size:0x4
 sKaleidoScopeDrawFunc = 0x8011B3C4; // size:0x4
 gBossMarkScale = 0x8011B3C8; // size:0x4
 gLoadedPauseMarkDataTable = 0x8011B3D0; // size:0x4
-sTransitionTile = 0x8011B3E0; // size:0xE0
+gTransitionTile = 0x8011B3E0; // size:0xE0
 gTransitionTileState = 0x8011B4C0; // size:0x4
-sPlayVisMono = 0x8011B4C8; // size:0x18
+gPlayVisMono = 0x8011B4C8; // size:0x18
 gVisMonoColor = 0x8011B4E0; // size:0x4
 D_801664D0 = 0x8011B4F0; // size:0x20
 sVisCvg = 0x8011B510; // size:0x10

tools/ido_block_numbers.py

@@ -222,7 +222,7 @@ def find_compiler_command_line(filename, oot_version):
     for line in make_output:
         parts = line.split()
         if "-o" in parts and str(filename) in parts:
-            makefile_command_line = parts
+            compiler_command_line = parts
             found += 1
 
     if found != 1:
@@ -231,49 +231,29 @@ def find_compiler_command_line(filename, oot_version):
         )
         sys.exit(1)
 
-    # Assume command line is of the form:
-    # tools/reencode.sh [COMPILER] [COMPILER_ARGS]
-    compiler_command_line = makefile_command_line[1:]
-
     print(f'Command line: {" ".join(compiler_command_line)}', file=sys.stderr)
     return compiler_command_line
 
 
 def generate_symbol_table(command_line):
-    # Find source file in compiler arguments
-    source_file = None
-    args = []
-    for arg in command_line:
-        if arg.endswith(".c"):
-            source_file = Path(arg)
-        else:
-            args.append(arg)
+    # Assume command line is of the form:
+    # python3 tools/preprocess.py [COMPILER] [COMPILER_ARGS] [INPUT_FILE]
+    input_file = Path(command_line[-1])
+    rest = command_line[:-1]
 
-    if source_file is None:
-        raise Exception("No source file found")
-
-    source_contents = source_file.read_text()
-
-    stem = "reencode_tmp"
-    input_file = Path(f"{stem}.c")
+    stem = input_file.stem
     symbol_table_file = Path(f"{stem}.T")
     ucode_file = Path(f"{stem}.B")
 
     try:
-        # Write temporary file with #line directive to simulate asm-processor
-        with open(input_file, "w") as f:
-            f.write('#line 1 "{}"\n'.format(source_file))
-            f.write(source_contents)
-
         # Invoke compiler
         # -Hf stops compilation after cfe so we can inspect the symbol table
-        subprocess.run(args + ["-Hf", input_file], check=True)
+        subprocess.run(rest + ["-Hf", input_file], check=True)
 
         # Read symbol table
         return symbol_table_file.read_bytes()
     finally:
         # Cleanup
-        input_file.unlink(missing_ok=True)
         symbol_table_file.unlink(missing_ok=True)
         ucode_file.unlink(missing_ok=True)
 

tools/preprocess.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: © 2024 ZeldaRET
+# SPDX-License-Identifier: CC0-1.0
+
+# Usage: preprocess.py [compile command minus input file...] [single input file]
+# Preprocess a C file to:
+# * Re-encode from UTF-8 to EUC-JP (the repo uses UTF-8 for text encoding, but
+#   the strings in the ROM are encoded in EUC-JP)
+# * Replace `#pragma increment_block_number N` with `N` fake structs for
+#   controlling BSS ordering
+
+from pathlib import Path
+import os
+import tempfile
+import subprocess
+import sys
+
+
+def fail(message):
+    print(message, file=sys.stderr)
+    sys.exit(1)
+
+
+def process_file(filename, input, output):
+    output.write(f'#line 1 "{filename}"\n')
+    for i, line in enumerate(input, start=1):
+        if line.startswith("#pragma increment_block_number"):
+            parts = line.split()
+            if len(parts) != 3:
+                fail(
+                    f"{filename}:{i}: increment_block_number must be followed by an integer"
+                )
+            try:
+                amount = int(parts[2])
+            except ValueError:
+                fail(
+                    f"{filename}:{i}: increment_block_number must be followed by an integer"
+                )
+            # Write fake structs for BSS ordering
+            for j in range(amount):
+                output.write(f"struct DummyStruct_{i:05}_{j:03};\n")
+            output.write(f'#line {i + 1} "{filename}"\n')
+        else:
+            output.write(line)
+
+
+def main():
+    filename = Path(sys.argv[-1])
+    with tempfile.TemporaryDirectory(prefix="oot_") as tmpdir:
+        tmpfile = Path(tmpdir) / filename.name
+
+        with open(filename, mode="r", encoding="utf-8") as input:
+            with open(tmpfile, mode="w", encoding="euc-jp") as output:
+                process_file(filename, input, output)
+
+        compile_command = sys.argv[1:-1] + ["-I", filename.parent, tmpfile]
+        process = subprocess.run(compile_command)
+        return process.returncode
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        sys.exit(1)

tools/reencode.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-# The repo uses UTF-8 for text encoding, but the strings in the ROM are encoded in EUC-JP.
-# This means for matching the source must be re-encoded to EUC-JP before IDO compiles it.
-# This is conceptually equivalent to `gcc --finput-charset=UTF-8 --fexec-charset=EUC-JP`,
-# except IDO has no equivalent arguments.
-# Usage: reencode.sh [compile command minus input file...] [single input file]
-
-set -e
-
-# The last argument, the input source file to be compiled
-srcfile="${@: -1}"
-
-# Create a temporary file, and remove it on script exit
-tempfile=`mktemp`_oot.c
-trap "rm -f $tempfile" EXIT
-
-# Re-encode from UTF-8 to EUC-JP
-{
-    printf '#line 1 "%s"\n' "$srcfile"  # linemarker
-    cat "$srcfile"
-} | iconv -f UTF-8 -t EUC-JP > "$tempfile"
-
-# All arguments but the last, forming the compile command
-# Also include the source file's directory to have the include path as if we compiled the original source.
-# Pass the EUC-JP encoded temporary file for compilation.
-"${@:1:$# - 1}" -I `dirname $srcfile` $tempfile