From d7c9d8d86ffb50fd8b7812b2e76cd54012c6660c Mon Sep 17 00:00:00 2001 From: Roman971 <32455037+Roman971@users.noreply.github.com> Date: Sun, 7 Jun 2020 16:37:48 +0200 Subject: [PATCH] Add sym_info.py and first_diff.py scripts for convenience (#192) --- first_diff.py | 246 +++++++++++++++++++++++++++++++++++++++++++++++++ spec | 3 +- sym_info.py | 139 ++++++++++++++++++++++++++++ tools/diff.txt | 0 4 files changed, 387 insertions(+), 1 deletion(-) create mode 100644 first_diff.py create mode 100644 sym_info.py delete mode 100644 tools/diff.txt diff --git a/first_diff.py b/first_diff.py new file mode 100644 index 0000000000..09eb68290d --- /dev/null +++ b/first_diff.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 + +import os.path +import argparse +from subprocess import check_call + +parser = argparse.ArgumentParser( + description="Find the first difference(s) between the built ROM and the base ROM." +) +parser.add_argument( + "-c", + "--count", + type=int, + default=5, + help="find up to this many instruction difference(s)", +) +parser.add_argument( + "-d", + "--diff", + dest="diff_args", + nargs="?", + action="store", + default=False, + const="prompt", + help="run diff.py on the result with the provided arguments" +) +args = parser.parse_args() + +diff_count = args.count + +baseimg = f"baserom.z64" +basemap = f"expected/build/z64.map" + +myimg = f"zelda_ocarina_mq_dbg.z64" +mymap = f"build/z64.map" + +if not os.path.isfile(baseimg): + print(f"{baseimg} must exist.") + exit(1) +if not os.path.isfile(myimg) or not os.path.isfile(mymap): + print(f"{myimg} and {mymap} must exist.") + exit(1) + +mybin = open(myimg, "rb").read() +basebin = open(baseimg, "rb").read() + +if len(mybin) != len(basebin): + print("Modified ROM has different size...") + exit(1) + +if mybin == basebin: + print("No differences!") + exit(0) + + +def search_address(target_addr): + is_ram = target_addr & 0x80000000 + ram_offset = None + prev_ram = 0 + prev_rom = 0 + prev_sym = "" + cur_file = "" + prev_file = cur_file + prev_line = "" + with open(mymap) as f: + for line in f: + if "load address" in line: + if "noload" in line or "noload" in prev_line: + ram_offset = None + continue + ram = int(line[16 : 16 + 18], 0) + rom = int(line[59 : 59 + 18], 0) + ram_offset = ram - rom + continue + prev_line = line + + if ( + ram_offset is None + or "=" in line + or "*fill*" in line + or " 0x" not in line + ): + continue + + ram = int(line[16 : 16 + 18], 0) + rom = ram - ram_offset + sym = line.split()[-1] + + if "0x" in sym: + ram_offset = None + continue + if "/" in sym: + cur_file = sym + continue + + if rom > target_addr or (is_ram and ram > target_addr): + return f"{prev_sym} (RAM 0x{prev_ram:X}, ROM 0x{prev_rom:X}, {prev_file})" + + prev_ram = ram + prev_rom = rom + prev_sym = sym + prev_file = cur_file + + return "at end of rom?" + + +def parse_map(map_fname): + ram_offset = None + cur_file = "" + syms = {} + prev_sym = None + prev_line = "" + with open(map_fname) as f: + for line in f: + if "load address" in line: + if "noload" in line or "noload" in prev_line: + ram_offset = None + continue + ram = int(line[16 : 16 + 18], 0) + rom = int(line[59 : 59 + 18], 0) + ram_offset = ram - rom + continue + prev_line = line + + if ( + ram_offset is None + or "=" in line + or "*fill*" in line + or " 0x" not in line + ): + continue + + ram = int(line[16 : 16 + 18], 0) + rom = ram - ram_offset + sym = line.split()[-1] + + if "0x" in sym: + ram_offset = None + continue + elif "/" in sym: + cur_file = sym + continue + + syms[sym] = (rom, cur_file, prev_sym, ram) + prev_sym = sym + + return syms + + +def map_diff(): + map1 = parse_map(mymap) + map2 = parse_map(basemap) + min_ram = None + found = None + for sym, addr in map1.items(): + if sym not in map2: + continue + if addr[0] != map2[sym][0]: + if min_ram is None or addr[0] < min_ram: + min_ram = addr[0] + found = (sym, addr[1], addr[2]) + if min_ram is None: + return False + else: + print( + f"Map appears to have shifted just before {found[0]} ({found[1]}) -- in {found[2]}?" + ) + if found[2] is not None and found[2] not in map2: + print( + f"(Base map file {basemap} out of date due to new or renamed symbols, so result may be imprecise.)" + ) + return True + + +def hexbytes(bs): + return ":".join("{:02X}".format(c) for c in bs) + + +found_instr_diff = [] +map_search_diff = [] +diffs = 0 +shift_cap = 1000 +for i in range(24, len(mybin), 4): + # (mybin[i:i+4] != basebin[i:i+4], but that's slightly slower in CPython...) + if diffs <= shift_cap and ( + mybin[i] != basebin[i] + or mybin[i + 1] != basebin[i + 1] + or mybin[i + 2] != basebin[i + 2] + or mybin[i + 3] != basebin[i + 3] + ): + if diffs == 0: + print(f"First difference at ROM addr 0x{i:X}, {search_address(i)}") + print( + f"Bytes: {hexbytes(mybin[i : i + 4])} vs {hexbytes(basebin[i : i + 4])}" + ) + diffs += 1 + if ( + len(found_instr_diff) < diff_count + and mybin[i] >> 2 != basebin[i] >> 2 + and not search_address(i) in map_search_diff + ): + found_instr_diff.append(i) + map_search_diff.append(search_address(i)) + +if diffs == 0: + print("No differences but ROMs differ?") + exit() + +if len(found_instr_diff) > 0: + for i in found_instr_diff: + print(f"Instruction difference at ROM addr 0x{i:X}, {search_address(i)}") + print( + f"Bytes: {hexbytes(mybin[i : i + 4])} vs {hexbytes(basebin[i : i + 4])}" + ) +print() + +definite_shift = diffs > shift_cap +if definite_shift: + print(f"Over {shift_cap} differing words, must be a shifted ROM.") +else: + print(f"{diffs} differing word(s).") + +if diffs > 100: + if not os.path.isfile(basemap): + print( + f"To find ROM shifts, copy a clean .map file to {basemap} and rerun this script." + ) + elif not map_diff(): + print(f"No ROM shift{' (!?)' if definite_shift else ''}") + +if args.diff_args: + diff_sym = search_address(found_instr_diff[0]).split()[0] + if args.diff_args == "prompt": + diff_args = input("Call diff.py with which arguments? ") or "--" + else: + diff_args = args.diff_args + if diff_args[0] != "-": + diff_args = "-" + diff_args + check_call( + [ + "python3", + "diff.py", + diff_args, + diff_sym, + ] + ) diff --git a/spec b/spec index 471096a8f9..92921997d2 100644 --- a/spec +++ b/spec @@ -121,6 +121,7 @@ endseg beginseg name "Audiobank" + address 0x10 // fake RAM address to avoid map lookup inaccuracies include "build/baserom/Audiobank.o" endseg @@ -337,7 +338,7 @@ beginseg include "build/src/code/z_map_data.o" include "build/src/code/z_parameter.o" include "build/src/code/z_path.o" - include "build/src/code//code_8008E6A0.o" + include "build/src/code/code_8008E6A0.o" include "build/src/code/z_player_lib.o" include "build/data/z_player_lib.data.o" include "build/data/z_player_lib.bss.o" diff --git a/sym_info.py b/sym_info.py new file mode 100644 index 0000000000..b8bdcd2375 --- /dev/null +++ b/sym_info.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 + +import os.path +import argparse + +parser = argparse.ArgumentParser( + description="Display various information about a symbol or address." +) +parser.add_argument( + "name", + type=str, + default="", + help="symbol name or ROM/RAM address to lookup" +) +parser.add_argument( + "-e", + "--expected", + dest="use_expected", + action="store_true", + help="use the map file in expected/build/ instead of build/" +) +args = parser.parse_args() + +mymap = "build/z64.map" +if args.use_expected: + mymap = f"expected/{mymap}" + +if not os.path.isfile(mymap): + print(f"{mymap} must exist.") + exit(1) + + +def search_address(target_addr): + is_ram = target_addr & 0x80000000 + ram_offset = None + prev_ram = 0 + prev_rom = 0 + prev_sym = "" + cur_file = "" + prev_file = cur_file + prev_line = "" + with open(mymap) as f: + for line in f: + if "load address" in line: + if "noload" in line or "noload" in prev_line: + ram_offset = None + continue + ram = int(line[16 : 16 + 18], 0) + rom = int(line[59 : 59 + 18], 0) + ram_offset = ram - rom + continue + prev_line = line + + if ( + ram_offset is None + or "=" in line + or "*fill*" in line + or " 0x" not in line + ): + continue + + ram = int(line[16 : 16 + 18], 0) + rom = ram - ram_offset + sym = line.split()[-1] + + if "0x" in sym: + ram_offset = None + continue + if "/" in sym: + cur_file = sym + continue + + if rom == target_addr or (is_ram and ram == target_addr): + return f"{sym} (RAM 0x{ram:X}, ROM 0x{rom:X}, {cur_file})" + if rom > target_addr or (is_ram and ram > target_addr): + offset = target_addr - prev_ram if is_ram else target_addr - prev_rom + return f"at 0x{offset:X} bytes inside {prev_sym} (RAM 0x{prev_ram:X}, ROM 0x{prev_rom:X}, {prev_file})" + + prev_ram = ram + prev_rom = rom + prev_sym = sym + prev_file = cur_file + + return "at end of rom?" + + +def search_symbol(target_sym): + ram_offset = None + cur_file = "" + prev_line = "" + with open(mymap) as f: + for line in f: + if "load address" in line: + if "noload" in line or "noload" in prev_line: + ram_offset = None + continue + ram = int(line[16 : 16 + 18], 0) + rom = int(line[59 : 59 + 18], 0) + ram_offset = ram - rom + continue + prev_line = line + + if ( + ram_offset is None + or "=" in line + or "*fill*" in line + or " 0x" not in line + ): + continue + + ram = int(line[16 : 16 + 18], 0) + rom = ram - ram_offset + sym = line.split()[-1] + + if "0x" in sym: + ram_offset = None + continue + elif "/" in sym: + cur_file = sym + continue + + if sym == target_sym: + return (rom, cur_file, ram) + + return None + + +try: + target_addr = int(args.name, 0) + print(args.name, "is", search_address(target_addr)) +except ValueError: + sym_info = search_symbol(args.name) + if sym_info is not None: + sym_rom = sym_info[0] + sym_file = sym_info[1] + sym_ram = sym_info[2] + print(f"Symbol {args.name} (RAM: 0x{sym_ram:08X}, ROM: 0x{sym_rom:06X}, {sym_file})") + else: + print(f"Symbol {args.name} not found in map file {mymap}") diff --git a/tools/diff.txt b/tools/diff.txt deleted file mode 100644 index e69de29bb2..0000000000