Generate and match relocations on StaticR.rel (#230)

* WIP: Seeky ppcdis disassembler

* Remove pickles files from git

* Remove caseD symbols

* Matching REL relocatable asm

* Matching REL asm, error on DOL

* Add ppcdis as required module instead of submodule

* Match trivial empty functions

* Update REL inline assembly

* Change ppcdis requirement to fork

* (WIP) missing jump labels

* automatic extern declarations for REL

* Remove shiftable DOL support

* Remove shiftable DOL support

* Migration to seeky's elf2rel and some symbol fixes

* Migrate to symbols.yml

* dump_func_symbols->dump_rel_elf_symbols

* WIP - resolving symbols

* resolve lbl_80890030

* WIP: more symbol resolutions

* Add ResourceManager bss

* WIP: ResourceManager static instance union

* StaticR.rel matching without relocations

* relocation fixes

* MATCH

* Update README and CI with new build procedure

* pin ppcdis version

* Fix ppcdis version

* delete_ppcdis_init

* Seeky review changes

* Readd DOL analysis

Co-authored-by: Theodoros Tyrovouzis <[email protected]>

.gitignore

@@ -64,7 +64,7 @@ tmp
 *.map
 out.html
 tools.zip
-pack/new_symbols.txt
+pack/new_symbols.yml
 
 # Editors
 .vscode/

.pre-commit-config.yaml

@@ -21,3 +21,5 @@ repos:
         types: [file]
         pass_filenames: true
         files: 'symbols\.txt'
+
+exclude: '.*\.j2$'

build.py

@@ -32,11 +32,13 @@
 from mkwutil.progress.percent_decompiled import build_stats
 from mkwutil.gen_asm import gen_asm
 from mkwutil.project import load_dol_slices, load_rel_slices
+from mkwutil.ppcdis_adapter.ppcdis_analyse import analyse_bins
 
 
 def parse_args():
     parser = argparse.ArgumentParser(description="Build main.dol and StaticR.rel.")
     parser.add_argument("--regen_asm", action="store_true", help="Regenerate all ASM")
+    parser.add_argument("--force_analyse", action="store_true", help="Force run original binary analysis")
     parser.add_argument(
         "-j",
         "--concurrency",
@@ -183,7 +185,7 @@ def patch_compilers():
     "-nostdinc",
     "-msgstyle gcc -lang=c99 -DREVOKART",
     "-func_align 4",
-    "-sym dwarf-2",
+    #"-sym dwarf-2",
 ]
 
 # Hack: $@ doesn't behave properly with this
@@ -323,7 +325,7 @@ def link(
         ]
     )
     if partial:
-        cmd.append("-r")
+        cmd.append("-r1")
     command = " ".join(map(str, cmd))
     lines, returncode = run_windows_cmd(command)
     for line in lines:
@@ -409,14 +411,16 @@ def link_rel(o_files: list[Path]):
     map_path = dest_dir / "StaticR.map"
     link(elf_path, o_files, dst_lcf_path, map_path, partial=True)
     # Convert ELF to REL.
+    dol_elf_path = dest_dir / "main.elf"
     rel_path = dest_dir / "StaticR.rel"
     orig_dir = Path("artifacts", "orig")
-    pack_staticr_rel(elf_path, rel_path, orig_dir)
+    orig_rel_yml_path = Path("mkwutil", "ppcdis_adapter", "rel.yaml")
+    pack_staticr_rel(elf_path, rel_path, orig_rel_yml_path, dol_elf_path)
     return rel_path
 
 
 def build(args):
-    # Start by running gen_asm.
+    analyse_bins(args.force_analyse)
     gen_asm(args.regen_asm)
 
     """Builds the game."""

diff.py

@@ -7,7 +7,7 @@
 
 Edits:
     Dol & rel ram-rom conversion added using mkwutil
-    Check symbols.txt (pack/symbols.txt) for symbol name and size definitions
+    Check symbols.yml (pack/symbols.yml) for symbol name and size definitions
     Convert runtime VMAs of REL and DOL to offsets in binary file if start or end > 0x8000000
 """
 
@@ -388,7 +388,7 @@ class ProjectSettings:
     build_command: List[str]
     map_format: str
     mw_build_dir: str
-    symbols_txt: str
+    symbols_yml: str
     baseimg: Optional[str]
     myimg: Optional[str]
     mapfile: Optional[str]
@@ -447,7 +447,7 @@ def create_project_settings(settings: Dict[str, Any]) -> ProjectSettings:
         baseimg=settings.get("baseimg"),
         myimg=settings.get("myimg"),
         mapfile=settings.get("mapfile"),
-        symbols_txt=settings.get("symbols_txt"),
+        symbols_yml=settings.get("symbols_yml"),
         build_command=settings.get(
             "make_command", ["make", *settings.get("makeflags", [])]
         ),
@@ -1383,12 +1383,12 @@ def dump_objfile(
     )
 
 
-def search_symbols_txt(start, start_addr, project, config):
+def search_symbols_yml(start, start_addr, project, config):
     from mkwutil.lib.symbols import SymbolsList
 
     symbols = SymbolsList()
-    with open(project.symbols_txt, "r") as f:
-        symbols.read_from(f)
+    with open(project.symbols_yml, "r") as f:
+        symbols.read_from_yaml(f)
     symbols.derive_sizes(0x8100_0000)
 
     if start_addr is not None:
@@ -1415,9 +1415,9 @@ def dump_binary(
         run_make(project.myimg, project)
     start_addr = maybe_eval_int(start)
 
-    start_addr, end_addr = search_symbols_txt(start, start_addr, project, config)
+    start_addr, end_addr = search_symbols_yml(start, start_addr, project, config)
 
-    if start_addr is None: # if symbol not found in symbols.txt AND start arg not an int, search map file
+    if start_addr is None: # if symbol not found in symbols.yml AND start arg not an int, search map file
         _, start_addr = search_map_file(start, project, config)
         if start_addr is None:
             fail("Not able to find function in map file.")

diff_settings.py

@@ -16,7 +16,7 @@ def apply(config: dict, args):
         config["mapfile"] = "artifacts/target/pal/main.map"
         config["myimg"] = "artifacts/target/pal/main.dol"
         config["baseimg"] = "artifacts/orig/pal/main.dol"
-    config["symbols_txt"] = "pack/symbols.txt"
+    config["symbols_yml"] = "pack/symbols.yml"
     config["make_command"] = [executable, "build.py", "--diff_py"]
     config["makeflags"] = []
     config["source_directories"] = ["source"]

ghidra/ExportFunctionInfoScriptToSymbolsTxt.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
-# Run this to convert a dump from ExportFunctionInfoScript.java to a symbols.txt dump.
+# Run this to convert a dump from ExportFunctionInfoScript.java to a symbols.yml dump.
 exec jq -r '.[] | "0x\(.entry) \(.name)"'

ghidra/ImportSymbolsTxt.py

@@ -1,5 +1,5 @@
 ##
-# Imports symbols.txt from mkw
+# Imports symbols.yml from mkw
 # @author Richard Patel <[email protected]>
 # @category Data
 #