Switch slices from CSV to YML

HACK.md

@@ -21,10 +21,10 @@ Uses `mkwutil` to rebuild MKW from decompiled sources and verifies the result.
 Specifies which sections are decompiled and holds information needed to reconstruct final binaries.
 
 - `./pack/dol_segments.csv`: Table of main.dol segments
-- `./pack/dol_slices.csv`: Table of main.dol decompiled sections
+- `./pack/dol_slices.yml`: Table of main.dol decompiled sections
 - `./pack/dol_objects.txt`: List of main.dol reconstructed objects (in order)
 - `./pack/rel_segments.csv`: Table of StaticR.rel segments
-- `./pack/rel_slices.csv`: Table of StaticR.rel decompiled sections
+- `./pack/rel_slices.yml`: Table of StaticR.rel decompiled sections
 - `./pack/dol_objects.txt`: List of StaticR.rel reconstructed objects (in order)
 
 ### `./asm`: Binary blobs

configure.py

@@ -334,7 +334,7 @@ def link_dol(dol_objects_path: Path, n: Writer):
     # Generate LCF.
     src_lcf_path = Path("pack", "dol.lcf.j2")
     dst_lcf_path = Path("pack", "dol.lcf")
-    slices_path = Path("pack", "dol_slices.csv")
+    slices_path = Path("pack", "dol_slices.yml")
     n.build(
         str(dst_lcf_path),
         rule = "lcfgen",
@@ -421,7 +421,7 @@ def configure(args):
         orig_dol_path = Path("artifacts", "orig", "pal", "main.dol")
         orig_rel_path = Path("artifacts", "orig", "pal", "StaticR.rel")
 
-        slices_path = Path("pack", "rel_slices.csv")
+        slices_path = Path("pack", "rel_slices.yml")
         n.variable("slices", str(slices_path))
         n.rule(
             "lcfgen",

mkwutil/attic/format_slices.py

@@ -1,5 +1,8 @@
-from mkwutil.project import load_dol_slices, save_dol_slices
+from mkwutil.project import load_dol_slices, save_dol_slices, load_rel_slices, save_rel_slices
 
 slices = load_dol_slices()
 objects = slices.object_slices()
 save_dol_slices(objects)
+slices = load_rel_slices()
+objects = slices.object_slices()
+save_rel_slices(objects)

mkwutil/gen_asm.py

@@ -702,7 +702,7 @@ def __gen_includes(self, c_source_name: str, out_path: Path):
             c_text = c_file.read()
             for addr_str in re.findall(self.inline_asm_pattern, c_text):
                 addr = int(addr_str, 16)
-                inline_asm = self.disaser.function_to_text(addr, inline=True, extra=False, 
+                inline_asm = self.disaser.function_to_text(addr, inline=True, extra=False,
                                                            hashable=False, declare_mangled=False)
                 out_file = out_path / f"{addr:x}.s"
                 with open(out_file, "w") as asm_file:
@@ -725,7 +725,7 @@ def gen_asm(regen_asm=False, regen_inline=False):
 
     dol = read_dol(binary_dir / "main.dol")
     dol_disaser = get_dol_disaser()
-    dol_slices = read_enabled_slices(dol, pack_dir / "dol_slices.csv")
+    dol_slices = read_enabled_slices(dol, pack_dir / "dol_slices.yml")
 
     # Disassemble DOL sections.
     dol_asm_dir = asm_dir / "dol"

mkwutil/gen_lcf.py

@@ -30,9 +30,9 @@ def gen_lcf(
     slices = SliceTable.load_path(slices_path)
     slices = slices.filter(SliceTable.ONLY_ENABLED)
     # For dol and rel, we can also fill in section names.
-    if slices_path.name == "dol_slices.csv":
+    if slices_path.name == "dol_slices.yml":
         slices.set_sections(DOL_SECTIONS)
-    elif slices_path.name == "rel_slices.csv":
+    elif slices_path.name == "rel_slices.yml":
         slices.set_sections(REL_SECTIONS)
     else:
         print(f"WARN: Using linker-defined symbols instead of inline assembly")

mkwutil/lib/slices.py

@@ -453,63 +453,31 @@ def set_sections(self, sections: list) -> None:
             _slice.section = sections[i].name
 
 
-class SlicesCSVReader:
-    """Reads a list of slices from slices.csv."""
+import yaml
 
+class SlicesCSVReader:
     def __init__(self, file):
-        self.reader = csv.reader(file)
-        # Read CSV header.
-        header = next(self.reader)
-        self.cols = len(header)
-        # The name field separates tags and ranges.
-        name_idx = header.index("name")
-        self.tag_idx = header[:name_idx]
-        section_fields = header[name_idx + 1 :]
-        assert (
-            len(section_fields) > 0 and len(section_fields) % 2 == 0
-        ), "Odd number of fields"
-        # Remember section names.
-        self.sections = []
-        for i in range(0, len(section_fields), 2):
-            assert section_fields[i].endswith("Start")
-            assert section_fields[i + 1].endswith("End")
-            section = section_fields[i].removesuffix("Start")
-            assert section == section_fields[i + 1].removesuffix("End")
-            self.sections.append(section)
-
-    def __iter__(self) -> Generator[Slice, None, None]:
-        """Returns all slices in the file."""
-        for row in self.reader:
-            for slice in self.parse_row(row):
-                yield slice
-
-    def parse_row(self, row: str) -> Generator[Slice, None, None]:
-        """Returns all slices in a row."""
-        if len(row) == 0:
-            return
-        assert len(row) == self.cols, "Unexpected number of fields"
-        # Read flags.
-        flags = set()
-        for i, tag in enumerate(self.tag_idx):
-            if row[i] == "1":
-                flags.add(tag)
-        # Read name.
-        name = row[len(self.tag_idx)]
-        # Read section ranges.
-        ranges = row[len(self.tag_idx) + 1 :]
-        for i, section in enumerate(self.sections):
-            start_str, stop_str = ranges[2 * i : 2 * i + 2]
-            start_str = start_str.strip()
-            stop_str = stop_str.strip()
-            if start_str == "":
+        self.file = file
+
+    def __iter__(self):
+        slices = []
+        data = yaml.safe_load(self.file)
+
+        for source_file, regions in data.items():
+            if regions.get('enabled', 0) != 1:
                 continue
-            if stop_str == "":
-                print(f"column {i} start_str {start_str} stop_str {stop_str}")
 
-            assert stop_str != ""
-            start = int(start_str.strip(), 16)
-            stop = int(stop_str, 16)
-            yield Slice(start, stop, name, section, flags)
+            tags = []
+            for k, v in regions.items():
+                if v == 1:
+                    tags.append(k)
+
+            for key, value in regions.items():
+                if "Start" in key:
+                    start = value
+                    stop_key = key.replace("Start", "End")
+                    stop = regions[stop_key]
+                    yield Slice(start=start, stop=stop, name=source_file, section=key.replace("Start", ""), tags=tags)
 
 
 class SlicesCSVWriter:

mkwutil/project.py

@@ -6,11 +6,12 @@
 from pathlib import Path
 
 
-dol_slices_path = Path(__file__).parent / ".." / "pack" / "dol_slices.csv"
+dol_slices_path = Path(__file__).parent / ".." / "pack" / "dol_slices.yml"
+rel_slices_path = Path(__file__).parent / ".." / "pack" / "rel_slices.yml"
 
 
 def load_dol_slices(sections=None) -> "SliceTable":
-    """Loads pack/dol_slices.csv in the default DOL region."""
+    """Loads pack/dol_slices.yml in the default DOL region."""
     return SliceTable.load_path(dol_slices_path, sections=sections)
 
 
@@ -20,11 +21,15 @@ def save_dol_slices(objects: ObjectSlices):
 
 
 def load_rel_slices(sections=None) -> "SliceTable":
-    """Loads pack/rel_slices.csv in the default DOL region."""
+    """Loads pack/rel_slices.yml in the default DOL region."""
     return SliceTable.load_path(
-        Path(__file__).parent / ".." / "pack" / "rel_slices.csv", sections=sections
+        Path(__file__).parent / ".." / "pack" / "rel_slices.yml", sections=sections
     )
 
+def save_rel_slices(objects: ObjectSlices):
+    with open(rel_slices_path, "w", newline="\n") as slices_file:
+        objects.write_to(slices_file, sections=SliceTable.SECTION_ORDER)
+
 
 def load_dol_binary_blob_slices(dir: Path):
     path = dir / "artifacts" / "target" / "pal" / "main.elf"
@@ -64,7 +69,7 @@ def read_enabled_slices(dol, slices_path):
 
         return dol_slices.filter(SliceTable.ONLY_ENABLED)
 
-    raise RuntimeError("Cannot find dol_slices.csv")
+    raise RuntimeError("Cannot find dol_slices.yml")
 
 
 def read_rel(rel_path):