feat: Refactor ingestion and pack library routes

- Introduced `ingest_shared.py` for shared ingestion helpers, improving code organization and reusability.
- Created `pack_library.py` to manage KnowledgePack CRUD operations, merging, and world application, separating concerns from `ingest.py`.
- Enhanced frontend components for applying packs and displaying worlds with loading states.
- Updated `SourcesPanel` to handle queued jobs and improved error handling in job listings.
- Added tests for ingestion locking and pack serialization, ensuring robust functionality.
- Updated pytest configuration for better test discovery and exclusion of unnecessary directories.

Author: spuentesp

packages/agents/src/monitor_agents/analyzer.py

@@ -634,16 +634,26 @@ async def _commit_to_neo4j(
                 return
 
             _REL_TYPE_MAP = {
-                "member_of": "MEMBER_OF", "part_of": "MEMBER_OF",
+                "member_of": "MEMBER_OF",
+                "part_of": "PART_OF",
+                "subgroup_of": "SUBGROUP_OF",
+                "affiliated_with": "AFFILIATED_WITH",
                 "allied_with": "ALLIED_WITH",
-                "enemy_of": "HOSTILE_TO", "opposes": "HOSTILE_TO", "hostile_to": "HOSTILE_TO",
+                "enemy_of": "HOSTILE_TO",
+                "opposes": "HOSTILE_TO",
+                "hostile_to": "HOSTILE_TO",
                 "owns": "OWNS",
-                "located_in": "LOCATED_IN", "contains": "LOCATED_IN",
+                "located_in": "LOCATED_IN",
+                "contains": "CONTAINS",
                 "leads": "LEADS",
-                "serves": "WORKS_FOR", "controlled_by": "WORKS_FOR", "controls": "WORKS_FOR",
+                "serves": "WORKS_FOR",
+                "controlled_by": "CONTROLLED_BY",
+                "controls": "CONTROLS",
                 "works_for": "WORKS_FOR",
-                "created_by": "DERIVES_FROM", "descends_from": "DERIVES_FROM",
-                "derives_from": "DERIVES_FROM", "instance_of": "DERIVES_FROM",
+                "created_by": "DERIVES_FROM",
+                "descends_from": "DERIVES_FROM",
+                "derives_from": "DERIVES_FROM",
+                "instance_of": "INSTANCE_OF",
                 "subtype_of": "SUBTYPE_OF",
                 "worships": "REVERES",
                 "participates_in": "PARTICIPATES_IN",

packages/agents/src/monitor_agents/canonkeeper.py

@@ -304,32 +304,63 @@ def parse_game_rules(reasoning: str) -> List[Dict[str, Any]]:
 # ---------------------------------------------------------------------------
 
 
+def _split_named_list(raw: str) -> List[str]:
+    """Split a comma/semicolon/bullet list of named system terms into clean items."""
+    cleaned = raw.strip().strip("[]")
+    cleaned = re.sub(r"\([^)]*\)", "", cleaned)
+    cleaned = cleaned.replace("•", ",")
+    items = [part.strip(" -\t") for part in re.split(r",|;", cleaned)]
+    return [item[:100] for item in items if item and item.lower() not in {"none", "n/a", "na"}]
+
+
 def parse_character_sheet(reasoning: str) -> Dict[str, Any]:
-    """
-    Parse DSPy character sheet output lines.
-    Format:
-      ATTR | name | abbrev | min | max | default | modifier_formula
-      SKILL | name | linked_attribute | description
-      RESOURCE | name | abbrev | min | recovers_on | depleted_effect
-      MECHANIC | type | formula | success_type
-    """
+    """Parse structured or semi-structured system-schema output from the analyzer prompt."""
     attributes: List[Dict[str, Any]] = []
     skills: List[Dict[str, Any]] = []
     resources: List[Dict[str, Any]] = []
+    powers: List[Dict[str, Any]] = []
+    subsystems: List[Dict[str, Any]] = []
     core_mechanic: Dict[str, Any] = {}
 
-    for line in reasoning.splitlines():
-        line = line.strip()
+    attr_aliases = {"ATTR", "ATTRIBUTE", "ATTRIBUTES", "ABILITY", "ABILITIES", "STAT", "STATS", "TRAIT", "TRAITS"}
+    skill_aliases = {"SKILL", "SKILLS", "KNACK", "KNACKS", "APPROACH", "APPROACHES"}
+    resource_aliases = {"RESOURCE", "RESOURCES", "TRACK", "TRACKS"}
+    power_aliases = {"POWER", "POWERS", "DISCIPLINE", "DISCIPLINES", "ARCANA", "SCHOOL", "SCHOOLS", "EDGE", "EDGES", "FEAT", "FEATS", "MERIT", "MERITS", "ADVANTAGE", "ADVANTAGES"}
+    subsystem_aliases = {"SUBSYSTEM", "SUBSYSTEMS", "MODULE", "MODULES", "MODE", "MODES"}
+    mechanic_aliases = {"MECHANIC", "MECHANICS", "CORE_MECHANIC", "SYSTEM"}
+
+    for raw_line in reasoning.splitlines():
+        line = raw_line.strip()
         if not line:
             continue
-        parts = [p.strip() for p in line.split("|")]
+
+        # Accept human-readable heading lines like "Attributes: Brawn, Finesse, Wits"
+        if "|" not in line and ":" in line:
+            heading, body = line.split(":", 1)
+            heading_key = heading.strip().upper().replace(" ", "_")
+            parts = [heading_key, body.strip()]
+        else:
+            parts = [p.strip() for p in line.split("|")]
+
         keyword = parts[0].upper() if parts else ""
 
-        if keyword == "ATTR" and len(parts) >= 3:
+        if keyword in attr_aliases and len(parts) >= 2:
             try:
+                if len(parts) == 2 or (len(parts) <= 3 and any(sep in parts[1] for sep in [",", ";"])):
+                    for name in _split_named_list(" ".join(parts[1:])):
+                        attributes.append({
+                            "name": name,
+                            "abbreviation": name[:10].upper(),
+                            "min_value": 1,
+                            "max_value": 20,
+                            "default_value": 10,
+                            "modifier_formula": None,
+                        })
+                    continue
+
                 attributes.append({
                     "name": parts[1][:100],
-                    "abbreviation": parts[2][:10] if len(parts) > 2 else parts[1][:10],
+                    "abbreviation": parts[2][:10] if len(parts) > 2 and parts[2] else parts[1][:10].upper(),
                     "min_value": int(parts[3]) if len(parts) > 3 and parts[3].lstrip("-").isdigit() else 1,
                     "max_value": int(parts[4]) if len(parts) > 4 and parts[4].lstrip("-").isdigit() else 20,
                     "default_value": int(parts[5]) if len(parts) > 5 and parts[5].lstrip("-").isdigit() else 10,
@@ -338,33 +369,84 @@ def parse_character_sheet(reasoning: str) -> Dict[str, Any]:
             except (ValueError, IndexError):
                 continue
 
-        elif keyword == "SKILL" and len(parts) >= 2:
+        elif keyword in skill_aliases and len(parts) >= 2:
             try:
+                if len(parts) == 2 or (len(parts) <= 3 and any(sep in parts[1] for sep in [",", ";"])):
+                    for name in _split_named_list(" ".join(parts[1:])):
+                        skills.append({
+                            "name": name,
+                            "linked_attribute": None,
+                            "description": None,
+                        })
+                    continue
+
                 skills.append({
                     "name": parts[1][:100],
                     "linked_attribute": parts[2][:100] if len(parts) > 2 and parts[2] else None,
-                    "description": parts[3][:500] if len(parts) > 3 else None,
+                    "description": parts[3][:500] if len(parts) > 3 and parts[3] else None,
                 })
             except (ValueError, IndexError):
                 continue
 
-        elif keyword == "RESOURCE" and len(parts) >= 2:
+        elif keyword in resource_aliases and len(parts) >= 2:
             try:
+                if len(parts) == 2 or (len(parts) <= 3 and any(sep in parts[1] for sep in [",", ";"])):
+                    for name in _split_named_list(" ".join(parts[1:])):
+                        resources.append({
+                            "name": name,
+                            "abbreviation": name[:10].upper(),
+                            "min_value": 0,
+                            "recovers_on": None,
+                            "depleted_effect": None,
+                            "calculation": None,
+                        })
+                    continue
+
                 resources.append({
                     "name": parts[1][:100],
-                    "abbreviation": parts[2][:10] if len(parts) > 2 and parts[2] else parts[1][:10],
+                    "abbreviation": parts[2][:10] if len(parts) > 2 and parts[2] else parts[1][:10].upper(),
                     "min_value": int(parts[3]) if len(parts) > 3 and parts[3].lstrip("-").isdigit() else 0,
                     "recovers_on": parts[4][:100] if len(parts) > 4 and parts[4] else None,
                     "depleted_effect": parts[5][:200] if len(parts) > 5 and parts[5] else None,
+                    "calculation": parts[6][:200] if len(parts) > 6 and parts[6] else None,
+                })
+            except (ValueError, IndexError):
+                continue
+
+        elif keyword in power_aliases and len(parts) >= 2:
+            try:
+                if len(parts) == 2 or (len(parts) <= 3 and any(sep in parts[1] for sep in [",", ";"])):
+                    for name in _split_named_list(" ".join(parts[1:])):
+                        powers.append({
+                            "name": name,
+                            "category": "power",
+                            "description": None,
+                        })
+                    continue
+
+                powers.append({
+                    "name": parts[1][:100],
+                    "category": parts[2][:100] if len(parts) > 2 and parts[2] else "power",
+                    "description": parts[3][:500] if len(parts) > 3 and parts[3] else None,
+                })
+            except (ValueError, IndexError):
+                continue
+
+        elif keyword in subsystem_aliases and len(parts) >= 2:
+            try:
+                subsystems.append({
+                    "name": parts[1][:100],
+                    "scope": parts[2][:100] if len(parts) > 2 and parts[2] else "custom",
+                    "description": parts[3][:500] if len(parts) > 3 and parts[3] else None,
                 })
             except (ValueError, IndexError):
                 continue
 
-        elif keyword == "MECHANIC" and len(parts) >= 2 and not core_mechanic:
+        elif keyword in mechanic_aliases and len(parts) >= 2 and not core_mechanic:
             try:
                 core_mechanic = {
                     "type": parts[1].lower() if len(parts) > 1 else "d20",
-                    "formula": parts[2][:200] if len(parts) > 2 and parts[2] else "Roll and compare",
+                    "formula": parts[2][:200] if len(parts) > 2 and parts[2] else (parts[1][:200] if len(parts) == 2 else "Roll and compare"),
                     "success_type": parts[3].lower() if len(parts) > 3 and parts[3] else "meet_or_beat",
                 }
             except (ValueError, IndexError):
@@ -374,6 +456,8 @@ def parse_character_sheet(reasoning: str) -> Dict[str, Any]:
         "attributes": attributes,
         "skills": skills,
         "resources": resources,
+        "powers": powers,
+        "subsystems": subsystems,
         "core_mechanic": core_mechanic,
     }