Merge pull request #106 from neph1/update-v0.39

Update v0.39

Author: neph1

llm_config.yaml

@@ -34,7 +34,9 @@ STORY_BACKGROUND_PROMPT: "[USER_START] For an RPG described as {story_type} set
 START_LOCATION_PROMPT: '[Story context: {story_context}]; Zone info: {zone_info}; Item json example: {{"name":"", "type":"", "short_descr":"10 words"}}, type can be "Weapon", "Wearable", "Other" or "Money"; Npc example: {{"name":"", "sentiment":"", "race":"", "gender":"m, f, or n", "level":(int), "description":"25 words"}} ; Exit json example: {{"direction":"", "name":"name of new location", "short_descr":"exit description"}}; [USER_START] For a {story_type}, come up with a name for the location with this description: {location_description}. {items_prompt} {spawn_prompt} Add a brief description, and one to three additional exits leading to new locations. Fill in this JSON template and do not write anything else: {{"name": "", "exits":[], "items":[], "npcs":[]}}.'
 STORY_PLOT_PROMPT: "[USER_START] For an RPG described as {story_type} set in a world described as {world_mood}, {world_info}. Based on the following background: {story_background} write an innovative and engaging plot that the player can become part of. Use less than 400 words."
 WORLD_ITEMS: '<context>{context}</context>\n[USER_START] Using the information supplied inside the <context> tags, come up with 7 common items that can be found in the world. Item example: {item_template}, type is one of: {item_types}; Reply with a list of items in JSON format and do not write anything else: {{"items": []}}.'
+WORLD_ITEM_SINGLE: '<context>{context}</context>\n[USER_START] Using the information supplied inside the <context> tags, come up with a common item that can be found in the world. {previously_generated}Item example: {item_template}, type is one of: {item_types}; Reply with a single item in JSON format and do not write anything else: {{"item": {{}}}}.'
 WORLD_CREATURES: '<context>{context}</context>\n[USER_START] Using the information supplied inside the <context> tags, come up with 5 creatures of various level and sentiment that can be found in the world. Consider the role of the creature in the story and whether it will be friendly or hostile to the player. Creature example: {creature_template}. Reply with a list of creatures in JSON format and do not write anything else: {{"creatures": []}}.'
+WORLD_CREATURE_SINGLE: '<context>{context}</context>\n[USER_START] Using the information supplied inside the <context> tags, come up with a creature that can be found in the world. {previously_generated}Consider the role of the creature in the story and whether it will be friendly or hostile to the player. Creature example: {creature_template}. Reply with a single creature in JSON format and do not write anything else: {{"creature": {{}}}}.'
 GOAL_PROMPT: '[Characters:{characters}][Sentiments towards characters: {sentiments}]  [Last action: {last_action}] [Location: {location}] [Known locations: {locations}][Acting character: {character}] [Actions available:{actions}] [USER_START] For {character_name}, describe a goal that goes along with their character description that involves an item, a character or a location in the prompt. Then construct up to three tasks that will lead towards the achievement of said goal. Fill in the following JSON template: {{"goal":"", "tasks":[{"action":"", "what":""}, {"action":"", "what":""}, {"action":"", "what":""}]}}'
 JSON_GRAMMAR: "root   ::= object\nvalue  ::= object | array | string | number | (\"true\" | \"false\" | \"null\") ws\n\nobject ::=\n  \"{\" ws (\n            string \":\" ws value\n    (\",\" ws string \":\" ws value)*\n  )? \"}\" ws\n\narray  ::=\n  \"[\" ws (\n            value\n    (\",\" ws value)*\n  )? \"]\" ws\n\nstring ::=\n  \"\\\"\" (\n    [^\"\\\\] |\n    \"\\\\\" ([\"\\\\/bfnrt] | \"u\" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes\n  )* \"\\\"\" ws\n\nnumber ::= (\"-\"? ([0-9] | [1-9] [0-9]*)) (\".\" [0-9]+)? ([eE] [-+]? [0-9]+)? ws\n\n# Optional space: by convention, applied in this grammar after literal chars when allowed\nws ::= ([ \\t\\n] ws)?"
 PLAYER_ENTER_PROMPT: '<context>{context}</context> Zone info: {zone_info}; Npc example: {npc_template}.\n[USER_START] The player has just re-entered this location: {location_info}. Consider whether any items, npcs or mobs should be spawned. For mobs, only enter the name of race. Fill in this JSON template and do not write anything else: {{"items":[], "npcs":[] "mobs":[]}}.'

stories/chat_room/story.py

@@ -1,19 +1,10 @@
 import pathlib
 import sys
-from typing import Optional, Generator
 
-import tale
 from tale import parse_utils
-from tale.base import Location
 from tale.driver import Driver
 from tale.json_story import JsonStory
-from tale.llm.llm_ext import DynamicStory
 from tale.main import run_from_cmdline
-from tale.player import Player, PlayerConnection
-from tale.charbuilder import PlayerNaming
-from tale.story import *
-from tale.skills.weapon_type import WeaponType
-from tale.zone import Zone
 
 class Story(JsonStory):
 

stories/prancingllama/story.py

@@ -14,7 +14,7 @@
 from tale.skills.skills import SkillType
 from tale.story import *
 from tale.skills.weapon_type import WeaponType
-from tale.story import StoryContext
+from tale.story_context import StoryContext
 from tale.zone import Zone
 
 class Story(DynamicStory):

tale/json_util.py

@@ -0,0 +1,58 @@
+import json
+import re
+from typing import Any, Optional
+
+def strip_markdown_fences(s: str) -> str:
+    """Remove ```json fences or plain ``` fences."""
+    return re.sub(r'^```(?:json)?|```$', '', s.strip(), flags=re.MULTILINE).strip()
+
+def extract_json_block(s: str) -> str:
+    """Extract the first {...} or [...] block if there is extra text around it."""
+    match = re.search(r"(\{.*\}|\[.*\])", s, flags=re.DOTALL)
+    return match.group(1) if match else s
+
+def unwrap_double_encoded(s: str) -> str:
+    """Unwrap when JSON is a quoted string containing JSON."""
+    try:
+        parsed = json.loads(s)
+        if isinstance(parsed, str):
+            return parsed
+    except Exception:
+        pass
+    return s
+
+def fix_trailing_commas(s: str) -> str:
+    """Remove trailing commas before ] or }."""
+    return re.sub(r',\s*([}\]])', r'\1', s)
+
+def normalize_literals(s: str) -> str:
+    """Normalize Python-style literals to JSON literals."""
+    return (s.replace("None", "null")
+             .replace("True", "true")
+             .replace("False", "false"))
+
+def sanitize_json(s: str) -> str:
+    """Apply a pipeline of cleanup steps before parsing."""
+    if not s:
+        return ""
+    s = strip_markdown_fences(s)
+    s = extract_json_block(s)
+    s = unwrap_double_encoded(s)
+    s = fix_trailing_commas(s)
+    s = normalize_literals(s)
+    return s.strip()
+
+def safe_load(s: str) -> Optional[Any]:
+    """
+    Try to parse JSON string from LLM output with progressive cleanup.
+    Returns Python object or raises last error if unrecoverable.
+    """
+    cleaned = sanitize_json(s)
+    try:
+        return json.loads(cleaned)
+    except json.JSONDecodeError:
+        # Try double-unwrapping in case of double-encoded JSON
+        try:
+            return json.loads(json.loads(cleaned))
+        except Exception:
+            raise

tale/llm/character.py

@@ -5,9 +5,8 @@
 import json
 import random
 
-from tale import _MudContext, parse_utils
+from tale import json_util, parse_utils
 from tale.base import Location
-from tale.errors import LlmResponseException
 from tale.llm import llm_config
 from tale.llm.contexts.ActionContext import ActionContext
 from tale.llm.contexts.CharacterContext import CharacterContext
@@ -58,7 +57,7 @@ def generate_dialogue(self,
         request_body['grammar'] = self.json_grammar
         response = self.io_util.synchronous_request(request_body, prompt=prompt, context=context.to_prompt_string())
         try:
-            json_result = json.loads(parse_utils.sanitize_json(response))
+            json_result = json_util.safe_load(response)
             text = json_result["response"]
             if isinstance(text, list):
                 text = text[0]
@@ -84,7 +83,7 @@ def generate_character(self, character_context: CharacterContext) -> CharacterV2
             request_body[self.json_grammar_key] = self.json_grammar
         result = self.io_util.synchronous_request(request_body, prompt=prompt, context=character_context.to_prompt_string())
         try:
-            json_result = json.loads(parse_utils.sanitize_json(result))
+            json_result = json_util.safe_load(result)
         except JSONDecodeError as exc:
             print(exc)
             return None
@@ -165,7 +164,7 @@ def free_form_action(self, action_context: ActionContext) -> list:
             text = self.io_util.synchronous_request(request_body, prompt=prompt, context=action_context.to_prompt_string())
             if not text:
                 return None
-            response = json.loads(parse_utils.sanitize_json(text))
+            response = json_util.safe_load(text)
             if isinstance(response, dict):
                 return [ActionResponse(response)]
             actions = []
@@ -191,5 +190,5 @@ def request_follow(self, follow_context: FollowContext) -> FollowResponse:
         text = self.io_util.synchronous_request(request_body, prompt=prompt)
         if not text:
             return None
-        return FollowResponse(json.loads(parse_utils.sanitize_json(text)))
+        return FollowResponse(json_util.safe_load(text))
 

tale/llm/llm_utils.py

@@ -206,20 +206,20 @@ def generate_start_zone(self, location_desc: str, story_type: str, story_context
         world_generation_context = WorldGenerationContext(story_context=story_context, story_type=story_type, world_info=world_info, world_mood=world_info['world_mood'])
         return self._world_building.generate_start_zone(location_desc, context=world_generation_context)
 
-    def generate_world_items(self, story_context: str = '', story_type: str = '', world_info: str = '', world_mood: int = None, item_types: list = []) -> WorldItemsResponse:
+    def generate_world_items(self, story_context: str = '', story_type: str = '', world_info: str = '', world_mood: int = None, item_types: list = [], count: int = 7) -> WorldItemsResponse:
         world_generation_context = WorldGenerationContext(story_context=story_context or self.__story_context,
                                                             story_type=story_type or self.__story_type,
                                                             world_info=world_info or self.__world_info,
                                                             world_mood=world_mood or self.__story.config.world_mood)
-        return self._world_building.generate_world_items(world_generation_context, item_types=item_types)
+        return self._world_building.generate_world_items(world_generation_context, item_types=item_types, count=count)
 
 
-    def generate_world_creatures(self, story_context: str = '', story_type: str = '', world_info: str = '', world_mood: int = None) -> WorldCreaturesResponse:
+    def generate_world_creatures(self, story_context: str = '', story_type: str = '', world_info: str = '', world_mood: int = None, count: int = 5) -> WorldCreaturesResponse:
         world_generation_context = WorldGenerationContext(story_context=story_context or self.__story_context,
                                                             story_type=story_type or self.__story_type,
                                                             world_info=world_info or self.__world_info,
                                                             world_mood=world_mood or self.__story.config.world_mood)
-        return self._world_building.generate_world_creatures(world_generation_context)
+        return self._world_building.generate_world_creatures(world_generation_context, count=count)
 
     def generate_random_spawn(self, location: Location, zone_info: dict) -> bool:
         return self._world_building.generate_random_spawn(location=location, 

tale/llm/quest_building.py

@@ -1,7 +1,6 @@
 
 from copy import deepcopy
-import json
-from tale import parse_utils
+from tale import json_util, parse_utils
 from tale.base import Location
 from tale.llm import llm_config
 from tale.llm.contexts.WorldGenerationContext import WorldGenerationContext
@@ -40,5 +39,5 @@ def generate_note_quest(self, context: WorldGenerationContext, zone_info: str) -
         if self.json_grammar_key:
             request_body[self.json_grammar_key] = self.json_grammar
         text = self.io_util.synchronous_request(request_body, prompt=prompt, context=context)
-        quest_data = json.loads(parse_utils.sanitize_json(text))
+        quest_data = json_util.safe_load(text)
         return Quest(name=quest_data['name'], type=QuestType[quest_data['type'].upper()], reason=quest_data['reason'], target=quest_data['target'])