Improved. Added pause_time.

Author: jayl940712

openhands/agenthub/gui_agent/osworld_agent.py

@@ -47,6 +47,10 @@ def convert_action_to_message(action: OSWorldInteractiveAction) -> Message:
     output_ids = getattr(llm_response.choices[0], 'output_ids', None)
     logprobs = getattr(llm_response.choices[0], 'logprobs', None)
 
+    text_content = assistant_msg.content
+    if text_content is None:
+        text_content = ''
+
     return Message(
         role=getattr(assistant_msg, 'role', 'assistant'),
         content=[TextContent(text=assistant_msg.content)],
@@ -63,11 +67,19 @@ def convert_message_action_to_message(
     ) -> Message:
     text_content = action.content
     if include_a11y_tree:
-        accessibility_tree = linearize_accessibility_tree(action.accessibility_tree)
-        text_content += f"\n\nAccessibility Tree:\n{accessibility_tree}"
+        accessibility_tree = action.accessibility_tree
+        if accessibility_tree is None or len(accessibility_tree) < 1:
+            logger.error('Accessibility tree is None or empty, skipping')
+        else:
+            accessibility_tree = linearize_accessibility_tree(accessibility_tree)
+            text_content += f"\n\nAccessibility Tree:\n{accessibility_tree}"
     content = [TextContent(text=text_content)]
     if include_screenshot:
-        content.append(ImageContent(image_urls=action.image_urls))
+        image_urls = action.image_urls
+        if image_urls is None or len(image_urls) < 1:
+            logger.error('Image urls is None or empty, skipping')
+        else:
+            content.append(ImageContent(image_urls=image_urls))
     return Message(
         role='user',
         content=content,
@@ -82,11 +94,19 @@ def convert_observation_to_message(
     if isinstance(observation, OSWorldOutputObservation):
         prompt_text = OSWORLD_OBSERVATION_FEEDBACK_PROMPT.format(instruction=instruction)
         if include_a11y_tree:
-            accessibility_tree = linearize_accessibility_tree(observation.accessibility_tree)
-            prompt_text += f"\n\nAccessibility Tree:\n{accessibility_tree}"
+            accessibility_tree = observation.accessibility_tree
+            if accessibility_tree and len(accessibility_tree) >= 1:
+                logger.error('Accessibility tree is None or empty, skipping')
+            else:
+                accessibility_tree = linearize_accessibility_tree(accessibility_tree)
+                prompt_text += f"\n\nAccessibility Tree:\n{accessibility_tree}"
         content = [TextContent(text=prompt_text)]
         if include_screenshot:
-            content.append(ImageContent(image_urls=observation.image_urls))
+            image_url = observation.image_urls
+            if image_url is None or len(image_url) < 1:
+                logger.error('Image urls is None or empty, skipping')
+            else:
+                content.append(ImageContent(image_urls=image_url))
         return Message(
             role='tool', # or user?
             content=content,
@@ -106,11 +126,13 @@ def convert_message_action_to_message_full_state(
     action: MessageAction,
     include_a11y_tree: bool = True,
     ) -> Message:
-    test_content = action.content
+    text_content = action.content
     if include_a11y_tree:
-        accessibility_tree = linearize_accessibility_tree(action.accessibility_tree)
-        test_content += f"\n\nAccessibility Tree:\n{accessibility_tree}"
-    content = [TextContent(text=action.content)]
+        accessibility_tree = action.accessibility_tree
+        if accessibility_tree and len(accessibility_tree) > 0:
+            accessibility_tree = linearize_accessibility_tree(action.accessibility_tree)
+            text_content += f"\n\nAccessibility Tree:\n{accessibility_tree}"
+    content = [TextContent(text=text_content)]
     content.append(ImageContent(image_urls=action.image_urls))
     content.append(TextContent(text=action.accessibility_tree))
     return Message(
@@ -126,8 +148,10 @@ def convert_observation_to_message_full_state(
     if isinstance(observation, OSWorldOutputObservation):
         prompt_text = OSWORLD_OBSERVATION_FEEDBACK_PROMPT.format(instruction=instruction)
         if include_a11y_tree:
-            accessibility_tree = linearize_accessibility_tree(observation.accessibility_tree)
-            prompt_text += f"\n\nAccessibility Tree:\n{accessibility_tree}"
+            accessibility_tree = observation.accessibility_tree
+            if accessibility_tree and len(accessibility_tree) > 0:
+                accessibility_tree = linearize_accessibility_tree(accessibility_tree)
+                prompt_text += f"\n\nAccessibility Tree:\n{accessibility_tree}"
         content = [TextContent(text=prompt_text)]
 
         # We always add screenshot and accessibility tree to the message
@@ -168,6 +192,7 @@ def __init__(
         """
         super().__init__(llm, config)
 
+        self.pause_time = 0.0
         self.system_prompt = os.path.join(os.path.dirname(__file__), 'prompts', 'system_prompt_osworld.j2')
         with open(self.system_prompt, 'r') as file:
             self.system_prompt = file.read()
@@ -296,10 +321,14 @@ def step(self, state: State) -> Action:
         }
         params['tools'] = self.tools
         params['extra_body'] = {'metadata': state.to_llm_metadata(agent_name=self.name)}
+        import pdb; pdb.set_trace()
         response = self.llm.completion(**params)
         import pdb; pdb.set_trace()
         logger.debug(f'Response from LLM: {response}')
         action = codeact_function_calling.response_to_actions(response, timeout=self.config.action_timeout)
+        if self.pause_time > 0.5:
+            logger.info(f'Setting pause time to {self.pause_time} seconds for agentic action')
+            action.pause_time = self.pause_time
         logger.debug(f'Actions after response_to_actions: {action}')
         return action
 

openhands/agenthub/gui_agent/prompts/osworld.py

@@ -1,10 +1,10 @@
-OSWORLD_OBSERVATION_FEEDBACK_PROMPT = """Action executed. Please generate the next move according to the UI screenshot and instruction. And you can refer to the previous actions and observations for reflection.
+OSWORLD_OBSERVATION_FEEDBACK_PROMPT = """Action executed. Please generate the next move according to the UI screenshot and instruction.
 
 Instruction: {instruction}
 """
 
-ERROR_OBSERVATION_FEEDBACK_PROMPT = """Action failed. Please refer to previous message for the UI screenshot. Please continue working on the task according to the instruction.
-Error message: {error_message}.
+ERROR_OBSERVATION_FEEDBACK_PROMPT = """Action failed. Please continue working on the task according to the instruction.
+Error message: {error_message}
 
 Instruction: {instruction}
 """

openhands/events/action/os.py

@@ -40,6 +40,7 @@ class OSWorldInteractiveAction(Action):
     action: str = ActionType.OSWORLD_INTERACTIVE
     runnable: ClassVar[bool] = True
     security_risk: ActionSecurityRisk | None = None
+    pause_time: float = 0.0
 
     def __post_init__(self):
         if self.params is None:

openhands/events/observation/osworld.py

@@ -10,8 +10,8 @@ class OSWorldOutputObservation(Observation):
     observation: str = ObservationType.OSWORLD
     command: str = field(default='')
     content: str = field(default='')
-    screenshot: str = field(repr=False, default='')  # don't show in repr, in base64 format
-    accessibility_tree: str = field(repr=False, default='')
+    screenshot: str | None = None
+    accessibility_tree: str | None = None
     tool_call_id: str | None = None
     name: str = ''
 
@@ -21,7 +21,10 @@ def message(self) -> str:
 
     @property
     def image_urls(self) -> list[str]:
-        return [f'data:image/png;base64,{self.screenshot}']
+        if self.screenshot:
+            return [f'data:image/png;base64,{self.screenshot}']
+        else:
+            return []
 
     def __str__(self) -> str:
         ret = (

openhands/nvidia/os_world/osworld_utils.py

@@ -2,6 +2,7 @@
 import time
 import os
 import pandas as pd
+import base64
 import numpy as np
 import asyncio
 from evaluation.utils.shared import (  # type: ignore
@@ -89,8 +90,8 @@ def get_config(
         ensure_thinking_end_properly=agent_config['ensure_thinking_end_properly'], # set to true only if using text based server for training.
         action_timeout=30.0, # 30 seconds per action
         strict_loop_detector=agent_config['strict_loop_detector'], # set to true only if training
-        enable_vision=False,
-        enable_a11y_tree=True,
+        enable_vision=agent_config['enable_vision'],
+        enable_a11y_tree=agent_config['enable_a11y_tree'],
     )
     config.set_agent_config(agent_config)
     return config
@@ -114,6 +115,7 @@ def get_instruction(instance: pd.Series | dict, metadata: EvalMetadata, runtime:
     if include_screenshot:
         image = runtime.get_vm_screenshot()
         if image:
+            image = base64.b64encode(image).decode('utf-8')
             image_url = [f'data:image/png;base64,{image}']
 
     return MessageAction(content=instruction, image_urls=image_url, accessibility_tree=accessibility_tree)
@@ -218,6 +220,9 @@ async def run_agent(
     message_action = get_instruction(instance, metadata, runtime)
     try:
         agent = create_agent(config)
+        # Set pause time for agent to wait for the screenshot to be taken
+        # You can play around with this value to find appropriate value for setup.
+        agent.pause_time = 4.0
         job_details.agent = agent
         controller, initial_state = create_controller(
             agent=agent,

openhands/nvidia/registry.py

@@ -17,6 +17,8 @@
     'max_iterations': 2,
     'ensure_thinking_end_properly': False,
     'strict_loop_detector': False,
+    'enable_vision': False,
+    'enable_a11y_tree': False,
 }
 
 

openhands/runtime/impl/singularity/osworld_singularity_runtime.py

@@ -858,7 +858,7 @@ def osworld_interactive(self, action) -> 'Observation':
             if method == 'execute_action':
                 return self._handle_execute_action(params)
             elif method == 'execute_agentic_action':
-                return self._handle_execute_agentic_action(params, action.tool_call_metadata)
+                return self._handle_execute_agentic_action(params, action.tool_call_metadata, action.pause_time)
             elif method == 'get_screenshot':
                 return self._handle_get_screenshot()
             elif method == 'get_accessibility_tree':
@@ -919,7 +919,7 @@ def _handle_execute_action(self, params: dict) -> 'Observation':
                 exit_code=1,
             )
 
-    def _handle_execute_agentic_action(self, params: dict, tool_call_metadata: ToolCallMetadata | None) -> 'Observation':
+    def _handle_execute_agentic_action(self, params: dict, tool_call_metadata: ToolCallMetadata | None, pause_time: float = 0.0) -> 'Observation':
         """Handle execute_action - PyAutoGUI actions like CLICK, TYPING, etc."""
         from openhands.events.observation.osworld import OSWorldOutputObservation  
         from openhands.events.observation import ErrorObservation   
@@ -946,15 +946,17 @@ def _handle_execute_agentic_action(self, params: dict, tool_call_metadata: ToolC
         result = self.execute_vm_action(action_data)
 
         if result.get('status') == 'success':
+            if pause_time > 0.5:
+                time.sleep(pause_time)
             if include_screenshot:
                 screenshot_bytes = self.get_vm_screenshot()
-                screenshot_bytes = base64.b64encode(screenshot_bytes).decode('utf-8')
+                if screenshot_bytes:
+                    screenshot_bytes = base64.b64encode(screenshot_bytes).decode('utf-8')
             else:
                 screenshot_bytes = None
 
             if include_a11y_tree:
                 accessibility_tree = self.get_vm_accessibility_tree()
-                #accessibility_tree = linearize_accessibility_tree(accessibility_tree)
             else:
                 accessibility_tree = None
 

scripts/tests/test_async_server_osworld.py

@@ -42,7 +42,7 @@ def test_server(
         'log_completions': False,
         'native_tool_calling': True,
         'temperature': 0.6,
-        'max_iterations': 35,
+        'max_iterations': 3,
     }
 
     print('Starting server')
@@ -77,7 +77,7 @@ def test_server(
     start = time.time()
     # set timeout approriate to terminate
     results = test_server(
-        total_jobs=1, max_parallel_jobs=1, allow_skip_eval=False, timeout=6000
+        total_jobs=4, max_parallel_jobs=4, allow_skip_eval=False, timeout=6000
     )
     # Don't print full messages
     print(f'Time taken: {time.time() - start}')

scripts/tests/test_osworld_utils.py

@@ -18,14 +18,14 @@
 async def run(instance):
     max_iterations = 35
     sampling_params = {
-        'model': 'deepseek/deepseek-chat',
-        'api_key': os.getenv('DEEPSEEK_API_KEY', ''),
+        'model': 'gpt-5-mini-2025-08-07',
+        'api_key': os.getenv('OPENAI_API_KEY', ''),
         'modify_params': False,
         'log_completions': True,
         'native_tool_calling': True,
-        'temperature': 0.6,
+        'temperature': 1,
     }
-    llm_config = LLMConfig(base_url='https://api.deepseek.com/chat/completions', **sampling_params)
+    llm_config = LLMConfig(base_url='https://api.openai.com/v1', **sampling_params)
 
 
     job_details = JobDetails(
@@ -34,6 +34,8 @@ async def run(instance):
         llm_config=llm_config,
     )
     job_details.agent_config['max_iterations'] = max_iterations
+    job_details.agent_config['enable_vision'] = True
+    job_details.agent_config['enable_a11y_tree'] = False
     job_details.timer = PausableTimer(timeout=500)
     job_details.timer.start()