refact: mcpbench -> mcpmark, deprecate openai trace

Author: zjwu0522

README.md

@@ -17,9 +17,6 @@ Before running MCPMark you need to prepare the environment for the MCP service y
 All environment variables **must** be set in a file named `.mcp_env` in your project root. Example:
 
 ```env
-# For OpenAI API tracing (optional)
-OPENAI_TRACE_API_KEY="sk-proj-xxx-xxx-xx"
-
 # Service Credentials
 ## Notion
 SOURCE_NOTION_API_KEY="your-source-notion-api-key"   # For Source Hub (templates)

pipeline.py

@@ -24,18 +24,16 @@
 
 def main():
     """Main entry point for the evaluation pipeline."""
-    parser = argparse.ArgumentParser(
-        description="MCPMark Unified Evaluation Pipeline."
-    )
+    parser = argparse.ArgumentParser(description="MCPMark Unified Evaluation Pipeline.")
 
-    supported_services = MCPServiceFactory.get_supported_services()
+    supported_mcp_services = MCPServiceFactory.get_supported_mcp_services()
     supported_models = ModelConfig.get_supported_models()
 
     # Main configuration
     parser.add_argument(
         "--mcp",
         default="notion",
-        choices=supported_services,
+        choices=supported_mcp_services,
         help="MCP service to use (default: notion)",
     )
     parser.add_argument(
@@ -59,7 +57,6 @@ def main():
         "--timeout", type=int, default=300, help="Timeout in seconds for each task"
     )
 
-
     # Output configuration
     parser.add_argument(
         "--output-dir",
@@ -71,7 +68,7 @@ def main():
     # Load arguments and environment variables
     args = parser.parse_args()
     load_dotenv(dotenv_path=".mcp_env", override=False)
-    
+
     # Generate default exp-name if not provided
     if args.exp_name is None:
         args.exp_name = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
@@ -81,13 +78,17 @@ def main():
     model_list = [m.strip() for m in args.models.split(",") if m.strip()]
     if not model_list:
         parser.error("No valid models provided")
-    
+
     # Log warning for unsupported models but don't error
     unsupported_models = [m for m in model_list if m not in supported_models]
     if unsupported_models:
-        logger.warning(f"Using unsupported models: {', '.join(unsupported_models)}. Will use OPENAI_BASE_URL and OPENAI_API_KEY from environment.")
+        logger.warning(
+            f"Using unsupported models: {', '.join(unsupported_models)}. Will use OPENAI_BASE_URL and OPENAI_API_KEY from environment."
+        )
 
-    logger.info(f"Running evaluation for {len(model_list)} model(s): {', '.join(model_list)}")
+    logger.info(
+        f"Running evaluation for {len(model_list)} model(s): {', '.join(model_list)}"
+    )
 
     # Run evaluation for each model
     for i, model in enumerate(model_list, 1):
@@ -97,17 +98,19 @@ def main():
 
         # Initialize and run the evaluation pipeline for this model
         pipeline = MCPEvaluator(
-            service=args.mcp,
+            mcp_service=args.mcp,
             model=model,
             timeout=args.timeout,
             exp_name=args.exp_name,
             output_dir=args.output_dir,
         )
 
         pipeline.run_evaluation(args.tasks)
-        logger.info(f"✓ Evaluation completed for {model}. Results saved in: {pipeline.base_experiment_dir}")
+        logger.info(
+            f"✓ Evaluation completed for {model}. Results saved in: {pipeline.base_experiment_dir}"
+        )
 
-    logger.info(f"\n{'='*60}")
+    logger.info(f"\n{'=' * 60}")
     logger.info(f"✓ All evaluations completed for {len(model_list)} model(s)")
     logger.info(f"{'=' * 60}")
 

results_parser.py

@@ -101,7 +101,9 @@ def validate_and_gather_metrics(
 
         # Check pipeline errors
         error_msg = meta.get("execution_result", {}).get("error_message")
-        if error_msg and any(err in error_msg for err in src.evaluator.PIPELINE_RETRY_ERRORS):
+        if error_msg and any(
+            err in error_msg for err in src.evaluator.PIPELINE_RETRY_ERRORS
+        ):
             has_retry_error = True
 
         # Collect metrics
@@ -138,11 +140,15 @@ def validate_and_gather_metrics(
     return True, metrics, None
 
 
-def plot_metrics(metrics: Dict[str, Dict[str, float]], exp_name: str, service: str, show: bool):
+def plot_metrics(
+    metrics: Dict[str, Dict[str, float]], exp_name: str, service: str, show: bool
+):
     """Create a bar chart visualizing success rate and avg tokens; annotate avg turns."""
 
     # Sort by success-rate (desc)
-    sorted_items = sorted(metrics.items(), key=lambda x: x[1]["success_rate"], reverse=True)
+    sorted_items = sorted(
+        metrics.items(), key=lambda x: x[1]["success_rate"], reverse=True
+    )
     models = [m for m, _ in sorted_items]
     success_rates = [item[1]["success_rate"] for item in sorted_items]
     avg_tokens = [item[1]["avg_tokens"] for item in sorted_items]
@@ -270,7 +276,9 @@ def main():
     # Discover expected tasks for this service
     expected_tasks = discover_all_tasks(args.mcp)
     if not expected_tasks:
-        print(f"[ERROR] Could not discover any tasks for service '{args.mcp}'. Exiting.")
+        print(
+            f"[ERROR] Could not discover any tasks for service '{args.mcp}'. Exiting."
+        )
         return
 
     metrics: Dict[str, Dict[str, float]] = {}