diff --git a/pyproject.toml b/pyproject.toml index a4151e5..cf983a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,3 +32,10 @@ allow-direct-references = true [tool.black] line-length = 120 skip-string-normalization = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = "-v --tb=short" diff --git a/spectrumlab/config/base_config.py b/spectrumlab/config/base_config.py index 11d54aa..c0640c9 100644 --- a/spectrumlab/config/base_config.py +++ b/spectrumlab/config/base_config.py @@ -11,6 +11,10 @@ @dataclass class Config: + # This api key is for testing closed MLLMs by Boyue Richdata + BOYUE_API_KEY: str = os.getenv("BOYUE_API_KEY") + BOYUE_BASE_URL: str = os.getenv("BOYUE_BASE_URL") + # DeepSeek API Configuration deepseek_api_key: str = os.getenv("DEEPSEEK_API_KEY") deepseek_base_url: str = os.getenv("DEEPSEEK_BASE_URL") @@ -25,3 +29,29 @@ class Config: internvl_api_key: str = os.getenv("INTERNVL_API_KEY") internvl_base_url: str = os.getenv("INTERNVL_BASE_URL") internvl_model_name: str = os.getenv("INTERNVL_MODEL_NAME") + + # Claude API Configuration + claude_api_key: str = BOYUE_API_KEY + claude_base_url: str = BOYUE_BASE_URL + claude_sonnet_3_5_model_name: str = os.getenv("CLAUDE_SONNET_3_5") + claude_opus_4_model_name: str = os.getenv("CLAUDE_OPUS_4") + claude_haiku_3_5_model_name: str = os.getenv("CLAUDE_HAIKU_3_5") + claude_sonnet_4_model_name: str = os.getenv("CLAUDE_SONNET_4") + + # GPT-4.1, GPT-4-Vision + gpt4_1_api_key: str = BOYUE_API_KEY + gpt4_1_base_url: str = BOYUE_BASE_URL + gpt4_1_model_name: str = os.getenv("GPT4_1") + gpt4_vision_api_key: str = BOYUE_API_KEY + gpt4_vision_base_url: str = BOYUE_BASE_URL + gpt4_vision_model_name: str = os.getenv("GPT4_VISION") + + # Grok-2-Vision + grok_2_vision_api_key: str = BOYUE_API_KEY + grok_2_vision_base_url: str = BOYUE_BASE_URL + grok_2_vision_model_name: str = os.getenv("GROK_2_VISION") + + # Qwen-VL-Max + qwen_vl_api_key: str = BOYUE_API_KEY + qwen_vl_base_url: str = BOYUE_BASE_URL + qwen_vl_model_name: str = os.getenv("QWEN_VL") diff --git a/spectrumlab/models/__init__.py b/spectrumlab/models/__init__.py index 93ce4d1..0750ae3 100644 --- a/spectrumlab/models/__init__.py +++ b/spectrumlab/models/__init__.py @@ -1,5 +1,10 @@ from .deepseek_api import DeepSeek from .gpt4o_api import GPT4o from .internvl_api import InternVL +from .claude_api import Claude_Sonnet_3_5, Claude_Opus_4, Claude_Haiku_3_5, Claude_Sonnet_4 +from .gpt4_v_api import GPT4_1, GPT4_Vision +from .grok_api import Grok_2_Vision +from .qwen_vl_api import Qwen_VL_Max -__all__ = ["DeepSeek", "GPT4o", "InternVL"] +__all__ = ["DeepSeek", "GPT4o", "InternVL", "Claude_Sonnet_3_5", "Claude_Opus_4", + "Claude_Haiku_3_5", "Claude_Sonnet_4", "GPT4_1", "GPT4_Vision", "Grok_2_Vision", "Qwen_VL_Max"] diff --git a/spectrumlab/models/claude_api.py b/spectrumlab/models/claude_api.py new file mode 100644 index 0000000..9711061 --- /dev/null +++ b/spectrumlab/models/claude_api.py @@ -0,0 +1,288 @@ +from typing import Optional, Union, Dict, Any +from .base_api import BaseAPIModel +from spectrumlab.config import Config +from openai import OpenAI + + +class Claude_Sonnet_3_5(BaseAPIModel): + def __init__( + self, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + model_name: Optional[str] = None, + **kwargs, + ): + config = Config() + + # Use provided parameters or fall back to config + self.api_key = api_key or config.BOYUE_API_KEY + self.base_url = base_url or config.BOYUE_BASE_URL + self.model_name = model_name or config.claude_sonnet_3_5_model_name + + # Validate that we have required configuration + if not self.api_key: + raise ValueError( + "Claude API key not found. Please set CLAUDE_API_KEY in your .env file " + "or provide api_key parameter." + ) + + self.client = OpenAI( + api_key=self.api_key, + base_url=self.base_url, + ) + + # Initialize parent class + super().__init__(model_name=self.model_name, **kwargs) + + def generate( + self, prompt: Union[str, Dict[str, Any]], max_out_len: int = 512 + ) -> str: + """ + Generate response supporting both text and multimodal input. + + Args: + prompt: Either text string or multimodal dict + max_out_len: Maximum tokens to generate + + Returns: + Generated response string + """ + + # Link: https://docs.anthropic.com/claude/reference/getting-started-with-the-api + messages = [] + + if isinstance(prompt, dict) and "images" in prompt: + content = [] + + content.append({"type": "text", "text": prompt["text"]}) + + for image_data in prompt["images"]: + content.append(image_data) + + messages.append({"role": "user", "content": content}) + else: + text_content = prompt if isinstance(prompt, str) else prompt.get("text", "") + messages.append({"role": "user", "content": text_content}) + + try: + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=max_out_len, + ) + return response.choices[0].message.content + except Exception as e: + raise RuntimeError(f"Claude API call failed: {e}") + + +class Claude_Opus_4(BaseAPIModel): + def __init__( + self, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + model_name: Optional[str] = None, + **kwargs, + ): + config = Config() + + # Use provided parameters or fall back to config + self.api_key = api_key or config.BOYUE_API_KEY + self.base_url = base_url or config.BOYUE_BASE_URL + self.model_name = model_name or config.claude_opus_4_model_name + + # Validate that we have required configuration + if not self.api_key: + raise ValueError( + "Claude API key not found. Please set CLAUDE_API_KEY in your .env file " + "or provide api_key parameter." + ) + + self.client = OpenAI( + api_key=self.api_key, + base_url=self.base_url, + ) + + # Initialize parent class + super().__init__(model_name=self.model_name, **kwargs) + + def generate( + self, prompt: Union[str, Dict[str, Any]], max_out_len: int = 512 + ) -> str: + """ + Generate response supporting both text and multimodal input. + + Args: + prompt: Either text string or multimodal dict + max_out_len: Maximum tokens to generate + + Returns: + Generated response string + """ + + # Link: https://docs.anthropic.com/claude/reference/getting-started-with-the-api + messages = [] + + if isinstance(prompt, dict) and "images" in prompt: + content = [] + + content.append({"type": "text", "text": prompt["text"]}) + + for image_data in prompt["images"]: + content.append(image_data) + + messages.append({"role": "user", "content": content}) + else: + text_content = prompt if isinstance(prompt, str) else prompt.get("text", "") + messages.append({"role": "user", "content": text_content}) + + try: + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=max_out_len, + ) + return response.choices[0].message.content + except Exception as e: + raise RuntimeError(f"Claude API call failed: {e}") + + +class Claude_Haiku_3_5(BaseAPIModel): + def __init__( + self, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + model_name: Optional[str] = None, + **kwargs, + ): + config = Config() + + # Use provided parameters or fall back to config + self.api_key = api_key or config.BOYUE_API_KEY + self.base_url = base_url or config.BOYUE_BASE_URL + self.model_name = model_name or config.claude_haiku_3_5_model_name + + # Validate that we have required configuration + if not self.api_key: + raise ValueError( + "Claude API key not found. Please set CLAUDE_API_KEY in your .env file " + "or provide api_key parameter." + ) + + self.client = OpenAI( + api_key=self.api_key, + base_url=self.base_url, + ) + + # Initialize parent class + super().__init__(model_name=self.model_name, **kwargs) + + def generate( + self, prompt: Union[str, Dict[str, Any]], max_out_len: int = 512 + ) -> str: + """ + Generate response supporting both text and multimodal input. + + Args: + prompt: Either text string or multimodal dict + max_out_len: Maximum tokens to generate + + Returns: + Generated response string + """ + + # Link: https://docs.anthropic.com/claude/reference/getting-started-with-the-api + messages = [] + + if isinstance(prompt, dict) and "images" in prompt: + content = [] + + content.append({"type": "text", "text": prompt["text"]}) + + for image_data in prompt["images"]: + content.append(image_data) + + messages.append({"role": "user", "content": content}) + else: + text_content = prompt if isinstance(prompt, str) else prompt.get("text", "") + messages.append({"role": "user", "content": text_content}) + + try: + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=max_out_len, + ) + return response.choices[0].message.content + except Exception as e: + raise RuntimeError(f"Claude API call failed: {e}") + +class Claude_Sonnet_4(BaseAPIModel): + def __init__( + self, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + model_name: Optional[str] = None, + **kwargs, + ): + config = Config() + + # Use provided parameters or fall back to config + self.api_key = api_key or config.BOYUE_API_KEY + self.base_url = base_url or config.BOYUE_BASE_URL + self.model_name = model_name or config.claude_sonnet_4_model_name + + # Validate that we have required configuration + if not self.api_key: + raise ValueError( + "Claude API key not found. Please set CLAUDE_API_KEY in your .env file " + "or provide api_key parameter." + ) + + self.client = OpenAI( + api_key=self.api_key, + base_url=self.base_url, + ) + + # Initialize parent class + super().__init__(model_name=self.model_name, **kwargs) + + def generate( + self, prompt: Union[str, Dict[str, Any]], max_out_len: int = 512 + ) -> str: + """ + Generate response supporting both text and multimodal input. + + Args: + prompt: Either text string or multimodal dict + max_out_len: Maximum tokens to generate + + Returns: + Generated response string + """ + + # Link: https://docs.anthropic.com/claude/reference/getting-started-with-the-api + messages = [] + + if isinstance(prompt, dict) and "images" in prompt: + content = [] + + content.append({"type": "text", "text": prompt["text"]}) + + for image_data in prompt["images"]: + content.append(image_data) + + messages.append({"role": "user", "content": content}) + else: + text_content = prompt if isinstance(prompt, str) else prompt.get("text", "") + messages.append({"role": "user", "content": text_content}) + + try: + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=max_out_len, + ) + return response.choices[0].message.content + except Exception as e: + raise RuntimeError(f"Claude API call failed: {e}") + diff --git a/spectrumlab/models/gpt4_v_api.py b/spectrumlab/models/gpt4_v_api.py new file mode 100644 index 0000000..c14312c --- /dev/null +++ b/spectrumlab/models/gpt4_v_api.py @@ -0,0 +1,147 @@ +from typing import Dict, Any, Optional, Union +from .base_api import BaseAPIModel +from spectrumlab.config import Config +from openai import OpenAI + + +class GPT4_1(BaseAPIModel): + def __init__( + self, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + model_name: Optional[str] = None, + **kwargs, + ): + config = Config() + + # Use provided parameters or fall back to config + self.api_key = api_key or config.gpt4_1_api_key + self.base_url = base_url or config.gpt4_1_base_url + self.model_name = model_name or config.gpt4_1_model_name + + # Validate that we have required configuration + if not self.api_key: + raise ValueError( + "GPT-4.1 API key not found. Please set GPT4_1_API_KEY in your .env file " + "or provide api_key parameter." + ) + + self.client = OpenAI( + api_key=self.api_key, + base_url=self.base_url, + ) + + # Initialize parent class + super().__init__(model_name=self.model_name, **kwargs) + + def generate( + self, prompt: Union[str, Dict[str, Any]], max_tokens: int = 512 + ) -> str: + """ + Generate response supporting both text and multimodal input. + + Args: + prompt: Either text string or multimodal dict + max_tokens: Maximum tokens to generate + + Returns: + Generated response string + """ + messages = [] + + # Handle multimodal vs text-only prompts + if isinstance(prompt, dict) and "images" in prompt: + # Multimodal prompt + content = [] + + content.append({"type": "text", "text": prompt["text"]}) + + for image_data in prompt["images"]: + content.append(image_data) + + messages.append({"role": "user", "content": content}) + else: + # Text-only prompt + text_content = prompt if isinstance(prompt, str) else prompt.get("text", "") + messages.append({"role": "user", "content": text_content}) + + try: + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=max_tokens, + ) + return response.choices[0].message.content + except Exception as e: + raise RuntimeError(f"GPT-4.1 API call failed: {e}") + +class GPT4_Vision(BaseAPIModel): + def __init__( + self, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + model_name: Optional[str] = None, + **kwargs, + ): + config = Config() + + # Use provided parameters or fall back to config + self.api_key = api_key or config.gpt4_vision_api_key + self.base_url = base_url or config.gpt4_vision_base_url + self.model_name = model_name or config.gpt4_vision_model_name + + # Validate that we have required configuration + if not self.api_key: + raise ValueError( + "GPT-4 Vision API key not found. Please set GPT4_VISION_API_KEY in your .env file " + "or provide api_key parameter." + ) + + self.client = OpenAI( + api_key=self.api_key, + base_url=self.base_url, + ) + + # Initialize parent class + super().__init__(model_name=self.model_name, **kwargs) + + def generate( + self, prompt: Union[str, Dict[str, Any]], max_tokens: int = 512 + ) -> str: + """ + Generate response supporting both text and multimodal input. + + Args: + prompt: Either text string or multimodal dict + max_tokens: Maximum tokens to generate + + Returns: + Generated response string + """ + messages = [] + + # Handle multimodal vs text-only prompts + if isinstance(prompt, dict) and "images" in prompt: + # Multimodal prompt + content = [] + + content.append({"type": "text", "text": prompt["text"]}) + + for image_data in prompt["images"]: + content.append(image_data) + + messages.append({"role": "user", "content": content}) + else: + # Text-only prompt + text_content = prompt if isinstance(prompt, str) else prompt.get("text", "") + messages.append({"role": "user", "content": text_content}) + + try: + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=max_tokens, + ) + return response.choices[0].message.content + except Exception as e: + raise RuntimeError(f"GPT-4 Vision API call failed: {e}") \ No newline at end of file diff --git a/spectrumlab/models/grok_api.py b/spectrumlab/models/grok_api.py new file mode 100644 index 0000000..243287a --- /dev/null +++ b/spectrumlab/models/grok_api.py @@ -0,0 +1,76 @@ +from typing import Dict, Any, Optional, Union +from .base_api import BaseAPIModel +from spectrumlab.config import Config +from openai import OpenAI + + +class Grok_2_Vision(BaseAPIModel): + def __init__( + self, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + model_name: Optional[str] = None, + **kwargs, + ): + config = Config() + + # Use provided parameters or fall back to config + self.api_key = api_key or config.grok_2_vision_api_key + self.base_url = base_url or config.grok_2_vision_base_url + self.model_name = model_name or config.grok_2_vision_model_name + + # Validate that we have required configuration + if not self.api_key: + raise ValueError( + "Grok-2-Vision API key not found. Please set GROK_2_VISION_API_KEY in your .env file " + "or provide api_key parameter." + ) + + self.client = OpenAI( + api_key=self.api_key, + base_url=self.base_url, + ) + + # Initialize parent class + super().__init__(model_name=self.model_name, **kwargs) + + def generate( + self, prompt: Union[str, Dict[str, Any]], max_tokens: int = 512 + ) -> str: + """ + Generate response supporting both text and multimodal input. + + Args: + prompt: Either text string or multimodal dict + max_tokens: Maximum tokens to generate + + Returns: + Generated response string + """ + messages = [] + + # Handle multimodal vs text-only prompts + if isinstance(prompt, dict) and "images" in prompt: + # Multimodal prompt + content = [] + + content.append({"type": "text", "text": prompt["text"]}) + + for image_data in prompt["images"]: + content.append(image_data) + + messages.append({"role": "user", "content": content}) + else: + # Text-only prompt + text_content = prompt if isinstance(prompt, str) else prompt.get("text", "") + messages.append({"role": "user", "content": text_content}) + + try: + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=max_tokens, + ) + return response.choices[0].message.content + except Exception as e: + raise RuntimeError(f"Grok-2-Vision API call failed: {e}") diff --git a/spectrumlab/models/qwen_vl_api.py b/spectrumlab/models/qwen_vl_api.py new file mode 100644 index 0000000..d4c6817 --- /dev/null +++ b/spectrumlab/models/qwen_vl_api.py @@ -0,0 +1,76 @@ +from typing import Dict, Any, Optional, Union +from .base_api import BaseAPIModel +from spectrumlab.config import Config +from openai import OpenAI + + +class Qwen_VL_Max(BaseAPIModel): + def __init__( + self, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + model_name: Optional[str] = None, + **kwargs, + ): + config = Config() + + # Use provided parameters or fall back to config + self.api_key = api_key or config.qwen_vl_api_key + self.base_url = base_url or config.qwen_vl_base_url + self.model_name = model_name or config.qwen_vl_model_name + + # Validate that we have required configuration + if not self.api_key: + raise ValueError( + "Qwen-VL-Max API key not found. Please set QWEN_VL_MAX_API_KEY in your .env file " + "or provide api_key parameter." + ) + + self.client = OpenAI( + api_key=self.api_key, + base_url=self.base_url, + ) + + # Initialize parent class + super().__init__(model_name=self.model_name, **kwargs) + + def generate( + self, prompt: Union[str, Dict[str, Any]], max_tokens: int = 512 + ) -> str: + """ + Generate response supporting both text and multimodal input. + + Args: + prompt: Either text string or multimodal dict + max_tokens: Maximum tokens to generate + + Returns: + Generated response string + """ + messages = [] + + # Handle multimodal vs text-only prompts + if isinstance(prompt, dict) and "images" in prompt: + # Multimodal prompt + content = [] + + content.append({"type": "text", "text": prompt["text"]}) + + for image_data in prompt["images"]: + content.append(image_data) + + messages.append({"role": "user", "content": content}) + else: + # Text-only prompt + text_content = prompt if isinstance(prompt, str) else prompt.get("text", "") + messages.append({"role": "user", "content": text_content}) + + try: + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=max_tokens, + ) + return response.choices[0].message.content + except Exception as e: + raise RuntimeError(f"Qwen-VL-Max API call failed: {e}") diff --git a/tests/models/test_claude_haiku_3_5.py b/tests/models/test_claude_haiku_3_5.py new file mode 100644 index 0000000..16ea288 --- /dev/null +++ b/tests/models/test_claude_haiku_3_5.py @@ -0,0 +1,40 @@ +from spectrumlab.models import Claude_Haiku_3_5 +from spectrumlab.utils.image_utils import encode_image_to_base64 +from spectrumlab.benchmark.signal_group import SignalGroup +from spectrumlab.evaluator.choice_evaluator import ChoiceEvaluator + + +def test_claude_text_generation(): + model = Claude_Haiku_3_5() + prompt = "What is spectroscopy?" + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_multimodal_generation(): + model = Claude_Haiku_3_5() + image_path = "playground/models/test.jpg" + image_base64 = encode_image_to_base64(image_path) + prompt = { + "text": "Please explain this spectroscopy image.", + "images": [ + { + "type": "image_url", + "image_url": {"url": f"data:image/jpg;base64,{image_base64}"}, + } + ], + } + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_signalgroup_evaluation(): + model = Claude_Haiku_3_5() + signal_group = SignalGroup("data") + data = signal_group.get_data_by_subcategories(["Spectrum Type Classification"]) + evaluator = ChoiceEvaluator() + results = evaluator.evaluate(data_items=data, model=model, save_path=None) + assert "metrics" in results + assert "overall" in results["metrics"] diff --git a/tests/models/test_claude_opus_4.py b/tests/models/test_claude_opus_4.py new file mode 100644 index 0000000..66b383b --- /dev/null +++ b/tests/models/test_claude_opus_4.py @@ -0,0 +1,40 @@ +from spectrumlab.models import Claude_Opus_4 +from spectrumlab.utils.image_utils import encode_image_to_base64 +from spectrumlab.benchmark.signal_group import SignalGroup +from spectrumlab.evaluator.choice_evaluator import ChoiceEvaluator + + +def test_claude_text_generation(): + model = Claude_Opus_4() + prompt = "What is spectroscopy?" + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_multimodal_generation(): + model = Claude_Opus_4() + image_path = "playground/models/test.jpg" + image_base64 = encode_image_to_base64(image_path) + prompt = { + "text": "Please explain this spectroscopy image.", + "images": [ + { + "type": "image_url", + "image_url": {"url": f"data:image/jpg;base64,{image_base64}"}, + } + ], + } + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_signalgroup_evaluation(): + model = Claude_Opus_4() + signal_group = SignalGroup("data") + data = signal_group.get_data_by_subcategories(["Spectrum Type Classification"]) + evaluator = ChoiceEvaluator() + results = evaluator.evaluate(data_items=data, model=model, save_path=None) + assert "metrics" in results + assert "overall" in results["metrics"] diff --git a/tests/models/test_claude_sonnet_3_5.py b/tests/models/test_claude_sonnet_3_5.py new file mode 100644 index 0000000..818470a --- /dev/null +++ b/tests/models/test_claude_sonnet_3_5.py @@ -0,0 +1,40 @@ +from spectrumlab.models import Claude_Sonnet_3_5 +from spectrumlab.utils.image_utils import encode_image_to_base64 +from spectrumlab.benchmark.signal_group import SignalGroup +from spectrumlab.evaluator.choice_evaluator import ChoiceEvaluator + + +def test_claude_text_generation(): + model = Claude_Sonnet_3_5() + prompt = "What is spectroscopy?" + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_multimodal_generation(): + model = Claude_Sonnet_3_5() + image_path = "playground/models/test.jpg" + image_base64 = encode_image_to_base64(image_path) + prompt = { + "text": "Please explain this spectroscopy image.", + "images": [ + { + "type": "image_url", + "image_url": {"url": f"data:image/jpg;base64,{image_base64}"}, + } + ], + } + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_signalgroup_evaluation(): + model = Claude_Sonnet_3_5() + signal_group = SignalGroup("data") + data = signal_group.get_data_by_subcategories(["Spectrum Type Classification"]) + evaluator = ChoiceEvaluator() + results = evaluator.evaluate(data_items=data, model=model, save_path=None) + assert "metrics" in results + assert "overall" in results["metrics"] diff --git a/tests/models/test_claude_sonnet_4.py b/tests/models/test_claude_sonnet_4.py new file mode 100644 index 0000000..1d284b3 --- /dev/null +++ b/tests/models/test_claude_sonnet_4.py @@ -0,0 +1,40 @@ +from spectrumlab.models import Claude_Sonnet_4 +from spectrumlab.utils.image_utils import encode_image_to_base64 +from spectrumlab.benchmark.signal_group import SignalGroup +from spectrumlab.evaluator.choice_evaluator import ChoiceEvaluator + + +def test_claude_text_generation(): + model = Claude_Sonnet_4() + prompt = "What is spectroscopy?" + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_multimodal_generation(): + model = Claude_Sonnet_4() + image_path = "playground/models/test.jpg" + image_base64 = encode_image_to_base64(image_path) + prompt = { + "text": "Please explain this spectroscopy image.", + "images": [ + { + "type": "image_url", + "image_url": {"url": f"data:image/jpg;base64,{image_base64}"}, + } + ], + } + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_signalgroup_evaluation(): + model = Claude_Sonnet_4() + signal_group = SignalGroup("data") + data = signal_group.get_data_by_subcategories(["Spectrum Type Classification"]) + evaluator = ChoiceEvaluator() + results = evaluator.evaluate(data_items=data, model=model, save_path=None) + assert "metrics" in results + assert "overall" in results["metrics"] diff --git a/tests/models/test_gpt_4_1.py b/tests/models/test_gpt_4_1.py new file mode 100644 index 0000000..1982848 --- /dev/null +++ b/tests/models/test_gpt_4_1.py @@ -0,0 +1,40 @@ +from spectrumlab.models import GPT4_1 +from spectrumlab.utils.image_utils import encode_image_to_base64 +from spectrumlab.benchmark.signal_group import SignalGroup +from spectrumlab.evaluator.choice_evaluator import ChoiceEvaluator + + +def test_claude_text_generation(): + model = GPT4_1() + prompt = "What is spectroscopy?" + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_multimodal_generation(): + model = GPT4_1() + image_path = "playground/models/test.jpg" + image_base64 = encode_image_to_base64(image_path) + prompt = { + "text": "Please explain this spectroscopy image.", + "images": [ + { + "type": "image_url", + "image_url": {"url": f"data:image/jpg;base64,{image_base64}"}, + } + ], + } + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_signalgroup_evaluation(): + model = GPT4_1() + signal_group = SignalGroup("data") + data = signal_group.get_data_by_subcategories(["Spectrum Type Classification"]) + evaluator = ChoiceEvaluator() + results = evaluator.evaluate(data_items=data, model=model, save_path=None) + assert "metrics" in results + assert "overall" in results["metrics"] diff --git a/tests/models/test_gpt_4_v.py b/tests/models/test_gpt_4_v.py new file mode 100644 index 0000000..9b52442 --- /dev/null +++ b/tests/models/test_gpt_4_v.py @@ -0,0 +1,40 @@ +from spectrumlab.models import GPT4_Vision +from spectrumlab.utils.image_utils import encode_image_to_base64 +from spectrumlab.benchmark.signal_group import SignalGroup +from spectrumlab.evaluator.choice_evaluator import ChoiceEvaluator + + +def test_claude_text_generation(): + model = GPT4_Vision() + prompt = "What is spectroscopy?" + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_multimodal_generation(): + model = GPT4_Vision() + image_path = "playground/models/test.jpg" + image_base64 = encode_image_to_base64(image_path) + prompt = { + "text": "Please explain this spectroscopy image.", + "images": [ + { + "type": "image_url", + "image_url": {"url": f"data:image/jpg;base64,{image_base64}"}, + } + ], + } + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_signalgroup_evaluation(): + model = GPT4_Vision() + signal_group = SignalGroup("data") + data = signal_group.get_data_by_subcategories(["Spectrum Type Classification"]) + evaluator = ChoiceEvaluator() + results = evaluator.evaluate(data_items=data, model=model, save_path=None) + assert "metrics" in results + assert "overall" in results["metrics"] diff --git a/tests/models/test_grok_2_v.py b/tests/models/test_grok_2_v.py new file mode 100644 index 0000000..29bb571 --- /dev/null +++ b/tests/models/test_grok_2_v.py @@ -0,0 +1,40 @@ +from spectrumlab.models import Grok_2_Vision +from spectrumlab.utils.image_utils import encode_image_to_base64 +from spectrumlab.benchmark.signal_group import SignalGroup +from spectrumlab.evaluator.choice_evaluator import ChoiceEvaluator + + +def test_claude_text_generation(): + model = Grok_2_Vision() + prompt = "What is spectroscopy?" + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_multimodal_generation(): + model = Grok_2_Vision() + image_path = "playground/models/test.jpg" + image_base64 = encode_image_to_base64(image_path) + prompt = { + "text": "Please explain this spectroscopy image.", + "images": [ + { + "type": "image_url", + "image_url": {"url": f"data:image/jpg;base64,{image_base64}"}, + } + ], + } + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_signalgroup_evaluation(): + model = Grok_2_Vision() + signal_group = SignalGroup("data") + data = signal_group.get_data_by_subcategories(["Spectrum Type Classification"]) + evaluator = ChoiceEvaluator() + results = evaluator.evaluate(data_items=data, model=model, save_path=None) + assert "metrics" in results + assert "overall" in results["metrics"] diff --git a/tests/models/test_internvl.py b/tests/models/test_internvl.py index f718961..f469c90 100644 --- a/tests/models/test_internvl.py +++ b/tests/models/test_internvl.py @@ -14,14 +14,14 @@ def test_internvl_text_generation(): def test_internvl_multimodal_generation(): model = InternVL() - image_path = "playground/models/test.png" + image_path = "playground/models/test.jpg" image_base64 = encode_image_to_base64(image_path) prompt = { "text": "Please explain this spectroscopy image.", "images": [ { "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{image_base64}"}, + "image_url": {"url": f"data:image/jpg;base64,{image_base64}"}, } ], } diff --git a/tests/models/test_qwen_vl.py b/tests/models/test_qwen_vl.py new file mode 100644 index 0000000..8310935 --- /dev/null +++ b/tests/models/test_qwen_vl.py @@ -0,0 +1,40 @@ +from spectrumlab.models import Qwen_VL_Max +from spectrumlab.utils.image_utils import encode_image_to_base64 +from spectrumlab.benchmark.signal_group import SignalGroup +from spectrumlab.evaluator.choice_evaluator import ChoiceEvaluator + + +def test_claude_text_generation(): + model = Qwen_VL_Max() + prompt = "What is spectroscopy?" + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_multimodal_generation(): + model = Qwen_VL_Max() + image_path = "playground/models/test.jpg" + image_base64 = encode_image_to_base64(image_path) + prompt = { + "text": "Please explain this spectroscopy image.", + "images": [ + { + "type": "image_url", + "image_url": {"url": f"data:image/jpg;base64,{image_base64}"}, + } + ], + } + response = model.generate(prompt) + assert isinstance(response, str) + assert len(response) > 0 + + +def test_claude_signalgroup_evaluation(): + model = Qwen_VL_Max() + signal_group = SignalGroup("data") + data = signal_group.get_data_by_subcategories(["Spectrum Type Classification"]) + evaluator = ChoiceEvaluator() + results = evaluator.evaluate(data_items=data, model=model, save_path=None) + assert "metrics" in results + assert "overall" in results["metrics"]