diff --git a/.gitignore b/.gitignore index 858b38d9..0df9fd68 100644 --- a/.gitignore +++ b/.gitignore @@ -147,3 +147,8 @@ dmypy.json html/ .vscode + +# Meilisearch +data.ms/ +*.ms +meili_data/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..121949db --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,197 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## πŸ—οΈ Project Overview + +This is the **Meilisearch Python SDK** - the official Python client library for the Meilisearch search engine. It provides a complete Python interface for all Meilisearch API operations including document management, search, index configuration, and administrative tasks. + +## πŸš€ Development Commands + +### Environment Setup +```bash +# Install dependencies +pipenv install --dev + +# Activate virtual environment +pipenv shell +``` + +### Testing +```bash +# Start Meilisearch server (required for tests) +curl -L https://install.meilisearch.com | sh +./meilisearch --master-key=masterKey --no-analytics + +# Run all tests +pipenv run pytest tests + +# Run tests with coverage +pipenv run pytest tests --cov=meilisearch --cov-report term-missing + +# Run specific test file +pipenv run pytest tests/client/test_client.py + +# Run with Docker (alternative) +docker pull getmeili/meilisearch:latest +docker run -p 7700:7700 getmeili/meilisearch:latest meilisearch --master-key=masterKey --no-analytics +``` + +### Code Quality +```bash +# Type checking +pipenv run mypy meilisearch + +# Linting +pipenv run pylint meilisearch tests + +# Code formatting +pipenv run black meilisearch tests +pipenv run isort meilisearch tests +``` + +### Testing All Python Versions +```bash +# Using tox (runs tests on all supported Python versions) +pipenv run tox +``` + +### Docker Development +```bash +# Run all checks with docker +docker-compose run --rm package bash -c "pipenv install --dev && pipenv run mypy meilisearch && pipenv run pylint meilisearch tests && pipenv run pytest tests" +``` + +## πŸ›οΈ Architecture Overview + +### Core Structure +- **`meilisearch/client.py`**: Main `Client` class - entry point for all API operations +- **`meilisearch/index.py`**: `Index` class - handles index-specific operations (search, documents, settings) +- **`meilisearch/task.py`**: Task management and waiting utilities +- **`meilisearch/_httprequests.py`**: HTTP request handling and error management +- **`meilisearch/models/`**: Pydantic models for API responses and configuration + +### API Client Pattern +The SDK follows a hierarchical client pattern: +```python +# Client -> Index -> Operations +client = meilisearch.Client('http://localhost:7700', 'masterKey') +index = client.index('movies') +results = index.search('query') +``` + +### Key Design Patterns +1. **Async Task Handling**: Most write operations return tasks that can be waited for +2. **Type Safety**: Full typing with mypy strict mode enabled +3. **Error Hierarchy**: Custom exceptions in `meilisearch/errors.py` +4. **HTTP Abstraction**: Centralized HTTP handling with automatic retries and error conversion +5. **Model Validation**: Pydantic models for request/response validation + +### Testing Strategy +- **Integration Tests**: Most tests run against real Meilisearch instance +- **Fixtures**: Automatic index cleanup between tests via `conftest.py` +- **Test Environment**: Uses `tests/common.py` for shared configuration +- **Coverage**: Tests cover client operations, index management, search, settings, and error handling + +## πŸ”§ Development Guidelines + +### Code Style +- **Black**: Line length 100, Python 3.8+ target +- **isort**: Black-compatible import sorting +- **mypy**: Strict type checking enabled +- **pylint**: Custom configuration in `pyproject.toml` + +### Testing Requirements +- Must have running Meilisearch server on `http://127.0.0.1:7700` with master key `masterKey` +- Tests automatically clean up indexes after each run +- Use `pytest` for all test execution +- Coverage reports required for new features + +### Error Handling +- All API errors convert to `MeilisearchApiError` with structured error information +- HTTP errors handled in `_httprequests.py` +- Timeout and communication errors have specific exception types + +### Type Hints +- All public methods must have complete type annotations +- Use `from __future__ import annotations` for forward references +- Models use Pydantic for runtime validation + +## πŸ“¦ SDK Architecture + +### Client Hierarchy +``` +Client (meilisearch/client.py) +β”œβ”€β”€ Index management (create, list, delete indexes) +β”œβ”€β”€ Global operations (health, version, stats, keys, tasks) +β”œβ”€β”€ Multi-search functionality +└── Index (meilisearch/index.py) + β”œβ”€β”€ Document operations (add, update, delete, get) + β”œβ”€β”€ Search operations (search, facet_search) + β”œβ”€β”€ Settings management (all index configuration) + └── Task operations (wait_for_task, get_tasks) +``` + +### Models Structure +- **`models/document.py`**: Document and search result models +- **`models/index.py`**: Index settings and statistics models +- **`models/key.py`**: API key management models +- **`models/task.py`**: Task status and batch operation models +- **`models/embedders.py`**: AI embedder configuration models + +### HTTP Layer +- `_httprequests.py` handles all HTTP communication +- Automatic JSON serialization/deserialization +- Custom serializer support for complex types (datetime, UUID) +- Centralized error handling and retry logic + +## πŸ§ͺ Test Organization + +### Test Structure +``` +tests/ +β”œβ”€β”€ client/ # Client-level operations +β”œβ”€β”€ index/ # Index-specific operations +β”œβ”€β”€ settings/ # Index settings tests +β”œβ”€β”€ models/ # Model validation tests +β”œβ”€β”€ errors/ # Error handling tests +└── conftest.py # Shared fixtures and cleanup +``` + +### Key Test Patterns +- Each test module focuses on specific functionality +- Tests use real Meilisearch server for integration testing +- Automatic cleanup ensures test isolation +- Tests verify both success and error cases + +## πŸ” Common Development Tasks + +### Adding New API Endpoints +1. Add method to appropriate class (`Client` for global, `Index` for index-specific) +2. Add type hints for parameters and return values +3. Add corresponding model classes if needed +4. Write integration tests covering success and error cases +5. Update documentation if it's a public feature + +### Running Single Test Categories +```bash +# Test specific functionality +pipenv run pytest tests/client/ # Client operations +pipenv run pytest tests/index/ # Index operations +pipenv run pytest tests/settings/ # Settings management +pipenv run pytest tests/models/ # Model validation +``` + +### Debugging +```python +import pdb +pdb.set_trace() # Add breakpoint for debugging +``` + +## πŸ“ Release Process + +Version management: +- Version defined in `meilisearch/version.py` +- Semantic versioning (MAJOR.MINOR.PATCH) +- Automated via GitHub Actions workflow +- Publishes to PyPI automatically on release \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a792165..e8d3ce3f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,10 +47,15 @@ Each PR should pass the tests, mypy type checking, and the linter to be accepted Your PR also needs to be formatted using black and isort. ```bash -# Tests +# Tests (Option 1 - Auto-launch) +# The SDK will automatically download and launch Meilisearch if needed +pipenv run pytest tests + +# Tests (Option 2 - Manual setup) curl -L https://install.meilisearch.com | sh # download Meilisearch ./meilisearch --master-key=masterKey --no-analytics # run Meilisearch pipenv run pytest tests + # MyPy pipenv run mypy meilisearch # Linter diff --git a/README.md b/README.md index c072c3e0..0571ea8f 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ To learn more about Meilisearch Python, refer to the in-depth [Meilisearch Pytho ## πŸ”§ Installation -**Note**: Python 3.8+ is required. +**Note**: Python 3.9+ is required. With `pip3` in command line: @@ -52,9 +52,19 @@ pip3 install meilisearch ### Run Meilisearch -⚑️ **Launch, scale, and streamline in minutes with Meilisearch Cloud**β€”no maintenance, no commitment, cancel anytime. [Try it free now](https://cloud.meilisearch.com/login?utm_campaign=oss&utm_source=github&utm_medium=meilisearch-python). +There are three ways to use Meilisearch: -πŸͺ¨ Prefer to self-host? [Download and deploy](https://www.meilisearch.com/docs/learn/self_hosted/getting_started_with_self_hosted_meilisearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch-python) our fast, open-source search engine on your own infrastructure. +1. **πŸ†• Auto-Launch (Easiest)**: Let the Python client automatically download and run Meilisearch for you: +```python +import meilisearch + +# No URL needed - Meilisearch will be automatically launched! +client = meilisearch.Client() +``` + +2. **☁️ Meilisearch Cloud**: Launch, scale, and streamline in minutesβ€”no maintenance, no commitment, cancel anytime. [Try it free now](https://cloud.meilisearch.com/login?utm_campaign=oss&utm_source=github&utm_medium=meilisearch-python). + +3. **πŸͺ¨ Self-Host**: [Download and deploy](https://www.meilisearch.com/docs/learn/self_hosted/getting_started_with_self_hosted_meilisearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch-python) our fast, open-source search engine on your own infrastructure. ## πŸš€ Getting started @@ -63,7 +73,11 @@ pip3 install meilisearch ```python import meilisearch -client = meilisearch.Client('http://127.0.0.1:7700', 'masterKey') +# Automatic launch - no setup required! +client = meilisearch.Client() + +# Or connect to an existing instance +# client = meilisearch.Client('http://127.0.0.1:7700', 'masterKey') # An index is where the documents are stored. index = client.index('movies') @@ -234,6 +248,44 @@ index.search( } ``` +### Auto-Launch Feature + +The Python SDK can automatically launch a local Meilisearch instance for you, making development even easier: + +```python +import meilisearch + +# No URL needed - Meilisearch will be automatically launched! +client = meilisearch.Client() + +# Use it like normal +index = client.index('products') +index.add_documents([{"id": 1, "name": "Laptop"}]) + +# The server will be automatically stopped when the client is destroyed +``` + +You can also use it with a context manager for automatic cleanup: + +```python +with meilisearch.Client() as client: + # Meilisearch is running + client.index('products').add_documents([{"id": 1, "name": "Laptop"}]) +# Meilisearch is automatically stopped here +``` + +With a custom master key: + +```python +client = meilisearch.Client(api_key='myMasterKey') +``` + +**Note**: The auto-launch feature will: +- Check if Meilisearch is installed in your PATH +- If not found, automatically download the latest version for your platform +- Store the binary in `~/.meilisearch/bin/` +- Create temporary data directories that are cleaned up on exit + ## πŸ€– Compatibility with Meilisearch This package guarantees compatibility with [version v1.2 and above of Meilisearch](https://github.com/meilisearch/meilisearch/releases/latest), but some features may not be present. Please check the [issues](https://github.com/meilisearch/meilisearch-python/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22+label%3Aenhancement) for more info. diff --git a/meilisearch/_local_server.py b/meilisearch/_local_server.py new file mode 100644 index 00000000..7f8307a8 --- /dev/null +++ b/meilisearch/_local_server.py @@ -0,0 +1,233 @@ +"""Local Meilisearch server management utilities.""" + +from __future__ import annotations + +import atexit +import os +import platform +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Optional, Tuple +from urllib.error import URLError +from urllib.parse import urlparse +from urllib.request import urlopen + + +class LocalMeilisearchServer: + """Manages a local Meilisearch server instance.""" + + def __init__( + self, + port: Optional[int] = None, + data_path: Optional[str] = None, + master_key: Optional[str] = None, + ): + self.port = port or self._find_available_port() + self.data_path = data_path or tempfile.mkdtemp(prefix="meilisearch_") + self.master_key = master_key + self.process: Optional[subprocess.Popen] = None + self.binary_path: Optional[str] = None + self.url = f"http://127.0.0.1:{self.port}" + self._temp_data_dir = data_path is None + + def _find_available_port(self) -> int: + """Find an available port to use.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("", 0)) + return sock.getsockname()[1] + + def _find_meilisearch_binary(self) -> Optional[str]: + """Find Meilisearch binary in PATH or common locations.""" + # First, check if 'meilisearch' is in PATH + binary_name = "meilisearch" if platform.system() != "Windows" else "meilisearch.exe" + binary_path = shutil.which(binary_name) + if binary_path: + return binary_path + + # Check common installation locations + common_paths = [] + if platform.system() == "Darwin": # macOS + common_paths.extend( + [ + "/usr/local/bin/meilisearch", + "/opt/homebrew/bin/meilisearch", + str(Path.home() / ".local" / "bin" / "meilisearch"), + ] + ) + elif platform.system() == "Linux": + common_paths.extend( + [ + "/usr/local/bin/meilisearch", + "/usr/bin/meilisearch", + str(Path.home() / ".local" / "bin" / "meilisearch"), + ] + ) + + for path in common_paths: + if Path(path).exists() and Path(path).is_file(): + return path + + return None + + def _download_meilisearch(self) -> str: + """Download Meilisearch binary for the current platform.""" + system = platform.system().lower() + machine = platform.machine().lower() + + # Map platform to Meilisearch release names + if system == "darwin": + if machine in ["arm64", "aarch64"]: + platform_str = "apple-silicon" + else: + platform_str = "amd64" + binary_name = f"meilisearch-macos-{platform_str}" + elif system == "linux": + if machine in ["x86_64", "amd64"]: + platform_str = "amd64" + elif machine in ["aarch64", "arm64"]: + platform_str = "aarch64" + else: + raise RuntimeError(f"Unsupported Linux architecture: {machine}") + binary_name = f"meilisearch-linux-{platform_str}" + else: + raise RuntimeError(f"Unsupported platform: {system}") + + # Download the latest release + download_url = ( + f"https://github.com/meilisearch/meilisearch/releases/latest/download/{binary_name}" + ) + + # Create a temporary directory for the binary + binary_dir = Path.home() / ".meilisearch" / "bin" + binary_dir.mkdir(parents=True, exist_ok=True) + binary_path = binary_dir / "meilisearch" + + # Download if not already present + if not binary_path.exists(): + print(f"Downloading Meilisearch binary from {download_url}...") + try: + with urlopen(download_url, timeout=300) as response: + with open(binary_path, "wb") as f: + f.write(response.read()) + # Make it executable + binary_path.chmod(0o755) + print(f"Downloaded Meilisearch to {binary_path}") + except Exception as e: + raise RuntimeError(f"Failed to download Meilisearch: {e}") from e + + return str(binary_path) + + def start(self) -> None: + """Start the local Meilisearch server.""" + if self.process and self.process.poll() is None: + return # Already running + + # Find or download Meilisearch binary + self.binary_path = self._find_meilisearch_binary() + if not self.binary_path: + try: + self.binary_path = self._download_meilisearch() + except Exception as e: + raise RuntimeError( + "Meilisearch binary not found in PATH. " + "Please install Meilisearch: https://www.meilisearch.com/docs/learn/getting_started/installation" + ) from e + + # Prepare command + cmd = [ + self.binary_path, + "--http-addr", + f"127.0.0.1:{self.port}", + "--db-path", + self.data_path, + "--no-analytics", + ] + + if self.master_key: + cmd.extend(["--master-key", self.master_key]) + + # Start the process + try: + # pylint: disable=consider-using-with + # We don't use 'with' here because we want the process to run in the background + self.process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + ) + + # Register cleanup on exit + atexit.register(self.stop) + + # Wait for server to be ready + self._wait_for_server() + + except Exception as e: + self.stop() + raise RuntimeError(f"Failed to start Meilisearch: {e}") from e + + def _wait_for_server(self, timeout: int = 30) -> None: + """Wait for the server to be ready.""" + start_time = time.time() + while time.time() - start_time < timeout: + try: + with urlopen(f"{self.url}/health", timeout=1) as response: + if response.status == 200: + return + except (OSError, URLError): + pass + + # Check if process has died + if self.process and self.process.poll() is not None: + stdout, stderr = self.process.communicate() + raise RuntimeError( + f"Meilisearch process died unexpectedly. " + f"stdout: {stdout.decode()}, stderr: {stderr.decode()}" + ) + + time.sleep(0.1) + + raise RuntimeError("Meilisearch server failed to start within timeout") + + def stop(self) -> None: + """Stop the local Meilisearch server.""" + if self.process and self.process.poll() is None: + # Try graceful shutdown first + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + # Force kill if needed + if sys.platform == "win32": + self.process.kill() + else: + os.killpg(os.getpgid(self.process.pid), signal.SIGKILL) + self.process.wait() + + self.process = None + + # Clean up temporary data directory + if self._temp_data_dir and os.path.exists(self.data_path): + try: + shutil.rmtree(self.data_path) + except OSError: + pass # Ignore cleanup errors + + def __del__(self) -> None: + """Ensure cleanup on object destruction.""" + self.stop() + + +def parse_url_components(url: str) -> Tuple[str, int]: + """Parse URL to extract host and port.""" + parsed = urlparse(url) + host = parsed.hostname or "127.0.0.1" + port = parsed.port or 7700 + return host, port diff --git a/meilisearch/client.py b/meilisearch/client.py index e3572d32..44bb8eb3 100644 --- a/meilisearch/client.py +++ b/meilisearch/client.py @@ -12,6 +12,7 @@ from urllib import parse from meilisearch._httprequests import HttpRequests +from meilisearch._local_server import LocalMeilisearchServer from meilisearch.config import Config from meilisearch.errors import MeilisearchError from meilisearch.index import Index @@ -30,7 +31,7 @@ class Client: def __init__( self, - url: str, + url: Optional[str] = None, api_key: Optional[str] = None, timeout: Optional[int] = None, client_agents: Optional[Tuple[str, ...]] = None, @@ -39,10 +40,12 @@ def __init__( """ Parameters ---------- - url: - The url to the Meilisearch API (ex: http://localhost:7700) - api_key: - The optional API key for Meilisearch + url (optional): + The url to the Meilisearch API (ex: http://localhost:7700). + If not provided, a local Meilisearch instance will be automatically launched. + api_key (optional): + The optional API key for Meilisearch. + If not provided when auto-launching, no master key will be set. timeout (optional): The amount of time in seconds that the client will wait for a response before timing out. @@ -52,6 +55,16 @@ def __init__( custom_headers (optional): Custom headers to add when sending data to Meilisearch. """ + self._local_server: Optional[LocalMeilisearchServer] = None + + # If no URL is provided, start a local Meilisearch instance + if url is None: + self._local_server = LocalMeilisearchServer(master_key=api_key) + self._local_server.start() + url = self._local_server.url + # Use the same API key for the client + if api_key is None and self._local_server.master_key is not None: + api_key = self._local_server.master_key self.config = Config(url, api_key, timeout=timeout, client_agents=client_agents) @@ -768,3 +781,21 @@ def _valid_uuid(uuid: str) -> bool: ) match = uuid4hex.match(uuid) return bool(match) + + def close(self) -> None: + """Close the client and stop the local Meilisearch server if it was auto-launched.""" + if self._local_server: + self._local_server.stop() + self._local_server = None + + def __enter__(self) -> "Client": + """Support using the client as a context manager.""" + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + """Clean up when exiting the context manager.""" + self.close() + + def __del__(self) -> None: + """Ensure cleanup on object destruction.""" + self.close() diff --git a/tests/auto_launch/conftest.py b/tests/auto_launch/conftest.py new file mode 100644 index 00000000..51905fb4 --- /dev/null +++ b/tests/auto_launch/conftest.py @@ -0,0 +1,21 @@ +# Override parent conftest.py fixtures for auto-launch tests + +import pytest + + +@pytest.fixture(scope="session") +def client(): + """Override client fixture to return None.""" + return None + + +@pytest.fixture(autouse=True) +def clear_indexes(): + """Override clear_indexes to do nothing.""" + yield + + +@pytest.fixture(autouse=True) +def clear_all_tasks(): + """Override clear_all_tasks to do nothing.""" + yield diff --git a/tests/auto_launch/test_client_auto_launch.py b/tests/auto_launch/test_client_auto_launch.py new file mode 100644 index 00000000..9188ab4b --- /dev/null +++ b/tests/auto_launch/test_client_auto_launch.py @@ -0,0 +1,98 @@ +"""Tests for auto-launch functionality.""" + +import time + +import meilisearch +from meilisearch._local_server import LocalMeilisearchServer + + +def test_client_auto_launch(): + """Test that client can auto-launch a local Meilisearch instance.""" + # Create client without URL - should auto-launch with a generated API key + with meilisearch.Client(api_key="test_auto_launch_key") as client: + # Verify the client has a local server + assert client._local_server is not None + assert isinstance(client._local_server, LocalMeilisearchServer) + + # Verify we can communicate with the server + assert client.is_healthy() + + # Test basic operations + # Create an index + task = client.create_index("test_index") + client.wait_for_task(task.task_uid) + + # Get the index + index = client.get_index("test_index") + assert index.uid == "test_index" + + # Add documents + documents = [{"id": 1, "title": "Test Document"}, {"id": 2, "title": "Another Document"}] + task = index.add_documents(documents) + client.wait_for_task(task.task_uid) + + # Give Meilisearch a moment to process + time.sleep(0.5) + + # Search + results = index.search("test") + assert len(results["hits"]) > 0 + + # After context manager exits, server should be stopped + # We can't easily test this without trying to connect again + + +def test_client_auto_launch_with_api_key(): + """Test auto-launch with a custom API key.""" + api_key = "test_master_key_123" + + with meilisearch.Client(api_key=api_key) as client: + assert client._local_server is not None + assert client._local_server.master_key == api_key + assert client.config.api_key == api_key + + # Should be able to use the client normally + assert client.is_healthy() + + +def test_client_with_url_no_auto_launch(): + """Test that providing a URL prevents auto-launch.""" + # This will connect to a specific URL (doesn't need to be running for this test) + client = meilisearch.Client("http://127.0.0.1:7700", "masterKey") + + # Should not have a local server + assert client._local_server is None + + # Should have the correct configuration + assert client.config.url == "http://127.0.0.1:7700" + assert client.config.api_key == "masterKey" + + +def test_client_del_cleanup(): + """Test that __del__ properly cleans up the local server.""" + client = meilisearch.Client(api_key="cleanup_test_key") + assert client._local_server is not None + + # Store reference to local server + local_server = client._local_server + + # Delete the client + del client + + # The local server should have been stopped + # (We can't easily verify the process is dead without platform-specific code) + + +def test_client_auto_launch_no_api_key(): + """Test auto-launch without API key for public operations.""" + with meilisearch.Client() as client: + # Verify the client has a local server + assert client._local_server is not None + assert client._local_server.master_key is None + + # Health check should work without authentication + assert client.is_healthy() + + # Version check should work without authentication + version = client.get_version() + assert "pkgVersion" in version