|
1 | 1 | # webfetch |
2 | | -AI-friendly webfetch tool, cli, mcp server, and lib |
| 2 | + |
| 3 | +AI-friendly web content fetching tool designed for LLM consumption. Rust library with CLI, MCP server, and Python bindings. |
| 4 | + |
| 5 | +## Features |
| 6 | + |
| 7 | +- **HTTP fetching** - GET and HEAD methods with streaming support |
| 8 | +- **HTML-to-Markdown** - Built-in conversion optimized for LLMs |
| 9 | +- **HTML-to-Text** - Plain text extraction with clean formatting |
| 10 | +- **Binary detection** - Returns metadata only for images, PDFs, etc. |
| 11 | +- **Timeout handling** - 1s first-byte, 30s body with partial content on timeout |
| 12 | +- **URL filtering** - Allow/block lists for controlled access |
| 13 | +- **MCP server** - Model Context Protocol support for AI tool integration |
| 14 | + |
| 15 | +## Installation |
| 16 | + |
| 17 | +### From Git (recommended) |
| 18 | + |
| 19 | +```bash |
| 20 | +cargo install --git https://github.com/anthropics/webfetch webfetch-cli |
| 21 | +``` |
| 22 | + |
| 23 | +### From Source |
| 24 | + |
| 25 | +```bash |
| 26 | +git clone https://github.com/anthropics/webfetch |
| 27 | +cd webfetch |
| 28 | +cargo install --path crates/webfetch-cli |
| 29 | +``` |
| 30 | + |
| 31 | +## CLI Usage |
| 32 | + |
| 33 | +```bash |
| 34 | +# Basic fetch |
| 35 | +webfetch --url https://example.com |
| 36 | + |
| 37 | +# Convert to markdown |
| 38 | +webfetch --url https://example.com --as-markdown |
| 39 | + |
| 40 | +# Convert to plain text |
| 41 | +webfetch --url https://example.com --as-text |
| 42 | + |
| 43 | +# HEAD request (metadata only) |
| 44 | +webfetch --url https://example.com --method HEAD |
| 45 | + |
| 46 | +# Custom user agent |
| 47 | +webfetch --url https://example.com --user-agent "MyBot/1.0" |
| 48 | + |
| 49 | +# Show full documentation |
| 50 | +webfetch --llmtxt |
| 51 | +``` |
| 52 | + |
| 53 | +Output is JSON to stdout: |
| 54 | + |
| 55 | +```json |
| 56 | +{ |
| 57 | + "url": "https://example.com", |
| 58 | + "status_code": 200, |
| 59 | + "content_type": "text/html", |
| 60 | + "size": 1256, |
| 61 | + "format": "markdown", |
| 62 | + "content": "# Example Domain\n\nThis domain is for use in illustrative examples...", |
| 63 | + "truncated": false, |
| 64 | + "method": "GET" |
| 65 | +} |
| 66 | +``` |
| 67 | + |
| 68 | +## MCP Server |
| 69 | + |
| 70 | +Run as a Model Context Protocol server: |
| 71 | + |
| 72 | +```bash |
| 73 | +webfetch mcp |
| 74 | +``` |
| 75 | + |
| 76 | +Exposes `webfetch` as a tool over JSON-RPC 2.0 stdio transport. Compatible with Claude Desktop and other MCP clients. |
| 77 | + |
| 78 | +## Library Usage |
| 79 | + |
| 80 | +Add to `Cargo.toml`: |
| 81 | + |
| 82 | +```toml |
| 83 | +[dependencies] |
| 84 | +webfetch = { git = "https://github.com/anthropics/webfetch" } |
| 85 | +``` |
| 86 | + |
| 87 | +### Basic Fetch |
| 88 | + |
| 89 | +```rust |
| 90 | +use webfetch::{fetch, WebFetchRequest}; |
| 91 | + |
| 92 | +#[tokio::main] |
| 93 | +async fn main() { |
| 94 | + let request = WebFetchRequest { |
| 95 | + url: "https://example.com".to_string(), |
| 96 | + method: None, |
| 97 | + as_markdown: Some(true), |
| 98 | + as_text: None, |
| 99 | + }; |
| 100 | + |
| 101 | + let response = fetch(request).await; |
| 102 | + println!("{}", response.content.unwrap_or_default()); |
| 103 | +} |
| 104 | +``` |
| 105 | + |
| 106 | +### With Tool Builder |
| 107 | + |
| 108 | +```rust |
| 109 | +use webfetch::Tool; |
| 110 | + |
| 111 | +let tool = Tool::builder() |
| 112 | + .enable_markdown(true) |
| 113 | + .enable_text(false) |
| 114 | + .user_agent("MyBot/1.0") |
| 115 | + .allow_prefix("https://docs.example.com") |
| 116 | + .block_prefix("https://internal.example.com") |
| 117 | + .build(); |
| 118 | + |
| 119 | +let response = tool.fetch(request).await; |
| 120 | +``` |
| 121 | + |
| 122 | +## Python Bindings |
| 123 | + |
| 124 | +```bash |
| 125 | +pip install webfetch |
| 126 | +``` |
| 127 | + |
| 128 | +```python |
| 129 | +from webfetch import fetch, WebFetchRequest, WebFetchTool |
| 130 | + |
| 131 | +# Simple fetch |
| 132 | +response = fetch("https://example.com", as_markdown=True) |
| 133 | +print(response.content) |
| 134 | + |
| 135 | +# With configuration |
| 136 | +tool = WebFetchTool( |
| 137 | + enable_markdown=True, |
| 138 | + user_agent="MyBot/1.0", |
| 139 | + allow_prefixes=["https://docs.example.com"] |
| 140 | +) |
| 141 | +response = tool.fetch(WebFetchRequest(url="https://example.com")) |
| 142 | +``` |
| 143 | + |
| 144 | +## Response Fields |
| 145 | + |
| 146 | +| Field | Type | Description | |
| 147 | +|-------|------|-------------| |
| 148 | +| `url` | string | Fetched URL | |
| 149 | +| `status_code` | int | HTTP status code | |
| 150 | +| `content_type` | string? | Content-Type header | |
| 151 | +| `size` | int? | Content size in bytes | |
| 152 | +| `last_modified` | string? | Last-Modified header | |
| 153 | +| `filename` | string? | From Content-Disposition | |
| 154 | +| `format` | string | "markdown", "text", or "raw" | |
| 155 | +| `content` | string? | Page content | |
| 156 | +| `truncated` | bool | True if content was cut off | |
| 157 | +| `method` | string | HTTP method used | |
| 158 | +| `error` | string? | Error message if failed | |
| 159 | + |
| 160 | +## Error Handling |
| 161 | + |
| 162 | +Errors are returned in the `error` field: |
| 163 | + |
| 164 | +- `InvalidUrl` - Malformed URL |
| 165 | +- `UrlBlocked` - URL blocked by filter |
| 166 | +- `NetworkError` - Connection failed |
| 167 | +- `Timeout` - Request timed out |
| 168 | +- `HttpError` - 4xx/5xx response |
| 169 | +- `ContentError` - Failed to read body |
| 170 | +- `BinaryContent` - Binary content not supported |
| 171 | + |
| 172 | +## Configuration |
| 173 | + |
| 174 | +### Timeouts |
| 175 | + |
| 176 | +- **First-byte**: 1 second (connect + initial response) |
| 177 | +- **Body**: 30 seconds total |
| 178 | + |
| 179 | +Partial content is returned on body timeout with `truncated: true`. |
| 180 | + |
| 181 | +### Binary Content |
| 182 | + |
| 183 | +Automatically detected and returns metadata only for: |
| 184 | +- Images, audio, video, fonts |
| 185 | +- PDFs, archives (zip, tar, rar, 7z) |
| 186 | +- Office documents |
| 187 | + |
| 188 | +### HTML Conversion |
| 189 | + |
| 190 | +**Markdown mode** (`--as-markdown`): |
| 191 | +- Headers: `h1-h6` → `#` to `######` |
| 192 | +- Lists: Proper nesting with 2-space indent |
| 193 | +- Code: Fenced blocks and inline backticks |
| 194 | +- Links: `[text](url)` format |
| 195 | +- Strips: scripts, styles, iframes, SVGs |
| 196 | + |
| 197 | +**Text mode** (`--as-text`): |
| 198 | +- Plain text extraction |
| 199 | +- Normalized whitespace |
| 200 | +- Newlines for block elements |
| 201 | + |
| 202 | +## License |
| 203 | + |
| 204 | +MIT |
0 commit comments