From 356c72a757b078f634941c5f2d54d4621b4cd01a Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 17 Apr 2025 12:56:11 -0700 Subject: [PATCH 1/3] with new tools --- chat-sample/package-lock.json | 143 +++++++++++++++- chat-sample/package.json | 33 +++- chat-sample/src/listSimulators.ts | 77 +++++++++ chat-sample/src/simple.ts | 117 +++++++------ chat-sample/src/toolParticipant.ts | 6 +- chat-sample/src/tools.ts | 44 ++++- chat-sample/src/toolsPrompt.tsx | 8 +- ...code.proposed.chatReferenceBinaryData.d.ts | 57 +++++++ .../vscode.propsed.languageModelDataPart.d.ts | 159 ++++++++++++++++++ 9 files changed, 565 insertions(+), 79 deletions(-) create mode 100644 chat-sample/src/listSimulators.ts create mode 100644 chat-sample/vscode.proposed.chatReferenceBinaryData.d.ts create mode 100644 chat-sample/vscode.propsed.languageModelDataPart.d.ts diff --git a/chat-sample/package-lock.json b/chat-sample/package-lock.json index 2de7b0b5da..051e450366 100644 --- a/chat-sample/package-lock.json +++ b/chat-sample/package-lock.json @@ -9,12 +9,14 @@ "version": "0.1.0", "dependencies": { "@vscode/chat-extension-utils": "^0.0.0-alpha.1", - "@vscode/prompt-tsx": "^0.3.0-alpha.12" + "@vscode/prompt-tsx": "file:vscode-prompt-tsx-0.3.0-alpha.23.tgz", + "screenshot-desktop": "^1.15.1" }, "devDependencies": { "@eslint/js": "^9.13.0", "@stylistic/eslint-plugin": "^2.9.0", "@types/node": "^20", + "@types/screenshot-desktop": "^1.12.3", "@types/vscode": "^1.95.0", "eslint": "^9.13.0", "typescript": "^5.8.2", @@ -293,6 +295,15 @@ "undici-types": "~5.26.4" } }, + "node_modules/@types/screenshot-desktop": { + "version": "1.12.3", + "resolved": "https://registry.npmjs.org/@types/screenshot-desktop/-/screenshot-desktop-1.12.3.tgz", + "integrity": "sha512-b1BoY60eEUwXgAE4le7gQFf78RWQxveF/ZKW5Ifh1+M4byPiyp3kP4qStVL+2pwrwaaJyN8qzeaczSDfz2WOPw==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/vscode": { "version": "1.95.0", "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.95.0.tgz", @@ -529,9 +540,9 @@ } }, "node_modules/@vscode/prompt-tsx": { - "version": "0.3.0-alpha.13", - "resolved": "https://registry.npmjs.org/@vscode/prompt-tsx/-/prompt-tsx-0.3.0-alpha.13.tgz", - "integrity": "sha512-0m9Hy2VqfGcFgXmY7xFV1nYngoq2zm2Wy/3YdesmR6bOwFrJed9xW87y43Ax7UFVHwtjZkpjn4M9HbFvxvzdWA==", + "version": "0.3.0-alpha.23", + "resolved": "file:vscode-prompt-tsx-0.3.0-alpha.23.tgz", + "integrity": "sha512-JKL2yh0mrJEQA8FrFQeOCtfB/ax8vdB9T6QWylx5YQ6gEqDRxLjWN3ChLfqnIWKEgIVqh0hR1JwOj7b5IhHu4Q==", "license": "MIT" }, "node_modules/acorn": { @@ -598,14 +609,12 @@ "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -673,7 +682,6 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, "license": "MIT" }, "node_modules/cross-spawn": { @@ -1033,6 +1041,31 @@ "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", "dev": true }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -1109,6 +1142,21 @@ "node": ">=0.8.19" } }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -1249,7 +1297,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -1258,6 +1305,25 @@ "node": "*" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -1270,6 +1336,14 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/optionator": { "version": "0.9.3", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", @@ -1339,6 +1413,14 @@ "node": ">=8" } }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -1422,6 +1504,18 @@ "node": ">=0.10.0" } }, + "node_modules/rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -1446,6 +1540,20 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/screenshot-desktop": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/screenshot-desktop/-/screenshot-desktop-1.15.1.tgz", + "integrity": "sha512-4j9bDZSFnkRqH53bAw5W+ZhdGKiTUcUDohO6NZjL6QSC/6AXbhUqJ9YXygjHrYcSOUD4IcLty6uQ1ES7PSsomg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/bencevans" + } + ], + "dependencies": { + "temp": "^0.9.4" + } + }, "node_modules/semver": { "version": "7.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", @@ -1505,6 +1613,18 @@ "node": ">=8" } }, + "node_modules/temp": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/temp/-/temp-0.9.4.tgz", + "integrity": "sha512-yYrrsWnrXMcdsnu/7YMYAofM1ktpL5By7vZhf15CrXijWWrEYZks5AXBudalfSWJLlnen/QUJUB5aoB0kqZUGA==", + "dependencies": { + "mkdirp": "^0.5.1", + "rimraf": "~2.6.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -1618,6 +1738,11 @@ "node": ">= 8" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/chat-sample/package.json b/chat-sample/package.json index 3315376bbc..770b1cca6e 100644 --- a/chat-sample/package.json +++ b/chat-sample/package.json @@ -1,6 +1,6 @@ { "name": "chat-sample", - "publisher": "vscode-samples", + "publisher": "vscode-samples-justin", "displayName": "Copilot Chat Sample", "description": "Sample chat extension, a trusty cat tutor that will can teach you computer science topics.", "repository": { @@ -15,6 +15,9 @@ "AI", "Chat" ], + "enabledApiProposals": [ + "languageModelDataPart" + ], "activationEvents": [], "contributes": { "chatParticipants": [ @@ -26,8 +29,8 @@ "isSticky": true, "commands": [ { - "name": "randomTeach", - "description": "Pick at random a computer science concept then explain it in purfect way of a cat" + "name": "catTakePhoto", + "description": "The cat is trying to take a photo." }, { "name": "play", @@ -155,6 +158,26 @@ "command" ] } + }, + { + "name": "chat-tools-sample_capture", + "tags": [ + "terminal", + "chat-tools-sample", + "screenshot", + "capture" + ], + "displayName": "Capture Screenshot", + "modelDescription": "Capture a screenshot and return the image", + "inputSchema": { + "type": "object", + "properties": { + "screenshot": { + "type": "string", + "description": "Take a screenshot of a simulator." + } + } + } } ], "commands": [ @@ -173,12 +196,14 @@ }, "dependencies": { "@vscode/chat-extension-utils": "^0.0.0-alpha.1", - "@vscode/prompt-tsx": "^0.3.0-alpha.12" + "@vscode/prompt-tsx": "file:vscode-prompt-tsx-0.3.0-alpha.23.tgz", + "screenshot-desktop": "^1.15.1" }, "devDependencies": { "@eslint/js": "^9.13.0", "@stylistic/eslint-plugin": "^2.9.0", "@types/node": "^20", + "@types/screenshot-desktop": "^1.12.3", "@types/vscode": "^1.95.0", "eslint": "^9.13.0", "typescript": "^5.8.2", diff --git a/chat-sample/src/listSimulators.ts b/chat-sample/src/listSimulators.ts new file mode 100644 index 0000000000..4099af294f --- /dev/null +++ b/chat-sample/src/listSimulators.ts @@ -0,0 +1,77 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import * as util from 'util'; +import { exec as execCallback } from 'child_process'; +import * as vscode from 'vscode'; + +const exec = util.promisify(execCallback); + +interface SimulatorDevice { + name: string; + udid: string; + state: string; +} + +export async function listSimulators(): Promise { + try { + const { stdout } = await exec('xcrun simctl list devices available --json'); + const result = JSON.parse(stdout); + const devices: SimulatorDevice[] = []; + + // Parse the devices from all runtimes + Object.values(result.devices).forEach((runtime: any) => { + (runtime as any[]).forEach(device => { + if (device.state === 'Booted' || device.state === 'Shutdown') { + devices.push({ + name: device.name, + udid: device.udid, + state: device.state + }); + } + }); + }); + + return devices; + } catch (error) { + console.error('Failed to list simulators:', error); + return []; + } +} + +export async function captureScreenshot(udid: string): Promise { + try { + // Ensure the simulator is booted + const devices = await listSimulators(); + const device = devices.find(d => d.udid === udid); + + if (!device) { + throw new Error(`No simulator found with UDID ${udid}`); + } + + if (device.state !== 'Booted') { + await exec(`xcrun simctl boot ${udid}`); + // Wait a moment for the simulator to fully boot + await new Promise(resolve => setTimeout(resolve, 5000)); + } + + // Create a temporary file for the screenshot + const tempDir = vscode.Uri.joinPath(vscode.Uri.file(process.env.TMPDIR || '/tmp'), 'vscode-simulator-capture'); + const tempFile = vscode.Uri.joinPath(tempDir, `${udid}.png`); + + // Ensure temp directory exists + await vscode.workspace.fs.createDirectory(tempDir); + + // Capture the screenshot + await exec(`xcrun simctl io ${udid} screenshot ${tempFile.fsPath}`); + + // Read the file + const imageData = await vscode.workspace.fs.readFile(tempFile); + + // Clean up + await vscode.workspace.fs.delete(tempFile); + + return imageData; + } catch (error) { + console.error('Failed to capture screenshot:', error); + return undefined; + } +} diff --git a/chat-sample/src/simple.ts b/chat-sample/src/simple.ts index db3caac697..bdbd5897c5 100644 --- a/chat-sample/src/simple.ts +++ b/chat-sample/src/simple.ts @@ -1,6 +1,7 @@ -import { renderPrompt } from '@vscode/prompt-tsx'; +import * as fs from 'fs/promises'; +import * as path from 'path'; +import screenshot from 'screenshot-desktop'; import * as vscode from 'vscode'; -import { PlayPrompt } from './play'; const CAT_NAMES_COMMAND_ID = 'cat.namesInEditor'; const CAT_PARTICIPANT_ID = 'chat-sample.cat'; @@ -18,13 +19,18 @@ export function registerSimpleParticipant(context: vscode.ExtensionContext) { // To talk to an LLM in your subcommand handler implementation, your // extension can use VS Code's `requestChatAccess` API to access the Copilot API. // The GitHub Copilot Chat extension implements this provider. - if (request.command === 'randomTeach') { - stream.progress('Picking the right topic to teach...'); - const topic = getTopic(context.history); + if (request.command === 'catTakePhoto') { + stream.progress('Somehow the cat is taking a photo...'); try { + // + const imageBuffer = await screenshot({ format: 'png' }); + const imageData = Uint8Array.from(imageBuffer); const messages = [ - vscode.LanguageModelChatMessage.User('You are a cat! Your job is to explain computer science concepts in the funny manner of a cat. Always start your response by stating what concept you are explaining. Always include code samples.'), - vscode.LanguageModelChatMessage.User(topic) + vscode.LanguageModelChatMessage2.User([new vscode.LanguageModelDataPart({ + data: imageData, + mimeType: vscode.ChatImageMimeType.PNG, + })]), + vscode.LanguageModelChatMessage.User('tell me about this image. make sure to be very detailed and start the sentence with "meow i am a cat"'), ]; const chatResponse = await request.model.sendRequest(messages, {}, token); @@ -36,40 +42,43 @@ export function registerSimpleParticipant(context: vscode.ExtensionContext) { handleError(logger, err, stream); } - stream.button({ - command: CAT_NAMES_COMMAND_ID, - title: vscode.l10n.t('Use Cat Names in Editor') - }); - - logger.logUsage('request', { kind: 'randomTeach' }); - return { metadata: { command: 'randomTeach' } }; + logger.logUsage('request', { kind: 'catTakePhoto' }); + return { metadata: { command: 'catTakePhoto' } }; } else if (request.command === 'play') { - stream.progress('Throwing away the computer science books and preparing to play with some Python code...'); - try { - // Here's an example of how to use the prompt-tsx library to build a prompt - const { messages } = await renderPrompt( - PlayPrompt, - { userQuery: request.prompt }, - { modelMaxPromptTokens: request.model.maxInputTokens }, - request.model); - - const chatResponse = await request.model.sendRequest(messages, {}, token); - for await (const fragment of chatResponse.text) { - stream.markdown(fragment); + stream.progress('Analysing image...'); + try { + const match = request.prompt.match(/(.*)'(.*)'/); + if (!match) { + stream.markdown(vscode.l10n.t('Please provide a question followed by a path to an image in quotes.')); + return { metadata: { command: 'play' } }; + } + + const imageBuffer = await fs.readFile(path.join(match[2])); + const imageData = Uint8Array.from(imageBuffer); + const messages = [ + vscode.LanguageModelChatMessage2.User([new vscode.LanguageModelDataPart({ + data: imageData, + mimeType: vscode.ChatImageMimeType.PNG, + })]), + vscode.LanguageModelChatMessage2.User(match[1]), + ]; + + const chatResponse = await request.model.sendRequest(messages, {}, token); + for await (const fragment of chatResponse.text) { + stream.markdown(fragment); + } + + } catch (err) { + handleError(logger, err, stream); } - - } catch (err) { - handleError(logger, err, stream); - } - - logger.logUsage('request', { kind: 'play' }); - return { metadata: { command: 'play' } }; + return { metadata: { command: 'play' } }; + } else { try { const messages = [ - vscode.LanguageModelChatMessage.User(`You are a cat! Think carefully and step by step like a cat would. + vscode.LanguageModelChatMessage2.User(`You are a cat! Think carefully and step by step like a cat would. Your job is to explain computer science concepts in the funny manner of a cat, using cat metaphors. Always start your response by stating what concept you are explaining. Always include code samples.`), - vscode.LanguageModelChatMessage.User(request.prompt) + vscode.LanguageModelChatMessage2.User(request.prompt) ]; const chatResponse = await request.model.sendRequest(messages, {}, token); @@ -141,9 +150,9 @@ export function registerSimpleParticipant(context: vscode.ExtensionContext) { } const messages = [ - vscode.LanguageModelChatMessage.User(`You are a cat! Think carefully and step by step like a cat would. + vscode.LanguageModelChatMessage2.User(`You are a cat! Think carefully and step by step like a cat would. Your job is to replace all variable names in the following code with funny cat variable names. Be creative. IMPORTANT respond just with code. Do not use markdown!`), - vscode.LanguageModelChatMessage.User(text) + vscode.LanguageModelChatMessage2.User(text) ]; chatResponse = await model.sendRequest(messages, {}, new vscode.CancellationTokenSource().token); @@ -204,20 +213,20 @@ function handleError(logger: vscode.TelemetryLogger, err: any, stream: vscode.Ch } // Get a random topic that the cat has not taught in the chat history yet -function getTopic(history: ReadonlyArray): string { - const topics = ['linked list', 'recursion', 'stack', 'queue', 'pointers']; - // Filter the chat history to get only the responses from the cat - const previousCatResponses = history.filter(h => { - return h instanceof vscode.ChatResponseTurn && h.participant === CAT_PARTICIPANT_ID; - }) as vscode.ChatResponseTurn[]; - // Filter the topics to get only the topics that have not been taught by the cat yet - const topicsNoRepetition = topics.filter(topic => { - return !previousCatResponses.some(catResponse => { - return catResponse.response.some(r => { - return r instanceof vscode.ChatResponseMarkdownPart && r.value.value.includes(topic); - }); - }); - }); - - return topicsNoRepetition[Math.floor(Math.random() * topicsNoRepetition.length)] || 'I have taught you everything I know. Meow!'; -} +// function getTopic(history: ReadonlyArray): string { +// const topics = ['linked list', 'recursion', 'stack', 'queue', 'pointers']; +// // Filter the chat history to get only the responses from the cat +// const previousCatResponses = history.filter(h => { +// return h instanceof vscode.ChatResponseTurn && h.participant === CAT_PARTICIPANT_ID; +// }) as vscode.ChatResponseTurn[]; +// // Filter the topics to get only the topics that have not been taught by the cat yet +// const topicsNoRepetition = topics.filter(topic => { +// return !previousCatResponses.some(catResponse => { +// return catResponse.response.some(r => { +// return r instanceof vscode.ChatResponseMarkdownPart && r.value.value.includes(topic); +// }); +// }); +// }); + +// return topicsNoRepetition[Math.floor(Math.random() * topicsNoRepetition.length)] || 'I have taught you everything I know. Meow!'; +// } diff --git a/chat-sample/src/toolParticipant.ts b/chat-sample/src/toolParticipant.ts index 39d41e366a..d09f5a2657 100644 --- a/chat-sample/src/toolParticipant.ts +++ b/chat-sample/src/toolParticipant.ts @@ -8,7 +8,7 @@ export interface TsxToolUserMetadata { export interface ToolCallsMetadata { toolCallRounds: ToolCallRound[]; - toolCallResults: Record; + toolCallResults: Record; } export function isTsxToolUserMetadata(obj: unknown): obj is TsxToolUserMetadata { @@ -62,7 +62,7 @@ export function registerToolUserChatParticipant(context: vscode.ExtensionContext }); const toolReferences = [...request.toolReferences]; - const accumulatedToolResults: Record = {}; + const accumulatedToolResults: Record = {}; const toolCallRounds: ToolCallRound[] = []; const runWithTools = async (): Promise => { // If a toolReference is present, force the model to call that tool @@ -135,4 +135,4 @@ export function registerToolUserChatParticipant(context: vscode.ExtensionContext const toolUser = vscode.chat.createChatParticipant('chat-tools-sample.tools', handler); toolUser.iconPath = new vscode.ThemeIcon('tools'); context.subscriptions.push(toolUser); -} \ No newline at end of file +} diff --git a/chat-sample/src/tools.ts b/chat-sample/src/tools.ts index 6306980633..a46c546adf 100644 --- a/chat-sample/src/tools.ts +++ b/chat-sample/src/tools.ts @@ -1,9 +1,11 @@ import * as vscode from 'vscode'; +import { captureScreenshot, listSimulators } from './listSimulators'; export function registerChatTools(context: vscode.ExtensionContext) { context.subscriptions.push(vscode.lm.registerTool('chat-tools-sample_tabCount', new TabCountTool())); context.subscriptions.push(vscode.lm.registerTool('chat-tools-sample_findFiles', new FindFilesTool())); context.subscriptions.push(vscode.lm.registerTool('chat-tools-sample_runInTerminal', new RunInTerminalTool())); + context.subscriptions.push(vscode.lm.registerTool('chat-tools-sample_capture', new CaptureTool())); } interface ITabCountParameters { @@ -26,10 +28,10 @@ export class TabCountTool implements vscode.LanguageModelTool f.fsPath).join('\n'); - return new vscode.LanguageModelToolResult([new vscode.LanguageModelTextPart(`Found ${files.length} files matching "${params.pattern}":\n${strFiles}`)]); + return new vscode.LanguageModelToolResult2([new vscode.LanguageModelTextPart(`Found ${files.length} files matching "${params.pattern}":\n${strFiles}`)]); } async prepareInvocation( @@ -126,7 +128,7 @@ export class RunInTerminalTool try { await waitForShellIntegration(terminal, 5000); } catch (e) { - return new vscode.LanguageModelToolResult([new vscode.LanguageModelTextPart((e as Error).message)]); + return new vscode.LanguageModelToolResult2([new vscode.LanguageModelTextPart((e as Error).message)]); } const execution = terminal.shellIntegration!.executeCommand(params.command); @@ -137,7 +139,7 @@ export class RunInTerminalTool terminalResult += chunk; } - return new vscode.LanguageModelToolResult([new vscode.LanguageModelTextPart(terminalResult)]); + return new vscode.LanguageModelToolResult2([new vscode.LanguageModelTextPart(terminalResult)]); } async prepareInvocation( @@ -158,3 +160,35 @@ export class RunInTerminalTool }; } } + +interface CaptureToolOptions { + prompt?: string; +} + +class CaptureTool implements vscode.LanguageModelTool { + async invoke(_options: vscode.LanguageModelToolInvocationOptions, _token: vscode.CancellationToken): Promise { + try { + // List available simulators + const simulators = await listSimulators(); + + if (simulators.length === 0) { + throw new Error('No iOS simulators found. Please make sure Xcode is installed and iOS simulators are set up.'); + } + + // Use the first available simulator + const simulator = simulators[0]; + const screenshot = await captureScreenshot(simulator.udid); + + if (!screenshot) { + throw new Error('Failed to capture simulator screenshot.'); + } + + return new vscode.LanguageModelToolResult2([new vscode.LanguageModelDataPart({ mimeType: vscode.ChatImageMimeType.PNG, data: screenshot})]); + } catch (error) { + throw error instanceof Error ? error : new Error(String(error)); + } + } +} + + +export function deactivate() { } diff --git a/chat-sample/src/toolsPrompt.tsx b/chat-sample/src/toolsPrompt.tsx index f533fbdabe..e5fe01d2d4 100644 --- a/chat-sample/src/toolsPrompt.tsx +++ b/chat-sample/src/toolsPrompt.tsx @@ -26,7 +26,7 @@ export interface ToolUserProps extends BasePromptElementProps { request: vscode.ChatRequest; context: vscode.ChatContext; toolCallRounds: ToolCallRound[]; - toolCallResults: Record; + toolCallResults: Record; } export class ToolUserPrompt extends PromptElement { @@ -65,7 +65,7 @@ export class ToolUserPrompt extends PromptElement { interface ToolCallsProps extends BasePromptElementProps { toolCallRounds: ToolCallRound[]; - toolCallResults: Record; + toolCallResults: Record; toolInvocationToken: vscode.ChatParticipantToolToken | undefined; } @@ -101,7 +101,7 @@ class ToolCalls extends PromptElement { interface ToolResultElementProps extends BasePromptElementProps { toolCall: vscode.LanguageModelToolCallPart; toolInvocationToken: vscode.ChatParticipantToolToken | undefined; - toolCallResult: vscode.LanguageModelToolResult | undefined; + toolCallResult: vscode.LanguageModelToolResult2 | undefined; } /** @@ -135,7 +135,7 @@ class ToolResultElement extends PromptElement { export class ToolResultMetadata extends PromptMetadata { constructor( public toolCallId: string, - public result: vscode.LanguageModelToolResult, + public result: vscode.LanguageModelToolResult2, ) { super(); } diff --git a/chat-sample/vscode.proposed.chatReferenceBinaryData.d.ts b/chat-sample/vscode.proposed.chatReferenceBinaryData.d.ts new file mode 100644 index 0000000000..3917935ba6 --- /dev/null +++ b/chat-sample/vscode.proposed.chatReferenceBinaryData.d.ts @@ -0,0 +1,57 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +declare module 'vscode' { + +/** + * A reference to a value that the user added to their chat request. + */ + export interface ChatPromptReference { + /** + * A unique identifier for this kind of reference. + */ + readonly id: string; + + /** + * The start and end index of the reference in the {@link ChatRequest.prompt prompt}. When undefined, the reference was not part of the prompt text. + * + * *Note* that the indices take the leading `#`-character into account which means they can + * used to modify the prompt as-is. + */ + readonly range?: [start: number, end: number]; + + /** + * A description of this value that could be used in an LLM prompt. + */ + readonly modelDescription?: string; + + /** + * The value of this reference. The `string | Uri | Location` types are used today, but this could expand in the future. + */ + readonly value: string | Uri | Location | ChatReferenceBinaryData| unknown; + } + + + export class ChatReferenceBinaryData { + /** + * The MIME type of the binary data. + */ + readonly mimeType: string; + + /** + * Retrieves the binary data of the reference. + * @returns A promise that resolves to the binary data as a Uint8Array. + */ + data(): Thenable; + + /** + * @param mimeType The MIME type of the binary data. + * @param data The binary data of the reference. + */ + constructor(mimeType: string, data: () => Thenable); + } +} + + diff --git a/chat-sample/vscode.propsed.languageModelDataPart.d.ts b/chat-sample/vscode.propsed.languageModelDataPart.d.ts new file mode 100644 index 0000000000..09a9c4e91b --- /dev/null +++ b/chat-sample/vscode.propsed.languageModelDataPart.d.ts @@ -0,0 +1,159 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +declare module 'vscode' { + + export interface LanguageModelChat { + sendRequest(messages: Array, options?: LanguageModelChatRequestOptions, token?: CancellationToken): Thenable; + countTokens(text: string | LanguageModelChatMessage | LanguageModelChatMessage2, token?: CancellationToken): Thenable; + } + + /** + * Represents a message in a chat. Can assume different roles, like user or assistant. + */ + export class LanguageModelChatMessage2 { + + /** + * Utility to create a new user message. + * + * @param content The content of the message. + * @param name The optional name of a user for the message. + */ + static User(content: string | Array, name?: string): LanguageModelChatMessage2; + + /** + * Utility to create a new assistant message. + * + * @param content The content of the message. + * @param name The optional name of a user for the message. + */ + static Assistant(content: string | Array, name?: string): LanguageModelChatMessage2; + + /** + * The role of this message. + */ + role: LanguageModelChatMessageRole; + + /** + * A string or heterogeneous array of things that a message can contain as content. Some parts may be message-type + * specific for some models. + */ + content: Array; + + /** + * The optional name of a user for this message. + */ + name: string | undefined; + + /** + * Create a new user message. + * + * @param role The role of the message. + * @param content The content of the message. + * @param name The optional name of a user for the message. + */ + constructor(role: LanguageModelChatMessageRole, content: string | Array, name?: string); + } + + /** + * A language model response part containing an image, returned from a {@link LanguageModelChatResponse}. + */ + export class LanguageModelDataPart { + /** + * The image content of the part. + */ + value: ChatImagePart; + + /** + * Construct an image part with the given content. + * @param value The image content of the part. + */ + constructor(value: ChatImagePart); + } + + /** + * Enum for supported image MIME types. + */ + export enum ChatImageMimeType { + PNG = 'image/png', + JPEG = 'image/jpeg', + GIF = 'image/gif', + WEBP = 'image/webp', + BMP = 'image/bmp', + } + + export interface ChatImagePart { + /** + * The image's MIME type. + */ + mimeType: ChatImageMimeType; + + /** + * The raw binary data of the image, encoded as a Uint8Array. Note: do not use base64 encoding. Maximum image size is 5MB. + */ + data: Uint8Array; + } + + + /** + * The result of a tool call. This is the counterpart of a {@link LanguageModelToolCallPart tool call} and + * it can only be included in the content of a User message + */ + export class LanguageModelToolResultPart2 { + /** + * The ID of the tool call. + * + * *Note* that this should match the {@link LanguageModelToolCallPart.callId callId} of a tool call part. + */ + callId: string; + + /** + * The value of the tool result. + */ + content: Array; + + /** + * @param callId The ID of the tool call. + * @param content The content of the tool result. + */ + constructor(callId: string, content: Array); + } + + + /** + * A tool that can be invoked by a call to a {@link LanguageModelChat}. + */ + export interface LanguageModelTool { + /** + * Invoke the tool with the given input and return a result. + * + * The provided {@link LanguageModelToolInvocationOptions.input} has been validated against the declared schema. + */ + invoke(options: LanguageModelToolInvocationOptions, token: CancellationToken): ProviderResult; + } + + /** + * A result returned from a tool invocation. If using `@vscode/prompt-tsx`, this result may be rendered using a `ToolResult`. + */ + export class LanguageModelToolResult2 { + /** + * A list of tool result content parts. Includes `unknown` becauses this list may be extended with new content types in + * the future. + * @see {@link lm.invokeTool}. + */ + content: Array; + + /** + * Create a LanguageModelToolResult + * @param content A list of tool result content parts + */ + constructor(content: Array); + } + + export namespace lm { + export function invokeTool(name: string, options: LanguageModelToolInvocationOptions, token?: CancellationToken): Thenable; + } + +} From 9bb9899a8b6427dc72d68ff25f85c668c08c7a2f Mon Sep 17 00:00:00 2001 From: justschen Date: Tue, 9 Sep 2025 16:14:45 -0700 Subject: [PATCH 2/3] new tools temp --- chat-sample/package-lock.json | 343 +++++++++++----- chat-sample/package.json | 4 +- chat-sample/src/simple.ts | 380 +++++++++--------- chat-sample/src/toolParticipant.ts | 2 +- chat-sample/src/tools.ts | 24 +- .../vscode.propsed.languageModelDataPart.d.ts | 148 +++---- 6 files changed, 502 insertions(+), 399 deletions(-) diff --git a/chat-sample/package-lock.json b/chat-sample/package-lock.json index 27f1930858..76f5cde866 100644 --- a/chat-sample/package-lock.json +++ b/chat-sample/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.0", "dependencies": { "@vscode/chat-extension-utils": "^0.0.0-alpha.1", - "@vscode/prompt-tsx": "file:vscode-prompt-tsx-0.3.0-alpha.23.tgz", + "@vscode/prompt-tsx": "^0.3.0-alpha.7", "screenshot-desktop": "^1.15.1" }, "devDependencies": { @@ -53,6 +53,19 @@ "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" } }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, "node_modules/@eslint-community/regexpp": { "version": "4.12.1", "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.1.tgz", @@ -125,6 +138,30 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/@eslint/eslintrc/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/@eslint/js": { "version": "9.31.0", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.31.0.tgz", @@ -186,11 +223,26 @@ "node": ">=18.18.0" } }, + "node_modules/@humanfs/node/node_modules/@humanwhocodes/retry": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.3.1.tgz", + "integrity": "sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, "node_modules/@humanwhocodes/module-importer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", "dev": true, + "license": "Apache-2.0", "engines": { "node": ">=12.22" }, @@ -200,9 +252,9 @@ } }, "node_modules/@humanwhocodes/retry": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.3.1.tgz", - "integrity": "sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA==", + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", "dev": true, "license": "Apache-2.0", "engines": { @@ -252,14 +304,15 @@ } }, "node_modules/@stylistic/eslint-plugin": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/@stylistic/eslint-plugin/-/eslint-plugin-2.9.0.tgz", - "integrity": "sha512-OrDyFAYjBT61122MIY1a3SfEgy3YCMgt2vL4eoPmvTwDBwyQhAXurxNQznlRD/jESNfYWfID8Ej+31LljvF7Xg==", + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/@stylistic/eslint-plugin/-/eslint-plugin-2.13.0.tgz", + "integrity": "sha512-RnO1SaiCFHn666wNz2QfZEFxvmiNRqhzaMXHXxXXKt+MEP7aajlPxUSMIQpKAaJfverpovEYqjBOXDq6dDcaOQ==", "dev": true, + "license": "MIT", "dependencies": { - "@typescript-eslint/utils": "^8.8.0", - "eslint-visitor-keys": "^4.1.0", - "espree": "^10.2.0", + "@typescript-eslint/utils": "^8.13.0", + "eslint-visitor-keys": "^4.2.0", + "espree": "^10.3.0", "estraverse": "^5.3.0", "picomatch": "^4.0.2" }, @@ -270,41 +323,19 @@ "eslint": ">=8.40.0" } }, - "node_modules/@stylistic/eslint-plugin/node_modules/eslint-visitor-keys": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.1.0.tgz", - "integrity": "sha512-Q7lok0mqMUSf5a/AdAZkA5a/gHcO6snwQClVNNvFKCAVlxXucdU8pKydU5ZVZjBx5xr37vGbFFWtLQYreLzrZg==", - "dev": true, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@stylistic/eslint-plugin/node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/@types/estree": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", - "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", - "dev": true + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.7.tgz", + "integrity": "sha512-w28IoSUCJpidD/TGviZwwMJckNESJZXFu7NBZ5YJ4mEUnNraUn9Pm8HSZm/jDF1pDWYKspWE7oVphigUPRakIQ==", + "dev": true, + "license": "MIT" }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/@types/node": { "version": "22.17.0", @@ -321,6 +352,7 @@ "resolved": "https://registry.npmjs.org/@types/screenshot-desktop/-/screenshot-desktop-1.12.3.tgz", "integrity": "sha512-b1BoY60eEUwXgAE4le7gQFf78RWQxveF/ZKW5Ifh1+M4byPiyp3kP4qStVL+2pwrwaaJyN8qzeaczSDfz2WOPw==", "dev": true, + "license": "MIT", "dependencies": { "@types/node": "*" } @@ -604,18 +636,18 @@ } }, "node_modules/@vscode/chat-extension-utils": { - "version": "0.0.0-alpha.1", - "resolved": "https://registry.npmjs.org/@vscode/chat-extension-utils/-/chat-extension-utils-0.0.0-alpha.1.tgz", - "integrity": "sha512-49eYur98d1iukPEQqMYQL4lJgaKnM0QFQB4/BFIFvuuKM+Kug2KNE/TSIJJQXrp5CrP0kDOmIIXvTnNRPtO2vg==", + "version": "0.0.0-alpha.5", + "resolved": "https://registry.npmjs.org/@vscode/chat-extension-utils/-/chat-extension-utils-0.0.0-alpha.5.tgz", + "integrity": "sha512-EkfetTIGMDyClZkIx8oMOhprlXufnj0b/G1W4QGg4jhkWVUBE7kLRZsqEnpNjmtxHTugzc61gPQwT3zgH3HXgA==", "license": "MIT", "dependencies": { - "@vscode/prompt-tsx": "^0.3.0-alpha.13" + "@vscode/prompt-tsx": "^0.3.0-alpha.14" } }, "node_modules/@vscode/prompt-tsx": { - "version": "0.3.0-alpha.23", - "resolved": "file:vscode-prompt-tsx-0.3.0-alpha.23.tgz", - "integrity": "sha512-JKL2yh0mrJEQA8FrFQeOCtfB/ax8vdB9T6QWylx5YQ6gEqDRxLjWN3ChLfqnIWKEgIVqh0hR1JwOj7b5IhHu4Q==", + "version": "0.3.0-alpha.24", + "resolved": "https://registry.npmjs.org/@vscode/prompt-tsx/-/prompt-tsx-0.3.0-alpha.24.tgz", + "integrity": "sha512-WUz6rPLcN6F64WxxwTiLzHOuhUcdLKBWMckppn43DBC1Ba67Lvd9RV+2LOxj938YzvEVOKGoAY/qgRtXd77I7Q==", "license": "MIT" }, "node_modules/acorn": { @@ -636,6 +668,7 @@ "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", "dev": true, + "license": "MIT", "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } @@ -662,6 +695,7 @@ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "dev": true, + "license": "MIT", "dependencies": { "color-convert": "^2.0.1" }, @@ -682,7 +716,8 @@ "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" }, "node_modules/brace-expansion": { "version": "1.1.12", @@ -691,8 +726,7 @@ "dev": true, "license": "MIT", "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "balanced-match": "^1.0.0" } }, "node_modules/braces": { @@ -723,6 +757,7 @@ "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", "dev": true, + "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -739,6 +774,7 @@ "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", "dev": true, + "license": "MIT", "dependencies": { "color-name": "~1.1.4" }, @@ -750,7 +786,8 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/concat-map": { "version": "0.0.1", @@ -774,12 +811,13 @@ } }, "node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", "dev": true, + "license": "MIT", "dependencies": { - "ms": "2.1.2" + "ms": "^2.1.3" }, "engines": { "node": ">=6.0" @@ -794,13 +832,15 @@ "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, @@ -925,6 +965,30 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/eslint/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/espree": { "version": "10.4.0", "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", @@ -957,10 +1021,11 @@ } }, "node_modules/esquery": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", - "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", + "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", "dev": true, + "license": "BSD-3-Clause", "dependencies": { "estraverse": "^5.1.0" }, @@ -986,6 +1051,7 @@ "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true, + "license": "BSD-2-Clause", "engines": { "node": ">=4.0" } @@ -995,6 +1061,7 @@ "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true, + "license": "BSD-2-Clause", "engines": { "node": ">=0.10.0" } @@ -1047,7 +1114,8 @@ "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/fastq": { "version": "1.19.1", @@ -1064,6 +1132,7 @@ "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", "dev": true, + "license": "MIT", "dependencies": { "flat-cache": "^4.0.0" }, @@ -1089,6 +1158,7 @@ "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", "dev": true, + "license": "MIT", "dependencies": { "locate-path": "^6.0.0", "path-exists": "^4.0.0" @@ -1105,6 +1175,7 @@ "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", "dev": true, + "license": "MIT", "dependencies": { "flatted": "^3.2.9", "keyv": "^4.5.4" @@ -1114,21 +1185,24 @@ } }, "node_modules/flatted": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", - "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", - "dev": true + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", + "integrity": "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==", + "dev": true, + "license": "ISC" }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC" }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "deprecated": "Glob versions prior to v9 are no longer supported", + "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1149,6 +1223,7 @@ "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", "dev": true, + "license": "ISC", "dependencies": { "is-glob": "^4.0.3" }, @@ -1156,6 +1231,28 @@ "node": ">=10.13.0" } }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/globals": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", @@ -1181,15 +1278,17 @@ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/ignore": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", - "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", "dev": true, + "license": "MIT", "engines": { "node": ">= 4" } @@ -1216,6 +1315,7 @@ "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.8.19" } @@ -1225,6 +1325,7 @@ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -1233,13 +1334,15 @@ "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.10.0" } @@ -1249,6 +1352,7 @@ "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", "dev": true, + "license": "MIT", "dependencies": { "is-extglob": "^2.1.1" }, @@ -1270,7 +1374,8 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/js-yaml": { "version": "4.1.0", @@ -1289,7 +1394,8 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/json-schema-traverse": { "version": "0.4.1", @@ -1302,13 +1408,15 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", "dev": true, + "license": "MIT", "dependencies": { "json-buffer": "3.0.1" } @@ -1318,6 +1426,7 @@ "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", "dev": true, + "license": "MIT", "dependencies": { "prelude-ls": "^1.2.1", "type-check": "~0.4.0" @@ -1331,6 +1440,7 @@ "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", "dev": true, + "license": "MIT", "dependencies": { "p-locate": "^5.0.0" }, @@ -1345,7 +1455,8 @@ "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/merge2": { "version": "1.4.1", @@ -1371,22 +1482,40 @@ "node": ">=8.6" } }, + "node_modules/micromatch/node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, "license": "ISC", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^2.0.1" }, "engines": { - "node": "*" + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -1395,6 +1524,7 @@ "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "license": "MIT", "dependencies": { "minimist": "^1.2.6" }, @@ -1403,37 +1533,41 @@ } }, "node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", "dependencies": { "wrappy": "1" } }, "node_modules/optionator": { - "version": "0.9.3", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", - "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==", + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", "dev": true, + "license": "MIT", "dependencies": { - "@aashutoshrathi/word-wrap": "^1.2.3", "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", - "type-check": "^0.4.0" + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" }, "engines": { "node": ">= 0.8.0" @@ -1444,6 +1578,7 @@ "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", "dev": true, + "license": "MIT", "dependencies": { "yocto-queue": "^0.1.0" }, @@ -1459,6 +1594,7 @@ "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", "dev": true, + "license": "MIT", "dependencies": { "p-limit": "^3.0.2" }, @@ -1487,6 +1623,7 @@ "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -1495,6 +1632,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", "engines": { "node": ">=0.10.0" } @@ -1504,18 +1642,19 @@ "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", "dev": true, "license": "MIT", "engines": { - "node": ">=8.6" + "node": ">=12" }, "funding": { "url": "https://github.com/sponsors/jonschlinkert" @@ -1526,6 +1665,7 @@ "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", "dev": true, + "license": "MIT", "engines": { "node": ">= 0.8.0" } @@ -1587,6 +1727,7 @@ "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", "dependencies": { "glob": "^7.1.3" }, @@ -1628,6 +1769,7 @@ "url": "https://github.com/sponsors/bencevans" } ], + "license": "MIT", "dependencies": { "temp": "^0.9.4" } @@ -1650,6 +1792,7 @@ "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dev": true, + "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" }, @@ -1662,6 +1805,7 @@ "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -1684,6 +1828,7 @@ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", "dev": true, + "license": "MIT", "dependencies": { "has-flag": "^4.0.0" }, @@ -1722,6 +1867,7 @@ "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", "dev": true, + "license": "MIT", "dependencies": { "prelude-ls": "^1.2.1" }, @@ -1789,6 +1935,7 @@ "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, + "license": "ISC", "dependencies": { "isexe": "^2.0.0" }, @@ -1799,16 +1946,28 @@ "node": ">= 8" } }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, diff --git a/chat-sample/package.json b/chat-sample/package.json index 076dff1749..9cfbf9bddb 100644 --- a/chat-sample/package.json +++ b/chat-sample/package.json @@ -174,7 +174,7 @@ "properties": { "screenshot": { "type": "string", - "description": "Take a screenshot of a simulator." + "description": "Take a screenshot of the current window or screen or whatever is available." } } } @@ -196,7 +196,7 @@ }, "dependencies": { "@vscode/chat-extension-utils": "^0.0.0-alpha.1", - "@vscode/prompt-tsx": "file:vscode-prompt-tsx-0.3.0-alpha.23.tgz", + "@vscode/prompt-tsx": "^0.3.0-alpha.7", "screenshot-desktop": "^1.15.1" }, "devDependencies": { diff --git a/chat-sample/src/simple.ts b/chat-sample/src/simple.ts index bdbd5897c5..0cc152db12 100644 --- a/chat-sample/src/simple.ts +++ b/chat-sample/src/simple.ts @@ -7,209 +7,203 @@ const CAT_NAMES_COMMAND_ID = 'cat.namesInEditor'; const CAT_PARTICIPANT_ID = 'chat-sample.cat'; interface ICatChatResult extends vscode.ChatResult { - metadata: { - command: string; - } + metadata: { + command: string; + } } export function registerSimpleParticipant(context: vscode.ExtensionContext) { - // Define a Cat chat handler. - const handler: vscode.ChatRequestHandler = async (request: vscode.ChatRequest, context: vscode.ChatContext, stream: vscode.ChatResponseStream, token: vscode.CancellationToken): Promise => { - // To talk to an LLM in your subcommand handler implementation, your - // extension can use VS Code's `requestChatAccess` API to access the Copilot API. - // The GitHub Copilot Chat extension implements this provider. - if (request.command === 'catTakePhoto') { - stream.progress('Somehow the cat is taking a photo...'); - try { - // - const imageBuffer = await screenshot({ format: 'png' }); - const imageData = Uint8Array.from(imageBuffer); - const messages = [ - vscode.LanguageModelChatMessage2.User([new vscode.LanguageModelDataPart({ - data: imageData, - mimeType: vscode.ChatImageMimeType.PNG, - })]), - vscode.LanguageModelChatMessage.User('tell me about this image. make sure to be very detailed and start the sentence with "meow i am a cat"'), - ]; - - const chatResponse = await request.model.sendRequest(messages, {}, token); - for await (const fragment of chatResponse.text) { - stream.markdown(fragment); - } - - } catch (err) { - handleError(logger, err, stream); - } - - logger.logUsage('request', { kind: 'catTakePhoto' }); - return { metadata: { command: 'catTakePhoto' } }; - } else if (request.command === 'play') { - stream.progress('Analysing image...'); - try { - const match = request.prompt.match(/(.*)'(.*)'/); - if (!match) { - stream.markdown(vscode.l10n.t('Please provide a question followed by a path to an image in quotes.')); - return { metadata: { command: 'play' } }; - } - - const imageBuffer = await fs.readFile(path.join(match[2])); - const imageData = Uint8Array.from(imageBuffer); - const messages = [ - vscode.LanguageModelChatMessage2.User([new vscode.LanguageModelDataPart({ - data: imageData, - mimeType: vscode.ChatImageMimeType.PNG, - })]), - vscode.LanguageModelChatMessage2.User(match[1]), - ]; - - const chatResponse = await request.model.sendRequest(messages, {}, token); - for await (const fragment of chatResponse.text) { - stream.markdown(fragment); - } - - } catch (err) { - handleError(logger, err, stream); - } - return { metadata: { command: 'play' } }; - - } else { - try { - const messages = [ - vscode.LanguageModelChatMessage2.User(`You are a cat! Think carefully and step by step like a cat would. + // Define a Cat chat handler. + const handler: vscode.ChatRequestHandler = async (request: vscode.ChatRequest, context: vscode.ChatContext, stream: vscode.ChatResponseStream, token: vscode.CancellationToken): Promise => { + // To talk to an LLM in your subcommand handler implementation, your + // extension can use VS Code's `requestChatAccess` API to access the Copilot API. + // The GitHub Copilot Chat extension implements this provider. + if (request.command === 'catTakePhoto') { + stream.progress('Somehow the cat is taking a photo...'); + try { + // + const imageBuffer = await screenshot({ format: 'png' }); + const imageData = Uint8Array.from(imageBuffer); + const messages = [ + vscode.LanguageModelChatMessage2.User([new vscode.LanguageModelDataPart(imageData, 'image/png')]), + vscode.LanguageModelChatMessage2.User('tell me about this image. make sure to be very detailed and start the sentence with "meow i am a cat"'), + ]; + + const chatResponse = await request.model.sendRequest(messages, {}, token); + for await (const fragment of chatResponse.text) { + stream.markdown(fragment); + } + + } catch (err) { + handleError(logger, err, stream); + } + + logger.logUsage('request', { kind: 'catTakePhoto' }); + return { metadata: { command: 'catTakePhoto' } }; + } else if (request.command === 'play') { + stream.progress('Analysing image...'); + try { + const match = request.prompt.match(/(.*)'(.*)'/); + if (!match) { + stream.markdown(vscode.l10n.t('Please provide a question followed by a path to an image in quotes.')); + return { metadata: { command: 'play' } }; + } + + const imageBuffer = await fs.readFile(path.join(match[2])); + const imageData = Uint8Array.from(imageBuffer); + const messages = [ + vscode.LanguageModelChatMessage2.User([new vscode.LanguageModelDataPart(imageData, 'image/png')]), + vscode.LanguageModelChatMessage2.User(match[1]), + ]; + + const chatResponse = await request.model.sendRequest(messages, {}, token); + for await (const fragment of chatResponse.text) { + stream.markdown(fragment); + } + + } catch (err) { + handleError(logger, err, stream); + } + return { metadata: { command: 'play' } }; + + } else { + try { + const messages = [ + vscode.LanguageModelChatMessage2.User(`You are a cat! Think carefully and step by step like a cat would. Your job is to explain computer science concepts in the funny manner of a cat, using cat metaphors. Always start your response by stating what concept you are explaining. Always include code samples.`), - vscode.LanguageModelChatMessage2.User(request.prompt) - ]; - - const chatResponse = await request.model.sendRequest(messages, {}, token); - for await (const fragment of chatResponse.text) { - // Process the output from the language model - // Replace all python function definitions with cat sounds to make the user stop looking at the code and start playing with the cat - const catFragment = fragment.replaceAll('def', 'meow'); - stream.markdown(catFragment); - } - } catch (err) { - handleError(logger, err, stream); - } - - logger.logUsage('request', { kind: '' }); - return { metadata: { command: '' } }; - } - }; - - // Chat participants appear as top-level options in the chat input - // when you type `@`, and can contribute sub-commands in the chat input - // that appear when you type `/`. - const cat = vscode.chat.createChatParticipant(CAT_PARTICIPANT_ID, handler); - cat.iconPath = vscode.Uri.joinPath(context.extensionUri, 'cat.jpeg'); - cat.followupProvider = { - provideFollowups(_result: ICatChatResult, _context: vscode.ChatContext, _token: vscode.CancellationToken) { - return [{ - prompt: 'let us play', - label: vscode.l10n.t('Play with the cat'), - command: 'play' - } satisfies vscode.ChatFollowup]; - } - }; - - const logger = vscode.env.createTelemetryLogger({ - sendEventData(eventName, data) { - // Capture event telemetry - console.log(`Event: ${eventName}`); - console.log(`Data: ${JSON.stringify(data)}`); - }, - sendErrorData(error, data) { - // Capture error telemetry - console.error(`Error: ${error}`); - console.error(`Data: ${JSON.stringify(data)}`); - } - }); - - context.subscriptions.push(cat.onDidReceiveFeedback((feedback: vscode.ChatResultFeedback) => { - // Log chat result feedback to be able to compute the success matric of the participant - // unhelpful / totalRequests is a good success metric - logger.logUsage('chatResultFeedback', { - kind: feedback.kind - }); - })); - - context.subscriptions.push( - cat, - // Register the command handler for the /meow followup - vscode.commands.registerTextEditorCommand(CAT_NAMES_COMMAND_ID, async (textEditor: vscode.TextEditor) => { - // Replace all variables in active editor with cat names and words - const text = textEditor.document.getText(); - - let chatResponse: vscode.LanguageModelChatResponse | undefined; - try { - // Use gpt-4o since it is fast and high quality. - const [model] = await vscode.lm.selectChatModels({ vendor: 'copilot', family: 'gpt-4o' }); - if (!model) { - console.log('Model not found. Please make sure the GitHub Copilot Chat extension is installed and enabled.'); - return; - } - - const messages = [ - vscode.LanguageModelChatMessage2.User(`You are a cat! Think carefully and step by step like a cat would. + vscode.LanguageModelChatMessage2.User(request.prompt) + ]; + + const chatResponse = await request.model.sendRequest(messages, {}, token); + for await (const fragment of chatResponse.text) { + // Process the output from the language model + // Replace all python function definitions with cat sounds to make the user stop looking at the code and start playing with the cat + const catFragment = fragment.replaceAll('def', 'meow'); + stream.markdown(catFragment); + } + } catch (err) { + handleError(logger, err, stream); + } + + logger.logUsage('request', { kind: '' }); + return { metadata: { command: '' } }; + } + }; + + // Chat participants appear as top-level options in the chat input + // when you type `@`, and can contribute sub-commands in the chat input + // that appear when you type `/`. + const cat = vscode.chat.createChatParticipant(CAT_PARTICIPANT_ID, handler); + cat.iconPath = vscode.Uri.joinPath(context.extensionUri, 'cat.jpeg'); + cat.followupProvider = { + provideFollowups(_result: ICatChatResult, _context: vscode.ChatContext, _token: vscode.CancellationToken) { + return [{ + prompt: 'let us play', + label: vscode.l10n.t('Play with the cat'), + command: 'play' + } satisfies vscode.ChatFollowup]; + } + }; + + const logger = vscode.env.createTelemetryLogger({ + sendEventData(eventName, data) { + // Capture event telemetry + console.log(`Event: ${eventName}`); + console.log(`Data: ${JSON.stringify(data)}`); + }, + sendErrorData(error, data) { + // Capture error telemetry + console.error(`Error: ${error}`); + console.error(`Data: ${JSON.stringify(data)}`); + } + }); + + context.subscriptions.push(cat.onDidReceiveFeedback((feedback: vscode.ChatResultFeedback) => { + // Log chat result feedback to be able to compute the success matric of the participant + // unhelpful / totalRequests is a good success metric + logger.logUsage('chatResultFeedback', { + kind: feedback.kind + }); + })); + + context.subscriptions.push( + cat, + // Register the command handler for the /meow followup + vscode.commands.registerTextEditorCommand(CAT_NAMES_COMMAND_ID, async (textEditor: vscode.TextEditor) => { + // Replace all variables in active editor with cat names and words + const text = textEditor.document.getText(); + + let chatResponse: vscode.LanguageModelChatResponse | undefined; + try { + // Use gpt-4o since it is fast and high quality. + const [model] = await vscode.lm.selectChatModels({ vendor: 'copilot', family: 'gpt-4o' }); + if (!model) { + console.log('Model not found. Please make sure the GitHub Copilot Chat extension is installed and enabled.'); + return; + } + + const messages = [ + vscode.LanguageModelChatMessage2.User(`You are a cat! Think carefully and step by step like a cat would. Your job is to replace all variable names in the following code with funny cat variable names. Be creative. IMPORTANT respond just with code. Do not use markdown!`), - vscode.LanguageModelChatMessage2.User(text) - ]; - chatResponse = await model.sendRequest(messages, {}, new vscode.CancellationTokenSource().token); - - } catch (err) { - if (err instanceof vscode.LanguageModelError) { - console.log(err.message, err.code, err.cause); - } else { - throw err; - } - return; - } - - // Clear the editor content before inserting new content - await textEditor.edit(edit => { - const start = new vscode.Position(0, 0); - const end = new vscode.Position(textEditor.document.lineCount - 1, textEditor.document.lineAt(textEditor.document.lineCount - 1).text.length); - edit.delete(new vscode.Range(start, end)); - }); - - // Stream the code into the editor as it is coming in from the Language Model - try { - for await (const fragment of chatResponse.text) { - await textEditor.edit(edit => { - const lastLine = textEditor.document.lineAt(textEditor.document.lineCount - 1); - const position = new vscode.Position(lastLine.lineNumber, lastLine.text.length); - edit.insert(position, fragment); - }); - } - } catch (err) { - // async response stream may fail, e.g network interruption or server side error - await textEditor.edit(edit => { - const lastLine = textEditor.document.lineAt(textEditor.document.lineCount - 1); - const position = new vscode.Position(lastLine.lineNumber, lastLine.text.length); - edit.insert(position, (err as Error).message); - }); - } - }), - ); + vscode.LanguageModelChatMessage2.User(text) + ]; + chatResponse = await model.sendRequest(messages, {}, new vscode.CancellationTokenSource().token); + + } catch (err) { + if (err instanceof vscode.LanguageModelError) { + console.log(err.message, err.code, err.cause); + } else { + throw err; + } + return; + } + + // Clear the editor content before inserting new content + await textEditor.edit(edit => { + const start = new vscode.Position(0, 0); + const end = new vscode.Position(textEditor.document.lineCount - 1, textEditor.document.lineAt(textEditor.document.lineCount - 1).text.length); + edit.delete(new vscode.Range(start, end)); + }); + + // Stream the code into the editor as it is coming in from the Language Model + try { + for await (const fragment of chatResponse.text) { + await textEditor.edit(edit => { + const lastLine = textEditor.document.lineAt(textEditor.document.lineCount - 1); + const position = new vscode.Position(lastLine.lineNumber, lastLine.text.length); + edit.insert(position, fragment); + }); + } + } catch (err) { + // async response stream may fail, e.g network interruption or server side error + await textEditor.edit(edit => { + const lastLine = textEditor.document.lineAt(textEditor.document.lineCount - 1); + const position = new vscode.Position(lastLine.lineNumber, lastLine.text.length); + edit.insert(position, (err as Error).message); + }); + } + }), + ); } // eslint-disable-next-line @typescript-eslint/no-explicit-any function handleError(logger: vscode.TelemetryLogger, err: any, stream: vscode.ChatResponseStream): void { - // making the chat request might fail because - // - model does not exist - // - user consent not given - // - quote limits exceeded - logger.logError(err); - - if (err instanceof vscode.LanguageModelError) { - console.log(err.message, err.code, err.cause); - if (err.cause instanceof Error && err.cause.message.includes('off_topic')) { - stream.markdown(vscode.l10n.t('I\'m sorry, I can only explain computer science concepts.')); - } - } else { - // re-throw other errors so they show up in the UI - throw err; - } + // making the chat request might fail because + // - model does not exist + // - user consent not given + // - quote limits exceeded + logger.logError(err); + + if (err instanceof vscode.LanguageModelError) { + console.log(err.message, err.code, err.cause); + if (err.cause instanceof Error && err.cause.message.includes('off_topic')) { + stream.markdown(vscode.l10n.t('I\'m sorry, I can only explain computer science concepts.')); + } + } else { + // re-throw other errors so they show up in the UI + throw err; + } } // Get a random topic that the cat has not taught in the chat history yet diff --git a/chat-sample/src/toolParticipant.ts b/chat-sample/src/toolParticipant.ts index d09f5a2657..e110eb436d 100644 --- a/chat-sample/src/toolParticipant.ts +++ b/chat-sample/src/toolParticipant.ts @@ -108,7 +108,7 @@ export function registerToolUserChatParticipant(context: vscode.ExtensionContext { modelMaxPromptTokens: model.maxInputTokens }, model)); messages = result.messages; - const toolResultMetadata = result.metadatas.getAll(ToolResultMetadata); + const toolResultMetadata = result.metadata.getAll(ToolResultMetadata); if (toolResultMetadata?.length) { // Cache tool results for later, so they can be incorporated into later prompts without calling the tool again toolResultMetadata.forEach(meta => accumulatedToolResults[meta.toolCallId] = meta.result); diff --git a/chat-sample/src/tools.ts b/chat-sample/src/tools.ts index a46c546adf..f34d852d25 100644 --- a/chat-sample/src/tools.ts +++ b/chat-sample/src/tools.ts @@ -1,5 +1,5 @@ import * as vscode from 'vscode'; -import { captureScreenshot, listSimulators } from './listSimulators'; +import screenshot from 'screenshot-desktop'; export function registerChatTools(context: vscode.ExtensionContext) { context.subscriptions.push(vscode.lm.registerTool('chat-tools-sample_tabCount', new TabCountTool())); @@ -168,22 +168,18 @@ interface CaptureToolOptions { class CaptureTool implements vscode.LanguageModelTool { async invoke(_options: vscode.LanguageModelToolInvocationOptions, _token: vscode.CancellationToken): Promise { try { - // List available simulators - const simulators = await listSimulators(); - - if (simulators.length === 0) { - throw new Error('No iOS simulators found. Please make sure Xcode is installed and iOS simulators are set up.'); - } - - // Use the first available simulator - const simulator = simulators[0]; - const screenshot = await captureScreenshot(simulator.udid); - - if (!screenshot) { + const imageBuffer = await screenshot({ format: 'png' }); + const imageData = Uint8Array.from(imageBuffer); + // const messages = [ + // vscode.LanguageModelChatMessage2.User([new vscode.LanguageModelDataPart(imageData, vscode.ChatImageMimeType.PNG)]), + // vscode.LanguageModelChatMessage2.User('tell me about this image. make sure to be very detailed and start the sentence with "meow i am a cat"'), + // ]; + + if (!imageBuffer) { throw new Error('Failed to capture simulator screenshot.'); } - return new vscode.LanguageModelToolResult2([new vscode.LanguageModelDataPart({ mimeType: vscode.ChatImageMimeType.PNG, data: screenshot})]); + return new vscode.LanguageModelToolResult2([new vscode.LanguageModelDataPart(imageData, 'image/png')]); } catch (error) { throw error instanceof Error ? error : new Error(String(error)); } diff --git a/chat-sample/vscode.propsed.languageModelDataPart.d.ts b/chat-sample/vscode.propsed.languageModelDataPart.d.ts index 09a9c4e91b..545b5242f7 100644 --- a/chat-sample/vscode.propsed.languageModelDataPart.d.ts +++ b/chat-sample/vscode.propsed.languageModelDataPart.d.ts @@ -1,10 +1,54 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/consistent-indexed-object-style */ /*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +// version: 1 + declare module 'vscode' { + /** + * A language model response part containing thinking/reasoning content. + * Thinking tokens represent the model's internal reasoning process that + * typically streams before the final response. + */ + export class LanguageModelThinkingPart { + /** + * The thinking/reasoning text content. + */ + value: string | string[]; + + /** + * Optional unique identifier for this thinking sequence. + * This ID is typically provided at the end of the thinking stream + * and can be used for retrieval or reference purposes. + */ + id?: string; + + /** + * Optional metadata associated with this thinking sequence. + */ + metadata?: { readonly [key: string]: any }; + + /** + * Construct a thinking part with the given content. + * @param value The thinking text content. + * @param id Optional unique identifier for this thinking sequence. + * @param metadata Optional metadata associated with this thinking sequence. + */ + constructor(value: string | string[], id?: string, metadata?: { readonly [key: string]: any }); + } + + export interface LanguageModelChatResponse { + /** + * An async iterable that is a stream of text, thinking, and tool-call parts forming the overall response. + * This includes {@link LanguageModelThinkingPart} which represents the model's internal reasoning process. + */ + stream: AsyncIterable; + } + export interface LanguageModelChat { sendRequest(messages: Array, options?: LanguageModelChatRequestOptions, token?: CancellationToken): Thenable; countTokens(text: string | LanguageModelChatMessage | LanguageModelChatMessage2, token?: CancellationToken): Thenable; @@ -21,7 +65,7 @@ declare module 'vscode' { * @param content The content of the message. * @param name The optional name of a user for the message. */ - static User(content: string | Array, name?: string): LanguageModelChatMessage2; + static User(content: string | Array, name?: string): LanguageModelChatMessage2; /** * Utility to create a new assistant message. @@ -40,7 +84,7 @@ declare module 'vscode' { * A string or heterogeneous array of things that a message can contain as content. Some parts may be message-type * specific for some models. */ - content: Array; + content: Array; /** * The optional name of a user for this message. @@ -54,106 +98,16 @@ declare module 'vscode' { * @param content The content of the message. * @param name The optional name of a user for the message. */ - constructor(role: LanguageModelChatMessageRole, content: string | Array, name?: string); + constructor(role: LanguageModelChatMessageRole, content: string | Array, name?: string); } /** - * A language model response part containing an image, returned from a {@link LanguageModelChatResponse}. + * Temporary alias for LanguageModelToolResultPart to avoid breaking changes in chat. */ - export class LanguageModelDataPart { - /** - * The image content of the part. - */ - value: ChatImagePart; - - /** - * Construct an image part with the given content. - * @param value The image content of the part. - */ - constructor(value: ChatImagePart); - } + export class LanguageModelToolResultPart2 extends LanguageModelToolResultPart { } /** - * Enum for supported image MIME types. + * Temporary alias for LanguageModelToolResult to avoid breaking changes in chat. */ - export enum ChatImageMimeType { - PNG = 'image/png', - JPEG = 'image/jpeg', - GIF = 'image/gif', - WEBP = 'image/webp', - BMP = 'image/bmp', - } - - export interface ChatImagePart { - /** - * The image's MIME type. - */ - mimeType: ChatImageMimeType; - - /** - * The raw binary data of the image, encoded as a Uint8Array. Note: do not use base64 encoding. Maximum image size is 5MB. - */ - data: Uint8Array; - } - - - /** - * The result of a tool call. This is the counterpart of a {@link LanguageModelToolCallPart tool call} and - * it can only be included in the content of a User message - */ - export class LanguageModelToolResultPart2 { - /** - * The ID of the tool call. - * - * *Note* that this should match the {@link LanguageModelToolCallPart.callId callId} of a tool call part. - */ - callId: string; - - /** - * The value of the tool result. - */ - content: Array; - - /** - * @param callId The ID of the tool call. - * @param content The content of the tool result. - */ - constructor(callId: string, content: Array); - } - - - /** - * A tool that can be invoked by a call to a {@link LanguageModelChat}. - */ - export interface LanguageModelTool { - /** - * Invoke the tool with the given input and return a result. - * - * The provided {@link LanguageModelToolInvocationOptions.input} has been validated against the declared schema. - */ - invoke(options: LanguageModelToolInvocationOptions, token: CancellationToken): ProviderResult; - } - - /** - * A result returned from a tool invocation. If using `@vscode/prompt-tsx`, this result may be rendered using a `ToolResult`. - */ - export class LanguageModelToolResult2 { - /** - * A list of tool result content parts. Includes `unknown` becauses this list may be extended with new content types in - * the future. - * @see {@link lm.invokeTool}. - */ - content: Array; - - /** - * Create a LanguageModelToolResult - * @param content A list of tool result content parts - */ - constructor(content: Array); - } - - export namespace lm { - export function invokeTool(name: string, options: LanguageModelToolInvocationOptions, token?: CancellationToken): Thenable; - } - + export class LanguageModelToolResult2 extends LanguageModelToolResult { } } From a287f4f47696cadacffb99222ee4c7453311907d Mon Sep 17 00:00:00 2001 From: justschen Date: Tue, 9 Sep 2025 16:17:48 -0700 Subject: [PATCH 3/3] fix prompt tsx --- chat-sample/package-lock.json | 116 ++++++++++------------------------ chat-sample/package.json | 4 +- 2 files changed, 35 insertions(+), 85 deletions(-) diff --git a/chat-sample/package-lock.json b/chat-sample/package-lock.json index 76f5cde866..eb3f790d0a 100644 --- a/chat-sample/package-lock.json +++ b/chat-sample/package-lock.json @@ -9,13 +9,14 @@ "version": "0.1.0", "dependencies": { "@vscode/chat-extension-utils": "^0.0.0-alpha.1", - "@vscode/prompt-tsx": "^0.3.0-alpha.7", + "@vscode/prompt-tsx": "^0.3.0-alpha.12", "screenshot-desktop": "^1.15.1" }, "devDependencies": { "@eslint/js": "^9.13.0", "@stylistic/eslint-plugin": "^2.9.0", "@types/node": "^22", + "@types/screenshot-desktop": "^1.12.3", "@types/vscode": "^1.100.0", "eslint": "^9.13.0", "typescript": "^5.9.2", @@ -25,15 +26,6 @@ "vscode": "^1.100.0" } }, - "node_modules/@aashutoshrathi/word-wrap": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz", - "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/@eslint-community/eslint-utils": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.7.0.tgz", @@ -91,6 +83,19 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@eslint/config-array/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/@eslint/config-helpers": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.3.0.tgz", @@ -251,20 +256,6 @@ "url": "https://github.com/sponsors/nzakas" } }, - "node_modules/@humanwhocodes/retry": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", - "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=18.18" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -622,19 +613,6 @@ "url": "https://opencollective.com/typescript-eslint" } }, - "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", - "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, "node_modules/@vscode/chat-extension-utils": { "version": "0.0.0-alpha.5", "resolved": "https://registry.npmjs.org/@vscode/chat-extension-utils/-/chat-extension-utils-0.0.0-alpha.5.tgz", @@ -927,12 +905,13 @@ } }, "node_modules/eslint-visitor-keys": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", - "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", "dev": true, + "license": "Apache-2.0", "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" }, "funding": { "url": "https://opencollective.com/eslint" @@ -952,19 +931,6 @@ "url": "https://github.com/sponsors/nzakas" } }, - "node_modules/eslint/node_modules/eslint-visitor-keys": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", - "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, "node_modules/eslint/node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -1007,19 +973,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/espree/node_modules/eslint-visitor-keys": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", - "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, "node_modules/esquery": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", @@ -1495,22 +1448,6 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", @@ -1836,6 +1773,19 @@ "node": ">=8" } }, + "node_modules/temp": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/temp/-/temp-0.9.4.tgz", + "integrity": "sha512-yYrrsWnrXMcdsnu/7YMYAofM1ktpL5By7vZhf15CrXijWWrEYZks5AXBudalfSWJLlnen/QUJUB5aoB0kqZUGA==", + "license": "MIT", + "dependencies": { + "mkdirp": "^0.5.1", + "rimraf": "~2.6.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", diff --git a/chat-sample/package.json b/chat-sample/package.json index 9cfbf9bddb..a4b20f6d60 100644 --- a/chat-sample/package.json +++ b/chat-sample/package.json @@ -196,7 +196,7 @@ }, "dependencies": { "@vscode/chat-extension-utils": "^0.0.0-alpha.1", - "@vscode/prompt-tsx": "^0.3.0-alpha.7", + "@vscode/prompt-tsx": "^0.3.0-alpha.12", "screenshot-desktop": "^1.15.1" }, "devDependencies": { @@ -209,4 +209,4 @@ "typescript": "^5.9.2", "typescript-eslint": "^8.39.0" } -} \ No newline at end of file +}