feat: implement rate limiting for LLM requests

src/vs/workbench/contrib/void/browser/autocompleteService.ts

@@ -795,67 +795,71 @@ export class AutocompleteService extends Disposable implements IAutocompleteServ
 		const modelSelectionOptions = modelSelection ? this._settingsService.state.optionsOfModelSelection[featureName][modelSelection.providerName]?.[modelSelection.modelName] : undefined
 
 		// set parameters of `newAutocompletion` appropriately
-		newAutocompletion.llmPromise = new Promise((resolve, reject) => {
-
-			const requestId = this._llmMessageService.sendLLMMessage({
-				messagesType: 'FIMMessage',
-				messages: this._convertToLLMMessageService.prepareFIMMessage({
-					messages: {
-						prefix: llmPrefix,
-						suffix: llmSuffix,
-						stopTokens: stopTokens,
-					}
-				}),
-				modelSelection,
-				modelSelectionOptions,
-				overridesOfModel,
-				logging: { loggingName: 'Autocomplete' },
-				onText: () => { }, // unused in FIMMessage
-				// onText: async ({ fullText, newText }) => {
-
-				// 	newAutocompletion.insertText = fullText
-
-				// 	// count newlines in newText
-				// 	const numNewlines = newText.match(/\n|\r\n/g)?.length || 0
-				// 	newAutocompletion._newlineCount += numNewlines
-
-				// 	// if too many newlines, resolve up to last newline
-				// 	if (newAutocompletion._newlineCount > 10) {
-				// 		const lastNewlinePos = fullText.lastIndexOf('\n')
-				// 		newAutocompletion.insertText = fullText.substring(0, lastNewlinePos)
-				// 		resolve(newAutocompletion.insertText)
-				// 		return
-				// 	}
-
-				// 	// if (!getAutocompletionMatchup({ prefix: this._lastPrefix, autocompletion: newAutocompletion })) {
-				// 	// 	reject('LLM response did not match user\'s text.')
-				// 	// }
-				// },
-				onFinalMessage: ({ fullText }) => {
-
-					// console.log('____res: ', JSON.stringify(newAutocompletion.insertText))
-
-					newAutocompletion.endTime = Date.now()
-					newAutocompletion.status = 'finished'
-					const [text, _] = extractCodeFromRegular({ text: fullText, recentlyAddedTextLen: 0 })
-					newAutocompletion.insertText = processStartAndEndSpaces(text)
-
-					// handle special case for predicting starting on the next line, add a newline character
-					if (newAutocompletion.type === 'multi-line-start-on-next-line') {
-						newAutocompletion.insertText = _ln + newAutocompletion.insertText
-					}
-
-					resolve(newAutocompletion.insertText)
-
-				},
-				onError: ({ message }) => {
-					newAutocompletion.endTime = Date.now()
-					newAutocompletion.status = 'error'
-					reject(message)
-				},
-				onAbort: () => { reject('Aborted autocomplete') },
-			})
-			newAutocompletion.requestId = requestId
+		newAutocompletion.llmPromise = new Promise(async (resolve, reject) => {
+
+			try {
+				const requestId = await this._llmMessageService.sendLLMMessage({
+					messagesType: 'FIMMessage',
+					messages: this._convertToLLMMessageService.prepareFIMMessage({
+						messages: {
+							prefix: llmPrefix,
+							suffix: llmSuffix,
+							stopTokens: stopTokens,
+						}
+					}),
+					modelSelection,
+					modelSelectionOptions,
+					overridesOfModel,
+					logging: { loggingName: 'Autocomplete' },
+					onText: () => { }, // unused in FIMMessage
+					// onText: async ({ fullText, newText }) => {
+
+					// 	newAutocompletion.insertText = fullText
+
+					// 	// count newlines in newText
+					// 	const numNewlines = newText.match(/\n|\r\n/g)?.length || 0
+					// 	newAutocompletion._newlineCount += numNewlines
+
+					// 	// if too many newlines, resolve up to last newline
+					// 	if (newAutocompletion._newlineCount > 10) {
+					// 		const lastNewlinePos = fullText.lastIndexOf('\n')
+					// 		newAutocompletion.insertText = fullText.substring(0, lastNewlinePos)
+					// 		resolve(newAutocompletion.insertText)
+					// 		return
+					// 	}
+
+					// 	// if (!getAutocompletionMatchup({ prefix: this._lastPrefix, autocompletion: newAutocompletion })) {
+					// 	// 	reject('LLM response did not match user\'s text.')
+					// 	// }
+					// },
+					onFinalMessage: ({ fullText }) => {
+
+						// console.log('____res: ', JSON.stringify(newAutocompletion.insertText))
+
+						newAutocompletion.endTime = Date.now()
+						newAutocompletion.status = 'finished'
+						const [text, _] = extractCodeFromRegular({ text: fullText, recentlyAddedTextLen: 0 })
+						newAutocompletion.insertText = processStartAndEndSpaces(text)
+
+						// handle special case for predicting starting on the next line, add a newline character
+						if (newAutocompletion.type === 'multi-line-start-on-next-line') {
+							newAutocompletion.insertText = _ln + newAutocompletion.insertText
+						}
+
+						resolve(newAutocompletion.insertText)
+
+					},
+					onError: ({ message }) => {
+						newAutocompletion.endTime = Date.now()
+						newAutocompletion.status = 'error'
+						reject(message)
+					},
+					onAbort: () => { reject('Aborted autocomplete') },
+				})
+				newAutocompletion.requestId = requestId
+			} catch (error) {
+				reject(error)
+			}
 
 			// if the request hasnt resolved in TIMEOUT_TIME seconds, reject it
 			setTimeout(() => {

src/vs/workbench/contrib/void/browser/chatThreadService.ts

@@ -799,33 +799,38 @@ class ChatThreadService extends Disposable implements IChatThreadService {
 					| { type: 'llmError', error?: { message: string; fullError: Error | null; } }
 					| { type: 'llmAborted' }
 
-				let resMessageIsDonePromise: (res: ResTypes) => void // resolves when user approves this tool use (or if tool doesn't require approval)
+				let resMessageIsDonePromise: (res: ResTypes) => void = () => { } // resolves when user approves this tool use (or if tool doesn't require approval)
 				const messageIsDonePromise = new Promise<ResTypes>((res, rej) => { resMessageIsDonePromise = res })
 
-				const llmCancelToken = this._llmMessageService.sendLLMMessage({
-					messagesType: 'chatMessages',
-					chatMode,
-					messages: messages,
-					modelSelection,
-					modelSelectionOptions,
-					overridesOfModel,
-					logging: { loggingName: `Chat - ${chatMode}`, loggingExtras: { threadId, nMessagesSent, chatMode } },
-					separateSystemMessage: separateSystemMessage,
-					onText: ({ fullText, fullReasoning, toolCall }) => {
-						this._setStreamState(threadId, { isRunning: 'LLM', llmInfo: { displayContentSoFar: fullText, reasoningSoFar: fullReasoning, toolCallSoFar: toolCall ?? null }, interrupt: Promise.resolve(() => { if (llmCancelToken) this._llmMessageService.abort(llmCancelToken) }) })
-					},
-					onFinalMessage: async ({ fullText, fullReasoning, toolCall, anthropicReasoning, }) => {
-						resMessageIsDonePromise({ type: 'llmDone', toolCall, info: { fullText, fullReasoning, anthropicReasoning } }) // resolve with tool calls
-					},
-					onError: async (error) => {
-						resMessageIsDonePromise({ type: 'llmError', error: error })
-					},
-					onAbort: () => {
-						// stop the loop to free up the promise, but don't modify state (already handled by whatever stopped it)
-						resMessageIsDonePromise({ type: 'llmAborted' })
-						this._metricsService.capture('Agent Loop Done (Aborted)', { nMessagesSent, chatMode })
-					},
-				})
+				let llmCancelToken: string | null = null;
+				try {
+					llmCancelToken = await this._llmMessageService.sendLLMMessage({
+						messagesType: 'chatMessages',
+						chatMode,
+						messages: messages,
+						modelSelection,
+						modelSelectionOptions,
+						overridesOfModel,
+						logging: { loggingName: `Chat - ${chatMode}`, loggingExtras: { threadId, nMessagesSent, chatMode } },
+						separateSystemMessage: separateSystemMessage,
+						onText: ({ fullText, fullReasoning, toolCall }) => {
+							this._setStreamState(threadId, { isRunning: 'LLM', llmInfo: { displayContentSoFar: fullText, reasoningSoFar: fullReasoning, toolCallSoFar: toolCall ?? null }, interrupt: Promise.resolve(() => { if (llmCancelToken) this._llmMessageService.abort(llmCancelToken) }) })
+						},
+						onFinalMessage: async ({ fullText, fullReasoning, toolCall, anthropicReasoning, }) => {
+							resMessageIsDonePromise({ type: 'llmDone', toolCall, info: { fullText, fullReasoning, anthropicReasoning } }) // resolve with tool calls
+						},
+						onError: async (error) => {
+							resMessageIsDonePromise({ type: 'llmError', error: error })
+						},
+						onAbort: () => {
+							// stop the loop to free up the promise, but don't modify state (already handled by whatever stopped it)
+							resMessageIsDonePromise({ type: 'llmAborted' })
+							this._metricsService.capture('Agent Loop Done (Aborted)', { nMessagesSent, chatMode })
+						},
+					})
+				} catch (error) {
+					resMessageIsDonePromise({ type: 'llmError', error: { message: `Rate limit check failed: ${error}`, fullError: error } })
+				}
 
 				// mark as streaming
 				if (!llmCancelToken) {

src/vs/workbench/contrib/void/browser/editCodeService.ts

@@ -1503,54 +1503,59 @@ class EditCodeService extends Disposable implements IEditCodeService {
 				let aborted = false
 				let weAreAborting = false
 
+				try {
+					streamRequestIdRef.current = await this._llmMessageService.sendLLMMessage({
+						messagesType: 'chatMessages',
+						logging: { loggingName: `Edit (Writeover) - ${from}` },
+						messages,
+						modelSelection,
+						modelSelectionOptions,
+						overridesOfModel,
+						separateSystemMessage,
+						chatMode: null, // not chat
+						onText: (params) => {
+							const { fullText: fullText_ } = params
+							const newText_ = fullText_.substring(fullTextSoFar.length, Infinity)
+
+							const newText = prevIgnoredSuffix + newText_ // add the previously ignored suffix because it's no longer the suffix!
+							fullTextSoFar += newText // full text, including ```, etc
+
+							const [croppedText, deltaCroppedText, croppedSuffix] = extractText(fullTextSoFar, newText.length)
+							const { endLineInLlmTextSoFar } = this._writeStreamedDiffZoneLLMText(uri, originalCode, croppedText, deltaCroppedText, latestStreamLocationMutable)
+							diffZone._streamState.line = (diffZone.startLine - 1) + endLineInLlmTextSoFar // change coordinate systems from originalCode to full file
+
+							this._refreshStylesAndDiffsInURI(uri)
+
+							prevIgnoredSuffix = croppedSuffix
+						},
+						onFinalMessage: (params) => {
+							const { fullText } = params
+							// console.log('DONE! FULL TEXT\n', extractText(fullText), diffZone.startLine, diffZone.endLine)
+							// at the end, re-write whole thing to make sure no sync errors
+							const [croppedText, _1, _2] = extractText(fullText, 0)
+							this._writeURIText(uri, croppedText,
+								{ startLineNumber: diffZone.startLine, startColumn: 1, endLineNumber: diffZone.endLine, endColumn: Number.MAX_SAFE_INTEGER }, // 1-indexed
+								{ shouldRealignDiffAreas: true }
+							)
 
-				streamRequestIdRef.current = this._llmMessageService.sendLLMMessage({
-					messagesType: 'chatMessages',
-					logging: { loggingName: `Edit (Writeover) - ${from}` },
-					messages,
-					modelSelection,
-					modelSelectionOptions,
-					overridesOfModel,
-					separateSystemMessage,
-					chatMode: null, // not chat
-					onText: (params) => {
-						const { fullText: fullText_ } = params
-						const newText_ = fullText_.substring(fullTextSoFar.length, Infinity)
-
-						const newText = prevIgnoredSuffix + newText_ // add the previously ignored suffix because it's no longer the suffix!
-						fullTextSoFar += newText // full text, including ```, etc
-
-						const [croppedText, deltaCroppedText, croppedSuffix] = extractText(fullTextSoFar, newText.length)
-						const { endLineInLlmTextSoFar } = this._writeStreamedDiffZoneLLMText(uri, originalCode, croppedText, deltaCroppedText, latestStreamLocationMutable)
-						diffZone._streamState.line = (diffZone.startLine - 1) + endLineInLlmTextSoFar // change coordinate systems from originalCode to full file
-
-						this._refreshStylesAndDiffsInURI(uri)
+							onDone()
+							resMessageDonePromise()
+						},
+						onError: (e) => {
+							onError(e)
+						},
+						onAbort: () => {
+							if (weAreAborting) return
+							// stop the loop to free up the promise, but don't modify state (already handled by whatever stopped it)
+							aborted = true
+							resMessageDonePromise()
+						},
+					})
+				} catch (error) {
+					onError({ message: `Rate limit check failed: ${error}`, fullError: error })
+					return
+				}
 
-						prevIgnoredSuffix = croppedSuffix
-					},
-					onFinalMessage: (params) => {
-						const { fullText } = params
-						// console.log('DONE! FULL TEXT\n', extractText(fullText), diffZone.startLine, diffZone.endLine)
-						// at the end, re-write whole thing to make sure no sync errors
-						const [croppedText, _1, _2] = extractText(fullText, 0)
-						this._writeURIText(uri, croppedText,
-							{ startLineNumber: diffZone.startLine, startColumn: 1, endLineNumber: diffZone.endLine, endColumn: Number.MAX_SAFE_INTEGER }, // 1-indexed
-							{ shouldRealignDiffAreas: true }
-						)
-
-						onDone()
-						resMessageDonePromise()
-					},
-					onError: (e) => {
-						onError(e)
-					},
-					onAbort: () => {
-						if (weAreAborting) return
-						// stop the loop to free up the promise, but don't modify state (already handled by whatever stopped it)
-						aborted = true
-						resMessageDonePromise()
-					},
-				})
 				// should never happen, just for safety
 				if (streamRequestIdRef.current === null) { return }
 
@@ -1950,7 +1955,7 @@ class EditCodeService extends Disposable implements IEditCodeService {
 					this._refreshStylesAndDiffsInURI(uri)
 				}
 
-				streamRequestIdRef.current = this._llmMessageService.sendLLMMessage({
+				streamRequestIdRef.current = await this._llmMessageService.sendLLMMessage({
 					messagesType: 'chatMessages',
 					logging: { loggingName: `Edit (Search/Replace) - ${from}` },
 					messages,

src/vs/workbench/contrib/void/browser/rateLimiter.contribution.ts

@@ -0,0 +1,6 @@
+import { registerSingleton, InstantiationType } from '../../../../platform/instantiation/common/extensions.js';
+import { IRateLimiterService } from '../common/rateLimiterService.js';
+import { RateLimiterService } from '../common/rateLimiterService.js';
+
+// Register the rate limiter service
+registerSingleton(IRateLimiterService, RateLimiterService, InstantiationType.Delayed);

src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/Settings.tsx

@@ -1387,6 +1387,21 @@ export const Settings = () => {
 
 							{/* General section */}
 							<div className={`${shouldShowTab('general') ? `` : 'hidden'} flex flex-col gap-12`}>
+								{/* API Rate Limiting */}
+								<div>
+									<h2 className='text-3xl mb-2'>API Rate Limiting</h2>
+									<h4 className='text-void-fg-3 mb-4'>
+										Control the maximum number of API requests per minute to manage usage and costs.
+									</h4>
+
+									<ErrorBoundary>
+										<div className='flex items-center gap-x-2 my-2'>
+											<span className='text-void-fg-3 text-xs'>Max Requests per Minute:</span>
+											<RateLimitSlider />
+										</div>
+									</ErrorBoundary>
+								</div>
+
 								{/* One-Click Switch section */}
 								<div>
 									<ErrorBoundary>
@@ -1531,3 +1546,32 @@ Use Model Context Protocol to provide Agent mode with more tools.
 		</div>
 	);
 }
+
+const RateLimitSlider = () => {
+	const accessor = useAccessor()
+	const voidSettingsService = accessor.get('IVoidSettingsService')
+	const settingsState = useSettingsState()
+
+
+	// console.log('RateLimitSlider value:', settingsState.globalSettings.maxRequestsPerMinute)
+
+	const onChangeValue = useCallback((newVal: number) => {
+		voidSettingsService.setGlobalSetting('maxRequestsPerMinute', newVal)
+	}, [voidSettingsService])
+
+	return (
+		<div className='flex items-center gap-x-2 my-2'>
+			<input
+				type="range"
+				min="1"
+				max="120"
+				value={settingsState.globalSettings.maxRequestsPerMinute}
+				onChange={(e) => onChangeValue(parseInt(e.target.value))}
+				className="w-32 bg-void-bg-2"
+			/>
+			<span className='text-void-fg-3 text-xs pointer-events-none'>
+				{settingsState.globalSettings.maxRequestsPerMinute} requests/min
+			</span>
+		</div>
+	)
+}

src/vs/workbench/contrib/void/browser/void.contribution.ts

@@ -14,6 +14,8 @@ import './sidebarPane.js'
 // register quick edit (Ctrl+K)
 import './quickEditActions.js'
 
+// register rate limiter service
+import './rateLimiter.contribution.js'
 
 // register Autocomplete
 import './autocompleteService.js'