Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions frontend/src/components/icons/ProviderLogos.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,32 @@ export function CloudflareLogo({ size = defaultSize, className, ...props }: Logo
)
}

export function SixtydbLogo({ size = defaultSize, className, ...props }: LogoProps) {
return (
<svg
viewBox="0 0 24 24"
width={size}
height={size}
className={className}
xmlns="http://www.w3.org/2000/svg"
{...props}
>
<circle cx="12" cy="12" r="10" fill="none" stroke="currentColor" strokeWidth="2" />
<text
x="12"
y="15.5"
textAnchor="middle"
fontSize="8"
fontWeight="700"
fontFamily="ui-sans-serif, system-ui, -apple-system"
fill="currentColor"
>
60db
</text>
</svg>
)
}

export function WhisperCppLogo({ size = defaultSize, className }: LogoProps) {
return (
<img
Expand All @@ -257,6 +283,7 @@ const PROVIDER_LOGO_MAP: Record<string, (props: AnyLogoProps) => JSX.Element> =
cloudflare: CloudflareLogo,
local_openai: OpenAILogo,
local_whisper_cpp: WhisperCppLogo,
sixtydb: SixtydbLogo,
}

export function getProviderLogo(
Expand Down
207 changes: 207 additions & 0 deletions frontend/src/providers/implementations/SixtydbProvider.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,207 @@
/**
* 60db STT Realtime ASR Provider.
*
* Connects through the local proxy at ws://localhost:23456/ws/sixtydb
* (parity with ElevenLabsProvider). The proxy hides the api key from
* the browser DevTools and normalizes 60db's two-phase finals into the
* standard partial/final contract.
*
* Docs: https://docs.60db.ai/api-reference/websocket/stt
*/

import { BaseASRProvider } from '../base'
import type {
ASRProviderInfo,
ProviderConfig,
ASRVendor,
} from '../../types/asr'
import {
SIXTYDB_SUPPORTED_LANGUAGES,
} from '../../types/asr/vendors/sixtydb'

const PROXY_WS_URL = 'ws://localhost:23456/ws/sixtydb'

export class SixtydbProvider extends BaseASRProvider {
readonly id: ASRVendor = 'sixtydb' as ASRVendor

readonly info: ASRProviderInfo = {
id: 'sixtydb' as ASRVendor,
name: '60db',
description:
'60db real-time speech-to-text. ~40 languages including Indic + English code-switching, sentence-based continuous mode, optional speaker diarization.',
type: 'cloud',
supportsStreaming: true,
capabilities: {
audioInputMode: 'pcm16',
audioProfile: {
payloadFormat: 'pcm16',
sampleRateHz: 16000,
channels: 1,
preferredChunkMs: 100,
},
transport: {
type: 'realtime',
captureRestartStrategy: 'reuse-session',
},
prompting: {
supportsLanguageHints: true,
},
workloads: {
liveCapture: {
availability: 'implemented',
executionMode: 'realtime-stream',
inputSources: ['system-audio'],
acceptedFileKinds: ['audio'],
},
fileTranscription: {
availability: 'compatible',
executionMode: 'single-request',
inputSources: ['file'],
acceptedFileKinds: ['audio', 'video'],
},
},
supportsConfigTest: true,
},
requiredConfigKeys: ['apiKey'],
supportedLanguages: [...SIXTYDB_SUPPORTED_LANGUAGES],
website: 'https://60db.ai',
docsUrl: 'https://docs.60db.ai/api-reference/websocket/stt',
configFields: [
{
key: 'apiKey',
label: 'API Key',
type: 'password',
required: true,
placeholder: 'sk_live_...',
description: 'Get your 60db API key from docs.60db.ai',
},
{
key: 'languageHints',
label: 'Language Hints',
type: 'text',
required: false,
placeholder: 'en, hi',
description: 'Comma-separated ISO 639-1 codes (max 5). Omit for auto-detect.',
},
],
}

private ws: WebSocket | null = null
private wsReady = false

async connect(config: ProviderConfig): Promise<void> {
const apiKey = config.apiKey as string

if (!apiKey) {
this.emitError(this.createError('MISSING_API_KEY', '60db API key is required'))
return
}
Comment on lines +92 to +98

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major | ⚡ Quick win

Reject promise when API key is missing.

When apiKey is missing, emitError is called but the Promise returned by connect() is never resolved or rejected. This leaves the caller hanging indefinitely.

Proposed fix
  async connect(config: ProviderConfig): Promise<void> {
    const apiKey = config.apiKey as string

    if (!apiKey) {
-     this.emitError(this.createError('MISSING_API_KEY', '60db API key is required'))
-     return
+     const error = this.createError('MISSING_API_KEY', '60db API key is required')
+     this.emitError(error)
+     throw new Error(error.message)
    }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
async connect(config: ProviderConfig): Promise<void> {
const apiKey = config.apiKey as string
if (!apiKey) {
this.emitError(this.createError('MISSING_API_KEY', '60db API key is required'))
return
}
async connect(config: ProviderConfig): Promise<void> {
const apiKey = config.apiKey as string
if (!apiKey) {
const error = this.createError('MISSING_API_KEY', '60db API key is required')
this.emitError(error)
throw new Error(error.message)
}
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@frontend/src/providers/implementations/SixtydbProvider.ts` around lines 92 -
98, The connect method in SixtydbProvider (connect(config: ProviderConfig))
calls this.emitError(this.createError(...)) when apiKey is missing but never
rejects or throws, leaving the caller hanging; after emitting the error you
should reject the Promise by throwing the created error (or returning
Promise.reject) so callers receive the failure. Update the connect
implementation to create the error via this.createError('MISSING_API_KEY', '60db
API key is required'), call this.emitError(error), then throw that error (or
return Promise.reject(error)) so the Promise is settled.


this._config = config
this.setState('connecting')

return new Promise((resolve, reject) => {
try {
const params = new URLSearchParams({
apiKey,
language: (config.language as string) || '',
})

const proxyUrl = `${PROXY_WS_URL}?${params.toString()}`
console.log('[SixtydbProvider] connecting to proxy...')

this.ws = new WebSocket(proxyUrl)

this.ws.onopen = () => {
console.log('[SixtydbProvider] proxy connected, awaiting 60db session_started...')
}

this.ws.onmessage = (event) => {
try {
const msg = JSON.parse(event.data)

switch (msg.type) {
case 'ready':
console.log('[SixtydbProvider] 60db session ready')
this.wsReady = true
this.setState('connected')
resolve()
break

case 'partial':
if (msg.text) {
console.log('[SixtydbProvider] partial:', msg.text.substring(0, 50))
this.emitPartial(msg.text)
}
break

case 'final':
console.log('[SixtydbProvider] final:', msg.text)
this.emitFinal(msg.text || '')
this.emitFinished()
break
Comment on lines +138 to +142

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major | ⚡ Quick win

Remove emitFinished() call on each final transcript.

Calling emitFinished() after every final message signals end of session repeatedly. In continuous transcription, multiple finals occur during a single session. emitFinished() should only be called when the session truly ends (e.g., on intentional disconnect or session_stopped).

Based on the downstream behavior in providerSession.ts, this could trigger unintended session-end callbacks multiple times.

Proposed fix
              case 'final':
                console.log('[SixtydbProvider] final:', msg.text)
                this.emitFinal(msg.text || '')
-               this.emitFinished()
                break
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
case 'final':
console.log('[SixtydbProvider] final:', msg.text)
this.emitFinal(msg.text || '')
this.emitFinished()
break
case 'final':
console.log('[SixtydbProvider] final:', msg.text)
this.emitFinal(msg.text || '')
break
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@frontend/src/providers/implementations/SixtydbProvider.ts` around lines 138 -
142, In the 'final' switch branch inside SixtydbProvider (the case handling
final messages) remove the call to this.emitFinished() so that
emitFinal(msg.text || '') is emitted without signaling session end; instead
ensure emitFinished() is only invoked from real session-termination paths (e.g.,
the session_stopped/disconnect handler or wherever session lifecycle is
explicitly closed) so multiple 'final' transcripts during continuous
transcription don't trigger end-of-session callbacks.


case 'error':
console.error('[SixtydbProvider] server error:', msg.message)
this.emitError(this.createError('SERVER_ERROR', msg.message || 'Server error'))
break
}
} catch (e) {
console.error('[SixtydbProvider] failed to parse message:', e)
}
}

this.ws.onerror = (error) => {
console.error('[SixtydbProvider] WebSocket error:', error)
this.emitError(this.createError('WEBSOCKET_ERROR', 'WebSocket connection error — make sure the local proxy is running'))
reject(new Error('WebSocket connection error'))
}

this.ws.onclose = (event) => {
console.log('[SixtydbProvider] WebSocket closed:', event.code, event.reason)
this.wsReady = false
this.setState('idle')
}
} catch (error) {
console.error('[SixtydbProvider] connect failed:', error)
this.emitError(this.createError('CONNECTION_ERROR', 'Connection failed'))
reject(error)
}
})
}

async disconnect(): Promise<void> {
console.log('[SixtydbProvider] disconnecting...')

if (this.ws && this.wsReady) {
this.ws.send(JSON.stringify({ type: 'audio_end' }))
}

await new Promise(resolve => setTimeout(resolve, 500))

if (this.ws) {
this.ws.close(1000, 'disconnect')
this.ws = null
}

this.wsReady = false
this.setState('idle')
}

sendAudio(data: Blob | ArrayBuffer): void {
if (!this.ws || !this.wsReady) {
console.warn('[SixtydbProvider] WebSocket not ready, dropping audio')
return
}

this.setState('recording')

if (data instanceof Blob) {
data.arrayBuffer().then(buffer => {
this.ws?.send(buffer)
})
} else {
this.ws.send(data)
}
}
}
1 change: 1 addition & 0 deletions frontend/src/providers/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,4 @@ export { AssemblyAIProvider } from './implementations/AssemblyAIProvider'
export { ElevenLabsProvider } from './implementations/ElevenLabsProvider'
export { LocalOpenAIProvider } from './implementations/LocalOpenAIProvider'
export { WhisperCppRuntimeProvider } from './implementations/WhisperCppRuntimeProvider'
export { SixtydbProvider } from './implementations/SixtydbProvider'
6 changes: 6 additions & 0 deletions frontend/src/providers/registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import { GladiaProvider } from './implementations/GladiaProvider'
import { CloudflareProvider } from './implementations/CloudflareProvider'
import { LocalOpenAIProvider } from './implementations/LocalOpenAIProvider'
import { WhisperCppRuntimeProvider } from './implementations/WhisperCppRuntimeProvider'
import { SixtydbProvider } from './implementations/SixtydbProvider'

// Provider 注册表
class ProviderRegistry {
Expand Down Expand Up @@ -131,6 +132,11 @@ function registerDefaultProviders(): void {
info: new WhisperCppRuntimeProvider().info,
create: () => new WhisperCppRuntimeProvider(),
})

providerRegistry.register({
info: new SixtydbProvider().info,
create: () => new SixtydbProvider(),
})
}

// 初始化注册
Expand Down
1 change: 1 addition & 0 deletions frontend/src/types/asr/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ export enum ASRVendor {
Cloudflare = 'cloudflare',
LocalOpenAI = 'local_openai',
LocalWhisperCpp = 'local_whisper_cpp',
Sixtydb = 'sixtydb',
}

export type ProviderType = 'cloud' | 'local'
Expand Down
60 changes: 60 additions & 0 deletions frontend/src/types/asr/vendors/sixtydb.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/**
* 60db STT Realtime ASR vendor-specific types.
*
* Docs: https://docs.60db.ai/api-reference/websocket/stt
*/

export const SIXTYDB_DEFAULT_MODEL = '60db-stt-v01'

// 60db supports these languages plus auto-detect. The "multi" entry is a
// placeholder for the auto-detect/multi-language session feature (up to 5
// languages per session) — when languages is omitted in the start message,
// 60db auto-detects.
export const SIXTYDB_SUPPORTED_LANGUAGES = [
'en', 'es', 'fr', 'de', 'it', 'pt', 'nl', 'pl', 'ru', 'uk',
'cs', 'sv', 'ar',
'hi', 'bn', 'mr', 'pa', 'gu', 'ta', 'te', 'kn', 'ml', 'or',
'as', 'ne', 'sa',
'multi',
] as const

export type SixtydbSupportedLanguage = typeof SIXTYDB_SUPPORTED_LANGUAGES[number]

// Server-emitted transcription event (minimal shape that the proxy parses).
// The proxy normalizes 60db's two-phase finals into a single 'partial' /
// 'final' contract before forwarding to the client, so the provider doesn't
// need to see this directly — but the type is kept here for documentation.
export interface SixtydbTranscriptionEvent {
type: 'transcription'
text: string
confidence?: number
language?: string
is_final?: boolean
speech_final?: boolean
sentence_id?: number
words?: Array<{
word: string
start: number
end: number
confidence?: number
}>
speakers?: Array<{ speaker: string; start: number; end: number }>
}

export interface SixtydbSessionStartedEvent {
type: 'session_started'
session_id: string
language?: string
model?: string
}

export interface SixtydbErrorEvent {
type: 'error'
error: string
error_code?: string
}

export type SixtydbServerEvent =
| SixtydbTranscriptionEvent
| SixtydbSessionStartedEvent
| SixtydbErrorEvent
8 changes: 8 additions & 0 deletions server/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { createDeepgramProxyServer } from './deepgramProxy.js'
import { createAssemblyAIProxyServer } from './assemblyaiProxy.js'
import { createElevenLabsProxyServer } from './elevenlabsProxy.js'
import { createGladiaProxyServer } from './gladiaProxy.js'
import { createSixtydbProxyServer } from './sixtydbProxy.js'

const __filename = fileURLToPath(import.meta.url)
const __dirname = path.dirname(__filename)
Expand Down Expand Up @@ -37,6 +38,9 @@ createElevenLabsProxyServer(elevenlabsWss)
const gladiaWss = new WebSocketServer({ noServer: true })
createGladiaProxyServer(gladiaWss)

const sixtydbWss = new WebSocketServer({ noServer: true })
createSixtydbProxyServer(sixtydbWss)

server.on('upgrade', (request, socket, head) => {
const { pathname } = new URL(request.url || '', `http://${request.headers.host}`)

Expand Down Expand Up @@ -64,6 +68,10 @@ server.on('upgrade', (request, socket, head) => {
gladiaWss.handleUpgrade(request, socket, head, (ws) => {
gladiaWss.emit('connection', ws, request)
})
} else if (pathname === '/ws/sixtydb') {
sixtydbWss.handleUpgrade(request, socket, head, (ws) => {
sixtydbWss.emit('connection', ws, request)
})
} else {
socket.destroy()
}
Expand Down
6 changes: 6 additions & 0 deletions server/src/sixtydbProxy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import type { WebSocketServer } from 'ws'
import { attachSixtydbProxyServer } from '../../shared/sixtydbProxyCore.js'

export function createSixtydbProxyServer(wss: WebSocketServer): void {
attachSixtydbProxyServer(wss)
}
Loading