Add live provider response validation

Author: eddmpython

docs/skills/architecture/live-provider-ops.md

@@ -40,7 +40,9 @@ Codaro의 기본 gate는 deterministic scripted provider를 사용한다. 실제
 - network timeout/connection: 네트워크 문제.
 - endpoint/header/SSE 변경: OAuth provider compatibility 점검 필요.
 
-HTTP/stream/UI 경계에는 `code`, `message`, `action`, `provider`, `detail`, `recoverable`을 가진 진단 payload를 넘긴다. 기본 화면은 `message`만 보여주고, raw detail은 trace나 확장 진단에서만 본다. editor는 `connect-provider`, `relogin-provider`, `restart-login`, `check-network`, `check-provider-compatibility` action을 구분해 설정 열기/재로그인/네트워크 점검/호환성 점검으로 안내한다.
+HTTP/stream/UI 경계에는 `code`, `message`, `action`, `provider`, `detail`, `recoverable`을 가진 진단 payload를 넘긴다. 기본 화면은 `message`만 보여주고, raw detail은 trace나 확장 진단에서만 본다. editor는 `connect-provider`, `relogin-provider`, `restart-login`, `configure-api-key`, `configure-base-url`, `check-network`, `check-provider-compatibility` action을 구분해 설정 열기/재로그인/키 입력/base URL 입력/네트워크 점검/호환성 점검으로 안내한다.
+
+Provider 설정의 검증은 `probe=availability`와 `probe=response`를 구분한다. 목록/상태 확인은 availability probe로 충분하지만, 로그인/저장/선택 직후와 사용자가 누르는 “응답 검증”은 response probe를 써서 짧은 실제 응답까지 확인한다. `openai`는 저장된 key 또는 `OPENAI_API_KEY`, `custom`은 base URL과 key, `ollama`는 로컬 서버와 모델 응답, `oauth-chatgpt`는 저장 token과 live endpoint 응답을 기준으로 실패 메시지를 갈라야 한다.
 
 ## Live Smoke Gate
 

editor/src/App.tsx

@@ -100,6 +100,7 @@ function App() {
     setAiProfile,
     setProviderSettingsOpen,
     startOauthProviderLogin,
+    validateAiProvider,
   } = useProviderConnection({ apiOnline, onNotice: setNotice });
 
   const {
@@ -319,6 +320,7 @@ function App() {
         onOpenChange={setProviderSettingsOpen}
         onSaveApiProvider={saveApiProvider}
         onSelectProvider={selectAiProvider}
+        onValidateProvider={validateAiProvider}
       />
     </SidebarProvider>
   );

editor/src/components/assistant/providerSettingsSheet.tsx

@@ -33,6 +33,7 @@ type ProviderSettingsSheetProps = {
   onOauthLogout: (providerId: string) => void;
   onSaveApiProvider: (providerId: string, apiKey: string, baseUrl?: string) => void;
   onSelectProvider: (providerId: string) => void;
+  onValidateProvider: (providerId: string) => void;
 };
 
 type ProviderRuntime = {
@@ -70,6 +71,7 @@ export function ProviderSettingsSheet({
   onOauthLogout,
   onSaveApiProvider,
   onSelectProvider,
+  onValidateProvider,
 }: ProviderSettingsSheetProps) {
   const providers = useMemo(() => providerCatalog(aiProfile), [aiProfile]);
   const runtime = useMemo(() => providerRuntime(aiProfile), [aiProfile]);
@@ -108,6 +110,7 @@ export function ProviderSettingsSheet({
                   onOauthLogout={onOauthLogout}
                   onSaveApiProvider={onSaveApiProvider}
                   onSelectProvider={onSelectProvider}
+                  onValidateProvider={onValidateProvider}
                 />
               ))
             ) : (
@@ -133,6 +136,7 @@ function ProviderCard({
   onOauthLogout,
   onSaveApiProvider,
   onSelectProvider,
+  onValidateProvider,
 }: {
   active: boolean;
   aiConnecting: boolean;
@@ -144,6 +148,7 @@ function ProviderCard({
   onOauthLogout: (providerId: string) => void;
   onSaveApiProvider: (providerId: string, apiKey: string, baseUrl?: string) => void;
   onSelectProvider: (providerId: string) => void;
+  onValidateProvider: (providerId: string) => void;
 }) {
   const providerId = provider.id ?? "";
   const copy = providerCopy(provider);
@@ -209,6 +214,17 @@ function ProviderCard({
                 로그아웃
               </Button>
             ) : null}
+            <Button
+              className="gap-1.5"
+              disabled={!apiOnline || aiConnecting}
+              size="sm"
+              type="button"
+              variant="outline"
+              onClick={() => onValidateProvider(providerId)}
+            >
+              {aiConnecting ? <Loader2 className="size-3.5 animate-spin" /> : <Check className="size-3.5" />}
+              응답 검증
+            </Button>
           </div>
         ) : null}
 
@@ -245,20 +261,44 @@ function ProviderCard({
                 저장된 키 사용
               </Button>
             ) : null}
+            <Button
+              className="gap-1.5"
+              disabled={!apiOnline || aiConnecting}
+              size="sm"
+              type="button"
+              variant="outline"
+              onClick={() => onValidateProvider(providerId)}
+            >
+              {aiConnecting ? <Loader2 className="size-3.5 animate-spin" /> : <Check className="size-3.5" />}
+              응답 검증
+            </Button>
           </div>
         ) : null}
 
         {authKind === "none" ? (
-          <Button disabled={!apiOnline || aiConnecting || active} size="sm" type="button" variant={active ? "secondary" : "outline"} onClick={() => onSelectProvider(providerId)}>
-            {active ? (
-              <>
-                <Check className="size-3.5" />
-                사용 중
-              </>
-            ) : (
-              "사용"
-            )}
-          </Button>
+          <div className="flex flex-wrap gap-2">
+            <Button disabled={!apiOnline || aiConnecting || active} size="sm" type="button" variant={active ? "secondary" : "outline"} onClick={() => onSelectProvider(providerId)}>
+              {active ? (
+                <>
+                  <Check className="size-3.5" />
+                  사용 중
+                </>
+              ) : (
+                "사용"
+              )}
+            </Button>
+            <Button
+              className="gap-1.5"
+              disabled={!apiOnline || aiConnecting}
+              size="sm"
+              type="button"
+              variant="outline"
+              onClick={() => onValidateProvider(providerId)}
+            >
+              {aiConnecting ? <Loader2 className="size-3.5 animate-spin" /> : <Check className="size-3.5" />}
+              응답 검증
+            </Button>
+          </div>
         ) : null}
       </div>
     </section>

editor/src/hooks/useProviderConnection.ts

@@ -6,6 +6,7 @@ import {
   providerAuthFailureNotice,
   saveApiProvider as saveApiProviderAction,
   selectProvider,
+  validateProviderAction,
   type ProviderActionResult,
 } from "@/lib/providerConnection";
 import type { AiProfile, AppNotice } from "@/types";
@@ -99,6 +100,18 @@ export function useProviderConnection({
     }
   }, [aiConnecting, apiOnline, applyProviderActionResult, onNotice]);
 
+  const validateAiProvider = useCallback(async (providerId: string) => {
+    if (!apiOnline || aiConnecting) return;
+    setAiConnecting(true);
+    try {
+      applyProviderActionResult(await validateProviderAction(providerId, providerModel(aiProfile, providerId)));
+    } catch (error) {
+      onNotice(providerAuthFailureNotice(error));
+    } finally {
+      setAiConnecting(false);
+    }
+  }, [aiConnecting, aiProfile, apiOnline, applyProviderActionResult, onNotice]);
+
   return {
     aiConnecting,
     aiProfile,
@@ -110,5 +123,13 @@ export function useProviderConnection({
     setAiProfile,
     setProviderSettingsOpen,
     startOauthProviderLogin,
+    validateAiProvider,
   };
 }
+
+function providerModel(profile: AiProfile | null, providerId: string): string | null {
+  const providers = profile?.providers;
+  if (!providers || typeof providers !== "object" || Array.isArray(providers)) return null;
+  const runtime = (providers as Record<string, { model?: unknown }>)[providerId];
+  return typeof runtime?.model === "string" ? runtime.model : null;
+}

editor/src/lib/api.ts

@@ -237,9 +237,10 @@ export const codaroApi = {
   aiProviders: () => requestJson<AiProviderCatalogPayload>("/api/ai/providers"),
   aiTools: () => requestJson<AiToolCatalogPayload>("/api/ai/tools"),
   aiProfile: () => requestJson<AiProfile>("/api/ai/profile"),
-  validateAiProvider: (provider: string, model?: string | null) => {
+  validateAiProvider: (provider: string, model?: string | null, probe = "availability") => {
     const params = new URLSearchParams({ provider });
     if (model) params.set("model", model);
+    if (probe) params.set("probe", probe);
     return postJson<ProviderValidationPayload>(`/api/ai/provider/validate?${params.toString()}`, {});
   },
   updateAiProfile: (payload: {

editor/src/lib/providerConnection.ts

@@ -96,6 +96,26 @@ export async function saveApiProvider(providerId: string, apiKey: string, baseUr
   });
 }
 
+export async function validateProviderAction(providerId: string, model?: string | null): Promise<ProviderActionResult> {
+  const validation = await validateProvider(providerId, model, "response");
+  if (validation.valid) {
+    return {
+      notice: {
+        tone: "success",
+        title: "Provider 응답 확인됨",
+        detail: validation.model ? `${providerId} · ${validation.model}` : providerId,
+      },
+    };
+  }
+  return {
+    notice: {
+      tone: "warning",
+      title: "Provider 확인 필요",
+      detail: validation.diagnostic?.message ?? validation.error ?? "Provider 응답 검증에 실패했습니다.",
+    },
+  };
+}
+
 export function providerAuthFailureNotice(error: unknown): AppNotice {
   const diagnostic = providerDiagnosticFromError(error);
   return {
@@ -124,7 +144,10 @@ export function providerAssistantFailure(error: unknown): ProviderAssistantFailu
 
 export function isProviderAuthError(error: unknown) {
   const diagnostic = providerDiagnosticFromError(error);
-  if (diagnostic?.action && ["connect-provider", "relogin-provider", "restart-login"].includes(diagnostic.action)) {
+  if (
+    diagnostic?.action
+    && ["connect-provider", "relogin-provider", "restart-login", "configure-api-key", "configure-base-url"].includes(diagnostic.action)
+  ) {
     return true;
   }
   const normalized = errorMessage(error).toLowerCase();
@@ -158,7 +181,7 @@ async function withProviderValidation(
       notice: { tone: "warning", title: "Provider 확인 필요", detail: "선택된 provider가 없습니다." },
     };
   }
-  const validation = await validateProvider(provider, profile.activeModel);
+  const validation = await validateProvider(provider, profile.activeModel, "response");
   if (validation.valid) {
     return {
       ...base,
@@ -169,8 +192,9 @@ async function withProviderValidation(
       },
     };
   }
+  const failureBase = base.closeSettings ? { ...base, closeSettings: false, openSettings: true } : base;
   return {
-    ...base,
+    ...failureBase,
     notice: {
       tone: "warning",
       title: "Provider 확인 필요",
@@ -179,9 +203,9 @@ async function withProviderValidation(
   };
 }
 
-async function validateProvider(provider: string, model?: unknown): Promise<ProviderValidationPayload> {
+async function validateProvider(provider: string, model?: unknown, probe = "availability"): Promise<ProviderValidationPayload> {
   try {
-    return await codaroApi.validateAiProvider(provider, typeof model === "string" ? model : undefined);
+    return await codaroApi.validateAiProvider(provider, typeof model === "string" ? model : undefined, probe);
   } catch (error) {
     const diagnostic = providerDiagnosticFromError(error);
     return {

editor/src/types.ts

@@ -249,6 +249,7 @@ export type ProviderValidationPayload = {
   model?: string | null;
   error?: string | null;
   diagnostic?: ProviderDiagnostic | null;
+  probe?: string;
 };
 
 export type AiToolCatalogPayload = {