Merge branch 'wecode-ai:main' into main

Author: Micro66

.gitignore

@@ -94,6 +94,7 @@ coverage/
 .claude/settings.local.json
 .history
 .wegentrc
+.worktrees/
 
 # PyInstaller
 *.spec

backend/.env.example

@@ -134,6 +134,12 @@ WEBSCRAPER_PROXY=
 #   - direct: Always use proxy for all requests (proxy must be configured)
 #   - fallback: Try direct connection first, use proxy only if direct fails
 WEBSCRAPER_PROXY_MODE=fallback
+# Web scraper site-specific configuration
+# Configuration for specific sites that require special handling
+# due to anti-bot detection, dynamic content loading, or navigation patterns
+# Format: JSON object with site URL patterns as keys
+# WEB_SCRAPER_SITE_CONFIG={"Example.com":{"wait_until":"networkidle","page_timeout":30000,"delay_before_return_html":3.0}}
+WEB_SCRAPER_SITE_CONFIG={}
 
 # Web search configuration
 # Enable/disable web search feature (default: False)

backend/app/api/api.py

@@ -20,6 +20,7 @@
     quota,
     repository,
     share,
+    skill_identity,
     skill_market,
     subtasks,
     tables,
@@ -177,6 +178,7 @@
 api_router.include_router(
     skill_market.router, prefix="/skill-market", tags=["skill-market"]
 )
+api_router.include_router(skill_identity.router, tags=["skill-identity"])
 api_router.include_router(k_router)
 
 # Internal API endpoints (for service-to-service communication)

backend/app/api/endpoints/internal/rag.py

@@ -737,7 +737,11 @@ async def read_document(
         Document content with pagination info
     """
     try:
-        from app.services.rag.document_read_service import document_read_service
+        from app.services.rag.document_read_service import (
+            DOCUMENT_READ_ERROR_ACCESS_DENIED,
+            DOCUMENT_READ_ERROR_NOT_FOUND,
+            document_read_service,
+        )
 
         results = document_read_service.read_documents(
             db=db,
@@ -748,12 +752,9 @@ async def read_document(
         )
         result = results[0] if results else None
 
-        if not result or result.get("error") == "Document not found":
+        if not result or result.get("error_code") == DOCUMENT_READ_ERROR_NOT_FOUND:
             raise HTTPException(status_code=404, detail="Document not found")
-        if (
-            result.get("error")
-            == "Access denied: document not in allowed knowledge bases"
-        ):
+        if result.get("error_code") == DOCUMENT_READ_ERROR_ACCESS_DENIED:
             raise HTTPException(
                 status_code=403,
                 detail="Access denied: document not in allowed knowledge bases",

backend/app/api/endpoints/knowledge.py

@@ -47,7 +47,10 @@
     KnowledgeService,
     knowledge_base_qa_service,
 )
-from app.services.knowledge.orchestrator import knowledge_orchestrator
+from app.services.knowledge.orchestrator import (
+    MAX_DOCUMENT_READ_LIMIT,
+    knowledge_orchestrator,
+)
 from shared.telemetry.decorators import (
     add_span_event,
     capture_trace_context,
@@ -61,6 +64,70 @@
 router = APIRouter()
 
 
+def _serialize_standalone_document_detail(
+    detail: DocumentDetailResponse | dict,
+    *,
+    include_content: bool,
+    include_summary: bool,
+) -> dict:
+    """Preserve the standalone endpoint's pre-refactor response shape."""
+    payload = detail.model_dump() if hasattr(detail, "model_dump") else dict(detail)
+    response = {
+        "document_id": payload["document_id"],
+    }
+    if include_content:
+        response["content"] = payload.get("content")
+        response["content_length"] = payload.get("content_length")
+        response["truncated"] = payload.get("truncated")
+    if include_summary:
+        response["summary"] = payload.get("summary")
+    return response
+
+
+def _raise_document_detail_http_error(error: ValueError) -> None:
+    """Map orchestrator detail errors to stable HTTP responses."""
+    error_msg = str(error)
+    if "not found" in error_msg.lower():
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=error_msg,
+        )
+    if "access denied" in error_msg.lower():
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail=error_msg,
+        )
+    raise HTTPException(
+        status_code=status.HTTP_400_BAD_REQUEST,
+        detail=error_msg,
+    )
+
+
+def _validate_knowledge_base_access_or_raise(
+    db: Session,
+    *,
+    knowledge_base_id: int,
+    user: User,
+):
+    """Restore the KB-scoped endpoint's KB-level error semantics."""
+    knowledge_base, has_access = KnowledgeService.get_knowledge_base(
+        db=db,
+        knowledge_base_id=knowledge_base_id,
+        user_id=user.id,
+    )
+    if not knowledge_base:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Knowledge base not found",
+        )
+    if not has_access:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Access denied to this knowledge base",
+        )
+    return knowledge_base
+
+
 # ============== Knowledge Base Endpoints ==============
 
 
@@ -695,8 +762,8 @@ async def update_document_content(
 
 
 @document_router.get("/{document_id}/detail")
-@trace_sync("get_document_detail_standalone", "knowledge.api")
-def get_document_detail_standalone(
+@trace_async("get_document_detail_standalone", "knowledge.api")
+async def get_document_detail_standalone(
     document_id: int,
     include_content: bool = Query(True, description="Include document content"),
     include_summary: bool = Query(True, description="Include document summary"),
@@ -709,76 +776,23 @@ def get_document_detail_standalone(
     This is a convenience endpoint for getting document content when the kb_id
     is not readily available (e.g., in citation tooltips).
     """
-    from app.models.knowledge import KnowledgeDocument
-
-    # Get document
-    document = (
-        db.query(KnowledgeDocument).filter(KnowledgeDocument.id == document_id).first()
-    )
-
-    if not document:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail="Document not found",
-        )
-
-    # Check access permission via knowledge base
-    kb, has_access = KnowledgeService.get_knowledge_base(
-        db=db,
-        knowledge_base_id=document.kind_id,
-        user_id=current_user.id,
-    )
-    if not kb:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail="Knowledge base not found",
+    try:
+        detail = await knowledge_orchestrator.get_document_detail(
+            db=db,
+            user=current_user,
+            document_id=document_id,
+            include_content=include_content,
+            include_summary=include_summary,
+            offset=0,
+            limit=MAX_DOCUMENT_READ_LIMIT,
         )
-    if not has_access:
-        raise HTTPException(
-            status_code=status.HTTP_403_FORBIDDEN,
-            detail="Access denied to this document",
+        return _serialize_standalone_document_detail(
+            detail,
+            include_content=include_content,
+            include_summary=include_summary,
         )
-
-    # Get content if requested
-    content = None
-    content_length = None
-    truncated = False
-
-    if include_content and document.attachment_id:
-        try:
-            from app.services.context import context_service
-
-            attachment = context_service.get_context_by_id(
-                db=db,
-                context_id=document.attachment_id,
-            )
-            if attachment and attachment.extracted_text:
-                full_content = attachment.extracted_text
-                content_length = len(full_content)
-                # Truncate if too large
-                max_length = 100000
-                if content_length > max_length:
-                    content = full_content[:max_length]
-                    truncated = True
-                else:
-                    content = full_content
-        except Exception as e:
-            logger.warning(f"Failed to get document content: {e}")
-
-    # Build response
-    response = {
-        "document_id": document_id,
-    }
-
-    if include_content:
-        response["content"] = content
-        response["content_length"] = content_length
-        response["truncated"] = truncated
-
-    if include_summary:
-        response["summary"] = document.summary
-
-    return response
+    except ValueError as error:
+        _raise_document_detail_http_error(error)
 
 
 @document_router.post("/batch/delete", response_model=BatchOperationResult)
@@ -1104,23 +1118,13 @@ async def get_document_detail(
     - summary: Document summary object (if include_summary=true)
     """
     from app.models.knowledge import KnowledgeDocument
-    from app.models.subtask_context import SubtaskContext
-    from app.services.knowledge import get_summary_service
 
-    # Validate KB access permission first
-    kb, has_access = KnowledgeService.get_knowledge_base(db, kb_id, current_user.id)
-    if not kb:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail="Knowledge base not found",
-        )
-    if not has_access:
-        raise HTTPException(
-            status_code=status.HTTP_403_FORBIDDEN,
-            detail="Access denied to this knowledge base",
-        )
+    _validate_knowledge_base_access_or_raise(
+        db,
+        knowledge_base_id=kb_id,
+        user=current_user,
+    )
 
-    # Validate document belongs to the specified knowledge base
     document = (
         db.query(KnowledgeDocument)
         .filter(
@@ -1135,53 +1139,18 @@ async def get_document_detail(
             detail="Document not found in the specified knowledge base",
         )
 
-    # Initialize response data
-    content = None
-    content_length = None
-    truncated = None
-    summary = None
-
-    # Get document content if requested
-    if include_content:
-        content = ""
-        truncated = False
-        max_length = 100000  # 100k characters limit for frontend display
-
-        if document.attachment_id:
-            context = (
-                db.query(SubtaskContext)
-                .filter(SubtaskContext.id == document.attachment_id)
-                .first()
-            )
-
-            if context and context.extracted_text:
-                content = context.extracted_text
-                # Truncate if too long
-                if len(content) > max_length:
-                    content = content[:max_length]
-                    truncated = True
-
-        content_length = len(content)
-
-    # Get document summary if requested
-    if include_summary:
-        summary_service = get_summary_service(db)
-        summary_obj = await summary_service.get_document_summary(doc_id)
-        # Convert DocumentSummary object to dict for response
-        if summary_obj:
-            summary = (
-                summary_obj.model_dump()
-                if hasattr(summary_obj, "model_dump")
-                else summary_obj
-            )
-
-    return DocumentDetailResponse(
-        document_id=doc_id,
-        content=content,
-        content_length=content_length,
-        truncated=truncated,
-        summary=summary,
-    )
+    try:
+        return await knowledge_orchestrator.get_document_detail(
+            db=db,
+            user=current_user,
+            document_id=doc_id,
+            include_content=include_content,
+            include_summary=include_summary,
+            offset=0,
+            limit=MAX_DOCUMENT_READ_LIMIT,
+        )
+    except ValueError as error:
+        _raise_document_detail_http_error(error)
 
 
 @summary_router.get("/{kb_id}/documents/{doc_id}/summary")

backend/app/api/endpoints/skill_identity.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: 2025 Weibo, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Public API for skill identity verification."""
+
+from fastapi import APIRouter, Depends
+from pydantic import BaseModel, Field
+
+from app.core import security
+from app.services.auth import verify_skill_identity_token
+
+router = APIRouter(prefix="/skill-identity", tags=["skill-identity"])
+
+
+class SkillIdentityVerifyRequest(BaseModel):
+    """Request schema for skill identity verification."""
+
+    token: str = Field(description="Skill identity JWT")
+    user_name: str = Field(description="Claimed user name")
+
+
+@router.post("/verify")
+def verify_skill_identity(
+    request: SkillIdentityVerifyRequest,
+    _: security.AuthContext = Depends(security.get_auth_context),
+) -> dict:
+    """Verify that a skill identity token belongs to the claimed user."""
+    if not request.user_name:
+        return {"matched": False, "reason": "missing_user_name"}
+
+    token_info = verify_skill_identity_token(request.token)
+    if token_info is None:
+        return {"matched": False, "reason": "invalid_token"}
+
+    if token_info.user_name != request.user_name:
+        return {"matched": False, "reason": "user_mismatch"}
+
+    return {"matched": True}