Merge pull request #196 from Ontos-AI/feat/wangbinqi/knowhere-v2-page-memory-api

Add v2 page-memory ingestion API

Author: suguanYang

apps/api/.env.example

@@ -98,10 +98,6 @@ MAX_PDF_PAGE_LIMIT=200
 OVERSIZED_PDF_SHARD_ENABLED=true
 OVERSIZED_PDF_SOFT_LIMIT=1500
 PDF_PROFILE_TOC_ENABLED=false
-RETRIEVAL_PAGE_MEMORY_ENABLED=false
-PAGE_MEMORY_ASSET_EXTRACTION_ENABLED=false
-PAGE_MEMORY_ASSET_SUMMARY_ENABLED=false
-PAGE_MEMORY_MAX_PAGES=1500
 MINERU_SHARD_CONCURRENCY=3
 PARSE_AGENT_PLAN_BUDGET=50000
 PARSE_AGENT_VISUAL_BUDGET=80000

apps/api/alembic/versions/fbe1c2d3e4f5_add_v2_job_polling_system_limit.py

@@ -0,0 +1,54 @@
+"""Add v2 job polling system limit.
+
+Revision ID: fbe1c2d3e4f5
+Revises: fae1b2c3d4e5
+Create Date: 2026-07-01 13:45:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "fbe1c2d3e4f5"
+down_revision: Union[str, Sequence[str], None] = "fae1b2c3d4e5"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Cap v2 job-result polling like v1 job-result polling."""
+    op.execute(
+        """
+        INSERT INTO system_limits (
+            method,
+            api_pattern,
+            priority,
+            rpm,
+            period,
+            description
+        )
+        VALUES (
+            'GET',
+            '/v2/jobs/*',
+            200,
+            200,
+            'minute',
+            'Job queries - prevent polling'
+        )
+        ON CONFLICT (method, api_pattern) DO NOTHING
+        """
+    )
+
+
+def downgrade() -> None:
+    """Remove the v2 job-result polling cap."""
+    op.execute(
+        """
+        DELETE FROM system_limits
+        WHERE method = 'GET'
+          AND api_pattern = '/v2/jobs/*'
+        """
+    )

apps/api/app/api/api_router.py

@@ -3,11 +3,13 @@
 """
 
 from app.api.v1.api_v1 import api_router as v1_router
+from app.api.v2.api_v2 import api_router as v2_router
 from fastapi import APIRouter
 
 api_router = APIRouter()
 
 # Register the versioned API routes.
 api_router.include_router(v1_router, prefix="/v1")
+api_router.include_router(v2_router, prefix="/v2")
 
 __all__ = ["api_router"]

apps/api/app/api/v1/routes/jobs.py

@@ -46,7 +46,7 @@ async def create_job(  # pyright: ignore[reportGeneralTypeIssues]
     """
     Create a parsing job.
     """
-    return await _document_ingestion_service.create_job(
+    return await _document_ingestion_service.create_v1_job(
         db,
         payload=payload,
         current_user=current_user,

apps/api/app/api/v2/__init__.py

@@ -0,0 +1 @@
+"""API v2 package."""

apps/api/app/api/v2/api_v2.py

@@ -0,0 +1,12 @@
+"""API v2 route registry."""
+
+from app.api.v2.routes import documents, jobs, retrieval
+from fastapi import APIRouter
+
+api_router = APIRouter()
+
+api_router.include_router(jobs.router, prefix="/jobs", tags=["Jobs"])
+api_router.include_router(retrieval.router, prefix="/retrieval", tags=["Retrieval"])
+api_router.include_router(documents.router, prefix="/documents", tags=["Documents"])
+
+__all__ = ["api_router"]

apps/api/app/api/v2/routes/__init__.py

@@ -0,0 +1 @@
+"""API v2 route package."""

apps/api/app/api/v2/routes/documents.py

@@ -0,0 +1,5 @@
+"""Documents API v2 routes."""
+
+from app.api.v1.routes.documents import router
+
+__all__ = ["router"]

apps/api/app/api/v2/routes/jobs.py

@@ -0,0 +1,114 @@
+"""Jobs API v2 routes."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Optional
+
+from app.api.dependencies.auth import require_write_permission
+from app.api.dependencies.current_user import with_current_user
+from app.api.dependencies.job_admission import require_billing_limits
+from app.services.document_ingestion import DocumentIngestionService
+from app.services.jobs import (
+    get_job_result_for_user,
+    list_jobs_for_user,
+)
+from app.services.rate_limit.data_structures import CurrentUser
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from shared.core.database import get_db
+from shared.models.schemas.job import (
+    ConfirmUploadRequest,
+    JobCreateV2,
+    JobList,
+    JobResponse,
+    JobResultResponse,
+)
+
+router = APIRouter(tags=["Jobs"])
+_document_ingestion_service = DocumentIngestionService()
+
+
+@router.post("", response_model=JobResponse, summary="Create a v2 parsing job")
+@router.post("/", include_in_schema=False)
+async def create_job(  # pyright: ignore[reportGeneralTypeIssues]
+    payload: JobCreateV2,
+    current_user: CurrentUser = Depends(require_billing_limits),
+    _write_permission: None = Depends(require_write_permission),
+    db: AsyncSession = Depends(get_db),
+):
+    """Create a v2 parsing job."""
+    return await _document_ingestion_service.create_v2_job(
+        db,
+        payload=payload,
+        current_user=current_user,
+    )
+
+
+@router.get("", response_model=JobList, summary="List jobs")
+@router.get("/page", response_model=JobList, include_in_schema=False)
+async def list_jobs(
+    page: int = Query(1, ge=1, description="Page number"),
+    page_size: int = Query(20, ge=1, le=100, description="Items per page"),
+    job_status: Optional[str] = Query(None, description="Status filter"),
+    job_type: Optional[str] = Query(None, description="Job type filter"),
+    recent_days: Optional[int] = Query(
+        None,
+        description="Recent-day filter; supported values are 1, 7, and 30",
+        enum=[1, 7, 30],
+    ),
+    start_time: Optional[datetime] = Query(
+        None, description="Start time in ISO format"
+    ),
+    end_time: Optional[datetime] = Query(None, description="End time in ISO format"),
+    current_user: CurrentUser = Depends(with_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    """List jobs for the current user."""
+    return await list_jobs_for_user(
+        db,
+        user_id=current_user.user_id,
+        page=page,
+        page_size=page_size,
+        job_status=job_status,
+        job_type=job_type,
+        recent_days=recent_days,
+        start_time=start_time,
+        end_time=end_time,
+    )
+
+
+@router.get("/{job_id}", response_model=JobResultResponse, summary="Get a job result")
+async def get_job_result(
+    job_id: str,
+    current_user: CurrentUser = Depends(with_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    """Return the result payload for one job."""
+    return await get_job_result_for_user(
+        db,
+        job_id=job_id,
+        user_id=current_user.user_id,
+    )
+
+
+@router.post(
+    "/{job_id}/confirm-upload",
+    response_model=dict,
+    summary="Confirm file upload",
+)
+async def confirm_upload(
+    job_id: str,
+    request: Optional[ConfirmUploadRequest] = None,
+    current_user: CurrentUser = Depends(with_current_user),
+    _write_permission: None = Depends(require_write_permission),
+    db: AsyncSession = Depends(get_db),
+):
+    """Confirm a completed file upload as a fallback path."""
+    return await _document_ingestion_service.confirm_upload(
+        db,
+        job_id=job_id,
+        request_payload=request,
+        user_id=current_user.user_id,
+    )

apps/api/app/api/v2/routes/retrieval.py

@@ -0,0 +1,5 @@
+"""Retrieval API v2 routes."""
+
+from app.api.v1.routes.retrieval import router
+
+__all__ = ["router"]