diff --git a/agents/workspace_layout.py b/agents/workspace_layout.py index 0aa24ca..75e139a 100644 --- a/agents/workspace_layout.py +++ b/agents/workspace_layout.py @@ -93,6 +93,26 @@ def _migrate_legacy_idea_workspace(workspace_root: Path) -> None: shutil.move(str(legacy_exp), str(suite_main)) +def _ensure_dir_or_current_link(path: Path) -> None: + """Create a directory unless the path is already a symlink/current pointer. + + ``current`` paths are allowed to be: + - a real directory + - a symlink to the canonical run / paper dir + - a placeholder directory later populated with ``CURRENT_RUN.txt`` + + ``Path.mkdir(exist_ok=True)`` still raises ``FileExistsError`` on symlinks + on some runtimes, so we special-case them here. + """ + if path.is_symlink(): + return + if path.exists(): + if path.is_dir(): + return + raise FileExistsError(f"Workspace path exists and is not a directory: {path}") + path.mkdir(parents=True, exist_ok=True) + + def _resolve_suite_for_run(run_id: int) -> str: row = db.fetchone("SELECT experiment_suite FROM experiment_runs WHERE id=?", (int(run_id),)) if row: @@ -137,14 +157,14 @@ def get_idea_workspace(insight_id: int, insight: dict | None = None, *, create: "canonical_run_id": insight.get("canonical_run_id"), } if create: - workspace_root.mkdir(parents=True, exist_ok=True) - plan_root.mkdir(parents=True, exist_ok=True) + _ensure_dir_or_current_link(workspace_root) + _ensure_dir_or_current_link(plan_root) for key in ("paper_root", "paper_current_root", "paper_bundles_root", "paper_manifests_root"): - Path(layout[key]).mkdir(parents=True, exist_ok=True) - experiments_root.mkdir(parents=True, exist_ok=True) + _ensure_dir_or_current_link(Path(layout[key])) + _ensure_dir_or_current_link(experiments_root) for suite in KNOWN_EXPERIMENT_SUITES: - (experiments_root / suite / "runs").mkdir(parents=True, exist_ok=True) - (experiments_root / suite / "current").mkdir(parents=True, exist_ok=True) + _ensure_dir_or_current_link(experiments_root / suite / "runs") + _ensure_dir_or_current_link(experiments_root / suite / "current") if sync_db and insight.get("id") is not None: desired = { @@ -197,10 +217,10 @@ def ensure_run_workspace( "spec_root": run_root / "spec", "codex_root": run_root / "codex", } - suite_runs_root.mkdir(parents=True, exist_ok=True) - suite_current_root.mkdir(parents=True, exist_ok=True) + _ensure_dir_or_current_link(suite_runs_root) + _ensure_dir_or_current_link(suite_current_root) for key in ("run_root", "code_root", "results_root", "spec_root", "codex_root"): - Path(info[key]).mkdir(parents=True, exist_ok=True) + _ensure_dir_or_current_link(Path(info[key])) return info diff --git a/pyproject.toml b/pyproject.toml index 57af97a..6e3be64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "numpy>=1.26", "pydantic>=2.0", "pymupdf>=1.24", + "PyYAML>=6.0", "waitress>=3.0", ] keywords = ["knowledge-graph", "science", "literature-mining", "arxiv", "research-ideas", "open-science"] diff --git a/requirements.txt b/requirements.txt index ede5b4b..e9ba2f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ pymupdf>=1.24 psycopg[binary]>=3.1 matplotlib>=3.8 numpy>=1.26 +PyYAML>=6.0 waitress>=3.0 diff --git a/tests/test_workspace_layout.py b/tests/test_workspace_layout.py index ab83997..7d50f95 100644 --- a/tests/test_workspace_layout.py +++ b/tests/test_workspace_layout.py @@ -79,6 +79,22 @@ def test_promote_canonical_run_falls_back_to_marker_when_current_dir_locked(self self.assertTrue(marker.exists()) self.assertEqual(marker.read_text(encoding="utf-8"), str(layout["run_root"])) + def test_get_idea_workspace_tolerates_existing_current_symlink(self): + database.execute("INSERT INTO deep_insights (id, tier, title) VALUES (1, 2, 'Idea Workspace')") + database.commit() + + root = self.workspace_root / "idea_1" + target = root / "legacy_target" + target.mkdir(parents=True, exist_ok=True) + current_link = root / "experiments" / "main" / "current" + current_link.parent.mkdir(parents=True, exist_ok=True) + current_link.symlink_to(target, target_is_directory=True) + + layout = workspace_layout.get_idea_workspace(1) + + self.assertTrue(Path(layout["experiment_current_root"]).is_symlink()) + self.assertEqual(Path(layout["experiment_current_root"]).resolve(), target.resolve()) + def test_backfill_script_maps_legacy_run_and_manuscript_dirs(self): legacy_run = Path(self.tmpdir.name) / "legacy_run" (legacy_run / "code").mkdir(parents=True, exist_ok=True) diff --git a/web/static/css/style.css b/web/static/css/style.css index 93b1bbb..0067c4e 100644 --- a/web/static/css/style.css +++ b/web/static/css/style.css @@ -900,6 +900,76 @@ header#topBar { overflow-y: auto; } +.paper-mini-stats { + margin-bottom: 14px; +} + +.stat-card-mini { + min-height: 86px; +} + +.paper-flow-list { + display: flex; + flex-direction: column; + gap: 10px; +} + +.paper-flow-item { + border: 1px solid var(--border); + background: var(--bg-card); + border-radius: var(--radius); + padding: 14px 16px; + transition: border-color var(--transition), background var(--transition), transform var(--transition); +} + +.paper-flow-item:hover { + border-color: var(--border-hover); + background: var(--bg-card-hover); + transform: translateY(-1px); +} + +.paper-flow-head { + display: flex; + justify-content: space-between; + align-items: flex-start; + gap: 12px; + margin-bottom: 8px; +} + +.paper-flow-title { + font-size: 0.9rem; + font-weight: 600; + color: var(--text-primary); + line-height: 1.45; +} + +.paper-flow-meta { + display: flex; + flex-wrap: wrap; + gap: 12px; + color: var(--text-dim); + font-size: 0.72rem; + margin-bottom: 8px; +} + +.paper-flow-note { + color: var(--text-secondary); + font-size: 0.78rem; + line-height: 1.6; + margin-top: 6px; +} + +.paper-flow-error { + color: var(--red); +} + +.paper-flow-actions { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-top: 12px; +} + .paper-row { padding: 12px 14px; border-radius: 8px; diff --git a/web/static/js/app.js b/web/static/js/app.js index 6b615b3..75d4ab0 100644 --- a/web/static/js/app.js +++ b/web/static/js/app.js @@ -26,6 +26,8 @@ let providerTimer = null; let papersLoaded = false; let oppsLoaded = false; let providersLoaded = false; +let paperProgressLoaded = false; +let generatedPapersLoaded = false; let sidebarCollapsed = false; // ── Helpers ────────────────────────────────────────────────────────── @@ -70,6 +72,13 @@ function timeAgo(ts) { return Math.floor(s / 86400) + 'd ago'; } +function fmtDateTime(ts) { + if (!ts) return ''; + const d = new Date(ts); + if (Number.isNaN(d.getTime())) return String(ts); + return d.toLocaleString(); +} + // ── Tab Navigation ─────────────────────────────────────────────────── function switchTab(tab) { @@ -102,6 +111,14 @@ function onTabActivated(tab) { case 'papers': if (!papersLoaded) loadPapers(); break; + case 'paper-progress': + loadPaperProgressTab(); + paperProgressLoaded = true; + break; + case 'generated-papers': + loadGeneratedPapersTab(); + generatedPapersLoaded = true; + break; case 'discoveries': loadDiscoveriesTab(); break; @@ -1058,6 +1075,231 @@ function renderPapers() { }).join(''); } +// ── Paper Progress Tabs ───────────────────────────────────────────── + +function statusBadge(label, tone = 'dim') { + const cls = { + green: 'badge-green', + red: 'badge-red', + gold: 'badge-gold', + accent: 'badge-accent', + dim: 'badge-dim', + }[tone] || 'badge-dim'; + return `${esc(label)}`; +} + +function toneForPaperStage(stage) { + const key = String(stage || '').toLowerCase(); + if (key.includes('reasoned') || key.includes('done') || key.includes('bundle_ready')) return 'green'; + if (key.includes('error') || key.includes('failed') || key.includes('blocked')) return 'red'; + if (key.includes('research') || key.includes('writing') || key.includes('verify')) return 'accent'; + if (key.includes('experiment') || key.includes('gpu') || key.includes('review')) return 'gold'; + return 'dim'; +} + +function paperPreviewHref(insightId, kind = 'index') { + if (!insightId) return ''; + if (kind === 'pdf') return `/papers/${insightId}/pdf`; + if (kind === 'tex') return `/papers/${insightId}/tex`; + return `/papers/${insightId}`; +} + +function renderMiniStatGrid(targetId, items) { + const root = el(targetId); + if (!root) return; + root.innerHTML = items.map(item => ` +
+
${esc(String(item.value ?? 0))}
+
${esc(item.label)}
+
+ `).join(''); +} + +function renderPaperPipelineRows(rows) { + const list = el('paperProgressPipelineList'); + const count = el('paperProgressPipelineCount'); + if (!list || !count) return; + const papers = (rows || []).slice(0, 20); + count.textContent = papers.length; + if (!papers.length) { + list.innerHTML = '

No papers are moving through the pipeline right now.

'; + return; + } + list.innerHTML = papers.map(item => ` +
+
+
${esc(trunc(item.title || item.id || 'Untitled paper', 120))}
+ ${statusBadge(item.processing_stage || item.status || 'queued', toneForPaperStage(item.processing_stage || item.status))} +
+
+ ${esc(item.id || '-')} + ${esc(item.status || 'unknown')} + ${item.updated_at ? `${esc(timeAgo(item.updated_at))}` : ''} +
+ ${item.stage_last_error ? `
${esc(trunc(item.stage_last_error, 240))}
` : ''} +
+ `).join(''); +} + +function renderPaperGenerationRows(jobs, manuscripts) { + const list = el('paperProgressGenerationList'); + const count = el('paperProgressGenerationCount'); + if (!list || !count) return; + + const activeJobs = (jobs || []).filter(job => !['completed', 'failed'].includes(String(job.status || '').toLowerCase())); + const activeManuscripts = (manuscripts || []).filter(row => { + const status = String(row.status || '').toLowerCase(); + return status && !['stale', 'completed', 'ready'].includes(status); + }); + const total = activeJobs.length + activeManuscripts.length; + count.textContent = total; + + if (!total) { + list.innerHTML = '

No paper-generation jobs are active right now.

'; + return; + } + + const jobCards = activeJobs.map(job => { + const stage = friendlyAutomationStage(job.status, job.stage); + const previewUrl = paperPreviewHref(job.deep_insight_id, 'index'); + return ` +
+
+
${esc(trunc(job.title || `Idea #${job.deep_insight_id}`, 120))}
+ ${statusBadge(stage, toneForPaperStage(job.stage || job.status))} +
+
+ Idea #${esc(job.deep_insight_id || '-')} + ${job.experiment_status ? `Experiment: ${esc(job.experiment_status)}` : ''} + ${job.updated_at ? `${esc(timeAgo(job.updated_at))}` : ''} +
+ ${job.last_note ? `
${esc(trunc(job.last_note, 220))}
` : ''} + ${job.last_error ? `
${esc(trunc(job.last_error, 220))}
` : ''} +
+ + ${previewUrl ? `` : ''} +
+
+ `; + }); + + const manuscriptCards = activeManuscripts.map(row => ` +
+
+
${esc(trunc(row.insight_title || `Manuscript #${row.id}`, 120))}
+ ${statusBadge(row.status || 'drafting', toneForPaperStage(row.status))} +
+
+ Manuscript #${esc(row.id || '-')} + ${row.hypothesis_verdict ? `Verdict: ${esc(row.hypothesis_verdict)}` : ''} + ${row.updated_at ? `${esc(timeAgo(row.updated_at))}` : ''} +
+ ${row.workdir ? `
${esc(trunc(row.workdir, 220))}
` : ''} +
+ ${row.deep_insight_id ? `` : ''} +
+
+ `); + + list.innerHTML = jobCards.concat(manuscriptCards).join(''); +} + +async function loadPaperProgressTab() { + try { + const [automation, jobs, manuscripts] = await Promise.all([ + api('/api/automation'), + api('/api/auto_research/jobs?limit=50'), + api('/api/manuscripts?limit=50'), + ]); + const current = (automation || {}).current_work || {}; + const activeManuscripts = (manuscripts || []).filter(row => { + const status = String(row.status || '').toLowerCase(); + return status && !['stale', 'completed', 'ready'].includes(status); + }); + renderMiniStatGrid('paperProgressStats', [ + { label: 'Pipeline Papers', value: (current.papers || []).length }, + { label: 'Paper Jobs', value: (jobs || []).filter(job => !['completed', 'failed'].includes(String(job.status || '').toLowerCase())).length }, + { label: 'Active Manuscripts', value: activeManuscripts.length }, + { label: 'Blocked Jobs', value: ((automation || {}).auto_research || {}).blocked || 0 }, + ]); + renderPaperPipelineRows(current.papers || []); + renderPaperGenerationRows(jobs || [], manuscripts || []); + } catch (e) { + const listA = el('paperProgressPipelineList'); + const listB = el('paperProgressGenerationList'); + if (listA) listA.innerHTML = `

Failed to load paper progress: ${esc(e.message)}

`; + if (listB) listB.innerHTML = `

Failed to load paper generation jobs: ${esc(e.message)}

`; + } +} + +function manuscriptTone(status) { + const key = String(status || '').toLowerCase(); + if (key === 'bundle_ready' || key === 'ready') return 'green'; + if (key === 'stale' || key === 'failed' || key === 'blocked') return 'red'; + if (key.includes('draft')) return 'accent'; + return 'dim'; +} + +function renderGeneratedPapers(manuscripts) { + const list = el('generatedPapersList'); + const count = el('generatedPapersCount'); + if (!list || !count) return; + const rows = (manuscripts || []).slice(0, 100); + count.textContent = rows.length; + if (!rows.length) { + list.innerHTML = '

No manuscript runs have been generated yet.

'; + return; + } + list.innerHTML = rows.map(row => { + const preview = row.deep_insight_id ? paperPreviewHref(row.deep_insight_id, 'index') : ''; + return ` +
+
+
${esc(trunc(row.insight_title || `Manuscript #${row.id}`, 130))}
+ ${statusBadge(row.status || 'generated', manuscriptTone(row.status))} +
+
+ Manuscript #${esc(row.id || '-')} + ${row.experiment_run_id ? `Run #${esc(row.experiment_run_id)}` : ''} + ${row.hypothesis_verdict ? `Verdict: ${esc(row.hypothesis_verdict)}` : ''} + ${row.updated_at ? `${esc(timeAgo(row.updated_at))}` : ''} +
+
+ ${row.created_at ? `Generated: ${esc(fmtDateTime(row.created_at))}` : ''} + ${row.updated_at ? `Updated: ${esc(fmtDateTime(row.updated_at))}` : ''} +
+ ${row.workdir ? `
${esc(trunc(row.workdir, 220))}
` : ''} +
Use the paper page to open only the assets that actually exist for this manuscript.
+
+ ${preview ? `` : ''} +
+
+ `; + }).join(''); +} + +async function loadGeneratedPapersTab() { + try { + const manuscripts = await api('/api/manuscripts?limit=100'); + const counts = (manuscripts || []).reduce((acc, row) => { + const key = String(row.status || 'unknown').toLowerCase(); + acc.total += 1; + acc[key] = (acc[key] || 0) + 1; + return acc; + }, { total: 0 }); + renderMiniStatGrid('generatedPapersStats', [ + { label: 'Total Manuscripts', value: counts.total || 0 }, + { label: 'Ready', value: counts.ready || counts.bundle_ready || 0 }, + { label: 'Stale', value: counts.stale || 0 }, + { label: 'Drafting', value: counts.drafting || 0 }, + ]); + renderGeneratedPapers(manuscripts || []); + } catch (e) { + const list = el('generatedPapersList'); + if (list) list.innerHTML = `

Failed to load generated papers: ${esc(e.message)}

`; + } +} + async function togglePaper(rowEl) { const isExpanded = rowEl.classList.contains('expanded'); @@ -2081,6 +2323,9 @@ window._dg = { togglePaper, updateMatrixMetric, searchNav, + viewPaperGeneration(insightId) { + return this.viewExperimentGroup(insightId); + }, async viewExperimentGroup(insightId) { try { @@ -2340,6 +2585,8 @@ function init() { setInterval(() => { if (activeTab === 'experiments') loadExperimentsTab(); + if (activeTab === 'paper-progress') loadPaperProgressTab(); + if (activeTab === 'generated-papers') loadGeneratedPapersTab(); if (activeTab === 'discoveries') loadDiscoveriesTab(); }, 10000); } diff --git a/web/templates/index.html b/web/templates/index.html index 665c274..dc6025b 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -69,7 +69,15 @@ + +