Add Atlos integration, incident export, and Docker full stack.

Stop tracking llm-wiki-vault and secrets; expand gitignore.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: popovvasile

.env.example

@@ -0,0 +1,27 @@
+APP_NAME=groupint
+
+# Neo4j (docker-compose.desktop.yml)
+NEO4J_URI=bolt://groupint-neo4j:7687
+NEO4J_USERNAME=neo4j
+NEO4J_PASSWORD=difficulties-pushup-gaps
+
+# Telegram (main app + incident worker)
+TELEGRAM_PHONE=
+TELEGRAM_API_ID=
+TELEGRAM_API_HASH=
+
+# Incident LLM pipeline
+INCIDENT_LLM_PROVIDER=anthropic
+ANTHROPIC_API_KEY=
+OPENAI_API_KEY=
+GOOGLE_MAPS_API_KEY=
+INCIDENT_POLL_INTERVAL_SEC=300
+
+# Atlos export (defaults: local Docker stack from scripts/up-full.sh)
+ATLOS_BASE_URL=http://atlos:4000
+ATLOS_API_TOKEN=
+
+# Atlos database (docker-compose.atlos.yml only)
+ATLOS_DB_USER=atlos
+ATLOS_DB_PASSWORD=atlos
+ATLOS_DB_NAME=atlos

.gitignore

@@ -61,3 +61,16 @@ venv
 .idea
 *.session
 *.session-journal
+
+# Local agent knowledge base (not for the public repo)
+llm-wiki-vault/
+
+# Secrets and local environment
+.env
+.streamlit/secrets.toml
+
+# Local lecture drafts
+lectures/
+
+# IDE / Cursor
+.cursor/

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "vendor/atlos"]
+	path = vendor/atlos
+	url = https://github.com/atlosdotorg/atlos.git
+	shallow = true

core/incidents/atlos_export.py

@@ -0,0 +1,188 @@
+"""Export Groupint incidents to Atlos API v2."""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any
+from urllib.parse import urljoin
+
+import httpx
+
+from core.incidents.config import (
+    apply_atlos_secrets,
+    default_atlos_api_token,
+    default_atlos_base_url,
+)
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_SENSITIVE = ["Not Sensitive"]
+DEFAULT_STATUS = "To Do"
+REQUEST_TIMEOUT = 60.0
+
+
+def normalize_base_url(url: str) -> str:
+    return (url or "").strip().rstrip("/")
+
+
+def atlos_config() -> dict[str, str]:
+    """Neo4j saved settings override env defaults (local Docker by default)."""
+    from db.dal import GraphManager
+
+    apply_atlos_secrets()
+    row = GraphManager.get_incident_monitor_config()
+    base = normalize_base_url(row.get("atlos_base_url") or "") or default_atlos_base_url()
+    token = (row.get("atlos_api_token") or "").strip() or default_atlos_api_token()
+    return {"base_url": base, "api_token": token}
+
+
+def incident_to_atlos_payload(
+    inc: dict,
+    *,
+    sensitive: list[str] | None = None,
+    status: str = DEFAULT_STATUS,
+) -> dict[str, Any]:
+    category = (inc.get("category") or "other").strip()
+    location = (inc.get("location_text") or "").strip()
+    summary = (inc.get("summary") or "").strip()
+    occurred = inc.get("occurred_at") or ""
+    parts = [f"[{category}]"]
+    if location:
+        parts.append(location)
+    if summary:
+        parts.append(summary)
+    if occurred:
+        parts.append(f"Occurred: {occurred}")
+    description = "\n\n".join(parts).strip()
+    if len(description) < 8:
+        description = (description + " — Groupint OSINT incident export").strip()
+        if len(description) < 8:
+            description = "Groupint OSINT incident export."
+
+    payload: dict[str, Any] = {
+        "description": description,
+        "sensitive": sensitive or DEFAULT_SENSITIVE,
+        "status": status,
+        "tags": [category],
+    }
+    lat, lon = inc.get("lat"), inc.get("lon")
+    if lat is not None and lon is not None:
+        payload["geolocation"] = f"{float(lat)},{float(lon)}"
+    urls = []
+    for u in inc.get("source_urls") or []:
+        s = str(u).strip()
+        if s.startswith("http://") or s.startswith("https://"):
+            urls.append(s)
+    if urls:
+        payload["urls"] = urls
+    return payload
+
+
+def _extract_slug(data: dict | list | None) -> str | None:
+    if not data:
+        return None
+    if isinstance(data, dict):
+        for key in ("slug", "incident_slug"):
+            if data.get(key):
+                return str(data[key])
+        for nested in ("incident", "data", "result"):
+            inner = data.get(nested)
+            if isinstance(inner, dict):
+                found = _extract_slug(inner)
+                if found:
+                    return found
+    return None
+
+
+def create_atlos_incident(
+    client: httpx.Client,
+    base_url: str,
+    token: str,
+    payload: dict[str, Any],
+) -> str:
+    url = urljoin(base_url + "/", "api/v2/incidents/new")
+    resp = client.post(
+        url,
+        json=payload,
+        headers={"Authorization": f"Bearer {token}"},
+        timeout=REQUEST_TIMEOUT,
+    )
+    resp.raise_for_status()
+    try:
+        body = resp.json()
+    except Exception:
+        body = {}
+    slug = _extract_slug(body if isinstance(body, dict) else {})
+    if slug:
+        return slug
+    raise ValueError(f"Atlos response missing slug: {body!r}")
+
+
+def test_atlos_connection(base_url: str, token: str) -> tuple[bool, str]:
+    if not token:
+        return False, "API token is empty."
+    base_url = normalize_base_url(base_url)
+    url = urljoin(base_url + "/", "api/v2/incidents")
+    try:
+        with httpx.Client(timeout=REQUEST_TIMEOUT) as client:
+            resp = client.get(
+                url,
+                headers={"Authorization": f"Bearer {token}"},
+                params={"limit": 1},
+            )
+        if resp.status_code in (200, 401, 403):
+            if resp.status_code == 200:
+                return True, "Connected to Atlos API."
+            return False, f"Atlos returned HTTP {resp.status_code}: check API token."
+        return False, f"Unexpected HTTP {resp.status_code}: {resp.text[:200]}"
+    except httpx.RequestError as exc:
+        return False, f"Could not reach Atlos: {exc}"
+
+
+def export_incidents_batch(
+    incidents: list[dict],
+    *,
+    base_url: str,
+    api_token: str,
+    sensitive: list[str] | None = None,
+    skip_exported: bool = True,
+    delay_sec: float = 0.3,
+) -> dict[str, Any]:
+    base_url = normalize_base_url(base_url)
+    if not api_token:
+        return {
+            "created": 0,
+            "skipped": len(incidents),
+            "failed": 0,
+            "errors": ["ATLOS_API_TOKEN is not set."],
+        }
+    from db.dal import GraphManager
+
+    created = 0
+    skipped = 0
+    failed = 0
+    errors: list[str] = []
+    with httpx.Client(timeout=REQUEST_TIMEOUT) as client:
+        for inc in incidents:
+            if skip_exported and inc.get("atlos_slug"):
+                skipped += 1
+                continue
+            try:
+                payload = incident_to_atlos_payload(inc, sensitive=sensitive)
+                slug = create_atlos_incident(client, base_url, api_token, payload)
+                GraphManager.set_incident_atlos_export(str(inc["id"]), slug)
+                created += 1
+                if delay_sec > 0:
+                    time.sleep(delay_sec)
+            except Exception as exc:
+                failed += 1
+                msg = f"{inc.get('id')}: {exc}"
+                errors.append(msg)
+                logger.warning("Atlos export failed: %s", msg)
+    return {
+        "created": created,
+        "skipped": skipped,
+        "failed": failed,
+        "errors": errors,
+    }

core/incidents/config.py

@@ -115,3 +115,35 @@ def apply_incidents_secrets() -> None:
         except Exception:
             pass
         break
+
+
+def default_atlos_base_url() -> str:
+    return (os.environ.get("ATLOS_BASE_URL") or "http://atlos:4000").strip().rstrip("/")
+
+
+def default_atlos_api_token() -> str:
+    return (os.environ.get("ATLOS_API_TOKEN") or "").strip()
+
+
+def apply_atlos_secrets() -> None:
+    """Load [atlos] from secrets.toml into env when not already set."""
+    for path in ("/app/.streamlit/secrets.toml", ".streamlit/secrets.toml"):
+        if not os.path.isfile(path):
+            continue
+        try:
+            import tomllib
+
+            with open(path, "rb") as fh:
+                atlos = tomllib.load(fh).get("atlos") or {}
+            for env_key, secret_key in (
+                ("ATLOS_BASE_URL", "base_url"),
+                ("ATLOS_API_TOKEN", "api_token"),
+            ):
+                if os.environ.get(env_key):
+                    continue
+                val = atlos.get(secret_key)
+                if val is not None and str(val).strip():
+                    os.environ[env_key] = str(val).strip()
+        except Exception:
+            pass
+        break

core/incidents/watchlist_channels.py

@@ -0,0 +1,116 @@
+"""Parse channel lists for bulk watchlist import."""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import re
+
+_TME_LINK = re.compile(
+    r"^(?:https?://)?(?:www\.)?t\.me/(?P<ref>\+?[\w-]+)/?$",
+    re.IGNORECASE,
+)
+
+
+def _normalize_channel(raw: str) -> str | None:
+    """Same rules as tg_api_connector.normalize_telegram_group_ref (no Telethon import)."""
+    value = (raw or "").strip()
+    if not value or value.startswith("#"):
+        return None
+    match = _TME_LINK.match(value)
+    if match:
+        ref = match.group("ref")
+    elif value.startswith("@"):
+        ref = value[1:]
+    else:
+        ref = value
+    if not ref or len(ref) < 2:
+        return None
+    return ref
+
+
+def parse_channel_lines(text: str) -> list[str]:
+    """Split pasted text by newlines, commas, or semicolons; dedupe in order."""
+    if not text or not str(text).strip():
+        return []
+    seen: set[str] = set()
+    out: list[str] = []
+    for line in str(text).splitlines():
+        for chunk in re.split(r"[,;]", line):
+            ref = _normalize_channel(chunk)
+            if ref and ref not in seen:
+                seen.add(ref)
+                out.append(ref)
+    return out
+
+
+def _channels_from_csv(raw: bytes) -> list[str]:
+    text = raw.decode("utf-8-sig", errors="replace")
+    reader = csv.reader(io.StringIO(text))
+    rows = list(reader)
+    if not rows:
+        return []
+    header = [c.strip().lower() for c in rows[0]]
+    channel_cols = {"channel_ref", "channel", "username", "url", "link", "t.me"}
+    start = 0
+    col_idx = 0
+    if header and any(h in channel_cols for h in header):
+        for i, h in enumerate(header):
+            if h in channel_cols:
+                col_idx = i
+                break
+        start = 1
+    seen: set[str] = set()
+    out: list[str] = []
+    for row in rows[start:]:
+        if not row or col_idx >= len(row):
+            continue
+        ref = _normalize_channel(row[col_idx])
+        if ref and ref not in seen:
+            seen.add(ref)
+            out.append(ref)
+    return out
+
+
+def _channels_from_json(raw: bytes) -> list[str]:
+    data = json.loads(raw.decode("utf-8-sig", errors="replace"))
+    items: list = []
+    if isinstance(data, list):
+        items = data
+    elif isinstance(data, dict):
+        for key in ("channels", "watchlist", "items"):
+            if key in data and isinstance(data[key], list):
+                items = data[key]
+                break
+    seen: set[str] = set()
+    out: list[str] = []
+    for item in items:
+        if isinstance(item, str):
+            raw_val = item
+        elif isinstance(item, dict):
+            raw_val = (
+                item.get("channel_ref")
+                or item.get("channel")
+                or item.get("url")
+                or item.get("link")
+                or ""
+            )
+        else:
+            continue
+        ref = _normalize_channel(str(raw_val))
+        if ref and ref not in seen:
+            seen.add(ref)
+            out.append(ref)
+    return out
+
+
+def parse_channels_from_upload(uploaded) -> list[str]:
+    """Parse .txt, .csv, or .json channel list from Streamlit upload."""
+    raw: bytes = uploaded.getvalue()
+    name = (uploaded.name or "").lower()
+    if name.endswith(".json"):
+        return _channels_from_json(raw)
+    if name.endswith(".csv"):
+        return _channels_from_csv(raw)
+    return parse_channel_lines(raw.decode("utf-8-sig", errors="replace"))