Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file removed foo
Empty file.
4 changes: 3 additions & 1 deletion src-tauri/Info.plist
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,13 @@
<string>Coco AI needs access to your microphone for voice input and audio recording features.</string>
<key>NSCameraUsageDescription</key>
<string>Coco AI requires camera access for scanning documents and capturing images.</string>
<key>NSCameraUseContinuityCameraDeviceType</key>
<true/>
<key>NSSpeechRecognitionUsageDescription</key>
<string>Coco AI uses speech recognition to convert your voice into text for a hands-free experience.</string>
<key>NSAppleEventsUsageDescription</key>
<string>Coco AI requires access to Apple Events to enable certain features, such as opening files and applications.</string>
<key>NSAccessibility</key>
<true/>
</dict>
</plist>
</plist>
119 changes: 108 additions & 11 deletions src-tauri/src/selection_monitor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ use once_cell::sync::Lazy;
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, Ordering};

/// Global toggle: selection monitoring disabled for this release.
static SELECTION_ENABLED: AtomicBool = AtomicBool::new(false);
/// Global toggle: selection monitoring enabled by default.
static SELECTION_ENABLED: AtomicBool = AtomicBool::new(true);

/// Ensure we only start the monitor thread once. Allows delayed start after
/// Accessibility permission is granted post-launch.
Expand Down Expand Up @@ -281,12 +281,19 @@ pub fn start_selection_monitor(app_handle: tauri::AppHandle) {
let mut stable_text = String::new();
let mut stable_count = 0;
let mut empty_count = 0;
let mut loop_idx: u64 = 0;

loop {
loop_idx = loop_idx.wrapping_add(1);
let verbose = loop_idx % 50 == 0; // Log roughly every 1.5s (30ms * 50)

std::thread::sleep(Duration::from_millis(30));

// If disabled: do not read AX / do not show popup; hide if currently visible.
if !is_selection_enabled() {
if verbose {
println!("[SelectionMonitor] Disabled");
}
if popup_visible {
let _ = app_handle.emit("selection-detected", "");
popup_visible = false;
Expand All @@ -301,6 +308,13 @@ pub fn start_selection_monitor(app_handle: tauri::AppHandle) {
// system-wide focused element belongs to this process.
let front_is_me = is_frontmost_app_me() || is_focused_element_me();

if verbose {
println!(
"[SelectionMonitor] Loop heartbeat. front_is_me={}",
front_is_me
);
}

// When Coco is frontmost, disable detection but do NOT hide the popup.
// Users may be clicking the popup; we must keep it visible.
if front_is_me {
Expand All @@ -313,9 +327,16 @@ pub fn start_selection_monitor(app_handle: tauri::AppHandle) {
// Lightweight retries to smooth out transient AX focus instability.
let selected_text = {
// Up to 2 retries, 35ms apart.
read_selected_text_with_retries(2, 35)
read_selected_text_with_retries(2, 35, verbose)
};

if verbose {
println!(
"[SelectionMonitor] read_selected_text result: {:?}",
selected_text
);
}

match selected_text {
Some(text) if !text.is_empty() => {
empty_count = 0;
Expand Down Expand Up @@ -425,6 +446,7 @@ fn ensure_accessibility_permission(app_handle: &tauri::AppHandle) -> bool {
}

// Still not trusted — notify frontend and deep-link to settings.
println!("[SelectionMonitor] Accessibility permission NOT granted. Opening System Settings...");
let _ = app_handle.emit("selection-permission-required", true);
log::debug!(target: "coco_lib::selection_monitor", "selection-permission-required emitted");

Expand Down Expand Up @@ -563,12 +585,16 @@ fn is_focused_element_me() -> bool {
/// Read the selected text of the frontmost application (without using the clipboard).
/// macOS only. Returns `None` when the frontmost app is Coco to avoid false empties.
#[cfg(target_os = "macos")]
fn read_selected_text() -> Option<String> {
fn read_selected_text(verbose: bool) -> Option<String> {
use objc2_app_kit::NSWorkspace;
use objc2_application_services::{AXError, AXUIElement};
use objc2_core_foundation::{CFRetained, CFString, CFType};
use std::ptr::NonNull;

if verbose {
println!("[SelectionMonitor] read_selected_text: Attempting to read...");
}

// Prefer system-wide focused element; if unavailable, fall back to app/window focused element.
let mut focused_ui_ptr: *const CFType = std::ptr::null();
let focused_attr = CFString::from_static_str("AXFocusedUIElement");
Expand All @@ -583,19 +609,33 @@ fn read_selected_text() -> Option<String> {
.copy_attribute_value(&focused_attr, NonNull::new(&mut focused_ui_ptr).unwrap())
};
if err != AXError::Success {
if verbose {
println!(
"[SelectionMonitor] Failed to get AXFocusedUIElement from system wide element: {:?}",
err
);
}
focused_ui_ptr = std::ptr::null();
}
} else if verbose {
println!("[SelectionMonitor] AXUIElementCreateSystemWide returned null");
}

// Fallback to the frontmost app's focused/window element.
if focused_ui_ptr.is_null() {
if verbose {
println!("[SelectionMonitor] System-wide focus not found, trying frontmost app...");
}
let workspace = unsafe { NSWorkspace::sharedWorkspace() };
let frontmost_app = unsafe { workspace.frontmostApplication() }?;
let pid = unsafe { frontmost_app.processIdentifier() };

// Skip if frontmost is Coco (this process).
let my_pid = std::process::id() as i32;
if pid == my_pid {
if verbose {
println!("[SelectionMonitor] Frontmost app is Coco, skipping.");
}
return None;
}

Expand All @@ -605,6 +645,12 @@ fn read_selected_text() -> Option<String> {
.copy_attribute_value(&focused_attr, NonNull::new(&mut focused_ui_ptr).unwrap())
};
if err != AXError::Success || focused_ui_ptr.is_null() {
if verbose {
println!(
"[SelectionMonitor] Failed to get AXFocusedUIElement from app (pid={}), trying AXFocusedWindow...",
pid
);
}
// Try `AXFocusedWindow` as a lightweight fallback.
let mut focused_window_ptr: *const CFType = std::ptr::null();
let focused_window_attr = CFString::from_static_str("AXFocusedWindow");
Expand All @@ -615,9 +661,41 @@ fn read_selected_text() -> Option<String> {
)
};
if w_err != AXError::Success || focused_window_ptr.is_null() {
if verbose {
println!(
"[SelectionMonitor] Failed to get AXFocusedWindow from app (pid={})",
pid
);
}
return None;
}
focused_ui_ptr = focused_window_ptr;
let focused_window_elem: CFRetained<AXUIElement> = unsafe {
CFRetained::from_raw(
NonNull::new(focused_window_ptr.cast::<AXUIElement>().cast_mut()).unwrap(),
)
};
let mut inner_focused_ptr: *const CFType = std::ptr::null();
let w_focused_err = unsafe {
focused_window_elem.copy_attribute_value(
&focused_attr,
NonNull::new(&mut inner_focused_ptr).unwrap(),
)
};
if w_focused_err == AXError::Success && !inner_focused_ptr.is_null() {
if verbose {
println!(
"[SelectionMonitor] Resolved AXFocusedUIElement via AXFocusedWindow (pid={})",
pid
);
}
focused_ui_ptr = inner_focused_ptr;
} else if verbose {
println!(
"[SelectionMonitor] Failed to get AXFocusedUIElement from focused window (pid={}), err={:?}",
pid, w_focused_err
);
}
}
}

Expand All @@ -635,6 +713,12 @@ fn read_selected_text() -> Option<String> {
)
};
if err != AXError::Success || selected_text_ptr.is_null() {
if verbose {
println!(
"[SelectionMonitor] Failed to get AXSelectedText (err={:?})",
err
);
}
return None;
}

Expand All @@ -643,27 +727,40 @@ fn read_selected_text() -> Option<String> {
CFRetained::from_raw(NonNull::new(selected_text_ptr.cast::<CFString>().cast_mut()).unwrap())
};

Some(selected_cfstr.to_string())
let s = selected_cfstr.to_string();
if verbose {
println!("[SelectionMonitor] Success! Found text length: {}", s.len());
}
Some(s)
}

/// Read selected text with lightweight retries to handle transient AX focus instability.
#[cfg(target_os = "macos")]
fn read_selected_text_with_retries(retries: u32, delay_ms: u64) -> Option<String> {
fn read_selected_text_with_retries(retries: u32, delay_ms: u64, verbose: bool) -> Option<String> {
use std::thread;
use std::time::Duration;
for attempt in 0..=retries {
if let Some(text) = read_selected_text() {
if let Some(text) = read_selected_text(verbose) {
if !text.is_empty() {
if attempt > 0 {
// log::info!(
// "read_selected_text: 第{}次重试成功,获取到选中文本",
// attempt
// );
log::info!(
"read_selected_text: 第{}次重试成功,获取到选中文本",
attempt
);
if verbose {
println!("[SelectionMonitor] Retry success on attempt {}", attempt);
}
}
return Some(text);
}
}
if attempt < retries {
if verbose {
println!(
"[SelectionMonitor] Attempt {} failed/empty, retrying...",
attempt
);
}
thread::sleep(Duration::from_millis(delay_ms));
}
}
Expand Down
26 changes: 13 additions & 13 deletions src/hooks/useTray.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ export const useTray = () => {
const showCocoShortcuts = useAppStore((state) => state.showCocoShortcuts);

const selectionEnabled = useSelectionStore((state) => state.selectionEnabled);
// const setSelectionEnabled = useSelectionStore((state) => state.setSelectionEnabled);
const setSelectionEnabled = useSelectionStore((state) => state.setSelectionEnabled);

useUpdateEffect(() => {
if (showCocoShortcuts.length === 0) return;
Expand Down Expand Up @@ -65,18 +65,18 @@ export const useTray = () => {

itemPromises.push(PredefinedMenuItem.new({ item: "Separator" }));

// if (isMac) {
// itemPromises.push(
// MenuItem.new({
// text: selectionEnabled
// ? t("tray.selectionDisable")
// : t("tray.selectionEnable"),
// action: async () => {
// setSelectionEnabled(!selectionEnabled);
// },
// })
// );
// }
if (isMac) {
itemPromises.push(
MenuItem.new({
text: selectionEnabled
? t("tray.selectionDisable")
: t("tray.selectionEnable"),
action: async () => {
setSelectionEnabled(!selectionEnabled);
},
})
);
}

itemPromises.push(
MenuItem.new({
Expand Down
4 changes: 2 additions & 2 deletions src/routes/outlet.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import { useExtensionsStore } from "@/stores/extensionsStore";
import { useSelectionStore, startSelectionStorePersistence } from "@/stores/selectionStore";
import { useServers } from "@/hooks/useServers";
import { useDeepLinkManager } from "@/hooks/useDeepLinkManager";
// import { useSelectionWindow } from "@/hooks/useSelectionWindow";
import { useSelectionWindow } from "@/hooks/useSelectionWindow";

export default function LayoutOutlet() {
const location = useLocation();
Expand Down Expand Up @@ -128,7 +128,7 @@ export default function LayoutOutlet() {
});

// --- Selection window ---
// useSelectionWindow();
useSelectionWindow();

return (
<>
Expand Down
2 changes: 1 addition & 1 deletion src/stores/selectionStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ export const useSelectionStore = create<SelectionStore>((set) => ({
setIconsOnly: (iconsOnly) => set({ iconsOnly }),
toolbarConfig: [],
setToolbarConfig: (toolbarConfig) => set({ toolbarConfig }),
selectionEnabled: false,
selectionEnabled: true,
setSelectionEnabled: (selectionEnabled) => set({ selectionEnabled }),
}));

Expand Down