diff --git a/Cargo.lock b/Cargo.lock index 9e694ba0..4bc0e01b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1863,6 +1863,7 @@ dependencies = [ "serde_ignored", "serde_json", "smithay-client-toolkit", + "tempfile", "tokio", "toml", "toml_edit", diff --git a/Cargo.toml b/Cargo.toml index ca972771..17a46aa4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,8 @@ serde = { version = "1.0", features = ["derive"] } serde_ignored = "0.1" schemars = { version = "1.2", features = ["derive"], optional = true } libc = "0.2" +# Securely created, self-deleting temporary files for the OCR engine handoff. +tempfile = "3" flate2 = "1.0" zune-jpeg = "0.5" # Grapheme/word segmentation for the text-editor caret and selection. diff --git a/README.md b/README.md index 85c0ba1b..c762c599 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ The v0.9.23+ prebuilt `wayscriber` packages require glibc 2.39 and GTK 4.12 — - Full-screen saves, active-window grabs, region capture - Copy to clipboard or save to file - Uses `grim`, `slurp`, `wl-clipboard` (installed automatically by deb/rpm/AUR packages; fallback: xdg-desktop-portal) +- Copy text from screen (OCR): drag a region of the shown desktop and get its text on the clipboard (needs `tesseract`; no default shortcut) ### Sessions and persistence - Session persistence is enabled by default for boards, undo/redo history, and tool state @@ -329,6 +330,9 @@ If you are on a stable NixOS channel and want the newest release, use [the proje grim slurp wl-clipboard + + # Optional: copy text from screen (OCR) + (tesseract.override { enableLanguages = [ "eng" ]; }) ]; } ``` @@ -538,6 +542,19 @@ sudo apt-get install wl-clipboard grim slurp # Debian/Ubuntu sudo dnf install wl-clipboard grim slurp # Fedora ``` +### Copy text from screen (OCR) + +`Copy text from screen` recognizes the text in a dragged screen region and copies +it to the clipboard. It is optional: the action has no default shortcut and its +toolbar button is hidden until you turn it on. Install Tesseract and the language +data you configure in `[capture].ocr_languages` (default `eng`): + +```bash +sudo pacman -S tesseract tesseract-data-eng # Arch/Omarchy +sudo apt-get install tesseract-ocr # Debian/Ubuntu +sudo dnf install tesseract tesseract-langpack-eng # Fedora +``` + --- ## First launch diff --git a/config.example.toml b/config.example.toml index 45843111..175108fe 100644 --- a/config.example.toml +++ b/config.example.toml @@ -275,6 +275,10 @@ export_all_boards_pdf_file = [] # Open the most recent capture folder open_capture_folder = ["Ctrl+Alt+O"] +# Select a screen region and copy the text recognized in it (needs Tesseract). +# Unbound by default: "O" is already the orange quick color. +copy_text_from_screen = [] + # Toggle frozen mode toggle_frozen_mode = ["Ctrl+Shift+F"] @@ -628,6 +632,9 @@ rebind_modifier = "ctrl_shift" [ui.toolbar.items] # Checked in the configurator means shown; IDs listed here are hidden. # The screenshot toolbar button is hidden by default. +# Copy text (OCR) is hidden by a built-in baseline instead of by this list, +# so upgrading a config written before it existed cannot reveal it; enable it +# through toolbar customization, which records an entry in `shown` below. # Section-level ids (side.group.*) act as explicit overrides that beat the # layout-mode baseline and survive mode switches. hidden = [ @@ -660,6 +667,7 @@ top_controls = [ "top.utility.text", "top.utility.sticky-note", "top.utility.screenshot", + "top.utility.ocr", "top.utility.clear-canvas", "top.utility.highlight", ] @@ -1384,6 +1392,11 @@ copy_to_clipboard = true # Use --no-exit-after-capture to keep the overlay open for a run. exit_after_capture = false +# Languages used by "Copy text from screen" (OCR), in Tesseract's +# plus-separated form, for example "eng" or "eng+deu". The matching Tesseract +# language packages must be installed. OCR also needs `enabled = true` above. +ocr_languages = "eng" + # ═══════════════════════════════════════════════════════════════════════════════ # EXPORT SETTINGS # ═══════════════════════════════════════════════════════════════════════════════ diff --git a/configurator/src/app/pages/capture.rs b/configurator/src/app/pages/capture.rs index 6bf07d59..bfbe4b61 100644 --- a/configurator/src/app/pages/capture.rs +++ b/configurator/src/app/pages/capture.rs @@ -15,6 +15,8 @@ use crate::models::{ PdfTransparentBackgroundOption, TabId, TextField, ToggleField, }; +use wayscriber::config::validate_ocr_languages; + use super::super::search::SearchArea; use super::super::state::ConfiguratorApp; use super::color_rows::{ResolvedColor, color_row}; @@ -56,6 +58,18 @@ pub(super) fn build(sender: &ComponentSender) -> BuiltPage { "", |app| app.draft.capture_exit_after, |value| Message::ToggleChanged(ToggleField::CaptureExitAfter, value), + ) + .entry_row_validated( + "OCR languages (e.g. eng, eng+deu \u{2014} needs the matching Tesseract packages)", + |app| app.draft.capture_ocr_languages.clone(), + |value| Message::TextChanged(TextField::CaptureOcrLanguages, value), + |app| { + validate_ocr_languages(&app.draft.capture_ocr_languages) + .err() + .map(|reason| { + format!("{reason}. Use Tesseract codes joined by '+', such as eng+deu.") + }) + }, ); page.group_in_area("PDF export", SearchArea::CapturePdf) diff --git a/configurator/src/models/config/draft/from_config.rs b/configurator/src/models/config/draft/from_config.rs index 44afe3f4..35dba0dd 100644 --- a/configurator/src/models/config/draft/from_config.rs +++ b/configurator/src/models/config/draft/from_config.rs @@ -247,6 +247,7 @@ impl ConfigDraft { capture_format: config.capture.format.clone(), capture_copy_to_clipboard: config.capture.copy_to_clipboard, capture_exit_after: config.capture.exit_after_capture, + capture_ocr_languages: config.capture.ocr_languages.clone(), export_pdf_filename_template: config .export .pdf diff --git a/configurator/src/models/config/draft/mod.rs b/configurator/src/models/config/draft/mod.rs index dbc71784..e7709cda 100644 --- a/configurator/src/models/config/draft/mod.rs +++ b/configurator/src/models/config/draft/mod.rs @@ -180,6 +180,7 @@ pub struct ConfigDraft { pub capture_format: String, pub capture_copy_to_clipboard: bool, pub capture_exit_after: bool, + pub capture_ocr_languages: String, pub export_pdf_filename_template: String, pub export_pdf_all_boards_filename_template: String, pub export_pdf_page_size: PdfPageSizeOption, diff --git a/configurator/src/models/config/setters.rs b/configurator/src/models/config/setters.rs index 9aa87d88..aafda684 100644 --- a/configurator/src/models/config/setters.rs +++ b/configurator/src/models/config/setters.rs @@ -366,6 +366,7 @@ impl ConfigDraft { TextField::CaptureSaveDirectory => self.capture_save_directory = value, TextField::CaptureFilename => self.capture_filename_template = value, TextField::CaptureFormat => self.capture_format = value, + TextField::CaptureOcrLanguages => self.capture_ocr_languages = value, TextField::ExportPdfFilenameTemplate => self.export_pdf_filename_template = value, TextField::ExportPdfAllBoardsFilenameTemplate => { self.export_pdf_all_boards_filename_template = value diff --git a/configurator/src/models/config/to_config/capture.rs b/configurator/src/models/config/to_config/capture.rs index 2f883413..bdb80e68 100644 --- a/configurator/src/models/config/to_config/capture.rs +++ b/configurator/src/models/config/to_config/capture.rs @@ -1,14 +1,21 @@ use super::super::draft::ConfigDraft; use crate::models::error::FormError; -use wayscriber::config::Config; +use wayscriber::config::{Config, validate_ocr_languages}; impl ConfigDraft { - pub(super) fn apply_capture(&self, config: &mut Config, _errors: &mut Vec) { + pub(super) fn apply_capture(&self, config: &mut Config, errors: &mut Vec) { config.capture.enabled = self.capture_enabled; config.capture.save_directory = self.capture_save_directory.clone(); config.capture.filename_template = self.capture_filename_template.clone(); config.capture.format = self.capture_format.clone(); config.capture.copy_to_clipboard = self.capture_copy_to_clipboard; config.capture.exit_after_capture = self.capture_exit_after; + match validate_ocr_languages(&self.capture_ocr_languages) { + Ok(languages) => config.capture.ocr_languages = languages, + Err(reason) => errors.push(FormError::new( + "capture.ocr_languages", + format!("OCR languages: {reason}."), + )), + } } } diff --git a/configurator/src/models/fields/toggles.rs b/configurator/src/models/fields/toggles.rs index 2b014355..5a9e2dd5 100644 --- a/configurator/src/models/fields/toggles.rs +++ b/configurator/src/models/fields/toggles.rs @@ -136,6 +136,7 @@ pub enum TextField { CaptureSaveDirectory, CaptureFilename, CaptureFormat, + CaptureOcrLanguages, ExportPdfFilenameTemplate, ExportPdfAllBoardsFilenameTemplate, ExportPdfCustomWidth, diff --git a/configurator/src/models/keybindings/field/config/read.rs b/configurator/src/models/keybindings/field/config/read.rs index ea622adb..6dbb7783 100644 --- a/configurator/src/models/keybindings/field/config/read.rs +++ b/configurator/src/models/keybindings/field/config/read.rs @@ -133,6 +133,7 @@ impl KeybindingField { Self::ExportBoardPdfFile => &config.capture.export_board_pdf_file, Self::ExportAllBoardsPdfFile => &config.capture.export_all_boards_pdf_file, Self::OpenCaptureFolder => &config.capture.open_capture_folder, + Self::CopyTextFromScreen => &config.capture.copy_text_from_screen, Self::ToggleFrozenMode => &config.zoom.toggle_frozen_mode, Self::ZoomIn => &config.zoom.zoom_in, Self::ZoomOut => &config.zoom.zoom_out, diff --git a/configurator/src/models/keybindings/field/config/write.rs b/configurator/src/models/keybindings/field/config/write.rs index 3f78aadc..09722411 100644 --- a/configurator/src/models/keybindings/field/config/write.rs +++ b/configurator/src/models/keybindings/field/config/write.rs @@ -136,6 +136,7 @@ impl KeybindingField { Self::ExportBoardPdfFile => config.capture.export_board_pdf_file = value, Self::ExportAllBoardsPdfFile => config.capture.export_all_boards_pdf_file = value, Self::OpenCaptureFolder => config.capture.open_capture_folder = value, + Self::CopyTextFromScreen => config.capture.copy_text_from_screen = value, Self::ToggleFrozenMode => config.zoom.toggle_frozen_mode = value, Self::ZoomIn => config.zoom.zoom_in = value, Self::ZoomOut => config.zoom.zoom_out = value, diff --git a/configurator/src/models/keybindings/field/labels.rs b/configurator/src/models/keybindings/field/labels.rs index b59d7f72..6422c956 100644 --- a/configurator/src/models/keybindings/field/labels.rs +++ b/configurator/src/models/keybindings/field/labels.rs @@ -128,6 +128,7 @@ impl KeybindingField { Self::ExportBoardPdfFile => "Export board: PDF file", Self::ExportAllBoardsPdfFile => "Export all boards: PDF file", Self::OpenCaptureFolder => "Open capture folder", + Self::CopyTextFromScreen => "Copy text from screen (OCR)", Self::ToggleFrozenMode => "Toggle freeze", Self::ZoomIn => "Zoom in", Self::ZoomOut => "Zoom out", @@ -281,6 +282,7 @@ impl KeybindingField { Self::ExportBoardPdfFile => "export_board_pdf_file", Self::ExportAllBoardsPdfFile => "export_all_boards_pdf_file", Self::OpenCaptureFolder => "open_capture_folder", + Self::CopyTextFromScreen => "copy_text_from_screen", Self::ToggleFrozenMode => "toggle_frozen_mode", Self::ZoomIn => "zoom_in", Self::ZoomOut => "zoom_out", diff --git a/configurator/src/models/keybindings/field/list.rs b/configurator/src/models/keybindings/field/list.rs index c0cb3189..7ad4f480 100644 --- a/configurator/src/models/keybindings/field/list.rs +++ b/configurator/src/models/keybindings/field/list.rs @@ -128,6 +128,7 @@ impl KeybindingField { Self::ExportBoardPdfFile, Self::ExportAllBoardsPdfFile, Self::OpenCaptureFolder, + Self::CopyTextFromScreen, Self::ToggleFrozenMode, Self::ZoomIn, Self::ZoomOut, diff --git a/configurator/src/models/keybindings/field/mod.rs b/configurator/src/models/keybindings/field/mod.rs index 9cc811b4..e3064d8b 100644 --- a/configurator/src/models/keybindings/field/mod.rs +++ b/configurator/src/models/keybindings/field/mod.rs @@ -113,6 +113,7 @@ pub enum KeybindingField { ExportBoardPdfFile, ExportAllBoardsPdfFile, OpenCaptureFolder, + CopyTextFromScreen, ToggleFrozenMode, ZoomIn, ZoomOut, diff --git a/configurator/src/models/keybindings/field/tab.rs b/configurator/src/models/keybindings/field/tab.rs index c785ae2b..abda464f 100644 --- a/configurator/src/models/keybindings/field/tab.rs +++ b/configurator/src/models/keybindings/field/tab.rs @@ -129,6 +129,7 @@ impl KeybindingField { | Self::ExportBoardPdfFile | Self::ExportAllBoardsPdfFile | Self::OpenCaptureFolder + | Self::CopyTextFromScreen | Self::ToggleFrozenMode | Self::ZoomIn | Self::ZoomOut diff --git a/docs/CONFIG.md b/docs/CONFIG.md index b9ce07f2..87fc9d7b 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -1380,12 +1380,47 @@ copy_to_clipboard = true # When false, clipboard-only captures still auto-exit by default. # Use --no-exit-after-capture to keep the overlay open for a run. exit_after_capture = false + +# Languages for "Copy text from screen" (OCR), in Tesseract's plus-separated +# form. The matching Tesseract language packages must be installed. +ocr_languages = "eng" ``` **Tips:** - Set `copy_to_clipboard = false` if you prefer file-only captures. - Clipboard-only shortcuts ignore the save directory automatically. - `wl-clipboard`, `grim`, and `slurp` are installed automatically by deb/rpm/AUR packages. For source/tarball installs, add them manually; otherwise wayscriber falls back to `xdg-desktop-portal`. + +#### Copy text from screen (OCR) + +`Copy text from screen` selects a region of the desktop image wayscriber is +already showing, recognizes the text in it with a local Tesseract, and copies +the result to the clipboard. It reads the underlying screen capture only — +never your annotations, the toolbars, or any other wayscriber chrome — and it +does not change the active tool, the drawing history, or the board. + +- The action is `copy_text_from_screen`. It has **no default shortcut**, because + `O` is already the orange quick color; bind one in the configurator or in + `[keybindings.capture]`. +- It is also in the command palette (search for "OCR"), and as an optional top + toolbar button (`top.utility.ocr`), hidden by default like Screenshot. +- `ocr_languages` accepts one language or several joined with `+` + (`eng`, `eng+deu`). Only letters, digits, `_` and `-` are accepted; anything + else falls back to `eng`. +- OCR obeys `enabled` above: with capture disabled, the action reports that and + does nothing. +- On a solid whiteboard or blackboard with no visible screen capture, OCR + refuses rather than reading the board. + +**Requirements:** the `tesseract` command plus the language data for every code +in `ocr_languages`, and `wl-copy` (from `wl-clipboard`) for the clipboard write. + +| Distribution | Packages | +| --- | --- | +| Arch / Omarchy | `tesseract`, `tesseract-data-eng` | +| Debian / Ubuntu | `tesseract-ocr` (depends on the English data) | +| Fedora | `tesseract`, `tesseract-langpack-eng` | +| Nix | the `tesseract` package/wrapper with `eng` enabled | - Use `--exit-after-capture` / `--no-exit-after-capture` to override exit behavior per run. ### `[export.pdf]` - PDF Export @@ -1839,6 +1874,10 @@ export_all_boards_pdf_file = [] # Open the most recent capture folder open_capture_folder = ["Ctrl+Alt+O"] +# Select a screen region and copy the text recognized in it (needs Tesseract). +# Unbound by default: "O" is already the orange quick color. +copy_text_from_screen = [] + # Toggle frozen mode toggle_frozen_mode = ["Ctrl+Shift+F"] diff --git a/docs/codebase-overview.md b/docs/codebase-overview.md index 6a6923d6..3ac017ec 100644 --- a/docs/codebase-overview.md +++ b/docs/codebase-overview.md @@ -470,6 +470,7 @@ Notifications are sent via `notification::send_notification_async`, keeping all | `src/config/` | Config parsing, defaults, keybinding map. | | `src/runtime_ui_state/` | Generated UI preference wire format, seed registry, guarded persistence, reset, and recovery state machines. | | `src/session/` | Configured and named session persistence, snapshots, sidecars, locks, and catalog metadata. | +| `src/ocr/` | Screen text recognition: capacity-one controller, Tesseract adapter, typed outcomes. | | `src/notification.rs` | Desktop notifications for capture results and the update notice. | | `src/update_check/` | Opt-outable "a newer release exists" check: version ordering, manifest trust rules, `curl`/`wget` transport, cache. Installs nothing. | | `src/about_window/` | Standalone About dialog: content/layout/interaction split, update card, diagnostics copy. | diff --git a/packaging/.SRCINFO b/packaging/.SRCINFO index 60d6d319..6ca91abe 100644 --- a/packaging/.SRCINFO +++ b/packaging/.SRCINFO @@ -19,6 +19,8 @@ pkgbase = wayscriber depends = wl-clipboard depends = grim depends = slurp + optdepends = tesseract: copy text from screen (OCR) + optdepends = tesseract-data-eng: English language data for OCR source = wayscriber-0.9.23.tar.gz::https://github.com/devmobasa/wayscriber/archive/refs/tags/v0.9.23.tar.gz sha256sums = SKIP diff --git a/packaging/PKGBUILD b/packaging/PKGBUILD index 6b01e7c8..1b4f5e8b 100644 --- a/packaging/PKGBUILD +++ b/packaging/PKGBUILD @@ -20,6 +20,10 @@ depends=( 'grim' 'slurp' ) +optdepends=( + 'tesseract: copy text from screen (OCR)' + 'tesseract-data-eng: English language data for OCR' +) makedepends=( 'cargo' ) diff --git a/packaging/package.wayscriber.yaml b/packaging/package.wayscriber.yaml index 68969816..2dc799d5 100644 --- a/packaging/package.wayscriber.yaml +++ b/packaging/package.wayscriber.yaml @@ -122,6 +122,10 @@ overrides: - wl-clipboard - grim - slurp + # Screen text recognition is opt-in (no default shortcut, hidden toolbar + # button), so the engine and its language data are not a hard dependency. + recommends: + - tesseract-ocr rpm: depends: # Keep the glibc floor aligned with MAX_GLIBC_VERSION in tools/package.sh. @@ -135,5 +139,8 @@ overrides: - wl-clipboard - grim - slurp + recommends: + - tesseract + - tesseract-langpack-eng rpm: arch: x86_64 diff --git a/src/AGENTS.md b/src/AGENTS.md index d42dd86c..a1db3655 100644 --- a/src/AGENTS.md +++ b/src/AGENTS.md @@ -8,7 +8,7 @@ - `src/main.rs` is a thin wrapper that returns `wayscriber::run_from_env()`. - `src/lib.rs` owns logging/error wiring and the canonical module graph; its public entry facade routes CLI outcomes through `src/app/` into `src/daemon/` or the active Wayland backend. - Reusable modules remain public for integration tests and the configurator, while runtime modules are private library implementation details. -- Major domains are `domain`, `backend`, `input`, `draw`, `ui`, `capture`, `config`, `session`, `daemon`, `canvas_export`, `paths`, `toolbar_icons`, `render_profiles`, and `runtime_capabilities`. +- Major domains are `domain`, `backend`, `input`, `draw`, `ui`, `capture`, `ocr`, `config`, `session`, `daemon`, `canvas_export`, `paths`, `toolbar_icons`, `render_profiles`, and `runtime_capabilities`. ## Invariants - Keep Wayland/runtime side effects in backend/about-window code; keep reusable data, validation, rendering helpers, and path logic in library modules. diff --git a/src/backend/wayland/backend/event_loop/capture.rs b/src/backend/wayland/backend/event_loop/capture.rs index d371f9bd..1a4528d4 100644 --- a/src/backend/wayland/backend/event_loop/capture.rs +++ b/src/backend/wayland/backend/event_loop/capture.rs @@ -92,9 +92,11 @@ pub(super) fn handle_pending_actions( state.poll_hex_copy_completion(); state.poll_text_copy_completion(); state.poll_text_paste_completion(); + state.poll_ocr_completion(); state.poll_session_file_dialog_completion(qh); state.drain_clipboard_requests(); state.handle_pending_eyedropper_toggle(); + state.handle_pending_ocr_request(); // Copy/paste-hex requests from the color picker popup's pointer release are // drained here: unlike the toolbar/key paths, that release has no other // drain site. An accepted quick-color recolor is queued from the same diff --git a/src/backend/wayland/handlers/compositor.rs b/src/backend/wayland/handlers/compositor.rs index d292d94b..cdf1b297 100644 --- a/src/backend/wayland/handlers/compositor.rs +++ b/src/backend/wayland/handlers/compositor.rs @@ -60,6 +60,7 @@ impl CompositorHandler for WaylandState { self.zoom .handle_resize(phys_w, phys_h, &mut self.input_state); self.cancel_eyedropper_if_source_missing(); + self.cancel_ocr_if_source_missing(); self.toolbar .maybe_update_scale(self.surface.current_output().as_ref(), scale); self.toolbar.mark_dirty(); @@ -170,6 +171,7 @@ impl CompositorHandler for WaylandState { self.zoom .handle_resize(phys_w, phys_h, &mut self.input_state); self.cancel_eyedropper_if_source_missing(); + self.cancel_ocr_if_source_missing(); // If freeze-on-start was requested, trigger it once the surface is configured and active. if self.pending_freeze_on_start() { diff --git a/src/backend/wayland/handlers/keyboard/mod.rs b/src/backend/wayland/handlers/keyboard/mod.rs index df370d79..98ff8c5d 100644 --- a/src/backend/wayland/handlers/keyboard/mod.rs +++ b/src/backend/wayland/handlers/keyboard/mod.rs @@ -117,6 +117,16 @@ impl KeyboardHandler for WaylandState { // Any fresh key press ends the previous auto-repeat; a repeatable one // re-arms it at the end of this handler. self.clear_key_repeat(); + if self.input_state.ocr_is_engaged() { + // Every other shortcut is swallowed while the selector is up so a + // key cannot change the active tool mid-drag. + if matches!(key, Key::Escape) + || self.input_state.action_for_key(key) == Some(Action::CopyTextFromScreen) + { + self.cancel_ocr(); + } + return; + } if self.input_state.eyedropper_is_engaged() { if matches!(key, Key::Escape) || self.input_state.action_for_key(key) == Some(Action::PickScreenColor) diff --git a/src/backend/wayland/handlers/layer.rs b/src/backend/wayland/handlers/layer.rs index 996c81d7..e98b1b0d 100644 --- a/src/backend/wayland/handlers/layer.rs +++ b/src/backend/wayland/handlers/layer.rs @@ -69,6 +69,7 @@ impl LayerShellHandler for WaylandState { self.zoom .handle_resize(phys_w, phys_h, &mut self.input_state); self.cancel_eyedropper_if_source_missing(); + self.cancel_ocr_if_source_missing(); // Refresh active geometry for portal fallback cropping using latest logical size/scale. let output_transform = self @@ -106,6 +107,7 @@ impl LayerShellHandler for WaylandState { self.zoom .handle_resize(phys_w, phys_h, &mut self.input_state); self.cancel_eyedropper_if_source_missing(); + self.cancel_ocr_if_source_missing(); // Re-apply toolbar offsets now that we have a configured surface size; avoids clamping to 0 // on startup before the compositor provides dimensions. diff --git a/src/backend/wayland/handlers/pointer/cursor.rs b/src/backend/wayland/handlers/pointer/cursor.rs index e388ba1e..d6254b7a 100644 --- a/src/backend/wayland/handlers/pointer/cursor.rs +++ b/src/backend/wayland/handlers/pointer/cursor.rs @@ -38,7 +38,7 @@ impl WaylandState { /// Computes the appropriate cursor icon based on current context. fn compute_cursor_icon(&mut self, toolbar_hover: bool) -> CursorIcon { - if self.input_state.eyedropper_is_active() && !toolbar_hover { + if self.input_state.screen_modal_is_active() && !toolbar_hover { return CursorIcon::Crosshair; } diff --git a/src/backend/wayland/handlers/pointer/mod.rs b/src/backend/wayland/handlers/pointer/mod.rs index 148f4aa3..afdeea98 100644 --- a/src/backend/wayland/handlers/pointer/mod.rs +++ b/src/backend/wayland/handlers/pointer/mod.rs @@ -4,6 +4,7 @@ use smithay_client_toolkit::seat::pointer::{PointerEvent, PointerEventKind, Poin use wayland_client::{Connection, QueueHandle, protocol::wl_pointer}; use crate::backend::wayland::state::{debug_toolbar_drag_logging_enabled, surface_id}; +use crate::input::state::OcrInputSource; use super::super::state::WaylandState; diff --git a/src/backend/wayland/handlers/pointer/motion.rs b/src/backend/wayland/handlers/pointer/motion.rs index 8b594447..792f9adf 100644 --- a/src/backend/wayland/handlers/pointer/motion.rs +++ b/src/backend/wayland/handlers/pointer/motion.rs @@ -16,6 +16,20 @@ impl WaylandState { on_toolbar: bool, inline_active: bool, ) { + if self.input_state.ocr_is_active() { + let screen_position = if on_toolbar { + self.toolbar_surface_screen_coords(&event.surface, event.position) + } else { + Some(event.position) + }; + if let Some((x, y)) = screen_position { + self.set_current_mouse(x.round() as i32, y.round() as i32); + self.update_ocr_selection(OcrInputSource::Pointer, x, y); + } + self.update_pointer_cursor(on_toolbar || self.pointer_over_toolbar(), conn); + return; + } + if self.input_state.eyedropper_is_active() { let inline_hover = !on_toolbar && self.inline_toolbars_active() diff --git a/src/backend/wayland/handlers/pointer/press.rs b/src/backend/wayland/handlers/pointer/press.rs index 6643cbee..ada2f717 100644 --- a/src/backend/wayland/handlers/pointer/press.rs +++ b/src/backend/wayland/handlers/pointer/press.rs @@ -38,6 +38,30 @@ impl WaylandState { .clear_help_overlay_press_for(help_press_source); } + if self.input_state.ocr_is_active() { + if on_toolbar || self.pointer_over_toolbar() { + // A toolbar interaction ends the region first, then runs + // normally; the click never lands on the selector. + self.cancel_ocr_for_toolbar_interaction(); + } else { + match button { + BTN_LEFT => { + self.begin_ocr_selection( + OcrInputSource::Pointer, + event.position.0, + event.position.1, + ); + } + BTN_RIGHT => { + self.cancel_ocr(); + self.set_suppress_next_release(true); + } + _ => {} + } + return; + } + } + if self.input_state.eyedropper_is_active() { if on_toolbar || self.pointer_over_toolbar() { self.cancel_eyedropper(); diff --git a/src/backend/wayland/handlers/pointer/release.rs b/src/backend/wayland/handlers/pointer/release.rs index e70ebebc..c3c61984 100644 --- a/src/backend/wayland/handlers/pointer/release.rs +++ b/src/backend/wayland/handlers/pointer/release.rs @@ -16,6 +16,20 @@ impl WaylandState { inline_active: bool, button: u32, ) { + if self.input_state.ocr_is_active() { + if button == BTN_LEFT { + let screen_position = if on_toolbar { + self.toolbar_surface_screen_coords(&event.surface, event.position) + } else { + Some(event.position) + }; + if let Some((x, y)) = screen_position { + self.finish_ocr_selection(OcrInputSource::Pointer, x, y); + } + } + return; + } + if self.input_state.eyedropper_is_active() { return; } diff --git a/src/backend/wayland/handlers/tablet/frame.rs b/src/backend/wayland/handlers/tablet/frame.rs index 08c88bea..00f6421b 100644 --- a/src/backend/wayland/handlers/tablet/frame.rs +++ b/src/backend/wayland/handlers/tablet/frame.rs @@ -10,7 +10,13 @@ const BTN_STYLUS: u32 = 331; const BTN_STYLUS2: u32 = 332; fn modal_blocks_stylus_barrel_actions(input_state: &crate::input::InputState) -> bool { - input_state.show_help || input_state.tour_active + input_state.show_help + || input_state.tour_active + // A screen-region modal owns pointer and keyboard input while it is up. + // A barrel button is bound to an ordinary action — undo, clear canvas, + // the radial menu — so letting one through would run it on the canvas + // behind the selector. + || input_state.screen_modal_is_engaged() } impl WaylandState { @@ -159,6 +165,12 @@ impl WaylandState { .clear_help_overlay_press_for(HelpOverlayPressSource::Stylus); } + if self.input_state.ocr_is_active() { + let (x, y) = self.current_stylus_position(); + self.begin_ocr_selection(crate::input::state::OcrInputSource::Stylus, x, y); + return; + } + if self.input_state.eyedropper_is_active() { let (x, y) = self.current_stylus_position(); self.sample_eyedropper(x, y); @@ -313,4 +325,29 @@ mod tests { state.tour_active = true; assert!(modal_blocks_stylus_barrel_actions(&state)); } + + /// Both screen-region modals swallow pointer and keyboard input, so a + /// barrel button must not run its bound action on the canvas behind them — + /// including while they are still waiting on a capture. + #[test] + fn screen_region_modals_block_stylus_barrel_actions() { + use crate::input::state::{EyedropperCaptureSource, OcrCaptureSource}; + + let mut state = make_test_input_state(); + assert!(!modal_blocks_stylus_barrel_actions(&state)); + + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + assert!(modal_blocks_stylus_barrel_actions(&state)); + state.activate_ocr(true); + assert!(modal_blocks_stylus_barrel_actions(&state)); + state.cancel_ocr(); + assert!(!modal_blocks_stylus_barrel_actions(&state)); + + state.set_eyedropper_pending_capture(EyedropperCaptureSource::Frozen); + assert!(modal_blocks_stylus_barrel_actions(&state)); + state.activate_eyedropper(true); + assert!(modal_blocks_stylus_barrel_actions(&state)); + state.cancel_eyedropper(); + assert!(!modal_blocks_stylus_barrel_actions(&state)); + } } diff --git a/src/backend/wayland/handlers/tablet/tool.rs b/src/backend/wayland/handlers/tablet/tool.rs index c44b85e2..1c276957 100644 --- a/src/backend/wayland/handlers/tablet/tool.rs +++ b/src/backend/wayland/handlers/tablet/tool.rs @@ -9,9 +9,34 @@ use crate::{ }; use crate::backend::wayland::state::WaylandState; +use crate::input::state::OcrInputSource; const STYLUS_CURSOR_DAMAGE_RADIUS: i32 = 64; +/// Whether a pressure sample must be dropped rather than queued for the next +/// tablet frame. +/// +/// A committed pressure sample runs `set_pressure_thickness_for_active_tool`, +/// so a sample that should not have reached the canvas resizes the drawing tool +/// — the one thing OCR promises not to touch. Three independent reasons to drop +/// one, none of which subsumes the others: +/// +/// * the pen is not over the overlay, so the sample is not ours to apply; +/// * the contact was disowned when a screen modal took over, and is only still +/// arriving because clearing our flags does not lift the pen; +/// * a screen modal is on screen and owns the pen outright. +/// +/// The modal term follows the *active* boundary, which keeps this an exact twin +/// of the tip and motion guards: a stroke begun while a capture is still pending +/// is a real stroke and keeps its pressure. The retired term is what stops that +/// allowance from also readmitting the contact the modal already took over. +fn drop_stylus_pressure( + on_overlay: bool, + contact_retired: bool, + input_state: &crate::input::InputState, +) -> bool { + !on_overlay || contact_retired || input_state.screen_modal_is_active() +} impl WaylandState { fn stylus_hover_cursor_pos(&self) -> Option<(f64, f64)> { self.stylus_hover_cursor_position() @@ -44,7 +69,7 @@ impl WaylandState { || (self.inline_toolbars_active() && self.toolbar.is_visible()) } - pub(super) fn mark_stylus_hover_cursor_dirty( + pub(in crate::backend::wayland) fn mark_stylus_hover_cursor_dirty( &mut self, previous: Option<(f64, f64)>, next: Option<(f64, f64)>, @@ -163,6 +188,15 @@ impl Dispatch for WaylandState { tool_id, tool_type ); state.commit_pending_stylus_frame(); + // The tip is gone and no Up is coming, so a region drag *this + // pen* started must end here rather than keeping the selector — + // and any OCR-owned freeze — alive until the user cancels by + // hand. A region the mouse or a finger is dragging is not ours + // to withdraw. + state.cancel_ocr_selection_from(OcrInputSource::Stylus); + // The pen is gone, so the tip-up the latch was waiting for is + // never coming; leaving it armed would swallow a later contact. + state.take_retired_stylus_contact(); let hover_cursor_pos = state.stylus_hover_cursor_pos(); state.stylus_tip_down = false; state.stylus_on_overlay = false; @@ -196,6 +230,28 @@ impl Dispatch for WaylandState { // Note: We keep the tool type in the map - tools persist across proximity events } Event::Down { .. } => { + // A contact a screen modal disowned is not the user pressing — + // it is the pen that was already down, still reported until it + // lifts. Swallow it whole rather than declining one branch: + // falling through queues the tip-down, and the frame commit + // would start a region (or a stroke) from it one hop later. + // + // Protocol-unreachable today, since the latch only clears on a + // tip-up or proximity-out and a press must follow one of those. + // It is enforced rather than argued because the surrounding + // pressure, motion, and release guards all rely on it. + if state.stylus_contact_retired { + return; + } + if state.input_state.ocr_is_active() { + if state.stylus_on_toolbar { + state.cancel_ocr_for_toolbar_interaction(); + } else if state.stylus_on_overlay { + let (x, y) = state.current_or_pending_stylus_position(); + state.begin_ocr_selection(OcrInputSource::Stylus, x, y); + return; + } + } if state.input_state.eyedropper_is_active() { if state.stylus_on_toolbar { state.cancel_eyedropper(); @@ -235,6 +291,24 @@ impl Dispatch for WaylandState { state.queue_stylus_down(); } Event::Up => { + // Read before the modal branches so the latch is consumed by + // the tip-up that actually ends the disowned contact, whatever + // else this Up does. + let retired_contact = state.take_retired_stylus_contact(); + if state.input_state.ocr_is_active() { + if state.stylus_on_overlay { + let (x, y) = state.current_or_pending_stylus_position(); + // A no-op unless this pen owns the region, so a retired + // contact lifting cannot submit one the mouse or a + // finger is still drawing. + state.finish_ocr_selection(OcrInputSource::Stylus, x, y); + } else { + // The tip left the overlay before lifting; there is no + // region of ours to submit, so withdraw only our own. + state.cancel_ocr_selection_from(OcrInputSource::Stylus); + } + return; + } let inline_active = state.inline_toolbars_active() && state.toolbar.is_visible(); if inline_active && state.stylus_on_toolbar { let (mx, my) = state.current_mouse(); @@ -250,12 +324,21 @@ impl Dispatch for WaylandState { state.end_toolbar_move_drag(); return; } - if !state.stylus_on_overlay { + if !state.stylus_on_overlay || retired_contact { return; } state.queue_stylus_up(); } Event::Motion { x, y } => { + if state.input_state.ocr_is_active() && state.stylus_on_overlay { + state.stylus_last_pos = Some((x, y)); + state.set_current_mouse(x.round() as i32, y.round() as i32); + // Dropped unless this pen owns the region: a pen hovering + // over the overlay, or one whose contact was retired, must + // not drag somebody else's selection around. + state.update_ocr_selection(OcrInputSource::Stylus, x, y); + return; + } if state.input_state.eyedropper_is_active() && state.stylus_on_overlay { state.stylus_last_pos = Some((x, y)); state.set_current_mouse(x.round() as i32, y.round() as i32); @@ -325,7 +408,11 @@ impl Dispatch for WaylandState { state.queue_stylus_motion(x, y); } Event::Pressure { pressure } => { - if !state.stylus_on_overlay { + if drop_stylus_pressure( + state.stylus_on_overlay, + state.stylus_contact_retired, + &state.input_state, + ) { return; } state.queue_stylus_pressure(pressure); @@ -371,6 +458,65 @@ impl Dispatch for WaylandState { mod tests { use super::*; + /// The modal term stops exactly where the tip and motion guards stop: once + /// the selector is on screen. While a capture is still pending the canvas + /// keeps the pen, so a stroke drawn then must keep its pressure — otherwise + /// a capture that fails would leave a silently flat stroke behind. + #[test] + fn stylus_pressure_stops_at_the_selector_not_at_the_request() { + use crate::input::state::test_support::make_test_input_state; + use crate::input::state::{EyedropperCaptureSource, OcrCaptureSource}; + + let fresh_contact = |state: &_| drop_stylus_pressure(true, false, state); + + let mut state = make_test_input_state(); + assert!(!fresh_contact(&state)); + + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + assert!( + !fresh_contact(&state), + "a stroke drawn while the capture is pending is still a real stroke" + ); + state.activate_ocr(true); + assert!(fresh_contact(&state)); + state.start_ocr_selection(OcrInputSource::Stylus, (10.0, 10.0)); + assert!(fresh_contact(&state)); + state.cancel_ocr(); + assert!(!fresh_contact(&state)); + + state.set_eyedropper_pending_capture(EyedropperCaptureSource::Frozen); + assert!(!fresh_contact(&state)); + state.activate_eyedropper(true); + assert!(fresh_contact(&state)); + state.cancel_eyedropper(); + assert!(!fresh_contact(&state)); + } + + /// A contact disowned by a modal keeps arriving until the pen lifts, and the + /// pending-capture allowance above must not readmit it: the canvas holds the + /// pen again during the wait, but this contact is not the user drawing. + #[test] + fn a_retired_contact_never_reaches_the_tool_whatever_the_modal_is_doing() { + use crate::input::state::OcrCaptureSource; + use crate::input::state::test_support::make_test_input_state; + + let mut state = make_test_input_state(); + + // The window the previous test allows, and the one that matters here. + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + assert!(!drop_stylus_pressure(true, false, &state)); + assert!(drop_stylus_pressure(true, true, &state)); + + // And it outlives the modal: cancelling before activation leaves the pen + // physically down with no modal to blame. + state.cancel_ocr(); + assert!(!drop_stylus_pressure(true, false, &state)); + assert!(drop_stylus_pressure(true, true, &state)); + + // Off the overlay nothing is ours either way. + assert!(drop_stylus_pressure(false, false, &state)); + } + #[test] fn stylus_cursor_damage_rect_covers_cursor_area() { let rect = stylus_cursor_damage_rect((100.2, 80.7), 400, 300).expect("rect"); diff --git a/src/backend/wayland/handlers/touch.rs b/src/backend/wayland/handlers/touch.rs index 99375e78..3818034c 100644 --- a/src/backend/wayland/handlers/touch.rs +++ b/src/backend/wayland/handlers/touch.rs @@ -10,7 +10,7 @@ use crate::backend::wayland::state::{ }; use crate::backend::wayland::toolbar_intent::intent_to_event; use crate::input::MouseButton; -use crate::input::state::HelpOverlayPressSource; +use crate::input::state::{HelpOverlayPressSource, OcrInputSource}; use crate::ui::ZoomChipPress; impl TouchHandler for WaylandState { @@ -114,6 +114,11 @@ impl WaylandState { } let target = self.active_touch.target(); + // A cancelled sequence never produces a release, so a region drag it + // started has to end here — otherwise the selector, and any freeze it + // owns, would outlive the touch that opened it. A region another device + // is dragging is untouched. + self.cancel_ocr_selection_from(OcrInputSource::Touch); self.set_pending_toast_press(None); self.set_pending_status_hud_press(false); self.set_pending_zoom_chip_press(ZoomChipPress::None); @@ -204,6 +209,26 @@ impl WaylandState { .clear_help_overlay_press_for(HelpOverlayPressSource::Touch); } + if self.input_state.ocr_is_active() { + let inline_active = self.inline_toolbars_active() && self.toolbar.is_visible(); + let inline_hit = target == TouchTarget::Overlay + && inline_active + && self.inline_toolbar_motion(screen_position); + if target == TouchTarget::Toolbar || inline_hit { + self.cancel_ocr_for_toolbar_interaction(); + } else if target == TouchTarget::Overlay { + self.begin_ocr_selection( + OcrInputSource::Touch, + screen_position.0, + screen_position.1, + ); + // Unlike the one-shot eyedropper sample, an OCR region is a + // drag: report the real target so motion and release still + // resolve to screen coordinates and reach the selector. + return TouchTarget::Overlay; + } + } + if self.input_state.eyedropper_is_active() { let inline_active = self.inline_toolbars_active() && self.toolbar.is_visible(); let inline_hit = target == TouchTarget::Overlay @@ -333,6 +358,17 @@ impl WaylandState { let screen_y = screen_position.1.round() as i32; self.set_current_mouse(screen_x, screen_y); + if self.input_state.ocr_is_active() { + if target == TouchTarget::Overlay { + self.update_ocr_selection( + OcrInputSource::Touch, + screen_position.0, + screen_position.1, + ); + } + return; + } + if self.input_state.eyedropper_is_active() { if target == TouchTarget::Overlay { self.update_eyedropper_hover(screen_position.0, screen_position.1); @@ -412,6 +448,15 @@ impl WaylandState { position: (f64, f64), target: TouchTarget, ) { + if self.input_state.ocr_is_active() { + if let Some((x, y)) = self.touch_screen_position(surface, position, target) { + self.finish_ocr_selection(OcrInputSource::Touch, x, y); + } else { + self.cancel_ocr_selection_from(OcrInputSource::Touch); + } + return; + } + if self.take_suppress_next_release() { self.set_pending_toast_press(None); self.set_pending_status_hud_press(false); diff --git a/src/backend/wayland/handlers/xdg.rs b/src/backend/wayland/handlers/xdg.rs index f6e1d474..c1bd64c2 100644 --- a/src/backend/wayland/handlers/xdg.rs +++ b/src/backend/wayland/handlers/xdg.rs @@ -121,6 +121,7 @@ impl WindowHandler for WaylandState { self.zoom .handle_resize(phys_w, phys_h, &mut self.input_state); self.cancel_eyedropper_if_source_missing(); + self.cancel_ocr_if_source_missing(); let output_transform = self .surface diff --git a/src/backend/wayland/state.rs b/src/backend/wayland/state.rs index 6c779b8a..d5c3744f 100644 --- a/src/backend/wayland/state.rs +++ b/src/backend/wayland/state.rs @@ -108,10 +108,12 @@ mod input_actions; mod input_hud; mod keybindings; pub(in crate::backend::wayland) use keybindings::queue_keybinding_edit; +mod ocr; mod onboarding; mod pdf_export; mod perf; mod render; +mod screen_image; mod text_clipboard; mod toolbar; #[cfg(feature = "toolbar-gtk")] @@ -263,6 +265,9 @@ pub(super) struct WaylandState { /// the current edit generation remain distinct; a new generation replaces /// stale queued requests from the old edit session. pub(super) pending_text_paste: VecDeque, + /// Capacity-one screen text recognition. A busy controller reports + /// busy rather than queuing a region the user has moved on from. + pub(super) ocr: crate::ocr::OcrController, /// GTK toolbar frontend; `None` means the built-in bars are in charge. pub(super) gtk_toolbar: Option, pub(super) onboarding: crate::onboarding::OnboardingStore, @@ -361,6 +366,13 @@ pub(super) struct WaylandState { pub(super) stylus_peak_thickness: Option, #[cfg(feature = "tablet-input")] pub(super) pending_stylus_frame: PendingStylusFrame, + /// The pen is still physically down, but the contact was disowned when a + /// screen-region modal took over. Wayscriber's own flags were cleared, yet + /// the compositor keeps reporting this contact until the tip lifts — so it + /// is tracked separately and ignored until then, rather than being mistaken + /// for a fresh press made during the wait. + #[cfg(feature = "tablet-input")] + pub(super) stylus_contact_retired: bool, /// Map of tool object IDs to their physical types (pen, eraser, etc.) #[cfg(feature = "tablet-input")] pub(super) stylus_tool_types: std::collections::HashMap< diff --git a/src/backend/wayland/state/capture.rs b/src/backend/wayland/state/capture.rs index a5b88550..dc6bcad9 100644 --- a/src/backend/wayland/state/capture.rs +++ b/src/backend/wayland/state/capture.rs @@ -43,6 +43,7 @@ impl WaylandState { self.finish_pending_eyedropper_capture( crate::input::state::EyedropperCaptureSource::Frozen, ); + self.finish_pending_ocr_capture(crate::input::state::OcrCaptureSource::Frozen); } if self.zoom.take_capture_done() { log::info!( @@ -54,6 +55,7 @@ impl WaylandState { self.finish_pending_eyedropper_capture( crate::input::state::EyedropperCaptureSource::Zoom, ); + self.finish_pending_ocr_capture(crate::input::state::OcrCaptureSource::Zoom); } } diff --git a/src/backend/wayland/state/color_picker.rs b/src/backend/wayland/state/color_picker.rs index 013b079d..bb91b38a 100644 --- a/src/backend/wayland/state/color_picker.rs +++ b/src/backend/wayland/state/color_picker.rs @@ -2,10 +2,12 @@ use super::{ClipboardOperationController, WaylandState}; use crate::backend::wayland::clipboard::ClipboardPoll; +use crate::clipboard_text::{ + ClipboardTextError, copy_text_via_command, read_clipboard_text_via_command, +}; use crate::draw::Color; use crate::input::state::{HexPasteTarget, Toast, ToastPriority}; use crate::input::state::{color_to_hex, parse_hex_color}; -use std::ffi::OsStr; use std::time::Duration; impl WaylandState { @@ -204,80 +206,6 @@ pub(super) fn submit_pending_clipboard_copy_if_idle( start_clipboard_copy(controller, hex, operation) } -pub(super) enum ClipboardTextError { - Empty, - Other(String), -} - -pub(super) fn copy_text_via_command(text: &str) -> Result<(), String> { - let output = crate::process_broker::current() - .and_then(|broker| { - broker.publish( - crate::process_broker::HelperKind::WlCopy, - OsStr::new("wl-copy"), - [OsStr::new("--type"), OsStr::new("text/plain;charset=utf-8")], - text.as_bytes().to_vec(), - Duration::from_secs(5), - ) - }) - .map_err(|error| format!("Failed to run wl-copy: {error:#}"))?; - if output.timed_out { - return Err("wl-copy timed out".to_string()); - } - if output.status != 0 { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - return if stderr.is_empty() { - Err("wl-copy exited unsuccessfully".to_string()) - } else { - Err(format!("wl-copy exited unsuccessfully: {stderr}")) - }; - } - Ok(()) -} - -pub(super) fn read_clipboard_text_via_command() -> Result { - let output = crate::process_broker::current() - .and_then(|broker| { - broker.run( - crate::process_broker::HelperKind::WlPaste, - OsStr::new("wl-paste"), - clipboard_text_read_args(), - Vec::new(), - Duration::from_secs(5), - 1024 * 1024, - ) - }) - .map_err(|err| ClipboardTextError::Other(format!("Failed to run wl-paste: {err:#}")))?; - - if !output.timed_out && output.status == 0 { - Ok(String::from_utf8_lossy(&output.stdout).to_string()) - } else { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - if stderr.to_ascii_lowercase().contains("nothing is copied") - || stderr.to_ascii_lowercase().contains("clipboard is empty") - { - Err(ClipboardTextError::Empty) - } else if stderr.is_empty() { - Err(ClipboardTextError::Other( - "wl-paste exited unsuccessfully".to_string(), - )) - } else { - Err(ClipboardTextError::Other(format!( - "wl-paste exited unsuccessfully: {}", - stderr - ))) - } - } -} - -fn clipboard_text_read_args() -> [&'static OsStr; 3] { - [ - OsStr::new("--no-newline"), - OsStr::new("--type"), - OsStr::new("text"), - ] -} - #[cfg(test)] mod tests { use std::sync::mpsc; @@ -287,18 +215,6 @@ mod tests { use crate::backend::wayland::RuntimeWakeSource; use crate::backend::wayland::clipboard::ClipboardOperationIdSource; - #[test] - fn clipboard_text_read_requests_only_a_text_mime() { - assert_eq!( - clipboard_text_read_args(), - [ - OsStr::new("--no-newline"), - OsStr::new("--type"), - OsStr::new("text"), - ] - ); - } - #[test] fn hex_copy_submission_stays_off_the_event_thread_until_completion() { let wake = RuntimeWakeSource::new().unwrap(); diff --git a/src/backend/wayland/state/core/accessors.rs b/src/backend/wayland/state/core/accessors.rs index 3c794b67..edf915a4 100644 --- a/src/backend/wayland/state/core/accessors.rs +++ b/src/backend/wayland/state/core/accessors.rs @@ -71,6 +71,41 @@ impl WaylandState { } } + /// Retire in-flight stylus input without dispatching a canvas release. + /// + /// A screen-region modal cancels the gesture the pen was making, and then + /// consumes the tip-up the compositor still owes us — so nothing else + /// retires the contact. Left set, `stylus_tip_down` hides the hover cursor + /// and defers session saving until some unrelated release or proximity-out + /// happens to clear it. The stroke was cancelled rather than finished, so + /// its peak pressure is dropped instead of being committed to the tool. + #[cfg(feature = "tablet-input")] + pub(in crate::backend::wayland) fn retire_stylus_contact(&mut self) { + // Tablet events are coalesced until their frame commits, so input that + // physically happened before the modal opened can still be sitting in + // the buffer. Replaying it afterwards would start a region from a press + // that predates the selector, or apply that batch's pressure and barrel + // actions behind it. Dropping the batch is unconditional: an uncommitted + // tip-down is exactly the case where no contact is logically held yet. + let had_contact = self.stylus_tip_down || self.pending_stylus_frame.down; + self.pending_stylus_frame = Default::default(); + // Clearing our own flags does not lift the pen. The compositor keeps + // reporting this contact — pressure included — until the tip rises, and + // a pending capture leaves the canvas holding the pen, so without this + // the disowned stroke would go on resizing the tool and would commit its + // peak on the way up. + self.stylus_contact_retired |= had_contact; + if !self.stylus_tip_down { + return; + } + let previous_hover = self.stylus_hover_cursor_position(); + self.stylus_tip_down = false; + self.stylus_pressure_thickness = None; + self.stylus_peak_thickness = None; + let next_hover = self.stylus_hover_cursor_position(); + self.mark_stylus_hover_cursor_dirty(previous_hover, next_hover); + } + #[cfg(not(feature = "tablet-input"))] pub(in crate::backend::wayland) fn stylus_hover_cursor_visible(&self) -> bool { false @@ -81,6 +116,19 @@ impl WaylandState { None } + /// Consume the disowned-contact latch, reporting whether the tip that just + /// lifted belonged to a contact a screen modal had taken over. + /// + /// A disowned tip-up dispatches no canvas release and commits no thickness; + /// the next press after it is an ordinary fresh contact. + #[cfg(feature = "tablet-input")] + pub(in crate::backend::wayland) fn take_retired_stylus_contact(&mut self) -> bool { + std::mem::take(&mut self.stylus_contact_retired) + } + + #[cfg(not(feature = "tablet-input"))] + pub(in crate::backend::wayland) fn retire_stylus_contact(&mut self) {} + pub(in crate::backend::wayland) fn set_pointer_focus(&mut self, value: bool) { self.data.has_pointer_focus = value; } diff --git a/src/backend/wayland/state/core/init.rs b/src/backend/wayland/state/core/init.rs index 05cc8cc0..252f08ce 100644 --- a/src/backend/wayland/state/core/init.rs +++ b/src/backend/wayland/state/core/init.rs @@ -120,6 +120,7 @@ impl WaylandState { ); let clipboard_text_paste = ClipboardOperationController::new(clipboard_operation_ids, runtime_wake.clone()); + let ocr = crate::ocr::OcrController::new(runtime_wake.clone()); Self { registry_state, @@ -152,6 +153,7 @@ impl WaylandState { pending_text_copy: Default::default(), clipboard_text_paste, pending_text_paste: Default::default(), + ocr, gtk_toolbar: None, onboarding, config_edits: super::super::super::config_edits::ConfigEditWorker::new( @@ -227,6 +229,8 @@ impl WaylandState { #[cfg(feature = "tablet-input")] pending_stylus_frame: crate::backend::wayland::state::PendingStylusFrame::default(), #[cfg(feature = "tablet-input")] + stylus_contact_retired: false, + #[cfg(feature = "tablet-input")] stylus_tool_types: std::collections::HashMap::new(), #[cfg(feature = "tablet-input")] stylus_auto_switched_to_eraser: false, diff --git a/src/backend/wayland/state/eyedropper.rs b/src/backend/wayland/state/eyedropper.rs index 87a68581..b01c0d8d 100644 --- a/src/backend/wayland/state/eyedropper.rs +++ b/src/backend/wayland/state/eyedropper.rs @@ -4,88 +4,10 @@ use crate::input::state::EyedropperCaptureSource; use crate::input::state::{Toast, ToastPriority}; use super::WaylandState; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum BackgroundImageKind { - Zoom, - Frozen, -} - -/// The captured image that the canvas renderer actually displays. -pub(super) struct BackgroundImageSource<'a> { - pub image: &'a FrozenImage, - pub kind: BackgroundImageKind, - pub zoom_transformed: bool, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum EyedropperEntryDecision { - Activate, - WaitForZoom, - AutoFreeze, - RefuseWhileZoomedOnSolidBoard, - RefuseSolidBoard, - CaptureUnavailable, - ZoomImageUnavailable, -} - -fn eyedropper_entry_decision( - has_source: bool, - board_is_transparent: bool, - zoom_engaged: bool, - zoom_active: bool, - frozen_enabled: bool, -) -> EyedropperEntryDecision { - if has_source { - EyedropperEntryDecision::Activate - } else if !board_is_transparent && zoom_engaged { - EyedropperEntryDecision::RefuseWhileZoomedOnSolidBoard - } else if !board_is_transparent { - EyedropperEntryDecision::RefuseSolidBoard - } else if zoom_engaged && !zoom_active { - EyedropperEntryDecision::WaitForZoom - } else if !frozen_enabled { - EyedropperEntryDecision::CaptureUnavailable - } else if zoom_active { - EyedropperEntryDecision::ZoomImageUnavailable - } else { - EyedropperEntryDecision::AutoFreeze - } -} - -pub(super) fn background_image_source<'a>( - zoom: &'a crate::backend::wayland::zoom::ZoomState, - frozen: &'a crate::backend::wayland::frozen::FrozenState, - board_is_transparent: bool, -) -> Option> { - let allow_background_image = !zoom.is_engaged() || board_is_transparent; - if !allow_background_image { - return None; - } - - if zoom.active { - if let Some(image) = zoom.image() { - return Some(BackgroundImageSource { - image, - kind: BackgroundImageKind::Zoom, - zoom_transformed: true, - }); - } - if let Some(image) = frozen.image() { - return Some(BackgroundImageSource { - image, - kind: BackgroundImageKind::Frozen, - zoom_transformed: true, - }); - } - } - - frozen.image().map(|image| BackgroundImageSource { - image, - kind: BackgroundImageKind::Frozen, - zoom_transformed: false, - }) -} +use super::screen_image::{ + DisplayedScreenImage, ScreenSourceEntry, displayed_screen_image, image_point_for_screen_point, + screen_source_entry, +}; fn sample_at(image: &FrozenImage, image_x: f64, image_y: f64) -> Option { if image.width == 0 || image.height == 0 || image.stride <= 0 { @@ -127,7 +49,10 @@ impl WaylandState { return; } - self.input_state.prepare_for_eyedropper(); + // The two screen modals are mutually exclusive; entering one ends the + // other, including any temporary freeze that one owned. + self.cancel_ocr(); + self.input_state.prepare_for_screen_modal(); self.zoom.stop_pan(); self.stop_board_pan(); self.set_board_pan_key_held(false); @@ -135,8 +60,12 @@ impl WaylandState { // toolbar move; it is not an accepted drop. self.cancel_toolbar_move_drag(); self.unlock_pointer(); + // Same as OCR: the cancelled gesture may belong to a pen that is still + // down, and the pending branches below do not activate, so a tip-up + // during the wait would commit that stroke's peak pressure to the tool. + self.retire_stylus_contact(); - let decision = eyedropper_entry_decision( + let decision = screen_source_entry( self.background_image_source().is_some(), self.input_state.board_is_transparent(), self.zoom.is_engaged(), @@ -144,16 +73,16 @@ impl WaylandState { self.frozen_enabled(), ); match decision { - EyedropperEntryDecision::Activate => self.input_state.activate_eyedropper(false), - EyedropperEntryDecision::WaitForZoom => self + ScreenSourceEntry::Activate => self.activate_eyedropper_sampler(false), + ScreenSourceEntry::WaitForZoom => self .input_state .set_eyedropper_pending_capture(EyedropperCaptureSource::Zoom), - EyedropperEntryDecision::AutoFreeze => { + ScreenSourceEntry::AutoFreeze => { self.input_state .set_eyedropper_pending_capture(EyedropperCaptureSource::Frozen); self.input_state.request_frozen_toggle(); } - EyedropperEntryDecision::RefuseWhileZoomedOnSolidBoard => { + ScreenSourceEntry::RefuseWhileZoomedOnSolidBoard => { self.input_state.push_toast( ToastPriority::Action, "eyedropper", @@ -164,10 +93,10 @@ impl WaylandState { ), ); } - EyedropperEntryDecision::RefuseSolidBoard => { + ScreenSourceEntry::RefuseSolidBoard => { self.input_state.push_toast(ToastPriority::Action, "eyedropper", Toast::info("Screen eyedropper requires a transparent board or an active screen freeze.").action("Switch to transparent", crate::config::Action::ReturnToTransparent)); } - EyedropperEntryDecision::CaptureUnavailable => { + ScreenSourceEntry::CaptureUnavailable => { self.input_state.push_toast( ToastPriority::Info, "eyedropper", @@ -176,14 +105,14 @@ impl WaylandState { ), ); } - EyedropperEntryDecision::ZoomImageUnavailable => { + ScreenSourceEntry::ZoomImageUnavailable => { self.input_state.push_toast(ToastPriority::Info, "eyedropper", Toast::warning("Screen eyedropper is unavailable because zoom has no captured screen image.")); } } } - fn background_image_source(&self) -> Option> { - background_image_source( + fn background_image_source(&self) -> Option> { + displayed_screen_image( &self.zoom, &self.frozen, self.input_state.board_is_transparent(), @@ -198,8 +127,10 @@ impl WaylandState { return; } if self.background_image_source().is_some() { - self.input_state - .activate_eyedropper(matches!(capture_source, EyedropperCaptureSource::Frozen)); + self.activate_eyedropper_sampler(matches!( + capture_source, + EyedropperCaptureSource::Frozen + )); } else { self.cancel_eyedropper(); self.input_state @@ -207,6 +138,14 @@ impl WaylandState { } } + /// Arm the screen sampler. Retires an in-flight stylus contact for the same + /// reason OCR does: from here the modal swallows the tip-up that would + /// otherwise end it. + fn activate_eyedropper_sampler(&mut self, auto_froze: bool) { + self.retire_stylus_contact(); + self.input_state.activate_eyedropper(auto_froze); + } + pub(in crate::backend::wayland) fn update_eyedropper_hover(&mut self, x: f64, y: f64) { if self.input_state.eyedropper_is_active() { self.input_state.update_eyedropper_hover((x, y)); @@ -244,18 +183,15 @@ impl WaylandState { fn eyedropper_image_coords( &self, - source: &BackgroundImageSource<'_>, + source: &DisplayedScreenImage<'_>, x: f64, y: f64, ) -> (f64, f64) { - let (world_x, world_y) = if source.zoom_transformed { - self.zoom.screen_to_world(x, y) - } else { - (x, y) - }; - ( - world_x * f64::from(source.image.width) / f64::from(self.surface.width()).max(1.0), - world_y * f64::from(source.image.height) / f64::from(self.surface.height()).max(1.0), + image_point_for_screen_point( + source, + &self.zoom, + (self.surface.width(), self.surface.height()), + (x, y), ) } @@ -427,70 +363,43 @@ mod tests { assert_eq!(sample_at(&image, 20.0, 20.0).unwrap().b, 1.0); } - #[test] - fn background_source_truth_table_matches_rendering_contract() { - let mut zoom = crate::backend::wayland::zoom::ZoomState::new(None); - let mut frozen = crate::backend::wayland::frozen::FrozenState::new(None); - assert!(background_image_source(&zoom, &frozen, true).is_none()); - - frozen.set_image(image([0, 0, 255, 255])); - let source = background_image_source(&zoom, &frozen, false).unwrap(); - assert_eq!(source.kind, BackgroundImageKind::Frozen); - assert!(!source.zoom_transformed); - - zoom.active = true; - let source = background_image_source(&zoom, &frozen, true).unwrap(); - assert_eq!(source.kind, BackgroundImageKind::Frozen); - assert!(source.zoom_transformed); - assert!(background_image_source(&zoom, &frozen, false).is_none()); - - zoom.set_image(image([255, 0, 0, 255])); - let source = background_image_source(&zoom, &frozen, true).unwrap(); - assert_eq!(source.kind, BackgroundImageKind::Zoom); - assert!(source.zoom_transformed); - - zoom.active = false; - zoom.request_activation(); - assert!(background_image_source(&zoom, &frozen, false).is_none()); - } - #[test] fn entry_waits_for_pending_zoom_instead_of_starting_frozen_capture() { assert_eq!( - eyedropper_entry_decision(false, true, true, false, true), - EyedropperEntryDecision::WaitForZoom + screen_source_entry(false, true, true, false, true), + ScreenSourceEntry::WaitForZoom ); } #[test] fn entry_distinguishes_solid_board_zoom_refusal() { assert_eq!( - eyedropper_entry_decision(false, false, true, true, true), - EyedropperEntryDecision::RefuseWhileZoomedOnSolidBoard + screen_source_entry(false, false, true, true, true), + ScreenSourceEntry::RefuseWhileZoomedOnSolidBoard ); assert_eq!( - eyedropper_entry_decision(false, false, false, false, true), - EyedropperEntryDecision::RefuseSolidBoard + screen_source_entry(false, false, false, false, true), + ScreenSourceEntry::RefuseSolidBoard ); } #[test] fn entry_uses_existing_source_before_board_policy() { assert_eq!( - eyedropper_entry_decision(true, false, false, false, true), - EyedropperEntryDecision::Activate + screen_source_entry(true, false, false, false, true), + ScreenSourceEntry::Activate ); } #[test] fn entry_reports_missing_zoom_image_separately_from_missing_capture_support() { assert_eq!( - eyedropper_entry_decision(false, true, true, true, true), - EyedropperEntryDecision::ZoomImageUnavailable + screen_source_entry(false, true, true, true, true), + ScreenSourceEntry::ZoomImageUnavailable ); assert_eq!( - eyedropper_entry_decision(false, true, false, false, false), - EyedropperEntryDecision::CaptureUnavailable + screen_source_entry(false, true, false, false, false), + ScreenSourceEntry::CaptureUnavailable ); } } diff --git a/src/backend/wayland/state/ocr.rs b/src/backend/wayland/state/ocr.rs new file mode 100644 index 00000000..1ac8ee75 --- /dev/null +++ b/src/backend/wayland/state/ocr.rs @@ -0,0 +1,495 @@ +//! `Copy text from screen`: capture ownership, region selection, and outcomes. +//! +//! OCR reads the desktop image the renderer is already displaying, so it reuses +//! the eyedropper's freeze/zoom ownership rule: a freeze OCR created is a freeze +//! OCR releases, and a user-owned freeze or zoom survives untouched. Nothing +//! here mutates annotations, history, boards, sessions, or the active tool. + +use crate::input::state::{OcrCaptureSource, OcrInputSource, Toast, ToastPriority}; +use crate::ocr::{OcrFailure, OcrLanguages, OcrPoll, OcrRequest, OcrSubmitError, OcrSuccess}; + +use super::WaylandState; +use super::screen_image::{ + CropError, DisplayedScreenImage, ScreenSourceEntry, copy_image_rect, displayed_screen_image, + image_rect_for_screen_rect, screen_source_entry, +}; + +/// Drags below this many logical pixels on either axis are treated as a stray +/// click and never reach the engine. +const MIN_REGION_LOGICAL_PIXELS: f64 = 4.0; + +const TOAST_SOURCE: &str = "ocr"; + +impl WaylandState { + /// Drain a `Copy text from screen` request into the region selector. + pub(in crate::backend::wayland) fn handle_pending_ocr_request(&mut self) { + // Read unconditionally so the latch never outlives the batch that set it. + let dismissed_by_toolbar = self.input_state.take_ocr_cancelled_by_toolbar(); + if !self.input_state.take_pending_ocr_request() { + return; + } + if self.input_state.ocr_state().is_engaged() { + // A second invocation while the selector is up cancels it, matching + // the eyedropper's toggle behavior. + self.cancel_ocr(); + return; + } + if dismissed_by_toolbar { + // The toolbar path already cancelled the selector on the way in, so + // this request is the same click toggling it off. + return; + } + if !self.config.capture.enabled { + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::warning("Screen text recognition is off because capture is disabled."), + ); + return; + } + if self.ocr.is_active() { + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::info("OCR is already running"), + ); + return; + } + + // The two screen modals are mutually exclusive; entering one ends the + // other, including any temporary freeze that one owned. + self.cancel_eyedropper(); + self.input_state.prepare_for_screen_modal(); + self.zoom.stop_pan(); + self.stop_board_pan(); + self.set_board_pan_key_held(false); + self.cancel_toolbar_move_drag(); + self.unlock_pointer(); + // The gesture just cancelled above may belong to a pen that is still + // down. Activation retires the contact on its own, but the pending + // branches below do not activate: without this, a tip-up during the + // wait would commit the cancelled stroke's peak pressure to the tool. + self.retire_stylus_contact(); + + match screen_source_entry( + self.ocr_screen_source().is_some(), + self.input_state.board_is_transparent(), + self.zoom.is_engaged(), + self.zoom.active, + self.frozen_enabled(), + ) { + ScreenSourceEntry::Activate => self.activate_ocr_selector(false), + ScreenSourceEntry::WaitForZoom => { + self.input_state + .set_ocr_pending_capture(OcrCaptureSource::Zoom); + } + ScreenSourceEntry::AutoFreeze => { + self.input_state + .set_ocr_pending_capture(OcrCaptureSource::Frozen); + self.input_state.request_frozen_toggle(); + } + ScreenSourceEntry::RefuseWhileZoomedOnSolidBoard + | ScreenSourceEntry::RefuseSolidBoard => { + self.input_state.push_toast( + ToastPriority::Action, + TOAST_SOURCE, + Toast::info("OCR needs a visible screen image.").action( + "Switch to transparent", + crate::config::Action::ReturnToTransparent, + ), + ); + } + ScreenSourceEntry::CaptureUnavailable => { + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::warning( + "Screen text recognition is unavailable because screen capture is not available.", + ), + ); + } + ScreenSourceEntry::ZoomImageUnavailable => { + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::warning( + "Screen text recognition is unavailable because zoom has no captured screen image.", + ), + ); + } + } + } + + fn ocr_screen_source(&self) -> Option> { + displayed_screen_image( + &self.zoom, + &self.frozen, + self.input_state.board_is_transparent(), + ) + } + + pub(in crate::backend::wayland) fn finish_pending_ocr_capture( + &mut self, + capture_source: OcrCaptureSource, + ) { + if self.input_state.ocr_state().pending_source() != Some(capture_source) { + return; + } + if self.ocr_screen_source().is_some() { + self.activate_ocr_selector(matches!(capture_source, OcrCaptureSource::Frozen)); + } else { + self.cancel_ocr(); + self.input_state.report_ocr_capture_failure_if_unreported(); + } + } + + /// Arm the region selector. + /// + /// Activation cancels whatever gesture was in flight, and from that moment + /// the selector swallows the events that would have ended it — including a + /// stylus tip-up. The contact is retired alongside, so the two can never be + /// separated: a capture that lands seconds after the request arms the + /// selector against a gesture that started during the wait, not the one the + /// request itself cancelled. + fn activate_ocr_selector(&mut self, auto_froze: bool) { + self.retire_stylus_contact(); + self.input_state.activate_ocr(auto_froze); + } + + /// Leave OCR selection, releasing only a freeze OCR created itself. + pub(in crate::backend::wayland) fn cancel_ocr(&mut self) -> bool { + let state = self.input_state.ocr_state(); + let was_engaged = state.is_engaged(); + let pending_source = state.pending_source(); + let auto_froze = self.input_state.cancel_ocr(); + self.release_ocr_capture(auto_froze, pending_source); + was_engaged + } + + /// Give back the temporary freeze OCR created. A user-owned freeze or an + /// active zoom is never touched here. + fn release_ocr_capture(&mut self, auto_froze: bool, pending_source: Option) { + if !auto_froze { + return; + } + self.restore_xdg_after_frozen(); + if pending_source == Some(OcrCaptureSource::Frozen) && self.frozen.is_in_progress() { + self.frozen.cancel(&mut self.input_state); + self.exit_overlay_suppression(super::OverlaySuppression::Frozen); + } else { + self.frozen.unfreeze(&mut self.input_state); + } + } + + /// Dismiss the selector because a toolbar interaction took over. + /// + /// Recorded rather than just cancelled: when the interaction is a click on + /// the OCR button itself, the request that follows is the button toggling + /// off, and without this the pending handler would see `Inactive` and open a + /// fresh selector instead. + pub(in crate::backend::wayland) fn cancel_ocr_for_toolbar_interaction(&mut self) -> bool { + if !self.cancel_ocr() { + return false; + } + self.input_state.note_ocr_cancelled_by_toolbar(); + true + } + + /// A display or source change invalidates the pixels the user is selecting. + pub(in crate::backend::wayland) fn cancel_ocr_if_source_missing(&mut self) { + if self.input_state.ocr_is_active() && self.ocr_screen_source().is_none() { + self.cancel_ocr(); + } + } + + pub(in crate::backend::wayland) fn begin_ocr_selection( + &mut self, + owner: OcrInputSource, + x: f64, + y: f64, + ) -> bool { + self.input_state.start_ocr_selection(owner, (x, y)) + } + + pub(in crate::backend::wayland) fn update_ocr_selection( + &mut self, + source: OcrInputSource, + x: f64, + y: f64, + ) { + self.input_state.update_ocr_selection(source, (x, y)); + } + + /// Discard a region because the device dragging it went away — the pen left + /// proximity, the touch sequence was cancelled. Devices that are not + /// dragging have nothing to withdraw, so this leaves the selector armed for + /// whichever one is. + pub(in crate::backend::wayland) fn cancel_ocr_selection_from( + &mut self, + source: OcrInputSource, + ) -> bool { + if !self.input_state.ocr_selection_is_owned_by(source) { + return false; + } + self.cancel_ocr() + } + + /// End the drag: own the selected pixels, release an OCR-created freeze, + /// and submit. Returns whether `source` had a region drag to finish. + pub(in crate::backend::wayland) fn finish_ocr_selection( + &mut self, + source: OcrInputSource, + x: f64, + y: f64, + ) -> bool { + // A release from a device that is not dragging must not submit — or + // even see — the region another one is still drawing. + if !self.input_state.ocr_selection_is_owned_by(source) { + return false; + } + self.input_state.update_ocr_selection(source, (x, y)); + let Some(selection) = self.input_state.ocr_state().selection() else { + return false; + }; + + // The crop is taken while the capture is still held: releasing first + // would leave the worker reading pixels that no longer exist. + match self.crop_ocr_selection(selection.start, selection.end) { + Ok(None) => self.input_state.rearm_ocr_selection(), + Ok(Some(pixels)) => { + let auto_froze = self.input_state.cancel_ocr(); + self.release_ocr_capture(auto_froze, None); + self.submit_ocr_request(pixels); + } + Err(message) => { + let auto_froze = self.input_state.cancel_ocr(); + self.release_ocr_capture(auto_froze, None); + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::warning(message), + ); + } + } + true + } + + /// `Ok(None)` means the drag was too small to be a region — a stray click, + /// not a failure worth a toast. + fn crop_ocr_selection( + &self, + start: (f64, f64), + end: (f64, f64), + ) -> Result, &'static str> { + if (end.0 - start.0).abs() < MIN_REGION_LOGICAL_PIXELS + || (end.1 - start.1).abs() < MIN_REGION_LOGICAL_PIXELS + { + return Ok(None); + } + let Some(source) = self.ocr_screen_source() else { + return Err("The screen image for text recognition is no longer available."); + }; + let Some(rect) = image_rect_for_screen_rect( + &source, + &self.zoom, + (self.surface.width(), self.surface.height()), + start, + end, + ) else { + return Ok(None); + }; + copy_image_rect(source.image, rect) + .map(Some) + .map_err(|error| match error { + CropError::Empty => "That selection has no screen pixels.", + CropError::OutOfBounds => "Could not read that region of the screen image.", + }) + } + + fn submit_ocr_request(&mut self, pixels: crate::ocr::OcrPixels) { + let languages = OcrLanguages::from_validated(self.config.capture.resolved_ocr_languages()); + log::debug!( + "OCR submitting {}x{} crop for languages {}", + pixels.width, + pixels.height, + languages.as_str() + ); + match self.ocr.try_submit(OcrRequest { pixels, languages }) { + Ok(id) => { + // wl-copy needs the overlay to stay alive long enough to serve + // the selection it publishes. + self.suppress_focus_exit_for(std::time::Duration::from_millis(1500)); + log::debug!("OCR request {id} started"); + } + Err(OcrSubmitError::Busy { .. }) => { + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::info("OCR is already running"), + ); + } + Err(error) => { + log::warn!("Failed to start screen text recognition: {error}"); + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::warning("Could not start screen text recognition."), + ); + } + } + } + + /// Turn a finished recognition into one stable toast. The completion never + /// carries recognized text, so nothing here can leak screen content. + pub(in crate::backend::wayland) fn poll_ocr_completion(&mut self) { + match self.ocr.poll() { + OcrPoll::Idle | OcrPoll::Pending => {} + OcrPoll::Ready { + id, + outcome: + Ok(OcrSuccess::Copied { + character_count, + replaced_invalid_utf8, + }), + } => { + log::debug!("OCR request {id} copied {character_count} characters"); + let message = if replaced_invalid_utf8 { + "Text copied (some characters were unreadable)" + } else { + "Text copied" + }; + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::info(message), + ); + } + OcrPoll::Ready { + outcome: Ok(OcrSuccess::NoTextFound), + .. + } => { + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::info("No text found"), + ); + } + OcrPoll::Ready { + id, + outcome: Err(failure), + } => { + log::warn!("OCR request {id} failed: {failure:?}"); + let toast = match failure { + OcrFailure::EngineMissing | OcrFailure::LanguageMissing { .. } => { + Toast::warning(failure.message()) + } + _ => Toast::error(failure.message()), + }; + self.input_state + .push_toast(ToastPriority::Critical, TOAST_SOURCE, toast); + } + OcrPoll::WorkerLost { id, reason } => { + log::error!("OCR request {id} lost its worker: {reason}"); + self.input_state.push_toast( + ToastPriority::Critical, + TOAST_SOURCE, + Toast::error("Screen text recognition failed."), + ); + } + } + } + + /// Paint the region selector: a scrim over everything outside the drag and + /// scan-corner brackets around it. Never baked into the source crop — this + /// runs in the UI pass, over a copy the worker already owns. + pub(in crate::backend::wayland) fn render_ocr_selection( + &self, + ctx: &cairo::Context, + screen_width: u32, + screen_height: u32, + ) { + if !self.input_state.ocr_is_active() { + return; + } + let width = f64::from(screen_width); + let height = f64::from(screen_height); + let selection = self.input_state.ocr_state().selection(); + + let _ = ctx.save(); + ctx.set_source_rgba(0.02, 0.03, 0.05, 0.35); + ctx.rectangle(0.0, 0.0, width, height); + if let Some(selection) = selection { + let (x, y, w, h) = normalized_rect(selection.start, selection.end); + // Even-odd keeps the selected region unscrimmed, so the user reads + // the real screen pixels the crop will take. + ctx.rectangle(x, y, w, h); + ctx.set_fill_rule(cairo::FillRule::EvenOdd); + } + let _ = ctx.fill(); + ctx.set_fill_rule(cairo::FillRule::Winding); + + if let Some(selection) = selection { + let (x, y, w, h) = normalized_rect(selection.start, selection.end); + ctx.set_source_rgba(1.0, 1.0, 1.0, 0.9); + ctx.set_line_width(1.0); + ctx.rectangle(x + 0.5, y + 0.5, (w - 1.0).max(0.0), (h - 1.0).max(0.0)); + let _ = ctx.stroke(); + draw_scan_corners(ctx, x, y, w, h); + } + let _ = ctx.restore(); + } +} + +fn normalized_rect(start: (f64, f64), end: (f64, f64)) -> (f64, f64, f64, f64) { + let x = start.0.min(end.0); + let y = start.1.min(end.1); + (x, y, (end.0 - start.0).abs(), (end.1 - start.1).abs()) +} + +/// Four bracket corners, the visual language a scan frame reads as. The arm +/// length shrinks with the region so a small selection is not swallowed by it. +fn draw_scan_corners(ctx: &cairo::Context, x: f64, y: f64, w: f64, h: f64) { + let arm = (w.min(h) / 4.0).clamp(4.0, 20.0); + ctx.set_source_rgba(1.0, 1.0, 1.0, 1.0); + ctx.set_line_width(2.0); + ctx.set_line_cap(cairo::LineCap::Square); + for (corner_x, corner_y, dx, dy) in [ + (x, y, 1.0, 1.0), + (x + w, y, -1.0, 1.0), + (x, y + h, 1.0, -1.0), + (x + w, y + h, -1.0, -1.0), + ] { + ctx.move_to(corner_x + dx * arm, corner_y); + ctx.line_to(corner_x, corner_y); + ctx.line_to(corner_x, corner_y + dy * arm); + let _ = ctx.stroke(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalization_orders_any_pair_of_drag_corners() { + assert_eq!( + normalized_rect((30.0, 40.0), (10.0, 20.0)), + (10.0, 20.0, 20.0, 20.0) + ); + assert_eq!( + normalized_rect((10.0, 20.0), (30.0, 40.0)), + (10.0, 20.0, 20.0, 20.0) + ); + } + + #[test] + fn scan_corner_arms_stay_inside_a_small_region() { + // The clamp floor is 4px; the guard is that the arm never spans more + // than the region it decorates. + for size in [1.0_f64, 8.0, 40.0, 400.0] { + let arm = (size / 4.0).clamp(4.0, 20.0); + assert!(arm <= 20.0); + assert!(arm >= 4.0); + } + } +} diff --git a/src/backend/wayland/state/render/canvas/background.rs b/src/backend/wayland/state/render/canvas/background.rs index b0cc62ac..7aa5485a 100644 --- a/src/backend/wayland/state/render/canvas/background.rs +++ b/src/backend/wayland/state/render/canvas/background.rs @@ -1,5 +1,5 @@ use super::super::super::*; -use crate::backend::wayland::state::eyedropper::{BackgroundImageKind, background_image_source}; +use crate::backend::wayland::state::screen_image::{ScreenImageKind, displayed_screen_image}; use crate::draw::Color; pub(super) struct CanvasEraserContext { @@ -39,15 +39,15 @@ impl WaylandState { let mut logical_to_image_scale_x = 1.0; let mut logical_to_image_scale_y = 1.0; - let background_image = background_image_source( + let background_image = displayed_screen_image( &self.zoom, &self.frozen, self.input_state.board_is_transparent(), ) .map(|source| { let cache_key = match source.kind { - BackgroundImageKind::Zoom => (self.zoom.image_generation() << 1) | 1, - BackgroundImageKind::Frozen => self.frozen.image_generation() << 1, + ScreenImageKind::Zoom => (self.zoom.image_generation() << 1) | 1, + ScreenImageKind::Frozen => self.frozen.image_generation() << 1, }; (source.image, cache_key, source.zoom_transformed) }); diff --git a/src/backend/wayland/state/render/ui.rs b/src/backend/wayland/state/render/ui.rs index 4448ab2b..e4e589cd 100644 --- a/src/backend/wayland/state/render/ui.rs +++ b/src/backend/wayland/state/render/ui.rs @@ -227,6 +227,7 @@ impl WaylandState { } self.render_eyedropper_loupe(ctx, width, height); + self.render_ocr_selection(ctx, width, height); if self.input_state.is_radial_menu_open() { // Layout (and with it hit-testing) is live from the moment diff --git a/src/backend/wayland/state/screen_image.rs b/src/backend/wayland/state/screen_image.rs new file mode 100644 index 00000000..877c7a67 --- /dev/null +++ b/src/backend/wayland/state/screen_image.rs @@ -0,0 +1,473 @@ +//! The captured desktop image the canvas renderer actually displays. +//! +//! The screen eyedropper samples one pixel of it and OCR crops a rectangle out +//! of it. Both must read exactly what Wayscriber is showing rather than a fresh +//! screenshot, so source resolution, the logical→image mapping, and the checked +//! rectangle copy live here instead of inside either feature. + +use crate::backend::wayland::frozen::{FrozenImage, FrozenState}; +use crate::backend::wayland::zoom::ZoomState; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ScreenImageKind { + Zoom, + Frozen, +} + +/// The captured image that the canvas renderer actually displays. +pub(super) struct DisplayedScreenImage<'a> { + pub image: &'a FrozenImage, + pub kind: ScreenImageKind, + /// The renderer is painting this image through the zoom transform, so + /// screen points must be un-zoomed before they are scaled into the image. + pub zoom_transformed: bool, +} + +/// A rectangle in source-image pixels, guaranteed non-empty and in bounds of +/// the image it was resolved against. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct ImageRect { + pub x: u32, + pub y: u32, + pub width: u32, + pub height: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum CropError { + /// The rectangle has no pixels, or the source image is empty. + Empty, + /// The rectangle leaves the source image, or its size does not fit memory. + OutOfBounds, +} + +/// What a modal that needs the displayed screen image should do when the user +/// asks for it. Shared by the eyedropper and OCR so the two cannot disagree +/// about when a temporary freeze is created or refused; each owns only the +/// wording it shows. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ScreenSourceEntry { + Activate, + WaitForZoom, + AutoFreeze, + RefuseWhileZoomedOnSolidBoard, + RefuseSolidBoard, + CaptureUnavailable, + ZoomImageUnavailable, +} + +pub(super) fn screen_source_entry( + has_source: bool, + board_is_transparent: bool, + zoom_engaged: bool, + zoom_active: bool, + frozen_enabled: bool, +) -> ScreenSourceEntry { + if has_source { + ScreenSourceEntry::Activate + } else if !board_is_transparent && zoom_engaged { + ScreenSourceEntry::RefuseWhileZoomedOnSolidBoard + } else if !board_is_transparent { + ScreenSourceEntry::RefuseSolidBoard + } else if zoom_engaged && !zoom_active { + ScreenSourceEntry::WaitForZoom + } else if !frozen_enabled { + ScreenSourceEntry::CaptureUnavailable + } else if zoom_active { + ScreenSourceEntry::ZoomImageUnavailable + } else { + ScreenSourceEntry::AutoFreeze + } +} + +pub(super) fn displayed_screen_image<'a>( + zoom: &'a ZoomState, + frozen: &'a FrozenState, + board_is_transparent: bool, +) -> Option> { + let allow_background_image = !zoom.is_engaged() || board_is_transparent; + if !allow_background_image { + return None; + } + + if zoom.active { + if let Some(image) = zoom.image() { + return Some(DisplayedScreenImage { + image, + kind: ScreenImageKind::Zoom, + zoom_transformed: true, + }); + } + if let Some(image) = frozen.image() { + return Some(DisplayedScreenImage { + image, + kind: ScreenImageKind::Frozen, + zoom_transformed: true, + }); + } + } + + frozen.image().map(|image| DisplayedScreenImage { + image, + kind: ScreenImageKind::Frozen, + zoom_transformed: false, + }) +} + +/// Map a logical screen point onto source-image pixel coordinates. The result +/// is unclamped on purpose: samplers clamp to the nearest pixel, while region +/// selection clips against the image bounds. +pub(super) fn image_point_for_screen_point( + source: &DisplayedScreenImage<'_>, + zoom: &ZoomState, + surface: (u32, u32), + point: (f64, f64), +) -> (f64, f64) { + let (world_x, world_y) = if source.zoom_transformed { + zoom.screen_to_world(point.0, point.1) + } else { + point + }; + ( + world_x * f64::from(source.image.width) / f64::from(surface.0).max(1.0), + world_y * f64::from(source.image.height) / f64::from(surface.1).max(1.0), + ) +} + +/// Map a logical screen rectangle (given by any two opposite corners) onto a +/// non-empty in-bounds source-image rectangle, or `None` when the two corners +/// resolve to nothing the image can supply. +pub(super) fn image_rect_for_screen_rect( + source: &DisplayedScreenImage<'_>, + zoom: &ZoomState, + surface: (u32, u32), + start: (f64, f64), + end: (f64, f64), +) -> Option { + let first = image_point_for_screen_point(source, zoom, surface, start); + let second = image_point_for_screen_point(source, zoom, surface, end); + image_rect_from_points(first, second, source.image.width, source.image.height) +} + +/// Clip two unordered image-space points into an integer pixel rectangle. +/// Reversed drags, partly out-of-bounds drags, and non-finite coordinates all +/// resolve deterministically here rather than at the call sites. +fn image_rect_from_points( + first: (f64, f64), + second: (f64, f64), + image_width: u32, + image_height: u32, +) -> Option { + if image_width == 0 || image_height == 0 { + return None; + } + if ![first.0, first.1, second.0, second.1] + .iter() + .all(|value| value.is_finite()) + { + return None; + } + + let width = f64::from(image_width); + let height = f64::from(image_height); + let left = first.0.min(second.0).floor().clamp(0.0, width); + let top = first.1.min(second.1).floor().clamp(0.0, height); + let right = first.0.max(second.0).ceil().clamp(0.0, width); + let bottom = first.1.max(second.1).ceil().clamp(0.0, height); + if right <= left || bottom <= top { + return None; + } + + Some(ImageRect { + x: left as u32, + y: top as u32, + width: (right - left) as u32, + height: (bottom - top) as u32, + }) +} + +/// Copy `rect` out of `source` into a tightly packed owned ARGB snapshot. +/// +/// The source may carry row padding; the copy never does, so the worker can +/// hand it straight to Cairo without re-deriving a stride. +pub(super) fn copy_image_rect( + source: &FrozenImage, + rect: ImageRect, +) -> Result { + if rect.width == 0 || rect.height == 0 || source.width == 0 || source.height == 0 { + return Err(CropError::Empty); + } + if rect + .x + .checked_add(rect.width) + .ok_or(CropError::OutOfBounds)? + > source.width + || rect + .y + .checked_add(rect.height) + .ok_or(CropError::OutOfBounds)? + > source.height + { + return Err(CropError::OutOfBounds); + } + let source_stride = usize::try_from(source.stride).map_err(|_| CropError::OutOfBounds)?; + let row_bytes = usize::try_from(rect.width) + .ok() + .and_then(|width| width.checked_mul(4)) + .ok_or(CropError::OutOfBounds)?; + let total_bytes = usize::try_from(rect.height) + .ok() + .and_then(|height| height.checked_mul(row_bytes)) + .ok_or(CropError::OutOfBounds)?; + let stride = i32::try_from(row_bytes).map_err(|_| CropError::OutOfBounds)?; + + let mut data = Vec::with_capacity(total_bytes); + let start_column = usize::try_from(rect.x) + .ok() + .and_then(|x| x.checked_mul(4)) + .ok_or(CropError::OutOfBounds)?; + for row in 0..rect.height { + let source_row = usize::try_from(rect.y.saturating_add(row)) + .ok() + .and_then(|row| row.checked_mul(source_stride)) + .ok_or(CropError::OutOfBounds)?; + let start = source_row + .checked_add(start_column) + .ok_or(CropError::OutOfBounds)?; + let end = start.checked_add(row_bytes).ok_or(CropError::OutOfBounds)?; + data.extend_from_slice(source.data.get(start..end).ok_or(CropError::OutOfBounds)?); + } + + Ok(crate::ocr::OcrPixels { + width: rect.width, + height: rect.height, + stride, + data, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn image(width: u32, height: u32, stride: i32, data: Vec) -> FrozenImage { + FrozenImage { + width, + height, + stride, + data, + } + } + + fn opaque(width: u32, height: u32) -> FrozenImage { + let mut data = Vec::with_capacity((width * height * 4) as usize); + for y in 0..height { + for x in 0..width { + data.extend_from_slice(&[x as u8, y as u8, 0, 0xFF]); + } + } + image(width, height, (width * 4) as i32, data) + } + + fn zoom_state() -> ZoomState { + ZoomState::new(None) + } + + #[test] + fn displayed_source_truth_table_matches_the_rendering_contract() { + let mut zoom = zoom_state(); + let mut frozen = FrozenState::new(None); + assert!(displayed_screen_image(&zoom, &frozen, true).is_none()); + + frozen.set_image(opaque(2, 2)); + let source = displayed_screen_image(&zoom, &frozen, false).unwrap(); + assert_eq!(source.kind, ScreenImageKind::Frozen); + assert!(!source.zoom_transformed); + + zoom.active = true; + let source = displayed_screen_image(&zoom, &frozen, true).unwrap(); + assert_eq!(source.kind, ScreenImageKind::Frozen); + assert!(source.zoom_transformed); + assert!(displayed_screen_image(&zoom, &frozen, false).is_none()); + + zoom.set_image(opaque(2, 2)); + let source = displayed_screen_image(&zoom, &frozen, true).unwrap(); + assert_eq!(source.kind, ScreenImageKind::Zoom); + assert!(source.zoom_transformed); + + zoom.active = false; + zoom.request_activation(); + assert!(displayed_screen_image(&zoom, &frozen, false).is_none()); + } + + #[test] + fn unzoomed_points_scale_by_the_image_to_surface_ratio() { + let mut frozen = FrozenState::new(None); + frozen.set_image(opaque(200, 100)); + let zoom = zoom_state(); + let source = displayed_screen_image(&zoom, &frozen, true).unwrap(); + + let point = image_point_for_screen_point(&source, &zoom, (100, 50), (50.0, 25.0)); + assert!((point.0 - 100.0).abs() < f64::EPSILON); + assert!((point.1 - 50.0).abs() < f64::EPSILON); + } + + #[test] + fn zoomed_points_use_the_same_transform_the_renderer_applied() { + let mut zoom = zoom_state(); + zoom.active = true; + zoom.set_image(opaque(100, 100)); + zoom.scale = 2.0; + zoom.view_offset = (10.0, 20.0); + let frozen = FrozenState::new(None); + let source = displayed_screen_image(&zoom, &frozen, true).unwrap(); + + let point = image_point_for_screen_point(&source, &zoom, (100, 100), (40.0, 60.0)); + // screen_to_world(40, 60) = (10 + 20, 20 + 30) with a 1:1 image scale. + assert!((point.0 - 30.0).abs() < f64::EPSILON); + assert!((point.1 - 50.0).abs() < f64::EPSILON); + } + + #[test] + fn reversed_drags_normalize_and_partly_offscreen_drags_clip() { + let rect = image_rect_from_points((30.0, 40.0), (10.0, 20.0), 100, 100).unwrap(); + assert_eq!( + rect, + ImageRect { + x: 10, + y: 20, + width: 20, + height: 20, + } + ); + + let clipped = image_rect_from_points((-50.0, -50.0), (10.5, 10.5), 100, 100).unwrap(); + assert_eq!( + clipped, + ImageRect { + x: 0, + y: 0, + width: 11, + height: 11, + } + ); + } + + #[test] + fn degenerate_rectangles_are_rejected_instead_of_clamped_to_one_pixel() { + assert!(image_rect_from_points((10.0, 10.0), (10.0, 10.0), 100, 100).is_none()); + assert!(image_rect_from_points((-20.0, -20.0), (-5.0, -5.0), 100, 100).is_none()); + assert!(image_rect_from_points((0.0, 0.0), (10.0, 10.0), 0, 10).is_none()); + assert!(image_rect_from_points((f64::NAN, 0.0), (10.0, 10.0), 100, 100).is_none()); + assert!(image_rect_from_points((0.0, 0.0), (f64::INFINITY, 10.0), 100, 100).is_none()); + } + + #[test] + fn crop_repacks_a_padded_source_row_by_row() { + let source = image( + 2, + 2, + 12, + vec![ + 1, 1, 1, 255, 2, 2, 2, 255, 9, 9, 9, 9, // + 3, 3, 3, 255, 4, 4, 4, 255, 8, 8, 8, 8, + ], + ); + + let crop = copy_image_rect( + &source, + ImageRect { + x: 1, + y: 0, + width: 1, + height: 2, + }, + ) + .unwrap(); + + assert_eq!((crop.width, crop.height, crop.stride), (1, 2, 4)); + assert_eq!(crop.data, vec![2, 2, 2, 255, 4, 4, 4, 255]); + } + + #[test] + fn crop_preserves_premultiplied_and_transparent_pixels_verbatim() { + let source = image(2, 1, 8, vec![25, 50, 100, 128, 0, 0, 0, 0]); + + let crop = copy_image_rect( + &source, + ImageRect { + x: 0, + y: 0, + width: 2, + height: 1, + }, + ) + .unwrap(); + + assert_eq!(crop.data, vec![25, 50, 100, 128, 0, 0, 0, 0]); + } + + #[test] + fn crop_rejects_empty_and_out_of_bounds_rectangles() { + let source = opaque(4, 4); + let error = |rect| copy_image_rect(&source, rect).err(); + + assert_eq!( + error(ImageRect { + x: 0, + y: 0, + width: 0, + height: 2, + }), + Some(CropError::Empty) + ); + assert_eq!( + error(ImageRect { + x: 3, + y: 0, + width: 2, + height: 1, + }), + Some(CropError::OutOfBounds) + ); + assert_eq!( + error(ImageRect { + x: u32::MAX, + y: 0, + width: 2, + height: 1, + }), + Some(CropError::OutOfBounds) + ); + } + + #[test] + fn crop_of_a_rotated_capture_keeps_the_transformed_orientation() { + use wayland_client::protocol::wl_output; + + // Two rows of three distinct pixels; rotating to 270 makes the source + // 2x3, and the crop must read that post-transform layout. + let mut data = Vec::new(); + for value in [1u8, 2, 3, 4, 5, 6] { + data.extend_from_slice(&[value, 0, 0, 0xFF]); + } + let rotated = image(3, 2, 12, data).with_output_transform(wl_output::Transform::_270); + assert_eq!((rotated.width, rotated.height), (2, 3)); + + let crop = copy_image_rect( + &rotated, + ImageRect { + x: 0, + y: 0, + width: 2, + height: 1, + }, + ) + .unwrap(); + + assert_eq!( + crop.data.chunks_exact(4).map(|p| p[0]).collect::>(), + vec![4, 1] + ); + } +} diff --git a/src/backend/wayland/state/text_clipboard.rs b/src/backend/wayland/state/text_clipboard.rs index f65b2fae..2a1edea2 100644 --- a/src/backend/wayland/state/text_clipboard.rs +++ b/src/backend/wayland/state/text_clipboard.rs @@ -8,11 +8,11 @@ //! handlers, drained each input cycle, fulfill it against the compositor //! clipboard (reusing the generic clipboard worker pipeline). -use super::color_picker::{ - ClipboardTextError, copy_text_via_command, read_clipboard_text_via_command, -}; use super::{ClipboardOperationController, WaylandState}; use crate::backend::wayland::clipboard::ClipboardPoll; +use crate::clipboard_text::{ + ClipboardTextError, copy_text_via_command, read_clipboard_text_via_command, +}; use crate::input::state::{ TextClipboardRequest, TextPasteEdit, TextPasteTarget, Toast, ToastPriority, }; diff --git a/src/backend/wayland/state/toolbar/events.rs b/src/backend/wayland/state/toolbar/events.rs index 48e8a25a..ae14ab49 100644 --- a/src/backend/wayland/state/toolbar/events.rs +++ b/src/backend/wayland/state/toolbar/events.rs @@ -134,6 +134,7 @@ impl WaylandState { // A toolbar interaction replaces the modal sampler. Do this before // shortcut capture so the capture modal owns subsequent keys. self.cancel_eyedropper(); + self.cancel_ocr_for_toolbar_interaction(); if rebind_requested && let Some(action) = crate::ui::toolbar::model::action_for_event(&event) { diff --git a/src/clipboard_text.rs b/src/clipboard_text.rs new file mode 100644 index 00000000..28c7a74f --- /dev/null +++ b/src/clipboard_text.rs @@ -0,0 +1,99 @@ +//! Generic system text-clipboard commands (`wl-copy` / `wl-paste`). +//! +//! Blocking process-broker calls with no Wayland state of their own, shared by +//! the hex-color picker, the text editor, and screen text recognition. Every +//! caller runs them on a worker thread; none of them may run on the event loop. + +use std::ffi::OsStr; +use std::time::Duration; + +pub(crate) enum ClipboardTextError { + Empty, + Other(String), +} + +pub(crate) fn copy_text_via_command(text: &str) -> Result<(), String> { + let output = crate::process_broker::current() + .and_then(|broker| { + broker.publish( + crate::process_broker::HelperKind::WlCopy, + OsStr::new("wl-copy"), + [OsStr::new("--type"), OsStr::new("text/plain;charset=utf-8")], + text.as_bytes().to_vec(), + Duration::from_secs(5), + ) + }) + .map_err(|error| format!("Failed to run wl-copy: {error:#}"))?; + if output.timed_out { + return Err("wl-copy timed out".to_string()); + } + if output.status != 0 { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + return if stderr.is_empty() { + Err("wl-copy exited unsuccessfully".to_string()) + } else { + Err(format!("wl-copy exited unsuccessfully: {stderr}")) + }; + } + Ok(()) +} + +pub(crate) fn read_clipboard_text_via_command() -> Result { + let output = crate::process_broker::current() + .and_then(|broker| { + broker.run( + crate::process_broker::HelperKind::WlPaste, + OsStr::new("wl-paste"), + clipboard_text_read_args(), + Vec::new(), + Duration::from_secs(5), + 1024 * 1024, + ) + }) + .map_err(|err| ClipboardTextError::Other(format!("Failed to run wl-paste: {err:#}")))?; + + if !output.timed_out && output.status == 0 { + Ok(String::from_utf8_lossy(&output.stdout).to_string()) + } else { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if stderr.to_ascii_lowercase().contains("nothing is copied") + || stderr.to_ascii_lowercase().contains("clipboard is empty") + { + Err(ClipboardTextError::Empty) + } else if stderr.is_empty() { + Err(ClipboardTextError::Other( + "wl-paste exited unsuccessfully".to_string(), + )) + } else { + Err(ClipboardTextError::Other(format!( + "wl-paste exited unsuccessfully: {}", + stderr + ))) + } + } +} + +fn clipboard_text_read_args() -> [&'static OsStr; 3] { + [ + OsStr::new("--no-newline"), + OsStr::new("--type"), + OsStr::new("text"), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clipboard_text_read_requests_only_a_text_mime() { + assert_eq!( + clipboard_text_read_args(), + [ + OsStr::new("--no-newline"), + OsStr::new("--type"), + OsStr::new("text"), + ] + ); + } +} diff --git a/src/config/action_meta/entries/capture.rs b/src/config/action_meta/entries/capture.rs index 0e4c288c..b7dea692 100644 --- a/src/config/action_meta/entries/capture.rs +++ b/src/config/action_meta/entries/capture.rs @@ -168,4 +168,15 @@ pub const ENTRIES: &[ActionMeta] = &[ true, false ), + meta!( + CopyTextFromScreen, + "Copy Text From Screen", + Some("Copy Text"), + "Select a screen region and copy recognized text", + Capture, + true, + true, + true, + &["ocr", "recognize text", "screen text", "copy text"] + ), ]; diff --git a/src/config/action_meta/tests.rs b/src/config/action_meta/tests.rs index bfa85ad4..f7f127b2 100644 --- a/src/config/action_meta/tests.rs +++ b/src/config/action_meta/tests.rs @@ -258,6 +258,7 @@ const EXPECTED_COMMAND_PALETTE_ACTIONS: &[Action] = &[ Action::ExportBoardPdfFile, Action::ExportAllBoardsPdfFile, Action::OpenCaptureFolder, + Action::CopyTextFromScreen, Action::ToggleFrozenMode, Action::ZoomIn, Action::ZoomOut, diff --git a/src/config/keybindings/config/map/capture.rs b/src/config/keybindings/config/map/capture.rs index 58013c85..7df961f6 100644 --- a/src/config/keybindings/config/map/capture.rs +++ b/src/config/keybindings/config/map/capture.rs @@ -49,6 +49,10 @@ impl KeybindingsConfig { Action::ExportAllBoardsPdfFile, )?; inserter.insert_all(&self.capture.open_capture_folder, Action::OpenCaptureFolder)?; + inserter.insert_all( + &self.capture.copy_text_from_screen, + Action::CopyTextFromScreen, + )?; Ok(()) } } diff --git a/src/config/keybindings/config/map/edit.rs b/src/config/keybindings/config/map/edit.rs index a2da29c9..5065d158 100644 --- a/src/config/keybindings/config/map/edit.rs +++ b/src/config/keybindings/config/map/edit.rs @@ -189,6 +189,7 @@ define_action_binding_accessors! { ExportBoardPdfFile => capture.export_board_pdf_file, ExportAllBoardsPdfFile => capture.export_all_boards_pdf_file, OpenCaptureFolder => capture.open_capture_folder, + CopyTextFromScreen => capture.copy_text_from_screen, ToggleFrozenMode => zoom.toggle_frozen_mode, ZoomIn => zoom.zoom_in, ZoomOut => zoom.zoom_out, diff --git a/src/config/keybindings/config/types/bindings/capture.rs b/src/config/keybindings/config/types/bindings/capture.rs index db72ee2c..6390239b 100644 --- a/src/config/keybindings/config/types/bindings/capture.rs +++ b/src/config/keybindings/config/types/bindings/capture.rs @@ -49,6 +49,11 @@ pub struct CaptureKeybindingsConfig { #[serde(default = "default_open_capture_folder")] pub open_capture_folder: Vec, + + /// Screen text recognition. Unbound by default: `O` already selects the + /// orange quick color, so a default here would silently repurpose it. + #[serde(default = "default_copy_text_from_screen")] + pub copy_text_from_screen: Vec, } impl Default for CaptureKeybindingsConfig { @@ -69,6 +74,7 @@ impl Default for CaptureKeybindingsConfig { export_board_pdf_file: default_export_board_pdf_file(), export_all_boards_pdf_file: default_export_all_boards_pdf_file(), open_capture_folder: default_open_capture_folder(), + copy_text_from_screen: default_copy_text_from_screen(), } } } diff --git a/src/config/keybindings/defaults/capture.rs b/src/config/keybindings/defaults/capture.rs index 8d4246f5..b640603e 100644 --- a/src/config/keybindings/defaults/capture.rs +++ b/src/config/keybindings/defaults/capture.rs @@ -57,3 +57,9 @@ pub(crate) fn default_export_all_boards_pdf_file() -> Vec { pub(crate) fn default_open_capture_folder() -> Vec { vec!["Ctrl+Alt+O".to_string()] } + +/// Deliberately empty: `O` is the orange quick color and no other +/// conflict-free chord is obviously right, so the user picks one. +pub(crate) fn default_copy_text_from_screen() -> Vec { + Vec::new() +} diff --git a/src/config/keybindings/tests.rs b/src/config/keybindings/tests.rs index b5aa53b7..cc533c7d 100644 --- a/src/config/keybindings/tests.rs +++ b/src/config/keybindings/tests.rs @@ -360,6 +360,30 @@ fn screen_eyedropper_defaults_to_i_and_maps_when_reconfigured() { ); } +#[test] +fn screen_text_recognition_is_unbound_by_default_and_leaves_o_to_orange() { + let mut config = KeybindingsConfig::default(); + assert!(config.capture.copy_text_from_screen.is_empty()); + + let default_map = config.build_action_map().unwrap(); + assert_eq!( + default_map.get(&KeyBinding::parse("O").unwrap()), + Some(&Action::SetColorOrange) + ); + assert!( + !default_map + .values() + .any(|action| *action == Action::CopyTextFromScreen) + ); + + config.capture.copy_text_from_screen = vec!["Ctrl+Alt+T".to_string()]; + let map = config.build_action_map().unwrap(); + assert_eq!( + map.get(&KeyBinding::parse("Ctrl+Alt+T").unwrap()), + Some(&Action::CopyTextFromScreen) + ); +} + #[test] fn canvas_export_actions_deserialize_from_config_names() { #[derive(serde::Deserialize)] @@ -637,6 +661,9 @@ const DEFAULT_BINDING_SNAPSHOT: &[(&str, &[&str])] = &[ ("export_board_pdf_file", &[]), ("export_all_boards_pdf_file", &[]), ("open_capture_folder", &["Ctrl+Alt+O"]), + // Intentionally unbound: `O` is the orange quick color, and no other + // conflict-free chord is obviously right, so the user picks one. + ("copy_text_from_screen", &[]), ("toggle_frozen_mode", &["Ctrl+Shift+F"]), ("zoom_in", &["Ctrl+Alt++", "Ctrl+Alt+="]), ("zoom_out", &["Ctrl+Alt+-", "Ctrl+Alt+_"]), diff --git a/src/config/mod.rs b/src/config/mod.rs index 76833915..502d8158 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -58,28 +58,28 @@ pub use migration::{MigrationChange, MigrationPreview}; #[allow(unused_imports)] pub use types::{ ArrowConfig, BoardBackgroundConfig, BoardColorConfig, BoardConfig, BoardItemConfig, - BoardsConfig, CaptureConfig, ClickHighlightConfig, DragButtonConfig, DrawingConfig, - ExportConfig, HelpOverlayStyle, HistoryConfig, InputHudConfig, InputHudMode, InputHudPosition, - MouseDragToolsConfig, PDF_LABEL_APP_BOARD, PDF_LABEL_APP_BOARDS, PDF_LABEL_BOARD_NAME, - PDF_LABEL_DEFAULT_TEMPLATE, PDF_LABEL_DOCUMENT_PAGE, PDF_LABEL_DOCUMENT_PAGES, - PDF_LABEL_EXPORT_BOARD, PDF_LABEL_EXPORT_BOARDS, PDF_LABEL_PAGE, PDF_LABEL_PAGE_NAME, - PDF_LABEL_PAGES, PDF_LABEL_PLACEHOLDERS, PRESET_SLOTS_MAX, PRESET_SLOTS_MIN, PdfExportConfig, - PdfFitMode, PdfLabelConfig, PdfLabelContentMode, PdfLabelPosition, PdfOrientation, PdfPageSize, - PdfTransparentBackground, PerformanceConfig, PresenterModeConfig, PresenterToolBehavior, - PresenterToolbarMode, PresetSlotsConfig, PresetToolSettingConfig, PresetToolStatesConfig, - QUICK_COLOR_RENDER_LIMIT, QuickColorConfig, QuickColorPalette, QuickColorPaletteEntry, - QuickColorSlot, QuickColorWrite, QuickColorsConfig, RenderColorMappingConfig, - RenderProfileConfig, RenderProfileExportMode, RenderProfilesConfig, ResolvedToolbarItems, - SessionCompression, SessionConfig, SessionStorageMode, SpotlightConfig, StatusBarItem, - StatusBarStyle, ToolPresetConfig, ToolbarBackendKind, ToolbarConfig, ToolbarGroupId, - ToolbarItemCategory, ToolbarItemDefinition, ToolbarItemId, ToolbarItemOrderConfig, - ToolbarItemOrderGroup, ToolbarItemSurface, ToolbarItemsConfig, ToolbarLayoutMode, - ToolbarModeOverride, ToolbarModeOverrides, ToolbarRebindModifier, ToolbarSectionFlag, - ToolbarSectionVisibility, TopDisplayMode, TrayConfig, TrayIconStyle, UiConfig, UpdatesConfig, - ZoomChipDisplay, default_quick_color_for_index, fold_legacy_section_flags, - resolve_section_visibility, section_flag_for_item, set_section_visibility, - toolbar_item_definitions, toolbar_item_ids, toolbar_item_order_group, - validate_pdf_label_template, + BoardsConfig, CaptureConfig, ClickHighlightConfig, DEFAULT_OCR_LANGUAGES, DragButtonConfig, + DrawingConfig, ExportConfig, HelpOverlayStyle, HistoryConfig, InputHudConfig, InputHudMode, + InputHudPosition, MouseDragToolsConfig, PDF_LABEL_APP_BOARD, PDF_LABEL_APP_BOARDS, + PDF_LABEL_BOARD_NAME, PDF_LABEL_DEFAULT_TEMPLATE, PDF_LABEL_DOCUMENT_PAGE, + PDF_LABEL_DOCUMENT_PAGES, PDF_LABEL_EXPORT_BOARD, PDF_LABEL_EXPORT_BOARDS, PDF_LABEL_PAGE, + PDF_LABEL_PAGE_NAME, PDF_LABEL_PAGES, PDF_LABEL_PLACEHOLDERS, PRESET_SLOTS_MAX, + PRESET_SLOTS_MIN, PdfExportConfig, PdfFitMode, PdfLabelConfig, PdfLabelContentMode, + PdfLabelPosition, PdfOrientation, PdfPageSize, PdfTransparentBackground, PerformanceConfig, + PresenterModeConfig, PresenterToolBehavior, PresenterToolbarMode, PresetSlotsConfig, + PresetToolSettingConfig, PresetToolStatesConfig, QUICK_COLOR_RENDER_LIMIT, QuickColorConfig, + QuickColorPalette, QuickColorPaletteEntry, QuickColorSlot, QuickColorWrite, QuickColorsConfig, + RenderColorMappingConfig, RenderProfileConfig, RenderProfileExportMode, RenderProfilesConfig, + ResolvedToolbarItems, SessionCompression, SessionConfig, SessionStorageMode, SpotlightConfig, + StatusBarItem, StatusBarStyle, ToolPresetConfig, ToolbarBackendKind, ToolbarConfig, + ToolbarGroupId, ToolbarItemCategory, ToolbarItemDefinition, ToolbarItemId, + ToolbarItemOrderConfig, ToolbarItemOrderGroup, ToolbarItemSurface, ToolbarItemsConfig, + ToolbarLayoutMode, ToolbarModeOverride, ToolbarModeOverrides, ToolbarRebindModifier, + ToolbarSectionFlag, ToolbarSectionVisibility, TopDisplayMode, TrayConfig, TrayIconStyle, + UiConfig, UpdatesConfig, ZoomChipDisplay, default_quick_color_for_index, + fold_legacy_section_flags, resolve_section_visibility, section_flag_for_item, + set_section_visibility, toolbar_item_definitions, toolbar_item_ids, toolbar_item_order_group, + validate_ocr_languages, validate_pdf_label_template, }; #[cfg(feature = "tablet-input")] #[allow(unused_imports)] diff --git a/src/config/types/capture.rs b/src/config/types/capture.rs index a052d52b..f6cf910e 100644 --- a/src/config/types/capture.rs +++ b/src/config/types/capture.rs @@ -31,6 +31,12 @@ pub struct CaptureConfig { /// When false, clipboard-only captures still auto-exit by default. #[serde(default = "default_capture_exit_after")] pub exit_after_capture: bool, + + /// Tesseract languages used by `Copy text from screen`, in Tesseract's + /// plus-separated form (for example `eng` or `eng+deu`). The matching + /// Tesseract language packages must be installed. + #[serde(default = "default_capture_ocr_languages")] + pub ocr_languages: String, } impl Default for CaptureConfig { @@ -42,8 +48,65 @@ impl Default for CaptureConfig { format: default_capture_format(), copy_to_clipboard: default_capture_clipboard(), exit_after_capture: default_capture_exit_after(), + ocr_languages: default_capture_ocr_languages(), + } + } +} + +impl CaptureConfig { + /// The configured OCR languages, or the default when the authored value is + /// unusable. Recognition never passes an unvalidated string to the engine. + pub fn resolved_ocr_languages(&self) -> String { + match validate_ocr_languages(&self.ocr_languages) { + Ok(languages) => languages, + Err(reason) => { + log::warn!( + "Ignoring capture.ocr_languages {:?}: {reason}; using {}", + self.ocr_languages, + DEFAULT_OCR_LANGUAGES + ); + DEFAULT_OCR_LANGUAGES.to_string() + } + } + } +} + +pub const DEFAULT_OCR_LANGUAGES: &str = "eng"; + +/// Normalize a Tesseract language argument, rejecting anything that could +/// change how the engine is invoked. +/// +/// Accepts the plus-separated form Tesseract itself uses (`eng`, `eng+deu`). +/// Tokens are restricted to the character set real language codes use, so a +/// value can never carry a path, an option, or shell syntax into the argument +/// vector. +pub fn validate_ocr_languages(raw: &str) -> Result { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return Err("value is empty".to_string()); + } + let tokens: Vec<&str> = trimmed.split('+').collect(); + for token in &tokens { + if token.is_empty() { + return Err("value has an empty language between '+' separators".to_string()); + } + if token.starts_with('-') { + return Err(format!("language {token:?} looks like a command option")); + } + if !token + .chars() + .all(|character| character.is_ascii_alphanumeric() || matches!(character, '_' | '-')) + { + return Err(format!( + "language {token:?} may only contain letters, digits, '_' and '-'" + )); } } + Ok(tokens.join("+")) +} + +fn default_capture_ocr_languages() -> String { + DEFAULT_OCR_LANGUAGES.to_string() } fn default_capture_enabled() -> bool { @@ -69,3 +132,53 @@ fn default_capture_clipboard() -> bool { fn default_capture_exit_after() -> bool { false } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_ocr_language_is_english() { + assert_eq!(CaptureConfig::default().ocr_languages, "eng"); + assert_eq!(CaptureConfig::default().resolved_ocr_languages(), "eng"); + } + + #[test] + fn plus_separated_languages_round_trip_after_trimming() { + assert_eq!(validate_ocr_languages(" eng+deu \n").unwrap(), "eng+deu"); + assert_eq!(validate_ocr_languages("chi_sim").unwrap(), "chi_sim"); + assert_eq!(validate_ocr_languages("eng").unwrap(), "eng"); + } + + #[test] + fn unsafe_or_empty_values_are_rejected() { + for value in [ + "", + " ", + "eng+", + "+eng", + "eng deu", + "../tessdata/eng", + "/usr/share/eng", + "eng;rm -rf /", + "eng$(id)", + "--psm", + "eng\0", + ] { + assert!( + validate_ocr_languages(value).is_err(), + "{value:?} must be rejected" + ); + } + } + + #[test] + fn an_invalid_authored_value_falls_back_to_the_default() { + let config = CaptureConfig { + ocr_languages: "../evil".to_string(), + ..CaptureConfig::default() + }; + + assert_eq!(config.resolved_ocr_languages(), DEFAULT_OCR_LANGUAGES); + } +} diff --git a/src/config/types/mod.rs b/src/config/types/mod.rs index 9d823a42..4ed016df 100644 --- a/src/config/types/mod.rs +++ b/src/config/types/mod.rs @@ -28,7 +28,7 @@ mod updates; pub use arrow::ArrowConfig; pub use board::BoardConfig; pub use boards::{BoardBackgroundConfig, BoardColorConfig, BoardItemConfig, BoardsConfig}; -pub use capture::CaptureConfig; +pub use capture::{CaptureConfig, DEFAULT_OCR_LANGUAGES, validate_ocr_languages}; pub use click_highlight::ClickHighlightConfig; pub use context_menu::ContextMenuUiConfig; pub use drawing::{ diff --git a/src/config/types/toolbar/ids.rs b/src/config/types/toolbar/ids.rs index c3128336..a330e0d5 100644 --- a/src/config/types/toolbar/ids.rs +++ b/src/config/types/toolbar/ids.rs @@ -36,6 +36,7 @@ pub const TOP_UTILITY_CLEAR_CANVAS: ToolbarItemId = ToolbarItemId::from_known("top.utility.clear-canvas"); pub const TOP_UTILITY_SCREENSHOT: ToolbarItemId = ToolbarItemId::from_known("top.utility.screenshot"); +pub const TOP_UTILITY_OCR: ToolbarItemId = ToolbarItemId::from_known("top.utility.ocr"); pub const TOP_UTILITY_HIGHLIGHT: ToolbarItemId = ToolbarItemId::from_known("top.utility.highlight"); pub const TOP_UTILITY_HIGHLIGHT_RING: ToolbarItemId = ToolbarItemId::from_known("top.utility.highlight-ring"); diff --git a/src/config/types/toolbar/items.rs b/src/config/types/toolbar/items.rs index 66f969e7..73afe923 100644 --- a/src/config/types/toolbar/items.rs +++ b/src/config/types/toolbar/items.rs @@ -54,6 +54,21 @@ fn drain_without(raw_ids: Vec, id: ToolbarItemId) -> Vec { const DEFAULT_HIDDEN_TOOLBAR_ITEM_IDS: &[ToolbarItemId] = &[ids::TOP_UTILITY_SCREENSHOT]; +/// Items hidden by a code-level baseline rather than by the shipped `hidden` +/// list, overridable with an explicit `shown` entry. +/// +/// A `hidden` list authored before an item existed cannot name it, and serde +/// applies the struct default only to a *missing* field — so shipping a new +/// default-hidden id reaches new configs and silently exposes the item on every +/// installed one. Screenshot stays list-driven because every existing config +/// already names it; anything added afterwards belongs here instead. +const BASELINE_HIDDEN_TOOLBAR_ITEM_IDS: &[ToolbarItemId] = &[ids::TOP_UTILITY_OCR]; + +/// Whether `id` is hidden unless the user explicitly shows it. +pub(crate) fn toolbar_item_hidden_by_baseline(id: ToolbarItemId) -> bool { + BASELINE_HIDDEN_TOOLBAR_ITEM_IDS.contains(&id) +} + const DEFAULT_TOP_TOOLS_ORDER: &[ToolbarItemId] = &[ ids::TOP_TOOL_SELECT, ids::TOP_TOOL_PEN, @@ -77,6 +92,7 @@ const DEFAULT_TOP_CONTROLS_ORDER: &[ToolbarItemId] = &[ ids::TOP_UTILITY_TEXT, ids::TOP_UTILITY_STICKY_NOTE, ids::TOP_UTILITY_SCREENSHOT, + ids::TOP_UTILITY_OCR, ids::TOP_UTILITY_CLEAR_CANVAS, ids::TOP_UTILITY_HIGHLIGHT, ]; @@ -133,9 +149,12 @@ impl ToolbarItemsConfig { self.shown = drain_without(std::mem::take(&mut self.shown), id); if hidden { self.hidden.push(id.as_str().to_string()); - } else if super::visibility::section_flag_for_item(id).is_some() { - // Section-level ids have a layout-mode baseline that may hide - // them again; an explicit "shown" entry pins them visible. + } else if super::visibility::section_flag_for_item(id).is_some() + || toolbar_item_hidden_by_baseline(id) + { + // These have a baseline that would hide them again — a layout mode + // for section ids, a code-level default for the rest — so an + // explicit "shown" entry is what pins them visible. self.shown.push(id.as_str().to_string()); } } @@ -147,6 +166,13 @@ impl ToolbarItemsConfig { id: ToolbarItemId, setting: ToolbarItemVisibilitySetting, ) -> bool { + // Re-applying the setting an item already has must be a no-op. The + // rewrite below moves the id to the end of its list, so without this + // guard a second identical call would report a change forever — which + // is what a batch reset over several items does. + if item_visibility_setting(&self.resolved(), id) == setting { + return false; + } let before_hidden = self.hidden.clone(); let before_shown = self.shown.clone(); self.hidden = drain_without(std::mem::take(&mut self.hidden), id); @@ -250,7 +276,12 @@ pub struct ResolvedToolbarItems { impl ResolvedToolbarItems { pub fn is_hidden(&self, id: ToolbarItemId) -> bool { - self.hidden.contains(&id) + if self.hidden.contains(&id) { + return true; + } + // A baseline-hidden item needs an explicit `shown` entry, so upgrading + // a config written before the item existed cannot reveal it. + toolbar_item_hidden_by_baseline(id) && !self.shown.contains(&id) } } diff --git a/src/config/types/toolbar/items/definitions.rs b/src/config/types/toolbar/items/definitions.rs index 73336a31..3191b8d0 100644 --- a/src/config/types/toolbar/items/definitions.rs +++ b/src/config/types/toolbar/items/definitions.rs @@ -80,6 +80,7 @@ const TOOLBAR_ITEM_DEFINITIONS: &[ToolbarItemDefinition] = &[ Utility, None, ), + item(ids::TOP_UTILITY_OCR, "Copy text", Top, Utility, None), item(ids::TOP_UTILITY_HIGHLIGHT, "Highlight", Top, Utility, None), item( ids::TOP_UTILITY_HIGHLIGHT_RING, diff --git a/src/config/types/toolbar/items/tests.rs b/src/config/types/toolbar/items/tests.rs index 144b8ac5..305e9601 100644 --- a/src/config/types/toolbar/items/tests.rs +++ b/src/config/types/toolbar/items/tests.rs @@ -28,6 +28,94 @@ fn default_hidden_items_hide_screenshot_tool() { ); } +#[test] +fn the_ocr_utility_is_defined_once_ordered_and_hidden_by_default() { + let definitions: Vec<_> = toolbar_item_definitions() + .iter() + .filter(|definition| definition.id == ids::TOP_UTILITY_OCR) + .collect(); + assert_eq!(definitions.len(), 1); + assert_eq!(definitions[0].label, "Copy text"); + + let order = ToolbarItemOrderConfig::default().resolved(); + let top_controls = order.ordered_ids(ToolbarItemOrderGroup::TopControls); + assert_eq!( + top_controls + .iter() + .filter(|id| **id == ids::TOP_UTILITY_OCR) + .count(), + 1 + ); + + // Hidden by the code-level baseline, which reads as no explicit setting: + // nothing is written to `hidden`, so a config that predates the item still + // resolves as hiding it. + let resolved = ToolbarItemsConfig::default().resolved(); + assert!(resolved.is_hidden(ids::TOP_UTILITY_OCR)); + assert_eq!( + item_visibility_setting(&resolved, ids::TOP_UTILITY_OCR), + ToolbarItemVisibilitySetting::Default + ); + assert!( + !ToolbarItemsConfig::default() + .hidden + .iter() + .any(|raw| raw == ids::TOP_UTILITY_OCR.as_str()) + ); +} + +/// The regression the baseline exists for: an install predating OCR carries its +/// own `hidden` list, and serde applies the struct default only to a *missing* +/// field — so a shipped default-hidden entry would never reach it. +#[test] +fn a_config_written_before_ocr_existed_still_hides_it() { + let upgraded: ToolbarItemsConfig = + toml::from_str("hidden = [\"top.utility.screenshot\"]\nshown = []\n") + .expect("previous-release toolbar items parse"); + + let resolved = upgraded.resolved(); + assert!(resolved.is_hidden(ids::TOP_UTILITY_SCREENSHOT)); + assert!( + resolved.is_hidden(ids::TOP_UTILITY_OCR), + "upgrading exposed the opt-in OCR button" + ); +} + +/// Turning it on has to stick, which needs an explicit `shown` entry: merely +/// dropping it from `hidden` would leave the baseline hiding it again. +#[test] +fn showing_the_ocr_utility_records_it_as_explicitly_shown() { + let mut config = ToolbarItemsConfig::default(); + config.set_hidden(ids::TOP_UTILITY_OCR, false); + + assert!( + config + .shown + .iter() + .any(|raw| raw == ids::TOP_UTILITY_OCR.as_str()) + ); + assert!(!config.resolved().is_hidden(ids::TOP_UTILITY_OCR)); + + config.set_hidden(ids::TOP_UTILITY_OCR, true); + assert!(config.resolved().is_hidden(ids::TOP_UTILITY_OCR)); +} + +#[test] +fn showing_the_ocr_utility_is_an_explicit_customization_that_survives_reads() { + let mut config = ToolbarItemsConfig::default(); + + assert!( + config.set_visibility_setting(ids::TOP_UTILITY_OCR, ToolbarItemVisibilitySetting::Shown) + ); + assert!(!config.resolved().is_hidden(ids::TOP_UTILITY_OCR)); + // Re-applying the same setting is not a change; only a different one is. + assert!( + !config.set_visibility_setting(ids::TOP_UTILITY_OCR, ToolbarItemVisibilitySetting::Shown) + ); + assert!(config.reset_known_hidden_to_defaults()); + assert!(config.resolved().is_hidden(ids::TOP_UTILITY_OCR)); +} + #[test] fn visibility_setting_is_hidden_first_and_known_mutation_canonicalizes_conflicts() { let mut config = ToolbarItemsConfig { diff --git a/src/configurator_destination.rs b/src/configurator_destination.rs index 301149e9..cfc517c8 100644 --- a/src/configurator_destination.rs +++ b/src/configurator_destination.rs @@ -200,6 +200,7 @@ pub fn keybindings_section_for_action(action: Action) -> Option { + // The backend owns capture ownership and the region selector, + // so this only records the intent. It selects no tool and + // touches no drawing state. + log::debug!("Copy text from screen requested"); + self.request_copy_text_from_screen(); + true + } Action::ToggleFrozenMode => { log::info!("Toggle frozen mode requested"); self.request_frozen_toggle(); diff --git a/src/input/state/core/base/state/init.rs b/src/input/state/core/base/state/init.rs index a8acc45a..8274ed6c 100644 --- a/src/input/state/core/base/state/init.rs +++ b/src/input/state/core/base/state/init.rs @@ -292,6 +292,9 @@ impl InputState { pending_frozen_toggle: false, eyedropper_ui_state: crate::input::state::core::EyedropperUiState::Inactive, pending_eyedropper_toggle: false, + ocr_ui_state: crate::input::state::core::OcrUiState::Inactive, + pending_ocr_request: false, + ocr_cancelled_by_toolbar: false, zoom_active: false, zoom_locked: false, zoom_scale: 1.0, diff --git a/src/input/state/core/base/state/structs.rs b/src/input/state/core/base/state/structs.rs index 387a49ed..258c1007 100644 --- a/src/input/state/core/base/state/structs.rs +++ b/src/input/state/core/base/state/structs.rs @@ -586,6 +586,14 @@ pub struct InputState { crate::input::state::core::EyedropperUiState, /// Pending eyedropper activation request for the Wayland backend. pub(in crate::input::state::core) pending_eyedropper_toggle: bool, + /// Screen text recognition (OCR) region-selector lifecycle. + pub(in crate::input::state::core) ocr_ui_state: crate::input::state::core::OcrUiState, + /// Pending `Copy text from screen` request for the Wayland backend. + pub(in crate::input::state::core) pending_ocr_request: bool, + /// A toolbar interaction dismissed the selector in this input batch, so a + /// `Copy text from screen` request produced by that same interaction is the + /// button toggling itself off — not a fresh invocation to honor. + pub(in crate::input::state::core) ocr_cancelled_by_toolbar: bool, /// Whether zoom mode is currently active pub(in crate::input::state::core) zoom_active: bool, /// Whether zoom view is locked diff --git a/src/input/state/core/eyedropper.rs b/src/input/state/core/eyedropper.rs index 087af856..788f74e5 100644 --- a/src/input/state/core/eyedropper.rs +++ b/src/input/state/core/eyedropper.rs @@ -86,7 +86,10 @@ impl InputState { .flatten() } - pub(crate) fn prepare_for_eyedropper(&mut self) { + /// Close everything a screen-region modal must not compete with, and + /// cancel any unfinished gesture. Shared by the eyedropper and OCR: both + /// take over pointer input entirely while they are up. + pub(crate) fn prepare_for_screen_modal(&mut self) { self.cancel_active_interaction(); if self.show_help { self.toggle_help_overlay(); @@ -117,7 +120,7 @@ impl InputState { // A capture can take long enough for another interaction to begin while // the eyedropper is pending. Entering the modal state must cancel it so // the eyedropper cannot swallow the matching release event. - self.prepare_for_eyedropper(); + self.prepare_for_screen_modal(); self.eyedropper_ui_state = EyedropperUiState::Active { hover: None, auto_froze, @@ -226,6 +229,23 @@ mod tests { assert!(state.eyedropper_is_active()); } + /// The eyedropper swallows every key press it receives, so a key held from + /// before it opened must not keep repeating into the canvas behind it. + #[test] + fn an_engaged_eyedropper_stops_canvas_key_repeat() { + let mut state = make_test_input_state(); + assert!(!state.modal_blocks_canvas_key_repeat()); + + state.set_eyedropper_pending_capture(EyedropperCaptureSource::Frozen); + assert!(state.modal_blocks_canvas_key_repeat()); + + state.activate_eyedropper(true); + assert!(state.modal_blocks_canvas_key_repeat()); + + state.cancel_eyedropper(); + assert!(!state.modal_blocks_canvas_key_repeat()); + } + #[test] fn default_i_key_resolves_to_screen_eyedropper() { let state = make_test_input_state(); diff --git a/src/input/state/core/mod.rs b/src/input/state/core/mod.rs index 3e2d80d7..e59d2d57 100644 --- a/src/input/state/core/mod.rs +++ b/src/input/state/core/mod.rs @@ -12,6 +12,7 @@ mod index; mod input_hud_controls; mod menus; pub(crate) mod modal; +mod ocr; mod properties; pub(crate) mod radial_menu; mod selection; @@ -71,6 +72,7 @@ pub use ime::{ImeCompositionState, ImePreedit}; pub use menus::{ ContextMenuCursorHint, ContextMenuEntry, ContextMenuKind, ContextMenuState, MenuCommand, }; +pub use ocr::{OcrCaptureSource, OcrInputSource, OcrSelection, OcrUiState}; pub use properties::{SelectionPropertyEntry, SelectionPropertyKind}; pub use radial_menu::{ COMPASS_SLICES as RADIAL_COMPASS_SLICES, CompassDir, RADIAL_PAINT_DELAY, RadialMenuLayout, diff --git a/src/input/state/core/modal.rs b/src/input/state/core/modal.rs index 7e541fde..ecae28a8 100644 --- a/src/input/state/core/modal.rs +++ b/src/input/state/core/modal.rs @@ -135,18 +135,49 @@ impl InputState { pub fn modal_owns_text_input(&self) -> bool { self.engaged_modal().is_some() // The keybinding capture chord can be armed after the palette - // closes, and the eyedropper is not a popup surface; both still - // own the keyboard. + // closes, and the screen-region modals are not popup surfaces; + // all of them still own the keyboard. || self.command_palette_is_engaged() - || self.eyedropper_is_engaged() + || self.screen_modal_is_engaged() } /// Modal paths whose editing/repeat behavior must not be driven by the /// backend's canvas-oriented manual repeat timer. pub fn modal_blocks_canvas_key_repeat(&self) -> bool { self.command_palette_is_engaged() + // A screen-region modal can open from the toolbar, the command + // palette, or a mouse path while a canvas key is still held. It + // swallows every key press it receives, so leaving the repeat timer + // armed would keep feeding the canvas behind the selector. + || self.screen_modal_is_engaged() || ModalSurface::ALL .into_iter() .any(|surface| surface.blocks_canvas_key_repeat() && self.modal_is_open(surface)) } + + /// Whether either screen-region modal — the eyedropper or the OCR region + /// selector — has been asked for, including while it still waits on a + /// capture. + /// + /// This is the *keyboard* boundary: the key that requested the modal is the + /// last one the canvas sees, and every press from then on is swallowed, so + /// key repeat, IME ownership, and stylus barrel actions all stop here. + /// + /// It is deliberately not the pen boundary — see + /// [`Self::screen_modal_is_active`]. + pub(crate) fn screen_modal_is_engaged(&self) -> bool { + self.eyedropper_is_engaged() || self.ocr_is_engaged() + } + + /// Whether a screen-region modal is on screen and owns pointer input. + /// + /// The *pen* boundary, and narrower than [`Self::screen_modal_is_engaged`]: + /// while a modal is still waiting on its capture there is nothing drawn + /// over the canvas, and every pointer, touch, and stylus path still routes + /// to drawing — so a stroke started during that wait is a real stroke and + /// keeps its pressure. Activation cancels it; a capture that fails or is + /// cancelled first leaves it intact. + pub(crate) fn screen_modal_is_active(&self) -> bool { + self.eyedropper_is_active() || self.ocr_is_active() + } } diff --git a/src/input/state/core/ocr.rs b/src/input/state/core/ocr.rs new file mode 100644 index 00000000..51ffdbf3 --- /dev/null +++ b/src/input/state/core/ocr.rs @@ -0,0 +1,531 @@ +use super::InputState; +use crate::input::state::{Toast, ToastPriority}; + +/// Which capture the OCR selector is waiting on before it can arm. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OcrCaptureSource { + Frozen, + Zoom, +} + +/// Which physical input started a region drag. +/// +/// A region is one gesture by one device. Wayscriber is modal but the seat is +/// not: a pen can hover or lift while the region is being dragged with the +/// mouse, and without an owner those stray events would move — or submit — +/// somebody else's region. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OcrInputSource { + Pointer, + Touch, + Stylus, +} + +/// A finished or in-progress region drag in logical screen coordinates. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct OcrSelection { + pub start: (f64, f64), + pub end: (f64, f64), + /// OCR created the freeze it selected against, so OCR must release it. + pub auto_froze: bool, +} + +/// UI-facing lifecycle for the modal `Copy text from screen` region selector. +/// +/// It owns transient input and render state only. The selected pixels and the +/// recognition work belong outside `InputState`, so cancelling here can never +/// discard a request that is already running. +#[derive(Debug, Default, Clone, Copy, PartialEq)] +pub enum OcrUiState { + #[default] + Inactive, + PendingCapture { + source: OcrCaptureSource, + auto_froze: bool, + }, + /// The selector is live and waiting for the press that starts a region. + Armed { auto_froze: bool }, + /// A region drag is in progress between `start` and `current`, owned by the + /// device that pressed. Only that device may move, release, or cancel it. + Selecting { + owner: OcrInputSource, + start: (f64, f64), + current: (f64, f64), + auto_froze: bool, + }, +} + +impl OcrUiState { + pub fn is_active(self) -> bool { + matches!(self, Self::Armed { .. } | Self::Selecting { .. }) + } + + pub fn is_pending(self) -> bool { + matches!(self, Self::PendingCapture { .. }) + } + + pub fn is_engaged(self) -> bool { + self.is_active() || self.is_pending() + } + + pub fn is_selecting(self) -> bool { + matches!(self, Self::Selecting { .. }) + } + + /// The region drag in logical screen coordinates, if one is in progress. + pub fn selection(self) -> Option { + match self { + Self::Selecting { + start, + current, + auto_froze, + .. + } => Some(OcrSelection { + start, + end: current, + auto_froze, + }), + Self::Inactive | Self::PendingCapture { .. } | Self::Armed { .. } => None, + } + } + + /// The device dragging the region, if one is in progress. + pub fn selection_owner(self) -> Option { + match self { + Self::Selecting { owner, .. } => Some(owner), + Self::Inactive | Self::PendingCapture { .. } | Self::Armed { .. } => None, + } + } + + pub fn pending_source(self) -> Option { + match self { + Self::PendingCapture { source, .. } => Some(source), + Self::Inactive | Self::Armed { .. } | Self::Selecting { .. } => None, + } + } + + fn auto_froze(self) -> bool { + match self { + Self::PendingCapture { auto_froze, .. } + | Self::Armed { auto_froze } + | Self::Selecting { auto_froze, .. } => auto_froze, + Self::Inactive => false, + } + } +} + +impl InputState { + pub(crate) fn request_copy_text_from_screen(&mut self) { + self.pending_ocr_request = true; + } + + pub(crate) fn take_pending_ocr_request(&mut self) -> bool { + std::mem::take(&mut self.pending_ocr_request) + } + + /// Record that a toolbar interaction dismissed the selector. + pub(crate) fn note_ocr_cancelled_by_toolbar(&mut self) { + self.ocr_cancelled_by_toolbar = true; + } + + pub(crate) fn take_ocr_cancelled_by_toolbar(&mut self) -> bool { + std::mem::take(&mut self.ocr_cancelled_by_toolbar) + } + + pub fn ocr_state(&self) -> OcrUiState { + self.ocr_ui_state + } + + pub fn ocr_is_active(&self) -> bool { + self.ocr_ui_state.is_active() + } + + pub fn ocr_is_engaged(&self) -> bool { + self.ocr_ui_state.is_engaged() + } + + pub(crate) fn set_ocr_pending_capture(&mut self, source: OcrCaptureSource) { + self.ocr_ui_state = OcrUiState::PendingCapture { + source, + auto_froze: matches!(source, OcrCaptureSource::Frozen), + }; + self.mark_ocr_dirty(); + } + + pub(crate) fn activate_ocr(&mut self, auto_froze: bool) { + // A capture can take long enough for another interaction to begin while + // OCR is pending. Entering the modal state must cancel it so the + // selector cannot swallow the matching release event. + self.prepare_for_screen_modal(); + self.ocr_ui_state = OcrUiState::Armed { auto_froze }; + self.mark_ocr_dirty(); + } + + /// Begin a region drag owned by `owner`. + /// + /// Ignored unless the selector is armed, so a stray press during capture + /// cannot start a region against a stale image — and, because a drag in + /// progress is no longer armed, a second device cannot take one over. + pub(crate) fn start_ocr_selection(&mut self, owner: OcrInputSource, point: (f64, f64)) -> bool { + let OcrUiState::Armed { auto_froze } = self.ocr_ui_state else { + return false; + }; + self.ocr_ui_state = OcrUiState::Selecting { + owner, + start: point, + current: point, + auto_froze, + }; + self.mark_ocr_dirty(); + true + } + + /// Move the region, if `source` is the device dragging it. Motion from any + /// other device is somebody else's and is dropped. + pub(crate) fn update_ocr_selection(&mut self, source: OcrInputSource, point: (f64, f64)) { + if let OcrUiState::Selecting { owner, current, .. } = &mut self.ocr_ui_state + && *owner == source + && *current != point + { + *current = point; + self.mark_ocr_dirty(); + } + } + + /// Whether `source` is the device dragging the region right now. The + /// release and per-device cancellation paths check this before acting, so + /// one device cannot submit or discard another's region. + pub(crate) fn ocr_selection_is_owned_by(&self, source: OcrInputSource) -> bool { + self.ocr_ui_state.selection_owner() == Some(source) + } + + /// Abandon a drag that was too small to be a region and wait for the next + /// one. A mis-click should not drop the user out of the selector, and it + /// must not release a capture the selector is still using. + pub(crate) fn rearm_ocr_selection(&mut self) { + if let OcrUiState::Selecting { auto_froze, .. } = self.ocr_ui_state { + self.ocr_ui_state = OcrUiState::Armed { auto_froze }; + self.mark_ocr_dirty(); + } + } + + /// Leave OCR selection and report whether it created frozen mode itself. + pub(crate) fn cancel_ocr(&mut self) -> bool { + let auto_froze = self.ocr_ui_state.auto_froze(); + if !matches!(self.ocr_ui_state, OcrUiState::Inactive) { + self.ocr_ui_state = OcrUiState::Inactive; + self.mark_ocr_dirty(); + } + auto_froze + } + + pub(crate) fn report_ocr_capture_failure_if_unreported(&mut self) { + if self.ui_toast.is_none() { + self.push_toast( + ToastPriority::Critical, + "ocr", + Toast::error("Screen capture for text recognition failed."), + ); + } + } + + /// The selector paints a full-surface scrim with the selected region cut + /// out of it, so every change repaints the whole surface: an incremental + /// buffer would otherwise keep the scrim from an earlier frame. + fn mark_ocr_dirty(&mut self) { + self.dirty_tracker.mark_full(); + self.needs_redraw = true; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::input::state::test_support::make_test_input_state; + use crate::input::{DrawingState, MouseButton, Tool}; + + #[test] + fn cancel_reports_auto_frozen_ownership() { + let mut state = make_test_input_state(); + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + state.activate_ocr(true); + + assert!(state.cancel_ocr()); + assert_eq!(state.ocr_state(), OcrUiState::Inactive); + } + + #[test] + fn waiting_for_zoom_does_not_claim_frozen_mode() { + let mut state = make_test_input_state(); + state.set_ocr_pending_capture(OcrCaptureSource::Zoom); + + assert!(!state.cancel_ocr()); + assert_eq!(state.ocr_state(), OcrUiState::Inactive); + } + + #[test] + fn a_drag_records_both_corners_and_the_capture_it_owns() { + let mut state = make_test_input_state(); + state.activate_ocr(true); + + assert!(state.start_ocr_selection(OcrInputSource::Pointer, (10.0, 20.0))); + state.update_ocr_selection(OcrInputSource::Pointer, (60.0, 90.0)); + assert!(state.ocr_state().is_selecting()); + + assert_eq!( + state.ocr_state().selection(), + Some(OcrSelection { + start: (10.0, 20.0), + end: (60.0, 90.0), + auto_froze: true, + }) + ); + } + + /// A pen already drawing when the selector opens leaves it `Armed`, never + /// `Selecting`: the tip-up that follows has no region to submit and is + /// swallowed. That is exactly why the backend retires the stylus contact + /// when the modal opens instead of waiting for an `Up` it will consume. + #[test] + fn opening_the_selector_mid_stroke_cancels_the_stroke_and_arms_without_a_region() { + let mut state = make_test_input_state(); + state.on_mouse_press(MouseButton::Left, 10, 20); + state.on_mouse_motion(30, 40); + assert!(matches!(state.state, DrawingState::Drawing { .. })); + + state.activate_ocr(true); + + assert!(matches!(state.state, DrawingState::Idle)); + assert_eq!(state.ocr_state(), OcrUiState::Armed { auto_froze: true }); + assert!(state.ocr_state().selection().is_none()); + } + + #[test] + fn a_press_before_the_capture_lands_does_not_start_a_region() { + let mut state = make_test_input_state(); + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + + assert!(!state.start_ocr_selection(OcrInputSource::Pointer, (10.0, 20.0))); + assert!(state.ocr_state().selection().is_none()); + } + + #[test] + fn cancelling_mid_drag_yields_no_region() { + let mut state = make_test_input_state(); + state.activate_ocr(true); + state.start_ocr_selection(OcrInputSource::Pointer, (10.0, 20.0)); + + assert!(state.cancel_ocr()); + assert_eq!(state.ocr_state().selection(), None); + // The state is gone, so a second cancel cannot release the freeze twice. + assert!(!state.cancel_ocr()); + } + + /// Clicking the OCR toolbar button while the selector is up toggles it off. + /// + /// The toolbar paths dismiss the selector on the way in — that is what makes + /// every *other* button dismiss it — so by the time the button's own action + /// is drained the state reads `Inactive`. The latch is what tells those two + /// halves apart, and it must not survive the batch that set it. + #[test] + fn the_toolbar_cancel_latch_marks_one_batch_and_then_clears() { + let mut state = make_test_input_state(); + assert!(!state.take_ocr_cancelled_by_toolbar()); + + state.activate_ocr(true); + state.cancel_ocr(); + state.note_ocr_cancelled_by_toolbar(); + state.request_copy_text_from_screen(); + + assert!(state.take_ocr_cancelled_by_toolbar()); + assert!(state.take_pending_ocr_request()); + assert!( + !state.take_ocr_cancelled_by_toolbar(), + "the latch outlived the interaction that set it" + ); + } + + /// The seat is not modal even though Wayscriber is: a pen can hover, and a + /// contact retired when the selector opened keeps reporting until it lifts. + /// Neither may steer a region the mouse is dragging. + #[test] + fn only_the_device_that_started_a_region_can_move_or_finish_it() { + let mut state = make_test_input_state(); + state.activate_ocr(true); + assert!(state.start_ocr_selection(OcrInputSource::Pointer, (10.0, 20.0))); + + state.update_ocr_selection(OcrInputSource::Stylus, (500.0, 500.0)); + state.update_ocr_selection(OcrInputSource::Touch, (400.0, 400.0)); + assert_eq!( + state.ocr_state().selection().map(|region| region.end), + Some((10.0, 20.0)), + "another device dragged the pointer's region" + ); + + state.update_ocr_selection(OcrInputSource::Pointer, (60.0, 90.0)); + assert_eq!( + state.ocr_state().selection().map(|region| region.end), + Some((60.0, 90.0)) + ); + + assert!(!state.ocr_selection_is_owned_by(OcrInputSource::Stylus)); + assert!(!state.ocr_selection_is_owned_by(OcrInputSource::Touch)); + assert!(state.ocr_selection_is_owned_by(OcrInputSource::Pointer)); + } + + /// A second device pressing mid-drag must not take the region over: the + /// selector is no longer armed, so its press finds nothing to start. + #[test] + fn a_second_device_cannot_take_over_a_region_in_progress() { + let mut state = make_test_input_state(); + state.activate_ocr(true); + assert!(state.start_ocr_selection(OcrInputSource::Touch, (10.0, 20.0))); + + assert!(!state.start_ocr_selection(OcrInputSource::Pointer, (200.0, 200.0))); + + assert!(state.ocr_selection_is_owned_by(OcrInputSource::Touch)); + assert_eq!( + state.ocr_state().selection().map(|region| region.start), + Some((10.0, 20.0)) + ); + } + + /// Ownership is per drag, not per session: after one device's region ends, + /// the re-armed selector accepts the next press from any device. + #[test] + fn ownership_resets_with_each_region() { + let mut state = make_test_input_state(); + state.activate_ocr(true); + state.start_ocr_selection(OcrInputSource::Stylus, (10.0, 20.0)); + state.rearm_ocr_selection(); + + assert!(state.ocr_state().selection_owner().is_none()); + assert!(state.start_ocr_selection(OcrInputSource::Pointer, (30.0, 40.0))); + assert!(state.ocr_selection_is_owned_by(OcrInputSource::Pointer)); + } + + #[test] + fn a_mis_click_rearms_the_selector_instead_of_leaving_it() { + let mut state = make_test_input_state(); + state.activate_ocr(true); + state.start_ocr_selection(OcrInputSource::Pointer, (10.0, 20.0)); + + state.rearm_ocr_selection(); + + assert_eq!(state.ocr_state(), OcrUiState::Armed { auto_froze: true }); + assert!(state.ocr_is_active()); + } + + #[test] + fn activation_cancels_an_interaction_started_while_capture_was_pending() { + let mut state = make_test_input_state(); + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + state.on_mouse_press(MouseButton::Left, 10, 20); + assert!(matches!(state.state, DrawingState::Drawing { .. })); + + state.activate_ocr(true); + + assert!(matches!(state.state, DrawingState::Idle)); + assert!(state.active_drag_button.is_none()); + assert!(state.ocr_is_active()); + // The gesture cancelled here started during the wait, not at the + // request — and the selector arms with no region, so the release that + // ends it is swallowed. Activation is therefore the only point that can + // retire the backend contact behind it. + assert_eq!(state.ocr_state(), OcrUiState::Armed { auto_froze: true }); + assert!(state.ocr_state().selection().is_none()); + } + + /// OCR is an action, not a tool: whatever the user was drawing with before + /// must still be selected after the selector opens, drags, and closes. + #[test] + fn the_whole_selector_lifecycle_preserves_every_representative_tool() { + for tool in [ + Tool::Pen, + Tool::Select, + Tool::Marker, + Tool::Eraser, + Tool::Arrow, + Tool::Highlight, + Tool::Blur, + ] { + let mut state = make_test_input_state(); + state.set_tool_override(Some(tool)); + + state.request_copy_text_from_screen(); + assert!(state.take_pending_ocr_request()); + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + state.activate_ocr(true); + state.start_ocr_selection(OcrInputSource::Pointer, (10.0, 10.0)); + state.update_ocr_selection(OcrInputSource::Pointer, (80.0, 60.0)); + state.cancel_ocr(); + + assert_eq!( + state.active_tool(), + tool, + "OCR changed the tool away from {tool:?}" + ); + } + } + + /// Nothing about a recognition is a canvas edit. One undo after the whole + /// lifecycle must remove the stroke drawn before it — if OCR had pushed a + /// history entry, that undo would have been spent on OCR instead. + #[test] + fn the_selector_lifecycle_adds_no_shape_and_no_history_entry() { + let mut state = make_test_input_state(); + state.on_mouse_press(MouseButton::Left, 0, 0); + state.on_mouse_motion(10, 10); + state.on_mouse_release(MouseButton::Left, 10, 10); + assert_eq!(state.boards.active_frame().shapes.len(), 1); + + state.activate_ocr(true); + state.start_ocr_selection(OcrInputSource::Pointer, (10.0, 10.0)); + state.update_ocr_selection(OcrInputSource::Pointer, (120.0, 90.0)); + state.cancel_ocr(); + assert_eq!(state.boards.active_frame().shapes.len(), 1); + + state.handle_action(crate::domain::Action::Undo); + + assert_eq!(state.boards.active_frame().shapes.len(), 0); + } + + /// The selector swallows every key press it receives. A key still held + /// from before it opened — from the toolbar, the palette, or a mouse path — + /// must not keep repeating into the canvas behind it. + #[test] + fn an_engaged_selector_owns_the_keyboard_and_stops_canvas_key_repeat() { + let mut state = make_test_input_state(); + assert!(!state.modal_blocks_canvas_key_repeat()); + assert!(!state.modal_owns_text_input()); + + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + assert!(state.modal_blocks_canvas_key_repeat()); + assert!(state.modal_owns_text_input()); + + state.activate_ocr(true); + assert!(state.modal_blocks_canvas_key_repeat()); + assert!(state.modal_owns_text_input()); + + state.cancel_ocr(); + assert!(!state.modal_blocks_canvas_key_repeat()); + assert!(!state.modal_owns_text_input()); + } + + #[test] + fn capture_failure_preserves_a_more_specific_existing_error() { + let mut state = make_test_input_state(); + state.push_toast( + ToastPriority::Critical, + "ocr", + Toast::error("Freeze failed after the display changed size"), + ); + + state.report_ocr_capture_failure_if_unreported(); + + assert_eq!( + state.ui_toast.as_ref().map(|toast| toast.message.as_str()), + Some("Freeze failed after the display changed size") + ); + } +} diff --git a/src/input/state/core/tool_controls/toolbar.rs b/src/input/state/core/tool_controls/toolbar.rs index 10723639..60ea75da 100644 --- a/src/input/state/core/tool_controls/toolbar.rs +++ b/src/input/state/core/tool_controls/toolbar.rs @@ -574,6 +574,7 @@ mod tests { state .toolbar_items .set_hidden(ids::TOP_UTILITY_SCREENSHOT, false); + state.toolbar_items.set_hidden(ids::TOP_UTILITY_OCR, false); state.toolbar_items.set_hidden(ids::TOP_TOOL_PEN, true); state.toolbar_items.set_hidden(section, true); state @@ -589,6 +590,9 @@ mod tests { let resolved = state.toolbar_items.resolved(); assert!(resolved.hidden.contains(&ids::TOP_UTILITY_SCREENSHOT)); + // Restored to its baseline rather than to an explicit entry. + assert!(resolved.is_hidden(ids::TOP_UTILITY_OCR)); + assert!(!resolved.hidden.contains(&ids::TOP_UTILITY_OCR)); assert!(!resolved.hidden.contains(&ids::TOP_TOOL_PEN)); assert!(resolved.hidden.contains(§ion)); assert!(resolved.hidden.contains(&ids::TOP_CHROME_OVERFLOW)); diff --git a/src/input/state/core/utility/launcher.rs b/src/input/state/core/utility/launcher.rs index 373e817c..3f081ed7 100644 --- a/src/input/state/core/utility/launcher.rs +++ b/src/input/state/core/utility/launcher.rs @@ -5,12 +5,18 @@ use crate::env_vars::CONFIGURATOR_ENV; use crate::input::state::{Toast, ToastPriority}; use std::ffi::{OsStr, OsString}; +/// Launch a helper from the Wayland callback thread. +/// +/// Deliberately the non-blocking spawn: every launcher here runs inside event +/// dispatch, so waiting for the broker transport would stall input and redraw +/// for the length of an unrelated helper. A contended launch is reported and +/// the user retries. fn spawn_detached( kind: crate::process_broker::HelperKind, program: &OsStr, arguments: &[OsString], ) -> anyhow::Result { - crate::process_broker::current()?.spawn( + crate::process_broker::current()?.try_spawn( kind, crate::process_broker::HelperLifetime::DetachedAfterExec, program, @@ -19,6 +25,23 @@ fn spawn_detached( ) } +/// Whether a launch failed only because the broker transport was busy. +/// +/// Spawn requests give up rather than stall the event thread behind a long +/// helper, so this is a "try again in a moment", not a broken install. +fn launch_deferred_by_busy_broker(error: &anyhow::Error) -> bool { + format!("{error:#}").contains(crate::process_broker::BROKER_BUSY) +} + +/// Message for a failed launch: retryable when the transport was merely busy. +fn launch_failure_message(error: &anyhow::Error, failed: &'static str) -> &'static str { + if launch_deferred_by_busy_broker(error) { + "Busy with another task — try again in a moment." + } else { + failed + } +} + fn opener_arguments(path: &std::path::Path) -> (OsString, Vec) { if cfg!(target_os = "macos") { ("open".into(), vec![path.as_os_str().into()]) @@ -72,7 +95,10 @@ impl InputState { self.push_toast( ToastPriority::Critical, "launcher", - Toast::error("Failed to open About (see logs)."), + Toast::error(launch_failure_message( + &err, + "Failed to open About (see logs).", + )), ); } } @@ -102,8 +128,12 @@ impl InputState { log::error!("Set {CONFIGURATOR_ENV} to override the executable path if needed."); // The fallback hands the file to the desktop's default editor, // which has no concept of a configurator screen: the user still - // reaches the settings, just not the destination. - if self.open_config_file_default() { + // reaches the settings, just not the destination. A busy + // transport is not the configurator being unavailable, though — + // falling back there would open a text editor for what is really + // "try again", and would take a second run at the same busy + // transport to do it. + if !launch_deferred_by_busy_broker(&err) && self.open_config_file_default() { log::info!( "Opened config file with default application because wayscriber-configurator was unavailable" ); @@ -111,7 +141,10 @@ impl InputState { self.push_toast( ToastPriority::Critical, "launcher", - Toast::error("Failed to launch configurator (see logs)."), + Toast::error(launch_failure_message( + &err, + "Failed to launch configurator (see logs).", + )), ); } } @@ -165,7 +198,10 @@ impl InputState { self.push_toast( ToastPriority::Critical, "launcher", - Toast::error("Failed to open capture folder."), + Toast::error(launch_failure_message( + &err, + "Failed to open capture folder.", + )), ); } } @@ -206,7 +242,7 @@ impl InputState { self.push_toast( ToastPriority::Critical, "launcher", - Toast::error("Failed to open config file."), + Toast::error(launch_failure_message(&err, "Failed to open config file.")), ); false } diff --git a/src/input/state/interaction/actions.rs b/src/input/state/interaction/actions.rs index f2b1dc69..78655524 100644 --- a/src/input/state/interaction/actions.rs +++ b/src/input/state/interaction/actions.rs @@ -139,6 +139,7 @@ pub(crate) fn classify_action(action: Action) -> ActionRoute { | Action::ExportCanvasClipboardAndFile | Action::ExportBoardPdfFile | Action::ExportAllBoardsPdfFile + | Action::CopyTextFromScreen | Action::ToggleFrozenMode | Action::ZoomIn | Action::ZoomOut diff --git a/src/input/state/mod.rs b/src/input/state/mod.rs index 7c1310bb..3106acf6 100644 --- a/src/input/state/mod.rs +++ b/src/input/state/mod.rs @@ -29,8 +29,9 @@ pub use core::{ ContextMenuEntry, ContextMenuKind, ContextMenuState, DesktopEnvironment, DrawingState, EyedropperCaptureSource, EyedropperUiState, HelpOverlayClick, HelpOverlayCursorHint, HelpOverlayReleaseOutcome, ImeCompositionState, ImePreedit, InputState, MAX_STROKE_THICKNESS, - MIN_STROKE_THICKNESS, OutputFocusAction, PRESET_FEEDBACK_DURATION_MS, PRESET_TOAST_DURATION_MS, - PickerDrag, PrecisionEntryState, PresetAction, PresetFeedbackKind, PressureThicknessEditMode, + MIN_STROKE_THICKNESS, OcrCaptureSource, OcrInputSource, OcrSelection, OcrUiState, + OutputFocusAction, PRESET_FEEDBACK_DURATION_MS, PRESET_TOAST_DURATION_MS, PickerDrag, + PrecisionEntryState, PresetAction, PresetFeedbackKind, PressureThicknessEditMode, PressureThicknessEntryMode, QuickColorEdit, RADIAL_COMPASS_SLICES, RADIAL_PAINT_DELAY, RADIAL_TOOL_SEGMENT_COUNT, RadialMenuLayout, RadialMenuState, RadialParent, RadialRingSwatch, RadialSegmentId, RadialSlice, RadialSliceKind, SIZE_RING_ARC_SPAN, SIZE_RING_ARC_START, diff --git a/src/lib.rs b/src/lib.rs index a00096c3..a76a98ef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,7 @@ pub mod build_info; pub mod canvas_export; pub mod capture; mod cli; +pub(crate) mod clipboard_text; pub mod config; pub mod configurator_destination; mod daemon; @@ -25,6 +26,7 @@ pub mod input; mod label_format; mod logger; mod notification; +mod ocr; mod onboarding; pub mod palette_recents; pub mod paths; diff --git a/src/ocr/AGENTS.md b/src/ocr/AGENTS.md new file mode 100644 index 00000000..2eb4a4f1 --- /dev/null +++ b/src/ocr/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS.md + +## Scope +- Applies to screen text recognition (`Copy text from screen`) under `src/ocr/`. +- Covers the capacity-one controller, the Tesseract/`wl-copy` adapters, and the typed outcomes the event loop consumes. +- Region selection, capture ownership, and toasts live in `src/backend/wayland/state/ocr.rs`, not here. + +## Architecture +- `mod.rs` owns the request/outcome vocabulary and `run_request`, which encodes, recognizes, and publishes inside one worker stack frame. +- `controller.rs` is the identified capacity-one transport: one in-flight request, a terminal worker message, and an event-loop wake. +- `tesseract.rs` is the production adapter pair; tests substitute `TextRecognizer`/`OcrTextPublisher` fakes. + +## Invariants +- Recognized text must never reach application state, a log line, or a `Debug` rendering. `RecognizedText` redacts its own `Debug`; keep it that way. +- Never invoke a shell. Tesseract takes an explicit argument vector through the process broker's `HelperKind::Tesseract` allowlist. +- Language values arrive already validated by `config::validate_ocr_languages`; this module does not decide what is acceptable. +- The temporary PNG is deleted on every path, including failures and panics. +- Capacity one is deliberate: report busy rather than queueing a screen region the user has moved on from. + +## Coupled Changes +- Failure categories couple to the toasts in `src/backend/wayland/state/ocr.rs`. +- The invocation policy couples to `src/process_broker/manifest.rs` and its tests. +- `capture.ocr_languages` couples to `src/config/types/capture.rs`, the configurator Capture page, `config.example.toml`, and `docs/CONFIG.md`. +- Packaging guidance for Tesseract lives in `packaging/` and `README.md`. + +## Validation +- `cargo test ocr` covers the controller, adapters, and privacy contract. +- Run `./tools/lint-and-test.sh` for changes that touch the broker policy or config surface. +- Engine-dependent behavior (missing language data, real recognition quality) needs the live Wayland check; the unit tests use fakes. diff --git a/src/ocr/controller.rs b/src/ocr/controller.rs new file mode 100644 index 00000000..c548a938 --- /dev/null +++ b/src/ocr/controller.rs @@ -0,0 +1,567 @@ +//! Capacity-one completion transport for screen text recognition. +//! +//! Modelled on the clipboard controller: one identified request at a time, a +//! worker thread that always publishes a terminal message, and an event-loop +//! wake so the answer is never left waiting for unrelated input. Capacity one +//! is deliberate — a queued request would recognize a screen region the user +//! has already moved on from. + +use std::fmt; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::mpsc::{Receiver, SyncSender, TryRecvError, TrySendError}; + +use crate::backend::wayland::RuntimeWakeHandle; + +use super::{ + OcrOutcome, OcrRequest, OcrTextPublisher, TesseractRecognizer, TextRecognizer, WlCopyPublisher, + run_request, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct OcrRequestId(u64); + +impl fmt::Display for OcrRequestId { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(formatter) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum OcrSubmitError { + /// Another recognition is still running. + Busy { + active_id: OcrRequestId, + }, + IdentityExhausted, + Unhealthy, + SpawnFailed { + reason: String, + }, +} + +impl fmt::Display for OcrSubmitError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Busy { active_id } => { + write!(formatter, "OCR request {active_id} is still active") + } + Self::IdentityExhausted => formatter.write_str("OCR request IDs exhausted"), + Self::Unhealthy => formatter.write_str("OCR controller is unhealthy"), + Self::SpawnFailed { reason } => { + write!(formatter, "failed to spawn OCR worker: {reason}") + } + } + } +} + +#[derive(Debug)] +pub(crate) enum OcrPoll { + Idle, + /// A request is running. The identity stays with the controller until it + /// completes, so callers only need to know that OCR is busy. + Pending, + Ready { + id: OcrRequestId, + outcome: OcrOutcome, + }, + /// The worker panicked or exited without publishing an outcome. + WorkerLost { + id: OcrRequestId, + reason: String, + }, +} + +enum WorkerMessage { + Ready { + id: OcrRequestId, + outcome: OcrOutcome, + }, + Panicked { + id: OcrRequestId, + reason: String, + }, +} + +struct ActiveRequest { + id: OcrRequestId, + receiver: Receiver, +} + +pub(crate) struct OcrController { + next_id: Option, + runtime_wake: RuntimeWakeHandle, + active: Option, + healthy: bool, +} + +impl OcrController { + pub(crate) fn new(runtime_wake: RuntimeWakeHandle) -> Self { + Self { + next_id: Some(1), + runtime_wake, + active: None, + healthy: true, + } + } + + pub(crate) fn is_active(&self) -> bool { + self.active.is_some() + } + + /// Submit a request to the production Tesseract/`wl-copy` adapters. + pub(crate) fn try_submit( + &mut self, + request: OcrRequest, + ) -> Result { + self.try_submit_with(request, TesseractRecognizer, WlCopyPublisher) + } + + pub(crate) fn try_submit_with( + &mut self, + request: OcrRequest, + recognizer: impl TextRecognizer + Send + 'static, + publisher: impl OcrTextPublisher + Send + 'static, + ) -> Result { + self.try_submit_with_spawner(request, recognizer, publisher, |job| { + std::thread::Builder::new() + .name("wayscriber-ocr".to_string()) + .spawn(job) + .map(drop) + }) + } + + fn try_submit_with_spawner( + &mut self, + request: OcrRequest, + recognizer: impl TextRecognizer + Send + 'static, + publisher: impl OcrTextPublisher + Send + 'static, + spawn: impl FnOnce(Box) -> std::io::Result<()>, + ) -> Result { + if !self.healthy { + return Err(OcrSubmitError::Unhealthy); + } + if let Some(active) = &self.active { + return Err(OcrSubmitError::Busy { + active_id: active.id, + }); + } + let value = self.next_id.ok_or(OcrSubmitError::IdentityExhausted)?; + let id = OcrRequestId(value); + + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let runtime_wake = self.runtime_wake.clone(); + let job = Box::new(move || { + let guard = WorkerExitGuard::new(id, sender, runtime_wake); + let message = match catch_unwind(AssertUnwindSafe(|| { + run_request(request, &recognizer, &publisher) + })) { + Ok(outcome) => WorkerMessage::Ready { id, outcome }, + Err(payload) => WorkerMessage::Panicked { + id, + reason: panic_reason(&payload), + }, + }; + guard.publish(message); + }); + if let Err(err) = spawn(job) { + return Err(OcrSubmitError::SpawnFailed { + reason: err.to_string(), + }); + } + + self.next_id = value.checked_add(1); + self.active = Some(ActiveRequest { id, receiver }); + Ok(id) + } + + pub(crate) fn poll(&mut self) -> OcrPoll { + let Some(active) = self.active.as_ref() else { + return OcrPoll::Idle; + }; + let active_id = active.id; + match active.receiver.try_recv() { + Err(TryRecvError::Empty) => OcrPoll::Pending, + Err(TryRecvError::Disconnected) => { + self.active = None; + OcrPoll::WorkerLost { + id: active_id, + reason: "OCR worker exited without an outcome".to_string(), + } + } + Ok(WorkerMessage::Ready { id, outcome }) if id == active_id => { + self.active = None; + OcrPoll::Ready { id, outcome } + } + Ok(WorkerMessage::Panicked { id, reason }) if id == active_id => { + self.active = None; + OcrPoll::WorkerLost { id, reason } + } + Ok(WorkerMessage::Ready { id, .. } | WorkerMessage::Panicked { id, .. }) => { + self.healthy = false; + self.active = None; + OcrPoll::WorkerLost { + id: active_id, + reason: format!( + "OCR worker reported request identity {id}, expected {active_id}" + ), + } + } + } + } +} + +fn panic_reason(payload: &Box) -> String { + if let Some(message) = payload.downcast_ref::<&'static str>() { + (*message).to_string() + } else if let Some(message) = payload.downcast_ref::() { + message.clone() + } else { + "OCR worker panicked with a non-string payload".to_string() + } +} + +struct WorkerExitGuard { + id: OcrRequestId, + sender: Option>, + runtime_wake: RuntimeWakeHandle, + terminal_published: bool, +} + +impl WorkerExitGuard { + fn new( + id: OcrRequestId, + sender: SyncSender, + runtime_wake: RuntimeWakeHandle, + ) -> Self { + Self { + id, + sender: Some(sender), + runtime_wake, + terminal_published: false, + } + } + + fn publish(mut self, message: WorkerMessage) { + let result = self + .sender + .as_ref() + .expect("OCR worker sender retained until publication") + .try_send(message); + self.sender.take(); + self.terminal_published = true; + match result { + Ok(()) | Err(TrySendError::Disconnected(_)) => {} + Err(TrySendError::Full(_)) => { + log::error!("OCR worker {} found an impossible full channel", self.id); + } + } + if let Err(err) = self.runtime_wake.wake() { + log::error!("Failed to wake runtime for OCR request {}: {err}", self.id); + } + } +} + +impl Drop for WorkerExitGuard { + fn drop(&mut self) { + if self.terminal_published { + return; + } + // Closing the worker side is the terminal publication for a disconnect. + self.sender.take(); + if let Err(err) = self.runtime_wake.wake() { + log::error!( + "Failed to wake runtime for disconnected OCR request {}: {err}", + self.id + ); + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::mpsc; + use std::time::Duration; + + use super::*; + use crate::backend::wayland::RuntimeWakeSource; + use crate::ocr::{ + OcrFailure, OcrLanguages, OcrPixels, OcrSuccess, RecognizedOutput, RecognizedText, + }; + + struct FakeRecognizer { + outcome: fn() -> Result, + } + + impl TextRecognizer for FakeRecognizer { + fn recognize( + &self, + _png: &[u8], + _languages: &OcrLanguages, + ) -> Result { + (self.outcome)() + } + } + + struct BlockingRecognizer { + started: mpsc::Sender<()>, + release: std::sync::Mutex>, + } + + impl TextRecognizer for BlockingRecognizer { + fn recognize( + &self, + _png: &[u8], + _languages: &OcrLanguages, + ) -> Result { + self.started.send(()).unwrap(); + self.release + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(5)) + .unwrap(); + Ok(RecognizedOutput { + text: RecognizedText::trimmed("done"), + replaced_invalid_utf8: false, + }) + } + } + + struct PanickingRecognizer; + + impl TextRecognizer for PanickingRecognizer { + fn recognize( + &self, + _png: &[u8], + _languages: &OcrLanguages, + ) -> Result { + panic!("expected OCR worker panic"); + } + } + + struct NoopPublisher; + + impl OcrTextPublisher for NoopPublisher { + fn publish(&self, _text: &str) -> Result<(), OcrFailure> { + Ok(()) + } + } + + fn request() -> OcrRequest { + OcrRequest { + pixels: OcrPixels { + width: 2, + height: 2, + stride: 8, + data: vec![0xFF; 16], + }, + languages: OcrLanguages::from_validated("eng".to_string()), + } + } + + fn controller() -> (RuntimeWakeSource, OcrController) { + let wake = RuntimeWakeSource::new().unwrap(); + let controller = OcrController::new(wake.handle()); + (wake, controller) + } + + fn wait_for_wake(wake: &RuntimeWakeSource) { + assert!( + wake.wait_readable(Some(Duration::from_secs(5))).unwrap(), + "OCR completion did not wake the event loop" + ); + } + + #[test] + fn one_request_runs_off_thread_and_wakes_the_event_loop() { + let (wake, mut controller) = controller(); + let id = controller + .try_submit_with( + request(), + FakeRecognizer { + outcome: || { + Ok(RecognizedOutput { + text: RecognizedText::trimmed("copied text"), + replaced_invalid_utf8: false, + }) + }, + }, + NoopPublisher, + ) + .unwrap(); + + wait_for_wake(&wake); + let poll = controller.poll(); + assert!(matches!( + poll, + OcrPoll::Ready { + id: ready, + outcome: Ok(OcrSuccess::Copied { + character_count: 11, + replaced_invalid_utf8: false, + }), + } if ready == id + )); + assert!(!controller.is_active()); + } + + #[test] + fn a_second_request_while_active_is_rejected_rather_than_queued() { + let (_wake, mut controller) = controller(); + let (started_tx, started_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + + let first = controller + .try_submit_with( + request(), + BlockingRecognizer { + started: started_tx, + release: std::sync::Mutex::new(release_rx), + }, + NoopPublisher, + ) + .unwrap(); + started_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + + assert!(matches!(controller.poll(), OcrPoll::Pending)); + let error = controller + .try_submit_with( + request(), + FakeRecognizer { + outcome: || panic!("busy submission must not run"), + }, + NoopPublisher, + ) + .unwrap_err(); + assert_eq!(error, OcrSubmitError::Busy { active_id: first }); + + release_tx.send(()).unwrap(); + } + + #[test] + fn engine_failures_reach_the_event_loop_as_typed_outcomes() { + let (wake, mut controller) = controller(); + controller + .try_submit_with( + request(), + FakeRecognizer { + outcome: || Err(OcrFailure::TimedOut), + }, + NoopPublisher, + ) + .unwrap(); + + wait_for_wake(&wake); + assert!(matches!( + controller.poll(), + OcrPoll::Ready { + outcome: Err(OcrFailure::TimedOut), + .. + } + )); + } + + #[test] + fn worker_panic_is_reported_without_poisoning_the_controller() { + let (wake, mut controller) = controller(); + let id = controller + .try_submit_with(request(), PanickingRecognizer, NoopPublisher) + .unwrap(); + + wait_for_wake(&wake); + assert!(matches!( + controller.poll(), + OcrPoll::WorkerLost { id: lost, reason } if lost == id && reason.contains("expected OCR worker panic") + )); + assert!( + controller + .try_submit_with( + request(), + FakeRecognizer { + outcome: || { + Ok(RecognizedOutput { + text: RecognizedText::trimmed("after panic"), + replaced_invalid_utf8: false, + }) + }, + }, + NoopPublisher, + ) + .is_ok() + ); + } + + #[test] + fn a_worker_that_exits_without_publishing_is_reported_as_lost() { + let (wake, mut controller) = controller(); + let id = OcrRequestId(9); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + controller.active = Some(ActiveRequest { id, receiver }); + + drop(WorkerExitGuard::new(id, sender, wake.handle())); + + wait_for_wake(&wake); + assert!(matches!( + controller.poll(), + OcrPoll::WorkerLost { id: lost, .. } if lost == id + )); + } + + #[test] + fn spawn_failure_leaves_no_active_request() { + let (_wake, mut controller) = controller(); + let error = controller + .try_submit_with_spawner( + request(), + FakeRecognizer { + outcome: || Err(OcrFailure::EngineFailed), + }, + NoopPublisher, + |_job| Err(std::io::Error::other("injected spawn failure")), + ) + .unwrap_err(); + + assert_eq!( + error, + OcrSubmitError::SpawnFailed { + reason: "injected spawn failure".to_string(), + } + ); + assert!(!controller.is_active()); + } + + #[test] + fn identity_is_never_reused_and_exhaustion_is_reported() { + let (wake, mut controller) = controller(); + controller.next_id = Some(u64::MAX); + let id = controller + .try_submit_with( + request(), + FakeRecognizer { + outcome: || { + Ok(RecognizedOutput { + text: RecognizedText::trimmed("x"), + replaced_invalid_utf8: false, + }) + }, + }, + NoopPublisher, + ) + .unwrap(); + assert_eq!(id, OcrRequestId(u64::MAX)); + + wait_for_wake(&wake); + assert!(matches!(controller.poll(), OcrPoll::Ready { .. })); + assert_eq!( + controller + .try_submit_with( + request(), + FakeRecognizer { + outcome: || Err(OcrFailure::EngineFailed), + }, + NoopPublisher, + ) + .unwrap_err(), + OcrSubmitError::IdentityExhausted + ); + } +} diff --git a/src/ocr/mod.rs b/src/ocr/mod.rs new file mode 100644 index 00000000..4be4ac61 --- /dev/null +++ b/src/ocr/mod.rs @@ -0,0 +1,413 @@ +//! One-shot text recognition for a selected screen region. +//! +//! The controller runs at most one request at a time. Its worker owns the whole +//! job — PNG encoding, the Tesseract invocation, and the clipboard publication — +//! so the event loop only ever learns a request identity and a privacy-safe +//! outcome. Recognized text never reaches application state, a log line, or a +//! `Debug` rendering. + +use std::fmt; + +mod controller; +mod tesseract; + +pub(crate) use controller::{OcrController, OcrPoll, OcrSubmitError}; +pub(crate) use tesseract::{TesseractRecognizer, WlCopyPublisher}; + +/// Turns encoded image bytes into text. The production implementation shells +/// out to Tesseract; tests substitute deterministic fakes. +pub(crate) trait TextRecognizer { + fn recognize( + &self, + png: &[u8], + languages: &OcrLanguages, + ) -> Result; +} + +/// Publishes recognized text to the system clipboard. +pub(crate) trait OcrTextPublisher { + fn publish(&self, text: &str) -> Result<(), OcrFailure>; +} + +/// Tightly packed premultiplied ARGB32 pixels in Cairo's native byte order. +/// +/// Owned by the request: the crop is taken from the displayed screen image on +/// the event thread and handed to the worker, so a later freeze, zoom, or +/// display change cannot change what is recognized. +pub(crate) struct OcrPixels { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) stride: i32, + pub(crate) data: Vec, +} + +impl fmt::Debug for OcrPixels { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OcrPixels") + .field("width", &self.width) + .field("height", &self.height) + .field("stride", &self.stride) + .field("bytes", &self.data.len()) + .finish() + } +} + +/// A validated Tesseract language argument, such as `eng` or `eng+deu`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct OcrLanguages(String); + +impl OcrLanguages { + /// Wrap an already validated value. Config parsing is the only place that + /// decides what is acceptable; see `config::capture::validate_ocr_languages`. + pub(crate) fn from_validated(value: String) -> Self { + Self(value) + } + + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Debug)] +pub(crate) struct OcrRequest { + pub(crate) pixels: OcrPixels, + pub(crate) languages: OcrLanguages, +} + +/// Recognized text on its way to the clipboard. +/// +/// Deliberately opaque: a plain `String` here would eventually reach a log line +/// or an error message and put whatever was on screen into a file. Only the +/// character count and the redacted `Debug` leave this type. +pub(crate) struct RecognizedText(String); + +impl RecognizedText { + /// Trim the recognizer's output. Tesseract always ends with a newline and + /// pads empty regions with whitespace, so the trimmed value is what decides + /// between a copy and `No text found`. + pub(crate) fn trimmed(raw: &str) -> Self { + Self(raw.trim().to_string()) + } + + pub(crate) fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub(crate) fn character_count(&self) -> usize { + self.0.chars().count() + } + + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for RecognizedText { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "RecognizedText()", + self.character_count() + ) + } +} + +/// What a finished recognition produced. Carries no recognized text. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum OcrSuccess { + Copied { + character_count: usize, + /// The engine returned bytes that were not valid UTF-8 and unreadable + /// sequences were replaced before copying. + replaced_invalid_utf8: bool, + }, + NoTextFound, +} + +/// Why a recognition did not copy anything. Each variant is a distinct +/// user-facing message; engine detail stays in the debug log. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum OcrFailure { + /// No `tesseract` executable on `PATH`. + EngineMissing, + /// Tesseract could not load the configured language data. + LanguageMissing { languages: String }, + /// Tesseract exited unsuccessfully for some other reason. + EngineFailed, + /// Tesseract did not finish inside the timeout. + TimedOut, + /// Recognized output exceeded the stdout cap. + OutputTooLarge, + /// The selected pixels could not be encoded as PNG. + EncodeFailed, + /// The temporary PNG could not be created or written. + TemporaryFileFailed, + /// The process broker was unavailable or rejected the request. + EngineUnavailable, + /// Text was recognized but `wl-copy` did not accept it. + ClipboardFailed, +} + +impl OcrFailure { + /// Stable user-facing message. Never includes engine output. + pub(crate) fn message(&self) -> String { + match self { + Self::EngineMissing => "Install Tesseract to use screen text recognition.".to_string(), + Self::LanguageMissing { languages } => { + format!("Tesseract has no language data for \"{languages}\".") + } + Self::EngineFailed => "Screen text recognition failed.".to_string(), + Self::TimedOut => "Screen text recognition timed out.".to_string(), + Self::OutputTooLarge => "That region produced too much text to copy.".to_string(), + Self::EncodeFailed => { + "Could not prepare the selected region for recognition.".to_string() + } + Self::TemporaryFileFailed => { + "Could not create a temporary file for recognition.".to_string() + } + Self::EngineUnavailable => "Screen text recognition is unavailable.".to_string(), + Self::ClipboardFailed => "Text recognized, but clipboard copy failed.".to_string(), + } + } +} + +pub(crate) type OcrOutcome = Result; + +/// Run one complete request on the worker thread. +/// +/// Encoding, recognition, and publication live together so the recognized text +/// is created and consumed inside a single stack frame and is dropped before +/// the outcome travels back to the event loop. +fn run_request( + request: OcrRequest, + recognizer: &dyn TextRecognizer, + publisher: &dyn OcrTextPublisher, +) -> OcrOutcome { + let png = encode_png(&request.pixels)?; + let recognized = recognizer.recognize(&png, &request.languages)?; + if recognized.text.is_empty() { + return Ok(OcrSuccess::NoTextFound); + } + let character_count = recognized.text.character_count(); + publisher.publish(recognized.text.as_str())?; + Ok(OcrSuccess::Copied { + character_count, + replaced_invalid_utf8: recognized.replaced_invalid_utf8, + }) +} + +/// A recognizer result: the trimmed text plus whether the engine's bytes had to +/// be repaired to become UTF-8. +pub(crate) struct RecognizedOutput { + pub(crate) text: RecognizedText, + pub(crate) replaced_invalid_utf8: bool, +} + +fn encode_png(pixels: &OcrPixels) -> Result, OcrFailure> { + if pixels.width == 0 || pixels.height == 0 { + return Err(OcrFailure::EncodeFailed); + } + let width = i32::try_from(pixels.width).map_err(|_| OcrFailure::EncodeFailed)?; + let height = i32::try_from(pixels.height).map_err(|_| OcrFailure::EncodeFailed)?; + let surface = cairo::ImageSurface::create_for_data( + pixels.data.clone(), + cairo::Format::ARgb32, + width, + height, + pixels.stride, + ) + .map_err(|err| { + log::warn!("OCR crop surface creation failed: {err}"); + OcrFailure::EncodeFailed + })?; + let mut bytes = Vec::new(); + surface.write_to_png(&mut bytes).map_err(|err| { + log::warn!("OCR crop PNG encoding failed: {err}"); + OcrFailure::EncodeFailed + })?; + Ok(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + struct StubRecognizer(fn() -> Result); + + impl TextRecognizer for StubRecognizer { + fn recognize( + &self, + _png: &[u8], + _languages: &OcrLanguages, + ) -> Result { + (self.0)() + } + } + + struct RecordingPublisher(std::sync::Mutex>); + + impl OcrTextPublisher for RecordingPublisher { + fn publish(&self, text: &str) -> Result<(), OcrFailure> { + self.0.lock().unwrap().push(text.to_string()); + Ok(()) + } + } + + struct FailingPublisher; + + impl OcrTextPublisher for FailingPublisher { + fn publish(&self, _text: &str) -> Result<(), OcrFailure> { + Err(OcrFailure::ClipboardFailed) + } + } + + pub(super) fn pixels(width: u32, height: u32) -> OcrPixels { + OcrPixels { + width, + height, + stride: (width * 4) as i32, + data: vec![0xFF; (width * height * 4) as usize], + } + } + + fn request() -> OcrRequest { + OcrRequest { + pixels: pixels(2, 2), + languages: OcrLanguages::from_validated("eng".to_string()), + } + } + + #[test] + fn recognized_text_is_trimmed_and_never_formatted_into_debug_output() { + let text = RecognizedText::trimmed(" secret account number \n"); + assert_eq!(text.as_str(), "secret account number"); + assert_eq!(text.character_count(), 21); + + let rendered = format!("{text:?}"); + assert_eq!(rendered, "RecognizedText()"); + assert!(!rendered.contains("secret")); + } + + #[test] + fn pixel_debug_reports_geometry_without_the_captured_bytes() { + let rendered = format!("{:?}", pixels(3, 4)); + assert!(rendered.contains("width: 3")); + assert!(rendered.contains("bytes: 48")); + assert!(!rendered.contains("255")); + } + + #[test] + fn empty_recognition_does_not_publish_an_empty_clipboard_value() { + let publisher = RecordingPublisher(std::sync::Mutex::new(Vec::new())); + let outcome = run_request( + request(), + &StubRecognizer(|| { + Ok(RecognizedOutput { + text: RecognizedText::trimmed(" \n\t "), + replaced_invalid_utf8: false, + }) + }), + &publisher, + ); + + assert_eq!(outcome, Ok(OcrSuccess::NoTextFound)); + assert!(publisher.0.lock().unwrap().is_empty()); + } + + #[test] + fn successful_recognition_publishes_text_but_reports_only_a_character_count() { + let publisher = RecordingPublisher(std::sync::Mutex::new(Vec::new())); + let outcome = run_request( + request(), + &StubRecognizer(|| { + Ok(RecognizedOutput { + text: RecognizedText::trimmed("hello\n"), + replaced_invalid_utf8: false, + }) + }), + &publisher, + ); + + assert_eq!( + outcome, + Ok(OcrSuccess::Copied { + character_count: 5, + replaced_invalid_utf8: false, + }) + ); + assert_eq!(publisher.0.lock().unwrap().as_slice(), ["hello"]); + assert!(!format!("{outcome:?}").contains("hello")); + } + + #[test] + fn clipboard_failure_is_distinct_from_engine_failure() { + let outcome = run_request( + request(), + &StubRecognizer(|| { + Ok(RecognizedOutput { + text: RecognizedText::trimmed("hello"), + replaced_invalid_utf8: false, + }) + }), + &FailingPublisher, + ); + + assert_eq!(outcome, Err(OcrFailure::ClipboardFailed)); + assert_ne!( + OcrFailure::ClipboardFailed.message(), + OcrFailure::EngineFailed.message() + ); + } + + #[test] + fn every_failure_category_has_its_own_user_facing_message() { + let messages = [ + OcrFailure::EngineMissing, + OcrFailure::LanguageMissing { + languages: "deu".to_string(), + }, + OcrFailure::EngineFailed, + OcrFailure::TimedOut, + OcrFailure::OutputTooLarge, + OcrFailure::EncodeFailed, + OcrFailure::TemporaryFileFailed, + OcrFailure::EngineUnavailable, + OcrFailure::ClipboardFailed, + ] + .map(|failure| failure.message()); + let unique: std::collections::BTreeSet<_> = messages.iter().collect(); + + assert_eq!(unique.len(), messages.len()); + assert!(messages[1].contains("deu")); + } + + #[test] + fn encoding_rejects_an_empty_crop_before_touching_cairo() { + assert_eq!(encode_png(&pixels(0, 4)), Err(OcrFailure::EncodeFailed)); + assert_eq!(encode_png(&pixels(4, 0)), Err(OcrFailure::EncodeFailed)); + } + + #[test] + fn encoding_preserves_opaque_transparent_and_premultiplied_pixels() { + let source = OcrPixels { + width: 3, + height: 1, + stride: 12, + // Cairo ARgb32 is native-endian premultiplied BGRA on little-endian + // targets: opaque blue, fully transparent, half-alpha grey. + data: vec![255, 0, 0, 255, 0, 0, 0, 0, 64, 64, 64, 128], + }; + + let png = encode_png(&source).expect("crop encodes"); + let decoded = + cairo::ImageSurface::create_from_png(&mut png.as_slice()).expect("encoded PNG decodes"); + assert_eq!((decoded.width(), decoded.height()), (3, 1)); + + let mut decoded = decoded; + let stride = decoded.stride() as usize; + let data = decoded.data().expect("decoded pixels"); + assert_eq!(&data[..12], &source.data[..12]); + assert!(stride >= 12); + } +} diff --git a/src/ocr/tesseract.rs b/src/ocr/tesseract.rs new file mode 100644 index 00000000..d5ce3f7c --- /dev/null +++ b/src/ocr/tesseract.rs @@ -0,0 +1,325 @@ +//! Production adapters: local Tesseract for recognition, `wl-copy` for publication. + +use std::ffi::OsStr; +use std::io::Write; +use std::path::Path; +use std::time::Duration; + +use crate::process_broker::{HelperKind, STDOUT_CAP_EXCEEDED}; + +use super::{ + OcrFailure, OcrLanguages, OcrTextPublisher, RecognizedOutput, RecognizedText, TextRecognizer, +}; + +const TESSERACT_PROGRAM: &str = "tesseract"; +const TESSERACT_TIMEOUT: Duration = Duration::from_secs(30); +/// Recognized text is plain UTF-8; 4 MiB is far past any plausible screen +/// region and keeps a runaway engine from filling memory. +const TESSERACT_STDOUT_CAP: usize = 4 * 1024 * 1024; + +pub(crate) struct TesseractRecognizer; + +impl TextRecognizer for TesseractRecognizer { + fn recognize( + &self, + png: &[u8], + languages: &OcrLanguages, + ) -> Result { + if !program_on_path(TESSERACT_PROGRAM) { + return Err(OcrFailure::EngineMissing); + } + + // A securely created temporary file with automatic deletion on every + // path. A file rather than broker stdin because the broker's input cap + // is 16 MiB and a lossless desktop crop can exceed it. + let mut input = tempfile::Builder::new() + .prefix("wayscriber-ocr-") + .suffix(".png") + .tempfile() + .map_err(|err| { + log::warn!("OCR temporary file creation failed: {err}"); + OcrFailure::TemporaryFileFailed + })?; + input + .write_all(png) + .and_then(|()| input.as_file_mut().sync_all()) + .map_err(|err| { + log::warn!("OCR temporary file write failed: {err}"); + OcrFailure::TemporaryFileFailed + })?; + + let output = run_tesseract(input.path(), languages); + // Explicit close so the deletion error is observable rather than + // swallowed by `Drop`; either way the file is gone before returning. + if let Err(err) = input.close() { + log::warn!("OCR temporary file cleanup failed: {err}"); + } + output + } +} + +fn run_tesseract(input: &Path, languages: &OcrLanguages) -> Result { + let output = crate::process_broker::current() + .and_then(|broker| { + broker.run( + HelperKind::Tesseract, + OsStr::new(TESSERACT_PROGRAM), + tesseract_arguments(input, languages), + Vec::new(), + TESSERACT_TIMEOUT, + TESSERACT_STDOUT_CAP, + ) + }) + .map_err(|err| { + log::warn!("Failed to run tesseract: {err:#}"); + classify_broker_error(&err) + })?; + + if output.timed_out { + return Err(OcrFailure::TimedOut); + } + if output.status != 0 { + let stderr = String::from_utf8_lossy(&output.stderr); + // Engine detail is a debug/warning concern; the user sees a stable + // category message instead. + log::warn!( + "tesseract exited with status {}: {}", + output.status, + stderr.trim() + ); + return Err(classify_stderr(&stderr, languages)); + } + + // Never log stdout: it is the recognized screen content. + let replaced_invalid_utf8 = std::str::from_utf8(&output.stdout).is_err(); + let text = RecognizedText::trimmed(&String::from_utf8_lossy(&output.stdout)); + Ok(RecognizedOutput { + text, + replaced_invalid_utf8, + }) +} + +/// The invocation itself: an explicit argument vector, never a shell line. +fn tesseract_arguments<'a>(input: &'a Path, languages: &'a OcrLanguages) -> Vec<&'a OsStr> { + vec![ + input.as_os_str(), + OsStr::new("stdout"), + OsStr::new("--oem"), + OsStr::new("1"), + OsStr::new("--psm"), + OsStr::new("6"), + OsStr::new("-l"), + OsStr::new(languages.as_str()), + OsStr::new("--dpi"), + OsStr::new("300"), + OsStr::new("-c"), + OsStr::new("preserve_interword_spaces=1"), + ] +} + +/// A run that outgrows the stdout cap is rejected by the broker rather than +/// truncated, so "too much text" arrives as a transport error and has to be +/// separated from a broker that is simply unavailable. +fn classify_broker_error(error: &anyhow::Error) -> OcrFailure { + if format!("{error:#}").contains(STDOUT_CAP_EXCEEDED) { + OcrFailure::OutputTooLarge + } else { + OcrFailure::EngineUnavailable + } +} + +fn classify_stderr(stderr: &str, languages: &OcrLanguages) -> OcrFailure { + let lowered = stderr.to_ascii_lowercase(); + if lowered.contains("failed loading language") + || lowered.contains("could not load any languages") + || lowered.contains("couldn't load any languages") + || lowered.contains("error opening data file") + { + OcrFailure::LanguageMissing { + languages: languages.as_str().to_string(), + } + } else { + OcrFailure::EngineFailed + } +} + +/// Whether `program` resolves to an executable file on `PATH`. +/// +/// Checked before invoking so "not installed" is its own actionable message +/// rather than a generic spawn failure. +fn program_on_path(program: &str) -> bool { + std::env::var_os("PATH").is_some_and(|path| program_in_search_path(program, &path)) +} + +/// The search itself, over an explicit path list. Split out so tests can probe +/// a temporary directory without mutating the process-wide `PATH`, which every +/// concurrently running test shares. +fn program_in_search_path(program: &str, search_path: &OsStr) -> bool { + use std::os::unix::fs::PermissionsExt; + + std::env::split_paths(search_path).any(|directory| { + if directory.as_os_str().is_empty() { + return false; + } + std::fs::metadata(directory.join(program)) + .map(|metadata| metadata.is_file() && metadata.permissions().mode() & 0o111 != 0) + .unwrap_or(false) + }) +} + +pub(crate) struct WlCopyPublisher; + +impl OcrTextPublisher for WlCopyPublisher { + fn publish(&self, text: &str) -> Result<(), OcrFailure> { + crate::clipboard_text::copy_text_via_command(text).map_err(|err| { + log::warn!("wl-copy failed for recognized text: {err}"); + OcrFailure::ClipboardFailed + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn languages() -> OcrLanguages { + OcrLanguages::from_validated("eng+deu".to_string()) + } + + #[test] + fn invocation_matches_the_documented_argument_vector() { + let path = Path::new("/tmp/wayscriber-ocr-test.png"); + let arguments: Vec<_> = tesseract_arguments(path, &languages()) + .into_iter() + .map(|argument| argument.to_string_lossy().into_owned()) + .collect(); + + assert_eq!( + arguments, + [ + "/tmp/wayscriber-ocr-test.png", + "stdout", + "--oem", + "1", + "--psm", + "6", + "-l", + "eng+deu", + "--dpi", + "300", + "-c", + "preserve_interword_spaces=1", + ] + ); + } + + #[test] + fn missing_language_data_is_distinguished_from_a_generic_engine_failure() { + assert_eq!( + classify_stderr("Failed loading language 'deu'\n", &languages()), + OcrFailure::LanguageMissing { + languages: "eng+deu".to_string(), + } + ); + assert_eq!( + classify_stderr( + "Error opening data file /usr/share/tessdata/deu.traineddata", + &languages() + ), + OcrFailure::LanguageMissing { + languages: "eng+deu".to_string(), + } + ); + assert_eq!( + classify_stderr("Image too large to process", &languages()), + OcrFailure::EngineFailed + ); + } + + #[test] + fn path_probe_accepts_only_executable_files() { + use std::os::unix::fs::PermissionsExt; + + let directory = crate::test_temp::tempdir().unwrap(); + let program = directory.path().join("wayscriber-fake-ocr"); + std::fs::write(&program, b"#!/bin/sh\n").unwrap(); + let search_path = directory.path().as_os_str(); + + assert!( + !program_in_search_path("wayscriber-fake-ocr", search_path), + "a non-executable file is not a usable engine" + ); + + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(program_in_search_path("wayscriber-fake-ocr", search_path)); + assert!(!program_in_search_path( + "wayscriber-absent-ocr", + search_path + )); + } + + #[test] + fn path_probe_skips_empty_entries_and_searches_every_directory() { + use std::os::unix::fs::PermissionsExt; + + let directory = crate::test_temp::tempdir().unwrap(); + let program = directory.path().join("wayscriber-fake-ocr"); + std::fs::write(&program, b"#!/bin/sh\n").unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o755)).unwrap(); + + // An empty entry means "current directory" to some shells; treating it + // as one would make the probe depend on the working directory. + let search_path = + std::env::join_paths(["".as_ref(), Path::new("/nonexistent"), directory.path()]) + .unwrap(); + + assert!(program_in_search_path("wayscriber-fake-ocr", &search_path)); + } + + #[test] + fn an_oversized_result_is_distinguished_from_an_unavailable_broker() { + assert_eq!( + classify_broker_error(&anyhow::anyhow!(STDOUT_CAP_EXCEEDED)), + OcrFailure::OutputTooLarge + ); + assert_eq!( + classify_broker_error( + &anyhow::anyhow!(STDOUT_CAP_EXCEEDED).context("process broker rejected request") + ), + OcrFailure::OutputTooLarge + ); + assert_eq!( + classify_broker_error(&anyhow::anyhow!("runtime process broker is not active")), + OcrFailure::EngineUnavailable + ); + } + + #[test] + fn a_temporary_input_file_is_removed_on_every_path() { + // The recognizer owns the file through `tempfile`; the guarantee under + // test is that no wayscriber-ocr file survives a run that fails before, + // during, or after the engine call. Without a broker the run fails at + // the invocation step, which is the hardest path to clean up. + let before = temporary_ocr_files(); + let outcome = TesseractRecognizer.recognize(b"not a png", &languages()); + assert!(outcome.is_err()); + assert_eq!(temporary_ocr_files(), before); + } + + fn temporary_ocr_files() -> Vec { + let Ok(entries) = std::fs::read_dir(std::env::temp_dir()) else { + return Vec::new(); + }; + let mut paths: Vec<_> = entries + .flatten() + .map(|entry| entry.path()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("wayscriber-ocr-")) + }) + .collect(); + paths.sort(); + paths + } +} diff --git a/src/process_broker/client.rs b/src/process_broker/client.rs index e54fa18d..174b8347 100644 --- a/src/process_broker/client.rs +++ b/src/process_broker/client.rs @@ -2,7 +2,7 @@ use std::collections::VecDeque; use std::ffi::{OsStr, OsString}; use std::os::fd::{AsRawFd, OwnedFd, RawFd}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, OnceLock, Weak}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock, TryLockError, Weak}; use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; @@ -53,6 +53,38 @@ struct RunOptions { output_mode: OutputMode, } +/// Failure text used when a caller declines to wait for the transport. +/// Callers match on it to offer "try again" rather than a generic failure. +pub(crate) const BROKER_BUSY: &str = "process broker is busy running another helper"; + +/// How a caller wants the single-socket transport acquired. +/// +/// One socket serializes every exchange, and a `Run` can hold it for as long as +/// its helper lives — thirty seconds for screen text recognition, two minutes +/// for an interactive selector. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ExchangeWait { + /// Queue until the transport is free. Everything on a worker thread, the + /// daemon, and startup: those requests have to happen, and waiting for them + /// costs nothing the user can feel. + Queue, + /// Take the transport only if it is free right now. + /// + /// For requests issued from the Wayland callback thread, where any wait at + /// all stalls input and redraw dispatch. A contended launch is reported so + /// the user can retry, which is strictly better than the UI locking up for + /// the length of somebody else's helper. + Immediate, +} + +/// The transport policy and watchdog a spawn request carries, kept together so +/// the private spawn path stays inside the argument-count lint. +#[derive(Debug, Clone, Copy)] +struct SpawnOptions { + watchdog: Option, + wait: ExchangeWait, +} + static ACTIVE_BROKER: OnceLock>> = OnceLock::new(); fn active_slot() -> &'static Mutex> { @@ -150,8 +182,26 @@ impl ProcessBrokerGuard { } impl ProcessBroker { + fn lock_exchange(&self, wait: ExchangeWait) -> Result> { + match wait { + ExchangeWait::Queue => Ok(self + .inner + .exchange_lock + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner())), + // One attempt, never a timed retry: sleeping here would be the very + // stall this policy exists to avoid. + ExchangeWait::Immediate => match self.inner.exchange_lock.try_lock() { + Ok(guard) => Ok(guard), + Err(TryLockError::Poisoned(poisoned)) => Ok(poisoned.into_inner()), + Err(TryLockError::WouldBlock) => bail!(BROKER_BUSY), + }, + } + } + pub(super) fn request(&self, operation: BrokerOperation) -> Result { - let (outcome, descriptors) = self.request_with_descriptors(operation, &[])?; + let (outcome, descriptors) = + self.request_with_descriptors(operation, &[], ExchangeWait::Queue)?; if !descriptors.is_empty() { bail!("broker returned unexpected descriptors"); } @@ -162,6 +212,7 @@ impl ProcessBroker { &self, operation: BrokerOperation, descriptors: &[RawFd], + wait: ExchangeWait, ) -> Result<(BrokerOutcome, Vec)> { let request_id = crate::daemon::protocol_v2::ProtocolId::generate()?.to_string(); let exchange_timeout = broker_exchange_timeout(&operation); @@ -173,11 +224,7 @@ impl ProcessBroker { if packet.len() > MAX_PACKET_BYTES { bail!("broker request exceeds packet cap"); } - let _guard = self - .inner - .exchange_lock - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); + let _guard = self.lock_exchange(wait)?; if !self.inner.healthy.load(Ordering::Acquire) { bail!("process broker transport is no longer usable"); } @@ -290,6 +337,7 @@ impl ProcessBroker { output_mode: options.output_mode, }, &request_descriptors, + ExchangeWait::Queue, )?; match outcome { BrokerOutcome::Output { @@ -350,6 +398,7 @@ impl ProcessBroker { timeout_ms: u64::try_from(timeout.as_millis()).unwrap_or(u64::MAX), }, &request_descriptors, + ExchangeWait::Queue, )?; if !descriptors.is_empty() { bail!("broker returned unexpected publication descriptors"); @@ -384,7 +433,49 @@ impl ProcessBroker { I: IntoIterator, S: AsRef, { - self.spawn_inner(kind, lifetime, program, arguments, environment, None) + self.spawn_inner( + kind, + lifetime, + program, + arguments, + environment, + SpawnOptions { + watchdog: None, + wait: ExchangeWait::Queue, + }, + ) + } + + /// Spawn without waiting for the transport, failing with [`BROKER_BUSY`] + /// when another exchange holds it. + /// + /// For the Wayland callback thread only. Everything else — the daemon's + /// overlay spawn, the tray, startup detach — must keep queueing: those + /// requests have to happen, and failing them behind an unrelated helper + /// would turn a wait into a spurious error and a retry backoff. + pub(crate) fn try_spawn( + &self, + kind: HelperKind, + lifetime: HelperLifetime, + program: &OsStr, + arguments: I, + environment: Vec<(OsString, Option)>, + ) -> Result + where + I: IntoIterator, + S: AsRef, + { + self.spawn_inner( + kind, + lifetime, + program, + arguments, + environment, + SpawnOptions { + watchdog: None, + wait: ExchangeWait::Immediate, + }, + ) } pub(crate) fn spawn_with_watchdog( @@ -406,7 +497,10 @@ impl ProcessBroker { program, arguments, environment, - Some(watchdog), + SpawnOptions { + watchdog: Some(watchdog), + wait: ExchangeWait::Queue, + }, ) } @@ -417,7 +511,7 @@ impl ProcessBroker { program: &OsStr, arguments: I, environment: Vec<(OsString, Option)>, - watchdog: Option, + options: SpawnOptions, ) -> Result where I: IntoIterator, @@ -426,7 +520,7 @@ impl ProcessBroker { let operation = BrokerOperation::Spawn { kind, lifetime, - watchdog: watchdog.is_some(), + watchdog: options.watchdog.is_some(), program: OsWire::from_os(program)?, arguments: arguments .into_iter() @@ -443,7 +537,7 @@ impl ProcessBroker { .collect::>>()?, }; let (outcome, descriptors) = - self.request_with_descriptors(operation, watchdog.as_slice())?; + self.request_with_descriptors(operation, options.watchdog.as_slice(), options.wait)?; if !descriptors.is_empty() { bail!("broker returned unexpected spawn descriptors"); } diff --git a/src/process_broker/execution.rs b/src/process_broker/execution.rs index 4f8dd937..d9c02edf 100644 --- a/src/process_broker/execution.rs +++ b/src/process_broker/execution.rs @@ -9,7 +9,7 @@ use std::time::Duration; use anyhow::{Context, Result, anyhow}; use super::transport::shutdown_requested; -use super::wire::{HelperKind, MAX_STDERR_BYTES, OutputMode}; +use super::wire::{HelperKind, MAX_STDERR_BYTES, OutputMode, STDOUT_CAP_EXCEEDED}; pub(super) struct BoundedOutput { pub(super) status: ExitStatus, @@ -361,7 +361,7 @@ pub(super) fn run_bounded( } let stdout_limit_reached = stdout_limit_reached.load(Ordering::Acquire); if output_mode == OutputMode::Complete && stdout_limit_reached { - return Err(anyhow!("broker helper stdout exceeded output cap")); + return Err(anyhow!(STDOUT_CAP_EXCEEDED)); } if stderr_overflow.load(Ordering::Acquire) { return Err(anyhow!("broker helper stderr exceeded output cap")); diff --git a/src/process_broker/manifest.rs b/src/process_broker/manifest.rs index cae1ed32..c86607be 100644 --- a/src/process_broker/manifest.rs +++ b/src/process_broker/manifest.rs @@ -57,6 +57,7 @@ pub(super) fn validate( HelperKind::Grim => basename == "grim", HelperKind::Hyprctl => basename == "hyprctl", HelperKind::Slurp => basename == "slurp", + HelperKind::Tesseract => basename == "tesseract", HelperKind::WlPaste => basename == "wl-paste", HelperKind::WlCopy => basename == "wl-copy", HelperKind::SessionZenity => basename == "zenity", diff --git a/src/process_broker/mod.rs b/src/process_broker/mod.rs index 27ca0976..fa7c998c 100644 --- a/src/process_broker/mod.rs +++ b/src/process_broker/mod.rs @@ -15,6 +15,6 @@ mod wire; #[cfg(test)] mod tests; -pub(crate) use client::{BrokerChild, current, start_for_runtime}; +pub(crate) use client::{BROKER_BUSY, BrokerChild, current, start_for_runtime}; pub(crate) use server::run_internal_broker_if_requested; -pub(crate) use wire::{BrokerOutput, HelperKind, HelperLifetime}; +pub(crate) use wire::{BrokerOutput, HelperKind, HelperLifetime, STDOUT_CAP_EXCEEDED}; diff --git a/src/process_broker/tests.rs b/src/process_broker/tests.rs index 00f23e33..a7b84256 100644 --- a/src/process_broker/tests.rs +++ b/src/process_broker/tests.rs @@ -83,6 +83,100 @@ fn update_fetcher_manifest_allows_only_curl_and_wget() { ); } +#[test] +fn tesseract_manifest_allows_only_the_tesseract_basename() { + for program in ["/usr/bin/tesseract", "/usr/local/bin/tesseract"] { + let program = super::wire::OsWire::from_os(OsStr::new(program)).unwrap(); + super::manifest::validate(HelperKind::Tesseract, &program, &[], &[], &[]).unwrap(); + } + + for program in ["/usr/bin/sh", "/usr/bin/tesseract-wrapper", "/usr/bin/grim"] { + let program = super::wire::OsWire::from_os(OsStr::new(program)).unwrap(); + assert!( + super::manifest::validate(HelperKind::Tesseract, &program, &[], &[], &[]).is_err(), + "{program:?} must not be allowed for OCR" + ); + } +} + +#[test] +fn tesseract_reads_the_complete_recognized_output_rather_than_a_prefix() { + // Prefix mode stops the helper once the cap fills; OCR must instead see the + // cap as a distinct "too much text" failure, which needs Complete mode. + assert!(!super::manifest::supports_prefix_output( + HelperKind::Tesseract + )); + assert_eq!( + super::manifest::input_cap(HelperKind::Tesseract), + super::wire::MAX_INPUT_BYTES + ); +} + +/// A launcher click must not stall behind a long helper, and nothing else may +/// be made to fail by that policy. +/// +/// `try_spawn` is issued from the Wayland callback thread while OCR keeps the +/// overlay interactive, so queueing behind a thirty-second recognition would +/// hold input and redraw dispatch for its whole run. Plain `spawn` — the +/// daemon's overlay start, the tray, startup detach — must keep queueing, or an +/// unrelated background helper would turn a short wait into a spurious failure +/// and a retry backoff. +#[test] +fn only_the_callback_thread_spawn_declines_to_wait_for_the_transport() { + let guard = start_for_runtime().unwrap(); + let broker = guard.broker().clone(); + let (running_tx, running_rx) = std::sync::mpsc::channel(); + + let long_helper = std::thread::spawn(move || { + running_tx.send(()).unwrap(); + broker.run( + HelperKind::TestSleep, + OsStr::new("sleep"), + [OsStr::new("1")], + Vec::new(), + Duration::from_secs(10), + 1024, + ) + }); + running_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + // The helper thread has to hold the exchange before the policy is testable. + std::thread::sleep(Duration::from_millis(200)); + + let started = Instant::now(); + let declined = guard.broker().try_spawn( + HelperKind::TestSleep, + HelperLifetime::OperationBound, + OsStr::new("sleep"), + [OsStr::new("0")], + Vec::new(), + ); + let waited = started.elapsed(); + + let error = declined.expect_err("try_spawn must not queue behind the long helper"); + assert!( + format!("{error:#}").contains(crate::process_broker::BROKER_BUSY), + "unexpected try_spawn failure: {error:#}" + ); + assert!( + waited < Duration::from_millis(250), + "try_spawn slept for {waited:?} instead of declining" + ); + + // The queueing path waits it out and still succeeds. + guard + .broker() + .spawn( + HelperKind::TestSleep, + HelperLifetime::OperationBound, + OsStr::new("sleep"), + [OsStr::new("0")], + Vec::new(), + ) + .expect("spawn must queue behind the long helper rather than fail"); + + let _ = long_helper.join(); +} + #[test] fn prelock_broker_runs_bounded_helpers_and_owns_reaping() { let guard = start_for_runtime().unwrap(); diff --git a/src/process_broker/wire.rs b/src/process_broker/wire.rs index 30a4c3c9..88a655f6 100644 --- a/src/process_broker/wire.rs +++ b/src/process_broker/wire.rs @@ -20,6 +20,13 @@ pub(super) const MAX_PACKET_DESCRIPTORS: usize = 2; pub(super) const REQUIRED_MEMFD_SEALS: i32 = libc::F_SEAL_WRITE | libc::F_SEAL_GROW | libc::F_SEAL_SHRINK | libc::F_SEAL_SEAL; +/// Failure text used when a `Complete`-mode helper writes past its stdout cap. +/// +/// The broker rejects that run rather than returning a truncated result, so the +/// only signal a caller gets is this message. Callers that distinguish +/// "produced too much output" from other transport failures match on it. +pub(crate) const STDOUT_CAP_EXCEEDED: &str = "broker helper stdout exceeded output cap"; + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub(crate) enum HelperKind { @@ -29,6 +36,7 @@ pub(crate) enum HelperKind { Grim, Hyprctl, Slurp, + Tesseract, WlPaste, WlCopy, SessionZenity, diff --git a/src/toolbar_icons/actions.rs b/src/toolbar_icons/actions.rs index 240a254c..83348f59 100644 --- a/src/toolbar_icons/actions.rs +++ b/src/toolbar_icons/actions.rs @@ -383,6 +383,11 @@ pub fn draw_icon_screenshot(ctx: &Context, x: f64, y: f64, size: f64) { super::svg::render_screenshot(ctx, x, y, size); } +/// Draw a screen text recognition (OCR) icon. +pub fn draw_icon_ocr(ctx: &Context, x: f64, y: f64, size: f64) { + super::svg::render_ocr(ctx, x, y, size); +} + /// Draw a visibility/eye icon. pub fn draw_icon_visibility(ctx: &Context, x: f64, y: f64, size: f64) { let s = size; diff --git a/src/toolbar_icons/mod.rs b/src/toolbar_icons/mod.rs index 59aba8a2..30a40cb0 100644 --- a/src/toolbar_icons/mod.rs +++ b/src/toolbar_icons/mod.rs @@ -104,6 +104,7 @@ pub(crate) fn top_toolbar_icon_painter( I::Text => draw_icon_text, I::StickyNote => draw_icon_note, I::Screenshot => draw_icon_screenshot, + I::Ocr => draw_icon_ocr, I::Highlight => draw_icon_highlight, I::ClearCanvas => draw_icon_clear, I::Undo => draw_icon_undo, @@ -153,7 +154,7 @@ mod painter_tests { /// Every public painter. `svg.rs` covers the newer family through its own /// `render_*` entry points; this covers the shipped surface callers use, /// including the older proportional-style painters that had no coverage. - const PAINTERS: [(&str, IconPainter); 63] = [ + const PAINTERS: [(&str, IconPainter); 64] = [ ("arrow", draw_icon_arrow), ("blur", draw_icon_blur), ("board", draw_icon_board), @@ -187,6 +188,7 @@ mod painter_tests { ("minus", draw_icon_minus), ("more", draw_icon_more), ("note", draw_icon_note), + ("ocr", draw_icon_ocr), ("parallelogram", draw_icon_parallelogram), ("paste", draw_icon_paste), ("pen", draw_icon_pen), diff --git a/src/toolbar_icons/svg.rs b/src/toolbar_icons/svg.rs index 4b109e8c..e6d7d39a 100644 --- a/src/toolbar_icons/svg.rs +++ b/src/toolbar_icons/svg.rs @@ -273,6 +273,34 @@ fn draw_screenshot(ctx: &Context) { stroke(ctx); } +/// Screen text recognition: a scan frame around three text lines. +/// +/// Deliberately not a port of any other project's geometry — this is the same +/// normalized 24x24 Cairo style as its neighbours, with shorter corner arms +/// than `draw_screenshot` so the text block inside stays readable at 18px. +fn draw_ocr(ctx: &Context) { + for (x1, y1, x2, y2, x3, y3) in [ + (3.25, 7.0, 3.25, 3.25, 7.0, 3.25), + (17.0, 3.25, 20.75, 3.25, 20.75, 7.0), + (20.75, 17.0, 20.75, 20.75, 17.0, 20.75), + (7.0, 20.75, 3.25, 20.75, 3.25, 17.0), + ] { + ctx.move_to(x1, y1); + ctx.line_to(x2, y2); + ctx.line_to(x3, y3); + stroke(ctx); + } + + // A ragged right edge is what reads as text rather than as a stack of bars. + ctx.set_line_width(1.6); + for (y, end) in [(8.75, 16.75), (12.0, 16.75), (15.25, 13.5)] { + ctx.move_to(7.25, y); + ctx.line_to(end, y); + stroke(ctx); + } + ctx.set_line_width(2.0); +} + fn draw_highlight(ctx: &Context) { circle(ctx, 12.0, 12.0, 3.25); stroke(ctx); @@ -565,6 +593,7 @@ renderers!( (render_text, draw_text), (render_note, draw_sticky_note), (render_screenshot, draw_screenshot), + (render_ocr, draw_ocr), (render_highlight, draw_highlight), (render_undo, draw_undo), (render_redo, draw_redo), @@ -596,7 +625,7 @@ mod tests { type IconRender = fn(&Context, f64, f64, f64); const SIZES: [i32; 5] = [18, 20, 22, 24, 28]; - const ICONS: [(&str, IconRender); 33] = [ + const ICONS: [(&str, IconRender); 34] = [ ("drag", render_drag), ("select", render_select), ("pen", render_pen), @@ -609,6 +638,7 @@ mod tests { ("text", render_text), ("sticky_note", render_note), ("screenshot", render_screenshot), + ("ocr", render_ocr), ("highlight", render_highlight), ("undo", render_undo), ("redo", render_redo), diff --git a/src/ui/toolbar/apply/actions.rs b/src/ui/toolbar/apply/actions.rs index 5a7c779b..851fbf89 100644 --- a/src/ui/toolbar/apply/actions.rs +++ b/src/ui/toolbar/apply/actions.rs @@ -57,6 +57,12 @@ impl InputState { true } + pub(super) fn apply_toolbar_copy_text_from_screen(&mut self) -> bool { + self.handle_action(Action::CopyTextFromScreen); + self.close_top_toolbar_menus(); + true + } + pub(super) fn apply_toolbar_open_command_palette(&mut self) -> bool { self.handle_action(Action::ToggleCommandPalette); self.close_top_toolbar_menus(); diff --git a/src/ui/toolbar/apply/mod.rs b/src/ui/toolbar/apply/mod.rs index 455fe9de..fdd77457 100644 --- a/src/ui/toolbar/apply/mod.rs +++ b/src/ui/toolbar/apply/mod.rs @@ -94,6 +94,7 @@ impl InputState { ToolbarEvent::CustomRedo => self.apply_toolbar_custom_redo(), ToolbarEvent::ClearCanvas { instant } => self.apply_toolbar_clear_canvas(instant), ToolbarEvent::CaptureScreenshot => self.apply_toolbar_capture_screenshot(), + ToolbarEvent::CopyTextFromScreen => self.apply_toolbar_copy_text_from_screen(), ToolbarEvent::PagePrev => self.apply_toolbar_page_prev(), ToolbarEvent::PageNext => self.apply_toolbar_page_next(), ToolbarEvent::PageNew => self.apply_toolbar_page_new(), diff --git a/src/ui/toolbar/bindings.rs b/src/ui/toolbar/bindings.rs index 0e934e7f..459a3fb2 100644 --- a/src/ui/toolbar/bindings.rs +++ b/src/ui/toolbar/bindings.rs @@ -167,6 +167,14 @@ mod tests { ); } + #[test] + fn action_for_event_maps_screen_text_recognition() { + assert_eq!( + action_for_event(&ToolbarEvent::CopyTextFromScreen), + Some(Action::CopyTextFromScreen) + ); + } + #[test] fn action_for_event_returns_none_for_layout_only_events() { assert_eq!(action_for_event(&ToolbarEvent::OpenConfigFile), None); diff --git a/src/ui/toolbar/events.rs b/src/ui/toolbar/events.rs index 6e00766d..89253665 100644 --- a/src/ui/toolbar/events.rs +++ b/src/ui/toolbar/events.rs @@ -111,6 +111,8 @@ pub enum ToolbarEvent { instant: bool, }, CaptureScreenshot, + /// Select a screen region and copy the text recognized in it. + CopyTextFromScreen, PagePrev, PageNext, PageNew, diff --git a/src/ui/toolbar/model/event_policy.rs b/src/ui/toolbar/model/event_policy.rs index 00861e40..4e71debe 100644 --- a/src/ui/toolbar/model/event_policy.rs +++ b/src/ui/toolbar/model/event_policy.rs @@ -148,6 +148,7 @@ pub(crate) fn action_for_event(event: &ToolbarEvent) -> Option { ToolbarEvent::RedoAllDelayed => Some(Action::RedoAllDelayed), ToolbarEvent::ClearCanvas { .. } => Some(Action::ClearCanvas), ToolbarEvent::CaptureScreenshot => Some(Action::CaptureSelection), + ToolbarEvent::CopyTextFromScreen => Some(Action::CopyTextFromScreen), ToolbarEvent::PagePrev => Some(Action::PagePrev), ToolbarEvent::PageNext => Some(Action::PageNext), ToolbarEvent::PageNew => Some(Action::PageNew), @@ -516,6 +517,7 @@ fn persistence_for_event(event: &ToolbarEvent) -> ToolbarPersistence { | ToolbarEvent::Redo | ToolbarEvent::ClearCanvas { .. } | ToolbarEvent::CaptureScreenshot + | ToolbarEvent::CopyTextFromScreen | ToolbarEvent::PagePrev | ToolbarEvent::PageNext | ToolbarEvent::PageNew diff --git a/src/ui/toolbar/model/mod.rs b/src/ui/toolbar/model/mod.rs index a8df1a37..603a09e8 100644 --- a/src/ui/toolbar/model/mod.rs +++ b/src/ui/toolbar/model/mod.rs @@ -52,7 +52,7 @@ pub(crate) use tools::{ default_polygon_tool, default_shape_tool, is_fill_tool, is_polygon_tool, polygon_tools, semantic_icon_for_tool, shape_tools, tool_visible, toolbar_item_id_for_tool, toolbar_item_visible, top_clear_canvas_visible, top_fill_visible, top_highlight_ring_visible, - top_highlight_visible, top_screenshot_visible, top_shape_picker_visible, + top_highlight_visible, top_ocr_visible, top_screenshot_visible, top_shape_picker_visible, top_sticky_note_visible, top_text_visible, top_tool_buttons, top_tool_group, visible_shape_picker_max_row_len, visible_shape_picker_row_count, visible_shape_picker_rows, visible_tool_count, visible_top_tool_buttons, visible_top_utility_buttons, diff --git a/src/ui/toolbar/model/tools.rs b/src/ui/toolbar/model/tools.rs index 09a1a63d..96cd76c4 100644 --- a/src/ui/toolbar/model/tools.rs +++ b/src/ui/toolbar/model/tools.rs @@ -145,6 +145,10 @@ pub(crate) fn top_screenshot_visible(snapshot: &ToolbarSnapshot) -> bool { toolbar_item_visible(snapshot, ids::TOP_UTILITY_SCREENSHOT) } +pub(crate) fn top_ocr_visible(snapshot: &ToolbarSnapshot) -> bool { + toolbar_item_visible(snapshot, ids::TOP_UTILITY_OCR) +} + pub(crate) fn top_highlight_visible(snapshot: &ToolbarSnapshot) -> bool { toolbar_item_visible(snapshot, ids::TOP_UTILITY_HIGHLIGHT) } @@ -213,15 +217,17 @@ pub(crate) enum TopUtilityButton { Text, StickyNote, Screenshot, + Ocr, ClearCanvas, Highlight, IconMode, } -const DEFAULT_TOP_UTILITY_BUTTONS: [TopUtilityButton; 5] = [ +const DEFAULT_TOP_UTILITY_BUTTONS: [TopUtilityButton; 6] = [ TopUtilityButton::Text, TopUtilityButton::StickyNote, TopUtilityButton::Screenshot, + TopUtilityButton::Ocr, TopUtilityButton::Highlight, TopUtilityButton::ClearCanvas, ]; @@ -232,6 +238,7 @@ impl TopUtilityButton { Self::Text => ids::TOP_UTILITY_TEXT, Self::StickyNote => ids::TOP_UTILITY_STICKY_NOTE, Self::Screenshot => ids::TOP_UTILITY_SCREENSHOT, + Self::Ocr => ids::TOP_UTILITY_OCR, Self::ClearCanvas => ids::TOP_UTILITY_CLEAR_CANVAS, Self::Highlight => ids::TOP_UTILITY_HIGHLIGHT, Self::IconMode if snapshot.use_icons => ids::TOP_UTILITY_ICON_MODE_TEXT, @@ -278,6 +285,8 @@ fn top_utility_button_for_id( Some(TopUtilityButton::StickyNote) } else if id == ids::TOP_UTILITY_SCREENSHOT { Some(TopUtilityButton::Screenshot) + } else if id == ids::TOP_UTILITY_OCR { + Some(TopUtilityButton::Ocr) } else if id == ids::TOP_UTILITY_CLEAR_CANVAS { Some(TopUtilityButton::ClearCanvas) } else if id == ids::TOP_UTILITY_HIGHLIGHT { @@ -302,6 +311,7 @@ fn top_utility_button_visible( TopUtilityButton::Text => top_text_visible(snapshot), TopUtilityButton::StickyNote => top_sticky_note_visible(snapshot), TopUtilityButton::Screenshot => top_screenshot_visible(snapshot), + TopUtilityButton::Ocr => top_ocr_visible(snapshot), TopUtilityButton::ClearCanvas => !simple && top_clear_canvas_visible(snapshot), TopUtilityButton::Highlight => !simple && use_icons && top_highlight_visible(snapshot), TopUtilityButton::IconMode => false, diff --git a/src/ui/toolbar/model/top_spec/control.rs b/src/ui/toolbar/model/top_spec/control.rs index 5858718d..6a088e6e 100644 --- a/src/ui/toolbar/model/top_spec/control.rs +++ b/src/ui/toolbar/model/top_spec/control.rs @@ -117,6 +117,7 @@ impl TopToolbarControl { TopToolbarUtility::Text => ids::TOP_UTILITY_TEXT, TopToolbarUtility::StickyNote => ids::TOP_UTILITY_STICKY_NOTE, TopToolbarUtility::Screenshot => ids::TOP_UTILITY_SCREENSHOT, + TopToolbarUtility::Ocr => ids::TOP_UTILITY_OCR, TopToolbarUtility::Highlight => ids::TOP_UTILITY_HIGHLIGHT, }, Self::Preset(index) => return TopToolbarControlId::Preset(index), @@ -196,7 +197,7 @@ impl TopToolbarControl { Self::Utility(TopToolbarUtility::Text) => snapshot.text_active, Self::Utility(TopToolbarUtility::StickyNote) => snapshot.note_active, Self::Utility(TopToolbarUtility::Highlight) => snapshot.any_highlight_active, - Self::Utility(TopToolbarUtility::Screenshot) => false, + Self::Utility(TopToolbarUtility::Screenshot | TopToolbarUtility::Ocr) => false, // A filled slot reads as active while it is the applied preset. Self::Preset(index) => snapshot.active_preset_slot == Some(index + 1), Self::Pin => snapshot.top_pinned, @@ -267,6 +268,7 @@ impl TopToolbarControl { Self::Utility(TopToolbarUtility::Text) => TopToolbarIcon::Text, Self::Utility(TopToolbarUtility::StickyNote) => TopToolbarIcon::StickyNote, Self::Utility(TopToolbarUtility::Screenshot) => TopToolbarIcon::Screenshot, + Self::Utility(TopToolbarUtility::Ocr) => TopToolbarIcon::Ocr, Self::ClearCanvas => TopToolbarIcon::ClearCanvas, Self::Utility(TopToolbarUtility::Highlight) => TopToolbarIcon::Highlight, Self::Undo => TopToolbarIcon::Undo, diff --git a/src/ui/toolbar/model/top_spec/control_meta.rs b/src/ui/toolbar/model/top_spec/control_meta.rs index e37c43f0..d3265563 100644 --- a/src/ui/toolbar/model/top_spec/control_meta.rs +++ b/src/ui/toolbar/model/top_spec/control_meta.rs @@ -16,6 +16,7 @@ pub(crate) enum TopToolbarUtility { Text, StickyNote, Screenshot, + Ocr, Highlight, } @@ -25,6 +26,7 @@ impl TopToolbarUtility { TopUtilityButton::Text => Some(Self::Text), TopUtilityButton::StickyNote => Some(Self::StickyNote), TopUtilityButton::Screenshot => Some(Self::Screenshot), + TopUtilityButton::Ocr => Some(Self::Ocr), TopUtilityButton::Highlight => Some(Self::Highlight), TopUtilityButton::ClearCanvas | TopUtilityButton::IconMode => None, } @@ -64,6 +66,8 @@ pub(crate) enum TopToolbarIcon { Text, StickyNote, Screenshot, + /// Screen text recognition: a scan frame around three text lines. + Ocr, Highlight, ClearCanvas, Undo, @@ -109,6 +113,7 @@ pub(super) fn utility_event( TopToolbarUtility::Text => ToolbarEvent::EnterTextMode, TopToolbarUtility::StickyNote => ToolbarEvent::EnterStickyNoteMode, TopToolbarUtility::Screenshot => ToolbarEvent::CaptureScreenshot, + TopToolbarUtility::Ocr => ToolbarEvent::CopyTextFromScreen, TopToolbarUtility::Highlight => { ToolbarEvent::ToggleAllHighlight(!snapshot.any_highlight_active) } @@ -120,6 +125,7 @@ pub(super) fn utility_action(utility: TopToolbarUtility) -> Action { TopToolbarUtility::Text => Action::EnterTextMode, TopToolbarUtility::StickyNote => Action::EnterStickyNoteMode, TopToolbarUtility::Screenshot => Action::CaptureSelection, + TopToolbarUtility::Ocr => Action::CopyTextFromScreen, TopToolbarUtility::Highlight => Action::ToggleHighlightTool, } } @@ -127,6 +133,7 @@ pub(super) fn utility_action(utility: TopToolbarUtility) -> Action { pub(super) fn utility_short_label(utility: TopToolbarUtility) -> &'static str { match utility { TopToolbarUtility::Screenshot => "Shot", + TopToolbarUtility::Ocr => "Copy text", TopToolbarUtility::Highlight => "Highlight", _ => action_short_label(utility_action(utility)), } diff --git a/tools/check-nixpkgs-recipe.py b/tools/check-nixpkgs-recipe.py index 07d83b17..f08bfc49 100755 --- a/tools/check-nixpkgs-recipe.py +++ b/tools/check-nixpkgs-recipe.py @@ -60,6 +60,8 @@ "serde_ignored": (), "serde_json": (), "smithay-client-toolkit": ("libxkbcommon",), + # Pure Rust: temporary files for the OCR engine handoff. + "tempfile": (), "tokio": (), "toml": (), "toml_edit": (), diff --git a/tools/lint-and-test.sh b/tools/lint-and-test.sh index 7e8373c9..21b92fe0 100755 --- a/tools/lint-and-test.sh +++ b/tools/lint-and-test.sh @@ -16,5 +16,9 @@ cargo fmt --all -- --check cargo clippy --workspace --all-targets --all-features -- -D warnings cargo build --workspace --all-features --bins cargo test --workspace --all-features +# Linted as strictly as the all-features build: code reachable only behind an +# optional feature leaves its callers dead without it, and building alone does +# not promote that to an error. +cargo clippy --workspace --all-targets --no-default-features -- -D warnings cargo build --workspace --no-default-features --bins cargo test --workspace --no-default-features