From 4e46dc9ca4b60fec337c5c0612ea4faf1141c233 Mon Sep 17 00:00:00 2001 From: devmobasa <4170275+devmobasa@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:42:14 +0200 Subject: [PATCH 1/3] feat(ocr): copy text from a selected screen region MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `Copy text from screen`: drag a region of the desktop image Wayscriber is already displaying, recognize its text with a local Tesseract, and put the result on the clipboard. It is a one-shot action rather than a drawing tool, so the active tool, the drawing history, boards, and sessions are all untouched. The region is cropped from the frozen or zoomed capture the renderer is showing, never from annotations or Wayscriber chrome, and a freeze OCR creates is a freeze OCR releases — a user-owned freeze or an active zoom survives untouched. Recognition runs off the event loop through a capacity-one controller; a second request reports busy rather than queueing a region the user has moved on from. Recognized text never reaches application state, a log line, or a `Debug` rendering: the worker owns encoding, the engine call, and the clipboard write in one stack frame, and only a character count comes back. Two shared seams come out of this rather than being copied: - `state/screen_image.rs` now owns displayed-source resolution, the checked logical-to-image mapping, and the rectangle crop, for the eyedropper, the canvas renderer, and OCR alike. - `clipboard_text.rs` holds the generic `wl-copy`/`wl-paste` commands that used to live in the color-picker module. Input ownership is explicit throughout. A region records the device that started it, so a hovering pen cannot drag a mouse-owned selection and a stray release cannot submit one. Both screen-region modals now share one keyboard boundary (engaged, including the capture wait) and one pen boundary (active, once the selector is on screen), and a pen already down when a modal opens is disowned until it lifts so it cannot resize the drawing tool behind the selector. Discoverability is opt-in: the action is in the command palette and the keybinding configurator with no default shortcut, since `O` is already the orange quick color, and the `top.utility.ocr` toolbar button ships hidden like Screenshot. `capture.ocr_languages` validates the Tesseract language argument so an authored value can never carry a path, an option, or shell syntax into the argument vector; the engine is invoked through the process broker under a new `HelperKind::Tesseract` allowlist with an explicit argument vector and a self-deleting temporary PNG. Tesseract is packaged as an optional dependency (optdepends on Arch, recommends on deb/rpm) because the feature is opt-in at every level. Also fixes a latent bug in `ToolbarItemsConfig::set_visibility_setting`, which rewrote an item to the end of its list even when the setting was unchanged. It was invisible while only one item was hidden by default; a second one made a batch reset report "changed" forever. --- Cargo.lock | 1 + Cargo.toml | 2 + README.md | 17 + config.example.toml | 13 +- configurator/src/app/pages/capture.rs | 14 + .../src/models/config/draft/from_config.rs | 1 + configurator/src/models/config/draft/mod.rs | 1 + configurator/src/models/config/setters.rs | 1 + .../src/models/config/to_config/capture.rs | 11 +- configurator/src/models/fields/toggles.rs | 1 + .../models/keybindings/field/config/read.rs | 1 + .../models/keybindings/field/config/write.rs | 1 + .../src/models/keybindings/field/labels.rs | 2 + .../src/models/keybindings/field/list.rs | 1 + .../src/models/keybindings/field/mod.rs | 1 + .../src/models/keybindings/field/tab.rs | 1 + docs/CONFIG.md | 39 ++ docs/codebase-overview.md | 1 + packaging/.SRCINFO | 2 + packaging/PKGBUILD | 4 + packaging/package.wayscriber.yaml | 7 + src/AGENTS.md | 2 +- .../wayland/backend/event_loop/capture.rs | 2 + src/backend/wayland/handlers/compositor.rs | 2 + src/backend/wayland/handlers/keyboard/mod.rs | 10 + src/backend/wayland/handlers/layer.rs | 2 + .../wayland/handlers/pointer/cursor.rs | 2 +- src/backend/wayland/handlers/pointer/mod.rs | 1 + .../wayland/handlers/pointer/motion.rs | 14 + src/backend/wayland/handlers/pointer/press.rs | 24 + .../wayland/handlers/pointer/release.rs | 14 + src/backend/wayland/handlers/tablet/frame.rs | 39 +- src/backend/wayland/handlers/tablet/tool.rs | 152 ++++- src/backend/wayland/handlers/touch.rs | 47 +- src/backend/wayland/handlers/xdg.rs | 1 + src/backend/wayland/state.rs | 12 + src/backend/wayland/state/capture.rs | 2 + src/backend/wayland/state/color_picker.rs | 90 +-- src/backend/wayland/state/core/accessors.rs | 48 ++ src/backend/wayland/state/core/init.rs | 4 + src/backend/wayland/state/eyedropper.rs | 195 ++---- src/backend/wayland/state/ocr.rs | 474 +++++++++++++++ .../wayland/state/render/canvas/background.rs | 8 +- src/backend/wayland/state/render/ui.rs | 1 + src/backend/wayland/state/screen_image.rs | 473 +++++++++++++++ src/backend/wayland/state/text_clipboard.rs | 6 +- src/backend/wayland/state/toolbar/events.rs | 1 + src/clipboard_text.rs | 99 +++ src/config/action_meta/entries/capture.rs | 11 + src/config/action_meta/tests.rs | 1 + src/config/keybindings/config/map/capture.rs | 4 + src/config/keybindings/config/map/edit.rs | 1 + .../config/types/bindings/capture.rs | 6 + src/config/keybindings/defaults/capture.rs | 6 + src/config/keybindings/tests.rs | 27 + src/config/mod.rs | 44 +- src/config/types/capture.rs | 113 ++++ src/config/types/mod.rs | 2 +- src/config/types/toolbar/ids.rs | 1 + src/config/types/toolbar/items.rs | 11 +- src/config/types/toolbar/items/definitions.rs | 1 + src/config/types/toolbar/items/tests.rs | 44 ++ src/configurator_destination.rs | 1 + src/domain/action.rs | 2 + src/domain/tests.rs | 1 + src/input/mod.rs | 4 +- .../state/actions/action_capture_zoom.rs | 8 + src/input/state/core/base/state/init.rs | 2 + src/input/state/core/base/state/structs.rs | 4 + src/input/state/core/eyedropper.rs | 24 +- src/input/state/core/mod.rs | 2 + src/input/state/core/modal.rs | 37 +- src/input/state/core/ocr.rs | 498 +++++++++++++++ src/input/state/core/tool_controls/toolbar.rs | 2 + src/input/state/interaction/actions.rs | 1 + src/input/state/mod.rs | 5 +- src/lib.rs | 2 + src/ocr/AGENTS.md | 29 + src/ocr/controller.rs | 567 ++++++++++++++++++ src/ocr/mod.rs | 413 +++++++++++++ src/ocr/tesseract.rs | 325 ++++++++++ src/process_broker/execution.rs | 4 +- src/process_broker/manifest.rs | 1 + src/process_broker/mod.rs | 2 +- src/process_broker/tests.rs | 29 + src/process_broker/wire.rs | 8 + src/toolbar_icons/actions.rs | 5 + src/toolbar_icons/mod.rs | 4 +- src/toolbar_icons/svg.rs | 32 +- src/ui/toolbar/apply/actions.rs | 6 + src/ui/toolbar/apply/mod.rs | 1 + src/ui/toolbar/bindings.rs | 8 + src/ui/toolbar/events.rs | 2 + src/ui/toolbar/model/event_policy.rs | 2 + src/ui/toolbar/model/mod.rs | 2 +- src/ui/toolbar/model/tools.rs | 12 +- src/ui/toolbar/model/top_spec/control.rs | 4 +- src/ui/toolbar/model/top_spec/control_meta.rs | 7 + tools/check-nixpkgs-recipe.py | 2 + 99 files changed, 3889 insertions(+), 288 deletions(-) create mode 100644 src/backend/wayland/state/ocr.rs create mode 100644 src/backend/wayland/state/screen_image.rs create mode 100644 src/clipboard_text.rs create mode 100644 src/input/state/core/ocr.rs create mode 100644 src/ocr/AGENTS.md create mode 100644 src/ocr/controller.rs create mode 100644 src/ocr/mod.rs create mode 100644 src/ocr/tesseract.rs diff --git a/Cargo.lock b/Cargo.lock index 9e694ba0..4bc0e01b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1863,6 +1863,7 @@ dependencies = [ "serde_ignored", "serde_json", "smithay-client-toolkit", + "tempfile", "tokio", "toml", "toml_edit", diff --git a/Cargo.toml b/Cargo.toml index ca972771..17a46aa4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,8 @@ serde = { version = "1.0", features = ["derive"] } serde_ignored = "0.1" schemars = { version = "1.2", features = ["derive"], optional = true } libc = "0.2" +# Securely created, self-deleting temporary files for the OCR engine handoff. +tempfile = "3" flate2 = "1.0" zune-jpeg = "0.5" # Grapheme/word segmentation for the text-editor caret and selection. diff --git a/README.md b/README.md index 85c0ba1b..c762c599 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ The v0.9.23+ prebuilt `wayscriber` packages require glibc 2.39 and GTK 4.12 — - Full-screen saves, active-window grabs, region capture - Copy to clipboard or save to file - Uses `grim`, `slurp`, `wl-clipboard` (installed automatically by deb/rpm/AUR packages; fallback: xdg-desktop-portal) +- Copy text from screen (OCR): drag a region of the shown desktop and get its text on the clipboard (needs `tesseract`; no default shortcut) ### Sessions and persistence - Session persistence is enabled by default for boards, undo/redo history, and tool state @@ -329,6 +330,9 @@ If you are on a stable NixOS channel and want the newest release, use [the proje grim slurp wl-clipboard + + # Optional: copy text from screen (OCR) + (tesseract.override { enableLanguages = [ "eng" ]; }) ]; } ``` @@ -538,6 +542,19 @@ sudo apt-get install wl-clipboard grim slurp # Debian/Ubuntu sudo dnf install wl-clipboard grim slurp # Fedora ``` +### Copy text from screen (OCR) + +`Copy text from screen` recognizes the text in a dragged screen region and copies +it to the clipboard. It is optional: the action has no default shortcut and its +toolbar button is hidden until you turn it on. Install Tesseract and the language +data you configure in `[capture].ocr_languages` (default `eng`): + +```bash +sudo pacman -S tesseract tesseract-data-eng # Arch/Omarchy +sudo apt-get install tesseract-ocr # Debian/Ubuntu +sudo dnf install tesseract tesseract-langpack-eng # Fedora +``` + --- ## First launch diff --git a/config.example.toml b/config.example.toml index 45843111..a800b5bf 100644 --- a/config.example.toml +++ b/config.example.toml @@ -275,6 +275,10 @@ export_all_boards_pdf_file = [] # Open the most recent capture folder open_capture_folder = ["Ctrl+Alt+O"] +# Select a screen region and copy the text recognized in it (needs Tesseract). +# Unbound by default: "O" is already the orange quick color. +copy_text_from_screen = [] + # Toggle frozen mode toggle_frozen_mode = ["Ctrl+Shift+F"] @@ -627,11 +631,12 @@ rebind_modifier = "ctrl_shift" [ui.toolbar.items] # Checked in the configurator means shown; IDs listed here are hidden. -# The screenshot toolbar button is hidden by default. +# The screenshot and copy-text (OCR) toolbar buttons are hidden by default. # Section-level ids (side.group.*) act as explicit overrides that beat the # layout-mode baseline and survive mode switches. hidden = [ "top.utility.screenshot", + "top.utility.ocr", ] # IDs explicitly shown, overriding the layout-mode baseline (for example @@ -660,6 +665,7 @@ top_controls = [ "top.utility.text", "top.utility.sticky-note", "top.utility.screenshot", + "top.utility.ocr", "top.utility.clear-canvas", "top.utility.highlight", ] @@ -1384,6 +1390,11 @@ copy_to_clipboard = true # Use --no-exit-after-capture to keep the overlay open for a run. exit_after_capture = false +# Languages used by "Copy text from screen" (OCR), in Tesseract's +# plus-separated form, for example "eng" or "eng+deu". The matching Tesseract +# language packages must be installed. OCR also needs `enabled = true` above. +ocr_languages = "eng" + # ═══════════════════════════════════════════════════════════════════════════════ # EXPORT SETTINGS # ═══════════════════════════════════════════════════════════════════════════════ diff --git a/configurator/src/app/pages/capture.rs b/configurator/src/app/pages/capture.rs index 6bf07d59..bfbe4b61 100644 --- a/configurator/src/app/pages/capture.rs +++ b/configurator/src/app/pages/capture.rs @@ -15,6 +15,8 @@ use crate::models::{ PdfTransparentBackgroundOption, TabId, TextField, ToggleField, }; +use wayscriber::config::validate_ocr_languages; + use super::super::search::SearchArea; use super::super::state::ConfiguratorApp; use super::color_rows::{ResolvedColor, color_row}; @@ -56,6 +58,18 @@ pub(super) fn build(sender: &ComponentSender) -> BuiltPage { "", |app| app.draft.capture_exit_after, |value| Message::ToggleChanged(ToggleField::CaptureExitAfter, value), + ) + .entry_row_validated( + "OCR languages (e.g. eng, eng+deu \u{2014} needs the matching Tesseract packages)", + |app| app.draft.capture_ocr_languages.clone(), + |value| Message::TextChanged(TextField::CaptureOcrLanguages, value), + |app| { + validate_ocr_languages(&app.draft.capture_ocr_languages) + .err() + .map(|reason| { + format!("{reason}. Use Tesseract codes joined by '+', such as eng+deu.") + }) + }, ); page.group_in_area("PDF export", SearchArea::CapturePdf) diff --git a/configurator/src/models/config/draft/from_config.rs b/configurator/src/models/config/draft/from_config.rs index 44afe3f4..35dba0dd 100644 --- a/configurator/src/models/config/draft/from_config.rs +++ b/configurator/src/models/config/draft/from_config.rs @@ -247,6 +247,7 @@ impl ConfigDraft { capture_format: config.capture.format.clone(), capture_copy_to_clipboard: config.capture.copy_to_clipboard, capture_exit_after: config.capture.exit_after_capture, + capture_ocr_languages: config.capture.ocr_languages.clone(), export_pdf_filename_template: config .export .pdf diff --git a/configurator/src/models/config/draft/mod.rs b/configurator/src/models/config/draft/mod.rs index dbc71784..e7709cda 100644 --- a/configurator/src/models/config/draft/mod.rs +++ b/configurator/src/models/config/draft/mod.rs @@ -180,6 +180,7 @@ pub struct ConfigDraft { pub capture_format: String, pub capture_copy_to_clipboard: bool, pub capture_exit_after: bool, + pub capture_ocr_languages: String, pub export_pdf_filename_template: String, pub export_pdf_all_boards_filename_template: String, pub export_pdf_page_size: PdfPageSizeOption, diff --git a/configurator/src/models/config/setters.rs b/configurator/src/models/config/setters.rs index 9aa87d88..aafda684 100644 --- a/configurator/src/models/config/setters.rs +++ b/configurator/src/models/config/setters.rs @@ -366,6 +366,7 @@ impl ConfigDraft { TextField::CaptureSaveDirectory => self.capture_save_directory = value, TextField::CaptureFilename => self.capture_filename_template = value, TextField::CaptureFormat => self.capture_format = value, + TextField::CaptureOcrLanguages => self.capture_ocr_languages = value, TextField::ExportPdfFilenameTemplate => self.export_pdf_filename_template = value, TextField::ExportPdfAllBoardsFilenameTemplate => { self.export_pdf_all_boards_filename_template = value diff --git a/configurator/src/models/config/to_config/capture.rs b/configurator/src/models/config/to_config/capture.rs index 2f883413..bdb80e68 100644 --- a/configurator/src/models/config/to_config/capture.rs +++ b/configurator/src/models/config/to_config/capture.rs @@ -1,14 +1,21 @@ use super::super::draft::ConfigDraft; use crate::models::error::FormError; -use wayscriber::config::Config; +use wayscriber::config::{Config, validate_ocr_languages}; impl ConfigDraft { - pub(super) fn apply_capture(&self, config: &mut Config, _errors: &mut Vec) { + pub(super) fn apply_capture(&self, config: &mut Config, errors: &mut Vec) { config.capture.enabled = self.capture_enabled; config.capture.save_directory = self.capture_save_directory.clone(); config.capture.filename_template = self.capture_filename_template.clone(); config.capture.format = self.capture_format.clone(); config.capture.copy_to_clipboard = self.capture_copy_to_clipboard; config.capture.exit_after_capture = self.capture_exit_after; + match validate_ocr_languages(&self.capture_ocr_languages) { + Ok(languages) => config.capture.ocr_languages = languages, + Err(reason) => errors.push(FormError::new( + "capture.ocr_languages", + format!("OCR languages: {reason}."), + )), + } } } diff --git a/configurator/src/models/fields/toggles.rs b/configurator/src/models/fields/toggles.rs index 2b014355..5a9e2dd5 100644 --- a/configurator/src/models/fields/toggles.rs +++ b/configurator/src/models/fields/toggles.rs @@ -136,6 +136,7 @@ pub enum TextField { CaptureSaveDirectory, CaptureFilename, CaptureFormat, + CaptureOcrLanguages, ExportPdfFilenameTemplate, ExportPdfAllBoardsFilenameTemplate, ExportPdfCustomWidth, diff --git a/configurator/src/models/keybindings/field/config/read.rs b/configurator/src/models/keybindings/field/config/read.rs index ea622adb..6dbb7783 100644 --- a/configurator/src/models/keybindings/field/config/read.rs +++ b/configurator/src/models/keybindings/field/config/read.rs @@ -133,6 +133,7 @@ impl KeybindingField { Self::ExportBoardPdfFile => &config.capture.export_board_pdf_file, Self::ExportAllBoardsPdfFile => &config.capture.export_all_boards_pdf_file, Self::OpenCaptureFolder => &config.capture.open_capture_folder, + Self::CopyTextFromScreen => &config.capture.copy_text_from_screen, Self::ToggleFrozenMode => &config.zoom.toggle_frozen_mode, Self::ZoomIn => &config.zoom.zoom_in, Self::ZoomOut => &config.zoom.zoom_out, diff --git a/configurator/src/models/keybindings/field/config/write.rs b/configurator/src/models/keybindings/field/config/write.rs index 3f78aadc..09722411 100644 --- a/configurator/src/models/keybindings/field/config/write.rs +++ b/configurator/src/models/keybindings/field/config/write.rs @@ -136,6 +136,7 @@ impl KeybindingField { Self::ExportBoardPdfFile => config.capture.export_board_pdf_file = value, Self::ExportAllBoardsPdfFile => config.capture.export_all_boards_pdf_file = value, Self::OpenCaptureFolder => config.capture.open_capture_folder = value, + Self::CopyTextFromScreen => config.capture.copy_text_from_screen = value, Self::ToggleFrozenMode => config.zoom.toggle_frozen_mode = value, Self::ZoomIn => config.zoom.zoom_in = value, Self::ZoomOut => config.zoom.zoom_out = value, diff --git a/configurator/src/models/keybindings/field/labels.rs b/configurator/src/models/keybindings/field/labels.rs index b59d7f72..6422c956 100644 --- a/configurator/src/models/keybindings/field/labels.rs +++ b/configurator/src/models/keybindings/field/labels.rs @@ -128,6 +128,7 @@ impl KeybindingField { Self::ExportBoardPdfFile => "Export board: PDF file", Self::ExportAllBoardsPdfFile => "Export all boards: PDF file", Self::OpenCaptureFolder => "Open capture folder", + Self::CopyTextFromScreen => "Copy text from screen (OCR)", Self::ToggleFrozenMode => "Toggle freeze", Self::ZoomIn => "Zoom in", Self::ZoomOut => "Zoom out", @@ -281,6 +282,7 @@ impl KeybindingField { Self::ExportBoardPdfFile => "export_board_pdf_file", Self::ExportAllBoardsPdfFile => "export_all_boards_pdf_file", Self::OpenCaptureFolder => "open_capture_folder", + Self::CopyTextFromScreen => "copy_text_from_screen", Self::ToggleFrozenMode => "toggle_frozen_mode", Self::ZoomIn => "zoom_in", Self::ZoomOut => "zoom_out", diff --git a/configurator/src/models/keybindings/field/list.rs b/configurator/src/models/keybindings/field/list.rs index c0cb3189..7ad4f480 100644 --- a/configurator/src/models/keybindings/field/list.rs +++ b/configurator/src/models/keybindings/field/list.rs @@ -128,6 +128,7 @@ impl KeybindingField { Self::ExportBoardPdfFile, Self::ExportAllBoardsPdfFile, Self::OpenCaptureFolder, + Self::CopyTextFromScreen, Self::ToggleFrozenMode, Self::ZoomIn, Self::ZoomOut, diff --git a/configurator/src/models/keybindings/field/mod.rs b/configurator/src/models/keybindings/field/mod.rs index 9cc811b4..e3064d8b 100644 --- a/configurator/src/models/keybindings/field/mod.rs +++ b/configurator/src/models/keybindings/field/mod.rs @@ -113,6 +113,7 @@ pub enum KeybindingField { ExportBoardPdfFile, ExportAllBoardsPdfFile, OpenCaptureFolder, + CopyTextFromScreen, ToggleFrozenMode, ZoomIn, ZoomOut, diff --git a/configurator/src/models/keybindings/field/tab.rs b/configurator/src/models/keybindings/field/tab.rs index c785ae2b..abda464f 100644 --- a/configurator/src/models/keybindings/field/tab.rs +++ b/configurator/src/models/keybindings/field/tab.rs @@ -129,6 +129,7 @@ impl KeybindingField { | Self::ExportBoardPdfFile | Self::ExportAllBoardsPdfFile | Self::OpenCaptureFolder + | Self::CopyTextFromScreen | Self::ToggleFrozenMode | Self::ZoomIn | Self::ZoomOut diff --git a/docs/CONFIG.md b/docs/CONFIG.md index b9ce07f2..87fc9d7b 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -1380,12 +1380,47 @@ copy_to_clipboard = true # When false, clipboard-only captures still auto-exit by default. # Use --no-exit-after-capture to keep the overlay open for a run. exit_after_capture = false + +# Languages for "Copy text from screen" (OCR), in Tesseract's plus-separated +# form. The matching Tesseract language packages must be installed. +ocr_languages = "eng" ``` **Tips:** - Set `copy_to_clipboard = false` if you prefer file-only captures. - Clipboard-only shortcuts ignore the save directory automatically. - `wl-clipboard`, `grim`, and `slurp` are installed automatically by deb/rpm/AUR packages. For source/tarball installs, add them manually; otherwise wayscriber falls back to `xdg-desktop-portal`. + +#### Copy text from screen (OCR) + +`Copy text from screen` selects a region of the desktop image wayscriber is +already showing, recognizes the text in it with a local Tesseract, and copies +the result to the clipboard. It reads the underlying screen capture only — +never your annotations, the toolbars, or any other wayscriber chrome — and it +does not change the active tool, the drawing history, or the board. + +- The action is `copy_text_from_screen`. It has **no default shortcut**, because + `O` is already the orange quick color; bind one in the configurator or in + `[keybindings.capture]`. +- It is also in the command palette (search for "OCR"), and as an optional top + toolbar button (`top.utility.ocr`), hidden by default like Screenshot. +- `ocr_languages` accepts one language or several joined with `+` + (`eng`, `eng+deu`). Only letters, digits, `_` and `-` are accepted; anything + else falls back to `eng`. +- OCR obeys `enabled` above: with capture disabled, the action reports that and + does nothing. +- On a solid whiteboard or blackboard with no visible screen capture, OCR + refuses rather than reading the board. + +**Requirements:** the `tesseract` command plus the language data for every code +in `ocr_languages`, and `wl-copy` (from `wl-clipboard`) for the clipboard write. + +| Distribution | Packages | +| --- | --- | +| Arch / Omarchy | `tesseract`, `tesseract-data-eng` | +| Debian / Ubuntu | `tesseract-ocr` (depends on the English data) | +| Fedora | `tesseract`, `tesseract-langpack-eng` | +| Nix | the `tesseract` package/wrapper with `eng` enabled | - Use `--exit-after-capture` / `--no-exit-after-capture` to override exit behavior per run. ### `[export.pdf]` - PDF Export @@ -1839,6 +1874,10 @@ export_all_boards_pdf_file = [] # Open the most recent capture folder open_capture_folder = ["Ctrl+Alt+O"] +# Select a screen region and copy the text recognized in it (needs Tesseract). +# Unbound by default: "O" is already the orange quick color. +copy_text_from_screen = [] + # Toggle frozen mode toggle_frozen_mode = ["Ctrl+Shift+F"] diff --git a/docs/codebase-overview.md b/docs/codebase-overview.md index 6a6923d6..3ac017ec 100644 --- a/docs/codebase-overview.md +++ b/docs/codebase-overview.md @@ -470,6 +470,7 @@ Notifications are sent via `notification::send_notification_async`, keeping all | `src/config/` | Config parsing, defaults, keybinding map. | | `src/runtime_ui_state/` | Generated UI preference wire format, seed registry, guarded persistence, reset, and recovery state machines. | | `src/session/` | Configured and named session persistence, snapshots, sidecars, locks, and catalog metadata. | +| `src/ocr/` | Screen text recognition: capacity-one controller, Tesseract adapter, typed outcomes. | | `src/notification.rs` | Desktop notifications for capture results and the update notice. | | `src/update_check/` | Opt-outable "a newer release exists" check: version ordering, manifest trust rules, `curl`/`wget` transport, cache. Installs nothing. | | `src/about_window/` | Standalone About dialog: content/layout/interaction split, update card, diagnostics copy. | diff --git a/packaging/.SRCINFO b/packaging/.SRCINFO index 60d6d319..6ca91abe 100644 --- a/packaging/.SRCINFO +++ b/packaging/.SRCINFO @@ -19,6 +19,8 @@ pkgbase = wayscriber depends = wl-clipboard depends = grim depends = slurp + optdepends = tesseract: copy text from screen (OCR) + optdepends = tesseract-data-eng: English language data for OCR source = wayscriber-0.9.23.tar.gz::https://github.com/devmobasa/wayscriber/archive/refs/tags/v0.9.23.tar.gz sha256sums = SKIP diff --git a/packaging/PKGBUILD b/packaging/PKGBUILD index 6b01e7c8..1b4f5e8b 100644 --- a/packaging/PKGBUILD +++ b/packaging/PKGBUILD @@ -20,6 +20,10 @@ depends=( 'grim' 'slurp' ) +optdepends=( + 'tesseract: copy text from screen (OCR)' + 'tesseract-data-eng: English language data for OCR' +) makedepends=( 'cargo' ) diff --git a/packaging/package.wayscriber.yaml b/packaging/package.wayscriber.yaml index 68969816..2dc799d5 100644 --- a/packaging/package.wayscriber.yaml +++ b/packaging/package.wayscriber.yaml @@ -122,6 +122,10 @@ overrides: - wl-clipboard - grim - slurp + # Screen text recognition is opt-in (no default shortcut, hidden toolbar + # button), so the engine and its language data are not a hard dependency. + recommends: + - tesseract-ocr rpm: depends: # Keep the glibc floor aligned with MAX_GLIBC_VERSION in tools/package.sh. @@ -135,5 +139,8 @@ overrides: - wl-clipboard - grim - slurp + recommends: + - tesseract + - tesseract-langpack-eng rpm: arch: x86_64 diff --git a/src/AGENTS.md b/src/AGENTS.md index d42dd86c..a1db3655 100644 --- a/src/AGENTS.md +++ b/src/AGENTS.md @@ -8,7 +8,7 @@ - `src/main.rs` is a thin wrapper that returns `wayscriber::run_from_env()`. - `src/lib.rs` owns logging/error wiring and the canonical module graph; its public entry facade routes CLI outcomes through `src/app/` into `src/daemon/` or the active Wayland backend. - Reusable modules remain public for integration tests and the configurator, while runtime modules are private library implementation details. -- Major domains are `domain`, `backend`, `input`, `draw`, `ui`, `capture`, `config`, `session`, `daemon`, `canvas_export`, `paths`, `toolbar_icons`, `render_profiles`, and `runtime_capabilities`. +- Major domains are `domain`, `backend`, `input`, `draw`, `ui`, `capture`, `ocr`, `config`, `session`, `daemon`, `canvas_export`, `paths`, `toolbar_icons`, `render_profiles`, and `runtime_capabilities`. ## Invariants - Keep Wayland/runtime side effects in backend/about-window code; keep reusable data, validation, rendering helpers, and path logic in library modules. diff --git a/src/backend/wayland/backend/event_loop/capture.rs b/src/backend/wayland/backend/event_loop/capture.rs index d371f9bd..1a4528d4 100644 --- a/src/backend/wayland/backend/event_loop/capture.rs +++ b/src/backend/wayland/backend/event_loop/capture.rs @@ -92,9 +92,11 @@ pub(super) fn handle_pending_actions( state.poll_hex_copy_completion(); state.poll_text_copy_completion(); state.poll_text_paste_completion(); + state.poll_ocr_completion(); state.poll_session_file_dialog_completion(qh); state.drain_clipboard_requests(); state.handle_pending_eyedropper_toggle(); + state.handle_pending_ocr_request(); // Copy/paste-hex requests from the color picker popup's pointer release are // drained here: unlike the toolbar/key paths, that release has no other // drain site. An accepted quick-color recolor is queued from the same diff --git a/src/backend/wayland/handlers/compositor.rs b/src/backend/wayland/handlers/compositor.rs index d292d94b..cdf1b297 100644 --- a/src/backend/wayland/handlers/compositor.rs +++ b/src/backend/wayland/handlers/compositor.rs @@ -60,6 +60,7 @@ impl CompositorHandler for WaylandState { self.zoom .handle_resize(phys_w, phys_h, &mut self.input_state); self.cancel_eyedropper_if_source_missing(); + self.cancel_ocr_if_source_missing(); self.toolbar .maybe_update_scale(self.surface.current_output().as_ref(), scale); self.toolbar.mark_dirty(); @@ -170,6 +171,7 @@ impl CompositorHandler for WaylandState { self.zoom .handle_resize(phys_w, phys_h, &mut self.input_state); self.cancel_eyedropper_if_source_missing(); + self.cancel_ocr_if_source_missing(); // If freeze-on-start was requested, trigger it once the surface is configured and active. if self.pending_freeze_on_start() { diff --git a/src/backend/wayland/handlers/keyboard/mod.rs b/src/backend/wayland/handlers/keyboard/mod.rs index df370d79..98ff8c5d 100644 --- a/src/backend/wayland/handlers/keyboard/mod.rs +++ b/src/backend/wayland/handlers/keyboard/mod.rs @@ -117,6 +117,16 @@ impl KeyboardHandler for WaylandState { // Any fresh key press ends the previous auto-repeat; a repeatable one // re-arms it at the end of this handler. self.clear_key_repeat(); + if self.input_state.ocr_is_engaged() { + // Every other shortcut is swallowed while the selector is up so a + // key cannot change the active tool mid-drag. + if matches!(key, Key::Escape) + || self.input_state.action_for_key(key) == Some(Action::CopyTextFromScreen) + { + self.cancel_ocr(); + } + return; + } if self.input_state.eyedropper_is_engaged() { if matches!(key, Key::Escape) || self.input_state.action_for_key(key) == Some(Action::PickScreenColor) diff --git a/src/backend/wayland/handlers/layer.rs b/src/backend/wayland/handlers/layer.rs index 996c81d7..e98b1b0d 100644 --- a/src/backend/wayland/handlers/layer.rs +++ b/src/backend/wayland/handlers/layer.rs @@ -69,6 +69,7 @@ impl LayerShellHandler for WaylandState { self.zoom .handle_resize(phys_w, phys_h, &mut self.input_state); self.cancel_eyedropper_if_source_missing(); + self.cancel_ocr_if_source_missing(); // Refresh active geometry for portal fallback cropping using latest logical size/scale. let output_transform = self @@ -106,6 +107,7 @@ impl LayerShellHandler for WaylandState { self.zoom .handle_resize(phys_w, phys_h, &mut self.input_state); self.cancel_eyedropper_if_source_missing(); + self.cancel_ocr_if_source_missing(); // Re-apply toolbar offsets now that we have a configured surface size; avoids clamping to 0 // on startup before the compositor provides dimensions. diff --git a/src/backend/wayland/handlers/pointer/cursor.rs b/src/backend/wayland/handlers/pointer/cursor.rs index e388ba1e..d6254b7a 100644 --- a/src/backend/wayland/handlers/pointer/cursor.rs +++ b/src/backend/wayland/handlers/pointer/cursor.rs @@ -38,7 +38,7 @@ impl WaylandState { /// Computes the appropriate cursor icon based on current context. fn compute_cursor_icon(&mut self, toolbar_hover: bool) -> CursorIcon { - if self.input_state.eyedropper_is_active() && !toolbar_hover { + if self.input_state.screen_modal_is_active() && !toolbar_hover { return CursorIcon::Crosshair; } diff --git a/src/backend/wayland/handlers/pointer/mod.rs b/src/backend/wayland/handlers/pointer/mod.rs index 148f4aa3..afdeea98 100644 --- a/src/backend/wayland/handlers/pointer/mod.rs +++ b/src/backend/wayland/handlers/pointer/mod.rs @@ -4,6 +4,7 @@ use smithay_client_toolkit::seat::pointer::{PointerEvent, PointerEventKind, Poin use wayland_client::{Connection, QueueHandle, protocol::wl_pointer}; use crate::backend::wayland::state::{debug_toolbar_drag_logging_enabled, surface_id}; +use crate::input::state::OcrInputSource; use super::super::state::WaylandState; diff --git a/src/backend/wayland/handlers/pointer/motion.rs b/src/backend/wayland/handlers/pointer/motion.rs index 8b594447..792f9adf 100644 --- a/src/backend/wayland/handlers/pointer/motion.rs +++ b/src/backend/wayland/handlers/pointer/motion.rs @@ -16,6 +16,20 @@ impl WaylandState { on_toolbar: bool, inline_active: bool, ) { + if self.input_state.ocr_is_active() { + let screen_position = if on_toolbar { + self.toolbar_surface_screen_coords(&event.surface, event.position) + } else { + Some(event.position) + }; + if let Some((x, y)) = screen_position { + self.set_current_mouse(x.round() as i32, y.round() as i32); + self.update_ocr_selection(OcrInputSource::Pointer, x, y); + } + self.update_pointer_cursor(on_toolbar || self.pointer_over_toolbar(), conn); + return; + } + if self.input_state.eyedropper_is_active() { let inline_hover = !on_toolbar && self.inline_toolbars_active() diff --git a/src/backend/wayland/handlers/pointer/press.rs b/src/backend/wayland/handlers/pointer/press.rs index 6643cbee..38b94592 100644 --- a/src/backend/wayland/handlers/pointer/press.rs +++ b/src/backend/wayland/handlers/pointer/press.rs @@ -38,6 +38,30 @@ impl WaylandState { .clear_help_overlay_press_for(help_press_source); } + if self.input_state.ocr_is_active() { + if on_toolbar || self.pointer_over_toolbar() { + // A toolbar interaction ends the region first, then runs + // normally; the click never lands on the selector. + self.cancel_ocr(); + } else { + match button { + BTN_LEFT => { + self.begin_ocr_selection( + OcrInputSource::Pointer, + event.position.0, + event.position.1, + ); + } + BTN_RIGHT => { + self.cancel_ocr(); + self.set_suppress_next_release(true); + } + _ => {} + } + return; + } + } + if self.input_state.eyedropper_is_active() { if on_toolbar || self.pointer_over_toolbar() { self.cancel_eyedropper(); diff --git a/src/backend/wayland/handlers/pointer/release.rs b/src/backend/wayland/handlers/pointer/release.rs index e70ebebc..c3c61984 100644 --- a/src/backend/wayland/handlers/pointer/release.rs +++ b/src/backend/wayland/handlers/pointer/release.rs @@ -16,6 +16,20 @@ impl WaylandState { inline_active: bool, button: u32, ) { + if self.input_state.ocr_is_active() { + if button == BTN_LEFT { + let screen_position = if on_toolbar { + self.toolbar_surface_screen_coords(&event.surface, event.position) + } else { + Some(event.position) + }; + if let Some((x, y)) = screen_position { + self.finish_ocr_selection(OcrInputSource::Pointer, x, y); + } + } + return; + } + if self.input_state.eyedropper_is_active() { return; } diff --git a/src/backend/wayland/handlers/tablet/frame.rs b/src/backend/wayland/handlers/tablet/frame.rs index 08c88bea..00f6421b 100644 --- a/src/backend/wayland/handlers/tablet/frame.rs +++ b/src/backend/wayland/handlers/tablet/frame.rs @@ -10,7 +10,13 @@ const BTN_STYLUS: u32 = 331; const BTN_STYLUS2: u32 = 332; fn modal_blocks_stylus_barrel_actions(input_state: &crate::input::InputState) -> bool { - input_state.show_help || input_state.tour_active + input_state.show_help + || input_state.tour_active + // A screen-region modal owns pointer and keyboard input while it is up. + // A barrel button is bound to an ordinary action — undo, clear canvas, + // the radial menu — so letting one through would run it on the canvas + // behind the selector. + || input_state.screen_modal_is_engaged() } impl WaylandState { @@ -159,6 +165,12 @@ impl WaylandState { .clear_help_overlay_press_for(HelpOverlayPressSource::Stylus); } + if self.input_state.ocr_is_active() { + let (x, y) = self.current_stylus_position(); + self.begin_ocr_selection(crate::input::state::OcrInputSource::Stylus, x, y); + return; + } + if self.input_state.eyedropper_is_active() { let (x, y) = self.current_stylus_position(); self.sample_eyedropper(x, y); @@ -313,4 +325,29 @@ mod tests { state.tour_active = true; assert!(modal_blocks_stylus_barrel_actions(&state)); } + + /// Both screen-region modals swallow pointer and keyboard input, so a + /// barrel button must not run its bound action on the canvas behind them — + /// including while they are still waiting on a capture. + #[test] + fn screen_region_modals_block_stylus_barrel_actions() { + use crate::input::state::{EyedropperCaptureSource, OcrCaptureSource}; + + let mut state = make_test_input_state(); + assert!(!modal_blocks_stylus_barrel_actions(&state)); + + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + assert!(modal_blocks_stylus_barrel_actions(&state)); + state.activate_ocr(true); + assert!(modal_blocks_stylus_barrel_actions(&state)); + state.cancel_ocr(); + assert!(!modal_blocks_stylus_barrel_actions(&state)); + + state.set_eyedropper_pending_capture(EyedropperCaptureSource::Frozen); + assert!(modal_blocks_stylus_barrel_actions(&state)); + state.activate_eyedropper(true); + assert!(modal_blocks_stylus_barrel_actions(&state)); + state.cancel_eyedropper(); + assert!(!modal_blocks_stylus_barrel_actions(&state)); + } } diff --git a/src/backend/wayland/handlers/tablet/tool.rs b/src/backend/wayland/handlers/tablet/tool.rs index c44b85e2..e1b93bd6 100644 --- a/src/backend/wayland/handlers/tablet/tool.rs +++ b/src/backend/wayland/handlers/tablet/tool.rs @@ -9,9 +9,34 @@ use crate::{ }; use crate::backend::wayland::state::WaylandState; +use crate::input::state::OcrInputSource; const STYLUS_CURSOR_DAMAGE_RADIUS: i32 = 64; +/// Whether a pressure sample must be dropped rather than queued for the next +/// tablet frame. +/// +/// A committed pressure sample runs `set_pressure_thickness_for_active_tool`, +/// so a sample that should not have reached the canvas resizes the drawing tool +/// — the one thing OCR promises not to touch. Three independent reasons to drop +/// one, none of which subsumes the others: +/// +/// * the pen is not over the overlay, so the sample is not ours to apply; +/// * the contact was disowned when a screen modal took over, and is only still +/// arriving because clearing our flags does not lift the pen; +/// * a screen modal is on screen and owns the pen outright. +/// +/// The modal term follows the *active* boundary, which keeps this an exact twin +/// of the tip and motion guards: a stroke begun while a capture is still pending +/// is a real stroke and keeps its pressure. The retired term is what stops that +/// allowance from also readmitting the contact the modal already took over. +fn drop_stylus_pressure( + on_overlay: bool, + contact_retired: bool, + input_state: &crate::input::InputState, +) -> bool { + !on_overlay || contact_retired || input_state.screen_modal_is_active() +} impl WaylandState { fn stylus_hover_cursor_pos(&self) -> Option<(f64, f64)> { self.stylus_hover_cursor_position() @@ -44,7 +69,7 @@ impl WaylandState { || (self.inline_toolbars_active() && self.toolbar.is_visible()) } - pub(super) fn mark_stylus_hover_cursor_dirty( + pub(in crate::backend::wayland) fn mark_stylus_hover_cursor_dirty( &mut self, previous: Option<(f64, f64)>, next: Option<(f64, f64)>, @@ -163,6 +188,15 @@ impl Dispatch for WaylandState { tool_id, tool_type ); state.commit_pending_stylus_frame(); + // The tip is gone and no Up is coming, so a region drag *this + // pen* started must end here rather than keeping the selector — + // and any OCR-owned freeze — alive until the user cancels by + // hand. A region the mouse or a finger is dragging is not ours + // to withdraw. + state.cancel_ocr_selection_from(OcrInputSource::Stylus); + // The pen is gone, so the tip-up the latch was waiting for is + // never coming; leaving it armed would swallow a later contact. + state.take_retired_stylus_contact(); let hover_cursor_pos = state.stylus_hover_cursor_pos(); state.stylus_tip_down = false; state.stylus_on_overlay = false; @@ -196,6 +230,28 @@ impl Dispatch for WaylandState { // Note: We keep the tool type in the map - tools persist across proximity events } Event::Down { .. } => { + // A contact a screen modal disowned is not the user pressing — + // it is the pen that was already down, still reported until it + // lifts. Swallow it whole rather than declining one branch: + // falling through queues the tip-down, and the frame commit + // would start a region (or a stroke) from it one hop later. + // + // Protocol-unreachable today, since the latch only clears on a + // tip-up or proximity-out and a press must follow one of those. + // It is enforced rather than argued because the surrounding + // pressure, motion, and release guards all rely on it. + if state.stylus_contact_retired { + return; + } + if state.input_state.ocr_is_active() { + if state.stylus_on_toolbar { + state.cancel_ocr(); + } else if state.stylus_on_overlay { + let (x, y) = state.current_or_pending_stylus_position(); + state.begin_ocr_selection(OcrInputSource::Stylus, x, y); + return; + } + } if state.input_state.eyedropper_is_active() { if state.stylus_on_toolbar { state.cancel_eyedropper(); @@ -235,6 +291,24 @@ impl Dispatch for WaylandState { state.queue_stylus_down(); } Event::Up => { + // Read before the modal branches so the latch is consumed by + // the tip-up that actually ends the disowned contact, whatever + // else this Up does. + let retired_contact = state.take_retired_stylus_contact(); + if state.input_state.ocr_is_active() { + if state.stylus_on_overlay { + let (x, y) = state.current_or_pending_stylus_position(); + // A no-op unless this pen owns the region, so a retired + // contact lifting cannot submit one the mouse or a + // finger is still drawing. + state.finish_ocr_selection(OcrInputSource::Stylus, x, y); + } else { + // The tip left the overlay before lifting; there is no + // region of ours to submit, so withdraw only our own. + state.cancel_ocr_selection_from(OcrInputSource::Stylus); + } + return; + } let inline_active = state.inline_toolbars_active() && state.toolbar.is_visible(); if inline_active && state.stylus_on_toolbar { let (mx, my) = state.current_mouse(); @@ -250,12 +324,21 @@ impl Dispatch for WaylandState { state.end_toolbar_move_drag(); return; } - if !state.stylus_on_overlay { + if !state.stylus_on_overlay || retired_contact { return; } state.queue_stylus_up(); } Event::Motion { x, y } => { + if state.input_state.ocr_is_active() && state.stylus_on_overlay { + state.stylus_last_pos = Some((x, y)); + state.set_current_mouse(x.round() as i32, y.round() as i32); + // Dropped unless this pen owns the region: a pen hovering + // over the overlay, or one whose contact was retired, must + // not drag somebody else's selection around. + state.update_ocr_selection(OcrInputSource::Stylus, x, y); + return; + } if state.input_state.eyedropper_is_active() && state.stylus_on_overlay { state.stylus_last_pos = Some((x, y)); state.set_current_mouse(x.round() as i32, y.round() as i32); @@ -325,7 +408,11 @@ impl Dispatch for WaylandState { state.queue_stylus_motion(x, y); } Event::Pressure { pressure } => { - if !state.stylus_on_overlay { + if drop_stylus_pressure( + state.stylus_on_overlay, + state.stylus_contact_retired, + &state.input_state, + ) { return; } state.queue_stylus_pressure(pressure); @@ -371,6 +458,65 @@ impl Dispatch for WaylandState { mod tests { use super::*; + /// The modal term stops exactly where the tip and motion guards stop: once + /// the selector is on screen. While a capture is still pending the canvas + /// keeps the pen, so a stroke drawn then must keep its pressure — otherwise + /// a capture that fails would leave a silently flat stroke behind. + #[test] + fn stylus_pressure_stops_at_the_selector_not_at_the_request() { + use crate::input::state::test_support::make_test_input_state; + use crate::input::state::{EyedropperCaptureSource, OcrCaptureSource}; + + let fresh_contact = |state: &_| drop_stylus_pressure(true, false, state); + + let mut state = make_test_input_state(); + assert!(!fresh_contact(&state)); + + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + assert!( + !fresh_contact(&state), + "a stroke drawn while the capture is pending is still a real stroke" + ); + state.activate_ocr(true); + assert!(fresh_contact(&state)); + state.start_ocr_selection(OcrInputSource::Stylus, (10.0, 10.0)); + assert!(fresh_contact(&state)); + state.cancel_ocr(); + assert!(!fresh_contact(&state)); + + state.set_eyedropper_pending_capture(EyedropperCaptureSource::Frozen); + assert!(!fresh_contact(&state)); + state.activate_eyedropper(true); + assert!(fresh_contact(&state)); + state.cancel_eyedropper(); + assert!(!fresh_contact(&state)); + } + + /// A contact disowned by a modal keeps arriving until the pen lifts, and the + /// pending-capture allowance above must not readmit it: the canvas holds the + /// pen again during the wait, but this contact is not the user drawing. + #[test] + fn a_retired_contact_never_reaches_the_tool_whatever_the_modal_is_doing() { + use crate::input::state::OcrCaptureSource; + use crate::input::state::test_support::make_test_input_state; + + let mut state = make_test_input_state(); + + // The window the previous test allows, and the one that matters here. + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + assert!(!drop_stylus_pressure(true, false, &state)); + assert!(drop_stylus_pressure(true, true, &state)); + + // And it outlives the modal: cancelling before activation leaves the pen + // physically down with no modal to blame. + state.cancel_ocr(); + assert!(!drop_stylus_pressure(true, false, &state)); + assert!(drop_stylus_pressure(true, true, &state)); + + // Off the overlay nothing is ours either way. + assert!(drop_stylus_pressure(false, false, &state)); + } + #[test] fn stylus_cursor_damage_rect_covers_cursor_area() { let rect = stylus_cursor_damage_rect((100.2, 80.7), 400, 300).expect("rect"); diff --git a/src/backend/wayland/handlers/touch.rs b/src/backend/wayland/handlers/touch.rs index 99375e78..d63d0799 100644 --- a/src/backend/wayland/handlers/touch.rs +++ b/src/backend/wayland/handlers/touch.rs @@ -10,7 +10,7 @@ use crate::backend::wayland::state::{ }; use crate::backend::wayland::toolbar_intent::intent_to_event; use crate::input::MouseButton; -use crate::input::state::HelpOverlayPressSource; +use crate::input::state::{HelpOverlayPressSource, OcrInputSource}; use crate::ui::ZoomChipPress; impl TouchHandler for WaylandState { @@ -114,6 +114,11 @@ impl WaylandState { } let target = self.active_touch.target(); + // A cancelled sequence never produces a release, so a region drag it + // started has to end here — otherwise the selector, and any freeze it + // owns, would outlive the touch that opened it. A region another device + // is dragging is untouched. + self.cancel_ocr_selection_from(OcrInputSource::Touch); self.set_pending_toast_press(None); self.set_pending_status_hud_press(false); self.set_pending_zoom_chip_press(ZoomChipPress::None); @@ -204,6 +209,26 @@ impl WaylandState { .clear_help_overlay_press_for(HelpOverlayPressSource::Touch); } + if self.input_state.ocr_is_active() { + let inline_active = self.inline_toolbars_active() && self.toolbar.is_visible(); + let inline_hit = target == TouchTarget::Overlay + && inline_active + && self.inline_toolbar_motion(screen_position); + if target == TouchTarget::Toolbar || inline_hit { + self.cancel_ocr(); + } else if target == TouchTarget::Overlay { + self.begin_ocr_selection( + OcrInputSource::Touch, + screen_position.0, + screen_position.1, + ); + // Unlike the one-shot eyedropper sample, an OCR region is a + // drag: report the real target so motion and release still + // resolve to screen coordinates and reach the selector. + return TouchTarget::Overlay; + } + } + if self.input_state.eyedropper_is_active() { let inline_active = self.inline_toolbars_active() && self.toolbar.is_visible(); let inline_hit = target == TouchTarget::Overlay @@ -333,6 +358,17 @@ impl WaylandState { let screen_y = screen_position.1.round() as i32; self.set_current_mouse(screen_x, screen_y); + if self.input_state.ocr_is_active() { + if target == TouchTarget::Overlay { + self.update_ocr_selection( + OcrInputSource::Touch, + screen_position.0, + screen_position.1, + ); + } + return; + } + if self.input_state.eyedropper_is_active() { if target == TouchTarget::Overlay { self.update_eyedropper_hover(screen_position.0, screen_position.1); @@ -412,6 +448,15 @@ impl WaylandState { position: (f64, f64), target: TouchTarget, ) { + if self.input_state.ocr_is_active() { + if let Some((x, y)) = self.touch_screen_position(surface, position, target) { + self.finish_ocr_selection(OcrInputSource::Touch, x, y); + } else { + self.cancel_ocr_selection_from(OcrInputSource::Touch); + } + return; + } + if self.take_suppress_next_release() { self.set_pending_toast_press(None); self.set_pending_status_hud_press(false); diff --git a/src/backend/wayland/handlers/xdg.rs b/src/backend/wayland/handlers/xdg.rs index f6e1d474..c1bd64c2 100644 --- a/src/backend/wayland/handlers/xdg.rs +++ b/src/backend/wayland/handlers/xdg.rs @@ -121,6 +121,7 @@ impl WindowHandler for WaylandState { self.zoom .handle_resize(phys_w, phys_h, &mut self.input_state); self.cancel_eyedropper_if_source_missing(); + self.cancel_ocr_if_source_missing(); let output_transform = self .surface diff --git a/src/backend/wayland/state.rs b/src/backend/wayland/state.rs index 6c779b8a..d5c3744f 100644 --- a/src/backend/wayland/state.rs +++ b/src/backend/wayland/state.rs @@ -108,10 +108,12 @@ mod input_actions; mod input_hud; mod keybindings; pub(in crate::backend::wayland) use keybindings::queue_keybinding_edit; +mod ocr; mod onboarding; mod pdf_export; mod perf; mod render; +mod screen_image; mod text_clipboard; mod toolbar; #[cfg(feature = "toolbar-gtk")] @@ -263,6 +265,9 @@ pub(super) struct WaylandState { /// the current edit generation remain distinct; a new generation replaces /// stale queued requests from the old edit session. pub(super) pending_text_paste: VecDeque, + /// Capacity-one screen text recognition. A busy controller reports + /// busy rather than queuing a region the user has moved on from. + pub(super) ocr: crate::ocr::OcrController, /// GTK toolbar frontend; `None` means the built-in bars are in charge. pub(super) gtk_toolbar: Option, pub(super) onboarding: crate::onboarding::OnboardingStore, @@ -361,6 +366,13 @@ pub(super) struct WaylandState { pub(super) stylus_peak_thickness: Option, #[cfg(feature = "tablet-input")] pub(super) pending_stylus_frame: PendingStylusFrame, + /// The pen is still physically down, but the contact was disowned when a + /// screen-region modal took over. Wayscriber's own flags were cleared, yet + /// the compositor keeps reporting this contact until the tip lifts — so it + /// is tracked separately and ignored until then, rather than being mistaken + /// for a fresh press made during the wait. + #[cfg(feature = "tablet-input")] + pub(super) stylus_contact_retired: bool, /// Map of tool object IDs to their physical types (pen, eraser, etc.) #[cfg(feature = "tablet-input")] pub(super) stylus_tool_types: std::collections::HashMap< diff --git a/src/backend/wayland/state/capture.rs b/src/backend/wayland/state/capture.rs index a5b88550..dc6bcad9 100644 --- a/src/backend/wayland/state/capture.rs +++ b/src/backend/wayland/state/capture.rs @@ -43,6 +43,7 @@ impl WaylandState { self.finish_pending_eyedropper_capture( crate::input::state::EyedropperCaptureSource::Frozen, ); + self.finish_pending_ocr_capture(crate::input::state::OcrCaptureSource::Frozen); } if self.zoom.take_capture_done() { log::info!( @@ -54,6 +55,7 @@ impl WaylandState { self.finish_pending_eyedropper_capture( crate::input::state::EyedropperCaptureSource::Zoom, ); + self.finish_pending_ocr_capture(crate::input::state::OcrCaptureSource::Zoom); } } diff --git a/src/backend/wayland/state/color_picker.rs b/src/backend/wayland/state/color_picker.rs index 013b079d..bb91b38a 100644 --- a/src/backend/wayland/state/color_picker.rs +++ b/src/backend/wayland/state/color_picker.rs @@ -2,10 +2,12 @@ use super::{ClipboardOperationController, WaylandState}; use crate::backend::wayland::clipboard::ClipboardPoll; +use crate::clipboard_text::{ + ClipboardTextError, copy_text_via_command, read_clipboard_text_via_command, +}; use crate::draw::Color; use crate::input::state::{HexPasteTarget, Toast, ToastPriority}; use crate::input::state::{color_to_hex, parse_hex_color}; -use std::ffi::OsStr; use std::time::Duration; impl WaylandState { @@ -204,80 +206,6 @@ pub(super) fn submit_pending_clipboard_copy_if_idle( start_clipboard_copy(controller, hex, operation) } -pub(super) enum ClipboardTextError { - Empty, - Other(String), -} - -pub(super) fn copy_text_via_command(text: &str) -> Result<(), String> { - let output = crate::process_broker::current() - .and_then(|broker| { - broker.publish( - crate::process_broker::HelperKind::WlCopy, - OsStr::new("wl-copy"), - [OsStr::new("--type"), OsStr::new("text/plain;charset=utf-8")], - text.as_bytes().to_vec(), - Duration::from_secs(5), - ) - }) - .map_err(|error| format!("Failed to run wl-copy: {error:#}"))?; - if output.timed_out { - return Err("wl-copy timed out".to_string()); - } - if output.status != 0 { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - return if stderr.is_empty() { - Err("wl-copy exited unsuccessfully".to_string()) - } else { - Err(format!("wl-copy exited unsuccessfully: {stderr}")) - }; - } - Ok(()) -} - -pub(super) fn read_clipboard_text_via_command() -> Result { - let output = crate::process_broker::current() - .and_then(|broker| { - broker.run( - crate::process_broker::HelperKind::WlPaste, - OsStr::new("wl-paste"), - clipboard_text_read_args(), - Vec::new(), - Duration::from_secs(5), - 1024 * 1024, - ) - }) - .map_err(|err| ClipboardTextError::Other(format!("Failed to run wl-paste: {err:#}")))?; - - if !output.timed_out && output.status == 0 { - Ok(String::from_utf8_lossy(&output.stdout).to_string()) - } else { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - if stderr.to_ascii_lowercase().contains("nothing is copied") - || stderr.to_ascii_lowercase().contains("clipboard is empty") - { - Err(ClipboardTextError::Empty) - } else if stderr.is_empty() { - Err(ClipboardTextError::Other( - "wl-paste exited unsuccessfully".to_string(), - )) - } else { - Err(ClipboardTextError::Other(format!( - "wl-paste exited unsuccessfully: {}", - stderr - ))) - } - } -} - -fn clipboard_text_read_args() -> [&'static OsStr; 3] { - [ - OsStr::new("--no-newline"), - OsStr::new("--type"), - OsStr::new("text"), - ] -} - #[cfg(test)] mod tests { use std::sync::mpsc; @@ -287,18 +215,6 @@ mod tests { use crate::backend::wayland::RuntimeWakeSource; use crate::backend::wayland::clipboard::ClipboardOperationIdSource; - #[test] - fn clipboard_text_read_requests_only_a_text_mime() { - assert_eq!( - clipboard_text_read_args(), - [ - OsStr::new("--no-newline"), - OsStr::new("--type"), - OsStr::new("text"), - ] - ); - } - #[test] fn hex_copy_submission_stays_off_the_event_thread_until_completion() { let wake = RuntimeWakeSource::new().unwrap(); diff --git a/src/backend/wayland/state/core/accessors.rs b/src/backend/wayland/state/core/accessors.rs index 3c794b67..edf915a4 100644 --- a/src/backend/wayland/state/core/accessors.rs +++ b/src/backend/wayland/state/core/accessors.rs @@ -71,6 +71,41 @@ impl WaylandState { } } + /// Retire in-flight stylus input without dispatching a canvas release. + /// + /// A screen-region modal cancels the gesture the pen was making, and then + /// consumes the tip-up the compositor still owes us — so nothing else + /// retires the contact. Left set, `stylus_tip_down` hides the hover cursor + /// and defers session saving until some unrelated release or proximity-out + /// happens to clear it. The stroke was cancelled rather than finished, so + /// its peak pressure is dropped instead of being committed to the tool. + #[cfg(feature = "tablet-input")] + pub(in crate::backend::wayland) fn retire_stylus_contact(&mut self) { + // Tablet events are coalesced until their frame commits, so input that + // physically happened before the modal opened can still be sitting in + // the buffer. Replaying it afterwards would start a region from a press + // that predates the selector, or apply that batch's pressure and barrel + // actions behind it. Dropping the batch is unconditional: an uncommitted + // tip-down is exactly the case where no contact is logically held yet. + let had_contact = self.stylus_tip_down || self.pending_stylus_frame.down; + self.pending_stylus_frame = Default::default(); + // Clearing our own flags does not lift the pen. The compositor keeps + // reporting this contact — pressure included — until the tip rises, and + // a pending capture leaves the canvas holding the pen, so without this + // the disowned stroke would go on resizing the tool and would commit its + // peak on the way up. + self.stylus_contact_retired |= had_contact; + if !self.stylus_tip_down { + return; + } + let previous_hover = self.stylus_hover_cursor_position(); + self.stylus_tip_down = false; + self.stylus_pressure_thickness = None; + self.stylus_peak_thickness = None; + let next_hover = self.stylus_hover_cursor_position(); + self.mark_stylus_hover_cursor_dirty(previous_hover, next_hover); + } + #[cfg(not(feature = "tablet-input"))] pub(in crate::backend::wayland) fn stylus_hover_cursor_visible(&self) -> bool { false @@ -81,6 +116,19 @@ impl WaylandState { None } + /// Consume the disowned-contact latch, reporting whether the tip that just + /// lifted belonged to a contact a screen modal had taken over. + /// + /// A disowned tip-up dispatches no canvas release and commits no thickness; + /// the next press after it is an ordinary fresh contact. + #[cfg(feature = "tablet-input")] + pub(in crate::backend::wayland) fn take_retired_stylus_contact(&mut self) -> bool { + std::mem::take(&mut self.stylus_contact_retired) + } + + #[cfg(not(feature = "tablet-input"))] + pub(in crate::backend::wayland) fn retire_stylus_contact(&mut self) {} + pub(in crate::backend::wayland) fn set_pointer_focus(&mut self, value: bool) { self.data.has_pointer_focus = value; } diff --git a/src/backend/wayland/state/core/init.rs b/src/backend/wayland/state/core/init.rs index 05cc8cc0..252f08ce 100644 --- a/src/backend/wayland/state/core/init.rs +++ b/src/backend/wayland/state/core/init.rs @@ -120,6 +120,7 @@ impl WaylandState { ); let clipboard_text_paste = ClipboardOperationController::new(clipboard_operation_ids, runtime_wake.clone()); + let ocr = crate::ocr::OcrController::new(runtime_wake.clone()); Self { registry_state, @@ -152,6 +153,7 @@ impl WaylandState { pending_text_copy: Default::default(), clipboard_text_paste, pending_text_paste: Default::default(), + ocr, gtk_toolbar: None, onboarding, config_edits: super::super::super::config_edits::ConfigEditWorker::new( @@ -227,6 +229,8 @@ impl WaylandState { #[cfg(feature = "tablet-input")] pending_stylus_frame: crate::backend::wayland::state::PendingStylusFrame::default(), #[cfg(feature = "tablet-input")] + stylus_contact_retired: false, + #[cfg(feature = "tablet-input")] stylus_tool_types: std::collections::HashMap::new(), #[cfg(feature = "tablet-input")] stylus_auto_switched_to_eraser: false, diff --git a/src/backend/wayland/state/eyedropper.rs b/src/backend/wayland/state/eyedropper.rs index 87a68581..b01c0d8d 100644 --- a/src/backend/wayland/state/eyedropper.rs +++ b/src/backend/wayland/state/eyedropper.rs @@ -4,88 +4,10 @@ use crate::input::state::EyedropperCaptureSource; use crate::input::state::{Toast, ToastPriority}; use super::WaylandState; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum BackgroundImageKind { - Zoom, - Frozen, -} - -/// The captured image that the canvas renderer actually displays. -pub(super) struct BackgroundImageSource<'a> { - pub image: &'a FrozenImage, - pub kind: BackgroundImageKind, - pub zoom_transformed: bool, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum EyedropperEntryDecision { - Activate, - WaitForZoom, - AutoFreeze, - RefuseWhileZoomedOnSolidBoard, - RefuseSolidBoard, - CaptureUnavailable, - ZoomImageUnavailable, -} - -fn eyedropper_entry_decision( - has_source: bool, - board_is_transparent: bool, - zoom_engaged: bool, - zoom_active: bool, - frozen_enabled: bool, -) -> EyedropperEntryDecision { - if has_source { - EyedropperEntryDecision::Activate - } else if !board_is_transparent && zoom_engaged { - EyedropperEntryDecision::RefuseWhileZoomedOnSolidBoard - } else if !board_is_transparent { - EyedropperEntryDecision::RefuseSolidBoard - } else if zoom_engaged && !zoom_active { - EyedropperEntryDecision::WaitForZoom - } else if !frozen_enabled { - EyedropperEntryDecision::CaptureUnavailable - } else if zoom_active { - EyedropperEntryDecision::ZoomImageUnavailable - } else { - EyedropperEntryDecision::AutoFreeze - } -} - -pub(super) fn background_image_source<'a>( - zoom: &'a crate::backend::wayland::zoom::ZoomState, - frozen: &'a crate::backend::wayland::frozen::FrozenState, - board_is_transparent: bool, -) -> Option> { - let allow_background_image = !zoom.is_engaged() || board_is_transparent; - if !allow_background_image { - return None; - } - - if zoom.active { - if let Some(image) = zoom.image() { - return Some(BackgroundImageSource { - image, - kind: BackgroundImageKind::Zoom, - zoom_transformed: true, - }); - } - if let Some(image) = frozen.image() { - return Some(BackgroundImageSource { - image, - kind: BackgroundImageKind::Frozen, - zoom_transformed: true, - }); - } - } - - frozen.image().map(|image| BackgroundImageSource { - image, - kind: BackgroundImageKind::Frozen, - zoom_transformed: false, - }) -} +use super::screen_image::{ + DisplayedScreenImage, ScreenSourceEntry, displayed_screen_image, image_point_for_screen_point, + screen_source_entry, +}; fn sample_at(image: &FrozenImage, image_x: f64, image_y: f64) -> Option { if image.width == 0 || image.height == 0 || image.stride <= 0 { @@ -127,7 +49,10 @@ impl WaylandState { return; } - self.input_state.prepare_for_eyedropper(); + // The two screen modals are mutually exclusive; entering one ends the + // other, including any temporary freeze that one owned. + self.cancel_ocr(); + self.input_state.prepare_for_screen_modal(); self.zoom.stop_pan(); self.stop_board_pan(); self.set_board_pan_key_held(false); @@ -135,8 +60,12 @@ impl WaylandState { // toolbar move; it is not an accepted drop. self.cancel_toolbar_move_drag(); self.unlock_pointer(); + // Same as OCR: the cancelled gesture may belong to a pen that is still + // down, and the pending branches below do not activate, so a tip-up + // during the wait would commit that stroke's peak pressure to the tool. + self.retire_stylus_contact(); - let decision = eyedropper_entry_decision( + let decision = screen_source_entry( self.background_image_source().is_some(), self.input_state.board_is_transparent(), self.zoom.is_engaged(), @@ -144,16 +73,16 @@ impl WaylandState { self.frozen_enabled(), ); match decision { - EyedropperEntryDecision::Activate => self.input_state.activate_eyedropper(false), - EyedropperEntryDecision::WaitForZoom => self + ScreenSourceEntry::Activate => self.activate_eyedropper_sampler(false), + ScreenSourceEntry::WaitForZoom => self .input_state .set_eyedropper_pending_capture(EyedropperCaptureSource::Zoom), - EyedropperEntryDecision::AutoFreeze => { + ScreenSourceEntry::AutoFreeze => { self.input_state .set_eyedropper_pending_capture(EyedropperCaptureSource::Frozen); self.input_state.request_frozen_toggle(); } - EyedropperEntryDecision::RefuseWhileZoomedOnSolidBoard => { + ScreenSourceEntry::RefuseWhileZoomedOnSolidBoard => { self.input_state.push_toast( ToastPriority::Action, "eyedropper", @@ -164,10 +93,10 @@ impl WaylandState { ), ); } - EyedropperEntryDecision::RefuseSolidBoard => { + ScreenSourceEntry::RefuseSolidBoard => { self.input_state.push_toast(ToastPriority::Action, "eyedropper", Toast::info("Screen eyedropper requires a transparent board or an active screen freeze.").action("Switch to transparent", crate::config::Action::ReturnToTransparent)); } - EyedropperEntryDecision::CaptureUnavailable => { + ScreenSourceEntry::CaptureUnavailable => { self.input_state.push_toast( ToastPriority::Info, "eyedropper", @@ -176,14 +105,14 @@ impl WaylandState { ), ); } - EyedropperEntryDecision::ZoomImageUnavailable => { + ScreenSourceEntry::ZoomImageUnavailable => { self.input_state.push_toast(ToastPriority::Info, "eyedropper", Toast::warning("Screen eyedropper is unavailable because zoom has no captured screen image.")); } } } - fn background_image_source(&self) -> Option> { - background_image_source( + fn background_image_source(&self) -> Option> { + displayed_screen_image( &self.zoom, &self.frozen, self.input_state.board_is_transparent(), @@ -198,8 +127,10 @@ impl WaylandState { return; } if self.background_image_source().is_some() { - self.input_state - .activate_eyedropper(matches!(capture_source, EyedropperCaptureSource::Frozen)); + self.activate_eyedropper_sampler(matches!( + capture_source, + EyedropperCaptureSource::Frozen + )); } else { self.cancel_eyedropper(); self.input_state @@ -207,6 +138,14 @@ impl WaylandState { } } + /// Arm the screen sampler. Retires an in-flight stylus contact for the same + /// reason OCR does: from here the modal swallows the tip-up that would + /// otherwise end it. + fn activate_eyedropper_sampler(&mut self, auto_froze: bool) { + self.retire_stylus_contact(); + self.input_state.activate_eyedropper(auto_froze); + } + pub(in crate::backend::wayland) fn update_eyedropper_hover(&mut self, x: f64, y: f64) { if self.input_state.eyedropper_is_active() { self.input_state.update_eyedropper_hover((x, y)); @@ -244,18 +183,15 @@ impl WaylandState { fn eyedropper_image_coords( &self, - source: &BackgroundImageSource<'_>, + source: &DisplayedScreenImage<'_>, x: f64, y: f64, ) -> (f64, f64) { - let (world_x, world_y) = if source.zoom_transformed { - self.zoom.screen_to_world(x, y) - } else { - (x, y) - }; - ( - world_x * f64::from(source.image.width) / f64::from(self.surface.width()).max(1.0), - world_y * f64::from(source.image.height) / f64::from(self.surface.height()).max(1.0), + image_point_for_screen_point( + source, + &self.zoom, + (self.surface.width(), self.surface.height()), + (x, y), ) } @@ -427,70 +363,43 @@ mod tests { assert_eq!(sample_at(&image, 20.0, 20.0).unwrap().b, 1.0); } - #[test] - fn background_source_truth_table_matches_rendering_contract() { - let mut zoom = crate::backend::wayland::zoom::ZoomState::new(None); - let mut frozen = crate::backend::wayland::frozen::FrozenState::new(None); - assert!(background_image_source(&zoom, &frozen, true).is_none()); - - frozen.set_image(image([0, 0, 255, 255])); - let source = background_image_source(&zoom, &frozen, false).unwrap(); - assert_eq!(source.kind, BackgroundImageKind::Frozen); - assert!(!source.zoom_transformed); - - zoom.active = true; - let source = background_image_source(&zoom, &frozen, true).unwrap(); - assert_eq!(source.kind, BackgroundImageKind::Frozen); - assert!(source.zoom_transformed); - assert!(background_image_source(&zoom, &frozen, false).is_none()); - - zoom.set_image(image([255, 0, 0, 255])); - let source = background_image_source(&zoom, &frozen, true).unwrap(); - assert_eq!(source.kind, BackgroundImageKind::Zoom); - assert!(source.zoom_transformed); - - zoom.active = false; - zoom.request_activation(); - assert!(background_image_source(&zoom, &frozen, false).is_none()); - } - #[test] fn entry_waits_for_pending_zoom_instead_of_starting_frozen_capture() { assert_eq!( - eyedropper_entry_decision(false, true, true, false, true), - EyedropperEntryDecision::WaitForZoom + screen_source_entry(false, true, true, false, true), + ScreenSourceEntry::WaitForZoom ); } #[test] fn entry_distinguishes_solid_board_zoom_refusal() { assert_eq!( - eyedropper_entry_decision(false, false, true, true, true), - EyedropperEntryDecision::RefuseWhileZoomedOnSolidBoard + screen_source_entry(false, false, true, true, true), + ScreenSourceEntry::RefuseWhileZoomedOnSolidBoard ); assert_eq!( - eyedropper_entry_decision(false, false, false, false, true), - EyedropperEntryDecision::RefuseSolidBoard + screen_source_entry(false, false, false, false, true), + ScreenSourceEntry::RefuseSolidBoard ); } #[test] fn entry_uses_existing_source_before_board_policy() { assert_eq!( - eyedropper_entry_decision(true, false, false, false, true), - EyedropperEntryDecision::Activate + screen_source_entry(true, false, false, false, true), + ScreenSourceEntry::Activate ); } #[test] fn entry_reports_missing_zoom_image_separately_from_missing_capture_support() { assert_eq!( - eyedropper_entry_decision(false, true, true, true, true), - EyedropperEntryDecision::ZoomImageUnavailable + screen_source_entry(false, true, true, true, true), + ScreenSourceEntry::ZoomImageUnavailable ); assert_eq!( - eyedropper_entry_decision(false, true, false, false, false), - EyedropperEntryDecision::CaptureUnavailable + screen_source_entry(false, true, false, false, false), + ScreenSourceEntry::CaptureUnavailable ); } } diff --git a/src/backend/wayland/state/ocr.rs b/src/backend/wayland/state/ocr.rs new file mode 100644 index 00000000..4f2ea6bc --- /dev/null +++ b/src/backend/wayland/state/ocr.rs @@ -0,0 +1,474 @@ +//! `Copy text from screen`: capture ownership, region selection, and outcomes. +//! +//! OCR reads the desktop image the renderer is already displaying, so it reuses +//! the eyedropper's freeze/zoom ownership rule: a freeze OCR created is a freeze +//! OCR releases, and a user-owned freeze or zoom survives untouched. Nothing +//! here mutates annotations, history, boards, sessions, or the active tool. + +use crate::input::state::{OcrCaptureSource, OcrInputSource, Toast, ToastPriority}; +use crate::ocr::{OcrFailure, OcrLanguages, OcrPoll, OcrRequest, OcrSubmitError, OcrSuccess}; + +use super::WaylandState; +use super::screen_image::{ + CropError, DisplayedScreenImage, ScreenSourceEntry, copy_image_rect, displayed_screen_image, + image_rect_for_screen_rect, screen_source_entry, +}; + +/// Drags below this many logical pixels on either axis are treated as a stray +/// click and never reach the engine. +const MIN_REGION_LOGICAL_PIXELS: f64 = 4.0; + +const TOAST_SOURCE: &str = "ocr"; + +impl WaylandState { + /// Drain a `Copy text from screen` request into the region selector. + pub(in crate::backend::wayland) fn handle_pending_ocr_request(&mut self) { + if !self.input_state.take_pending_ocr_request() { + return; + } + if self.input_state.ocr_state().is_engaged() { + // A second invocation while the selector is up cancels it, matching + // the eyedropper's toggle behavior. + self.cancel_ocr(); + return; + } + if !self.config.capture.enabled { + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::warning("Screen text recognition is off because capture is disabled."), + ); + return; + } + if self.ocr.is_active() { + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::info("OCR is already running"), + ); + return; + } + + // The two screen modals are mutually exclusive; entering one ends the + // other, including any temporary freeze that one owned. + self.cancel_eyedropper(); + self.input_state.prepare_for_screen_modal(); + self.zoom.stop_pan(); + self.stop_board_pan(); + self.set_board_pan_key_held(false); + self.cancel_toolbar_move_drag(); + self.unlock_pointer(); + // The gesture just cancelled above may belong to a pen that is still + // down. Activation retires the contact on its own, but the pending + // branches below do not activate: without this, a tip-up during the + // wait would commit the cancelled stroke's peak pressure to the tool. + self.retire_stylus_contact(); + + match screen_source_entry( + self.ocr_screen_source().is_some(), + self.input_state.board_is_transparent(), + self.zoom.is_engaged(), + self.zoom.active, + self.frozen_enabled(), + ) { + ScreenSourceEntry::Activate => self.activate_ocr_selector(false), + ScreenSourceEntry::WaitForZoom => { + self.input_state + .set_ocr_pending_capture(OcrCaptureSource::Zoom); + } + ScreenSourceEntry::AutoFreeze => { + self.input_state + .set_ocr_pending_capture(OcrCaptureSource::Frozen); + self.input_state.request_frozen_toggle(); + } + ScreenSourceEntry::RefuseWhileZoomedOnSolidBoard + | ScreenSourceEntry::RefuseSolidBoard => { + self.input_state.push_toast( + ToastPriority::Action, + TOAST_SOURCE, + Toast::info("OCR needs a visible screen image.").action( + "Switch to transparent", + crate::config::Action::ReturnToTransparent, + ), + ); + } + ScreenSourceEntry::CaptureUnavailable => { + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::warning( + "Screen text recognition is unavailable because screen capture is not available.", + ), + ); + } + ScreenSourceEntry::ZoomImageUnavailable => { + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::warning( + "Screen text recognition is unavailable because zoom has no captured screen image.", + ), + ); + } + } + } + + fn ocr_screen_source(&self) -> Option> { + displayed_screen_image( + &self.zoom, + &self.frozen, + self.input_state.board_is_transparent(), + ) + } + + pub(in crate::backend::wayland) fn finish_pending_ocr_capture( + &mut self, + capture_source: OcrCaptureSource, + ) { + if self.input_state.ocr_state().pending_source() != Some(capture_source) { + return; + } + if self.ocr_screen_source().is_some() { + self.activate_ocr_selector(matches!(capture_source, OcrCaptureSource::Frozen)); + } else { + self.cancel_ocr(); + self.input_state.report_ocr_capture_failure_if_unreported(); + } + } + + /// Arm the region selector. + /// + /// Activation cancels whatever gesture was in flight, and from that moment + /// the selector swallows the events that would have ended it — including a + /// stylus tip-up. The contact is retired alongside, so the two can never be + /// separated: a capture that lands seconds after the request arms the + /// selector against a gesture that started during the wait, not the one the + /// request itself cancelled. + fn activate_ocr_selector(&mut self, auto_froze: bool) { + self.retire_stylus_contact(); + self.input_state.activate_ocr(auto_froze); + } + + /// Leave OCR selection, releasing only a freeze OCR created itself. + pub(in crate::backend::wayland) fn cancel_ocr(&mut self) -> bool { + let state = self.input_state.ocr_state(); + let was_engaged = state.is_engaged(); + let pending_source = state.pending_source(); + let auto_froze = self.input_state.cancel_ocr(); + self.release_ocr_capture(auto_froze, pending_source); + was_engaged + } + + /// Give back the temporary freeze OCR created. A user-owned freeze or an + /// active zoom is never touched here. + fn release_ocr_capture(&mut self, auto_froze: bool, pending_source: Option) { + if !auto_froze { + return; + } + self.restore_xdg_after_frozen(); + if pending_source == Some(OcrCaptureSource::Frozen) && self.frozen.is_in_progress() { + self.frozen.cancel(&mut self.input_state); + self.exit_overlay_suppression(super::OverlaySuppression::Frozen); + } else { + self.frozen.unfreeze(&mut self.input_state); + } + } + + /// A display or source change invalidates the pixels the user is selecting. + pub(in crate::backend::wayland) fn cancel_ocr_if_source_missing(&mut self) { + if self.input_state.ocr_is_active() && self.ocr_screen_source().is_none() { + self.cancel_ocr(); + } + } + + pub(in crate::backend::wayland) fn begin_ocr_selection( + &mut self, + owner: OcrInputSource, + x: f64, + y: f64, + ) -> bool { + self.input_state.start_ocr_selection(owner, (x, y)) + } + + pub(in crate::backend::wayland) fn update_ocr_selection( + &mut self, + source: OcrInputSource, + x: f64, + y: f64, + ) { + self.input_state.update_ocr_selection(source, (x, y)); + } + + /// Discard a region because the device dragging it went away — the pen left + /// proximity, the touch sequence was cancelled. Devices that are not + /// dragging have nothing to withdraw, so this leaves the selector armed for + /// whichever one is. + pub(in crate::backend::wayland) fn cancel_ocr_selection_from( + &mut self, + source: OcrInputSource, + ) -> bool { + if !self.input_state.ocr_selection_is_owned_by(source) { + return false; + } + self.cancel_ocr() + } + + /// End the drag: own the selected pixels, release an OCR-created freeze, + /// and submit. Returns whether `source` had a region drag to finish. + pub(in crate::backend::wayland) fn finish_ocr_selection( + &mut self, + source: OcrInputSource, + x: f64, + y: f64, + ) -> bool { + // A release from a device that is not dragging must not submit — or + // even see — the region another one is still drawing. + if !self.input_state.ocr_selection_is_owned_by(source) { + return false; + } + self.input_state.update_ocr_selection(source, (x, y)); + let Some(selection) = self.input_state.ocr_state().selection() else { + return false; + }; + + // The crop is taken while the capture is still held: releasing first + // would leave the worker reading pixels that no longer exist. + match self.crop_ocr_selection(selection.start, selection.end) { + Ok(None) => self.input_state.rearm_ocr_selection(), + Ok(Some(pixels)) => { + let auto_froze = self.input_state.cancel_ocr(); + self.release_ocr_capture(auto_froze, None); + self.submit_ocr_request(pixels); + } + Err(message) => { + let auto_froze = self.input_state.cancel_ocr(); + self.release_ocr_capture(auto_froze, None); + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::warning(message), + ); + } + } + true + } + + /// `Ok(None)` means the drag was too small to be a region — a stray click, + /// not a failure worth a toast. + fn crop_ocr_selection( + &self, + start: (f64, f64), + end: (f64, f64), + ) -> Result, &'static str> { + if (end.0 - start.0).abs() < MIN_REGION_LOGICAL_PIXELS + || (end.1 - start.1).abs() < MIN_REGION_LOGICAL_PIXELS + { + return Ok(None); + } + let Some(source) = self.ocr_screen_source() else { + return Err("The screen image for text recognition is no longer available."); + }; + let Some(rect) = image_rect_for_screen_rect( + &source, + &self.zoom, + (self.surface.width(), self.surface.height()), + start, + end, + ) else { + return Ok(None); + }; + copy_image_rect(source.image, rect) + .map(Some) + .map_err(|error| match error { + CropError::Empty => "That selection has no screen pixels.", + CropError::OutOfBounds => "Could not read that region of the screen image.", + }) + } + + fn submit_ocr_request(&mut self, pixels: crate::ocr::OcrPixels) { + let languages = OcrLanguages::from_validated(self.config.capture.resolved_ocr_languages()); + log::debug!( + "OCR submitting {}x{} crop for languages {}", + pixels.width, + pixels.height, + languages.as_str() + ); + match self.ocr.try_submit(OcrRequest { pixels, languages }) { + Ok(id) => { + // wl-copy needs the overlay to stay alive long enough to serve + // the selection it publishes. + self.suppress_focus_exit_for(std::time::Duration::from_millis(1500)); + log::debug!("OCR request {id} started"); + } + Err(OcrSubmitError::Busy { .. }) => { + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::info("OCR is already running"), + ); + } + Err(error) => { + log::warn!("Failed to start screen text recognition: {error}"); + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::warning("Could not start screen text recognition."), + ); + } + } + } + + /// Turn a finished recognition into one stable toast. The completion never + /// carries recognized text, so nothing here can leak screen content. + pub(in crate::backend::wayland) fn poll_ocr_completion(&mut self) { + match self.ocr.poll() { + OcrPoll::Idle | OcrPoll::Pending => {} + OcrPoll::Ready { + id, + outcome: + Ok(OcrSuccess::Copied { + character_count, + replaced_invalid_utf8, + }), + } => { + log::debug!("OCR request {id} copied {character_count} characters"); + let message = if replaced_invalid_utf8 { + "Text copied (some characters were unreadable)" + } else { + "Text copied" + }; + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::info(message), + ); + } + OcrPoll::Ready { + outcome: Ok(OcrSuccess::NoTextFound), + .. + } => { + self.input_state.push_toast( + ToastPriority::Info, + TOAST_SOURCE, + Toast::info("No text found"), + ); + } + OcrPoll::Ready { + id, + outcome: Err(failure), + } => { + log::warn!("OCR request {id} failed: {failure:?}"); + let toast = match failure { + OcrFailure::EngineMissing | OcrFailure::LanguageMissing { .. } => { + Toast::warning(failure.message()) + } + _ => Toast::error(failure.message()), + }; + self.input_state + .push_toast(ToastPriority::Critical, TOAST_SOURCE, toast); + } + OcrPoll::WorkerLost { id, reason } => { + log::error!("OCR request {id} lost its worker: {reason}"); + self.input_state.push_toast( + ToastPriority::Critical, + TOAST_SOURCE, + Toast::error("Screen text recognition failed."), + ); + } + } + } + + /// Paint the region selector: a scrim over everything outside the drag and + /// scan-corner brackets around it. Never baked into the source crop — this + /// runs in the UI pass, over a copy the worker already owns. + pub(in crate::backend::wayland) fn render_ocr_selection( + &self, + ctx: &cairo::Context, + screen_width: u32, + screen_height: u32, + ) { + if !self.input_state.ocr_is_active() { + return; + } + let width = f64::from(screen_width); + let height = f64::from(screen_height); + let selection = self.input_state.ocr_state().selection(); + + let _ = ctx.save(); + ctx.set_source_rgba(0.02, 0.03, 0.05, 0.35); + ctx.rectangle(0.0, 0.0, width, height); + if let Some(selection) = selection { + let (x, y, w, h) = normalized_rect(selection.start, selection.end); + // Even-odd keeps the selected region unscrimmed, so the user reads + // the real screen pixels the crop will take. + ctx.rectangle(x, y, w, h); + ctx.set_fill_rule(cairo::FillRule::EvenOdd); + } + let _ = ctx.fill(); + ctx.set_fill_rule(cairo::FillRule::Winding); + + if let Some(selection) = selection { + let (x, y, w, h) = normalized_rect(selection.start, selection.end); + ctx.set_source_rgba(1.0, 1.0, 1.0, 0.9); + ctx.set_line_width(1.0); + ctx.rectangle(x + 0.5, y + 0.5, (w - 1.0).max(0.0), (h - 1.0).max(0.0)); + let _ = ctx.stroke(); + draw_scan_corners(ctx, x, y, w, h); + } + let _ = ctx.restore(); + } +} + +fn normalized_rect(start: (f64, f64), end: (f64, f64)) -> (f64, f64, f64, f64) { + let x = start.0.min(end.0); + let y = start.1.min(end.1); + (x, y, (end.0 - start.0).abs(), (end.1 - start.1).abs()) +} + +/// Four bracket corners, the visual language a scan frame reads as. The arm +/// length shrinks with the region so a small selection is not swallowed by it. +fn draw_scan_corners(ctx: &cairo::Context, x: f64, y: f64, w: f64, h: f64) { + let arm = (w.min(h) / 4.0).clamp(4.0, 20.0); + ctx.set_source_rgba(1.0, 1.0, 1.0, 1.0); + ctx.set_line_width(2.0); + ctx.set_line_cap(cairo::LineCap::Square); + for (corner_x, corner_y, dx, dy) in [ + (x, y, 1.0, 1.0), + (x + w, y, -1.0, 1.0), + (x, y + h, 1.0, -1.0), + (x + w, y + h, -1.0, -1.0), + ] { + ctx.move_to(corner_x + dx * arm, corner_y); + ctx.line_to(corner_x, corner_y); + ctx.line_to(corner_x, corner_y + dy * arm); + let _ = ctx.stroke(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalization_orders_any_pair_of_drag_corners() { + assert_eq!( + normalized_rect((30.0, 40.0), (10.0, 20.0)), + (10.0, 20.0, 20.0, 20.0) + ); + assert_eq!( + normalized_rect((10.0, 20.0), (30.0, 40.0)), + (10.0, 20.0, 20.0, 20.0) + ); + } + + #[test] + fn scan_corner_arms_stay_inside_a_small_region() { + // The clamp floor is 4px; the guard is that the arm never spans more + // than the region it decorates. + for size in [1.0_f64, 8.0, 40.0, 400.0] { + let arm = (size / 4.0).clamp(4.0, 20.0); + assert!(arm <= 20.0); + assert!(arm >= 4.0); + } + } +} diff --git a/src/backend/wayland/state/render/canvas/background.rs b/src/backend/wayland/state/render/canvas/background.rs index b0cc62ac..7aa5485a 100644 --- a/src/backend/wayland/state/render/canvas/background.rs +++ b/src/backend/wayland/state/render/canvas/background.rs @@ -1,5 +1,5 @@ use super::super::super::*; -use crate::backend::wayland::state::eyedropper::{BackgroundImageKind, background_image_source}; +use crate::backend::wayland::state::screen_image::{ScreenImageKind, displayed_screen_image}; use crate::draw::Color; pub(super) struct CanvasEraserContext { @@ -39,15 +39,15 @@ impl WaylandState { let mut logical_to_image_scale_x = 1.0; let mut logical_to_image_scale_y = 1.0; - let background_image = background_image_source( + let background_image = displayed_screen_image( &self.zoom, &self.frozen, self.input_state.board_is_transparent(), ) .map(|source| { let cache_key = match source.kind { - BackgroundImageKind::Zoom => (self.zoom.image_generation() << 1) | 1, - BackgroundImageKind::Frozen => self.frozen.image_generation() << 1, + ScreenImageKind::Zoom => (self.zoom.image_generation() << 1) | 1, + ScreenImageKind::Frozen => self.frozen.image_generation() << 1, }; (source.image, cache_key, source.zoom_transformed) }); diff --git a/src/backend/wayland/state/render/ui.rs b/src/backend/wayland/state/render/ui.rs index 4448ab2b..e4e589cd 100644 --- a/src/backend/wayland/state/render/ui.rs +++ b/src/backend/wayland/state/render/ui.rs @@ -227,6 +227,7 @@ impl WaylandState { } self.render_eyedropper_loupe(ctx, width, height); + self.render_ocr_selection(ctx, width, height); if self.input_state.is_radial_menu_open() { // Layout (and with it hit-testing) is live from the moment diff --git a/src/backend/wayland/state/screen_image.rs b/src/backend/wayland/state/screen_image.rs new file mode 100644 index 00000000..877c7a67 --- /dev/null +++ b/src/backend/wayland/state/screen_image.rs @@ -0,0 +1,473 @@ +//! The captured desktop image the canvas renderer actually displays. +//! +//! The screen eyedropper samples one pixel of it and OCR crops a rectangle out +//! of it. Both must read exactly what Wayscriber is showing rather than a fresh +//! screenshot, so source resolution, the logical→image mapping, and the checked +//! rectangle copy live here instead of inside either feature. + +use crate::backend::wayland::frozen::{FrozenImage, FrozenState}; +use crate::backend::wayland::zoom::ZoomState; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ScreenImageKind { + Zoom, + Frozen, +} + +/// The captured image that the canvas renderer actually displays. +pub(super) struct DisplayedScreenImage<'a> { + pub image: &'a FrozenImage, + pub kind: ScreenImageKind, + /// The renderer is painting this image through the zoom transform, so + /// screen points must be un-zoomed before they are scaled into the image. + pub zoom_transformed: bool, +} + +/// A rectangle in source-image pixels, guaranteed non-empty and in bounds of +/// the image it was resolved against. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct ImageRect { + pub x: u32, + pub y: u32, + pub width: u32, + pub height: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum CropError { + /// The rectangle has no pixels, or the source image is empty. + Empty, + /// The rectangle leaves the source image, or its size does not fit memory. + OutOfBounds, +} + +/// What a modal that needs the displayed screen image should do when the user +/// asks for it. Shared by the eyedropper and OCR so the two cannot disagree +/// about when a temporary freeze is created or refused; each owns only the +/// wording it shows. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ScreenSourceEntry { + Activate, + WaitForZoom, + AutoFreeze, + RefuseWhileZoomedOnSolidBoard, + RefuseSolidBoard, + CaptureUnavailable, + ZoomImageUnavailable, +} + +pub(super) fn screen_source_entry( + has_source: bool, + board_is_transparent: bool, + zoom_engaged: bool, + zoom_active: bool, + frozen_enabled: bool, +) -> ScreenSourceEntry { + if has_source { + ScreenSourceEntry::Activate + } else if !board_is_transparent && zoom_engaged { + ScreenSourceEntry::RefuseWhileZoomedOnSolidBoard + } else if !board_is_transparent { + ScreenSourceEntry::RefuseSolidBoard + } else if zoom_engaged && !zoom_active { + ScreenSourceEntry::WaitForZoom + } else if !frozen_enabled { + ScreenSourceEntry::CaptureUnavailable + } else if zoom_active { + ScreenSourceEntry::ZoomImageUnavailable + } else { + ScreenSourceEntry::AutoFreeze + } +} + +pub(super) fn displayed_screen_image<'a>( + zoom: &'a ZoomState, + frozen: &'a FrozenState, + board_is_transparent: bool, +) -> Option> { + let allow_background_image = !zoom.is_engaged() || board_is_transparent; + if !allow_background_image { + return None; + } + + if zoom.active { + if let Some(image) = zoom.image() { + return Some(DisplayedScreenImage { + image, + kind: ScreenImageKind::Zoom, + zoom_transformed: true, + }); + } + if let Some(image) = frozen.image() { + return Some(DisplayedScreenImage { + image, + kind: ScreenImageKind::Frozen, + zoom_transformed: true, + }); + } + } + + frozen.image().map(|image| DisplayedScreenImage { + image, + kind: ScreenImageKind::Frozen, + zoom_transformed: false, + }) +} + +/// Map a logical screen point onto source-image pixel coordinates. The result +/// is unclamped on purpose: samplers clamp to the nearest pixel, while region +/// selection clips against the image bounds. +pub(super) fn image_point_for_screen_point( + source: &DisplayedScreenImage<'_>, + zoom: &ZoomState, + surface: (u32, u32), + point: (f64, f64), +) -> (f64, f64) { + let (world_x, world_y) = if source.zoom_transformed { + zoom.screen_to_world(point.0, point.1) + } else { + point + }; + ( + world_x * f64::from(source.image.width) / f64::from(surface.0).max(1.0), + world_y * f64::from(source.image.height) / f64::from(surface.1).max(1.0), + ) +} + +/// Map a logical screen rectangle (given by any two opposite corners) onto a +/// non-empty in-bounds source-image rectangle, or `None` when the two corners +/// resolve to nothing the image can supply. +pub(super) fn image_rect_for_screen_rect( + source: &DisplayedScreenImage<'_>, + zoom: &ZoomState, + surface: (u32, u32), + start: (f64, f64), + end: (f64, f64), +) -> Option { + let first = image_point_for_screen_point(source, zoom, surface, start); + let second = image_point_for_screen_point(source, zoom, surface, end); + image_rect_from_points(first, second, source.image.width, source.image.height) +} + +/// Clip two unordered image-space points into an integer pixel rectangle. +/// Reversed drags, partly out-of-bounds drags, and non-finite coordinates all +/// resolve deterministically here rather than at the call sites. +fn image_rect_from_points( + first: (f64, f64), + second: (f64, f64), + image_width: u32, + image_height: u32, +) -> Option { + if image_width == 0 || image_height == 0 { + return None; + } + if ![first.0, first.1, second.0, second.1] + .iter() + .all(|value| value.is_finite()) + { + return None; + } + + let width = f64::from(image_width); + let height = f64::from(image_height); + let left = first.0.min(second.0).floor().clamp(0.0, width); + let top = first.1.min(second.1).floor().clamp(0.0, height); + let right = first.0.max(second.0).ceil().clamp(0.0, width); + let bottom = first.1.max(second.1).ceil().clamp(0.0, height); + if right <= left || bottom <= top { + return None; + } + + Some(ImageRect { + x: left as u32, + y: top as u32, + width: (right - left) as u32, + height: (bottom - top) as u32, + }) +} + +/// Copy `rect` out of `source` into a tightly packed owned ARGB snapshot. +/// +/// The source may carry row padding; the copy never does, so the worker can +/// hand it straight to Cairo without re-deriving a stride. +pub(super) fn copy_image_rect( + source: &FrozenImage, + rect: ImageRect, +) -> Result { + if rect.width == 0 || rect.height == 0 || source.width == 0 || source.height == 0 { + return Err(CropError::Empty); + } + if rect + .x + .checked_add(rect.width) + .ok_or(CropError::OutOfBounds)? + > source.width + || rect + .y + .checked_add(rect.height) + .ok_or(CropError::OutOfBounds)? + > source.height + { + return Err(CropError::OutOfBounds); + } + let source_stride = usize::try_from(source.stride).map_err(|_| CropError::OutOfBounds)?; + let row_bytes = usize::try_from(rect.width) + .ok() + .and_then(|width| width.checked_mul(4)) + .ok_or(CropError::OutOfBounds)?; + let total_bytes = usize::try_from(rect.height) + .ok() + .and_then(|height| height.checked_mul(row_bytes)) + .ok_or(CropError::OutOfBounds)?; + let stride = i32::try_from(row_bytes).map_err(|_| CropError::OutOfBounds)?; + + let mut data = Vec::with_capacity(total_bytes); + let start_column = usize::try_from(rect.x) + .ok() + .and_then(|x| x.checked_mul(4)) + .ok_or(CropError::OutOfBounds)?; + for row in 0..rect.height { + let source_row = usize::try_from(rect.y.saturating_add(row)) + .ok() + .and_then(|row| row.checked_mul(source_stride)) + .ok_or(CropError::OutOfBounds)?; + let start = source_row + .checked_add(start_column) + .ok_or(CropError::OutOfBounds)?; + let end = start.checked_add(row_bytes).ok_or(CropError::OutOfBounds)?; + data.extend_from_slice(source.data.get(start..end).ok_or(CropError::OutOfBounds)?); + } + + Ok(crate::ocr::OcrPixels { + width: rect.width, + height: rect.height, + stride, + data, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn image(width: u32, height: u32, stride: i32, data: Vec) -> FrozenImage { + FrozenImage { + width, + height, + stride, + data, + } + } + + fn opaque(width: u32, height: u32) -> FrozenImage { + let mut data = Vec::with_capacity((width * height * 4) as usize); + for y in 0..height { + for x in 0..width { + data.extend_from_slice(&[x as u8, y as u8, 0, 0xFF]); + } + } + image(width, height, (width * 4) as i32, data) + } + + fn zoom_state() -> ZoomState { + ZoomState::new(None) + } + + #[test] + fn displayed_source_truth_table_matches_the_rendering_contract() { + let mut zoom = zoom_state(); + let mut frozen = FrozenState::new(None); + assert!(displayed_screen_image(&zoom, &frozen, true).is_none()); + + frozen.set_image(opaque(2, 2)); + let source = displayed_screen_image(&zoom, &frozen, false).unwrap(); + assert_eq!(source.kind, ScreenImageKind::Frozen); + assert!(!source.zoom_transformed); + + zoom.active = true; + let source = displayed_screen_image(&zoom, &frozen, true).unwrap(); + assert_eq!(source.kind, ScreenImageKind::Frozen); + assert!(source.zoom_transformed); + assert!(displayed_screen_image(&zoom, &frozen, false).is_none()); + + zoom.set_image(opaque(2, 2)); + let source = displayed_screen_image(&zoom, &frozen, true).unwrap(); + assert_eq!(source.kind, ScreenImageKind::Zoom); + assert!(source.zoom_transformed); + + zoom.active = false; + zoom.request_activation(); + assert!(displayed_screen_image(&zoom, &frozen, false).is_none()); + } + + #[test] + fn unzoomed_points_scale_by_the_image_to_surface_ratio() { + let mut frozen = FrozenState::new(None); + frozen.set_image(opaque(200, 100)); + let zoom = zoom_state(); + let source = displayed_screen_image(&zoom, &frozen, true).unwrap(); + + let point = image_point_for_screen_point(&source, &zoom, (100, 50), (50.0, 25.0)); + assert!((point.0 - 100.0).abs() < f64::EPSILON); + assert!((point.1 - 50.0).abs() < f64::EPSILON); + } + + #[test] + fn zoomed_points_use_the_same_transform_the_renderer_applied() { + let mut zoom = zoom_state(); + zoom.active = true; + zoom.set_image(opaque(100, 100)); + zoom.scale = 2.0; + zoom.view_offset = (10.0, 20.0); + let frozen = FrozenState::new(None); + let source = displayed_screen_image(&zoom, &frozen, true).unwrap(); + + let point = image_point_for_screen_point(&source, &zoom, (100, 100), (40.0, 60.0)); + // screen_to_world(40, 60) = (10 + 20, 20 + 30) with a 1:1 image scale. + assert!((point.0 - 30.0).abs() < f64::EPSILON); + assert!((point.1 - 50.0).abs() < f64::EPSILON); + } + + #[test] + fn reversed_drags_normalize_and_partly_offscreen_drags_clip() { + let rect = image_rect_from_points((30.0, 40.0), (10.0, 20.0), 100, 100).unwrap(); + assert_eq!( + rect, + ImageRect { + x: 10, + y: 20, + width: 20, + height: 20, + } + ); + + let clipped = image_rect_from_points((-50.0, -50.0), (10.5, 10.5), 100, 100).unwrap(); + assert_eq!( + clipped, + ImageRect { + x: 0, + y: 0, + width: 11, + height: 11, + } + ); + } + + #[test] + fn degenerate_rectangles_are_rejected_instead_of_clamped_to_one_pixel() { + assert!(image_rect_from_points((10.0, 10.0), (10.0, 10.0), 100, 100).is_none()); + assert!(image_rect_from_points((-20.0, -20.0), (-5.0, -5.0), 100, 100).is_none()); + assert!(image_rect_from_points((0.0, 0.0), (10.0, 10.0), 0, 10).is_none()); + assert!(image_rect_from_points((f64::NAN, 0.0), (10.0, 10.0), 100, 100).is_none()); + assert!(image_rect_from_points((0.0, 0.0), (f64::INFINITY, 10.0), 100, 100).is_none()); + } + + #[test] + fn crop_repacks_a_padded_source_row_by_row() { + let source = image( + 2, + 2, + 12, + vec![ + 1, 1, 1, 255, 2, 2, 2, 255, 9, 9, 9, 9, // + 3, 3, 3, 255, 4, 4, 4, 255, 8, 8, 8, 8, + ], + ); + + let crop = copy_image_rect( + &source, + ImageRect { + x: 1, + y: 0, + width: 1, + height: 2, + }, + ) + .unwrap(); + + assert_eq!((crop.width, crop.height, crop.stride), (1, 2, 4)); + assert_eq!(crop.data, vec![2, 2, 2, 255, 4, 4, 4, 255]); + } + + #[test] + fn crop_preserves_premultiplied_and_transparent_pixels_verbatim() { + let source = image(2, 1, 8, vec![25, 50, 100, 128, 0, 0, 0, 0]); + + let crop = copy_image_rect( + &source, + ImageRect { + x: 0, + y: 0, + width: 2, + height: 1, + }, + ) + .unwrap(); + + assert_eq!(crop.data, vec![25, 50, 100, 128, 0, 0, 0, 0]); + } + + #[test] + fn crop_rejects_empty_and_out_of_bounds_rectangles() { + let source = opaque(4, 4); + let error = |rect| copy_image_rect(&source, rect).err(); + + assert_eq!( + error(ImageRect { + x: 0, + y: 0, + width: 0, + height: 2, + }), + Some(CropError::Empty) + ); + assert_eq!( + error(ImageRect { + x: 3, + y: 0, + width: 2, + height: 1, + }), + Some(CropError::OutOfBounds) + ); + assert_eq!( + error(ImageRect { + x: u32::MAX, + y: 0, + width: 2, + height: 1, + }), + Some(CropError::OutOfBounds) + ); + } + + #[test] + fn crop_of_a_rotated_capture_keeps_the_transformed_orientation() { + use wayland_client::protocol::wl_output; + + // Two rows of three distinct pixels; rotating to 270 makes the source + // 2x3, and the crop must read that post-transform layout. + let mut data = Vec::new(); + for value in [1u8, 2, 3, 4, 5, 6] { + data.extend_from_slice(&[value, 0, 0, 0xFF]); + } + let rotated = image(3, 2, 12, data).with_output_transform(wl_output::Transform::_270); + assert_eq!((rotated.width, rotated.height), (2, 3)); + + let crop = copy_image_rect( + &rotated, + ImageRect { + x: 0, + y: 0, + width: 2, + height: 1, + }, + ) + .unwrap(); + + assert_eq!( + crop.data.chunks_exact(4).map(|p| p[0]).collect::>(), + vec![4, 1] + ); + } +} diff --git a/src/backend/wayland/state/text_clipboard.rs b/src/backend/wayland/state/text_clipboard.rs index f65b2fae..2a1edea2 100644 --- a/src/backend/wayland/state/text_clipboard.rs +++ b/src/backend/wayland/state/text_clipboard.rs @@ -8,11 +8,11 @@ //! handlers, drained each input cycle, fulfill it against the compositor //! clipboard (reusing the generic clipboard worker pipeline). -use super::color_picker::{ - ClipboardTextError, copy_text_via_command, read_clipboard_text_via_command, -}; use super::{ClipboardOperationController, WaylandState}; use crate::backend::wayland::clipboard::ClipboardPoll; +use crate::clipboard_text::{ + ClipboardTextError, copy_text_via_command, read_clipboard_text_via_command, +}; use crate::input::state::{ TextClipboardRequest, TextPasteEdit, TextPasteTarget, Toast, ToastPriority, }; diff --git a/src/backend/wayland/state/toolbar/events.rs b/src/backend/wayland/state/toolbar/events.rs index 48e8a25a..287fa62b 100644 --- a/src/backend/wayland/state/toolbar/events.rs +++ b/src/backend/wayland/state/toolbar/events.rs @@ -134,6 +134,7 @@ impl WaylandState { // A toolbar interaction replaces the modal sampler. Do this before // shortcut capture so the capture modal owns subsequent keys. self.cancel_eyedropper(); + self.cancel_ocr(); if rebind_requested && let Some(action) = crate::ui::toolbar::model::action_for_event(&event) { diff --git a/src/clipboard_text.rs b/src/clipboard_text.rs new file mode 100644 index 00000000..28c7a74f --- /dev/null +++ b/src/clipboard_text.rs @@ -0,0 +1,99 @@ +//! Generic system text-clipboard commands (`wl-copy` / `wl-paste`). +//! +//! Blocking process-broker calls with no Wayland state of their own, shared by +//! the hex-color picker, the text editor, and screen text recognition. Every +//! caller runs them on a worker thread; none of them may run on the event loop. + +use std::ffi::OsStr; +use std::time::Duration; + +pub(crate) enum ClipboardTextError { + Empty, + Other(String), +} + +pub(crate) fn copy_text_via_command(text: &str) -> Result<(), String> { + let output = crate::process_broker::current() + .and_then(|broker| { + broker.publish( + crate::process_broker::HelperKind::WlCopy, + OsStr::new("wl-copy"), + [OsStr::new("--type"), OsStr::new("text/plain;charset=utf-8")], + text.as_bytes().to_vec(), + Duration::from_secs(5), + ) + }) + .map_err(|error| format!("Failed to run wl-copy: {error:#}"))?; + if output.timed_out { + return Err("wl-copy timed out".to_string()); + } + if output.status != 0 { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + return if stderr.is_empty() { + Err("wl-copy exited unsuccessfully".to_string()) + } else { + Err(format!("wl-copy exited unsuccessfully: {stderr}")) + }; + } + Ok(()) +} + +pub(crate) fn read_clipboard_text_via_command() -> Result { + let output = crate::process_broker::current() + .and_then(|broker| { + broker.run( + crate::process_broker::HelperKind::WlPaste, + OsStr::new("wl-paste"), + clipboard_text_read_args(), + Vec::new(), + Duration::from_secs(5), + 1024 * 1024, + ) + }) + .map_err(|err| ClipboardTextError::Other(format!("Failed to run wl-paste: {err:#}")))?; + + if !output.timed_out && output.status == 0 { + Ok(String::from_utf8_lossy(&output.stdout).to_string()) + } else { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if stderr.to_ascii_lowercase().contains("nothing is copied") + || stderr.to_ascii_lowercase().contains("clipboard is empty") + { + Err(ClipboardTextError::Empty) + } else if stderr.is_empty() { + Err(ClipboardTextError::Other( + "wl-paste exited unsuccessfully".to_string(), + )) + } else { + Err(ClipboardTextError::Other(format!( + "wl-paste exited unsuccessfully: {}", + stderr + ))) + } + } +} + +fn clipboard_text_read_args() -> [&'static OsStr; 3] { + [ + OsStr::new("--no-newline"), + OsStr::new("--type"), + OsStr::new("text"), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clipboard_text_read_requests_only_a_text_mime() { + assert_eq!( + clipboard_text_read_args(), + [ + OsStr::new("--no-newline"), + OsStr::new("--type"), + OsStr::new("text"), + ] + ); + } +} diff --git a/src/config/action_meta/entries/capture.rs b/src/config/action_meta/entries/capture.rs index 0e4c288c..b7dea692 100644 --- a/src/config/action_meta/entries/capture.rs +++ b/src/config/action_meta/entries/capture.rs @@ -168,4 +168,15 @@ pub const ENTRIES: &[ActionMeta] = &[ true, false ), + meta!( + CopyTextFromScreen, + "Copy Text From Screen", + Some("Copy Text"), + "Select a screen region and copy recognized text", + Capture, + true, + true, + true, + &["ocr", "recognize text", "screen text", "copy text"] + ), ]; diff --git a/src/config/action_meta/tests.rs b/src/config/action_meta/tests.rs index bfa85ad4..f7f127b2 100644 --- a/src/config/action_meta/tests.rs +++ b/src/config/action_meta/tests.rs @@ -258,6 +258,7 @@ const EXPECTED_COMMAND_PALETTE_ACTIONS: &[Action] = &[ Action::ExportBoardPdfFile, Action::ExportAllBoardsPdfFile, Action::OpenCaptureFolder, + Action::CopyTextFromScreen, Action::ToggleFrozenMode, Action::ZoomIn, Action::ZoomOut, diff --git a/src/config/keybindings/config/map/capture.rs b/src/config/keybindings/config/map/capture.rs index 58013c85..7df961f6 100644 --- a/src/config/keybindings/config/map/capture.rs +++ b/src/config/keybindings/config/map/capture.rs @@ -49,6 +49,10 @@ impl KeybindingsConfig { Action::ExportAllBoardsPdfFile, )?; inserter.insert_all(&self.capture.open_capture_folder, Action::OpenCaptureFolder)?; + inserter.insert_all( + &self.capture.copy_text_from_screen, + Action::CopyTextFromScreen, + )?; Ok(()) } } diff --git a/src/config/keybindings/config/map/edit.rs b/src/config/keybindings/config/map/edit.rs index a2da29c9..5065d158 100644 --- a/src/config/keybindings/config/map/edit.rs +++ b/src/config/keybindings/config/map/edit.rs @@ -189,6 +189,7 @@ define_action_binding_accessors! { ExportBoardPdfFile => capture.export_board_pdf_file, ExportAllBoardsPdfFile => capture.export_all_boards_pdf_file, OpenCaptureFolder => capture.open_capture_folder, + CopyTextFromScreen => capture.copy_text_from_screen, ToggleFrozenMode => zoom.toggle_frozen_mode, ZoomIn => zoom.zoom_in, ZoomOut => zoom.zoom_out, diff --git a/src/config/keybindings/config/types/bindings/capture.rs b/src/config/keybindings/config/types/bindings/capture.rs index db72ee2c..6390239b 100644 --- a/src/config/keybindings/config/types/bindings/capture.rs +++ b/src/config/keybindings/config/types/bindings/capture.rs @@ -49,6 +49,11 @@ pub struct CaptureKeybindingsConfig { #[serde(default = "default_open_capture_folder")] pub open_capture_folder: Vec, + + /// Screen text recognition. Unbound by default: `O` already selects the + /// orange quick color, so a default here would silently repurpose it. + #[serde(default = "default_copy_text_from_screen")] + pub copy_text_from_screen: Vec, } impl Default for CaptureKeybindingsConfig { @@ -69,6 +74,7 @@ impl Default for CaptureKeybindingsConfig { export_board_pdf_file: default_export_board_pdf_file(), export_all_boards_pdf_file: default_export_all_boards_pdf_file(), open_capture_folder: default_open_capture_folder(), + copy_text_from_screen: default_copy_text_from_screen(), } } } diff --git a/src/config/keybindings/defaults/capture.rs b/src/config/keybindings/defaults/capture.rs index 8d4246f5..b640603e 100644 --- a/src/config/keybindings/defaults/capture.rs +++ b/src/config/keybindings/defaults/capture.rs @@ -57,3 +57,9 @@ pub(crate) fn default_export_all_boards_pdf_file() -> Vec { pub(crate) fn default_open_capture_folder() -> Vec { vec!["Ctrl+Alt+O".to_string()] } + +/// Deliberately empty: `O` is the orange quick color and no other +/// conflict-free chord is obviously right, so the user picks one. +pub(crate) fn default_copy_text_from_screen() -> Vec { + Vec::new() +} diff --git a/src/config/keybindings/tests.rs b/src/config/keybindings/tests.rs index b5aa53b7..cc533c7d 100644 --- a/src/config/keybindings/tests.rs +++ b/src/config/keybindings/tests.rs @@ -360,6 +360,30 @@ fn screen_eyedropper_defaults_to_i_and_maps_when_reconfigured() { ); } +#[test] +fn screen_text_recognition_is_unbound_by_default_and_leaves_o_to_orange() { + let mut config = KeybindingsConfig::default(); + assert!(config.capture.copy_text_from_screen.is_empty()); + + let default_map = config.build_action_map().unwrap(); + assert_eq!( + default_map.get(&KeyBinding::parse("O").unwrap()), + Some(&Action::SetColorOrange) + ); + assert!( + !default_map + .values() + .any(|action| *action == Action::CopyTextFromScreen) + ); + + config.capture.copy_text_from_screen = vec!["Ctrl+Alt+T".to_string()]; + let map = config.build_action_map().unwrap(); + assert_eq!( + map.get(&KeyBinding::parse("Ctrl+Alt+T").unwrap()), + Some(&Action::CopyTextFromScreen) + ); +} + #[test] fn canvas_export_actions_deserialize_from_config_names() { #[derive(serde::Deserialize)] @@ -637,6 +661,9 @@ const DEFAULT_BINDING_SNAPSHOT: &[(&str, &[&str])] = &[ ("export_board_pdf_file", &[]), ("export_all_boards_pdf_file", &[]), ("open_capture_folder", &["Ctrl+Alt+O"]), + // Intentionally unbound: `O` is the orange quick color, and no other + // conflict-free chord is obviously right, so the user picks one. + ("copy_text_from_screen", &[]), ("toggle_frozen_mode", &["Ctrl+Shift+F"]), ("zoom_in", &["Ctrl+Alt++", "Ctrl+Alt+="]), ("zoom_out", &["Ctrl+Alt+-", "Ctrl+Alt+_"]), diff --git a/src/config/mod.rs b/src/config/mod.rs index 76833915..502d8158 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -58,28 +58,28 @@ pub use migration::{MigrationChange, MigrationPreview}; #[allow(unused_imports)] pub use types::{ ArrowConfig, BoardBackgroundConfig, BoardColorConfig, BoardConfig, BoardItemConfig, - BoardsConfig, CaptureConfig, ClickHighlightConfig, DragButtonConfig, DrawingConfig, - ExportConfig, HelpOverlayStyle, HistoryConfig, InputHudConfig, InputHudMode, InputHudPosition, - MouseDragToolsConfig, PDF_LABEL_APP_BOARD, PDF_LABEL_APP_BOARDS, PDF_LABEL_BOARD_NAME, - PDF_LABEL_DEFAULT_TEMPLATE, PDF_LABEL_DOCUMENT_PAGE, PDF_LABEL_DOCUMENT_PAGES, - PDF_LABEL_EXPORT_BOARD, PDF_LABEL_EXPORT_BOARDS, PDF_LABEL_PAGE, PDF_LABEL_PAGE_NAME, - PDF_LABEL_PAGES, PDF_LABEL_PLACEHOLDERS, PRESET_SLOTS_MAX, PRESET_SLOTS_MIN, PdfExportConfig, - PdfFitMode, PdfLabelConfig, PdfLabelContentMode, PdfLabelPosition, PdfOrientation, PdfPageSize, - PdfTransparentBackground, PerformanceConfig, PresenterModeConfig, PresenterToolBehavior, - PresenterToolbarMode, PresetSlotsConfig, PresetToolSettingConfig, PresetToolStatesConfig, - QUICK_COLOR_RENDER_LIMIT, QuickColorConfig, QuickColorPalette, QuickColorPaletteEntry, - QuickColorSlot, QuickColorWrite, QuickColorsConfig, RenderColorMappingConfig, - RenderProfileConfig, RenderProfileExportMode, RenderProfilesConfig, ResolvedToolbarItems, - SessionCompression, SessionConfig, SessionStorageMode, SpotlightConfig, StatusBarItem, - StatusBarStyle, ToolPresetConfig, ToolbarBackendKind, ToolbarConfig, ToolbarGroupId, - ToolbarItemCategory, ToolbarItemDefinition, ToolbarItemId, ToolbarItemOrderConfig, - ToolbarItemOrderGroup, ToolbarItemSurface, ToolbarItemsConfig, ToolbarLayoutMode, - ToolbarModeOverride, ToolbarModeOverrides, ToolbarRebindModifier, ToolbarSectionFlag, - ToolbarSectionVisibility, TopDisplayMode, TrayConfig, TrayIconStyle, UiConfig, UpdatesConfig, - ZoomChipDisplay, default_quick_color_for_index, fold_legacy_section_flags, - resolve_section_visibility, section_flag_for_item, set_section_visibility, - toolbar_item_definitions, toolbar_item_ids, toolbar_item_order_group, - validate_pdf_label_template, + BoardsConfig, CaptureConfig, ClickHighlightConfig, DEFAULT_OCR_LANGUAGES, DragButtonConfig, + DrawingConfig, ExportConfig, HelpOverlayStyle, HistoryConfig, InputHudConfig, InputHudMode, + InputHudPosition, MouseDragToolsConfig, PDF_LABEL_APP_BOARD, PDF_LABEL_APP_BOARDS, + PDF_LABEL_BOARD_NAME, PDF_LABEL_DEFAULT_TEMPLATE, PDF_LABEL_DOCUMENT_PAGE, + PDF_LABEL_DOCUMENT_PAGES, PDF_LABEL_EXPORT_BOARD, PDF_LABEL_EXPORT_BOARDS, PDF_LABEL_PAGE, + PDF_LABEL_PAGE_NAME, PDF_LABEL_PAGES, PDF_LABEL_PLACEHOLDERS, PRESET_SLOTS_MAX, + PRESET_SLOTS_MIN, PdfExportConfig, PdfFitMode, PdfLabelConfig, PdfLabelContentMode, + PdfLabelPosition, PdfOrientation, PdfPageSize, PdfTransparentBackground, PerformanceConfig, + PresenterModeConfig, PresenterToolBehavior, PresenterToolbarMode, PresetSlotsConfig, + PresetToolSettingConfig, PresetToolStatesConfig, QUICK_COLOR_RENDER_LIMIT, QuickColorConfig, + QuickColorPalette, QuickColorPaletteEntry, QuickColorSlot, QuickColorWrite, QuickColorsConfig, + RenderColorMappingConfig, RenderProfileConfig, RenderProfileExportMode, RenderProfilesConfig, + ResolvedToolbarItems, SessionCompression, SessionConfig, SessionStorageMode, SpotlightConfig, + StatusBarItem, StatusBarStyle, ToolPresetConfig, ToolbarBackendKind, ToolbarConfig, + ToolbarGroupId, ToolbarItemCategory, ToolbarItemDefinition, ToolbarItemId, + ToolbarItemOrderConfig, ToolbarItemOrderGroup, ToolbarItemSurface, ToolbarItemsConfig, + ToolbarLayoutMode, ToolbarModeOverride, ToolbarModeOverrides, ToolbarRebindModifier, + ToolbarSectionFlag, ToolbarSectionVisibility, TopDisplayMode, TrayConfig, TrayIconStyle, + UiConfig, UpdatesConfig, ZoomChipDisplay, default_quick_color_for_index, + fold_legacy_section_flags, resolve_section_visibility, section_flag_for_item, + set_section_visibility, toolbar_item_definitions, toolbar_item_ids, toolbar_item_order_group, + validate_ocr_languages, validate_pdf_label_template, }; #[cfg(feature = "tablet-input")] #[allow(unused_imports)] diff --git a/src/config/types/capture.rs b/src/config/types/capture.rs index a052d52b..f6cf910e 100644 --- a/src/config/types/capture.rs +++ b/src/config/types/capture.rs @@ -31,6 +31,12 @@ pub struct CaptureConfig { /// When false, clipboard-only captures still auto-exit by default. #[serde(default = "default_capture_exit_after")] pub exit_after_capture: bool, + + /// Tesseract languages used by `Copy text from screen`, in Tesseract's + /// plus-separated form (for example `eng` or `eng+deu`). The matching + /// Tesseract language packages must be installed. + #[serde(default = "default_capture_ocr_languages")] + pub ocr_languages: String, } impl Default for CaptureConfig { @@ -42,8 +48,65 @@ impl Default for CaptureConfig { format: default_capture_format(), copy_to_clipboard: default_capture_clipboard(), exit_after_capture: default_capture_exit_after(), + ocr_languages: default_capture_ocr_languages(), + } + } +} + +impl CaptureConfig { + /// The configured OCR languages, or the default when the authored value is + /// unusable. Recognition never passes an unvalidated string to the engine. + pub fn resolved_ocr_languages(&self) -> String { + match validate_ocr_languages(&self.ocr_languages) { + Ok(languages) => languages, + Err(reason) => { + log::warn!( + "Ignoring capture.ocr_languages {:?}: {reason}; using {}", + self.ocr_languages, + DEFAULT_OCR_LANGUAGES + ); + DEFAULT_OCR_LANGUAGES.to_string() + } + } + } +} + +pub const DEFAULT_OCR_LANGUAGES: &str = "eng"; + +/// Normalize a Tesseract language argument, rejecting anything that could +/// change how the engine is invoked. +/// +/// Accepts the plus-separated form Tesseract itself uses (`eng`, `eng+deu`). +/// Tokens are restricted to the character set real language codes use, so a +/// value can never carry a path, an option, or shell syntax into the argument +/// vector. +pub fn validate_ocr_languages(raw: &str) -> Result { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return Err("value is empty".to_string()); + } + let tokens: Vec<&str> = trimmed.split('+').collect(); + for token in &tokens { + if token.is_empty() { + return Err("value has an empty language between '+' separators".to_string()); + } + if token.starts_with('-') { + return Err(format!("language {token:?} looks like a command option")); + } + if !token + .chars() + .all(|character| character.is_ascii_alphanumeric() || matches!(character, '_' | '-')) + { + return Err(format!( + "language {token:?} may only contain letters, digits, '_' and '-'" + )); } } + Ok(tokens.join("+")) +} + +fn default_capture_ocr_languages() -> String { + DEFAULT_OCR_LANGUAGES.to_string() } fn default_capture_enabled() -> bool { @@ -69,3 +132,53 @@ fn default_capture_clipboard() -> bool { fn default_capture_exit_after() -> bool { false } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_ocr_language_is_english() { + assert_eq!(CaptureConfig::default().ocr_languages, "eng"); + assert_eq!(CaptureConfig::default().resolved_ocr_languages(), "eng"); + } + + #[test] + fn plus_separated_languages_round_trip_after_trimming() { + assert_eq!(validate_ocr_languages(" eng+deu \n").unwrap(), "eng+deu"); + assert_eq!(validate_ocr_languages("chi_sim").unwrap(), "chi_sim"); + assert_eq!(validate_ocr_languages("eng").unwrap(), "eng"); + } + + #[test] + fn unsafe_or_empty_values_are_rejected() { + for value in [ + "", + " ", + "eng+", + "+eng", + "eng deu", + "../tessdata/eng", + "/usr/share/eng", + "eng;rm -rf /", + "eng$(id)", + "--psm", + "eng\0", + ] { + assert!( + validate_ocr_languages(value).is_err(), + "{value:?} must be rejected" + ); + } + } + + #[test] + fn an_invalid_authored_value_falls_back_to_the_default() { + let config = CaptureConfig { + ocr_languages: "../evil".to_string(), + ..CaptureConfig::default() + }; + + assert_eq!(config.resolved_ocr_languages(), DEFAULT_OCR_LANGUAGES); + } +} diff --git a/src/config/types/mod.rs b/src/config/types/mod.rs index 9d823a42..4ed016df 100644 --- a/src/config/types/mod.rs +++ b/src/config/types/mod.rs @@ -28,7 +28,7 @@ mod updates; pub use arrow::ArrowConfig; pub use board::BoardConfig; pub use boards::{BoardBackgroundConfig, BoardColorConfig, BoardItemConfig, BoardsConfig}; -pub use capture::CaptureConfig; +pub use capture::{CaptureConfig, DEFAULT_OCR_LANGUAGES, validate_ocr_languages}; pub use click_highlight::ClickHighlightConfig; pub use context_menu::ContextMenuUiConfig; pub use drawing::{ diff --git a/src/config/types/toolbar/ids.rs b/src/config/types/toolbar/ids.rs index c3128336..a330e0d5 100644 --- a/src/config/types/toolbar/ids.rs +++ b/src/config/types/toolbar/ids.rs @@ -36,6 +36,7 @@ pub const TOP_UTILITY_CLEAR_CANVAS: ToolbarItemId = ToolbarItemId::from_known("top.utility.clear-canvas"); pub const TOP_UTILITY_SCREENSHOT: ToolbarItemId = ToolbarItemId::from_known("top.utility.screenshot"); +pub const TOP_UTILITY_OCR: ToolbarItemId = ToolbarItemId::from_known("top.utility.ocr"); pub const TOP_UTILITY_HIGHLIGHT: ToolbarItemId = ToolbarItemId::from_known("top.utility.highlight"); pub const TOP_UTILITY_HIGHLIGHT_RING: ToolbarItemId = ToolbarItemId::from_known("top.utility.highlight-ring"); diff --git a/src/config/types/toolbar/items.rs b/src/config/types/toolbar/items.rs index 66f969e7..afbc28de 100644 --- a/src/config/types/toolbar/items.rs +++ b/src/config/types/toolbar/items.rs @@ -52,7 +52,8 @@ fn drain_without(raw_ids: Vec, id: ToolbarItemId) -> Vec { next } -const DEFAULT_HIDDEN_TOOLBAR_ITEM_IDS: &[ToolbarItemId] = &[ids::TOP_UTILITY_SCREENSHOT]; +const DEFAULT_HIDDEN_TOOLBAR_ITEM_IDS: &[ToolbarItemId] = + &[ids::TOP_UTILITY_SCREENSHOT, ids::TOP_UTILITY_OCR]; const DEFAULT_TOP_TOOLS_ORDER: &[ToolbarItemId] = &[ ids::TOP_TOOL_SELECT, @@ -77,6 +78,7 @@ const DEFAULT_TOP_CONTROLS_ORDER: &[ToolbarItemId] = &[ ids::TOP_UTILITY_TEXT, ids::TOP_UTILITY_STICKY_NOTE, ids::TOP_UTILITY_SCREENSHOT, + ids::TOP_UTILITY_OCR, ids::TOP_UTILITY_CLEAR_CANVAS, ids::TOP_UTILITY_HIGHLIGHT, ]; @@ -147,6 +149,13 @@ impl ToolbarItemsConfig { id: ToolbarItemId, setting: ToolbarItemVisibilitySetting, ) -> bool { + // Re-applying the setting an item already has must be a no-op. The + // rewrite below moves the id to the end of its list, so without this + // guard a second identical call would report a change forever — which + // is what a batch reset over several items does. + if item_visibility_setting(&self.resolved(), id) == setting { + return false; + } let before_hidden = self.hidden.clone(); let before_shown = self.shown.clone(); self.hidden = drain_without(std::mem::take(&mut self.hidden), id); diff --git a/src/config/types/toolbar/items/definitions.rs b/src/config/types/toolbar/items/definitions.rs index 73336a31..3191b8d0 100644 --- a/src/config/types/toolbar/items/definitions.rs +++ b/src/config/types/toolbar/items/definitions.rs @@ -80,6 +80,7 @@ const TOOLBAR_ITEM_DEFINITIONS: &[ToolbarItemDefinition] = &[ Utility, None, ), + item(ids::TOP_UTILITY_OCR, "Copy text", Top, Utility, None), item(ids::TOP_UTILITY_HIGHLIGHT, "Highlight", Top, Utility, None), item( ids::TOP_UTILITY_HIGHLIGHT_RING, diff --git a/src/config/types/toolbar/items/tests.rs b/src/config/types/toolbar/items/tests.rs index 144b8ac5..433a28a1 100644 --- a/src/config/types/toolbar/items/tests.rs +++ b/src/config/types/toolbar/items/tests.rs @@ -28,6 +28,49 @@ fn default_hidden_items_hide_screenshot_tool() { ); } +#[test] +fn the_ocr_utility_is_defined_once_ordered_and_hidden_by_default() { + let definitions: Vec<_> = toolbar_item_definitions() + .iter() + .filter(|definition| definition.id == ids::TOP_UTILITY_OCR) + .collect(); + assert_eq!(definitions.len(), 1); + assert_eq!(definitions[0].label, "Copy text"); + + let order = ToolbarItemOrderConfig::default().resolved(); + let top_controls = order.ordered_ids(ToolbarItemOrderGroup::TopControls); + assert_eq!( + top_controls + .iter() + .filter(|id| **id == ids::TOP_UTILITY_OCR) + .count(), + 1 + ); + + let resolved = ToolbarItemsConfig::default().resolved(); + assert!(resolved.is_hidden(ids::TOP_UTILITY_OCR)); + assert_eq!( + item_visibility_setting(&resolved, ids::TOP_UTILITY_OCR), + ToolbarItemVisibilitySetting::Hidden + ); +} + +#[test] +fn showing_the_ocr_utility_is_an_explicit_customization_that_survives_reads() { + let mut config = ToolbarItemsConfig::default(); + + assert!( + config.set_visibility_setting(ids::TOP_UTILITY_OCR, ToolbarItemVisibilitySetting::Shown) + ); + assert!(!config.resolved().is_hidden(ids::TOP_UTILITY_OCR)); + // Re-applying the same setting is not a change; only a different one is. + assert!( + !config.set_visibility_setting(ids::TOP_UTILITY_OCR, ToolbarItemVisibilitySetting::Shown) + ); + assert!(config.reset_known_hidden_to_defaults()); + assert!(config.resolved().is_hidden(ids::TOP_UTILITY_OCR)); +} + #[test] fn visibility_setting_is_hidden_first_and_known_mutation_canonicalizes_conflicts() { let mut config = ToolbarItemsConfig { @@ -110,6 +153,7 @@ fn reset_known_hidden_restores_defaults_and_preserves_unknown_ids() { config.hidden, vec![ ids::TOP_UTILITY_SCREENSHOT.as_str().to_string(), + ids::TOP_UTILITY_OCR.as_str().to_string(), "future.toolbar.item".to_string() ] ); diff --git a/src/configurator_destination.rs b/src/configurator_destination.rs index 301149e9..cfc517c8 100644 --- a/src/configurator_destination.rs +++ b/src/configurator_destination.rs @@ -200,6 +200,7 @@ pub fn keybindings_section_for_action(action: Action) -> Option { + // The backend owns capture ownership and the region selector, + // so this only records the intent. It selects no tool and + // touches no drawing state. + log::debug!("Copy text from screen requested"); + self.request_copy_text_from_screen(); + true + } Action::ToggleFrozenMode => { log::info!("Toggle frozen mode requested"); self.request_frozen_toggle(); diff --git a/src/input/state/core/base/state/init.rs b/src/input/state/core/base/state/init.rs index a8acc45a..e93267e4 100644 --- a/src/input/state/core/base/state/init.rs +++ b/src/input/state/core/base/state/init.rs @@ -292,6 +292,8 @@ impl InputState { pending_frozen_toggle: false, eyedropper_ui_state: crate::input::state::core::EyedropperUiState::Inactive, pending_eyedropper_toggle: false, + ocr_ui_state: crate::input::state::core::OcrUiState::Inactive, + pending_ocr_request: false, zoom_active: false, zoom_locked: false, zoom_scale: 1.0, diff --git a/src/input/state/core/base/state/structs.rs b/src/input/state/core/base/state/structs.rs index 387a49ed..e469e521 100644 --- a/src/input/state/core/base/state/structs.rs +++ b/src/input/state/core/base/state/structs.rs @@ -586,6 +586,10 @@ pub struct InputState { crate::input::state::core::EyedropperUiState, /// Pending eyedropper activation request for the Wayland backend. pub(in crate::input::state::core) pending_eyedropper_toggle: bool, + /// Screen text recognition (OCR) region-selector lifecycle. + pub(in crate::input::state::core) ocr_ui_state: crate::input::state::core::OcrUiState, + /// Pending `Copy text from screen` request for the Wayland backend. + pub(in crate::input::state::core) pending_ocr_request: bool, /// Whether zoom mode is currently active pub(in crate::input::state::core) zoom_active: bool, /// Whether zoom view is locked diff --git a/src/input/state/core/eyedropper.rs b/src/input/state/core/eyedropper.rs index 087af856..788f74e5 100644 --- a/src/input/state/core/eyedropper.rs +++ b/src/input/state/core/eyedropper.rs @@ -86,7 +86,10 @@ impl InputState { .flatten() } - pub(crate) fn prepare_for_eyedropper(&mut self) { + /// Close everything a screen-region modal must not compete with, and + /// cancel any unfinished gesture. Shared by the eyedropper and OCR: both + /// take over pointer input entirely while they are up. + pub(crate) fn prepare_for_screen_modal(&mut self) { self.cancel_active_interaction(); if self.show_help { self.toggle_help_overlay(); @@ -117,7 +120,7 @@ impl InputState { // A capture can take long enough for another interaction to begin while // the eyedropper is pending. Entering the modal state must cancel it so // the eyedropper cannot swallow the matching release event. - self.prepare_for_eyedropper(); + self.prepare_for_screen_modal(); self.eyedropper_ui_state = EyedropperUiState::Active { hover: None, auto_froze, @@ -226,6 +229,23 @@ mod tests { assert!(state.eyedropper_is_active()); } + /// The eyedropper swallows every key press it receives, so a key held from + /// before it opened must not keep repeating into the canvas behind it. + #[test] + fn an_engaged_eyedropper_stops_canvas_key_repeat() { + let mut state = make_test_input_state(); + assert!(!state.modal_blocks_canvas_key_repeat()); + + state.set_eyedropper_pending_capture(EyedropperCaptureSource::Frozen); + assert!(state.modal_blocks_canvas_key_repeat()); + + state.activate_eyedropper(true); + assert!(state.modal_blocks_canvas_key_repeat()); + + state.cancel_eyedropper(); + assert!(!state.modal_blocks_canvas_key_repeat()); + } + #[test] fn default_i_key_resolves_to_screen_eyedropper() { let state = make_test_input_state(); diff --git a/src/input/state/core/mod.rs b/src/input/state/core/mod.rs index 3e2d80d7..e59d2d57 100644 --- a/src/input/state/core/mod.rs +++ b/src/input/state/core/mod.rs @@ -12,6 +12,7 @@ mod index; mod input_hud_controls; mod menus; pub(crate) mod modal; +mod ocr; mod properties; pub(crate) mod radial_menu; mod selection; @@ -71,6 +72,7 @@ pub use ime::{ImeCompositionState, ImePreedit}; pub use menus::{ ContextMenuCursorHint, ContextMenuEntry, ContextMenuKind, ContextMenuState, MenuCommand, }; +pub use ocr::{OcrCaptureSource, OcrInputSource, OcrSelection, OcrUiState}; pub use properties::{SelectionPropertyEntry, SelectionPropertyKind}; pub use radial_menu::{ COMPASS_SLICES as RADIAL_COMPASS_SLICES, CompassDir, RADIAL_PAINT_DELAY, RadialMenuLayout, diff --git a/src/input/state/core/modal.rs b/src/input/state/core/modal.rs index 7e541fde..ecae28a8 100644 --- a/src/input/state/core/modal.rs +++ b/src/input/state/core/modal.rs @@ -135,18 +135,49 @@ impl InputState { pub fn modal_owns_text_input(&self) -> bool { self.engaged_modal().is_some() // The keybinding capture chord can be armed after the palette - // closes, and the eyedropper is not a popup surface; both still - // own the keyboard. + // closes, and the screen-region modals are not popup surfaces; + // all of them still own the keyboard. || self.command_palette_is_engaged() - || self.eyedropper_is_engaged() + || self.screen_modal_is_engaged() } /// Modal paths whose editing/repeat behavior must not be driven by the /// backend's canvas-oriented manual repeat timer. pub fn modal_blocks_canvas_key_repeat(&self) -> bool { self.command_palette_is_engaged() + // A screen-region modal can open from the toolbar, the command + // palette, or a mouse path while a canvas key is still held. It + // swallows every key press it receives, so leaving the repeat timer + // armed would keep feeding the canvas behind the selector. + || self.screen_modal_is_engaged() || ModalSurface::ALL .into_iter() .any(|surface| surface.blocks_canvas_key_repeat() && self.modal_is_open(surface)) } + + /// Whether either screen-region modal — the eyedropper or the OCR region + /// selector — has been asked for, including while it still waits on a + /// capture. + /// + /// This is the *keyboard* boundary: the key that requested the modal is the + /// last one the canvas sees, and every press from then on is swallowed, so + /// key repeat, IME ownership, and stylus barrel actions all stop here. + /// + /// It is deliberately not the pen boundary — see + /// [`Self::screen_modal_is_active`]. + pub(crate) fn screen_modal_is_engaged(&self) -> bool { + self.eyedropper_is_engaged() || self.ocr_is_engaged() + } + + /// Whether a screen-region modal is on screen and owns pointer input. + /// + /// The *pen* boundary, and narrower than [`Self::screen_modal_is_engaged`]: + /// while a modal is still waiting on its capture there is nothing drawn + /// over the canvas, and every pointer, touch, and stylus path still routes + /// to drawing — so a stroke started during that wait is a real stroke and + /// keeps its pressure. Activation cancels it; a capture that fails or is + /// cancelled first leaves it intact. + pub(crate) fn screen_modal_is_active(&self) -> bool { + self.eyedropper_is_active() || self.ocr_is_active() + } } diff --git a/src/input/state/core/ocr.rs b/src/input/state/core/ocr.rs new file mode 100644 index 00000000..84dd3b20 --- /dev/null +++ b/src/input/state/core/ocr.rs @@ -0,0 +1,498 @@ +use super::InputState; +use crate::input::state::{Toast, ToastPriority}; + +/// Which capture the OCR selector is waiting on before it can arm. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OcrCaptureSource { + Frozen, + Zoom, +} + +/// Which physical input started a region drag. +/// +/// A region is one gesture by one device. Wayscriber is modal but the seat is +/// not: a pen can hover or lift while the region is being dragged with the +/// mouse, and without an owner those stray events would move — or submit — +/// somebody else's region. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OcrInputSource { + Pointer, + Touch, + Stylus, +} + +/// A finished or in-progress region drag in logical screen coordinates. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct OcrSelection { + pub start: (f64, f64), + pub end: (f64, f64), + /// OCR created the freeze it selected against, so OCR must release it. + pub auto_froze: bool, +} + +/// UI-facing lifecycle for the modal `Copy text from screen` region selector. +/// +/// It owns transient input and render state only. The selected pixels and the +/// recognition work belong outside `InputState`, so cancelling here can never +/// discard a request that is already running. +#[derive(Debug, Default, Clone, Copy, PartialEq)] +pub enum OcrUiState { + #[default] + Inactive, + PendingCapture { + source: OcrCaptureSource, + auto_froze: bool, + }, + /// The selector is live and waiting for the press that starts a region. + Armed { auto_froze: bool }, + /// A region drag is in progress between `start` and `current`, owned by the + /// device that pressed. Only that device may move, release, or cancel it. + Selecting { + owner: OcrInputSource, + start: (f64, f64), + current: (f64, f64), + auto_froze: bool, + }, +} + +impl OcrUiState { + pub fn is_active(self) -> bool { + matches!(self, Self::Armed { .. } | Self::Selecting { .. }) + } + + pub fn is_pending(self) -> bool { + matches!(self, Self::PendingCapture { .. }) + } + + pub fn is_engaged(self) -> bool { + self.is_active() || self.is_pending() + } + + pub fn is_selecting(self) -> bool { + matches!(self, Self::Selecting { .. }) + } + + /// The region drag in logical screen coordinates, if one is in progress. + pub fn selection(self) -> Option { + match self { + Self::Selecting { + start, + current, + auto_froze, + .. + } => Some(OcrSelection { + start, + end: current, + auto_froze, + }), + Self::Inactive | Self::PendingCapture { .. } | Self::Armed { .. } => None, + } + } + + /// The device dragging the region, if one is in progress. + pub fn selection_owner(self) -> Option { + match self { + Self::Selecting { owner, .. } => Some(owner), + Self::Inactive | Self::PendingCapture { .. } | Self::Armed { .. } => None, + } + } + + pub fn pending_source(self) -> Option { + match self { + Self::PendingCapture { source, .. } => Some(source), + Self::Inactive | Self::Armed { .. } | Self::Selecting { .. } => None, + } + } + + fn auto_froze(self) -> bool { + match self { + Self::PendingCapture { auto_froze, .. } + | Self::Armed { auto_froze } + | Self::Selecting { auto_froze, .. } => auto_froze, + Self::Inactive => false, + } + } +} + +impl InputState { + pub(crate) fn request_copy_text_from_screen(&mut self) { + self.pending_ocr_request = true; + } + + pub(crate) fn take_pending_ocr_request(&mut self) -> bool { + std::mem::take(&mut self.pending_ocr_request) + } + + pub fn ocr_state(&self) -> OcrUiState { + self.ocr_ui_state + } + + pub fn ocr_is_active(&self) -> bool { + self.ocr_ui_state.is_active() + } + + pub fn ocr_is_engaged(&self) -> bool { + self.ocr_ui_state.is_engaged() + } + + pub(crate) fn set_ocr_pending_capture(&mut self, source: OcrCaptureSource) { + self.ocr_ui_state = OcrUiState::PendingCapture { + source, + auto_froze: matches!(source, OcrCaptureSource::Frozen), + }; + self.mark_ocr_dirty(); + } + + pub(crate) fn activate_ocr(&mut self, auto_froze: bool) { + // A capture can take long enough for another interaction to begin while + // OCR is pending. Entering the modal state must cancel it so the + // selector cannot swallow the matching release event. + self.prepare_for_screen_modal(); + self.ocr_ui_state = OcrUiState::Armed { auto_froze }; + self.mark_ocr_dirty(); + } + + /// Begin a region drag owned by `owner`. + /// + /// Ignored unless the selector is armed, so a stray press during capture + /// cannot start a region against a stale image — and, because a drag in + /// progress is no longer armed, a second device cannot take one over. + pub(crate) fn start_ocr_selection(&mut self, owner: OcrInputSource, point: (f64, f64)) -> bool { + let OcrUiState::Armed { auto_froze } = self.ocr_ui_state else { + return false; + }; + self.ocr_ui_state = OcrUiState::Selecting { + owner, + start: point, + current: point, + auto_froze, + }; + self.mark_ocr_dirty(); + true + } + + /// Move the region, if `source` is the device dragging it. Motion from any + /// other device is somebody else's and is dropped. + pub(crate) fn update_ocr_selection(&mut self, source: OcrInputSource, point: (f64, f64)) { + if let OcrUiState::Selecting { owner, current, .. } = &mut self.ocr_ui_state + && *owner == source + && *current != point + { + *current = point; + self.mark_ocr_dirty(); + } + } + + /// Whether `source` is the device dragging the region right now. The + /// release and per-device cancellation paths check this before acting, so + /// one device cannot submit or discard another's region. + pub(crate) fn ocr_selection_is_owned_by(&self, source: OcrInputSource) -> bool { + self.ocr_ui_state.selection_owner() == Some(source) + } + + /// Abandon a drag that was too small to be a region and wait for the next + /// one. A mis-click should not drop the user out of the selector, and it + /// must not release a capture the selector is still using. + pub(crate) fn rearm_ocr_selection(&mut self) { + if let OcrUiState::Selecting { auto_froze, .. } = self.ocr_ui_state { + self.ocr_ui_state = OcrUiState::Armed { auto_froze }; + self.mark_ocr_dirty(); + } + } + + /// Leave OCR selection and report whether it created frozen mode itself. + pub(crate) fn cancel_ocr(&mut self) -> bool { + let auto_froze = self.ocr_ui_state.auto_froze(); + if !matches!(self.ocr_ui_state, OcrUiState::Inactive) { + self.ocr_ui_state = OcrUiState::Inactive; + self.mark_ocr_dirty(); + } + auto_froze + } + + pub(crate) fn report_ocr_capture_failure_if_unreported(&mut self) { + if self.ui_toast.is_none() { + self.push_toast( + ToastPriority::Critical, + "ocr", + Toast::error("Screen capture for text recognition failed."), + ); + } + } + + /// The selector paints a full-surface scrim with the selected region cut + /// out of it, so every change repaints the whole surface: an incremental + /// buffer would otherwise keep the scrim from an earlier frame. + fn mark_ocr_dirty(&mut self) { + self.dirty_tracker.mark_full(); + self.needs_redraw = true; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::input::state::test_support::make_test_input_state; + use crate::input::{DrawingState, MouseButton, Tool}; + + #[test] + fn cancel_reports_auto_frozen_ownership() { + let mut state = make_test_input_state(); + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + state.activate_ocr(true); + + assert!(state.cancel_ocr()); + assert_eq!(state.ocr_state(), OcrUiState::Inactive); + } + + #[test] + fn waiting_for_zoom_does_not_claim_frozen_mode() { + let mut state = make_test_input_state(); + state.set_ocr_pending_capture(OcrCaptureSource::Zoom); + + assert!(!state.cancel_ocr()); + assert_eq!(state.ocr_state(), OcrUiState::Inactive); + } + + #[test] + fn a_drag_records_both_corners_and_the_capture_it_owns() { + let mut state = make_test_input_state(); + state.activate_ocr(true); + + assert!(state.start_ocr_selection(OcrInputSource::Pointer, (10.0, 20.0))); + state.update_ocr_selection(OcrInputSource::Pointer, (60.0, 90.0)); + assert!(state.ocr_state().is_selecting()); + + assert_eq!( + state.ocr_state().selection(), + Some(OcrSelection { + start: (10.0, 20.0), + end: (60.0, 90.0), + auto_froze: true, + }) + ); + } + + /// A pen already drawing when the selector opens leaves it `Armed`, never + /// `Selecting`: the tip-up that follows has no region to submit and is + /// swallowed. That is exactly why the backend retires the stylus contact + /// when the modal opens instead of waiting for an `Up` it will consume. + #[test] + fn opening_the_selector_mid_stroke_cancels_the_stroke_and_arms_without_a_region() { + let mut state = make_test_input_state(); + state.on_mouse_press(MouseButton::Left, 10, 20); + state.on_mouse_motion(30, 40); + assert!(matches!(state.state, DrawingState::Drawing { .. })); + + state.activate_ocr(true); + + assert!(matches!(state.state, DrawingState::Idle)); + assert_eq!(state.ocr_state(), OcrUiState::Armed { auto_froze: true }); + assert!(state.ocr_state().selection().is_none()); + } + + #[test] + fn a_press_before_the_capture_lands_does_not_start_a_region() { + let mut state = make_test_input_state(); + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + + assert!(!state.start_ocr_selection(OcrInputSource::Pointer, (10.0, 20.0))); + assert!(state.ocr_state().selection().is_none()); + } + + #[test] + fn cancelling_mid_drag_yields_no_region() { + let mut state = make_test_input_state(); + state.activate_ocr(true); + state.start_ocr_selection(OcrInputSource::Pointer, (10.0, 20.0)); + + assert!(state.cancel_ocr()); + assert_eq!(state.ocr_state().selection(), None); + // The state is gone, so a second cancel cannot release the freeze twice. + assert!(!state.cancel_ocr()); + } + + /// The seat is not modal even though Wayscriber is: a pen can hover, and a + /// contact retired when the selector opened keeps reporting until it lifts. + /// Neither may steer a region the mouse is dragging. + #[test] + fn only_the_device_that_started_a_region_can_move_or_finish_it() { + let mut state = make_test_input_state(); + state.activate_ocr(true); + assert!(state.start_ocr_selection(OcrInputSource::Pointer, (10.0, 20.0))); + + state.update_ocr_selection(OcrInputSource::Stylus, (500.0, 500.0)); + state.update_ocr_selection(OcrInputSource::Touch, (400.0, 400.0)); + assert_eq!( + state.ocr_state().selection().map(|region| region.end), + Some((10.0, 20.0)), + "another device dragged the pointer's region" + ); + + state.update_ocr_selection(OcrInputSource::Pointer, (60.0, 90.0)); + assert_eq!( + state.ocr_state().selection().map(|region| region.end), + Some((60.0, 90.0)) + ); + + assert!(!state.ocr_selection_is_owned_by(OcrInputSource::Stylus)); + assert!(!state.ocr_selection_is_owned_by(OcrInputSource::Touch)); + assert!(state.ocr_selection_is_owned_by(OcrInputSource::Pointer)); + } + + /// A second device pressing mid-drag must not take the region over: the + /// selector is no longer armed, so its press finds nothing to start. + #[test] + fn a_second_device_cannot_take_over_a_region_in_progress() { + let mut state = make_test_input_state(); + state.activate_ocr(true); + assert!(state.start_ocr_selection(OcrInputSource::Touch, (10.0, 20.0))); + + assert!(!state.start_ocr_selection(OcrInputSource::Pointer, (200.0, 200.0))); + + assert!(state.ocr_selection_is_owned_by(OcrInputSource::Touch)); + assert_eq!( + state.ocr_state().selection().map(|region| region.start), + Some((10.0, 20.0)) + ); + } + + /// Ownership is per drag, not per session: after one device's region ends, + /// the re-armed selector accepts the next press from any device. + #[test] + fn ownership_resets_with_each_region() { + let mut state = make_test_input_state(); + state.activate_ocr(true); + state.start_ocr_selection(OcrInputSource::Stylus, (10.0, 20.0)); + state.rearm_ocr_selection(); + + assert!(state.ocr_state().selection_owner().is_none()); + assert!(state.start_ocr_selection(OcrInputSource::Pointer, (30.0, 40.0))); + assert!(state.ocr_selection_is_owned_by(OcrInputSource::Pointer)); + } + + #[test] + fn a_mis_click_rearms_the_selector_instead_of_leaving_it() { + let mut state = make_test_input_state(); + state.activate_ocr(true); + state.start_ocr_selection(OcrInputSource::Pointer, (10.0, 20.0)); + + state.rearm_ocr_selection(); + + assert_eq!(state.ocr_state(), OcrUiState::Armed { auto_froze: true }); + assert!(state.ocr_is_active()); + } + + #[test] + fn activation_cancels_an_interaction_started_while_capture_was_pending() { + let mut state = make_test_input_state(); + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + state.on_mouse_press(MouseButton::Left, 10, 20); + assert!(matches!(state.state, DrawingState::Drawing { .. })); + + state.activate_ocr(true); + + assert!(matches!(state.state, DrawingState::Idle)); + assert!(state.active_drag_button.is_none()); + assert!(state.ocr_is_active()); + // The gesture cancelled here started during the wait, not at the + // request — and the selector arms with no region, so the release that + // ends it is swallowed. Activation is therefore the only point that can + // retire the backend contact behind it. + assert_eq!(state.ocr_state(), OcrUiState::Armed { auto_froze: true }); + assert!(state.ocr_state().selection().is_none()); + } + + /// OCR is an action, not a tool: whatever the user was drawing with before + /// must still be selected after the selector opens, drags, and closes. + #[test] + fn the_whole_selector_lifecycle_preserves_every_representative_tool() { + for tool in [ + Tool::Pen, + Tool::Select, + Tool::Marker, + Tool::Eraser, + Tool::Arrow, + Tool::Highlight, + Tool::Blur, + ] { + let mut state = make_test_input_state(); + state.set_tool_override(Some(tool)); + + state.request_copy_text_from_screen(); + assert!(state.take_pending_ocr_request()); + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + state.activate_ocr(true); + state.start_ocr_selection(OcrInputSource::Pointer, (10.0, 10.0)); + state.update_ocr_selection(OcrInputSource::Pointer, (80.0, 60.0)); + state.cancel_ocr(); + + assert_eq!( + state.active_tool(), + tool, + "OCR changed the tool away from {tool:?}" + ); + } + } + + /// Nothing about a recognition is a canvas edit. One undo after the whole + /// lifecycle must remove the stroke drawn before it — if OCR had pushed a + /// history entry, that undo would have been spent on OCR instead. + #[test] + fn the_selector_lifecycle_adds_no_shape_and_no_history_entry() { + let mut state = make_test_input_state(); + state.on_mouse_press(MouseButton::Left, 0, 0); + state.on_mouse_motion(10, 10); + state.on_mouse_release(MouseButton::Left, 10, 10); + assert_eq!(state.boards.active_frame().shapes.len(), 1); + + state.activate_ocr(true); + state.start_ocr_selection(OcrInputSource::Pointer, (10.0, 10.0)); + state.update_ocr_selection(OcrInputSource::Pointer, (120.0, 90.0)); + state.cancel_ocr(); + assert_eq!(state.boards.active_frame().shapes.len(), 1); + + state.handle_action(crate::domain::Action::Undo); + + assert_eq!(state.boards.active_frame().shapes.len(), 0); + } + + /// The selector swallows every key press it receives. A key still held + /// from before it opened — from the toolbar, the palette, or a mouse path — + /// must not keep repeating into the canvas behind it. + #[test] + fn an_engaged_selector_owns_the_keyboard_and_stops_canvas_key_repeat() { + let mut state = make_test_input_state(); + assert!(!state.modal_blocks_canvas_key_repeat()); + assert!(!state.modal_owns_text_input()); + + state.set_ocr_pending_capture(OcrCaptureSource::Frozen); + assert!(state.modal_blocks_canvas_key_repeat()); + assert!(state.modal_owns_text_input()); + + state.activate_ocr(true); + assert!(state.modal_blocks_canvas_key_repeat()); + assert!(state.modal_owns_text_input()); + + state.cancel_ocr(); + assert!(!state.modal_blocks_canvas_key_repeat()); + assert!(!state.modal_owns_text_input()); + } + + #[test] + fn capture_failure_preserves_a_more_specific_existing_error() { + let mut state = make_test_input_state(); + state.push_toast( + ToastPriority::Critical, + "ocr", + Toast::error("Freeze failed after the display changed size"), + ); + + state.report_ocr_capture_failure_if_unreported(); + + assert_eq!( + state.ui_toast.as_ref().map(|toast| toast.message.as_str()), + Some("Freeze failed after the display changed size") + ); + } +} diff --git a/src/input/state/core/tool_controls/toolbar.rs b/src/input/state/core/tool_controls/toolbar.rs index 10723639..1c240a2d 100644 --- a/src/input/state/core/tool_controls/toolbar.rs +++ b/src/input/state/core/tool_controls/toolbar.rs @@ -574,6 +574,7 @@ mod tests { state .toolbar_items .set_hidden(ids::TOP_UTILITY_SCREENSHOT, false); + state.toolbar_items.set_hidden(ids::TOP_UTILITY_OCR, false); state.toolbar_items.set_hidden(ids::TOP_TOOL_PEN, true); state.toolbar_items.set_hidden(section, true); state @@ -589,6 +590,7 @@ mod tests { let resolved = state.toolbar_items.resolved(); assert!(resolved.hidden.contains(&ids::TOP_UTILITY_SCREENSHOT)); + assert!(resolved.hidden.contains(&ids::TOP_UTILITY_OCR)); assert!(!resolved.hidden.contains(&ids::TOP_TOOL_PEN)); assert!(resolved.hidden.contains(§ion)); assert!(resolved.hidden.contains(&ids::TOP_CHROME_OVERFLOW)); diff --git a/src/input/state/interaction/actions.rs b/src/input/state/interaction/actions.rs index f2b1dc69..78655524 100644 --- a/src/input/state/interaction/actions.rs +++ b/src/input/state/interaction/actions.rs @@ -139,6 +139,7 @@ pub(crate) fn classify_action(action: Action) -> ActionRoute { | Action::ExportCanvasClipboardAndFile | Action::ExportBoardPdfFile | Action::ExportAllBoardsPdfFile + | Action::CopyTextFromScreen | Action::ToggleFrozenMode | Action::ZoomIn | Action::ZoomOut diff --git a/src/input/state/mod.rs b/src/input/state/mod.rs index 7c1310bb..3106acf6 100644 --- a/src/input/state/mod.rs +++ b/src/input/state/mod.rs @@ -29,8 +29,9 @@ pub use core::{ ContextMenuEntry, ContextMenuKind, ContextMenuState, DesktopEnvironment, DrawingState, EyedropperCaptureSource, EyedropperUiState, HelpOverlayClick, HelpOverlayCursorHint, HelpOverlayReleaseOutcome, ImeCompositionState, ImePreedit, InputState, MAX_STROKE_THICKNESS, - MIN_STROKE_THICKNESS, OutputFocusAction, PRESET_FEEDBACK_DURATION_MS, PRESET_TOAST_DURATION_MS, - PickerDrag, PrecisionEntryState, PresetAction, PresetFeedbackKind, PressureThicknessEditMode, + MIN_STROKE_THICKNESS, OcrCaptureSource, OcrInputSource, OcrSelection, OcrUiState, + OutputFocusAction, PRESET_FEEDBACK_DURATION_MS, PRESET_TOAST_DURATION_MS, PickerDrag, + PrecisionEntryState, PresetAction, PresetFeedbackKind, PressureThicknessEditMode, PressureThicknessEntryMode, QuickColorEdit, RADIAL_COMPASS_SLICES, RADIAL_PAINT_DELAY, RADIAL_TOOL_SEGMENT_COUNT, RadialMenuLayout, RadialMenuState, RadialParent, RadialRingSwatch, RadialSegmentId, RadialSlice, RadialSliceKind, SIZE_RING_ARC_SPAN, SIZE_RING_ARC_START, diff --git a/src/lib.rs b/src/lib.rs index a00096c3..a76a98ef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,7 @@ pub mod build_info; pub mod canvas_export; pub mod capture; mod cli; +pub(crate) mod clipboard_text; pub mod config; pub mod configurator_destination; mod daemon; @@ -25,6 +26,7 @@ pub mod input; mod label_format; mod logger; mod notification; +mod ocr; mod onboarding; pub mod palette_recents; pub mod paths; diff --git a/src/ocr/AGENTS.md b/src/ocr/AGENTS.md new file mode 100644 index 00000000..2eb4a4f1 --- /dev/null +++ b/src/ocr/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS.md + +## Scope +- Applies to screen text recognition (`Copy text from screen`) under `src/ocr/`. +- Covers the capacity-one controller, the Tesseract/`wl-copy` adapters, and the typed outcomes the event loop consumes. +- Region selection, capture ownership, and toasts live in `src/backend/wayland/state/ocr.rs`, not here. + +## Architecture +- `mod.rs` owns the request/outcome vocabulary and `run_request`, which encodes, recognizes, and publishes inside one worker stack frame. +- `controller.rs` is the identified capacity-one transport: one in-flight request, a terminal worker message, and an event-loop wake. +- `tesseract.rs` is the production adapter pair; tests substitute `TextRecognizer`/`OcrTextPublisher` fakes. + +## Invariants +- Recognized text must never reach application state, a log line, or a `Debug` rendering. `RecognizedText` redacts its own `Debug`; keep it that way. +- Never invoke a shell. Tesseract takes an explicit argument vector through the process broker's `HelperKind::Tesseract` allowlist. +- Language values arrive already validated by `config::validate_ocr_languages`; this module does not decide what is acceptable. +- The temporary PNG is deleted on every path, including failures and panics. +- Capacity one is deliberate: report busy rather than queueing a screen region the user has moved on from. + +## Coupled Changes +- Failure categories couple to the toasts in `src/backend/wayland/state/ocr.rs`. +- The invocation policy couples to `src/process_broker/manifest.rs` and its tests. +- `capture.ocr_languages` couples to `src/config/types/capture.rs`, the configurator Capture page, `config.example.toml`, and `docs/CONFIG.md`. +- Packaging guidance for Tesseract lives in `packaging/` and `README.md`. + +## Validation +- `cargo test ocr` covers the controller, adapters, and privacy contract. +- Run `./tools/lint-and-test.sh` for changes that touch the broker policy or config surface. +- Engine-dependent behavior (missing language data, real recognition quality) needs the live Wayland check; the unit tests use fakes. diff --git a/src/ocr/controller.rs b/src/ocr/controller.rs new file mode 100644 index 00000000..c548a938 --- /dev/null +++ b/src/ocr/controller.rs @@ -0,0 +1,567 @@ +//! Capacity-one completion transport for screen text recognition. +//! +//! Modelled on the clipboard controller: one identified request at a time, a +//! worker thread that always publishes a terminal message, and an event-loop +//! wake so the answer is never left waiting for unrelated input. Capacity one +//! is deliberate — a queued request would recognize a screen region the user +//! has already moved on from. + +use std::fmt; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::mpsc::{Receiver, SyncSender, TryRecvError, TrySendError}; + +use crate::backend::wayland::RuntimeWakeHandle; + +use super::{ + OcrOutcome, OcrRequest, OcrTextPublisher, TesseractRecognizer, TextRecognizer, WlCopyPublisher, + run_request, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct OcrRequestId(u64); + +impl fmt::Display for OcrRequestId { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(formatter) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum OcrSubmitError { + /// Another recognition is still running. + Busy { + active_id: OcrRequestId, + }, + IdentityExhausted, + Unhealthy, + SpawnFailed { + reason: String, + }, +} + +impl fmt::Display for OcrSubmitError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Busy { active_id } => { + write!(formatter, "OCR request {active_id} is still active") + } + Self::IdentityExhausted => formatter.write_str("OCR request IDs exhausted"), + Self::Unhealthy => formatter.write_str("OCR controller is unhealthy"), + Self::SpawnFailed { reason } => { + write!(formatter, "failed to spawn OCR worker: {reason}") + } + } + } +} + +#[derive(Debug)] +pub(crate) enum OcrPoll { + Idle, + /// A request is running. The identity stays with the controller until it + /// completes, so callers only need to know that OCR is busy. + Pending, + Ready { + id: OcrRequestId, + outcome: OcrOutcome, + }, + /// The worker panicked or exited without publishing an outcome. + WorkerLost { + id: OcrRequestId, + reason: String, + }, +} + +enum WorkerMessage { + Ready { + id: OcrRequestId, + outcome: OcrOutcome, + }, + Panicked { + id: OcrRequestId, + reason: String, + }, +} + +struct ActiveRequest { + id: OcrRequestId, + receiver: Receiver, +} + +pub(crate) struct OcrController { + next_id: Option, + runtime_wake: RuntimeWakeHandle, + active: Option, + healthy: bool, +} + +impl OcrController { + pub(crate) fn new(runtime_wake: RuntimeWakeHandle) -> Self { + Self { + next_id: Some(1), + runtime_wake, + active: None, + healthy: true, + } + } + + pub(crate) fn is_active(&self) -> bool { + self.active.is_some() + } + + /// Submit a request to the production Tesseract/`wl-copy` adapters. + pub(crate) fn try_submit( + &mut self, + request: OcrRequest, + ) -> Result { + self.try_submit_with(request, TesseractRecognizer, WlCopyPublisher) + } + + pub(crate) fn try_submit_with( + &mut self, + request: OcrRequest, + recognizer: impl TextRecognizer + Send + 'static, + publisher: impl OcrTextPublisher + Send + 'static, + ) -> Result { + self.try_submit_with_spawner(request, recognizer, publisher, |job| { + std::thread::Builder::new() + .name("wayscriber-ocr".to_string()) + .spawn(job) + .map(drop) + }) + } + + fn try_submit_with_spawner( + &mut self, + request: OcrRequest, + recognizer: impl TextRecognizer + Send + 'static, + publisher: impl OcrTextPublisher + Send + 'static, + spawn: impl FnOnce(Box) -> std::io::Result<()>, + ) -> Result { + if !self.healthy { + return Err(OcrSubmitError::Unhealthy); + } + if let Some(active) = &self.active { + return Err(OcrSubmitError::Busy { + active_id: active.id, + }); + } + let value = self.next_id.ok_or(OcrSubmitError::IdentityExhausted)?; + let id = OcrRequestId(value); + + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let runtime_wake = self.runtime_wake.clone(); + let job = Box::new(move || { + let guard = WorkerExitGuard::new(id, sender, runtime_wake); + let message = match catch_unwind(AssertUnwindSafe(|| { + run_request(request, &recognizer, &publisher) + })) { + Ok(outcome) => WorkerMessage::Ready { id, outcome }, + Err(payload) => WorkerMessage::Panicked { + id, + reason: panic_reason(&payload), + }, + }; + guard.publish(message); + }); + if let Err(err) = spawn(job) { + return Err(OcrSubmitError::SpawnFailed { + reason: err.to_string(), + }); + } + + self.next_id = value.checked_add(1); + self.active = Some(ActiveRequest { id, receiver }); + Ok(id) + } + + pub(crate) fn poll(&mut self) -> OcrPoll { + let Some(active) = self.active.as_ref() else { + return OcrPoll::Idle; + }; + let active_id = active.id; + match active.receiver.try_recv() { + Err(TryRecvError::Empty) => OcrPoll::Pending, + Err(TryRecvError::Disconnected) => { + self.active = None; + OcrPoll::WorkerLost { + id: active_id, + reason: "OCR worker exited without an outcome".to_string(), + } + } + Ok(WorkerMessage::Ready { id, outcome }) if id == active_id => { + self.active = None; + OcrPoll::Ready { id, outcome } + } + Ok(WorkerMessage::Panicked { id, reason }) if id == active_id => { + self.active = None; + OcrPoll::WorkerLost { id, reason } + } + Ok(WorkerMessage::Ready { id, .. } | WorkerMessage::Panicked { id, .. }) => { + self.healthy = false; + self.active = None; + OcrPoll::WorkerLost { + id: active_id, + reason: format!( + "OCR worker reported request identity {id}, expected {active_id}" + ), + } + } + } + } +} + +fn panic_reason(payload: &Box) -> String { + if let Some(message) = payload.downcast_ref::<&'static str>() { + (*message).to_string() + } else if let Some(message) = payload.downcast_ref::() { + message.clone() + } else { + "OCR worker panicked with a non-string payload".to_string() + } +} + +struct WorkerExitGuard { + id: OcrRequestId, + sender: Option>, + runtime_wake: RuntimeWakeHandle, + terminal_published: bool, +} + +impl WorkerExitGuard { + fn new( + id: OcrRequestId, + sender: SyncSender, + runtime_wake: RuntimeWakeHandle, + ) -> Self { + Self { + id, + sender: Some(sender), + runtime_wake, + terminal_published: false, + } + } + + fn publish(mut self, message: WorkerMessage) { + let result = self + .sender + .as_ref() + .expect("OCR worker sender retained until publication") + .try_send(message); + self.sender.take(); + self.terminal_published = true; + match result { + Ok(()) | Err(TrySendError::Disconnected(_)) => {} + Err(TrySendError::Full(_)) => { + log::error!("OCR worker {} found an impossible full channel", self.id); + } + } + if let Err(err) = self.runtime_wake.wake() { + log::error!("Failed to wake runtime for OCR request {}: {err}", self.id); + } + } +} + +impl Drop for WorkerExitGuard { + fn drop(&mut self) { + if self.terminal_published { + return; + } + // Closing the worker side is the terminal publication for a disconnect. + self.sender.take(); + if let Err(err) = self.runtime_wake.wake() { + log::error!( + "Failed to wake runtime for disconnected OCR request {}: {err}", + self.id + ); + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::mpsc; + use std::time::Duration; + + use super::*; + use crate::backend::wayland::RuntimeWakeSource; + use crate::ocr::{ + OcrFailure, OcrLanguages, OcrPixels, OcrSuccess, RecognizedOutput, RecognizedText, + }; + + struct FakeRecognizer { + outcome: fn() -> Result, + } + + impl TextRecognizer for FakeRecognizer { + fn recognize( + &self, + _png: &[u8], + _languages: &OcrLanguages, + ) -> Result { + (self.outcome)() + } + } + + struct BlockingRecognizer { + started: mpsc::Sender<()>, + release: std::sync::Mutex>, + } + + impl TextRecognizer for BlockingRecognizer { + fn recognize( + &self, + _png: &[u8], + _languages: &OcrLanguages, + ) -> Result { + self.started.send(()).unwrap(); + self.release + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(5)) + .unwrap(); + Ok(RecognizedOutput { + text: RecognizedText::trimmed("done"), + replaced_invalid_utf8: false, + }) + } + } + + struct PanickingRecognizer; + + impl TextRecognizer for PanickingRecognizer { + fn recognize( + &self, + _png: &[u8], + _languages: &OcrLanguages, + ) -> Result { + panic!("expected OCR worker panic"); + } + } + + struct NoopPublisher; + + impl OcrTextPublisher for NoopPublisher { + fn publish(&self, _text: &str) -> Result<(), OcrFailure> { + Ok(()) + } + } + + fn request() -> OcrRequest { + OcrRequest { + pixels: OcrPixels { + width: 2, + height: 2, + stride: 8, + data: vec![0xFF; 16], + }, + languages: OcrLanguages::from_validated("eng".to_string()), + } + } + + fn controller() -> (RuntimeWakeSource, OcrController) { + let wake = RuntimeWakeSource::new().unwrap(); + let controller = OcrController::new(wake.handle()); + (wake, controller) + } + + fn wait_for_wake(wake: &RuntimeWakeSource) { + assert!( + wake.wait_readable(Some(Duration::from_secs(5))).unwrap(), + "OCR completion did not wake the event loop" + ); + } + + #[test] + fn one_request_runs_off_thread_and_wakes_the_event_loop() { + let (wake, mut controller) = controller(); + let id = controller + .try_submit_with( + request(), + FakeRecognizer { + outcome: || { + Ok(RecognizedOutput { + text: RecognizedText::trimmed("copied text"), + replaced_invalid_utf8: false, + }) + }, + }, + NoopPublisher, + ) + .unwrap(); + + wait_for_wake(&wake); + let poll = controller.poll(); + assert!(matches!( + poll, + OcrPoll::Ready { + id: ready, + outcome: Ok(OcrSuccess::Copied { + character_count: 11, + replaced_invalid_utf8: false, + }), + } if ready == id + )); + assert!(!controller.is_active()); + } + + #[test] + fn a_second_request_while_active_is_rejected_rather_than_queued() { + let (_wake, mut controller) = controller(); + let (started_tx, started_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + + let first = controller + .try_submit_with( + request(), + BlockingRecognizer { + started: started_tx, + release: std::sync::Mutex::new(release_rx), + }, + NoopPublisher, + ) + .unwrap(); + started_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + + assert!(matches!(controller.poll(), OcrPoll::Pending)); + let error = controller + .try_submit_with( + request(), + FakeRecognizer { + outcome: || panic!("busy submission must not run"), + }, + NoopPublisher, + ) + .unwrap_err(); + assert_eq!(error, OcrSubmitError::Busy { active_id: first }); + + release_tx.send(()).unwrap(); + } + + #[test] + fn engine_failures_reach_the_event_loop_as_typed_outcomes() { + let (wake, mut controller) = controller(); + controller + .try_submit_with( + request(), + FakeRecognizer { + outcome: || Err(OcrFailure::TimedOut), + }, + NoopPublisher, + ) + .unwrap(); + + wait_for_wake(&wake); + assert!(matches!( + controller.poll(), + OcrPoll::Ready { + outcome: Err(OcrFailure::TimedOut), + .. + } + )); + } + + #[test] + fn worker_panic_is_reported_without_poisoning_the_controller() { + let (wake, mut controller) = controller(); + let id = controller + .try_submit_with(request(), PanickingRecognizer, NoopPublisher) + .unwrap(); + + wait_for_wake(&wake); + assert!(matches!( + controller.poll(), + OcrPoll::WorkerLost { id: lost, reason } if lost == id && reason.contains("expected OCR worker panic") + )); + assert!( + controller + .try_submit_with( + request(), + FakeRecognizer { + outcome: || { + Ok(RecognizedOutput { + text: RecognizedText::trimmed("after panic"), + replaced_invalid_utf8: false, + }) + }, + }, + NoopPublisher, + ) + .is_ok() + ); + } + + #[test] + fn a_worker_that_exits_without_publishing_is_reported_as_lost() { + let (wake, mut controller) = controller(); + let id = OcrRequestId(9); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + controller.active = Some(ActiveRequest { id, receiver }); + + drop(WorkerExitGuard::new(id, sender, wake.handle())); + + wait_for_wake(&wake); + assert!(matches!( + controller.poll(), + OcrPoll::WorkerLost { id: lost, .. } if lost == id + )); + } + + #[test] + fn spawn_failure_leaves_no_active_request() { + let (_wake, mut controller) = controller(); + let error = controller + .try_submit_with_spawner( + request(), + FakeRecognizer { + outcome: || Err(OcrFailure::EngineFailed), + }, + NoopPublisher, + |_job| Err(std::io::Error::other("injected spawn failure")), + ) + .unwrap_err(); + + assert_eq!( + error, + OcrSubmitError::SpawnFailed { + reason: "injected spawn failure".to_string(), + } + ); + assert!(!controller.is_active()); + } + + #[test] + fn identity_is_never_reused_and_exhaustion_is_reported() { + let (wake, mut controller) = controller(); + controller.next_id = Some(u64::MAX); + let id = controller + .try_submit_with( + request(), + FakeRecognizer { + outcome: || { + Ok(RecognizedOutput { + text: RecognizedText::trimmed("x"), + replaced_invalid_utf8: false, + }) + }, + }, + NoopPublisher, + ) + .unwrap(); + assert_eq!(id, OcrRequestId(u64::MAX)); + + wait_for_wake(&wake); + assert!(matches!(controller.poll(), OcrPoll::Ready { .. })); + assert_eq!( + controller + .try_submit_with( + request(), + FakeRecognizer { + outcome: || Err(OcrFailure::EngineFailed), + }, + NoopPublisher, + ) + .unwrap_err(), + OcrSubmitError::IdentityExhausted + ); + } +} diff --git a/src/ocr/mod.rs b/src/ocr/mod.rs new file mode 100644 index 00000000..4be4ac61 --- /dev/null +++ b/src/ocr/mod.rs @@ -0,0 +1,413 @@ +//! One-shot text recognition for a selected screen region. +//! +//! The controller runs at most one request at a time. Its worker owns the whole +//! job — PNG encoding, the Tesseract invocation, and the clipboard publication — +//! so the event loop only ever learns a request identity and a privacy-safe +//! outcome. Recognized text never reaches application state, a log line, or a +//! `Debug` rendering. + +use std::fmt; + +mod controller; +mod tesseract; + +pub(crate) use controller::{OcrController, OcrPoll, OcrSubmitError}; +pub(crate) use tesseract::{TesseractRecognizer, WlCopyPublisher}; + +/// Turns encoded image bytes into text. The production implementation shells +/// out to Tesseract; tests substitute deterministic fakes. +pub(crate) trait TextRecognizer { + fn recognize( + &self, + png: &[u8], + languages: &OcrLanguages, + ) -> Result; +} + +/// Publishes recognized text to the system clipboard. +pub(crate) trait OcrTextPublisher { + fn publish(&self, text: &str) -> Result<(), OcrFailure>; +} + +/// Tightly packed premultiplied ARGB32 pixels in Cairo's native byte order. +/// +/// Owned by the request: the crop is taken from the displayed screen image on +/// the event thread and handed to the worker, so a later freeze, zoom, or +/// display change cannot change what is recognized. +pub(crate) struct OcrPixels { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) stride: i32, + pub(crate) data: Vec, +} + +impl fmt::Debug for OcrPixels { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OcrPixels") + .field("width", &self.width) + .field("height", &self.height) + .field("stride", &self.stride) + .field("bytes", &self.data.len()) + .finish() + } +} + +/// A validated Tesseract language argument, such as `eng` or `eng+deu`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct OcrLanguages(String); + +impl OcrLanguages { + /// Wrap an already validated value. Config parsing is the only place that + /// decides what is acceptable; see `config::capture::validate_ocr_languages`. + pub(crate) fn from_validated(value: String) -> Self { + Self(value) + } + + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Debug)] +pub(crate) struct OcrRequest { + pub(crate) pixels: OcrPixels, + pub(crate) languages: OcrLanguages, +} + +/// Recognized text on its way to the clipboard. +/// +/// Deliberately opaque: a plain `String` here would eventually reach a log line +/// or an error message and put whatever was on screen into a file. Only the +/// character count and the redacted `Debug` leave this type. +pub(crate) struct RecognizedText(String); + +impl RecognizedText { + /// Trim the recognizer's output. Tesseract always ends with a newline and + /// pads empty regions with whitespace, so the trimmed value is what decides + /// between a copy and `No text found`. + pub(crate) fn trimmed(raw: &str) -> Self { + Self(raw.trim().to_string()) + } + + pub(crate) fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub(crate) fn character_count(&self) -> usize { + self.0.chars().count() + } + + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for RecognizedText { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "RecognizedText()", + self.character_count() + ) + } +} + +/// What a finished recognition produced. Carries no recognized text. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum OcrSuccess { + Copied { + character_count: usize, + /// The engine returned bytes that were not valid UTF-8 and unreadable + /// sequences were replaced before copying. + replaced_invalid_utf8: bool, + }, + NoTextFound, +} + +/// Why a recognition did not copy anything. Each variant is a distinct +/// user-facing message; engine detail stays in the debug log. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum OcrFailure { + /// No `tesseract` executable on `PATH`. + EngineMissing, + /// Tesseract could not load the configured language data. + LanguageMissing { languages: String }, + /// Tesseract exited unsuccessfully for some other reason. + EngineFailed, + /// Tesseract did not finish inside the timeout. + TimedOut, + /// Recognized output exceeded the stdout cap. + OutputTooLarge, + /// The selected pixels could not be encoded as PNG. + EncodeFailed, + /// The temporary PNG could not be created or written. + TemporaryFileFailed, + /// The process broker was unavailable or rejected the request. + EngineUnavailable, + /// Text was recognized but `wl-copy` did not accept it. + ClipboardFailed, +} + +impl OcrFailure { + /// Stable user-facing message. Never includes engine output. + pub(crate) fn message(&self) -> String { + match self { + Self::EngineMissing => "Install Tesseract to use screen text recognition.".to_string(), + Self::LanguageMissing { languages } => { + format!("Tesseract has no language data for \"{languages}\".") + } + Self::EngineFailed => "Screen text recognition failed.".to_string(), + Self::TimedOut => "Screen text recognition timed out.".to_string(), + Self::OutputTooLarge => "That region produced too much text to copy.".to_string(), + Self::EncodeFailed => { + "Could not prepare the selected region for recognition.".to_string() + } + Self::TemporaryFileFailed => { + "Could not create a temporary file for recognition.".to_string() + } + Self::EngineUnavailable => "Screen text recognition is unavailable.".to_string(), + Self::ClipboardFailed => "Text recognized, but clipboard copy failed.".to_string(), + } + } +} + +pub(crate) type OcrOutcome = Result; + +/// Run one complete request on the worker thread. +/// +/// Encoding, recognition, and publication live together so the recognized text +/// is created and consumed inside a single stack frame and is dropped before +/// the outcome travels back to the event loop. +fn run_request( + request: OcrRequest, + recognizer: &dyn TextRecognizer, + publisher: &dyn OcrTextPublisher, +) -> OcrOutcome { + let png = encode_png(&request.pixels)?; + let recognized = recognizer.recognize(&png, &request.languages)?; + if recognized.text.is_empty() { + return Ok(OcrSuccess::NoTextFound); + } + let character_count = recognized.text.character_count(); + publisher.publish(recognized.text.as_str())?; + Ok(OcrSuccess::Copied { + character_count, + replaced_invalid_utf8: recognized.replaced_invalid_utf8, + }) +} + +/// A recognizer result: the trimmed text plus whether the engine's bytes had to +/// be repaired to become UTF-8. +pub(crate) struct RecognizedOutput { + pub(crate) text: RecognizedText, + pub(crate) replaced_invalid_utf8: bool, +} + +fn encode_png(pixels: &OcrPixels) -> Result, OcrFailure> { + if pixels.width == 0 || pixels.height == 0 { + return Err(OcrFailure::EncodeFailed); + } + let width = i32::try_from(pixels.width).map_err(|_| OcrFailure::EncodeFailed)?; + let height = i32::try_from(pixels.height).map_err(|_| OcrFailure::EncodeFailed)?; + let surface = cairo::ImageSurface::create_for_data( + pixels.data.clone(), + cairo::Format::ARgb32, + width, + height, + pixels.stride, + ) + .map_err(|err| { + log::warn!("OCR crop surface creation failed: {err}"); + OcrFailure::EncodeFailed + })?; + let mut bytes = Vec::new(); + surface.write_to_png(&mut bytes).map_err(|err| { + log::warn!("OCR crop PNG encoding failed: {err}"); + OcrFailure::EncodeFailed + })?; + Ok(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + struct StubRecognizer(fn() -> Result); + + impl TextRecognizer for StubRecognizer { + fn recognize( + &self, + _png: &[u8], + _languages: &OcrLanguages, + ) -> Result { + (self.0)() + } + } + + struct RecordingPublisher(std::sync::Mutex>); + + impl OcrTextPublisher for RecordingPublisher { + fn publish(&self, text: &str) -> Result<(), OcrFailure> { + self.0.lock().unwrap().push(text.to_string()); + Ok(()) + } + } + + struct FailingPublisher; + + impl OcrTextPublisher for FailingPublisher { + fn publish(&self, _text: &str) -> Result<(), OcrFailure> { + Err(OcrFailure::ClipboardFailed) + } + } + + pub(super) fn pixels(width: u32, height: u32) -> OcrPixels { + OcrPixels { + width, + height, + stride: (width * 4) as i32, + data: vec![0xFF; (width * height * 4) as usize], + } + } + + fn request() -> OcrRequest { + OcrRequest { + pixels: pixels(2, 2), + languages: OcrLanguages::from_validated("eng".to_string()), + } + } + + #[test] + fn recognized_text_is_trimmed_and_never_formatted_into_debug_output() { + let text = RecognizedText::trimmed(" secret account number \n"); + assert_eq!(text.as_str(), "secret account number"); + assert_eq!(text.character_count(), 21); + + let rendered = format!("{text:?}"); + assert_eq!(rendered, "RecognizedText()"); + assert!(!rendered.contains("secret")); + } + + #[test] + fn pixel_debug_reports_geometry_without_the_captured_bytes() { + let rendered = format!("{:?}", pixels(3, 4)); + assert!(rendered.contains("width: 3")); + assert!(rendered.contains("bytes: 48")); + assert!(!rendered.contains("255")); + } + + #[test] + fn empty_recognition_does_not_publish_an_empty_clipboard_value() { + let publisher = RecordingPublisher(std::sync::Mutex::new(Vec::new())); + let outcome = run_request( + request(), + &StubRecognizer(|| { + Ok(RecognizedOutput { + text: RecognizedText::trimmed(" \n\t "), + replaced_invalid_utf8: false, + }) + }), + &publisher, + ); + + assert_eq!(outcome, Ok(OcrSuccess::NoTextFound)); + assert!(publisher.0.lock().unwrap().is_empty()); + } + + #[test] + fn successful_recognition_publishes_text_but_reports_only_a_character_count() { + let publisher = RecordingPublisher(std::sync::Mutex::new(Vec::new())); + let outcome = run_request( + request(), + &StubRecognizer(|| { + Ok(RecognizedOutput { + text: RecognizedText::trimmed("hello\n"), + replaced_invalid_utf8: false, + }) + }), + &publisher, + ); + + assert_eq!( + outcome, + Ok(OcrSuccess::Copied { + character_count: 5, + replaced_invalid_utf8: false, + }) + ); + assert_eq!(publisher.0.lock().unwrap().as_slice(), ["hello"]); + assert!(!format!("{outcome:?}").contains("hello")); + } + + #[test] + fn clipboard_failure_is_distinct_from_engine_failure() { + let outcome = run_request( + request(), + &StubRecognizer(|| { + Ok(RecognizedOutput { + text: RecognizedText::trimmed("hello"), + replaced_invalid_utf8: false, + }) + }), + &FailingPublisher, + ); + + assert_eq!(outcome, Err(OcrFailure::ClipboardFailed)); + assert_ne!( + OcrFailure::ClipboardFailed.message(), + OcrFailure::EngineFailed.message() + ); + } + + #[test] + fn every_failure_category_has_its_own_user_facing_message() { + let messages = [ + OcrFailure::EngineMissing, + OcrFailure::LanguageMissing { + languages: "deu".to_string(), + }, + OcrFailure::EngineFailed, + OcrFailure::TimedOut, + OcrFailure::OutputTooLarge, + OcrFailure::EncodeFailed, + OcrFailure::TemporaryFileFailed, + OcrFailure::EngineUnavailable, + OcrFailure::ClipboardFailed, + ] + .map(|failure| failure.message()); + let unique: std::collections::BTreeSet<_> = messages.iter().collect(); + + assert_eq!(unique.len(), messages.len()); + assert!(messages[1].contains("deu")); + } + + #[test] + fn encoding_rejects_an_empty_crop_before_touching_cairo() { + assert_eq!(encode_png(&pixels(0, 4)), Err(OcrFailure::EncodeFailed)); + assert_eq!(encode_png(&pixels(4, 0)), Err(OcrFailure::EncodeFailed)); + } + + #[test] + fn encoding_preserves_opaque_transparent_and_premultiplied_pixels() { + let source = OcrPixels { + width: 3, + height: 1, + stride: 12, + // Cairo ARgb32 is native-endian premultiplied BGRA on little-endian + // targets: opaque blue, fully transparent, half-alpha grey. + data: vec![255, 0, 0, 255, 0, 0, 0, 0, 64, 64, 64, 128], + }; + + let png = encode_png(&source).expect("crop encodes"); + let decoded = + cairo::ImageSurface::create_from_png(&mut png.as_slice()).expect("encoded PNG decodes"); + assert_eq!((decoded.width(), decoded.height()), (3, 1)); + + let mut decoded = decoded; + let stride = decoded.stride() as usize; + let data = decoded.data().expect("decoded pixels"); + assert_eq!(&data[..12], &source.data[..12]); + assert!(stride >= 12); + } +} diff --git a/src/ocr/tesseract.rs b/src/ocr/tesseract.rs new file mode 100644 index 00000000..d5ce3f7c --- /dev/null +++ b/src/ocr/tesseract.rs @@ -0,0 +1,325 @@ +//! Production adapters: local Tesseract for recognition, `wl-copy` for publication. + +use std::ffi::OsStr; +use std::io::Write; +use std::path::Path; +use std::time::Duration; + +use crate::process_broker::{HelperKind, STDOUT_CAP_EXCEEDED}; + +use super::{ + OcrFailure, OcrLanguages, OcrTextPublisher, RecognizedOutput, RecognizedText, TextRecognizer, +}; + +const TESSERACT_PROGRAM: &str = "tesseract"; +const TESSERACT_TIMEOUT: Duration = Duration::from_secs(30); +/// Recognized text is plain UTF-8; 4 MiB is far past any plausible screen +/// region and keeps a runaway engine from filling memory. +const TESSERACT_STDOUT_CAP: usize = 4 * 1024 * 1024; + +pub(crate) struct TesseractRecognizer; + +impl TextRecognizer for TesseractRecognizer { + fn recognize( + &self, + png: &[u8], + languages: &OcrLanguages, + ) -> Result { + if !program_on_path(TESSERACT_PROGRAM) { + return Err(OcrFailure::EngineMissing); + } + + // A securely created temporary file with automatic deletion on every + // path. A file rather than broker stdin because the broker's input cap + // is 16 MiB and a lossless desktop crop can exceed it. + let mut input = tempfile::Builder::new() + .prefix("wayscriber-ocr-") + .suffix(".png") + .tempfile() + .map_err(|err| { + log::warn!("OCR temporary file creation failed: {err}"); + OcrFailure::TemporaryFileFailed + })?; + input + .write_all(png) + .and_then(|()| input.as_file_mut().sync_all()) + .map_err(|err| { + log::warn!("OCR temporary file write failed: {err}"); + OcrFailure::TemporaryFileFailed + })?; + + let output = run_tesseract(input.path(), languages); + // Explicit close so the deletion error is observable rather than + // swallowed by `Drop`; either way the file is gone before returning. + if let Err(err) = input.close() { + log::warn!("OCR temporary file cleanup failed: {err}"); + } + output + } +} + +fn run_tesseract(input: &Path, languages: &OcrLanguages) -> Result { + let output = crate::process_broker::current() + .and_then(|broker| { + broker.run( + HelperKind::Tesseract, + OsStr::new(TESSERACT_PROGRAM), + tesseract_arguments(input, languages), + Vec::new(), + TESSERACT_TIMEOUT, + TESSERACT_STDOUT_CAP, + ) + }) + .map_err(|err| { + log::warn!("Failed to run tesseract: {err:#}"); + classify_broker_error(&err) + })?; + + if output.timed_out { + return Err(OcrFailure::TimedOut); + } + if output.status != 0 { + let stderr = String::from_utf8_lossy(&output.stderr); + // Engine detail is a debug/warning concern; the user sees a stable + // category message instead. + log::warn!( + "tesseract exited with status {}: {}", + output.status, + stderr.trim() + ); + return Err(classify_stderr(&stderr, languages)); + } + + // Never log stdout: it is the recognized screen content. + let replaced_invalid_utf8 = std::str::from_utf8(&output.stdout).is_err(); + let text = RecognizedText::trimmed(&String::from_utf8_lossy(&output.stdout)); + Ok(RecognizedOutput { + text, + replaced_invalid_utf8, + }) +} + +/// The invocation itself: an explicit argument vector, never a shell line. +fn tesseract_arguments<'a>(input: &'a Path, languages: &'a OcrLanguages) -> Vec<&'a OsStr> { + vec![ + input.as_os_str(), + OsStr::new("stdout"), + OsStr::new("--oem"), + OsStr::new("1"), + OsStr::new("--psm"), + OsStr::new("6"), + OsStr::new("-l"), + OsStr::new(languages.as_str()), + OsStr::new("--dpi"), + OsStr::new("300"), + OsStr::new("-c"), + OsStr::new("preserve_interword_spaces=1"), + ] +} + +/// A run that outgrows the stdout cap is rejected by the broker rather than +/// truncated, so "too much text" arrives as a transport error and has to be +/// separated from a broker that is simply unavailable. +fn classify_broker_error(error: &anyhow::Error) -> OcrFailure { + if format!("{error:#}").contains(STDOUT_CAP_EXCEEDED) { + OcrFailure::OutputTooLarge + } else { + OcrFailure::EngineUnavailable + } +} + +fn classify_stderr(stderr: &str, languages: &OcrLanguages) -> OcrFailure { + let lowered = stderr.to_ascii_lowercase(); + if lowered.contains("failed loading language") + || lowered.contains("could not load any languages") + || lowered.contains("couldn't load any languages") + || lowered.contains("error opening data file") + { + OcrFailure::LanguageMissing { + languages: languages.as_str().to_string(), + } + } else { + OcrFailure::EngineFailed + } +} + +/// Whether `program` resolves to an executable file on `PATH`. +/// +/// Checked before invoking so "not installed" is its own actionable message +/// rather than a generic spawn failure. +fn program_on_path(program: &str) -> bool { + std::env::var_os("PATH").is_some_and(|path| program_in_search_path(program, &path)) +} + +/// The search itself, over an explicit path list. Split out so tests can probe +/// a temporary directory without mutating the process-wide `PATH`, which every +/// concurrently running test shares. +fn program_in_search_path(program: &str, search_path: &OsStr) -> bool { + use std::os::unix::fs::PermissionsExt; + + std::env::split_paths(search_path).any(|directory| { + if directory.as_os_str().is_empty() { + return false; + } + std::fs::metadata(directory.join(program)) + .map(|metadata| metadata.is_file() && metadata.permissions().mode() & 0o111 != 0) + .unwrap_or(false) + }) +} + +pub(crate) struct WlCopyPublisher; + +impl OcrTextPublisher for WlCopyPublisher { + fn publish(&self, text: &str) -> Result<(), OcrFailure> { + crate::clipboard_text::copy_text_via_command(text).map_err(|err| { + log::warn!("wl-copy failed for recognized text: {err}"); + OcrFailure::ClipboardFailed + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn languages() -> OcrLanguages { + OcrLanguages::from_validated("eng+deu".to_string()) + } + + #[test] + fn invocation_matches_the_documented_argument_vector() { + let path = Path::new("/tmp/wayscriber-ocr-test.png"); + let arguments: Vec<_> = tesseract_arguments(path, &languages()) + .into_iter() + .map(|argument| argument.to_string_lossy().into_owned()) + .collect(); + + assert_eq!( + arguments, + [ + "/tmp/wayscriber-ocr-test.png", + "stdout", + "--oem", + "1", + "--psm", + "6", + "-l", + "eng+deu", + "--dpi", + "300", + "-c", + "preserve_interword_spaces=1", + ] + ); + } + + #[test] + fn missing_language_data_is_distinguished_from_a_generic_engine_failure() { + assert_eq!( + classify_stderr("Failed loading language 'deu'\n", &languages()), + OcrFailure::LanguageMissing { + languages: "eng+deu".to_string(), + } + ); + assert_eq!( + classify_stderr( + "Error opening data file /usr/share/tessdata/deu.traineddata", + &languages() + ), + OcrFailure::LanguageMissing { + languages: "eng+deu".to_string(), + } + ); + assert_eq!( + classify_stderr("Image too large to process", &languages()), + OcrFailure::EngineFailed + ); + } + + #[test] + fn path_probe_accepts_only_executable_files() { + use std::os::unix::fs::PermissionsExt; + + let directory = crate::test_temp::tempdir().unwrap(); + let program = directory.path().join("wayscriber-fake-ocr"); + std::fs::write(&program, b"#!/bin/sh\n").unwrap(); + let search_path = directory.path().as_os_str(); + + assert!( + !program_in_search_path("wayscriber-fake-ocr", search_path), + "a non-executable file is not a usable engine" + ); + + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(program_in_search_path("wayscriber-fake-ocr", search_path)); + assert!(!program_in_search_path( + "wayscriber-absent-ocr", + search_path + )); + } + + #[test] + fn path_probe_skips_empty_entries_and_searches_every_directory() { + use std::os::unix::fs::PermissionsExt; + + let directory = crate::test_temp::tempdir().unwrap(); + let program = directory.path().join("wayscriber-fake-ocr"); + std::fs::write(&program, b"#!/bin/sh\n").unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o755)).unwrap(); + + // An empty entry means "current directory" to some shells; treating it + // as one would make the probe depend on the working directory. + let search_path = + std::env::join_paths(["".as_ref(), Path::new("/nonexistent"), directory.path()]) + .unwrap(); + + assert!(program_in_search_path("wayscriber-fake-ocr", &search_path)); + } + + #[test] + fn an_oversized_result_is_distinguished_from_an_unavailable_broker() { + assert_eq!( + classify_broker_error(&anyhow::anyhow!(STDOUT_CAP_EXCEEDED)), + OcrFailure::OutputTooLarge + ); + assert_eq!( + classify_broker_error( + &anyhow::anyhow!(STDOUT_CAP_EXCEEDED).context("process broker rejected request") + ), + OcrFailure::OutputTooLarge + ); + assert_eq!( + classify_broker_error(&anyhow::anyhow!("runtime process broker is not active")), + OcrFailure::EngineUnavailable + ); + } + + #[test] + fn a_temporary_input_file_is_removed_on_every_path() { + // The recognizer owns the file through `tempfile`; the guarantee under + // test is that no wayscriber-ocr file survives a run that fails before, + // during, or after the engine call. Without a broker the run fails at + // the invocation step, which is the hardest path to clean up. + let before = temporary_ocr_files(); + let outcome = TesseractRecognizer.recognize(b"not a png", &languages()); + assert!(outcome.is_err()); + assert_eq!(temporary_ocr_files(), before); + } + + fn temporary_ocr_files() -> Vec { + let Ok(entries) = std::fs::read_dir(std::env::temp_dir()) else { + return Vec::new(); + }; + let mut paths: Vec<_> = entries + .flatten() + .map(|entry| entry.path()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("wayscriber-ocr-")) + }) + .collect(); + paths.sort(); + paths + } +} diff --git a/src/process_broker/execution.rs b/src/process_broker/execution.rs index 4f8dd937..d9c02edf 100644 --- a/src/process_broker/execution.rs +++ b/src/process_broker/execution.rs @@ -9,7 +9,7 @@ use std::time::Duration; use anyhow::{Context, Result, anyhow}; use super::transport::shutdown_requested; -use super::wire::{HelperKind, MAX_STDERR_BYTES, OutputMode}; +use super::wire::{HelperKind, MAX_STDERR_BYTES, OutputMode, STDOUT_CAP_EXCEEDED}; pub(super) struct BoundedOutput { pub(super) status: ExitStatus, @@ -361,7 +361,7 @@ pub(super) fn run_bounded( } let stdout_limit_reached = stdout_limit_reached.load(Ordering::Acquire); if output_mode == OutputMode::Complete && stdout_limit_reached { - return Err(anyhow!("broker helper stdout exceeded output cap")); + return Err(anyhow!(STDOUT_CAP_EXCEEDED)); } if stderr_overflow.load(Ordering::Acquire) { return Err(anyhow!("broker helper stderr exceeded output cap")); diff --git a/src/process_broker/manifest.rs b/src/process_broker/manifest.rs index cae1ed32..c86607be 100644 --- a/src/process_broker/manifest.rs +++ b/src/process_broker/manifest.rs @@ -57,6 +57,7 @@ pub(super) fn validate( HelperKind::Grim => basename == "grim", HelperKind::Hyprctl => basename == "hyprctl", HelperKind::Slurp => basename == "slurp", + HelperKind::Tesseract => basename == "tesseract", HelperKind::WlPaste => basename == "wl-paste", HelperKind::WlCopy => basename == "wl-copy", HelperKind::SessionZenity => basename == "zenity", diff --git a/src/process_broker/mod.rs b/src/process_broker/mod.rs index 27ca0976..4ce84c41 100644 --- a/src/process_broker/mod.rs +++ b/src/process_broker/mod.rs @@ -17,4 +17,4 @@ mod tests; pub(crate) use client::{BrokerChild, current, start_for_runtime}; pub(crate) use server::run_internal_broker_if_requested; -pub(crate) use wire::{BrokerOutput, HelperKind, HelperLifetime}; +pub(crate) use wire::{BrokerOutput, HelperKind, HelperLifetime, STDOUT_CAP_EXCEEDED}; diff --git a/src/process_broker/tests.rs b/src/process_broker/tests.rs index 00f23e33..6146f839 100644 --- a/src/process_broker/tests.rs +++ b/src/process_broker/tests.rs @@ -83,6 +83,35 @@ fn update_fetcher_manifest_allows_only_curl_and_wget() { ); } +#[test] +fn tesseract_manifest_allows_only_the_tesseract_basename() { + for program in ["/usr/bin/tesseract", "/usr/local/bin/tesseract"] { + let program = super::wire::OsWire::from_os(OsStr::new(program)).unwrap(); + super::manifest::validate(HelperKind::Tesseract, &program, &[], &[], &[]).unwrap(); + } + + for program in ["/usr/bin/sh", "/usr/bin/tesseract-wrapper", "/usr/bin/grim"] { + let program = super::wire::OsWire::from_os(OsStr::new(program)).unwrap(); + assert!( + super::manifest::validate(HelperKind::Tesseract, &program, &[], &[], &[]).is_err(), + "{program:?} must not be allowed for OCR" + ); + } +} + +#[test] +fn tesseract_reads_the_complete_recognized_output_rather_than_a_prefix() { + // Prefix mode stops the helper once the cap fills; OCR must instead see the + // cap as a distinct "too much text" failure, which needs Complete mode. + assert!(!super::manifest::supports_prefix_output( + HelperKind::Tesseract + )); + assert_eq!( + super::manifest::input_cap(HelperKind::Tesseract), + super::wire::MAX_INPUT_BYTES + ); +} + #[test] fn prelock_broker_runs_bounded_helpers_and_owns_reaping() { let guard = start_for_runtime().unwrap(); diff --git a/src/process_broker/wire.rs b/src/process_broker/wire.rs index 30a4c3c9..88a655f6 100644 --- a/src/process_broker/wire.rs +++ b/src/process_broker/wire.rs @@ -20,6 +20,13 @@ pub(super) const MAX_PACKET_DESCRIPTORS: usize = 2; pub(super) const REQUIRED_MEMFD_SEALS: i32 = libc::F_SEAL_WRITE | libc::F_SEAL_GROW | libc::F_SEAL_SHRINK | libc::F_SEAL_SEAL; +/// Failure text used when a `Complete`-mode helper writes past its stdout cap. +/// +/// The broker rejects that run rather than returning a truncated result, so the +/// only signal a caller gets is this message. Callers that distinguish +/// "produced too much output" from other transport failures match on it. +pub(crate) const STDOUT_CAP_EXCEEDED: &str = "broker helper stdout exceeded output cap"; + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub(crate) enum HelperKind { @@ -29,6 +36,7 @@ pub(crate) enum HelperKind { Grim, Hyprctl, Slurp, + Tesseract, WlPaste, WlCopy, SessionZenity, diff --git a/src/toolbar_icons/actions.rs b/src/toolbar_icons/actions.rs index 240a254c..83348f59 100644 --- a/src/toolbar_icons/actions.rs +++ b/src/toolbar_icons/actions.rs @@ -383,6 +383,11 @@ pub fn draw_icon_screenshot(ctx: &Context, x: f64, y: f64, size: f64) { super::svg::render_screenshot(ctx, x, y, size); } +/// Draw a screen text recognition (OCR) icon. +pub fn draw_icon_ocr(ctx: &Context, x: f64, y: f64, size: f64) { + super::svg::render_ocr(ctx, x, y, size); +} + /// Draw a visibility/eye icon. pub fn draw_icon_visibility(ctx: &Context, x: f64, y: f64, size: f64) { let s = size; diff --git a/src/toolbar_icons/mod.rs b/src/toolbar_icons/mod.rs index 59aba8a2..30a40cb0 100644 --- a/src/toolbar_icons/mod.rs +++ b/src/toolbar_icons/mod.rs @@ -104,6 +104,7 @@ pub(crate) fn top_toolbar_icon_painter( I::Text => draw_icon_text, I::StickyNote => draw_icon_note, I::Screenshot => draw_icon_screenshot, + I::Ocr => draw_icon_ocr, I::Highlight => draw_icon_highlight, I::ClearCanvas => draw_icon_clear, I::Undo => draw_icon_undo, @@ -153,7 +154,7 @@ mod painter_tests { /// Every public painter. `svg.rs` covers the newer family through its own /// `render_*` entry points; this covers the shipped surface callers use, /// including the older proportional-style painters that had no coverage. - const PAINTERS: [(&str, IconPainter); 63] = [ + const PAINTERS: [(&str, IconPainter); 64] = [ ("arrow", draw_icon_arrow), ("blur", draw_icon_blur), ("board", draw_icon_board), @@ -187,6 +188,7 @@ mod painter_tests { ("minus", draw_icon_minus), ("more", draw_icon_more), ("note", draw_icon_note), + ("ocr", draw_icon_ocr), ("parallelogram", draw_icon_parallelogram), ("paste", draw_icon_paste), ("pen", draw_icon_pen), diff --git a/src/toolbar_icons/svg.rs b/src/toolbar_icons/svg.rs index 4b109e8c..e6d7d39a 100644 --- a/src/toolbar_icons/svg.rs +++ b/src/toolbar_icons/svg.rs @@ -273,6 +273,34 @@ fn draw_screenshot(ctx: &Context) { stroke(ctx); } +/// Screen text recognition: a scan frame around three text lines. +/// +/// Deliberately not a port of any other project's geometry — this is the same +/// normalized 24x24 Cairo style as its neighbours, with shorter corner arms +/// than `draw_screenshot` so the text block inside stays readable at 18px. +fn draw_ocr(ctx: &Context) { + for (x1, y1, x2, y2, x3, y3) in [ + (3.25, 7.0, 3.25, 3.25, 7.0, 3.25), + (17.0, 3.25, 20.75, 3.25, 20.75, 7.0), + (20.75, 17.0, 20.75, 20.75, 17.0, 20.75), + (7.0, 20.75, 3.25, 20.75, 3.25, 17.0), + ] { + ctx.move_to(x1, y1); + ctx.line_to(x2, y2); + ctx.line_to(x3, y3); + stroke(ctx); + } + + // A ragged right edge is what reads as text rather than as a stack of bars. + ctx.set_line_width(1.6); + for (y, end) in [(8.75, 16.75), (12.0, 16.75), (15.25, 13.5)] { + ctx.move_to(7.25, y); + ctx.line_to(end, y); + stroke(ctx); + } + ctx.set_line_width(2.0); +} + fn draw_highlight(ctx: &Context) { circle(ctx, 12.0, 12.0, 3.25); stroke(ctx); @@ -565,6 +593,7 @@ renderers!( (render_text, draw_text), (render_note, draw_sticky_note), (render_screenshot, draw_screenshot), + (render_ocr, draw_ocr), (render_highlight, draw_highlight), (render_undo, draw_undo), (render_redo, draw_redo), @@ -596,7 +625,7 @@ mod tests { type IconRender = fn(&Context, f64, f64, f64); const SIZES: [i32; 5] = [18, 20, 22, 24, 28]; - const ICONS: [(&str, IconRender); 33] = [ + const ICONS: [(&str, IconRender); 34] = [ ("drag", render_drag), ("select", render_select), ("pen", render_pen), @@ -609,6 +638,7 @@ mod tests { ("text", render_text), ("sticky_note", render_note), ("screenshot", render_screenshot), + ("ocr", render_ocr), ("highlight", render_highlight), ("undo", render_undo), ("redo", render_redo), diff --git a/src/ui/toolbar/apply/actions.rs b/src/ui/toolbar/apply/actions.rs index 5a7c779b..851fbf89 100644 --- a/src/ui/toolbar/apply/actions.rs +++ b/src/ui/toolbar/apply/actions.rs @@ -57,6 +57,12 @@ impl InputState { true } + pub(super) fn apply_toolbar_copy_text_from_screen(&mut self) -> bool { + self.handle_action(Action::CopyTextFromScreen); + self.close_top_toolbar_menus(); + true + } + pub(super) fn apply_toolbar_open_command_palette(&mut self) -> bool { self.handle_action(Action::ToggleCommandPalette); self.close_top_toolbar_menus(); diff --git a/src/ui/toolbar/apply/mod.rs b/src/ui/toolbar/apply/mod.rs index 455fe9de..fdd77457 100644 --- a/src/ui/toolbar/apply/mod.rs +++ b/src/ui/toolbar/apply/mod.rs @@ -94,6 +94,7 @@ impl InputState { ToolbarEvent::CustomRedo => self.apply_toolbar_custom_redo(), ToolbarEvent::ClearCanvas { instant } => self.apply_toolbar_clear_canvas(instant), ToolbarEvent::CaptureScreenshot => self.apply_toolbar_capture_screenshot(), + ToolbarEvent::CopyTextFromScreen => self.apply_toolbar_copy_text_from_screen(), ToolbarEvent::PagePrev => self.apply_toolbar_page_prev(), ToolbarEvent::PageNext => self.apply_toolbar_page_next(), ToolbarEvent::PageNew => self.apply_toolbar_page_new(), diff --git a/src/ui/toolbar/bindings.rs b/src/ui/toolbar/bindings.rs index 0e934e7f..459a3fb2 100644 --- a/src/ui/toolbar/bindings.rs +++ b/src/ui/toolbar/bindings.rs @@ -167,6 +167,14 @@ mod tests { ); } + #[test] + fn action_for_event_maps_screen_text_recognition() { + assert_eq!( + action_for_event(&ToolbarEvent::CopyTextFromScreen), + Some(Action::CopyTextFromScreen) + ); + } + #[test] fn action_for_event_returns_none_for_layout_only_events() { assert_eq!(action_for_event(&ToolbarEvent::OpenConfigFile), None); diff --git a/src/ui/toolbar/events.rs b/src/ui/toolbar/events.rs index 6e00766d..89253665 100644 --- a/src/ui/toolbar/events.rs +++ b/src/ui/toolbar/events.rs @@ -111,6 +111,8 @@ pub enum ToolbarEvent { instant: bool, }, CaptureScreenshot, + /// Select a screen region and copy the text recognized in it. + CopyTextFromScreen, PagePrev, PageNext, PageNew, diff --git a/src/ui/toolbar/model/event_policy.rs b/src/ui/toolbar/model/event_policy.rs index 00861e40..4e71debe 100644 --- a/src/ui/toolbar/model/event_policy.rs +++ b/src/ui/toolbar/model/event_policy.rs @@ -148,6 +148,7 @@ pub(crate) fn action_for_event(event: &ToolbarEvent) -> Option { ToolbarEvent::RedoAllDelayed => Some(Action::RedoAllDelayed), ToolbarEvent::ClearCanvas { .. } => Some(Action::ClearCanvas), ToolbarEvent::CaptureScreenshot => Some(Action::CaptureSelection), + ToolbarEvent::CopyTextFromScreen => Some(Action::CopyTextFromScreen), ToolbarEvent::PagePrev => Some(Action::PagePrev), ToolbarEvent::PageNext => Some(Action::PageNext), ToolbarEvent::PageNew => Some(Action::PageNew), @@ -516,6 +517,7 @@ fn persistence_for_event(event: &ToolbarEvent) -> ToolbarPersistence { | ToolbarEvent::Redo | ToolbarEvent::ClearCanvas { .. } | ToolbarEvent::CaptureScreenshot + | ToolbarEvent::CopyTextFromScreen | ToolbarEvent::PagePrev | ToolbarEvent::PageNext | ToolbarEvent::PageNew diff --git a/src/ui/toolbar/model/mod.rs b/src/ui/toolbar/model/mod.rs index a8df1a37..603a09e8 100644 --- a/src/ui/toolbar/model/mod.rs +++ b/src/ui/toolbar/model/mod.rs @@ -52,7 +52,7 @@ pub(crate) use tools::{ default_polygon_tool, default_shape_tool, is_fill_tool, is_polygon_tool, polygon_tools, semantic_icon_for_tool, shape_tools, tool_visible, toolbar_item_id_for_tool, toolbar_item_visible, top_clear_canvas_visible, top_fill_visible, top_highlight_ring_visible, - top_highlight_visible, top_screenshot_visible, top_shape_picker_visible, + top_highlight_visible, top_ocr_visible, top_screenshot_visible, top_shape_picker_visible, top_sticky_note_visible, top_text_visible, top_tool_buttons, top_tool_group, visible_shape_picker_max_row_len, visible_shape_picker_row_count, visible_shape_picker_rows, visible_tool_count, visible_top_tool_buttons, visible_top_utility_buttons, diff --git a/src/ui/toolbar/model/tools.rs b/src/ui/toolbar/model/tools.rs index 09a1a63d..96cd76c4 100644 --- a/src/ui/toolbar/model/tools.rs +++ b/src/ui/toolbar/model/tools.rs @@ -145,6 +145,10 @@ pub(crate) fn top_screenshot_visible(snapshot: &ToolbarSnapshot) -> bool { toolbar_item_visible(snapshot, ids::TOP_UTILITY_SCREENSHOT) } +pub(crate) fn top_ocr_visible(snapshot: &ToolbarSnapshot) -> bool { + toolbar_item_visible(snapshot, ids::TOP_UTILITY_OCR) +} + pub(crate) fn top_highlight_visible(snapshot: &ToolbarSnapshot) -> bool { toolbar_item_visible(snapshot, ids::TOP_UTILITY_HIGHLIGHT) } @@ -213,15 +217,17 @@ pub(crate) enum TopUtilityButton { Text, StickyNote, Screenshot, + Ocr, ClearCanvas, Highlight, IconMode, } -const DEFAULT_TOP_UTILITY_BUTTONS: [TopUtilityButton; 5] = [ +const DEFAULT_TOP_UTILITY_BUTTONS: [TopUtilityButton; 6] = [ TopUtilityButton::Text, TopUtilityButton::StickyNote, TopUtilityButton::Screenshot, + TopUtilityButton::Ocr, TopUtilityButton::Highlight, TopUtilityButton::ClearCanvas, ]; @@ -232,6 +238,7 @@ impl TopUtilityButton { Self::Text => ids::TOP_UTILITY_TEXT, Self::StickyNote => ids::TOP_UTILITY_STICKY_NOTE, Self::Screenshot => ids::TOP_UTILITY_SCREENSHOT, + Self::Ocr => ids::TOP_UTILITY_OCR, Self::ClearCanvas => ids::TOP_UTILITY_CLEAR_CANVAS, Self::Highlight => ids::TOP_UTILITY_HIGHLIGHT, Self::IconMode if snapshot.use_icons => ids::TOP_UTILITY_ICON_MODE_TEXT, @@ -278,6 +285,8 @@ fn top_utility_button_for_id( Some(TopUtilityButton::StickyNote) } else if id == ids::TOP_UTILITY_SCREENSHOT { Some(TopUtilityButton::Screenshot) + } else if id == ids::TOP_UTILITY_OCR { + Some(TopUtilityButton::Ocr) } else if id == ids::TOP_UTILITY_CLEAR_CANVAS { Some(TopUtilityButton::ClearCanvas) } else if id == ids::TOP_UTILITY_HIGHLIGHT { @@ -302,6 +311,7 @@ fn top_utility_button_visible( TopUtilityButton::Text => top_text_visible(snapshot), TopUtilityButton::StickyNote => top_sticky_note_visible(snapshot), TopUtilityButton::Screenshot => top_screenshot_visible(snapshot), + TopUtilityButton::Ocr => top_ocr_visible(snapshot), TopUtilityButton::ClearCanvas => !simple && top_clear_canvas_visible(snapshot), TopUtilityButton::Highlight => !simple && use_icons && top_highlight_visible(snapshot), TopUtilityButton::IconMode => false, diff --git a/src/ui/toolbar/model/top_spec/control.rs b/src/ui/toolbar/model/top_spec/control.rs index 5858718d..6a088e6e 100644 --- a/src/ui/toolbar/model/top_spec/control.rs +++ b/src/ui/toolbar/model/top_spec/control.rs @@ -117,6 +117,7 @@ impl TopToolbarControl { TopToolbarUtility::Text => ids::TOP_UTILITY_TEXT, TopToolbarUtility::StickyNote => ids::TOP_UTILITY_STICKY_NOTE, TopToolbarUtility::Screenshot => ids::TOP_UTILITY_SCREENSHOT, + TopToolbarUtility::Ocr => ids::TOP_UTILITY_OCR, TopToolbarUtility::Highlight => ids::TOP_UTILITY_HIGHLIGHT, }, Self::Preset(index) => return TopToolbarControlId::Preset(index), @@ -196,7 +197,7 @@ impl TopToolbarControl { Self::Utility(TopToolbarUtility::Text) => snapshot.text_active, Self::Utility(TopToolbarUtility::StickyNote) => snapshot.note_active, Self::Utility(TopToolbarUtility::Highlight) => snapshot.any_highlight_active, - Self::Utility(TopToolbarUtility::Screenshot) => false, + Self::Utility(TopToolbarUtility::Screenshot | TopToolbarUtility::Ocr) => false, // A filled slot reads as active while it is the applied preset. Self::Preset(index) => snapshot.active_preset_slot == Some(index + 1), Self::Pin => snapshot.top_pinned, @@ -267,6 +268,7 @@ impl TopToolbarControl { Self::Utility(TopToolbarUtility::Text) => TopToolbarIcon::Text, Self::Utility(TopToolbarUtility::StickyNote) => TopToolbarIcon::StickyNote, Self::Utility(TopToolbarUtility::Screenshot) => TopToolbarIcon::Screenshot, + Self::Utility(TopToolbarUtility::Ocr) => TopToolbarIcon::Ocr, Self::ClearCanvas => TopToolbarIcon::ClearCanvas, Self::Utility(TopToolbarUtility::Highlight) => TopToolbarIcon::Highlight, Self::Undo => TopToolbarIcon::Undo, diff --git a/src/ui/toolbar/model/top_spec/control_meta.rs b/src/ui/toolbar/model/top_spec/control_meta.rs index e37c43f0..d3265563 100644 --- a/src/ui/toolbar/model/top_spec/control_meta.rs +++ b/src/ui/toolbar/model/top_spec/control_meta.rs @@ -16,6 +16,7 @@ pub(crate) enum TopToolbarUtility { Text, StickyNote, Screenshot, + Ocr, Highlight, } @@ -25,6 +26,7 @@ impl TopToolbarUtility { TopUtilityButton::Text => Some(Self::Text), TopUtilityButton::StickyNote => Some(Self::StickyNote), TopUtilityButton::Screenshot => Some(Self::Screenshot), + TopUtilityButton::Ocr => Some(Self::Ocr), TopUtilityButton::Highlight => Some(Self::Highlight), TopUtilityButton::ClearCanvas | TopUtilityButton::IconMode => None, } @@ -64,6 +66,8 @@ pub(crate) enum TopToolbarIcon { Text, StickyNote, Screenshot, + /// Screen text recognition: a scan frame around three text lines. + Ocr, Highlight, ClearCanvas, Undo, @@ -109,6 +113,7 @@ pub(super) fn utility_event( TopToolbarUtility::Text => ToolbarEvent::EnterTextMode, TopToolbarUtility::StickyNote => ToolbarEvent::EnterStickyNoteMode, TopToolbarUtility::Screenshot => ToolbarEvent::CaptureScreenshot, + TopToolbarUtility::Ocr => ToolbarEvent::CopyTextFromScreen, TopToolbarUtility::Highlight => { ToolbarEvent::ToggleAllHighlight(!snapshot.any_highlight_active) } @@ -120,6 +125,7 @@ pub(super) fn utility_action(utility: TopToolbarUtility) -> Action { TopToolbarUtility::Text => Action::EnterTextMode, TopToolbarUtility::StickyNote => Action::EnterStickyNoteMode, TopToolbarUtility::Screenshot => Action::CaptureSelection, + TopToolbarUtility::Ocr => Action::CopyTextFromScreen, TopToolbarUtility::Highlight => Action::ToggleHighlightTool, } } @@ -127,6 +133,7 @@ pub(super) fn utility_action(utility: TopToolbarUtility) -> Action { pub(super) fn utility_short_label(utility: TopToolbarUtility) -> &'static str { match utility { TopToolbarUtility::Screenshot => "Shot", + TopToolbarUtility::Ocr => "Copy text", TopToolbarUtility::Highlight => "Highlight", _ => action_short_label(utility_action(utility)), } diff --git a/tools/check-nixpkgs-recipe.py b/tools/check-nixpkgs-recipe.py index 07d83b17..f08bfc49 100755 --- a/tools/check-nixpkgs-recipe.py +++ b/tools/check-nixpkgs-recipe.py @@ -60,6 +60,8 @@ "serde_ignored": (), "serde_json": (), "smithay-client-toolkit": ("libxkbcommon",), + # Pure Rust: temporary files for the OCR engine handoff. + "tempfile": (), "tokio": (), "toml": (), "toml_edit": (), From 9be7d30873f052c2b2ff9d7f6b56439656d23c51 Mon Sep 17 00:00:00 2001 From: devmobasa <4170275+devmobasa@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:42:22 +0200 Subject: [PATCH 2/3] build: lint the no-default-features configuration The gate built and tested that configuration but never linted it, so code reachable only behind an optional feature could leave its callers dead without it and still pass. Building alone does not promote that to an error. Caught exactly that during OCR work: a predicate used only from `tablet-input` code was dead in the default build, and only a manual clippy run found it. Passes clean today, so it is a free tripwire. --- tools/lint-and-test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/lint-and-test.sh b/tools/lint-and-test.sh index 7e8373c9..21b92fe0 100755 --- a/tools/lint-and-test.sh +++ b/tools/lint-and-test.sh @@ -16,5 +16,9 @@ cargo fmt --all -- --check cargo clippy --workspace --all-targets --all-features -- -D warnings cargo build --workspace --all-features --bins cargo test --workspace --all-features +# Linted as strictly as the all-features build: code reachable only behind an +# optional feature leaves its callers dead without it, and building alone does +# not promote that to an error. +cargo clippy --workspace --all-targets --no-default-features -- -D warnings cargo build --workspace --no-default-features --bins cargo test --workspace --no-default-features From 2c8d77925db37f9f65f82dbcfc718f0fb31a78af Mon Sep 17 00:00:00 2001 From: devmobasa <4170275+devmobasa@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:34:27 +0200 Subject: [PATCH 3/3] fix(ocr): address toolbar and broker regressions --- config.example.toml | 6 +- src/backend/wayland/handlers/pointer/press.rs | 2 +- src/backend/wayland/handlers/tablet/tool.rs | 2 +- src/backend/wayland/handlers/touch.rs | 2 +- src/backend/wayland/state/ocr.rs | 21 ++++ src/backend/wayland/state/toolbar/events.rs | 2 +- src/config/types/toolbar/items.rs | 34 ++++- src/config/types/toolbar/items/tests.rs | 48 ++++++- src/input/state/core/base/state/init.rs | 1 + src/input/state/core/base/state/structs.rs | 4 + src/input/state/core/ocr.rs | 33 +++++ src/input/state/core/tool_controls/toolbar.rs | 4 +- src/input/state/core/utility/launcher.rs | 50 ++++++-- src/process_broker/client.rs | 118 ++++++++++++++++-- src/process_broker/mod.rs | 2 +- src/process_broker/tests.rs | 65 ++++++++++ 16 files changed, 359 insertions(+), 35 deletions(-) diff --git a/config.example.toml b/config.example.toml index a800b5bf..175108fe 100644 --- a/config.example.toml +++ b/config.example.toml @@ -631,12 +631,14 @@ rebind_modifier = "ctrl_shift" [ui.toolbar.items] # Checked in the configurator means shown; IDs listed here are hidden. -# The screenshot and copy-text (OCR) toolbar buttons are hidden by default. +# The screenshot toolbar button is hidden by default. +# Copy text (OCR) is hidden by a built-in baseline instead of by this list, +# so upgrading a config written before it existed cannot reveal it; enable it +# through toolbar customization, which records an entry in `shown` below. # Section-level ids (side.group.*) act as explicit overrides that beat the # layout-mode baseline and survive mode switches. hidden = [ "top.utility.screenshot", - "top.utility.ocr", ] # IDs explicitly shown, overriding the layout-mode baseline (for example diff --git a/src/backend/wayland/handlers/pointer/press.rs b/src/backend/wayland/handlers/pointer/press.rs index 38b94592..ada2f717 100644 --- a/src/backend/wayland/handlers/pointer/press.rs +++ b/src/backend/wayland/handlers/pointer/press.rs @@ -42,7 +42,7 @@ impl WaylandState { if on_toolbar || self.pointer_over_toolbar() { // A toolbar interaction ends the region first, then runs // normally; the click never lands on the selector. - self.cancel_ocr(); + self.cancel_ocr_for_toolbar_interaction(); } else { match button { BTN_LEFT => { diff --git a/src/backend/wayland/handlers/tablet/tool.rs b/src/backend/wayland/handlers/tablet/tool.rs index e1b93bd6..1c276957 100644 --- a/src/backend/wayland/handlers/tablet/tool.rs +++ b/src/backend/wayland/handlers/tablet/tool.rs @@ -245,7 +245,7 @@ impl Dispatch for WaylandState { } if state.input_state.ocr_is_active() { if state.stylus_on_toolbar { - state.cancel_ocr(); + state.cancel_ocr_for_toolbar_interaction(); } else if state.stylus_on_overlay { let (x, y) = state.current_or_pending_stylus_position(); state.begin_ocr_selection(OcrInputSource::Stylus, x, y); diff --git a/src/backend/wayland/handlers/touch.rs b/src/backend/wayland/handlers/touch.rs index d63d0799..3818034c 100644 --- a/src/backend/wayland/handlers/touch.rs +++ b/src/backend/wayland/handlers/touch.rs @@ -215,7 +215,7 @@ impl WaylandState { && inline_active && self.inline_toolbar_motion(screen_position); if target == TouchTarget::Toolbar || inline_hit { - self.cancel_ocr(); + self.cancel_ocr_for_toolbar_interaction(); } else if target == TouchTarget::Overlay { self.begin_ocr_selection( OcrInputSource::Touch, diff --git a/src/backend/wayland/state/ocr.rs b/src/backend/wayland/state/ocr.rs index 4f2ea6bc..1ac8ee75 100644 --- a/src/backend/wayland/state/ocr.rs +++ b/src/backend/wayland/state/ocr.rs @@ -23,6 +23,8 @@ const TOAST_SOURCE: &str = "ocr"; impl WaylandState { /// Drain a `Copy text from screen` request into the region selector. pub(in crate::backend::wayland) fn handle_pending_ocr_request(&mut self) { + // Read unconditionally so the latch never outlives the batch that set it. + let dismissed_by_toolbar = self.input_state.take_ocr_cancelled_by_toolbar(); if !self.input_state.take_pending_ocr_request() { return; } @@ -32,6 +34,11 @@ impl WaylandState { self.cancel_ocr(); return; } + if dismissed_by_toolbar { + // The toolbar path already cancelled the selector on the way in, so + // this request is the same click toggling it off. + return; + } if !self.config.capture.enabled { self.input_state.push_toast( ToastPriority::Info, @@ -174,6 +181,20 @@ impl WaylandState { } } + /// Dismiss the selector because a toolbar interaction took over. + /// + /// Recorded rather than just cancelled: when the interaction is a click on + /// the OCR button itself, the request that follows is the button toggling + /// off, and without this the pending handler would see `Inactive` and open a + /// fresh selector instead. + pub(in crate::backend::wayland) fn cancel_ocr_for_toolbar_interaction(&mut self) -> bool { + if !self.cancel_ocr() { + return false; + } + self.input_state.note_ocr_cancelled_by_toolbar(); + true + } + /// A display or source change invalidates the pixels the user is selecting. pub(in crate::backend::wayland) fn cancel_ocr_if_source_missing(&mut self) { if self.input_state.ocr_is_active() && self.ocr_screen_source().is_none() { diff --git a/src/backend/wayland/state/toolbar/events.rs b/src/backend/wayland/state/toolbar/events.rs index 287fa62b..ae14ab49 100644 --- a/src/backend/wayland/state/toolbar/events.rs +++ b/src/backend/wayland/state/toolbar/events.rs @@ -134,7 +134,7 @@ impl WaylandState { // A toolbar interaction replaces the modal sampler. Do this before // shortcut capture so the capture modal owns subsequent keys. self.cancel_eyedropper(); - self.cancel_ocr(); + self.cancel_ocr_for_toolbar_interaction(); if rebind_requested && let Some(action) = crate::ui::toolbar::model::action_for_event(&event) { diff --git a/src/config/types/toolbar/items.rs b/src/config/types/toolbar/items.rs index afbc28de..73afe923 100644 --- a/src/config/types/toolbar/items.rs +++ b/src/config/types/toolbar/items.rs @@ -52,8 +52,22 @@ fn drain_without(raw_ids: Vec, id: ToolbarItemId) -> Vec { next } -const DEFAULT_HIDDEN_TOOLBAR_ITEM_IDS: &[ToolbarItemId] = - &[ids::TOP_UTILITY_SCREENSHOT, ids::TOP_UTILITY_OCR]; +const DEFAULT_HIDDEN_TOOLBAR_ITEM_IDS: &[ToolbarItemId] = &[ids::TOP_UTILITY_SCREENSHOT]; + +/// Items hidden by a code-level baseline rather than by the shipped `hidden` +/// list, overridable with an explicit `shown` entry. +/// +/// A `hidden` list authored before an item existed cannot name it, and serde +/// applies the struct default only to a *missing* field — so shipping a new +/// default-hidden id reaches new configs and silently exposes the item on every +/// installed one. Screenshot stays list-driven because every existing config +/// already names it; anything added afterwards belongs here instead. +const BASELINE_HIDDEN_TOOLBAR_ITEM_IDS: &[ToolbarItemId] = &[ids::TOP_UTILITY_OCR]; + +/// Whether `id` is hidden unless the user explicitly shows it. +pub(crate) fn toolbar_item_hidden_by_baseline(id: ToolbarItemId) -> bool { + BASELINE_HIDDEN_TOOLBAR_ITEM_IDS.contains(&id) +} const DEFAULT_TOP_TOOLS_ORDER: &[ToolbarItemId] = &[ ids::TOP_TOOL_SELECT, @@ -135,9 +149,12 @@ impl ToolbarItemsConfig { self.shown = drain_without(std::mem::take(&mut self.shown), id); if hidden { self.hidden.push(id.as_str().to_string()); - } else if super::visibility::section_flag_for_item(id).is_some() { - // Section-level ids have a layout-mode baseline that may hide - // them again; an explicit "shown" entry pins them visible. + } else if super::visibility::section_flag_for_item(id).is_some() + || toolbar_item_hidden_by_baseline(id) + { + // These have a baseline that would hide them again — a layout mode + // for section ids, a code-level default for the rest — so an + // explicit "shown" entry is what pins them visible. self.shown.push(id.as_str().to_string()); } } @@ -259,7 +276,12 @@ pub struct ResolvedToolbarItems { impl ResolvedToolbarItems { pub fn is_hidden(&self, id: ToolbarItemId) -> bool { - self.hidden.contains(&id) + if self.hidden.contains(&id) { + return true; + } + // A baseline-hidden item needs an explicit `shown` entry, so upgrading + // a config written before the item existed cannot reveal it. + toolbar_item_hidden_by_baseline(id) && !self.shown.contains(&id) } } diff --git a/src/config/types/toolbar/items/tests.rs b/src/config/types/toolbar/items/tests.rs index 433a28a1..305e9601 100644 --- a/src/config/types/toolbar/items/tests.rs +++ b/src/config/types/toolbar/items/tests.rs @@ -47,14 +47,59 @@ fn the_ocr_utility_is_defined_once_ordered_and_hidden_by_default() { 1 ); + // Hidden by the code-level baseline, which reads as no explicit setting: + // nothing is written to `hidden`, so a config that predates the item still + // resolves as hiding it. let resolved = ToolbarItemsConfig::default().resolved(); assert!(resolved.is_hidden(ids::TOP_UTILITY_OCR)); assert_eq!( item_visibility_setting(&resolved, ids::TOP_UTILITY_OCR), - ToolbarItemVisibilitySetting::Hidden + ToolbarItemVisibilitySetting::Default + ); + assert!( + !ToolbarItemsConfig::default() + .hidden + .iter() + .any(|raw| raw == ids::TOP_UTILITY_OCR.as_str()) + ); +} + +/// The regression the baseline exists for: an install predating OCR carries its +/// own `hidden` list, and serde applies the struct default only to a *missing* +/// field — so a shipped default-hidden entry would never reach it. +#[test] +fn a_config_written_before_ocr_existed_still_hides_it() { + let upgraded: ToolbarItemsConfig = + toml::from_str("hidden = [\"top.utility.screenshot\"]\nshown = []\n") + .expect("previous-release toolbar items parse"); + + let resolved = upgraded.resolved(); + assert!(resolved.is_hidden(ids::TOP_UTILITY_SCREENSHOT)); + assert!( + resolved.is_hidden(ids::TOP_UTILITY_OCR), + "upgrading exposed the opt-in OCR button" ); } +/// Turning it on has to stick, which needs an explicit `shown` entry: merely +/// dropping it from `hidden` would leave the baseline hiding it again. +#[test] +fn showing_the_ocr_utility_records_it_as_explicitly_shown() { + let mut config = ToolbarItemsConfig::default(); + config.set_hidden(ids::TOP_UTILITY_OCR, false); + + assert!( + config + .shown + .iter() + .any(|raw| raw == ids::TOP_UTILITY_OCR.as_str()) + ); + assert!(!config.resolved().is_hidden(ids::TOP_UTILITY_OCR)); + + config.set_hidden(ids::TOP_UTILITY_OCR, true); + assert!(config.resolved().is_hidden(ids::TOP_UTILITY_OCR)); +} + #[test] fn showing_the_ocr_utility_is_an_explicit_customization_that_survives_reads() { let mut config = ToolbarItemsConfig::default(); @@ -153,7 +198,6 @@ fn reset_known_hidden_restores_defaults_and_preserves_unknown_ids() { config.hidden, vec![ ids::TOP_UTILITY_SCREENSHOT.as_str().to_string(), - ids::TOP_UTILITY_OCR.as_str().to_string(), "future.toolbar.item".to_string() ] ); diff --git a/src/input/state/core/base/state/init.rs b/src/input/state/core/base/state/init.rs index e93267e4..8274ed6c 100644 --- a/src/input/state/core/base/state/init.rs +++ b/src/input/state/core/base/state/init.rs @@ -294,6 +294,7 @@ impl InputState { pending_eyedropper_toggle: false, ocr_ui_state: crate::input::state::core::OcrUiState::Inactive, pending_ocr_request: false, + ocr_cancelled_by_toolbar: false, zoom_active: false, zoom_locked: false, zoom_scale: 1.0, diff --git a/src/input/state/core/base/state/structs.rs b/src/input/state/core/base/state/structs.rs index e469e521..258c1007 100644 --- a/src/input/state/core/base/state/structs.rs +++ b/src/input/state/core/base/state/structs.rs @@ -590,6 +590,10 @@ pub struct InputState { pub(in crate::input::state::core) ocr_ui_state: crate::input::state::core::OcrUiState, /// Pending `Copy text from screen` request for the Wayland backend. pub(in crate::input::state::core) pending_ocr_request: bool, + /// A toolbar interaction dismissed the selector in this input batch, so a + /// `Copy text from screen` request produced by that same interaction is the + /// button toggling itself off — not a fresh invocation to honor. + pub(in crate::input::state::core) ocr_cancelled_by_toolbar: bool, /// Whether zoom mode is currently active pub(in crate::input::state::core) zoom_active: bool, /// Whether zoom view is locked diff --git a/src/input/state/core/ocr.rs b/src/input/state/core/ocr.rs index 84dd3b20..51ffdbf3 100644 --- a/src/input/state/core/ocr.rs +++ b/src/input/state/core/ocr.rs @@ -123,6 +123,15 @@ impl InputState { std::mem::take(&mut self.pending_ocr_request) } + /// Record that a toolbar interaction dismissed the selector. + pub(crate) fn note_ocr_cancelled_by_toolbar(&mut self) { + self.ocr_cancelled_by_toolbar = true; + } + + pub(crate) fn take_ocr_cancelled_by_toolbar(&mut self) -> bool { + std::mem::take(&mut self.ocr_cancelled_by_toolbar) + } + pub fn ocr_state(&self) -> OcrUiState { self.ocr_ui_state } @@ -312,6 +321,30 @@ mod tests { assert!(!state.cancel_ocr()); } + /// Clicking the OCR toolbar button while the selector is up toggles it off. + /// + /// The toolbar paths dismiss the selector on the way in — that is what makes + /// every *other* button dismiss it — so by the time the button's own action + /// is drained the state reads `Inactive`. The latch is what tells those two + /// halves apart, and it must not survive the batch that set it. + #[test] + fn the_toolbar_cancel_latch_marks_one_batch_and_then_clears() { + let mut state = make_test_input_state(); + assert!(!state.take_ocr_cancelled_by_toolbar()); + + state.activate_ocr(true); + state.cancel_ocr(); + state.note_ocr_cancelled_by_toolbar(); + state.request_copy_text_from_screen(); + + assert!(state.take_ocr_cancelled_by_toolbar()); + assert!(state.take_pending_ocr_request()); + assert!( + !state.take_ocr_cancelled_by_toolbar(), + "the latch outlived the interaction that set it" + ); + } + /// The seat is not modal even though Wayscriber is: a pen can hover, and a /// contact retired when the selector opened keeps reporting until it lifts. /// Neither may steer a region the mouse is dragging. diff --git a/src/input/state/core/tool_controls/toolbar.rs b/src/input/state/core/tool_controls/toolbar.rs index 1c240a2d..60ea75da 100644 --- a/src/input/state/core/tool_controls/toolbar.rs +++ b/src/input/state/core/tool_controls/toolbar.rs @@ -590,7 +590,9 @@ mod tests { let resolved = state.toolbar_items.resolved(); assert!(resolved.hidden.contains(&ids::TOP_UTILITY_SCREENSHOT)); - assert!(resolved.hidden.contains(&ids::TOP_UTILITY_OCR)); + // Restored to its baseline rather than to an explicit entry. + assert!(resolved.is_hidden(ids::TOP_UTILITY_OCR)); + assert!(!resolved.hidden.contains(&ids::TOP_UTILITY_OCR)); assert!(!resolved.hidden.contains(&ids::TOP_TOOL_PEN)); assert!(resolved.hidden.contains(§ion)); assert!(resolved.hidden.contains(&ids::TOP_CHROME_OVERFLOW)); diff --git a/src/input/state/core/utility/launcher.rs b/src/input/state/core/utility/launcher.rs index 373e817c..3f081ed7 100644 --- a/src/input/state/core/utility/launcher.rs +++ b/src/input/state/core/utility/launcher.rs @@ -5,12 +5,18 @@ use crate::env_vars::CONFIGURATOR_ENV; use crate::input::state::{Toast, ToastPriority}; use std::ffi::{OsStr, OsString}; +/// Launch a helper from the Wayland callback thread. +/// +/// Deliberately the non-blocking spawn: every launcher here runs inside event +/// dispatch, so waiting for the broker transport would stall input and redraw +/// for the length of an unrelated helper. A contended launch is reported and +/// the user retries. fn spawn_detached( kind: crate::process_broker::HelperKind, program: &OsStr, arguments: &[OsString], ) -> anyhow::Result { - crate::process_broker::current()?.spawn( + crate::process_broker::current()?.try_spawn( kind, crate::process_broker::HelperLifetime::DetachedAfterExec, program, @@ -19,6 +25,23 @@ fn spawn_detached( ) } +/// Whether a launch failed only because the broker transport was busy. +/// +/// Spawn requests give up rather than stall the event thread behind a long +/// helper, so this is a "try again in a moment", not a broken install. +fn launch_deferred_by_busy_broker(error: &anyhow::Error) -> bool { + format!("{error:#}").contains(crate::process_broker::BROKER_BUSY) +} + +/// Message for a failed launch: retryable when the transport was merely busy. +fn launch_failure_message(error: &anyhow::Error, failed: &'static str) -> &'static str { + if launch_deferred_by_busy_broker(error) { + "Busy with another task — try again in a moment." + } else { + failed + } +} + fn opener_arguments(path: &std::path::Path) -> (OsString, Vec) { if cfg!(target_os = "macos") { ("open".into(), vec![path.as_os_str().into()]) @@ -72,7 +95,10 @@ impl InputState { self.push_toast( ToastPriority::Critical, "launcher", - Toast::error("Failed to open About (see logs)."), + Toast::error(launch_failure_message( + &err, + "Failed to open About (see logs).", + )), ); } } @@ -102,8 +128,12 @@ impl InputState { log::error!("Set {CONFIGURATOR_ENV} to override the executable path if needed."); // The fallback hands the file to the desktop's default editor, // which has no concept of a configurator screen: the user still - // reaches the settings, just not the destination. - if self.open_config_file_default() { + // reaches the settings, just not the destination. A busy + // transport is not the configurator being unavailable, though — + // falling back there would open a text editor for what is really + // "try again", and would take a second run at the same busy + // transport to do it. + if !launch_deferred_by_busy_broker(&err) && self.open_config_file_default() { log::info!( "Opened config file with default application because wayscriber-configurator was unavailable" ); @@ -111,7 +141,10 @@ impl InputState { self.push_toast( ToastPriority::Critical, "launcher", - Toast::error("Failed to launch configurator (see logs)."), + Toast::error(launch_failure_message( + &err, + "Failed to launch configurator (see logs).", + )), ); } } @@ -165,7 +198,10 @@ impl InputState { self.push_toast( ToastPriority::Critical, "launcher", - Toast::error("Failed to open capture folder."), + Toast::error(launch_failure_message( + &err, + "Failed to open capture folder.", + )), ); } } @@ -206,7 +242,7 @@ impl InputState { self.push_toast( ToastPriority::Critical, "launcher", - Toast::error("Failed to open config file."), + Toast::error(launch_failure_message(&err, "Failed to open config file.")), ); false } diff --git a/src/process_broker/client.rs b/src/process_broker/client.rs index e54fa18d..174b8347 100644 --- a/src/process_broker/client.rs +++ b/src/process_broker/client.rs @@ -2,7 +2,7 @@ use std::collections::VecDeque; use std::ffi::{OsStr, OsString}; use std::os::fd::{AsRawFd, OwnedFd, RawFd}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, OnceLock, Weak}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock, TryLockError, Weak}; use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; @@ -53,6 +53,38 @@ struct RunOptions { output_mode: OutputMode, } +/// Failure text used when a caller declines to wait for the transport. +/// Callers match on it to offer "try again" rather than a generic failure. +pub(crate) const BROKER_BUSY: &str = "process broker is busy running another helper"; + +/// How a caller wants the single-socket transport acquired. +/// +/// One socket serializes every exchange, and a `Run` can hold it for as long as +/// its helper lives — thirty seconds for screen text recognition, two minutes +/// for an interactive selector. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ExchangeWait { + /// Queue until the transport is free. Everything on a worker thread, the + /// daemon, and startup: those requests have to happen, and waiting for them + /// costs nothing the user can feel. + Queue, + /// Take the transport only if it is free right now. + /// + /// For requests issued from the Wayland callback thread, where any wait at + /// all stalls input and redraw dispatch. A contended launch is reported so + /// the user can retry, which is strictly better than the UI locking up for + /// the length of somebody else's helper. + Immediate, +} + +/// The transport policy and watchdog a spawn request carries, kept together so +/// the private spawn path stays inside the argument-count lint. +#[derive(Debug, Clone, Copy)] +struct SpawnOptions { + watchdog: Option, + wait: ExchangeWait, +} + static ACTIVE_BROKER: OnceLock>> = OnceLock::new(); fn active_slot() -> &'static Mutex> { @@ -150,8 +182,26 @@ impl ProcessBrokerGuard { } impl ProcessBroker { + fn lock_exchange(&self, wait: ExchangeWait) -> Result> { + match wait { + ExchangeWait::Queue => Ok(self + .inner + .exchange_lock + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner())), + // One attempt, never a timed retry: sleeping here would be the very + // stall this policy exists to avoid. + ExchangeWait::Immediate => match self.inner.exchange_lock.try_lock() { + Ok(guard) => Ok(guard), + Err(TryLockError::Poisoned(poisoned)) => Ok(poisoned.into_inner()), + Err(TryLockError::WouldBlock) => bail!(BROKER_BUSY), + }, + } + } + pub(super) fn request(&self, operation: BrokerOperation) -> Result { - let (outcome, descriptors) = self.request_with_descriptors(operation, &[])?; + let (outcome, descriptors) = + self.request_with_descriptors(operation, &[], ExchangeWait::Queue)?; if !descriptors.is_empty() { bail!("broker returned unexpected descriptors"); } @@ -162,6 +212,7 @@ impl ProcessBroker { &self, operation: BrokerOperation, descriptors: &[RawFd], + wait: ExchangeWait, ) -> Result<(BrokerOutcome, Vec)> { let request_id = crate::daemon::protocol_v2::ProtocolId::generate()?.to_string(); let exchange_timeout = broker_exchange_timeout(&operation); @@ -173,11 +224,7 @@ impl ProcessBroker { if packet.len() > MAX_PACKET_BYTES { bail!("broker request exceeds packet cap"); } - let _guard = self - .inner - .exchange_lock - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); + let _guard = self.lock_exchange(wait)?; if !self.inner.healthy.load(Ordering::Acquire) { bail!("process broker transport is no longer usable"); } @@ -290,6 +337,7 @@ impl ProcessBroker { output_mode: options.output_mode, }, &request_descriptors, + ExchangeWait::Queue, )?; match outcome { BrokerOutcome::Output { @@ -350,6 +398,7 @@ impl ProcessBroker { timeout_ms: u64::try_from(timeout.as_millis()).unwrap_or(u64::MAX), }, &request_descriptors, + ExchangeWait::Queue, )?; if !descriptors.is_empty() { bail!("broker returned unexpected publication descriptors"); @@ -384,7 +433,49 @@ impl ProcessBroker { I: IntoIterator, S: AsRef, { - self.spawn_inner(kind, lifetime, program, arguments, environment, None) + self.spawn_inner( + kind, + lifetime, + program, + arguments, + environment, + SpawnOptions { + watchdog: None, + wait: ExchangeWait::Queue, + }, + ) + } + + /// Spawn without waiting for the transport, failing with [`BROKER_BUSY`] + /// when another exchange holds it. + /// + /// For the Wayland callback thread only. Everything else — the daemon's + /// overlay spawn, the tray, startup detach — must keep queueing: those + /// requests have to happen, and failing them behind an unrelated helper + /// would turn a wait into a spurious error and a retry backoff. + pub(crate) fn try_spawn( + &self, + kind: HelperKind, + lifetime: HelperLifetime, + program: &OsStr, + arguments: I, + environment: Vec<(OsString, Option)>, + ) -> Result + where + I: IntoIterator, + S: AsRef, + { + self.spawn_inner( + kind, + lifetime, + program, + arguments, + environment, + SpawnOptions { + watchdog: None, + wait: ExchangeWait::Immediate, + }, + ) } pub(crate) fn spawn_with_watchdog( @@ -406,7 +497,10 @@ impl ProcessBroker { program, arguments, environment, - Some(watchdog), + SpawnOptions { + watchdog: Some(watchdog), + wait: ExchangeWait::Queue, + }, ) } @@ -417,7 +511,7 @@ impl ProcessBroker { program: &OsStr, arguments: I, environment: Vec<(OsString, Option)>, - watchdog: Option, + options: SpawnOptions, ) -> Result where I: IntoIterator, @@ -426,7 +520,7 @@ impl ProcessBroker { let operation = BrokerOperation::Spawn { kind, lifetime, - watchdog: watchdog.is_some(), + watchdog: options.watchdog.is_some(), program: OsWire::from_os(program)?, arguments: arguments .into_iter() @@ -443,7 +537,7 @@ impl ProcessBroker { .collect::>>()?, }; let (outcome, descriptors) = - self.request_with_descriptors(operation, watchdog.as_slice())?; + self.request_with_descriptors(operation, options.watchdog.as_slice(), options.wait)?; if !descriptors.is_empty() { bail!("broker returned unexpected spawn descriptors"); } diff --git a/src/process_broker/mod.rs b/src/process_broker/mod.rs index 4ce84c41..fa7c998c 100644 --- a/src/process_broker/mod.rs +++ b/src/process_broker/mod.rs @@ -15,6 +15,6 @@ mod wire; #[cfg(test)] mod tests; -pub(crate) use client::{BrokerChild, current, start_for_runtime}; +pub(crate) use client::{BROKER_BUSY, BrokerChild, current, start_for_runtime}; pub(crate) use server::run_internal_broker_if_requested; pub(crate) use wire::{BrokerOutput, HelperKind, HelperLifetime, STDOUT_CAP_EXCEEDED}; diff --git a/src/process_broker/tests.rs b/src/process_broker/tests.rs index 6146f839..a7b84256 100644 --- a/src/process_broker/tests.rs +++ b/src/process_broker/tests.rs @@ -112,6 +112,71 @@ fn tesseract_reads_the_complete_recognized_output_rather_than_a_prefix() { ); } +/// A launcher click must not stall behind a long helper, and nothing else may +/// be made to fail by that policy. +/// +/// `try_spawn` is issued from the Wayland callback thread while OCR keeps the +/// overlay interactive, so queueing behind a thirty-second recognition would +/// hold input and redraw dispatch for its whole run. Plain `spawn` — the +/// daemon's overlay start, the tray, startup detach — must keep queueing, or an +/// unrelated background helper would turn a short wait into a spurious failure +/// and a retry backoff. +#[test] +fn only_the_callback_thread_spawn_declines_to_wait_for_the_transport() { + let guard = start_for_runtime().unwrap(); + let broker = guard.broker().clone(); + let (running_tx, running_rx) = std::sync::mpsc::channel(); + + let long_helper = std::thread::spawn(move || { + running_tx.send(()).unwrap(); + broker.run( + HelperKind::TestSleep, + OsStr::new("sleep"), + [OsStr::new("1")], + Vec::new(), + Duration::from_secs(10), + 1024, + ) + }); + running_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + // The helper thread has to hold the exchange before the policy is testable. + std::thread::sleep(Duration::from_millis(200)); + + let started = Instant::now(); + let declined = guard.broker().try_spawn( + HelperKind::TestSleep, + HelperLifetime::OperationBound, + OsStr::new("sleep"), + [OsStr::new("0")], + Vec::new(), + ); + let waited = started.elapsed(); + + let error = declined.expect_err("try_spawn must not queue behind the long helper"); + assert!( + format!("{error:#}").contains(crate::process_broker::BROKER_BUSY), + "unexpected try_spawn failure: {error:#}" + ); + assert!( + waited < Duration::from_millis(250), + "try_spawn slept for {waited:?} instead of declining" + ); + + // The queueing path waits it out and still succeeds. + guard + .broker() + .spawn( + HelperKind::TestSleep, + HelperLifetime::OperationBound, + OsStr::new("sleep"), + [OsStr::new("0")], + Vec::new(), + ) + .expect("spawn must queue behind the long helper rather than fail"); + + let _ = long_helper.join(); +} + #[test] fn prelock_broker_runs_bounded_helpers_and_owns_reaping() { let guard = start_for_runtime().unwrap();