Skip to content

feat: LLM post-processing for transcriptions #113

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
charlesdubt wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: LLM post-processing for transcriptions #113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 4 additions & 12 deletions speaktype.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
/* Begin PBXBuildFile section */
1C547F2B2F0E7DC0008120A6 /* WhisperKit in Frameworks */ = {isa = PBXBuildFile; productRef = 1C547F2A2F0E7DC0008120A6 /* WhisperKit */; };
1CE4CB3F2F0EF6AF00C01C62 /* KeyboardShortcuts in Frameworks */ = {isa = PBXBuildFile; productRef = 1CE4CB3E2F0EF6AF00C01C62 /* KeyboardShortcuts */; };
1CE4CB412F0EF86B00C01C62 /* whisperkit-cli in Frameworks */ = {isa = PBXBuildFile; productRef = 1CE4CB402F0EF86B00C01C62 /* whisperkit-cli */; };
FA0000000000000000000003 /* FluidAudio in Frameworks */ = {isa = PBXBuildFile; productRef = FA0000000000000000000002 /* FluidAudio */; };
/* End PBXBuildFile section */

Expand Down Expand Up @@ -74,7 +73,6 @@
files = (
1C547F2B2F0E7DC0008120A6 /* WhisperKit in Frameworks */,
1CE4CB3F2F0EF6AF00C01C62 /* KeyboardShortcuts in Frameworks */,
1CE4CB412F0EF86B00C01C62 /* whisperkit-cli in Frameworks */,
FA0000000000000000000003 /* FluidAudio in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
Expand Down Expand Up @@ -146,7 +144,6 @@
packageProductDependencies = (
1C547F2A2F0E7DC0008120A6 /* WhisperKit */,
1CE4CB3E2F0EF6AF00C01C62 /* KeyboardShortcuts */,
1CE4CB402F0EF86B00C01C62 /* whisperkit-cli */,
FA0000000000000000000002 /* FluidAudio */,
);
productName = speaktype;
Expand Down Expand Up @@ -431,19 +428,19 @@
362947442F0DBC5100BE895D /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
APP_DISPLAY_NAME = SpeakType;
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
CODE_SIGN_ENTITLEMENTS = speaktype/Resources/speaktype.entitlements;
CODE_SIGN_STYLE = Automatic;
COMBINE_HIDPI_IMAGES = YES;
CURRENT_PROJECT_VERSION = 27;
DEVELOPMENT_TEAM = PCV4UMSRZX;
DEVELOPMENT_TEAM = ZJ8C58LQA4;
ENABLE_APP_SANDBOX = NO;
ENABLE_HARDENED_RUNTIME = YES;
ENABLE_PREVIEWS = YES;
ENABLE_USER_SELECTED_FILES = readonly;
GENERATE_INFOPLIST_FILE = NO;
APP_DISPLAY_NAME = SpeakType;
INFOPLIST_FILE = speaktype/Resources/Info.plist;
INFOPLIST_KEY_NSHumanReadableCopyright = "";
LD_RUNPATH_SEARCH_PATHS = (
Expand All @@ -466,19 +463,19 @@
362947452F0DBC5100BE895D /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
APP_DISPLAY_NAME = SpeakType;
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
CODE_SIGN_ENTITLEMENTS = speaktype/Resources/speaktype.entitlements;
CODE_SIGN_STYLE = Automatic;
COMBINE_HIDPI_IMAGES = YES;
CURRENT_PROJECT_VERSION = 27;
DEVELOPMENT_TEAM = PCV4UMSRZX;
DEVELOPMENT_TEAM = ZJ8C58LQA4;
ENABLE_APP_SANDBOX = NO;
ENABLE_HARDENED_RUNTIME = YES;
ENABLE_PREVIEWS = YES;
ENABLE_USER_SELECTED_FILES = readonly;
GENERATE_INFOPLIST_FILE = NO;
APP_DISPLAY_NAME = SpeakType;
INFOPLIST_FILE = speaktype/Resources/Info.plist;
INFOPLIST_KEY_NSHumanReadableCopyright = "";
LD_RUNPATH_SEARCH_PATHS = (
Expand Down Expand Up @@ -653,11 +650,6 @@
package = 1CE4CB3D2F0EF6AF00C01C62 /* XCRemoteSwiftPackageReference "KeyboardShortcuts" */;
productName = KeyboardShortcuts;
};
1CE4CB402F0EF86B00C01C62 /* whisperkit-cli */ = {
isa = XCSwiftPackageProductDependency;
package = 1C547F222F0E7D07008120A6 /* XCRemoteSwiftPackageReference "WhisperKit" */;
productName = "whisperkit-cli";
};
FA0000000000000000000002 /* FluidAudio */ = {
isa = XCSwiftPackageProductDependency;
package = FA0000000000000000000001 /* XCRemoteSwiftPackageReference "FluidAudio" */;
Expand Down
127 changes: 127 additions & 0 deletions speaktype/Services/PostProcessingService.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
import Foundation

@Observable
class PostProcessingService {
static let shared = PostProcessingService()

private(set) var isProcessing = false

private static let enabledKey = "postProcessingEnabled"
private static let promptKey = "postProcessingPrompt"
private static let baseURLKey = "postProcessingBaseURL"
private static let modelKey = "postProcessingModel"
private static let keychainService = "sh.polar.speaktype.postprocessing"
private static let keychainAccount = "api_key"

static let defaultPrompt =
"You are a post-processor for voice dictation. Fix typos, remove filler words, correct grammar, and clean up the text while preserving the original meaning. Output ONLY the corrected text with no explanation."
static let defaultBaseURL = "http://localhost:11434/v1"
static let defaultModel = "qwen2.5:0.5b"

var isEnabled: Bool {
get { UserDefaults.standard.bool(forKey: Self.enabledKey) }
set { UserDefaults.standard.set(newValue, forKey: Self.enabledKey) }
}

var prompt: String {
get { UserDefaults.standard.string(forKey: Self.promptKey) ?? Self.defaultPrompt }
set { UserDefaults.standard.set(newValue, forKey: Self.promptKey) }
}

var baseURL: String {
get { UserDefaults.standard.string(forKey: Self.baseURLKey) ?? Self.defaultBaseURL }
set { UserDefaults.standard.set(newValue, forKey: Self.baseURLKey) }
}

var model: String {
get { UserDefaults.standard.string(forKey: Self.modelKey) ?? Self.defaultModel }
set { UserDefaults.standard.set(newValue, forKey: Self.modelKey) }
}

var apiKey: String {
get {
let query: [CFString: Any] = [
kSecClass: kSecClassGenericPassword,
kSecAttrService: Self.keychainService,
kSecAttrAccount: Self.keychainAccount,
kSecReturnData: true,
kSecMatchLimit: kSecMatchLimitOne,
]
var result: AnyObject?
let status = SecItemCopyMatching(query as CFDictionary, &result)
guard status == errSecSuccess, let data = result as? Data,
let key = String(data: data, encoding: .utf8)
else { return "" }
return key
}
set {
let deleteQuery: [CFString: Any] = [
kSecClass: kSecClassGenericPassword,
kSecAttrService: Self.keychainService,
kSecAttrAccount: Self.keychainAccount,
]
SecItemDelete(deleteQuery as CFDictionary)

guard !newValue.isEmpty, let data = newValue.data(using: .utf8) else { return }
let addQuery: [CFString: Any] = [
kSecClass: kSecClassGenericPassword,
kSecAttrService: Self.keychainService,
kSecAttrAccount: Self.keychainAccount,
kSecValueData: data,
kSecAttrAccessible: kSecAttrAccessibleAfterFirstUnlock,
]
SecItemAdd(addQuery as CFDictionary, nil)
}
}

private init() {}

func process(_ text: String) async throws -> String {
guard isEnabled else { return text }

let trimmedBase = baseURL.trimmingCharacters(in: CharacterSet(charactersIn: "/"))
guard let url = URL(string: "\(trimmedBase)/chat/completions") else { return text }

let body: [String: Any] = [
"model": model,
"messages": [
["role": "system", "content": prompt],
["role": "user", "content": text],
],
"temperature": 0.3,
]

var request = URLRequest(url: url)
request.httpMethod = "POST"
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
let key = apiKey
if !key.isEmpty {
request.setValue("Bearer \(key)", forHTTPHeaderField: "Authorization")
}
request.httpBody = try JSONSerialization.data(withJSONObject: body)
request.timeoutInterval = 30

isProcessing = true
defer { isProcessing = false }

let (data, response) = try await URLSession.shared.data(for: request)

guard let httpResponse = response as? HTTPURLResponse,
(200...299).contains(httpResponse.statusCode)
else {
return text
}

guard let json = try JSONSerialization.jsonObject(with: data) as? [String: Any],
let choices = json["choices"] as? [[String: Any]],
let first = choices.first,
let message = first["message"] as? [String: Any],
let content = message["content"] as? String
else {
return text
}

let result = content.trimmingCharacters(in: .whitespacesAndNewlines)
return result.isEmpty ? text : result
}
}
14 changes: 13 additions & 1 deletion speaktype/Views/Overlays/MiniRecorderView.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -914,9 +914,21 @@ struct MiniRecorderView: View {
if !cancelCommit {
await MainActor.run { statusMessage = "Transcribing..." }
}
let text = try await transcription.transcribe(audioFile: url, language: transcriptionLanguage)
var text = try await transcription.transcribe(audioFile: url, language: transcriptionLanguage)
debugLog("Transcription result: \(text.prefix(50))...")

if PostProcessingService.shared.isEnabled {
if !cancelCommit {
await MainActor.run { statusMessage = "Post-processing..." }
}
do {
text = try await PostProcessingService.shared.process(text)
debugLog("Post-processing result: \(text.prefix(50))...")
} catch {
debugLog("Post-processing failed, using original: \(error.localizedDescription)")
}
}

guard !text.isEmpty else {
debugLog("Empty text, cancelling")
await MainActor.run {
Expand Down
146 changes: 146 additions & 0 deletions speaktype/Views/Screens/Settings/SettingsView.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,6 +363,9 @@ struct GeneralSettingsTab: View {
}
}

// LLM Post-Processing
PostProcessingSettingsSection()

// Spoken Language
SettingsSection {
SettingsSectionHeader(
Expand Down Expand Up @@ -556,6 +559,149 @@ struct GeneralSettingsTab: View {
]
}

// MARK: - Post-Processing Settings Section

struct PostProcessingSettingsSection: View {
private var service: PostProcessingService { PostProcessingService.shared }
@State private var isEnabled: Bool = PostProcessingService.shared.isEnabled
@State private var prompt: String = PostProcessingService.shared.prompt
@State private var baseURL: String = PostProcessingService.shared.baseURL
@State private var model: String = PostProcessingService.shared.model
@State private var apiKey: String = PostProcessingService.shared.apiKey
@State private var showAPIKey = false

var body: some View {
SettingsSection {
SettingsSectionHeader(
icon: "brain",
title: "LLM Post-Processing",
subtitle: "Run an LLM on transcriptions to fix and reformat"
)

VStack(alignment: .leading, spacing: 14) {
HStack {
Text("Enable post-processing")
.font(Typography.bodyMedium)
.foregroundStyle(Color.textPrimary)
Spacer()
Toggle("", isOn: $isEnabled)
.labelsHidden()
.onChange(of: isEnabled) { service.isEnabled = isEnabled }
}

Text(
"After transcription, sends the text to an LLM to fix typos, remove filler words, and clean up formatting. Defaults to Ollama (local) — install it from ollama.com and run: ollama pull qwen2.5:0.5b"
)
.font(Typography.captionSmall)
.foregroundStyle(Color.textMuted)

if isEnabled {
Divider()

VStack(alignment: .leading, spacing: 6) {
Text("API Base URL")
.font(Typography.bodyMedium)
.foregroundStyle(Color.textPrimary)
TextField("http://localhost:11434/v1", text: $baseURL)
.textFieldStyle(.roundedBorder)
.font(Typography.bodySmall)
.onChange(of: baseURL) { service.baseURL = baseURL }
Text(
"OpenAI-compatible endpoint. Ollama runs locally at localhost:11434. Also works with OpenAI, LM Studio, or any compatible provider."
)
.font(Typography.captionSmall)
.foregroundStyle(Color.textMuted)
}

Divider()

VStack(alignment: .leading, spacing: 6) {
Text("Model")
.font(Typography.bodyMedium)
.foregroundStyle(Color.textPrimary)
TextField("qwen2.5:0.5b", text: $model)
.textFieldStyle(.roundedBorder)
.font(Typography.bodySmall)
.onChange(of: model) { service.model = model }
Text(
"For Ollama: qwen2.5:0.5b (fast, 290 MB) or gemma3:1b (better quality, 600 MB). For OpenAI: gpt-4o-mini."
)
.font(Typography.captionSmall)
.foregroundStyle(Color.textMuted)
}

Divider()

VStack(alignment: .leading, spacing: 6) {
Text("API Key (optional)")
.font(Typography.bodyMedium)
.foregroundStyle(Color.textPrimary)
HStack(spacing: 8) {
if showAPIKey {
TextField("Not needed for Ollama", text: $apiKey)
.textFieldStyle(.roundedBorder)
.font(Typography.bodySmall)
.onChange(of: apiKey) { service.apiKey = apiKey }
} else {
SecureField("Not needed for Ollama", text: $apiKey)
.textFieldStyle(.roundedBorder)
.font(Typography.bodySmall)
.onChange(of: apiKey) { service.apiKey = apiKey }
}
Button {
showAPIKey.toggle()
} label: {
Image(systemName: showAPIKey ? "eye.slash" : "eye")
.font(.system(size: 12))
.foregroundStyle(Color.textMuted)
}
.buttonStyle(.plain)
}
Text("Only needed for cloud providers like OpenAI. Leave empty for local Ollama.")
.font(Typography.captionSmall)
.foregroundStyle(Color.textMuted)
}

Divider()

VStack(alignment: .leading, spacing: 6) {
HStack {
Text("System Prompt")
.font(Typography.bodyMedium)
.foregroundStyle(Color.textPrimary)
Spacer()
Button("Reset to Default") {
prompt = PostProcessingService.defaultPrompt
service.prompt = prompt
}
.font(Typography.captionSmall)
.foregroundStyle(Color.textMuted)
.buttonStyle(.plain)
}
TextEditor(text: $prompt)
.font(Typography.bodySmall)
.frame(minHeight: 80, maxHeight: 160)
.scrollContentBackground(.hidden)
.padding(8)
.background(Color.bgHover)
.clipShape(RoundedRectangle(cornerRadius: 6))
.overlay(
RoundedRectangle(cornerRadius: 6)
.stroke(Color.border.opacity(0.5), lineWidth: 1)
)
.onChange(of: prompt) { service.prompt = prompt }
Text(
"The transcribed text is sent as the user message. Your prompt is the system instruction that tells the LLM how to reformat it."
)
.font(Typography.captionSmall)
.foregroundStyle(Color.textMuted)
}
}
}
}
}
}

// MARK: - Audio Settings Tab

struct AudioSettingsTab: View {
Expand Down