ai-tools/wisper-key
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Диктовка: нативный подчёркнутый черновик

Browse Source
This commit is contained in:
romanv
2026-07-29 15:09:17 +06:00
parent 5751743277
commit 81dc5db6e4
11 changed files with 607 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Sources/WhisperKey/AppDelegate.swift
+98 -10
View File
@@ -7,19 +7,28 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
private struct PendingChunk {
let url: URL
let continuesPhrase: Bool
let isFinalSession: Bool
}
private let statusItem = NSStatusBar.system.statusItem(withLength: NSStatusItem.variableLength)
private let recorder = AudioRecorder()
private let hotKeys = HotKeyManager()
private let composition = NativeCompositionBridge()
private let settings = AppSettings.shared
private var activeMode: TranscriptionMode?
private var startingSession = false
private var pendingChunks: [PendingChunk] = []
private var processingChunk = false
private var stopRequested = false
private var stitchNextChunk = false
private var nativeSession = false
private var nativeDraft = ""
func applicationDidFinishLaunching(_ notification: Notification) {
composition.onAvailabilityChange = { [weak self] in
self?.rebuildMenu()
}
composition.start()
configureStatusItem()
requestPermissions()
hotKeys.onLocal = { [weak self] in Task { @MainActor in await self?.toggle(.local) } }
@@ -64,6 +73,15 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
menu.addItem(.separator())
menu.addItem(item("Настройки…", #selector(openSettings)))
menu.addItem(item("Проверить локальный движок", #selector(checkLocalEngine)))
let nativeState: String
if composition.isAvailable {
nativeState = "Нативный ввод — активен"
} else if composition.isInstalled {
nativeState = "Нативный ввод — готов"
} else {
nativeState = "Установить нативный ввод…"
}
menu.addItem(item(nativeState, #selector(openInputSources)))
menu.addItem(.separator())
menu.addItem(item("Выход", #selector(quit), key: "q"))
statusItem.menu = menu
@@ -91,6 +109,15 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
await stopAndTranscribe()
return
}
guard !startingSession else { return }
startingSession = true
defer { startingSession = false }
var nativeSelected = composition.activateForDictation()
if !nativeSelected {
try? await Task.sleep(for: .milliseconds(250))
nativeSelected = composition.activateForDictation()
}
try? await Task.sleep(for: .milliseconds(180))
do {
activeMode = mode
lastMode = mode
@@ -98,7 +125,15 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
processingChunk = false
stopRequested = false
stitchNextChunk = false
nativeDraft = ""
nativeSession = nativeSelected && composition.isAvailable
if nativeSession {
composition.cancel(restoreInputSource: false)
} else {
composition.restoreInputSource()
}
try recorder.start(
chunkTiming: nativeSession ? .nativeDraft : .committed,
onChunk: { [weak self] chunk in
Task { @MainActor in self?.enqueue(chunk) }
},
@@ -125,20 +160,39 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
DictationSounds.playStop()
setStatus("обрабатываю…")
do {
if let final = try recorder.stop() {
enqueue(RecordedChunk(url: final, continuesPhrase: false))
let result = try recorder.stop()
if nativeSession, let fullSession = result.fullSessionURL {
if let finalChunk = result.finalChunkURL {
try? FileManager.default.removeItem(at: finalChunk)
}
enqueue(
RecordedChunk(url: fullSession, continuesPhrase: false),
isFinalSession: true
)
} else {
if let fullSession = result.fullSessionURL {
try? FileManager.default.removeItem(at: fullSession)
}
if let finalChunk = result.finalChunkURL {
enqueue(RecordedChunk(url: finalChunk, continuesPhrase: false))
}
}
if pendingChunks.isEmpty, !processingChunk {
if nativeSession { composition.cancel() }
completeLiveSession()
}
if pendingChunks.isEmpty, !processingChunk { completeLiveSession() }
} catch {
composition.cancel()
setStatus("ошибка")
showError("Диктовка не удалась", error.localizedDescription)
}
}
private func enqueue(_ chunk: RecordedChunk) {
private func enqueue(_ chunk: RecordedChunk, isFinalSession: Bool = false) {
pendingChunks.append(PendingChunk(
url: chunk.url,
continuesPhrase: chunk.continuesPhrase
continuesPhrase: chunk.continuesPhrase,
isFinalSession: isFinalSession
))
processNextChunk()
}
@@ -157,7 +211,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
Task {
do {
let wav = try AudioConverter.toWhisperWAV(recording)
saveDiagnosticRecording(wav)
if chunk.isFinalSession { saveDiagnosticRecording(wav) }
defer {
try? FileManager.default.removeItem(at: recording)
try? FileManager.default.removeItem(at: wav)
@@ -168,18 +222,43 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
case .cloud: raw = try await OpenAITranscriber.transcribe(wav: wav, settings: settings)
}
let cleaned = TextCleaner.clean(raw, removeFillers: settings.removeFillers, glossary: settings.glossary)
if !cleaned.isEmpty {
if chunk.isFinalSession {
if cleaned.isEmpty {
composition.cancel()
} else if !composition.commit(cleaned + " ") {
try TextInserter.insert(cleaned + " ")
}
nativeDraft = ""
stitchNextChunk = false
} else if !cleaned.isEmpty {
let joined = TranscriptJoiner.prepare(
cleaned,
continuesPrevious: stitchNextChunk,
continuesNext: chunk.continuesPhrase,
glossary: settings.glossary
)
if !joined.isEmpty { try TextInserter.insert(joined + " ") }
if !joined.isEmpty {
if nativeSession {
nativeDraft += (nativeDraft.isEmpty ? "" : " ") + joined
_ = composition.updateDraft(nativeDraft)
} else {
try TextInserter.insert(joined + " ")
}
}
stitchNextChunk = chunk.continuesPhrase
}
} catch {
showError("Фрагмент диктовки не распознан", error.localizedDescription)
if chunk.isFinalSession, !nativeDraft.isEmpty {
if !composition.commit(nativeDraft + " ") {
try? TextInserter.insert(nativeDraft + " ")
}
}
showError(
chunk.isFinalSession
? "Финальная диктовка не распознана"
: "Фрагмент диктовки не распознан",
error.localizedDescription
)
}
processingChunk = false
if pendingChunks.isEmpty {
@@ -208,12 +287,18 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
stopRequested = false
lastMode = nil
stitchNextChunk = false
nativeSession = false
nativeDraft = ""
setStatus("готово")
}
@objc private func startLocal() { Task { await toggle(.local) } }
@objc private func startCloud() { Task { await toggle(.cloud) } }
@objc private func openSettings() { SettingsWindowController.shared.show() }
@objc private func openInputSources() {
let url = URL(string: "x-apple.systempreferences:com.apple.Keyboard-Settings.extension")!
NSWorkspace.shared.open(url)
}
@objc private func checkLocalEngine() {
do {
try LocalWhisper.validate(settings: settings)
@@ -222,7 +307,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
showError("Локальный движок не готов", error.localizedDescription)
}
}
@objc private func quit() { NSApp.terminate(nil) }
@objc private func quit() {
composition.cancel()
NSApp.terminate(nil)
}
@objc private func handleWake() {
do {
Sources/WhisperKey/Audio.swift
+67 -8
View File
@@ -6,12 +6,37 @@ struct RecordedChunk: Sendable {
let continuesPhrase: Bool
}
struct RecordingResult: Sendable {
let finalChunkURL: URL?
let fullSessionURL: URL?
}
struct ChunkTiming: Sendable {
let minimumDuration: TimeInterval
let pauseDuration: TimeInterval
let maximumDuration: TimeInterval
static let committed = ChunkTiming(
minimumDuration: 2.5,
pauseDuration: 1.0,
maximumDuration: 4.0
)
static let nativeDraft = ChunkTiming(
minimumDuration: 1.5,
pauseDuration: 0.65,
maximumDuration: 2.25
)
}
final class AudioRecorder: @unchecked Sendable {
private var engine: AVAudioEngine?
private let lock = NSLock()
private var noSpeechTimeout: DispatchWorkItem?
private var currentURL: URL?
private var outputFile: AVAudioFile?
private var sessionURL: URL?
private var sessionFile: AVAudioFile?
private var sessionFrames: AVAudioFramePosition = 0
private var currentFrames: AVAudioFramePosition = 0
private var currentChunkHasSpeech = false
private var heardSpeech = false
@@ -21,8 +46,10 @@ final class AudioRecorder: @unchecked Sendable {
private var onChunk: (@Sendable (RecordedChunk) -> Void)?
private var onSilence: (@Sendable () -> Void)?
private var recordingFormat: AVAudioFormat?
private var chunkTiming = ChunkTiming.committed
func start(
chunkTiming: ChunkTiming,
onChunk: @escaping @Sendable (RecordedChunk) -> Void,
onSilence: @escaping @Sendable () -> Void
) throws {
@@ -33,8 +60,17 @@ final class AudioRecorder: @unchecked Sendable {
let format = input.outputFormat(forBus: 0)
self.onChunk = onChunk
self.onSilence = onSilence
self.chunkTiming = chunkTiming
recordingFormat = format
try beginChunk(format: format)
do {
try beginSession(format: format)
try beginChunk(format: format)
} catch {
sessionFile = nil
if let sessionURL { try? FileManager.default.removeItem(at: sessionURL) }
self.sessionURL = nil
throw error
}
input.installTap(onBus: 0, bufferSize: 4096, format: format) { [weak self] buffer, _ in
self?.consume(buffer, format: format)
}
@@ -47,13 +83,16 @@ final class AudioRecorder: @unchecked Sendable {
engine.reset()
self.engine = nil
if let currentURL { try? FileManager.default.removeItem(at: currentURL) }
if let sessionURL { try? FileManager.default.removeItem(at: sessionURL) }
outputFile = nil
currentURL = nil
sessionFile = nil
sessionURL = nil
throw error
}
}
func stop() throws -> URL? {
func stop() throws -> RecordingResult {
noSpeechTimeout?.cancel()
noSpeechTimeout = nil
if let engine {
@@ -66,19 +105,29 @@ final class AudioRecorder: @unchecked Sendable {
}
lock.lock()
defer { lock.unlock() }
let finalURL = currentFrames > 0 && currentChunkHasSpeech ? currentURL : nil
if finalURL == nil, let currentURL {
let finalChunkURL = currentFrames > 0 && currentChunkHasSpeech ? currentURL : nil
let fullSessionURL = sessionFrames > 0 && heardSpeech ? sessionURL : nil
if finalChunkURL == nil, let currentURL {
try? FileManager.default.removeItem(at: currentURL)
}
if fullSessionURL == nil, let sessionURL {
try? FileManager.default.removeItem(at: sessionURL)
}
outputFile = nil
currentURL = nil
sessionFile = nil
sessionURL = nil
sessionFrames = 0
currentFrames = 0
currentChunkHasSpeech = false
heardSpeech = false
silentFrames = 0
chunkSilentFrames = 0
silenceTriggered = false
return finalURL
return RecordingResult(
finalChunkURL: finalChunkURL,
fullSessionURL: fullSessionURL
)
}
private func scheduleNoSpeechTimeout() {
@@ -103,12 +152,22 @@ final class AudioRecorder: @unchecked Sendable {
chunkSilentFrames = 0
}
private func beginSession(format: AVAudioFormat) throws {
let url = FileManager.default.temporaryDirectory
.appendingPathComponent("whisperkey-session-\(UUID().uuidString).caf")
sessionFile = try AVAudioFile(forWriting: url, settings: format.settings)
sessionURL = url
sessionFrames = 0
}
private func consume(_ buffer: AVAudioPCMBuffer, format: AVAudioFormat) {
lock.lock()
defer { lock.unlock() }
guard let activeOutputFile = outputFile else { return }
try? activeOutputFile.write(from: buffer)
try? sessionFile?.write(from: buffer)
currentFrames += AVAudioFramePosition(buffer.frameLength)
sessionFrames += AVAudioFramePosition(buffer.frameLength)
let rms = bufferRMS(buffer)
if rms > 0.006 {
@@ -133,9 +192,9 @@ final class AudioRecorder: @unchecked Sendable {
// Keep enough context for Whisper, but insert completed phrases while
// the user is still dictating. Prefer a short natural pause; cap a
// continuous phrase so the input never stays blank indefinitely.
let minimumChunk = AVAudioFramePosition(format.sampleRate * 2.5)
let pauseBoundary = AVAudioFramePosition(format.sampleRate * 1.0)
let maximumChunk = AVAudioFramePosition(format.sampleRate * 4.0)
let minimumChunk = AVAudioFramePosition(format.sampleRate * chunkTiming.minimumDuration)
let pauseBoundary = AVAudioFramePosition(format.sampleRate * chunkTiming.pauseDuration)
let maximumChunk = AVAudioFramePosition(format.sampleRate * chunkTiming.maximumDuration)
let reachedPause =
currentFrames >= minimumChunk && chunkSilentFrames >= pauseBoundary
let reachedMaximum = currentFrames >= maximumChunk
Sources/WhisperKey/NativeCompositionBridge.swift
+171
View File
@@ -0,0 +1,171 @@
import AppKit
import Carbon
import Foundation
@MainActor
final class NativeCompositionBridge: NSObject {
private enum Message {
static let ping = Notification.Name("com.romanv.WhisperKeyInput.ping")
static let ready = Notification.Name("com.romanv.WhisperKeyInput.ready")
static let draft = Notification.Name("com.romanv.WhisperKeyInput.draft")
static let commit = Notification.Name("com.romanv.WhisperKeyInput.commit")
static let cancel = Notification.Name("com.romanv.WhisperKeyInput.cancel")
static let textKey = "text"
static let activeKey = "active"
}
private static let inputSourceID = "com.romanv.inputmethod.WhisperKey"
private static let installedBundle = FileManager.default.homeDirectoryForCurrentUser
.appendingPathComponent("Library/Input Methods/WhisperKeyInput.app", isDirectory: true)
var onAvailabilityChange: (() -> Void)?
private var previousInputSourceID: String?
private var restoreWorkItem: DispatchWorkItem?
private(set) var inputMethodActive = false {
didSet {
if oldValue != inputMethodActive { onAvailabilityChange?() }
}
}
var isAvailable: Bool {
inputMethodActive && currentInputSourceID() == Self.inputSourceID
}
var isInstalled: Bool {
FileManager.default.fileExists(atPath: Self.installedBundle.path)
}
func start() {
DistributedNotificationCenter.default().addObserver(
self,
selector: #selector(handleReady),
name: Message.ready,
object: nil
)
registerInstalledInputMethod()
launchInputMethod()
ping()
}
func activateForDictation() -> Bool {
restoreWorkItem?.cancel()
restoreWorkItem = nil
registerInstalledInputMethod()
launchInputMethod()
guard let source = inputSource(withID: Self.inputSourceID) else { return false }
let currentID = currentInputSourceID()
if currentID != Self.inputSourceID {
previousInputSourceID = currentID
}
guard TISEnableInputSource(source) == noErr,
TISSelectInputSource(source) == noErr else {
restoreInputSource()
return false
}
ping()
return true
}
func ping() {
DistributedNotificationCenter.default().post(
name: Message.ping,
object: nil,
userInfo: nil
)
}
@discardableResult
func updateDraft(_ text: String) -> Bool {
guard isAvailable else { return false }
post(name: Message.draft, text: text)
return true
}
@discardableResult
func commit(_ text: String) -> Bool {
let available = isAvailable
if available { post(name: Message.commit, text: text) }
scheduleInputSourceRestore()
return available
}
func cancel(restoreInputSource: Bool = true) {
if inputMethodActive {
DistributedNotificationCenter.default().post(
name: Message.cancel,
object: nil,
userInfo: nil
)
}
if restoreInputSource { scheduleInputSourceRestore() }
}
func restoreInputSource() {
restoreWorkItem?.cancel()
restoreWorkItem = nil
guard let previousInputSourceID else { return }
self.previousInputSourceID = nil
guard let source = inputSource(withID: previousInputSourceID) else { return }
_ = TISSelectInputSource(source)
}
private func post(name: Notification.Name, text: String) {
DistributedNotificationCenter.default().post(
name: name,
object: nil,
userInfo: [Message.textKey: text]
)
}
@objc private func handleReady(_ notification: Notification) {
inputMethodActive = notification.userInfo?[Message.activeKey] as? Bool ?? false
}
private func registerInstalledInputMethod() {
guard isInstalled else { return }
_ = TISRegisterInputSource(Self.installedBundle as CFURL)
}
private func launchInputMethod() {
guard isInstalled else { return }
let configuration = NSWorkspace.OpenConfiguration()
configuration.activates = false
NSWorkspace.shared.openApplication(
at: Self.installedBundle,
configuration: configuration
)
}
private func scheduleInputSourceRestore() {
restoreWorkItem?.cancel()
let work = DispatchWorkItem { [weak self] in
self?.restoreInputSource()
}
restoreWorkItem = work
DispatchQueue.main.asyncAfter(deadline: .now() + 0.3, execute: work)
}
private func inputSource(withID identifier: String) -> TISInputSource? {
guard let sources = TISCreateInputSourceList(nil, true)?.takeRetainedValue()
as? [TISInputSource] else {
return nil
}
return sources.first { inputSourceID(for: $0) == identifier }
}
private func currentInputSourceID() -> String? {
guard let source = TISCopyCurrentKeyboardInputSource()?.takeRetainedValue() else { return nil }
return inputSourceID(for: source)
}
private func inputSourceID(for source: TISInputSource) -> String? {
guard let rawValue = TISGetInputSourceProperty(
source,
kTISPropertyInputSourceID
) else { return nil }
return Unmanaged<CFString>
.fromOpaque(rawValue)
.takeUnretainedValue() as String
}
}
Sources/WhisperKey/SettingsWindow.swift
+1 -1
View File
@@ -44,7 +44,7 @@ final class SettingsWindowController: NSWindowController {
grid.column(at: 0).xPlacement = .trailing
modelPopup.addItems(withTitles: ["gpt-4o-transcribe", "gpt-4o-mini-transcribe", "whisper-1"])
let note = NSTextField(wrappingLabelWithString: "F5 — локально. Option+F5 — OpenAI. Вся реплика распознаётся целиком после 5 секунд тишины. Повторное нажатие останавливает её вручную.")
let note = NSTextField(wrappingLabelWithString: "F5 — локально. Option+F5 — OpenAI. С нативным источником ввода во время речи показывается подчёркнутый черновик, а после 5 секунд тишины он заменяется финальным текстом.")
note.textColor = .secondaryLabelColor
note.translatesAutoresizingMaskIntoConstraints = false
let save = NSButton(title: "Сохранить", target: self, action: #selector(save))
Sources/WhisperKeyInput/main.swift
+172
View File
@@ -0,0 +1,172 @@
import AppKit
import Carbon
import InputMethodKit
import OSLog
private let inputLog = Logger(
subsystem: "com.romanv.inputmethod.WhisperKey",
category: "composition"
)
private enum CompositionMessage {
static let ping = Notification.Name("com.romanv.WhisperKeyInput.ping")
static let ready = Notification.Name("com.romanv.WhisperKeyInput.ready")
static let draft = Notification.Name("com.romanv.WhisperKeyInput.draft")
static let commit = Notification.Name("com.romanv.WhisperKeyInput.commit")
static let cancel = Notification.Name("com.romanv.WhisperKeyInput.cancel")
static let textKey = "text"
static let activeKey = "active"
}
private final class CompositionCoordinator: @unchecked Sendable {
static let shared = CompositionCoordinator()
weak var activeController: WhisperKeyInputController? {
didSet { announceReady() }
}
private init() {
let center = DistributedNotificationCenter.default()
center.addObserver(
self,
selector: #selector(handlePing),
name: CompositionMessage.ping,
object: nil
)
center.addObserver(
self,
selector: #selector(handleDraft),
name: CompositionMessage.draft,
object: nil
)
center.addObserver(
self,
selector: #selector(handleCommit),
name: CompositionMessage.commit,
object: nil
)
center.addObserver(
self,
selector: #selector(handleCancel),
name: CompositionMessage.cancel,
object: nil
)
}
@objc private func handlePing() {
NSLog("WhisperKeyInput: ping")
announceReady()
}
@objc private func handleDraft(_ notification: Notification) {
guard let text = notification.userInfo?[CompositionMessage.textKey] as? String else { return }
NSLog("WhisperKeyInput: draft %ld", (text as NSString).length)
activeController?.setDraft(text)
}
@objc private func handleCommit(_ notification: Notification) {
guard let text = notification.userInfo?[CompositionMessage.textKey] as? String else { return }
NSLog("WhisperKeyInput: commit %ld", (text as NSString).length)
activeController?.commit(text)
}
@objc private func handleCancel() {
NSLog("WhisperKeyInput: cancel")
activeController?.cancelDraft()
}
private func announceReady() {
DistributedNotificationCenter.default().post(
name: CompositionMessage.ready,
object: nil,
userInfo: [CompositionMessage.activeKey: activeController != nil]
)
}
}
@objc(WhisperKeyInputController)
final class WhisperKeyInputController: IMKInputController {
private var draft = ""
override init!(server: IMKServer!, delegate: Any!, client inputClient: Any!) {
super.init(server: server, delegate: delegate, client: inputClient)
_ = CompositionCoordinator.shared
NSLog("WhisperKeyInput: controller initialized")
}
override func activateServer(_ sender: Any!) {
super.activateServer(sender)
inputLog.notice(
"controller activated for \(self.client().bundleIdentifier(), privacy: .public)"
)
CompositionCoordinator.shared.activeController = self
client().overrideKeyboard(withKeyboardNamed: "com.apple.keylayout.Russian")
}
override func deactivateServer(_ sender: Any!) {
cancelDraft()
if CompositionCoordinator.shared.activeController === self {
CompositionCoordinator.shared.activeController = nil
}
super.deactivateServer(sender)
}
override func inputControllerWillClose() {
if CompositionCoordinator.shared.activeController === self {
CompositionCoordinator.shared.activeController = nil
}
super.inputControllerWillClose()
}
func setDraft(_ text: String) {
inputLog.notice(
"setting marked text for \(self.client().bundleIdentifier(), privacy: .public)"
)
draft = text
let length = (text as NSString).length
client().setMarkedText(
text,
selectionRange: NSRange(location: length, length: 0),
replacementRange: activeReplacementRange()
)
}
func commit(_ text: String) {
inputLog.notice(
"inserting final text for \(self.client().bundleIdentifier(), privacy: .public)"
)
client().insertText(text, replacementRange: activeReplacementRange())
draft = ""
}
func cancelDraft() {
guard !draft.isEmpty else { return }
client().insertText("", replacementRange: activeReplacementRange())
draft = ""
}
private func activeReplacementRange() -> NSRange {
let marked = client().markedRange()
return marked.location == NSNotFound
? NSRange(location: NSNotFound, length: 0)
: marked
}
}
let app = NSApplication.shared
app.setActivationPolicy(.prohibited)
let connectionName =
Bundle.main.object(forInfoDictionaryKey: "InputMethodConnectionName") as? String
?? "com.romanv.inputmethod.WhisperKey_Connection"
let bundleIdentifier = Bundle.main.bundleIdentifier ?? "com.romanv.inputmethod.WhisperKey"
guard let inputServer = IMKServer(
name: connectionName,
bundleIdentifier: bundleIdentifier
) else {
fatalError("Не удалось запустить WhisperKey Input Method.")
}
_ = CompositionCoordinator.shared
NSLog("WhisperKeyInput: server started")
app.run()
withExtendedLifetime(inputServer) {}