From 4712c3f56ed7996081d1af565dbabf617420db74 Mon Sep 17 00:00:00 2001 From: Roman Varlakov Date: Wed, 29 Jul 2026 16:35:25 +0600 Subject: [PATCH] =?UTF-8?q?=D0=A3=D1=81=D0=BA=D0=BE=D1=80=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D0=BB=D0=BE=D0=BA=D0=B0=D0=BB=D1=8C=D0=BD=D1=83=D1=8E?= =?UTF-8?q?=20=D0=B4=D0=B8=D0=BA=D1=82=D0=BE=D0=B2=D0=BA=D1=83=20=D1=81?= =?UTF-8?q?=D1=82=D0=B0=D0=B1=D0=B8=D0=BB=D1=8C=D0=BD=D1=8B=D0=BC=20=D0=BF?= =?UTF-8?q?=D1=80=D0=B5=D1=84=D0=B8=D0=BA=D1=81=D0=BE=D0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 7 +- Sources/WhisperKey/AppDelegate.swift | 163 ++++++++++++++++++- Sources/WhisperKey/Audio.swift | 61 +++++-- Sources/WhisperKey/SettingsWindow.swift | 2 +- Sources/WhisperKey/TextPipeline.swift | 120 ++++++++++++++ Tests/WhisperKeyTests/TextCleanerTests.swift | 71 ++++++++ 6 files changed, 402 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 2bfddea..dcb4a93 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,10 @@ - `F5` — локальный whisper.cpp с полной несжатой моделью large-v3-turbo; - `Option+F5` — OpenAI Speech-to-Text API; - модель постоянно загружена в фоновом локальном сервере, поэтому не перечитывается с диска после каждого F5; -- во время длинной диктовки готовые фразы вставляются после паузы около секунды, но не позднее 4 секунд непрерывной речи; -- технические четырёхсекундные фрагменты склеиваются без искусственной точки и заглавной буквы; -- короткой нарезки по 1,5 секунды нет: ей не хватало контекста, и Whisper начинал домысливать; +- локальный F5 распознаёт перекрывающиеся окна примерно раз в 1,25 секунды и вставляет слова, совпавшие в двух последовательных проходах; +- нестабильный хвост остаётся внутри приложения до следующего прохода, поэтому ошибочная короткая гипотеза не попадает в поле; +- облачный Option+F5 сохраняет прежние крупные фрагменты и не создаёт дополнительные платные запросы; +- изолированных коротких запросов нет: каждый локальный проход получает перекрывающееся окно контекста до 7,5 секунды; - после 5 секунд тишины запись завершается автоматически, повторное нажатие останавливает её вручную; - результат очищается и вставляется в активное текстовое поле; - OpenAI API key хранится в macOS Keychain. diff --git a/Sources/WhisperKey/AppDelegate.swift b/Sources/WhisperKey/AppDelegate.swift index 329459f..ebefcdb 100644 --- a/Sources/WhisperKey/AppDelegate.swift +++ b/Sources/WhisperKey/AppDelegate.swift @@ -9,12 +9,21 @@ final class AppDelegate: NSObject, NSApplicationDelegate { let continuesPhrase: Bool } + private struct PendingSnapshot { + let urls: [URL] + let isFinal: Bool + let createdAt: ContinuousClock.Instant + } + private let statusItem = NSStatusBar.system.statusItem(withLength: NSStatusItem.variableLength) private let recorder = AudioRecorder() private let hotKeys = HotKeyManager() private let settings = AppSettings.shared private var activeMode: TranscriptionMode? private var pendingChunks: [PendingChunk] = [] + private var pendingSnapshots: [PendingSnapshot] = [] + private var stableSegments: [URL] = [] + private var stableTranscript = StableTranscript() private var processingChunk = false private var stopRequested = false private var stitchNextChunk = false @@ -91,16 +100,28 @@ final class AppDelegate: NSObject, NSApplicationDelegate { await stopAndTranscribe() return } + guard !stopRequested else { return } do { activeMode = mode lastMode = mode pendingChunks.removeAll() + pendingSnapshots.removeAll() + stableSegments.forEach { try? FileManager.default.removeItem(at: $0) } + stableSegments.removeAll() + stableTranscript = StableTranscript() processingChunk = false stopRequested = false stitchNextChunk = false try recorder.start( + chunking: mode == .local ? .stablePreviews : .committed, onChunk: { [weak self] chunk in - Task { @MainActor in self?.enqueue(chunk) } + Task { @MainActor in + if mode == .local { + self?.enqueueStableSegment(chunk.url) + } else { + self?.enqueueCommitted(chunk) + } + } }, onSilence: { [weak self] in Task { @MainActor in self?.finishLiveSession() } @@ -126,24 +147,74 @@ final class AppDelegate: NSObject, NSApplicationDelegate { setStatus("обрабатываю…") do { if let final = try recorder.stop() { - enqueue(RecordedChunk(url: final, continuesPhrase: false)) + if lastMode == .local { + stableSegments.append(final) + } else { + enqueueCommitted(RecordedChunk(url: final, continuesPhrase: false)) + } + } + if lastMode == .local { + Task { @MainActor [weak self] in + try? await Task.sleep(for: .milliseconds(100)) + self?.enqueueFinalStableSnapshot() + } + } else if pendingChunks.isEmpty, !processingChunk { + completeLiveSession() } - if pendingChunks.isEmpty, !processingChunk { completeLiveSession() } } catch { setStatus("ошибка") showError("Диктовка не удалась", error.localizedDescription) } } - private func enqueue(_ chunk: RecordedChunk) { + private func enqueueFinalStableSnapshot() { + pendingSnapshots.removeAll(where: { !$0.isFinal }) + if stableSegments.isEmpty { + completeLiveSession() + } else { + enqueueStableSnapshot( + PendingSnapshot( + urls: stableSegments, + isFinal: true, + createdAt: ContinuousClock.now + ) + ) + } + } + + private func enqueueCommitted(_ chunk: RecordedChunk) { pendingChunks.append(PendingChunk( url: chunk.url, continuesPhrase: chunk.continuesPhrase )) - processNextChunk() + processNextCommittedChunk() } - private func processNextChunk() { + private func enqueueStableSegment(_ url: URL) { + stableSegments.append(url) + guard !stopRequested else { return } + enqueueStableSnapshot( + PendingSnapshot( + urls: Array(stableSegments.suffix(6)), + isFinal: false, + createdAt: ContinuousClock.now + ) + ) + } + + private func enqueueStableSnapshot(_ snapshot: PendingSnapshot) { + if snapshot.isFinal { + pendingSnapshots.removeAll(where: { !$0.isFinal }) + pendingSnapshots.append(snapshot) + } else if let index = pendingSnapshots.lastIndex(where: { !$0.isFinal }) { + pendingSnapshots[index] = snapshot + } else { + pendingSnapshots.append(snapshot) + } + processNextStableSnapshot() + } + + private func processNextCommittedChunk() { guard !processingChunk, !pendingChunks.isEmpty else { return } processingChunk = true let chunk = pendingChunks.removeFirst() @@ -185,11 +256,83 @@ final class AppDelegate: NSObject, NSApplicationDelegate { if pendingChunks.isEmpty { if stopRequested { completeLiveSession() } } else { - processNextChunk() + processNextCommittedChunk() } } } + private func processNextStableSnapshot() { + guard !processingChunk, !pendingSnapshots.isEmpty else { return } + processingChunk = true + let snapshot = pendingSnapshots.removeFirst() + let mode = activeMode ?? (stopRequested ? lastMode : nil) + guard let mode else { + processingChunk = false + return + } + + Task { + do { + let processingStartedAt = ContinuousClock.now + let wav = try AudioConverter.toWhisperWAV(snapshot.urls) + if snapshot.isFinal { saveDiagnosticRecording(wav) } + defer { try? FileManager.default.removeItem(at: wav) } + + let raw: String + switch mode { + case .local: + raw = try await LocalWhisper.transcribe(wav: wav, settings: settings) + case .cloud: + raw = try await OpenAITranscriber.transcribe(wav: wav, settings: settings) + } + let cleaned = TextCleaner.clean( + raw, + removeFillers: settings.removeFillers, + glossary: settings.glossary + ) + if let update = stableTranscript.ingest(cleaned, final: snapshot.isFinal) { + try insertStableUpdate(update) + NSLog( + "WhisperKey stable commit: final=%d chars=%ld total=%@ processing=%@", + snapshot.isFinal ? 1 : 0, + (update.text as NSString).length, + String(describing: snapshot.createdAt.duration(to: .now)), + String(describing: processingStartedAt.duration(to: .now)) + ) + } + } catch { + if snapshot.isFinal, + let update = stableTranscript.ingest("", final: true) { + try? insertStableUpdate(update) + } + if snapshot.isFinal { + showError("Финальный фрагмент диктовки не распознан", error.localizedDescription) + } else { + NSLog("WhisperKey preview failed: %@", error.localizedDescription) + } + } + + processingChunk = false + if pendingSnapshots.isEmpty { + if stopRequested, snapshot.isFinal { completeLiveSession() } + } else { + processNextStableSnapshot() + } + } + } + + private func insertStableUpdate(_ update: StableTranscriptUpdate) throws { + let text = TranscriptJoiner.prepare( + update.text, + continuesPrevious: update.continuesPrevious, + continuesNext: false, + glossary: settings.glossary + ) + if !text.isEmpty { + try TextInserter.insert(text + " ") + } + } + private func saveDiagnosticRecording(_ wav: URL) { let directory = FileManager.default.urls( for: .applicationSupportDirectory, @@ -208,6 +351,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate { stopRequested = false lastMode = nil stitchNextChunk = false + pendingSnapshots.removeAll() + stableSegments.forEach { try? FileManager.default.removeItem(at: $0) } + stableSegments.removeAll() + stableTranscript = StableTranscript() setStatus("готово") } @@ -249,7 +396,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate { } } -enum TranscriptionMode { +enum TranscriptionMode: Sendable, Equatable { case local case cloud } diff --git a/Sources/WhisperKey/Audio.swift b/Sources/WhisperKey/Audio.swift index ea1f136..7d369b2 100644 --- a/Sources/WhisperKey/Audio.swift +++ b/Sources/WhisperKey/Audio.swift @@ -6,6 +6,11 @@ struct RecordedChunk: Sendable { let continuesPhrase: Bool } +enum AudioChunking: Sendable { + case stablePreviews + case committed +} + final class AudioRecorder: @unchecked Sendable { private var engine: AVAudioEngine? private let lock = NSLock() @@ -21,8 +26,10 @@ final class AudioRecorder: @unchecked Sendable { private var onChunk: (@Sendable (RecordedChunk) -> Void)? private var onSilence: (@Sendable () -> Void)? private var recordingFormat: AVAudioFormat? + private var chunking = AudioChunking.committed func start( + chunking: AudioChunking, onChunk: @escaping @Sendable (RecordedChunk) -> Void, onSilence: @escaping @Sendable () -> Void ) throws { @@ -33,6 +40,7 @@ final class AudioRecorder: @unchecked Sendable { let format = input.outputFormat(forBus: 0) self.onChunk = onChunk self.onSilence = onSilence + self.chunking = chunking recordingFormat = format try beginChunk(format: format) input.installTap(onBus: 0, bufferSize: 4096, format: format) { [weak self] buffer, _ in @@ -130,15 +138,22 @@ final class AudioRecorder: @unchecked Sendable { return } - // Keep enough context for Whisper, but insert completed phrases while - // the user is still dictating. Prefer a short natural pause; cap a - // continuous phrase so the input never stays blank indefinitely. - let minimumChunk = AVAudioFramePosition(format.sampleRate * 2.5) - let pauseBoundary = AVAudioFramePosition(format.sampleRate * 1.0) - let maximumChunk = AVAudioFramePosition(format.sampleRate * 4.0) - let reachedPause = - currentFrames >= minimumChunk && chunkSilentFrames >= pauseBoundary - let reachedMaximum = currentFrames >= maximumChunk + let reachedPause: Bool + let reachedMaximum: Bool + switch chunking { + case .stablePreviews: + reachedPause = false + reachedMaximum = currentFrames >= AVAudioFramePosition(format.sampleRate * 1.25) + case .committed: + // Cloud dictation keeps the old larger chunks so it does not issue + // overlapping paid requests. + let minimumChunk = AVAudioFramePosition(format.sampleRate * 2.5) + let pauseBoundary = AVAudioFramePosition(format.sampleRate * 1.0) + let maximumChunk = AVAudioFramePosition(format.sampleRate * 4.0) + reachedPause = + currentFrames >= minimumChunk && chunkSilentFrames >= pauseBoundary + reachedMaximum = currentFrames >= maximumChunk + } let shouldRotate = currentChunkHasSpeech && (reachedPause || reachedMaximum) if shouldRotate, let finishedURL = currentURL { @@ -174,6 +189,13 @@ final class AudioRecorder: @unchecked Sendable { enum AudioConverter { static func toWhisperWAV(_ input: URL) throws -> URL { + try toWhisperWAV([input]) + } + + static func toWhisperWAV(_ inputs: [URL]) throws -> URL { + guard !inputs.isEmpty else { + throw WhisperKeyError.processFailed("Нет аудиофрагментов для распознавания.") + } let output = FileManager.default.temporaryDirectory.appendingPathComponent("whisperkey-\(UUID().uuidString).wav") let ffmpegCandidates = ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg"] guard let ffmpeg = ffmpegCandidates.first(where: FileManager.default.isExecutableFile(atPath:)) else { @@ -181,7 +203,26 @@ enum AudioConverter { } let process = Process() process.executableURL = URL(fileURLWithPath: ffmpeg) - process.arguments = ["-hide_banner", "-loglevel", "error", "-y", "-i", input.path, "-ar", "16000", "-ac", "1", "-c:a", "pcm_s16le", output.path] + var arguments = ["-hide_banner", "-loglevel", "error", "-y"] + for input in inputs { + arguments.append(contentsOf: ["-i", input.path]) + } + if inputs.count > 1 { + let streams = inputs.indices.map { "[\($0):a:0]" }.joined() + arguments.append(contentsOf: [ + "-filter_complex", + "\(streams)concat=n=\(inputs.count):v=0:a=1[out]", + "-map", + "[out]", + ]) + } + arguments.append(contentsOf: [ + "-ar", "16000", + "-ac", "1", + "-c:a", "pcm_s16le", + output.path, + ]) + process.arguments = arguments let stderr = Pipe() process.standardError = stderr try process.run() diff --git a/Sources/WhisperKey/SettingsWindow.swift b/Sources/WhisperKey/SettingsWindow.swift index 4bcf283..f0611a6 100644 --- a/Sources/WhisperKey/SettingsWindow.swift +++ b/Sources/WhisperKey/SettingsWindow.swift @@ -44,7 +44,7 @@ final class SettingsWindowController: NSWindowController { grid.column(at: 0).xPlacement = .trailing modelPopup.addItems(withTitles: ["gpt-4o-transcribe", "gpt-4o-mini-transcribe", "whisper-1"]) - let note = NSTextField(wrappingLabelWithString: "F5 — локально. Option+F5 — OpenAI. Вся реплика распознаётся целиком после 5 секунд тишины. Повторное нажатие останавливает её вручную.") + let note = NSTextField(wrappingLabelWithString: "F5 — локально, подтверждённый текст появляется по мере речи. Option+F5 — OpenAI с обычными крупными фрагментами. После 5 секунд тишины запись завершается автоматически.") note.textColor = .secondaryLabelColor note.translatesAutoresizingMaskIntoConstraints = false let save = NSButton(title: "Сохранить", target: self, action: #selector(save)) diff --git a/Sources/WhisperKey/TextPipeline.swift b/Sources/WhisperKey/TextPipeline.swift index ee7610d..aba1b53 100644 --- a/Sources/WhisperKey/TextPipeline.swift +++ b/Sources/WhisperKey/TextPipeline.swift @@ -141,6 +141,126 @@ enum TranscriptJoiner { } } +struct StableTranscriptUpdate: Equatable { + let text: String + let continuesPrevious: Bool +} + +struct StableTranscript { + private struct Token: Equatable { + let surface: String + let key: String + } + + private var committed: [Token] = [] + private var previousPending: [Token] = [] + + mutating func ingest(_ hypothesis: String, final: Bool = false) -> StableTranscriptUpdate? { + let current = tokenize(hypothesis) + if current.isEmpty { + guard final, !previousPending.isEmpty else { return nil } + let pending = previousPending + previousPending.removeAll() + return commit(pending) + } + + let anchored = uncommittedTokens(in: current) + if !committed.isEmpty, !anchored.foundAnchor { + // A rolling window without the recent committed anchor is + // ambiguous. Keeping the previous tail is safer than duplicating + // text that is already in the target field. + if final, !previousPending.isEmpty { + let pending = previousPending + previousPending.removeAll() + return commit(pending) + } + return nil + } + + let pending = anchored.tokens + if final { + guard !pending.isEmpty else { return nil } + previousPending.removeAll() + return commit(pending) + } + + guard !previousPending.isEmpty else { + previousPending = pending + return nil + } + + let stableCount = commonPrefixCount(previousPending, pending) + guard stableCount > 0 else { + previousPending = pending + return nil + } + + let stable = Array(pending.prefix(stableCount)) + previousPending = Array(pending.dropFirst(stableCount)) + return commit(stable) + } + + private mutating func commit(_ tokens: [Token]) -> StableTranscriptUpdate? { + guard !tokens.isEmpty else { return nil } + let continuesPrevious = committed.last.map { + $0.surface.range( + of: #"[.!?…][”"'»)]*$"#, + options: .regularExpression + ) == nil + } ?? false + committed.append(contentsOf: tokens) + return StableTranscriptUpdate( + text: tokens.map(\.surface).joined(separator: " "), + continuesPrevious: continuesPrevious + ) + } + + private func uncommittedTokens(in current: [Token]) -> (tokens: [Token], foundAnchor: Bool) { + guard !committed.isEmpty else { return (current, true) } + + let maximumAnchor = min(8, committed.count, current.count) + if maximumAnchor >= 2 { + for length in stride(from: maximumAnchor, through: 2, by: -1) { + let anchor = committed.suffix(length).map(\.key) + for start in 0...(current.count - length) { + if current[start..<(start + length)].map(\.key) == anchor { + return (Array(current.dropFirst(start + length)), true) + } + } + } + } + + let lastKey = committed.last?.key + for start in 0.. Int { + var count = 0 + while count < lhs.count, count < rhs.count, lhs[count].key == rhs[count].key { + count += 1 + } + return count + } + + private func tokenize(_ text: String) -> [Token] { + text.split(whereSeparator: \.isWhitespace).compactMap { value in + let surface = String(value) + let key = surface + .lowercased() + .replacingOccurrences( + of: #"^[^\p{L}\p{N}+#]+|[^\p{L}\p{N}+#]+$"#, + with: "", + options: .regularExpression + ) + guard !key.isEmpty else { return nil } + return Token(surface: surface, key: key) + } + } +} + enum TextInserter { static func insert(_ text: String) throws { guard AXIsProcessTrusted() else { diff --git a/Tests/WhisperKeyTests/TextCleanerTests.swift b/Tests/WhisperKeyTests/TextCleanerTests.swift index 76a3db9..ef5c834 100644 --- a/Tests/WhisperKeyTests/TextCleanerTests.swift +++ b/Tests/WhisperKeyTests/TextCleanerTests.swift @@ -99,4 +99,75 @@ final class TextCleanerTests: XCTestCase { "Это новое предложение." ) } + + func testStableTranscriptWaitsForSecondMatchingPass() { + var transcript = StableTranscript() + + XCTAssertNil(transcript.ingest("Давай попробуем новый режим.")) + XCTAssertEqual( + transcript.ingest("Давай попробуем новый режим быстрее."), + StableTranscriptUpdate( + text: "Давай попробуем новый режим", + continuesPrevious: false + ) + ) + } + + func testStableTranscriptUsesCommittedAnchorWithoutDuplication() { + var transcript = StableTranscript() + _ = transcript.ingest("Давай попробуем новый режим.") + _ = transcript.ingest("Давай попробуем новый режим быстрее.") + + XCTAssertEqual( + transcript.ingest("Новый режим быстрее без задержки."), + StableTranscriptUpdate( + text: "быстрее", + continuesPrevious: true + ) + ) + XCTAssertEqual( + transcript.ingest( + "Давай попробуем новый режим быстрее без задержки.", + final: true + ), + StableTranscriptUpdate( + text: "без задержки.", + continuesPrevious: true + ) + ) + } + + func testStableTranscriptDoesNotCommitChangedTail() { + var transcript = StableTranscript() + + XCTAssertNil(transcript.ingest("Мы запускаем сбор лидов.")) + XCTAssertEqual( + transcript.ingest("Мы запускаем редактор субтитров."), + StableTranscriptUpdate( + text: "Мы запускаем", + continuesPrevious: false + ) + ) + XCTAssertNil(transcript.ingest("Мы запускаем сбор лидов сейчас.")) + XCTAssertEqual( + transcript.ingest("Мы запускаем сбор лидов сейчас.", final: true), + StableTranscriptUpdate( + text: "сбор лидов сейчас.", + continuesPrevious: true + ) + ) + } + + func testStableTranscriptFallsBackToPendingTailWhenFinalPassIsEmpty() { + var transcript = StableTranscript() + + XCTAssertNil(transcript.ingest("Проверяем последний фрагмент.")) + XCTAssertEqual( + transcript.ingest("", final: true), + StableTranscriptUpdate( + text: "Проверяем последний фрагмент.", + continuesPrevious: false + ) + ) + } }