Ускорить локальную диктовку стабильным префиксом

2026-07-29 16:35:25 +06:00
parent 67895c6157
commit 4712c3f56e
6 changed files with 402 additions and 22 deletions
@@ -5,9 +5,10 @@
 - `F5` — локальный whisper.cpp с полной несжатой моделью large-v3-turbo;
 - `Option+F5` — OpenAI Speech-to-Text API;
 - модель постоянно загружена в фоновом локальном сервере, поэтому не перечитывается с диска после каждого F5;
- во время длинной диктовки готовые фразы вставляются после паузы около секунды, но не позднее 4 секунд непрерывной речи;
- технические четырёхсекундные фрагменты склеиваются без искусственной точки и заглавной буквы;
- короткой нарезки по 1,5 секунды нет: ей не хватало контекста, и Whisper начинал домысливать;
+- локальный F5 распознаёт перекрывающиеся окна примерно раз в 1,25 секунды и вставляет слова, совпавшие в двух последовательных проходах;
+- нестабильный хвост остаётся внутри приложения до следующего прохода, поэтому ошибочная короткая гипотеза не попадает в поле;
+- облачный Option+F5 сохраняет прежние крупные фрагменты и не создаёт дополнительные платные запросы;
+- изолированных коротких запросов нет: каждый локальный проход получает перекрывающееся окно контекста до 7,5 секунды;
 - после 5 секунд тишины запись завершается автоматически, повторное нажатие останавливает её вручную;
 - результат очищается и вставляется в активное текстовое поле;
 - OpenAI API key хранится в macOS Keychain.
@@ -9,12 +9,21 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
        let continuesPhrase: Bool
    }

+    private struct PendingSnapshot {
+        let urls: [URL]
+        let isFinal: Bool
+        let createdAt: ContinuousClock.Instant
+    }
+
    private let statusItem = NSStatusBar.system.statusItem(withLength: NSStatusItem.variableLength)
    private let recorder = AudioRecorder()
    private let hotKeys = HotKeyManager()
    private let settings = AppSettings.shared
    private var activeMode: TranscriptionMode?
    private var pendingChunks: [PendingChunk] = []
+    private var pendingSnapshots: [PendingSnapshot] = []
+    private var stableSegments: [URL] = []
+    private var stableTranscript = StableTranscript()
    private var processingChunk = false
    private var stopRequested = false
    private var stitchNextChunk = false
@@ -91,16 +100,28 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
            await stopAndTranscribe()
            return
        }
+        guard !stopRequested else { return }
        do {
            activeMode = mode
            lastMode = mode
            pendingChunks.removeAll()
+            pendingSnapshots.removeAll()
+            stableSegments.forEach { try? FileManager.default.removeItem(at: $0) }
+            stableSegments.removeAll()
+            stableTranscript = StableTranscript()
            processingChunk = false
            stopRequested = false
            stitchNextChunk = false
            try recorder.start(
+                chunking: mode == .local ? .stablePreviews : .committed,
                onChunk: { [weak self] chunk in
-                    Task { @MainActor in self?.enqueue(chunk) }
+                    Task { @MainActor in
+                        if mode == .local {
+                            self?.enqueueStableSegment(chunk.url)
+                        } else {
+                            self?.enqueueCommitted(chunk)
+                        }
+                    }
                },
                onSilence: { [weak self] in
                    Task { @MainActor in self?.finishLiveSession() }
@@ -126,24 +147,74 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
        setStatus("обрабатываю…")
        do {
            if let final = try recorder.stop() {
-                enqueue(RecordedChunk(url: final, continuesPhrase: false))
+                if lastMode == .local {
+                    stableSegments.append(final)
+                } else {
+                    enqueueCommitted(RecordedChunk(url: final, continuesPhrase: false))
+                }
+            }
+            if lastMode == .local {
+                Task { @MainActor [weak self] in
+                    try? await Task.sleep(for: .milliseconds(100))
+                    self?.enqueueFinalStableSnapshot()
+                }
+            } else if pendingChunks.isEmpty, !processingChunk {
+                completeLiveSession()
            }
-            if pendingChunks.isEmpty, !processingChunk { completeLiveSession() }
        } catch {
            setStatus("ошибка")
            showError("Диктовка не удалась", error.localizedDescription)
        }
    }

-    private func enqueue(_ chunk: RecordedChunk) {
+    private func enqueueFinalStableSnapshot() {
+        pendingSnapshots.removeAll(where: { !$0.isFinal })
+        if stableSegments.isEmpty {
+            completeLiveSession()
+        } else {
+            enqueueStableSnapshot(
+                PendingSnapshot(
+                    urls: stableSegments,
+                    isFinal: true,
+                    createdAt: ContinuousClock.now
+                )
+            )
+        }
+    }
+
+    private func enqueueCommitted(_ chunk: RecordedChunk) {
        pendingChunks.append(PendingChunk(
            url: chunk.url,
            continuesPhrase: chunk.continuesPhrase
        ))
-        processNextChunk()
+        processNextCommittedChunk()
    }

-    private func processNextChunk() {
+    private func enqueueStableSegment(_ url: URL) {
+        stableSegments.append(url)
+        guard !stopRequested else { return }
+        enqueueStableSnapshot(
+            PendingSnapshot(
+                urls: Array(stableSegments.suffix(6)),
+                isFinal: false,
+                createdAt: ContinuousClock.now
+            )
+        )
+    }
+
+    private func enqueueStableSnapshot(_ snapshot: PendingSnapshot) {
+        if snapshot.isFinal {
+            pendingSnapshots.removeAll(where: { !$0.isFinal })
+            pendingSnapshots.append(snapshot)
+        } else if let index = pendingSnapshots.lastIndex(where: { !$0.isFinal }) {
+            pendingSnapshots[index] = snapshot
+        } else {
+            pendingSnapshots.append(snapshot)
+        }
+        processNextStableSnapshot()
+    }
+
+    private func processNextCommittedChunk() {
        guard !processingChunk, !pendingChunks.isEmpty else { return }
        processingChunk = true
        let chunk = pendingChunks.removeFirst()
@@ -185,11 +256,83 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
            if pendingChunks.isEmpty {
                if stopRequested { completeLiveSession() }
            } else {
-                processNextChunk()
+                processNextCommittedChunk()
            }
        }
    }

+    private func processNextStableSnapshot() {
+        guard !processingChunk, !pendingSnapshots.isEmpty else { return }
+        processingChunk = true
+        let snapshot = pendingSnapshots.removeFirst()
+        let mode = activeMode ?? (stopRequested ? lastMode : nil)
+        guard let mode else {
+            processingChunk = false
+            return
+        }
+
+        Task {
+            do {
+                let processingStartedAt = ContinuousClock.now
+                let wav = try AudioConverter.toWhisperWAV(snapshot.urls)
+                if snapshot.isFinal { saveDiagnosticRecording(wav) }
+                defer { try? FileManager.default.removeItem(at: wav) }
+
+                let raw: String
+                switch mode {
+                case .local:
+                    raw = try await LocalWhisper.transcribe(wav: wav, settings: settings)
+                case .cloud:
+                    raw = try await OpenAITranscriber.transcribe(wav: wav, settings: settings)
+                }
+                let cleaned = TextCleaner.clean(
+                    raw,
+                    removeFillers: settings.removeFillers,
+                    glossary: settings.glossary
+                )
+                if let update = stableTranscript.ingest(cleaned, final: snapshot.isFinal) {
+                    try insertStableUpdate(update)
+                    NSLog(
+                        "WhisperKey stable commit: final=%d chars=%ld total=%@ processing=%@",
+                        snapshot.isFinal ? 1 : 0,
+                        (update.text as NSString).length,
+                        String(describing: snapshot.createdAt.duration(to: .now)),
+                        String(describing: processingStartedAt.duration(to: .now))
+                    )
+                }
+            } catch {
+                if snapshot.isFinal,
+                   let update = stableTranscript.ingest("", final: true) {
+                    try? insertStableUpdate(update)
+                }
+                if snapshot.isFinal {
+                    showError("Финальный фрагмент диктовки не распознан", error.localizedDescription)
+                } else {
+                    NSLog("WhisperKey preview failed: %@", error.localizedDescription)
+                }
+            }
+
+            processingChunk = false
+            if pendingSnapshots.isEmpty {
+                if stopRequested, snapshot.isFinal { completeLiveSession() }
+            } else {
+                processNextStableSnapshot()
+            }
+        }
+    }
+
+    private func insertStableUpdate(_ update: StableTranscriptUpdate) throws {
+        let text = TranscriptJoiner.prepare(
+            update.text,
+            continuesPrevious: update.continuesPrevious,
+            continuesNext: false,
+            glossary: settings.glossary
+        )
+        if !text.isEmpty {
+            try TextInserter.insert(text + " ")
+        }
+    }
+
    private func saveDiagnosticRecording(_ wav: URL) {
        let directory = FileManager.default.urls(
            for: .applicationSupportDirectory,
@@ -208,6 +351,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
        stopRequested = false
        lastMode = nil
        stitchNextChunk = false
+        pendingSnapshots.removeAll()
+        stableSegments.forEach { try? FileManager.default.removeItem(at: $0) }
+        stableSegments.removeAll()
+        stableTranscript = StableTranscript()
        setStatus("готово")
    }

@@ -249,7 +396,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
    }
 }

-enum TranscriptionMode {
+enum TranscriptionMode: Sendable, Equatable {
    case local
    case cloud
 }
@@ -6,6 +6,11 @@ struct RecordedChunk: Sendable {
    let continuesPhrase: Bool
 }

+enum AudioChunking: Sendable {
+    case stablePreviews
+    case committed
+}
+
 final class AudioRecorder: @unchecked Sendable {
    private var engine: AVAudioEngine?
    private let lock = NSLock()
@@ -21,8 +26,10 @@ final class AudioRecorder: @unchecked Sendable {
    private var onChunk: (@Sendable (RecordedChunk) -> Void)?
    private var onSilence: (@Sendable () -> Void)?
    private var recordingFormat: AVAudioFormat?
+    private var chunking = AudioChunking.committed

    func start(
+        chunking: AudioChunking,
        onChunk: @escaping @Sendable (RecordedChunk) -> Void,
        onSilence: @escaping @Sendable () -> Void
    ) throws {
@@ -33,6 +40,7 @@ final class AudioRecorder: @unchecked Sendable {
        let format = input.outputFormat(forBus: 0)
        self.onChunk = onChunk
        self.onSilence = onSilence
+        self.chunking = chunking
        recordingFormat = format
        try beginChunk(format: format)
        input.installTap(onBus: 0, bufferSize: 4096, format: format) { [weak self] buffer, _ in
@@ -130,15 +138,22 @@ final class AudioRecorder: @unchecked Sendable {
            return
        }

-        // Keep enough context for Whisper, but insert completed phrases while
-        // the user is still dictating. Prefer a short natural pause; cap a
-        // continuous phrase so the input never stays blank indefinitely.
-        let minimumChunk = AVAudioFramePosition(format.sampleRate * 2.5)
-        let pauseBoundary = AVAudioFramePosition(format.sampleRate * 1.0)
-        let maximumChunk = AVAudioFramePosition(format.sampleRate * 4.0)
-        let reachedPause =
-            currentFrames >= minimumChunk && chunkSilentFrames >= pauseBoundary
-        let reachedMaximum = currentFrames >= maximumChunk
+        let reachedPause: Bool
+        let reachedMaximum: Bool
+        switch chunking {
+        case .stablePreviews:
+            reachedPause = false
+            reachedMaximum = currentFrames >= AVAudioFramePosition(format.sampleRate * 1.25)
+        case .committed:
+            // Cloud dictation keeps the old larger chunks so it does not issue
+            // overlapping paid requests.
+            let minimumChunk = AVAudioFramePosition(format.sampleRate * 2.5)
+            let pauseBoundary = AVAudioFramePosition(format.sampleRate * 1.0)
+            let maximumChunk = AVAudioFramePosition(format.sampleRate * 4.0)
+            reachedPause =
+                currentFrames >= minimumChunk && chunkSilentFrames >= pauseBoundary
+            reachedMaximum = currentFrames >= maximumChunk
+        }
        let shouldRotate = currentChunkHasSpeech && (reachedPause || reachedMaximum)

        if shouldRotate, let finishedURL = currentURL {
@@ -174,6 +189,13 @@ final class AudioRecorder: @unchecked Sendable {

 enum AudioConverter {
    static func toWhisperWAV(_ input: URL) throws -> URL {
+        try toWhisperWAV([input])
+    }
+
+    static func toWhisperWAV(_ inputs: [URL]) throws -> URL {
+        guard !inputs.isEmpty else {
+            throw WhisperKeyError.processFailed("Нет аудиофрагментов для распознавания.")
+        }
        let output = FileManager.default.temporaryDirectory.appendingPathComponent("whisperkey-\(UUID().uuidString).wav")
        let ffmpegCandidates = ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg"]
        guard let ffmpeg = ffmpegCandidates.first(where: FileManager.default.isExecutableFile(atPath:)) else {
@@ -181,7 +203,26 @@ enum AudioConverter {
        }
        let process = Process()
        process.executableURL = URL(fileURLWithPath: ffmpeg)
-        process.arguments = ["-hide_banner", "-loglevel", "error", "-y", "-i", input.path, "-ar", "16000", "-ac", "1", "-c:a", "pcm_s16le", output.path]
+        var arguments = ["-hide_banner", "-loglevel", "error", "-y"]
+        for input in inputs {
+            arguments.append(contentsOf: ["-i", input.path])
+        }
+        if inputs.count > 1 {
+            let streams = inputs.indices.map { "[\($0):a:0]" }.joined()
+            arguments.append(contentsOf: [
+                "-filter_complex",
+                "\(streams)concat=n=\(inputs.count):v=0:a=1[out]",
+                "-map",
+                "[out]",
+            ])
+        }
+        arguments.append(contentsOf: [
+            "-ar", "16000",
+            "-ac", "1",
+            "-c:a", "pcm_s16le",
+            output.path,
+        ])
+        process.arguments = arguments
        let stderr = Pipe()
        process.standardError = stderr
        try process.run()
@@ -44,7 +44,7 @@ final class SettingsWindowController: NSWindowController {
        grid.column(at: 0).xPlacement = .trailing
        modelPopup.addItems(withTitles: ["gpt-4o-transcribe", "gpt-4o-mini-transcribe", "whisper-1"])

-        let note = NSTextField(wrappingLabelWithString: "F5 — локально. Option+F5 — OpenAI. Вся реплика распознаётся целиком после 5 секунд тишины. Повторное нажатие останавливает её вручную.")
+        let note = NSTextField(wrappingLabelWithString: "F5 — локально, подтверждённый текст появляется по мере речи. Option+F5 — OpenAI с обычными крупными фрагментами. После 5 секунд тишины запись завершается автоматически.")
        note.textColor = .secondaryLabelColor
        note.translatesAutoresizingMaskIntoConstraints = false
        let save = NSButton(title: "Сохранить", target: self, action: #selector(save))
@@ -141,6 +141,126 @@ enum TranscriptJoiner {
    }
 }

+struct StableTranscriptUpdate: Equatable {
+    let text: String
+    let continuesPrevious: Bool
+}
+
+struct StableTranscript {
+    private struct Token: Equatable {
+        let surface: String
+        let key: String
+    }
+
+    private var committed: [Token] = []
+    private var previousPending: [Token] = []
+
+    mutating func ingest(_ hypothesis: String, final: Bool = false) -> StableTranscriptUpdate? {
+        let current = tokenize(hypothesis)
+        if current.isEmpty {
+            guard final, !previousPending.isEmpty else { return nil }
+            let pending = previousPending
+            previousPending.removeAll()
+            return commit(pending)
+        }
+
+        let anchored = uncommittedTokens(in: current)
+        if !committed.isEmpty, !anchored.foundAnchor {
+            // A rolling window without the recent committed anchor is
+            // ambiguous. Keeping the previous tail is safer than duplicating
+            // text that is already in the target field.
+            if final, !previousPending.isEmpty {
+                let pending = previousPending
+                previousPending.removeAll()
+                return commit(pending)
+            }
+            return nil
+        }
+
+        let pending = anchored.tokens
+        if final {
+            guard !pending.isEmpty else { return nil }
+            previousPending.removeAll()
+            return commit(pending)
+        }
+
+        guard !previousPending.isEmpty else {
+            previousPending = pending
+            return nil
+        }
+
+        let stableCount = commonPrefixCount(previousPending, pending)
+        guard stableCount > 0 else {
+            previousPending = pending
+            return nil
+        }
+
+        let stable = Array(pending.prefix(stableCount))
+        previousPending = Array(pending.dropFirst(stableCount))
+        return commit(stable)
+    }
+
+    private mutating func commit(_ tokens: [Token]) -> StableTranscriptUpdate? {
+        guard !tokens.isEmpty else { return nil }
+        let continuesPrevious = committed.last.map {
+            $0.surface.range(
+                of: #"[.!?…][”"'»)]*$"#,
+                options: .regularExpression
+            ) == nil
+        } ?? false
+        committed.append(contentsOf: tokens)
+        return StableTranscriptUpdate(
+            text: tokens.map(\.surface).joined(separator: " "),
+            continuesPrevious: continuesPrevious
+        )
+    }
+
+    private func uncommittedTokens(in current: [Token]) -> (tokens: [Token], foundAnchor: Bool) {
+        guard !committed.isEmpty else { return (current, true) }
+
+        let maximumAnchor = min(8, committed.count, current.count)
+        if maximumAnchor >= 2 {
+            for length in stride(from: maximumAnchor, through: 2, by: -1) {
+                let anchor = committed.suffix(length).map(\.key)
+                for start in 0...(current.count - length) {
+                    if current[start..<(start + length)].map(\.key) == anchor {
+                        return (Array(current.dropFirst(start + length)), true)
+                    }
+                }
+            }
+        }
+
+        let lastKey = committed.last?.key
+        for start in 0..<min(3, current.count) where current[start].key == lastKey {
+            return (Array(current.dropFirst(start + 1)), true)
+        }
+        return (current, false)
+    }
+
+    private func commonPrefixCount(_ lhs: [Token], _ rhs: [Token]) -> Int {
+        var count = 0
+        while count < lhs.count, count < rhs.count, lhs[count].key == rhs[count].key {
+            count += 1
+        }
+        return count
+    }
+
+    private func tokenize(_ text: String) -> [Token] {
+        text.split(whereSeparator: \.isWhitespace).compactMap { value in
+            let surface = String(value)
+            let key = surface
+                .lowercased()
+                .replacingOccurrences(
+                    of: #"^[^\p{L}\p{N}+#]+|[^\p{L}\p{N}+#]+$"#,
+                    with: "",
+                    options: .regularExpression
+                )
+            guard !key.isEmpty else { return nil }
+            return Token(surface: surface, key: key)
+        }
+    }
+}
+
 enum TextInserter {
    static func insert(_ text: String) throws {
        guard AXIsProcessTrusted() else {
@@ -99,4 +99,75 @@ final class TextCleanerTests: XCTestCase {
            "Это новое предложение."
        )
    }
+
+    func testStableTranscriptWaitsForSecondMatchingPass() {
+        var transcript = StableTranscript()
+
+        XCTAssertNil(transcript.ingest("Давай попробуем новый режим."))
+        XCTAssertEqual(
+            transcript.ingest("Давай попробуем новый режим быстрее."),
+            StableTranscriptUpdate(
+                text: "Давай попробуем новый режим",
+                continuesPrevious: false
+            )
+        )
+    }
+
+    func testStableTranscriptUsesCommittedAnchorWithoutDuplication() {
+        var transcript = StableTranscript()
+        _ = transcript.ingest("Давай попробуем новый режим.")
+        _ = transcript.ingest("Давай попробуем новый режим быстрее.")
+
+        XCTAssertEqual(
+            transcript.ingest("Новый режим быстрее без задержки."),
+            StableTranscriptUpdate(
+                text: "быстрее",
+                continuesPrevious: true
+            )
+        )
+        XCTAssertEqual(
+            transcript.ingest(
+                "Давай попробуем новый режим быстрее без задержки.",
+                final: true
+            ),
+            StableTranscriptUpdate(
+                text: "без задержки.",
+                continuesPrevious: true
+            )
+        )
+    }
+
+    func testStableTranscriptDoesNotCommitChangedTail() {
+        var transcript = StableTranscript()
+
+        XCTAssertNil(transcript.ingest("Мы запускаем сбор лидов."))
+        XCTAssertEqual(
+            transcript.ingest("Мы запускаем редактор субтитров."),
+            StableTranscriptUpdate(
+                text: "Мы запускаем",
+                continuesPrevious: false
+            )
+        )
+        XCTAssertNil(transcript.ingest("Мы запускаем сбор лидов сейчас."))
+        XCTAssertEqual(
+            transcript.ingest("Мы запускаем сбор лидов сейчас.", final: true),
+            StableTranscriptUpdate(
+                text: "сбор лидов сейчас.",
+                continuesPrevious: true
+            )
+        )
+    }
+
+    func testStableTranscriptFallsBackToPendingTailWhenFinalPassIsEmpty() {
+        var transcript = StableTranscript()
+
+        XCTAssertNil(transcript.ingest("Проверяем последний фрагмент."))
+        XCTAssertEqual(
+            transcript.ingest("", final: true),
+            StableTranscriptUpdate(
+                text: "Проверяем последний фрагмент.",
+                continuesPrevious: false
+            )
+        )
+    }
 }