From e2ef304d69a3ddcc69b4c05245343fe75ba1107e Mon Sep 17 00:00:00 2001 From: Ali Alavi Date: Fri, 2 Oct 2026 07:27:41 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20add=20Darwin=20notification=20triggers,?= =?UTF-8?q?=20in-stream=20end=20trigger=20('and'/'=D8=AA=D9=85=D8=A7=D9=85?= =?UTF-8?q?'),=20and=20wake/end=20word=20sanitization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/AppDelegate.swift | 79 ++++++++++++++++++-- src/InputBoxManager.swift | 153 ++++++++++++++++++++++++++++++++------ src/SonioxClient.swift | 119 +++++++++++++++++++---------- src/TextProcessor.swift | 30 ++++++++ 4 files changed, 312 insertions(+), 69 deletions(-) diff --git a/src/AppDelegate.swift b/src/AppDelegate.swift index a1d7d96..839c2d2 100644 --- a/src/AppDelegate.swift +++ b/src/AppDelegate.swift @@ -11,6 +11,7 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { private var pendingStopWorkItem: DispatchWorkItem? = nil private var isNetworkDegraded: Bool = false private var lastRecordedAudio: Data = Data() + private var finalizeWatchdogWorkItem: DispatchWorkItem? = nil public func applicationDidFinishLaunching(_ notification: Notification) { if UserDefaults.standard.object(forKey: "SonioxPlaySounds") == nil { @@ -91,6 +92,7 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { // Check permissions on start checkInitialPermissions() + setupDarwinNotifications() } private func checkInitialPermissions() { @@ -124,6 +126,8 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { public func startRecording() { if audioRecorder.isRecording { return } + finalizeWatchdogWorkItem?.cancel() + finalizeWatchdogWorkItem = nil if isBusyFinalizing { print("AppDelegate: isBusyFinalizing was active when startRecording invoked. Clearing stale lock.") isBusyFinalizing = false @@ -150,7 +154,21 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { } // Stream live finalized tokens directly into input box word-by-word - session.onCommittedDelta = { delta in + session.onCommittedDelta = { [weak self] delta in + guard let self = self else { return } + + // Check for in-stream end trigger ("and", "تمام", "end") + let deltaWords = delta.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } + if let lastWord = deltaWords.last?.replacingOccurrences(of: #"[.,!?:;،؛؟"'()\[\]{}«»–—\-]"#, with: "", options: .regularExpression).lowercased() { + if lastWord == "and" || lastWord == "تمام" || lastWord == "end" { + print("AppDelegate: In-speech end trigger detected ('\(lastWord)'). Finalizing speech...") + DispatchQueue.main.async { + self.stopRecording() + } + return + } + } + // If it's an executive slash command, do NOT type it into the input box if SonioxSettings.shared.voiceCommands && VoiceCommandExecutor.shared.isSlashCommand(text: delta) { return @@ -242,17 +260,25 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { HUDOverlayController.shared.show(state: .transcribing) - // Safety timeout watchdog: finalize must complete within 2.5s - DispatchQueue.main.asyncAfter(deadline: .now() + 2.5) { [weak self] in + // Dynamic safety timeout watchdog: scale with audio length & network transit + let durationSec = Double(pcmData.count) / 32000.0 + let finalizeTimeout = max(4.0, min(16.0, durationSec * 0.25 + 3.5)) + + let watchdog = DispatchWorkItem { [weak self] in guard let self = self, self.isBusyFinalizing else { return } - print("AppDelegate: Finalize safety watchdog triggered — starting Smart Audio Recovery.") - self.startSmartAudioRecovery(pcmData: self.lastRecordedAudio, reason: "Finalize timeout watchdog (2.5s)") + print("AppDelegate: Finalize safety watchdog triggered (\(String(format: "%.1f", finalizeTimeout))s) — starting Smart Audio Recovery.") + self.finalizeWatchdogWorkItem = nil + self.startSmartAudioRecovery(pcmData: self.lastRecordedAudio, reason: "Finalize timeout watchdog (\(String(format: "%.1f", finalizeTimeout))s)") } + self.finalizeWatchdogWorkItem = watchdog + DispatchQueue.main.asyncAfter(deadline: .now() + finalizeTimeout, execute: watchdog) activeSession?.finalizeStream() } private func handleFinalResult(_ result: Result) { + finalizeWatchdogWorkItem?.cancel() + finalizeWatchdogWorkItem = nil guard isBusyFinalizing else { return } self.activeSession = nil self.statusBarController.updateIcon(state: .idle) @@ -282,6 +308,8 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { } private func startSmartAudioRecovery(pcmData: Data, reason: String) { + finalizeWatchdogWorkItem?.cancel() + finalizeWatchdogWorkItem = nil guard !pcmData.isEmpty else { self.isBusyFinalizing = false self.statusBarController.updateIcon(state: .idle) @@ -369,4 +397,45 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { public func applicationWillTerminate(_ notification: Notification) { HotkeyManager.shared.unregisterHotkeys() } + + private func setupDarwinNotifications() { + let notifyCenter = CFNotificationCenterGetDarwinNotifyCenter() + let selfPtr = Unmanaged.passUnretained(self).toOpaque() + + let startCb: CFNotificationCallback = { (_, observer, _, _, _) in + guard let observer = observer else { return } + let app = Unmanaged.fromOpaque(observer).takeUnretainedValue() + DispatchQueue.main.async { + if !app.audioRecorder.isRecording { + print("AppDelegate: Received Darwin notification 'com.soniox.voice.start'. Starting recording...") + app.startRecording() + } + } + } + + let stopCb: CFNotificationCallback = { (_, observer, _, _, _) in + guard let observer = observer else { return } + let app = Unmanaged.fromOpaque(observer).takeUnretainedValue() + DispatchQueue.main.async { + if app.audioRecorder.isRecording { + print("AppDelegate: Received Darwin notification 'com.soniox.voice.stop'. Stopping recording...") + app.stopRecording() + } + } + } + + let toggleCb: CFNotificationCallback = { (_, observer, _, _, _) in + guard let observer = observer else { return } + let app = Unmanaged.fromOpaque(observer).takeUnretainedValue() + DispatchQueue.main.async { + print("AppDelegate: Received Darwin notification 'com.soniox.voice.toggle'.") + app.toggleRecording() + } + } + + CFNotificationCenterAddObserver(notifyCenter, selfPtr, startCb, "com.soniox.voice.start" as CFString, nil, .deliverImmediately) + CFNotificationCenterAddObserver(notifyCenter, selfPtr, stopCb, "com.soniox.voice.stop" as CFString, nil, .deliverImmediately) + CFNotificationCenterAddObserver(notifyCenter, selfPtr, toggleCb, "com.soniox.voice.toggle" as CFString, nil, .deliverImmediately) + } + } diff --git a/src/InputBoxManager.swift b/src/InputBoxManager.swift index 5ad3d23..ce10f77 100644 --- a/src/InputBoxManager.swift +++ b/src/InputBoxManager.swift @@ -75,6 +75,18 @@ public final class InputBoxManager { public func streamDelta(text: String) { guard !text.isEmpty else { return } + // Suppress trailing end triggers ("and", "تمام", "end") from live typing + let triggerWords = text.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } + if let last = triggerWords.last?.replacingOccurrences(of: #"[.,!?:;،؛؟"'()\[\]{}«»–—\-]"#, with: "", options: .regularExpression).lowercased(), + last == "and" || last == "تمام" || last == "end" { + if triggerWords.count <= 1 { + return // Drop completely if it was just the end word + } + let textWithoutEnd = triggerWords.dropLast().joined(separator: " ") + " " + streamDelta(text: textWithoutEnd) + return + } + lock.lock() if isPrefixResolved { var textToPaste = text @@ -219,9 +231,22 @@ public final class InputBoxManager { pasteQueue.async { [weak self] in guard let self = self else { return } + // Helper to restore full text onto clipboard for the user after injection completes + let restoreClipboardFullText = { + DispatchQueue.main.asyncAfter(deadline: .now() + 0.12) { + let pb = NSPasteboard.general + pb.clearContents() + pb.setString(textWithTrailing, forType: .string) + } + } + // Case 1: Nothing was streamed live -> Paste entire text if cleanStreamed.isEmpty { + let pb = NSPasteboard.general + pb.clearContents() + pb.setString(textWithTrailing, forType: .string) self.postCleanCmdV() + restoreClipboardFullText() return } @@ -233,6 +258,7 @@ public final class InputBoxManager { pb.setString(" ", forType: .string) self.postCleanCmdV() } + restoreClipboardFullText() return } @@ -244,40 +270,124 @@ public final class InputBoxManager { pb.clearContents() pb.setString(suffixToPaste, forType: .string) self.postCleanCmdV() + restoreClipboardFullText() return } - // Case 4: Word-level token overlap matching (robust to punctuation, ZWNJ, and acoustic model variations) + // Case 4: Intelligent Word-level Tail-Anchor Matching (robust to domain vocabulary, number formatting, punctuation, ZWNJ) let streamedWords = cleanStreamed.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } let finalWords = trimmed.components(separatedBy: .whitespacesAndNewlines).filter { !$0.isEmpty } + guard !finalWords.isEmpty else { + restoreClipboardFullText() + return + } + let cleanWord: (String) -> String = { w in - w.replacingOccurrences(of: "[.,!?:;،؛؟\\\"'()\\[\\]{}«»–—\\-]", with: "", options: .regularExpression).lowercased() + w.replacingOccurrences(of: "\u{200C}", with: "") + .replacingOccurrences(of: "\u{200D}", with: "") + .replacingOccurrences(of: "[.,!?:;،؛؟\\\"'()\\[\\]{}«»–—\\-]", with: "", options: .regularExpression) + .lowercased() + } + let normStreamed = streamedWords.map(cleanWord).filter { !$0.isEmpty } + let normFinal = finalWords.map(cleanWord).filter { !$0.isEmpty } + + var unstreamedStartIndex: Int? = nil + + // Strategy 4A: Search for 3-word tail anchor from streamed text in final words + if normStreamed.count >= 3 { + let a1 = normStreamed[normStreamed.count - 3] + let a2 = normStreamed[normStreamed.count - 2] + let a3 = normStreamed[normStreamed.count - 1] + for j in stride(from: normFinal.count - 1, through: 2, by: -1) { + if normFinal[j] == a3 && normFinal[j - 1] == a2 && normFinal[j - 2] == a1 { + unstreamedStartIndex = j + 1 + break + } + } } - let normStreamed = streamedWords.map(cleanWord) - let normFinal = finalWords.map(cleanWord) - var matchedWordCount = 0 - for i in 0..= 2 { + let a1 = normStreamed[normStreamed.count - 2] + let a2 = normStreamed[normStreamed.count - 1] + for j in stride(from: normFinal.count - 1, through: 1, by: -1) { + if normFinal[j] == a2 && normFinal[j - 1] == a1 { + unstreamedStartIndex = j + 1 + break + } + } + } + + // Strategy 4C: Search for 1-word tail anchor in recent final words + if unstreamedStartIndex == nil && !normStreamed.isEmpty { + let a1 = normStreamed[normStreamed.count - 1] + let searchStart = max(0, normFinal.count - max(15, normStreamed.count)) + for j in stride(from: normFinal.count - 1, through: searchStart, by: -1) { + if normFinal[j] == a1 { + unstreamedStartIndex = j + 1 + break + } + } + } + + // Strategy 4D: Forward prefix match fallback + if unstreamedStartIndex == nil { + var prefixMatchCount = 0 + for i in 0.. 0 { + unstreamedStartIndex = prefixMatchCount + } + } + + // Suffix Evaluation: + if let startIndex = unstreamedStartIndex { + if startIndex < finalWords.count { + // Only paste unstreamed trailing words + let remainingWords = Array(finalWords.dropFirst(startIndex)) + let suffix = remainingWords.joined(separator: " ") + let suffixToPaste = addTrailingSpace ? (" " + suffix + " ") : (" " + suffix) + let pb = NSPasteboard.general + pb.clearContents() + pb.setString(suffixToPaste, forType: .string) + self.postCleanCmdV() + restoreClipboardFullText() + return } else { - break + // startIndex >= finalWords.count: All words were ALREADY streamed! + if addTrailingSpace { + let pb = NSPasteboard.general + pb.clearContents() + pb.setString(" ", forType: .string) + self.postCleanCmdV() + } + restoreClipboardFullText() + return } } - if matchedWordCount > 0 && finalWords.count > matchedWordCount { - let remainingWords = Array(finalWords.dropFirst(matchedWordCount)) - let suffix = remainingWords.joined(separator: " ") - let suffixToPaste = addTrailingSpace ? (" " + suffix + " ") : (" " + suffix) - let pb = NSPasteboard.general - pb.clearContents() - pb.setString(suffixToPaste, forType: .string) - self.postCleanCmdV() + // Case 5: Anti-Duplication Safety Guard + // If more than half the words were streamed, the utterance is already in the input box! + // NEVER paste the full text, as that guarantees 100% duplication! + if streamedWords.count >= (finalWords.count / 2) { + print("InputBoxManager: Most words already streamed (\(streamedWords.count) vs \(finalWords.count)). Suppressing full paste to prevent duplication.") + if addTrailingSpace { + let pb = NSPasteboard.general + pb.clearContents() + pb.setString(" ", forType: .string) + self.postCleanCmdV() + } + restoreClipboardFullText() return } - // Case 5: Approximate fallback when streamed words were few and final words are many + // Case 6: Fallback when only very few words were streamed (<50%) and no anchor was found if finalWords.count > streamedWords.count { let remainingWords = Array(finalWords.dropFirst(streamedWords.count)) let suffix = remainingWords.joined(separator: " ") @@ -286,16 +396,11 @@ public final class InputBoxManager { pb.clearContents() pb.setString(suffixToPaste, forType: .string) self.postCleanCmdV() - return } - - // Case 6: Fallback - paste the full text so speech is NEVER discarded - self.postCleanCmdV() + restoreClipboardFullText() } } - /// Posts Cmd+V using privateState event source to cgAnnotatedSessionEventTap - /// This strips physical Option ⌥ / Shift / Control flags and pastes cleanly across all apps private func postCleanCmdV() { let src = CGEventSource(stateID: .privateState) let vKeyCode: CGKeyCode = 0x09 // 'v' diff --git a/src/SonioxClient.swift b/src/SonioxClient.swift index 7be1f66..fba75eb 100644 --- a/src/SonioxClient.swift +++ b/src/SonioxClient.swift @@ -461,7 +461,7 @@ public final class SonioxLiveSession { task.resume() var fullCommitted: [String] = [] - var fullNonFinal: [String] = [] + var latestNonFinal: [String] = [] var isDone = false let lock = NSLock() @@ -498,6 +498,7 @@ public final class SonioxLiveSession { let json = try? JSONSerialization.jsonObject(with: jData) as? [String: Any] { var gotFin = false + var packetNonFinal: [String] = [] if let parts = json["parts"] as? [[String: Any]] { for p in parts { let isTranslation = (p["translation_status"] as? String) == "translation" @@ -515,21 +516,28 @@ public final class SonioxLiveSession { } else if isFinal { if !t.isEmpty { fullCommitted.append(t) } } else { - if !t.isEmpty { fullNonFinal.append(t) } + if !t.isEmpty { packetNonFinal.append(t) } } } } - let combined = fullCommitted.joined() + fullNonFinal.joined() - if !combined.isEmpty { + lock.lock() + latestNonFinal = packetNonFinal + let currentProgress = fullCommitted.joined() + (gotFin ? "" : latestNonFinal.joined()) + lock.unlock() + + if !currentProgress.isEmpty { DispatchQueue.main.async { - onProgress?(combined) + onProgress?(currentProgress) } } let ended = (json["session_ended"] as? Bool ?? false) || ((json["type"] as? String) == "session_done") if gotFin || ended { - let sanitized = sanitizeText(combined) + lock.lock() + let textToReturn = fullCommitted.joined() + (gotFin ? "" : latestNonFinal.joined()) + lock.unlock() + let sanitized = sanitizeText(textToReturn) if !sanitized.isEmpty { finishOnce(with: .success(sanitized)) } else { @@ -542,8 +550,10 @@ public final class SonioxLiveSession { case .failure(let err): print("SonioxLiveSession: Burst receive error on \(endpoint):", err) - let combined = fullCommitted.joined() + fullNonFinal.joined() - let sanitized = sanitizeText(combined) + lock.lock() + let textToReturn = fullCommitted.joined() + latestNonFinal.joined() + lock.unlock() + let sanitized = sanitizeText(textToReturn) if !sanitized.isEmpty { finishOnce(with: .success(sanitized)) } else { @@ -555,39 +565,55 @@ public final class SonioxLiveSession { receiveNext() - // Pump audio in high-speed bursts (16KB chunks) - DispatchQueue.global(qos: .userInitiated).async { - let chunkSize = 16000 // 0.5s audio - var offset = 0 - while offset < pcmData.count { + // Handshake verification before pumping audio to avoid early drops & empty transcripts + task.sendPing { pingError in + lock.lock() + let alreadyDone = isDone + lock.unlock() + if alreadyDone { return } + + if let pingErr = pingError { + print("SonioxLiveSession: Burst connection ping failed on \(endpoint): \(pingErr)") + finishOnce(with: .failure(pingErr)) + return + } + + // Connection is active: stream audio in high-speed paced bursts + DispatchQueue.global(qos: .userInitiated).async { + let chunkSize = 16000 // 0.5s audio + var offset = 0 + while offset < pcmData.count { + lock.lock() + let done = isDone + lock.unlock() + if done { break } + + let length = min(chunkSize, pcmData.count - offset) + let chunk = pcmData.subdata(in: offset..<(offset + length)) + task.send(URLSessionWebSocketTask.Message.data(chunk)) { _ in } + offset += length + usleep(12000) // 12ms pace + } + lock.lock() let done = isDone lock.unlock() - if done { break } - - let length = min(chunkSize, pcmData.count - offset) - let chunk = pcmData.subdata(in: offset..<(offset + length)) - task.send(URLSessionWebSocketTask.Message.data(chunk)) { _ in } - offset += length - usleep(8000) // 8ms pace - } - - lock.lock() - let done = isDone - lock.unlock() - if !done { - // Send trailing silence + finalize - let silence = Data(count: 3840) - task.send(URLSessionWebSocketTask.Message.data(silence)) { _ in } - let finalizeMsg = URLSessionWebSocketTask.Message.string("{\"type\": \"finalize\"}") - task.send(finalizeMsg) { _ in } + if !done { + // Send trailing silence + finalize + let silence = Data(count: 4800) + task.send(URLSessionWebSocketTask.Message.data(silence)) { _ in } + let finalizeMsg = URLSessionWebSocketTask.Message.string("{\"type\": \"finalize\"}") + task.send(finalizeMsg) { _ in } + } } } // Dynamic Watchdog timeout DispatchQueue.global().asyncAfter(deadline: .now() + watchdogTimeout) { - let combined = fullCommitted.joined() + fullNonFinal.joined() - let sanitized = sanitizeText(combined) + lock.lock() + let textToReturn = fullCommitted.joined() + latestNonFinal.joined() + lock.unlock() + let sanitized = sanitizeText(textToReturn) if !sanitized.isEmpty { print("SonioxLiveSession: Burst watchdog fired, salvaging partial text: \(sanitized.prefix(30))...") finishOnce(with: .success(sanitized)) @@ -715,16 +741,29 @@ public final class SonioxSessionPool { let target = SonioxSettings.shared.targetLanguage.isEnabled ? SonioxSettings.shared.targetLanguage.rawValue : nil let session = SonioxLiveSession(languageHints: hints, targetLanguage: target) - // Allow 400ms for connection & initial ping - DispatchQueue.main.asyncAfter(deadline: .now() + 0.4) { [weak self] in + session.onConnectionStateChanged = { [weak self, weak session] connected in + guard let self = self, let session = session else { return } + self.lock.lock() + defer { self.lock.unlock() } + if connected { + session.startKeepAliveHeartbeat() + self.prewarmedSession = session + self.prewarmedSessionCreatedAt = Date() + self.isPrewarming = false + print("SonioxSessionPool: Fresh prewarmed session connected & ready.") + } else { + self.isPrewarming = false + } + } + + // Watchdog in case connection hangs indefinitely + DispatchQueue.global().asyncAfter(deadline: .now() + 15.0) { [weak self] in guard let self = self else { return } self.lock.lock() - session.startKeepAliveHeartbeat() - self.prewarmedSession = session - self.prewarmedSessionCreatedAt = Date() - self.isPrewarming = false + if self.isPrewarming && self.prewarmedSession == nil { + self.isPrewarming = false + } self.lock.unlock() - print("SonioxSessionPool: Fresh prewarmed session initialized & heartbeat started.") } } } diff --git a/src/TextProcessor.swift b/src/TextProcessor.swift index ce5dfa9..e7ab682 100644 --- a/src/TextProcessor.swift +++ b/src/TextProcessor.swift @@ -54,6 +54,10 @@ public final class TextProcessor { result = normalizeText(result) } + // 5. Wake Word & End Trigger sanitization + result = stripLeadingWakeWord(result) + result = stripTrailingEndTrigger(result) + return result.trimmingCharacters(in: .whitespacesAndNewlines) } @@ -253,4 +257,30 @@ public final class TextProcessor { } return t.trimmingCharacters(in: .whitespacesAndNewlines) } + + public func stripLeadingWakeWord(_ text: String) -> String { + var t = text.trimmingCharacters(in: .whitespacesAndNewlines) + let pattern = #"^(غلام|gholam|gulam)[،,:\s]+"# + if let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive) { + let range = NSRange(location: 0, length: t.utf16.count) + t = regex.stringByReplacingMatches(in: t, options: [], range: range, withTemplate: "") + } + return t.trimmingCharacters(in: .whitespacesAndNewlines) + } + + public func stripTrailingEndTrigger(_ text: String) -> String { + var t = text.trimmingCharacters(in: .whitespacesAndNewlines) + let pattern = #"[،,:\s]+(and|تمام|end)[.!?]?$"# + if let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive) { + let range = NSRange(location: 0, length: t.utf16.count) + t = regex.stringByReplacingMatches(in: t, options: [], range: range, withTemplate: "") + } + let singlePattern = #"^(and|تمام|end)[.!?]?$"# + if let regex = try? NSRegularExpression(pattern: singlePattern, options: .caseInsensitive) { + let range = NSRange(location: 0, length: t.utf16.count) + t = regex.stringByReplacingMatches(in: t, options: [], range: range, withTemplate: "") + } + return t.trimmingCharacters(in: .whitespacesAndNewlines) + } + }