From be80173395b141af7e88dad05f1dd441f696e752 Mon Sep 17 00:00:00 2001 From: Ali Alavi Date: Sun, 23 Aug 2026 19:24:59 +0000 Subject: [PATCH] fix(sync): universal synthetic paste engine, ghost sync suppression for web/electron apps and robust bidirectional state machine --- .../java/com/soniox/remotemic/MainActivity.kt | 36 +++- .../soniox/remotemic/StreamDictationClient.kt | 32 +-- mac/src/FocusedInputSync.swift | 201 +++++++++++++----- 3 files changed, 195 insertions(+), 74 deletions(-) diff --git a/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt b/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt index 165ffb6..a01d3ed 100644 --- a/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt +++ b/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt @@ -88,7 +88,7 @@ class MainActivity : AppCompatActivity() { setupUI() checkPermissions() - AppLogger.log("Main", "اپلیکیشن راه‌اندازی شد (طراحی ریسپانسیو و فوکوس کامل اینپوت‌باکس روی کیبورد)") + AppLogger.log("Main", "اپلیکیشن همگام‌سازی صوتی سانی‌اوکس راه‌اندازی شد (نسخه بهینه‌شده فوق‌سریع v5.2)") // Initialize Collaborative WebSocket Client streamDictationClient = StreamDictationClient( @@ -100,7 +100,7 @@ class MainActivity : AppCompatActivity() { ) }, onSyncStateReceived = { state -> - // Drop echoes and do not interrupt active user editing on phone + // Drop echoes originated from phone itself if (state.source != "android") { val timeSinceLocalEdit = System.currentTimeMillis() - lastLocalUserEditTime @@ -109,6 +109,19 @@ class MainActivity : AppCompatActivity() { return@StreamDictationClient } + // Monotonic revision check + if (state.revision > 0 && state.revision < currentRevision) { + return@StreamDictationClient + } + + // Ghost empty sync protection for opaque/Electron apps (e.g. Antigravity) + if (state.text.isEmpty() && lastLocalText.isNotEmpty() && state.source == "mac") { + if (state.app.isNotEmpty() && state.app != "Desktop" && state.app != "App") { + binding.tvMacStatus.text = "متصل به ${state.app} 🖥️" + } + return@StreamDictationClient + } + if (state.text != lastLocalText) { isApplyingRemoteUpdate = true lastLocalText = state.text @@ -212,10 +225,19 @@ class MainActivity : AppCompatActivity() { val prefix = if (start > 0) current.substring(0, start) else "" val suffix = if (end < current.length) current.substring(end) else "" - val formattedSpeech = if (prefix.isNotEmpty() && !prefix.endsWith(" ") && !speechText.startsWith(" ")) { - " $speechText" - } else { - speechText + val trimmedSpeech = speechText.trim() + if (trimmedSpeech.isEmpty()) return + + // Smart spacing for Persian / English word boundaries + val needsPreSpace = prefix.isNotEmpty() && !prefix.endsWith(" ") && !prefix.endsWith("\n") + val needsPostSpace = suffix.isNotEmpty() && !suffix.startsWith(" ") && !suffix.startsWith("\n") && + !suffix.startsWith(",") && !suffix.startsWith("،") && !suffix.startsWith(".") && + !suffix.startsWith("؟") && !suffix.startsWith("!") && !suffix.startsWith(":") + + val formattedSpeech = buildString { + if (needsPreSpace) append(" ") + append(trimmedSpeech) + if (needsPostSpace) append(" ") } val mergedText = "$prefix$formattedSpeech$suffix" @@ -228,7 +250,7 @@ class MainActivity : AppCompatActivity() { binding.etTranscript.setSelection(newCursor) isApplyingRemoteUpdate = false - AppLogger.log("Main", "تزریق گفتار در نشانگر: '$speechText' (موقعیت جدید: $newCursor)") + AppLogger.log("Main", "تزریق گفتار در نشانگر: '$trimmedSpeech' (موقعیت جدید: $newCursor)") streamDictationClient?.sendLocalSyncState(mergedText, newCursor) } diff --git a/android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt b/android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt index a7a1011..7cf0ede 100644 --- a/android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt +++ b/android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt @@ -52,7 +52,7 @@ class StreamDictationClient( .retryOnConnectionFailure(true) .build() - private var webSocket: WebSocket? = null + private var activeWebSocket: WebSocket? = null private val isConnected = AtomicBoolean(false) private var currentSessionId: String = "" private var isSessionActive = AtomicBoolean(false) @@ -63,21 +63,21 @@ class StreamDictationClient( @Synchronized fun connectWebSocket() { - if (isConnected.get() && webSocket != null) return + if (isConnected.get() && activeWebSocket != null) return val cleanHost = host.removePrefix("http://").removePrefix("https://").removePrefix("ws://").removePrefix("wss://") val wsUrl = "ws://$cleanHost/ws/stream" AppLogger.log(tag, "اتصال به سوکت همگام‌سازی بلادرنگ: $wsUrl") val req = Request.Builder().url(wsUrl).build() - webSocket = okHttpClient.newWebSocket(req, object : WebSocketListener() { - override fun onOpen(ws: WebSocket, response: Response) { + activeWebSocket = okHttpClient.newWebSocket(req, object : WebSocketListener() { + override fun onOpen(webSocket: WebSocket, response: Response) { AppLogger.log(tag, "🟢 سوکت همگام‌سازی بلادرنگ متصل شد") isConnected.set(true) mainHandler.post { onConnectionStateChanged(true) } } - override fun onMessage(ws: WebSocket, text: String) { + override fun onMessage(webSocket: WebSocket, text: String) { try { val json = JSONObject(text) val type = json.optString("type") @@ -121,18 +121,18 @@ class StreamDictationClient( } } - override fun onFailure(ws: WebSocket, t: Throwable, response: Response?) { + override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { AppLogger.log(tag, "🔴 قطع سوکت: ${t.message}. تلاش مجدد...") isConnected.set(false) - webSocket = null + activeWebSocket = null mainHandler.post { onConnectionStateChanged(false) } mainHandler.postDelayed({ connectWebSocket() }, 2000) } - override fun onClosed(ws: WebSocket, code: Int, reason: String) { + override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { AppLogger.log(tag, "سوکت بسته شد ($code). تلاش مجدد...") isConnected.set(false) - webSocket = null + activeWebSocket = null mainHandler.post { onConnectionStateChanged(false) } mainHandler.postDelayed({ connectWebSocket() }, 2000) } @@ -143,7 +143,7 @@ class StreamDictationClient( * Broadcasts phone's updated text & cursor to Mac in sub-15ms */ fun sendLocalSyncState(text: String, cursor: Int, selection: Int = 0) { - if (!isConnected.get() || webSocket == null) { + if (!isConnected.get() || activeWebSocket == null) { connectWebSocket() } val payload = JSONObject().apply { @@ -154,7 +154,7 @@ class StreamDictationClient( put("selection", selection) put("timestamp", System.currentTimeMillis() / 1000.0) }.toString() - webSocket?.send(payload) + activeWebSocket?.send(payload) } @SuppressLint("MissingPermission") @@ -165,7 +165,7 @@ class StreamDictationClient( isRecording.set(true) isSessionActive.set(true) - if (!isConnected.get() || webSocket == null) { + if (!isConnected.get() || activeWebSocket == null) { connectWebSocket() } @@ -174,7 +174,7 @@ class StreamDictationClient( put("session_id", currentSessionId) put("cursor_pos", cursorPos) }.toString() - webSocket?.send(startFrame) + activeWebSocket?.send(startFrame) AppLogger.log(tag, "🎙️ شروع استریم گفتار در موقعیت نشانگر $cursorPos...") val minBufferSize = AudioRecord.getMinBufferSize(sampleRate, channelConfig, audioFormat) @@ -207,7 +207,7 @@ class StreamDictationClient( val bytesRead = audioRecord?.read(chunk, 0, chunk.size) ?: -1 if (bytesRead > 0) { val slice = if (bytesRead == chunk.size) chunk.clone() else chunk.copyOf(bytesRead) - webSocket?.send(slice.toByteString()) + activeWebSocket?.send(slice.toByteString()) var sum = 0.0 val samplesCount = bytesRead / 2 @@ -255,7 +255,7 @@ class StreamDictationClient( put("session_id", currentSessionId) put("cursor_pos", cursorPos) }.toString() - webSocket?.send(stopFrame) + activeWebSocket?.send(stopFrame) AppLogger.log(tag, "⏹️ پایان ضبط. دریافت متن نهایی...") } @@ -263,7 +263,7 @@ class StreamDictationClient( isRecording.set(false) try { audioRecord?.release() - webSocket?.close(1000, "Client Shutdown") + activeWebSocket?.close(1000, "Client Shutdown") } catch (e: Exception) {} } } diff --git a/mac/src/FocusedInputSync.swift b/mac/src/FocusedInputSync.swift index a66b068..c298bc4 100644 --- a/mac/src/FocusedInputSync.swift +++ b/mac/src/FocusedInputSync.swift @@ -32,6 +32,16 @@ public final class FocusedInputSync { private var localRevision: Int64 = 0 private var remoteChangeExpiryTime: Double = 0 + // Known Web, Electron, Terminal, and Custom UI apps that do NOT accept direct AXValue writes + // but require fast, reliable native keystroke paste (Cmd+V / Cmd+A + Cmd+V) + private let pastePreferredApps: Set = [ + "antigravity", "antigravity helper", "google chrome", "chromium", "brave browser", + "arc", "microsoft edge", "firefox", "safari", "code", "cursor", "visual studio code", + "slack", "discord", "telegram", "whatsapp", "signal", "ghostty", "iterm2", "iterm", + "terminal", "alacritty", "kitty", "jetbrains", "idea", "webstorm", "pycharm", + "datagrip", "sublime text", "notion", "obsidian", "linear", "warp" + ] + private init() { self.systemWideElement = AXUIElementCreateSystemWide() enableGlobalAccessibility() @@ -42,7 +52,19 @@ public final class FocusedInputSync { AXUIElementSetAttributeValue(systemWideElement, "AXManualAccessibility" as CFString, kCFBooleanTrue) } - private func findFocusedDescendant(_ elem: AXUIElement) -> AXUIElement? { + private func isAppPastePreferred(_ appName: String) -> Bool { + let lower = appName.lowercased() + for pref in pastePreferredApps { + if lower.contains(pref) { + return true + } + } + return false + } + + private func findFocusedDescendant(_ elem: AXUIElement, depth: Int = 0) -> AXUIElement? { + if depth > 12 { return nil } + var isFocusedObj: CFTypeRef? if AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &isFocusedObj) == .success, let isFocused = isFocusedObj as? Bool, isFocused { @@ -58,7 +80,7 @@ public final class FocusedInputSync { if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, let children = childrenObj as? [AXUIElement] { for child in children { - if let found = findFocusedDescendant(child) { + if let found = findFocusedDescendant(child, depth: depth + 1) { return found } } @@ -66,8 +88,8 @@ public final class FocusedInputSync { return nil } - public func getFocusedElement() -> (AXUIElement, String)? { - guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil } + public func getFocusedElement() -> (AXUIElement?, String) { + guard let frontApp = NSWorkspace.shared.frontmostApplication else { return (nil, "App") } let appName = frontApp.localizedName ?? "App" let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) @@ -76,7 +98,7 @@ public final class FocusedInputSync { var targetElem: AXUIElement? - // 1. Try system wide focused element + // 1. Try system-wide focused element var focusedObj: CFTypeRef? if AXUIElementCopyAttributeValue(systemWideElement, kAXFocusedUIElementAttribute as CFString, &focusedObj) == .success, let obj = focusedObj { @@ -103,25 +125,59 @@ public final class FocusedInputSync { } } - // 3. Recursive search in tree + // 3. Try App focused window's focused element if targetElem == nil { - targetElem = findFocusedDescendant(appElem) + var focusedWinObj: CFTypeRef? + if AXUIElementCopyAttributeValue(appElem, kAXFocusedWindowAttribute as CFString, &focusedWinObj) == .success, + let win = focusedWinObj { + var winFocObj: CFTypeRef? + if AXUIElementCopyAttributeValue((win as! AXUIElement), kAXFocusedUIElementAttribute as CFString, &winFocObj) == .success, + let obj = winFocObj { + targetElem = (obj as! AXUIElement) + } + } } - if let elem = targetElem { - return (elem, appName) + // 4. Recursive search in tree + if targetElem == nil { + targetElem = findFocusedDescendant(appElem) } - return nil + + return (targetElem, appName) } - /// Inspects the current focused element and returns a state snapshot if changed + /// Inspects the current focused element and returns a state snapshot if changed on Mac public func inspectCurrentState() -> MacInputState? { let now = Date().timeIntervalSince1970 if isApplyingRemoteChange || now < remoteChangeExpiryTime { return nil } - guard let (elem, appName) = getFocusedElement() else { return nil } + let (elemOpt, appName) = getFocusedElement() + + // If app changed + let appChanged = (appName != lastObservedApp) + if appChanged { + lastObservedApp = appName + } + + guard let elem = elemOpt else { + // When no native AX element is available (e.g. Electron web canvas or Antigravity), + // DO NOT emit empty text which would wipe out the mobile client transcript! + if appChanged && !lastObservedText.isEmpty { + localRevision += 1 + return MacInputState(app: appName, text: lastObservedText, cursor: lastObservedCursor, selection: 0, revision: localRevision) + } + return nil + } + + // Check role + var roleObj: CFTypeRef? + AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) + let role = roleObj as? String ?? "" + + // Only inspect text if element is a recognized text-capable input + let isTextRole = (role == "AXTextField" || role == "AXTextArea" || role == "AXComboBox" || role == "AXSearchField") // Extract text var text = "" @@ -135,7 +191,7 @@ public final class FocusedInputSync { } } - if text.isEmpty { + if text.isEmpty && isTextRole { var countObj: CFTypeRef? if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countObj) == .success, let count = countObj as? Int, count > 0 { @@ -150,6 +206,12 @@ public final class FocusedInputSync { } } + // Ghost empty suppression: If an opaque web app reports empty string, but we had existing text, + // and element is NOT an explicit empty native text field, ignore the empty read. + if text.isEmpty && !lastObservedText.isEmpty && !isTextRole && isAppPastePreferred(appName) { + return nil + } + // Extract cursor & selection var cursor = text.count var selLen = 0 @@ -164,7 +226,7 @@ public final class FocusedInputSync { } // Check if text or cursor actually changed on Mac - if text == lastObservedText && cursor == lastObservedCursor && appName == lastObservedApp { + if text == lastObservedText && cursor == lastObservedCursor && !appChanged { return nil } @@ -180,10 +242,11 @@ public final class FocusedInputSync { @discardableResult public func applyRemoteUpdate(text: String, cursor: Int? = nil, isFullReplace: Bool = true) -> Bool { isApplyingRemoteChange = true - remoteChangeExpiryTime = Date().timeIntervalSince1970 + 0.45 // 450ms quiet window + // 750ms quiet window to prevent self-echo and race conditions + remoteChangeExpiryTime = Date().timeIntervalSince1970 + 0.75 defer { - DispatchQueue.main.asyncAfter(deadline: .now() + 0.45) { + DispatchQueue.main.asyncAfter(deadline: .now() + 0.75) { self.isApplyingRemoteChange = false } } @@ -198,62 +261,98 @@ public final class FocusedInputSync { lastObservedText = cleanText lastObservedCursor = targetCursor - guard let (elem, appName) = getFocusedElement() else { - // Fallback: clipboard paste - lastObservedApp = NSWorkspace.shared.frontmostApplication?.localizedName ?? "App" + let (elemOpt, appName) = getFocusedElement() + lastObservedApp = appName + + // If the target app is a Web/Electron/IDE application (e.g. Antigravity, Chrome, VS Code, Discord, Slack) + // or no native AX element was resolved, bypass AXValue write and execute rock-solid synthetic paste! + if isAppPastePreferred(appName) || elemOpt == nil { + print("FocusedInputSync: 🎯 Target app '\(appName)' is Web/Electron/Paste-preferred. Executing Universal Synthetic Paste.") return pasteViaKeystroke(cleanText, isFullReplace: isFullReplace) } - lastObservedApp = appName - if isFullReplace { - // Try setting AXValue directly - let setErr = AXUIElementSetAttributeValue(elem, kAXValueAttribute as CFString, cleanText as CFTypeRef) - if setErr == .success { - if let cursor = cursor { - var range = CFRange(location: min(cursor, cleanText.count), length: 0) - if let axRange = AXValueCreate(.cfRange, &range) { - AXUIElementSetAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, axRange) + // For Native AppKit/Cocoa apps (e.g. TextEdit, Notes, Finder): + if let elem = elemOpt { + if isFullReplace { + let setErr = AXUIElementSetAttributeValue(elem, kAXValueAttribute as CFString, cleanText as CFTypeRef) + if setErr == .success { + if let cursor = cursor { + var range = CFRange(location: min(cursor, cleanText.count), length: 0) + if let axRange = AXValueCreate(.cfRange, &range) { + AXUIElementSetAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, axRange) + } } + print("FocusedInputSync: ✅ Updated native AXValue for \(appName)") + return true + } + } else { + let setSelErr = AXUIElementSetAttributeValue(elem, kAXSelectedTextAttribute as CFString, cleanText as CFTypeRef) + if setSelErr == .success { + print("FocusedInputSync: ✅ Inserted native text via AXSelectedText for \(appName)") + return true } - print("FocusedInputSync: ✅ Updated AXValue for \(appName)") - return true - } - } else { - // Try setting selected text - let setSelErr = AXUIElementSetAttributeValue(elem, kAXSelectedTextAttribute as CFString, cleanText as CFTypeRef) - if setSelErr == .success { - print("FocusedInputSync: ✅ Inserted text via AXSelectedText for \(appName)") - return true } } + // Reliable fallback for any situation return pasteViaKeystroke(cleanText, isFullReplace: isFullReplace) } + /** + * Ultra-reliable synthetic keystroke injection (Cmd+A -> Cmd+V) + * Includes proper key-dwell times, modifier flag persistence, and clipboard preservation. + */ private func pasteViaKeystroke(_ text: String, isFullReplace: Bool) -> Bool { let pasteboard = NSPasteboard.general + + // 1. Snapshot previous clipboard to restore after paste + let oldString = pasteboard.string(forType: .string) + + // 2. Set new text to pasteboard pasteboard.clearContents() pasteboard.setString(text, forType: .string) - let src = CGEventSource(stateID: .hidSystemState) + let src = CGEventSource(stateID: .combinedSessionState) if isFullReplace { - // Select all: Cmd + A - let aDown = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: true) - aDown?.flags = .maskCommand - let aUp = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: false) - aDown?.post(tap: .cghidEventTap) - aUp?.post(tap: .cghidEventTap) + // Select all: Cmd + A (virtualKey 0 = 'a') + if let aDown = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: true) { + aDown.flags = .maskCommand + aDown.post(tap: .cghidEventTap) + } + usleep(12000) // 12ms key-down dwell time + + if let aUp = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: false) { + aUp.flags = .maskCommand + aUp.post(tap: .cghidEventTap) + } - usleep(20000) + // 35ms settling delay for Electron / DOM selection update + usleep(35000) + } + + // Paste: Cmd + V (virtualKey 9 = 'v') + if let vDown = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: true) { + vDown.flags = .maskCommand + vDown.post(tap: .cghidEventTap) } + usleep(12000) // 12ms key-down dwell time - // Paste: Cmd + V - let vDown = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: true) - vDown?.flags = .maskCommand - let vUp = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: false) - vDown?.post(tap: .cghidEventTap) - vUp?.post(tap: .cghidEventTap) + if let vUp = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: false) { + vUp.flags = .maskCommand + vUp.post(tap: .cghidEventTap) + } + + // 3. Asynchronously restore previous clipboard content after 250ms + if let previousText = oldString, previousText != text { + DispatchQueue.global(qos: .utility).asyncAfter(deadline: .now() + 0.25) { + // If pasteboard hasn't been changed by user in the meantime, restore + if pasteboard.string(forType: .string) == text { + pasteboard.clearContents() + pasteboard.setString(previousText, forType: .string) + } + } + } return true }