From d0e4a5d2a6544773e7046e9786e2a90e3f93e0b5 Mon Sep 17 00:00:00 2001 From: Ali Alavi Date: Mon, 24 Aug 2026 06:07:33 +0000 Subject: [PATCH] feat(sync): full multi-line paragraph & newline support with ultra-low latency 35ms mirroring --- .../java/com/soniox/remotemic/MainActivity.kt | 11 +++++++-- mac/src/FocusedInputSync.swift | 12 ++++------ server/relay_server.py | 23 +++++++++++++------ 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt b/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt index 98742d5..77ab98a 100644 --- a/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt +++ b/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt @@ -274,12 +274,19 @@ class MainActivity : AppCompatActivity() { lastLocalUserEditTime = System.currentTimeMillis() val cur = binding.etTranscript.selectionStart.coerceIn(0, text.length) - // 80ms debounce: sends clean replacement to Mac in real-time as user types/deletes + // Immediate sync for newlines (\n), crisp 35ms debounce for general typing + val isNewlineEdit = count == 1 && s?.subSequence(start, start + count)?.contains('\n') == true + val delayMs = if (isNewlineEdit) 0L else 35L + pendingSyncRunnable?.let { debounceHandler.removeCallbacks(it) } pendingSyncRunnable = Runnable { streamDictationClient?.sendPhoneEdit(text, cur) } - debounceHandler.postDelayed(pendingSyncRunnable!!, 80) + if (delayMs == 0L) { + debounceHandler.post(pendingSyncRunnable!!) + } else { + debounceHandler.postDelayed(pendingSyncRunnable!!, delayMs) + } } } override fun afterTextChanged(s: Editable?) {} diff --git a/mac/src/FocusedInputSync.swift b/mac/src/FocusedInputSync.swift index 90cbae9..294f47c 100644 --- a/mac/src/FocusedInputSync.swift +++ b/mac/src/FocusedInputSync.swift @@ -247,18 +247,16 @@ public final class FocusedInputSync { @discardableResult public func applyRemoteUpdate(text: String, cursor: Int? = nil, isFullReplace: Bool = true) -> Bool { isApplyingRemoteChange = true - remoteChangeExpiryTime = Date().timeIntervalSince1970 + 0.40 + remoteChangeExpiryTime = Date().timeIntervalSince1970 + 0.25 defer { - DispatchQueue.main.asyncAfter(deadline: .now() + 0.40) { + DispatchQueue.main.asyncAfter(deadline: .now() + 0.25) { self.isApplyingRemoteChange = false } } - // Normalize text - var cleanText = text.components(separatedBy: .newlines).joined(separator: " ") - cleanText = cleanText.replacingOccurrences(of: "\t", with: " ") - cleanText = cleanText.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) + // Normalize line endings and preserve intentional multiline text (\n) + var cleanText = text.replacingOccurrences(of: "\r\n", with: "\n").replacingOccurrences(of: "\r", with: "\n") cleanText = cleanText.trimmingCharacters(in: .whitespacesAndNewlines) let targetCursor = cursor ?? cleanText.count @@ -270,7 +268,7 @@ public final class FocusedInputSync { // Universal Quartz CGEvent Keystroke Engine (Cmd+A -> Cmd+V / Backspace or pure Cmd+V) // Works 100% reliably across native, web, and Electron/Chromium apps (e.g. Antigravity, VS Code, Slack, Firefox) - print("FocusedInputSync: ๐Ÿš€ Injecting text into '\(appName)' (replace: \(isFullReplace), len: \(cleanText.count))") + print("FocusedInputSync: ๐Ÿš€ Injecting text into '\(appName)' (replace: \(isFullReplace), len: \(cleanText.count), lines: \(cleanText.components(separatedBy: "\n").count))") if isFullReplace { return executeCleanFullReplace(cleanText) } else { diff --git a/server/relay_server.py b/server/relay_server.py index b73dd73..dea470e 100644 --- a/server/relay_server.py +++ b/server/relay_server.py @@ -105,17 +105,25 @@ class SonioxPool: soniox_pool = SonioxPool() -def sanitize_and_flatten_text(text: str) -> str: +def sanitize_text(text: str, allow_multiline: bool = True) -> str: if not text: return "" - flattened = re.sub(r"[\r\n\t]+", " ", text) - flattened = re.sub(r"\s+", " ", flattened).strip() + if allow_multiline: + normalized = text.replace("\r\n", "\n").replace("\r", "\n") + lines = [re.sub(r"[ \t]+", " ", line) for line in normalized.split("\n")] + cleaned = "\n".join(lines).strip("\r\n") + else: + cleaned = re.sub(r"[\r\n\t]+", " ", text) + cleaned = re.sub(r"\s+", " ", cleaned).strip() # Suppress lone punctuation artifacts (e.g. ยซ, ยป, ., ,, !, ?, etc.) - if re.fullmatch(r"[\sยซยป\.\,\ุŒ\ุ›\ุŸ\!\?\:\;\-\โ€“โ€”\"\'\(\)\[\]\{\}]+", flattened): + if re.fullmatch(r"[\sยซยป\.\,\ุŒ\ุ›\ุŸ\!\?\:\;\-\โ€“โ€”\"\'\(\)\[\]\{\}]+", cleaned): return "" - return flattened + return cleaned + +def sanitize_and_flatten_text(text: str) -> str: + return sanitize_text(text, allow_multiline=False) async def broadcast_state(payload_dict: dict, exclude_ws=None): """Broadcasts state snapshot to all connected clients (Mac and Phone) except sender.""" @@ -249,8 +257,9 @@ async def handle_phone_stream_ws(request): # ALL text / sync / insert operations must be broadcast to Mac! if msg_type in ("sync_state", "insert_speech", "speech_insert", "phone_input_edit", "update_input", "paste"): - clean_text = sanitize_and_flatten_text(data.get("text", "")) - if not clean_text and msg_type in ("insert_speech", "speech_insert"): + is_speech = msg_type in ("insert_speech", "speech_insert") + clean_text = sanitize_text(data.get("text", ""), allow_multiline=(not is_speech)) + if not clean_text and is_speech: continue # Do not broadcast empty speech or lone quotes data["text"] = clean_text data["source"] = "android"