17 Commits

Author SHA1 Message Date
Ali Alavi d0e4a5d2a6 feat(sync): full multi-line paragraph & newline support with ultra-low latency 35ms mirroring 3 days ago
Ali Alavi 8abb82ad84 fix(sync): live cursor tracking and instant touch reconnection for Figma/Docs mirroring 3 days ago
Ali Alavi 50d185e69c feat(network): resilient dual-host LAN/WAN failover, fast 3.5s connect timeout, and 4s keep-alive ping 3 days ago
Ali Alavi 52f5c119c0 fix(mac): ensure hotkey registration on launch and direct modern macOS Sequoia Accessibility pane opening 3 days ago
Ali Alavi 3491f30d23 fix(mac): eliminate false-success AXValue setter and enforce universal Quartz CGEvent injection for Chromium/Electron and Antigravity 3 days ago
Ali Alavi 1373f81406 feat(sync): real-time live bi-directional text mirroring with clean full replacement and instant backspace/clear sync 3 days ago
Ali Alavi 0effbaadcf fix(mac): separate passive state sync from active voice dictation and force replace 3 days ago
Ali Alavi cbff21002c fix: eliminate double paste, debounce interference and lone quote artifacts 3 days ago
Ali Alavi c23a2a5c08 fix(mac): explicit AXIsProcessTrustedWithOptions prompt trigger with TCC reset and stable codesign identifier 3 days ago
Ali Alavi 4c096cc65b feat(ui): elegant minimal black mic icon, spacious non-clipping ambient glow, and centered bottom voice layout 3 days ago
Ali Alavi 46a5362bf8 fix(gateway): route insert_speech and speech_insert payloads to Mac clients without drop 3 days ago
Ali Alavi 4aac4e2490 feat(ui): premium black studio microphone icon with Apple Mac accent badge on frost titanium button 3 days ago
Ali Alavi 3202d75da1 fix(sync): pure clean Quartz Cmd+V insertion for voice typing without Cmd+A clobber 3 days ago
Ali Alavi c246cd05f0 fix(mac): pure quartz cgevent modifier sequence injection with zero applescript automation dependency 3 days ago
Ali Alavi fd15d992c4 feat: resilient mobile websocket reconnection with exponential backoff and fast 350ms speech finalize 3 days ago
Ali Alavi 20bcb17c68 fix(mac): process-targeted AppleScript injection and non-activating floating HUD window 3 days ago
Ali Alavi be80173395 fix(sync): universal synthetic paste engine, ghost sync suppression for web/electron apps and robust bidirectional state machine 3 days ago
  1. 142
      android/app/src/main/java/com/soniox/remotemic/MainActivity.kt
  2. 104
      android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt
  3. 14
      android/app/src/main/res/drawable/bg_glow.xml
  4. 10
      android/app/src/main/res/drawable/bg_mic_button.xml
  5. 3
      android/app/src/main/res/drawable/bg_mic_button_active.xml
  6. 42
      android/app/src/main/res/drawable/ic_mic_mac.xml
  7. 16
      android/app/src/main/res/drawable/ic_mic_minimal.xml
  8. 47
      android/app/src/main/res/layout/activity_main.xml
  9. 5
      mac/package.sh
  10. 50
      mac/src/AppDelegate.swift
  11. 246
      mac/src/FocusedInputSync.swift
  12. 11
      mac/src/HUDOverlay.swift
  13. 20
      mac/src/RelayClient.swift
  14. 58
      server/relay_server.py

142
android/app/src/main/java/com/soniox/remotemic/MainActivity.kt

@ -41,6 +41,7 @@ import okhttp3.OkHttpClient
import okhttp3.Request import okhttp3.Request
import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.RequestBody.Companion.toRequestBody
import org.json.JSONObject import org.json.JSONObject
import java.util.concurrent.TimeUnit
class MainActivity : AppCompatActivity() { class MainActivity : AppCompatActivity() {
@ -88,7 +89,7 @@ class MainActivity : AppCompatActivity() {
setupUI() setupUI()
checkPermissions() checkPermissions()
AppLogger.log("Main", "اپلیکیشن راه‌اندازی شد (طراحی ریسپانسیو و فوکوس کامل اینپوت‌باکس روی کیبورد)")
AppLogger.log("Main", "اپلیکیشن همگام‌سازی صوتی سانی‌اوکس راه‌اندازی شد (نسخه بهینه‌شده فوق‌سریع v5.6)")
// Initialize Collaborative WebSocket Client // Initialize Collaborative WebSocket Client
streamDictationClient = StreamDictationClient( streamDictationClient = StreamDictationClient(
@ -100,12 +101,25 @@ class MainActivity : AppCompatActivity() {
) )
}, },
onSyncStateReceived = { state -> onSyncStateReceived = { state ->
// Drop echoes and do not interrupt active user editing on phone
// Drop echoes originated from phone itself
if (state.source != "android") { if (state.source != "android") {
val timeSinceLocalEdit = System.currentTimeMillis() - lastLocalUserEditTime val timeSinceLocalEdit = System.currentTimeMillis() - lastLocalUserEditTime
// If user is actively typing or backspacing right now on phone, do not override with remote echo
if (timeSinceLocalEdit < 1000L && binding.etTranscript.hasFocus()) {
// If user is actively typing or recording right now on phone, do not override
if ((timeSinceLocalEdit < 1000L && binding.etTranscript.hasFocus()) || isCurrentlyRecording) {
return@StreamDictationClient
}
// Monotonic revision check
if (state.revision > 0 && state.revision < currentRevision) {
return@StreamDictationClient
}
// Ghost empty sync protection for opaque apps
if (state.text.isEmpty() && lastLocalText.isNotEmpty() && state.source == "mac") {
if (state.app.isNotEmpty() && state.app != "Desktop" && state.app != "App") {
binding.tvMacStatus.text = "متصل به ${state.app} 🖥️"
}
return@StreamDictationClient return@StreamDictationClient
} }
@ -137,15 +151,17 @@ class MainActivity : AppCompatActivity() {
binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_blue)) binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_blue))
}, },
onAudioLevel = { level -> onAudioLevel = { level ->
val scale = 1.0f + (level * 0.35f)
if (isCurrentlyRecording) {
val scale = 1.0f + (level * 0.22f)
binding.viewGlow.scaleX = scale binding.viewGlow.scaleX = scale
binding.viewGlow.scaleY = scale binding.viewGlow.scaleY = scale
}
}, },
onSpeechCompleted = { finalText -> onSpeechCompleted = { finalText ->
vibrate(100) vibrate(100)
if (finalText.isNotEmpty()) { if (finalText.isNotEmpty()) {
insertSpeechAtCursor(finalText) insertSpeechAtCursor(finalText)
binding.tvInstruction.text = "✨ گفتار در نشانگر درج و با مک همگام شد"
binding.tvInstruction.text = "✨ گفتار با مک همگام شد"
binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_green)) binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_green))
} else { } else {
binding.tvInstruction.text = "صدایی تشخیص داده نشد" binding.tvInstruction.text = "صدایی تشخیص داده نشد"
@ -160,18 +176,15 @@ class MainActivity : AppCompatActivity() {
} }
/** /**
* Dual-engine keyboard visibility detector (WindowInsets + OnGlobalLayoutListener)
* Guarantees 100% detection on all Android versions and keyboards.
* Dual-engine keyboard visibility detector
*/ */
private fun setupKeyboardVisibilityDetection() { private fun setupKeyboardVisibilityDetection() {
// Engine 1: Modern WindowInsets
ViewCompat.setOnApplyWindowInsetsListener(binding.rootLayout) { _, insets -> ViewCompat.setOnApplyWindowInsetsListener(binding.rootLayout) { _, insets ->
val imeVisible = insets.isVisible(WindowInsetsCompat.Type.ime()) val imeVisible = insets.isVisible(WindowInsetsCompat.Type.ime())
updateKeyboardUIMode(imeVisible) updateKeyboardUIMode(imeVisible)
insets insets
} }
// Engine 2: Global Layout Frame Calculation (Fallback for OEM soft keyboards)
binding.rootLayout.viewTreeObserver.addOnGlobalLayoutListener(object : ViewTreeObserver.OnGlobalLayoutListener { binding.rootLayout.viewTreeObserver.addOnGlobalLayoutListener(object : ViewTreeObserver.OnGlobalLayoutListener {
private val r = Rect() private val r = Rect()
override fun onGlobalLayout() { override fun onGlobalLayout() {
@ -189,14 +202,11 @@ class MainActivity : AppCompatActivity() {
isKeyboardCurrentlyVisible = isKeyboardOpen isKeyboardCurrentlyVisible = isKeyboardOpen
if (isKeyboardOpen) { if (isKeyboardOpen) {
// KEYBOARD OPEN: Hide big mic circle, hide action buttons & divider completely!
// Give 100% of available space above keyboard exclusively to the huge text editor box!
binding.bottomVoiceSection.visibility = View.GONE binding.bottomVoiceSection.visibility = View.GONE
binding.actionDivider.visibility = View.GONE binding.actionDivider.visibility = View.GONE
binding.actionButtonsRow.visibility = View.GONE binding.actionButtonsRow.visibility = View.GONE
binding.tvSubtitle.visibility = View.GONE binding.tvSubtitle.visibility = View.GONE
} else { } else {
// KEYBOARD CLOSED: Restore spacious layout with large glowing mic button & action bar
binding.bottomVoiceSection.visibility = View.VISIBLE binding.bottomVoiceSection.visibility = View.VISIBLE
binding.actionDivider.visibility = View.VISIBLE binding.actionDivider.visibility = View.VISIBLE
binding.actionButtonsRow.visibility = View.VISIBLE binding.actionButtonsRow.visibility = View.VISIBLE
@ -205,6 +215,13 @@ class MainActivity : AppCompatActivity() {
} }
private fun insertSpeechAtCursor(speechText: String) { private fun insertSpeechAtCursor(speechText: String) {
val trimmedSpeech = speechText.trim()
// Suppress empty strings or lone quotes/punctuation marks
if (trimmedSpeech.isEmpty() || trimmedSpeech.matches("^[\\s«»\\.\\,\\،\\؛\\؟\\!\\?\\:\\;\\-\\–—\\\"\\'\\(\\)\\[\\]\\{\\}]+$".toRegex())) {
return
}
val current = binding.etTranscript.text?.toString() ?: "" val current = binding.etTranscript.text?.toString() ?: ""
val start = voiceInsertionCursorStart.coerceIn(0, current.length) val start = voiceInsertionCursorStart.coerceIn(0, current.length)
val end = voiceInsertionCursorEnd.coerceIn(0, current.length) val end = voiceInsertionCursorEnd.coerceIn(0, current.length)
@ -212,15 +229,24 @@ class MainActivity : AppCompatActivity() {
val prefix = if (start > 0) current.substring(0, start) else "" val prefix = if (start > 0) current.substring(0, start) else ""
val suffix = if (end < current.length) current.substring(end) else "" val suffix = if (end < current.length) current.substring(end) else ""
val formattedSpeech = if (prefix.isNotEmpty() && !prefix.endsWith(" ") && !speechText.startsWith(" ")) {
" $speechText"
} else {
speechText
val needsPreSpace = prefix.isNotEmpty() && !prefix.endsWith(" ") && !prefix.endsWith("\n")
val needsPostSpace = suffix.isNotEmpty() && !suffix.startsWith(" ") && !suffix.startsWith("\n") &&
!suffix.startsWith(",") && !suffix.startsWith("،") && !suffix.startsWith(".") &&
!suffix.startsWith("؟") && !suffix.startsWith("!") && !suffix.startsWith(":")
val formattedSpeech = buildString {
if (needsPreSpace) append(" ")
append(trimmedSpeech)
if (needsPostSpace) append(" ")
} }
val mergedText = "$prefix$formattedSpeech$suffix" val mergedText = "$prefix$formattedSpeech$suffix"
val newCursor = (start + formattedSpeech.length).coerceIn(0, mergedText.length) val newCursor = (start + formattedSpeech.length).coerceIn(0, mergedText.length)
// Cancel any pending debounced sync
pendingSyncRunnable?.let { debounceHandler.removeCallbacks(it) }
pendingSyncRunnable = null
isApplyingRemoteUpdate = true isApplyingRemoteUpdate = true
lastLocalText = mergedText lastLocalText = mergedText
lastLocalUserEditTime = System.currentTimeMillis() lastLocalUserEditTime = System.currentTimeMillis()
@ -228,9 +254,10 @@ class MainActivity : AppCompatActivity() {
binding.etTranscript.setSelection(newCursor) binding.etTranscript.setSelection(newCursor)
isApplyingRemoteUpdate = false isApplyingRemoteUpdate = false
AppLogger.log("Main", "تزریق گفتار در نشانگر: '$speechText' (موقعیت جدید: $newCursor)")
AppLogger.log("Main", "تزریق گفتار در نشانگر: '$trimmedSpeech' (موقعیت جدید: $newCursor)")
streamDictationClient?.sendLocalSyncState(mergedText, newCursor)
// Broadcast full mirrored text to Mac
streamDictationClient?.sendPhoneEdit(mergedText, newCursor)
} }
private fun setupUI() { private fun setupUI() {
@ -247,25 +274,36 @@ class MainActivity : AppCompatActivity() {
lastLocalUserEditTime = System.currentTimeMillis() lastLocalUserEditTime = System.currentTimeMillis()
val cur = binding.etTranscript.selectionStart.coerceIn(0, text.length) val cur = binding.etTranscript.selectionStart.coerceIn(0, text.length)
// Debounce sending to Mac by 60ms to allow 120Hz lag-free backspacing and typing
// Immediate sync for newlines (\n), crisp 35ms debounce for general typing
val isNewlineEdit = count == 1 && s?.subSequence(start, start + count)?.contains('\n') == true
val delayMs = if (isNewlineEdit) 0L else 35L
pendingSyncRunnable?.let { debounceHandler.removeCallbacks(it) } pendingSyncRunnable?.let { debounceHandler.removeCallbacks(it) }
pendingSyncRunnable = Runnable { pendingSyncRunnable = Runnable {
streamDictationClient?.sendLocalSyncState(text, cur)
streamDictationClient?.sendPhoneEdit(text, cur)
}
if (delayMs == 0L) {
debounceHandler.post(pendingSyncRunnable!!)
} else {
debounceHandler.postDelayed(pendingSyncRunnable!!, delayMs)
} }
debounceHandler.postDelayed(pendingSyncRunnable!!, 60)
} }
} }
override fun afterTextChanged(s: Editable?) {} override fun afterTextChanged(s: Editable?) {}
}) })
// Clear Button
// Clear Button (🗑️ پاک‌کردن)
binding.btnClearText.setOnClickListener { binding.btnClearText.setOnClickListener {
isApplyingRemoteUpdate = true isApplyingRemoteUpdate = true
binding.etTranscript.setText("") binding.etTranscript.setText("")
lastLocalText = "" lastLocalText = ""
lastLocalUserEditTime = System.currentTimeMillis() lastLocalUserEditTime = System.currentTimeMillis()
isApplyingRemoteUpdate = false isApplyingRemoteUpdate = false
streamDictationClient?.sendLocalSyncState("", 0)
pendingSyncRunnable?.let { debounceHandler.removeCallbacks(it) }
pendingSyncRunnable = null
streamDictationClient?.sendPhoneEdit("", 0)
binding.tvInstruction.text = getString(R.string.hold_to_speak) binding.tvInstruction.text = getString(R.string.hold_to_speak)
binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.text_secondary)) binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.text_secondary))
} }
@ -293,8 +331,8 @@ class MainActivity : AppCompatActivity() {
binding.tvInstruction.text = "در حال درج متن در مک..." binding.tvInstruction.text = "در حال درج متن در مک..."
binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_yellow)) binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_yellow))
// 1. Direct WebSocket broadcast
streamDictationClient?.sendLocalSyncState(text, cur)
// 1. Direct WebSocket broadcast with force_replace
streamDictationClient?.sendForceReplace(text, cur)
// 2. Direct HTTP Post guarantee // 2. Direct HTTP Post guarantee
lifecycleScope.launch { lifecycleScope.launch {
@ -323,12 +361,16 @@ class MainActivity : AppCompatActivity() {
return when (event.action) { return when (event.action) {
MotionEvent.ACTION_DOWN -> { MotionEvent.ACTION_DOWN -> {
if (checkAudioPermission()) { if (checkAudioPermission()) {
// Fast sync: ensure socket is connected immediately
streamDictationClient?.connectWebSocket()
val selStart = binding.etTranscript.selectionStart val selStart = binding.etTranscript.selectionStart
val selEnd = binding.etTranscript.selectionEnd val selEnd = binding.etTranscript.selectionEnd
val totalLen = binding.etTranscript.text?.length ?: 0
val currentText = binding.etTranscript.text?.toString() ?: ""
val totalLen = currentText.length
voiceInsertionCursorStart = if (selStart >= 0) selStart else totalLen
voiceInsertionCursorEnd = if (selEnd >= 0) selEnd else totalLen
voiceInsertionCursorStart = if (selStart in 0..totalLen) selStart else totalLen
voiceInsertionCursorEnd = if (selEnd in 0..totalLen) selEnd else totalLen
startRecording() startRecording()
} }
@ -336,6 +378,16 @@ class MainActivity : AppCompatActivity() {
} }
MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> { MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> {
if (isCurrentlyRecording) { if (isCurrentlyRecording) {
// Update insertion point to where cursor was right before processing
val selStart = binding.etTranscript.selectionStart
val selEnd = binding.etTranscript.selectionEnd
val currentText = binding.etTranscript.text?.toString() ?: ""
val totalLen = currentText.length
if (selStart in 0..totalLen && selEnd in 0..totalLen) {
voiceInsertionCursorStart = selStart
voiceInsertionCursorEnd = selEnd
}
stopRecordingAndProcess() stopRecordingAndProcess()
} }
true true
@ -346,7 +398,11 @@ class MainActivity : AppCompatActivity() {
private suspend fun sendDirectPaste(text: String, cursor: Int): Boolean { private suspend fun sendDirectPaste(text: String, cursor: Int): Boolean {
return try { return try {
val client = OkHttpClient()
val client = OkHttpClient.Builder()
.connectTimeout(2500, TimeUnit.MILLISECONDS)
.writeTimeout(3000, TimeUnit.MILLISECONDS)
.readTimeout(3000, TimeUnit.MILLISECONDS)
.build()
val json = JSONObject().apply { val json = JSONObject().apply {
put("text", text) put("text", text)
put("cursor_pos", cursor) put("cursor_pos", cursor)
@ -354,13 +410,25 @@ class MainActivity : AppCompatActivity() {
}.toString() }.toString()
val mediaType = "application/json; charset=utf-8".toMediaType() val mediaType = "application/json; charset=utf-8".toMediaType()
val body = json.toRequestBody(mediaType) val body = json.toRequestBody(mediaType)
val hosts = listOf(gatewayHost, "116.16.16.19:8999", "2.180.16.250:8999").distinct()
var success = false
for (h in hosts) {
try {
val req = Request.Builder() val req = Request.Builder()
.url("http://$gatewayHost/paste")
.url("http://$h/paste")
.post(body) .post(body)
.build() .build()
client.newCall(req).execute().use { resp -> client.newCall(req).execute().use { resp ->
resp.isSuccessful
if (resp.isSuccessful) {
success = true
return@use
}
}
if (success) break
} catch (_: Exception) {}
} }
success
} catch (e: Exception) { } catch (e: Exception) {
AppLogger.log("Main", "خطای ارسال دستی: ${e.message}") AppLogger.log("Main", "خطای ارسال دستی: ${e.message}")
false false
@ -456,12 +524,12 @@ class MainActivity : AppCompatActivity() {
} }
private fun startPulseAnimation() { private fun startPulseAnimation() {
val scaleX = PropertyValuesHolder.ofFloat(View.SCALE_X, 1.0f, 1.25f, 1.0f)
val scaleY = PropertyValuesHolder.ofFloat(View.SCALE_Y, 1.0f, 1.25f, 1.0f)
val alpha = PropertyValuesHolder.ofFloat(View.ALPHA, 0.3f, 0.7f, 0.3f)
val scaleX = PropertyValuesHolder.ofFloat(View.SCALE_X, 1.0f, 1.18f, 1.0f)
val scaleY = PropertyValuesHolder.ofFloat(View.SCALE_Y, 1.0f, 1.18f, 1.0f)
val alpha = PropertyValuesHolder.ofFloat(View.ALPHA, 0.4f, 0.85f, 0.4f)
pulseAnimator = ObjectAnimator.ofPropertyValuesHolder(binding.viewGlow, scaleX, scaleY, alpha).apply { pulseAnimator = ObjectAnimator.ofPropertyValuesHolder(binding.viewGlow, scaleX, scaleY, alpha).apply {
duration = 1000
duration = 900
repeatCount = ValueAnimator.INFINITE repeatCount = ValueAnimator.INFINITE
interpolator = AccelerateDecelerateInterpolator() interpolator = AccelerateDecelerateInterpolator()
start() start()
@ -472,7 +540,7 @@ class MainActivity : AppCompatActivity() {
pulseAnimator?.cancel() pulseAnimator?.cancel()
binding.viewGlow.scaleX = 1.0f binding.viewGlow.scaleX = 1.0f
binding.viewGlow.scaleY = 1.0f binding.viewGlow.scaleY = 1.0f
binding.viewGlow.alpha = 0.3f
binding.viewGlow.alpha = 0.0f
} }
private fun vibrate(durationMs: Long) { private fun vibrate(durationMs: Long) {

104
android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt

@ -44,18 +44,26 @@ class StreamDictationClient(
private val isRecording = AtomicBoolean(false) private val isRecording = AtomicBoolean(false)
private val mainHandler = Handler(Looper.getMainLooper()) private val mainHandler = Handler(Looper.getMainLooper())
private val candidateHosts: List<String> = listOf(
host,
if (host.contains("116.16.16.19")) "2.180.16.250:8999" else "116.16.16.19:8999"
).filter { it.isNotBlank() }.distinct()
private var currentHostIndex = 0
private val okHttpClient = OkHttpClient.Builder() private val okHttpClient = OkHttpClient.Builder()
.connectTimeout(5, TimeUnit.SECONDS)
.connectTimeout(3500, TimeUnit.MILLISECONDS)
.readTimeout(0, TimeUnit.MILLISECONDS) // Persistent WebSocket .readTimeout(0, TimeUnit.MILLISECONDS) // Persistent WebSocket
.writeTimeout(5, TimeUnit.SECONDS)
.pingInterval(10, TimeUnit.SECONDS)
.writeTimeout(5000, TimeUnit.MILLISECONDS)
.pingInterval(4, TimeUnit.SECONDS) // 4s active heartbeat to keep NAT tables alive
.retryOnConnectionFailure(true) .retryOnConnectionFailure(true)
.build() .build()
private var webSocket: WebSocket? = null
private var activeWebSocket: WebSocket? = null
private val isConnected = AtomicBoolean(false) private val isConnected = AtomicBoolean(false)
private val isConnecting = AtomicBoolean(false)
private var currentSessionId: String = "" private var currentSessionId: String = ""
private var isSessionActive = AtomicBoolean(false) private var isSessionActive = AtomicBoolean(false)
private var retryAttempt = 0
init { init {
connectWebSocket() connectWebSocket()
@ -63,21 +71,30 @@ class StreamDictationClient(
@Synchronized @Synchronized
fun connectWebSocket() { fun connectWebSocket() {
if (isConnected.get() && webSocket != null) return
if (isConnected.get() && activeWebSocket != null) return
if (isConnecting.getAndSet(true)) return
val cleanHost = host.removePrefix("http://").removePrefix("https://").removePrefix("ws://").removePrefix("wss://")
val targetHost = candidateHosts[currentHostIndex % candidateHosts.size]
val cleanHost = targetHost.removePrefix("http://").removePrefix("https://").removePrefix("ws://").removePrefix("wss://")
val wsUrl = "ws://$cleanHost/ws/stream" val wsUrl = "ws://$cleanHost/ws/stream"
AppLogger.log(tag, "اتصال به سوکت همگام‌سازی بلادرنگ: $wsUrl")
AppLogger.log(tag, "اتصال به سوکت همگام‌سازی بلادرنگ ($targetHost): $wsUrl")
val req = Request.Builder()
.url(wsUrl)
.header("User-Agent", "SonioxAndroidRemote/5.6")
.build()
val req = Request.Builder().url(wsUrl).build()
webSocket = okHttpClient.newWebSocket(req, object : WebSocketListener() {
override fun onOpen(ws: WebSocket, response: Response) {
AppLogger.log(tag, "🟢 سوکت همگام‌سازی بلادرنگ متصل شد")
activeWebSocket = okHttpClient.newWebSocket(req, object : WebSocketListener() {
override fun onOpen(webSocket: WebSocket, response: Response) {
val connectedHost = candidateHosts[currentHostIndex % candidateHosts.size]
AppLogger.log(tag, "🟢 سوکت همگام‌سازی بلادرنگ متصل شد ($connectedHost)")
isConnected.set(true) isConnected.set(true)
isConnecting.set(false)
retryAttempt = 0
mainHandler.post { onConnectionStateChanged(true) } mainHandler.post { onConnectionStateChanged(true) }
} }
override fun onMessage(ws: WebSocket, text: String) {
override fun onMessage(webSocket: WebSocket, text: String) {
try { try {
val json = JSONObject(text) val json = JSONObject(text)
val type = json.optString("type") val type = json.optString("type")
@ -121,40 +138,67 @@ class StreamDictationClient(
} }
} }
override fun onFailure(ws: WebSocket, t: Throwable, response: Response?) {
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
AppLogger.log(tag, "🔴 قطع سوکت: ${t.message}. تلاش مجدد...") AppLogger.log(tag, "🔴 قطع سوکت: ${t.message}. تلاش مجدد...")
isConnected.set(false)
webSocket = null
mainHandler.post { onConnectionStateChanged(false) }
mainHandler.postDelayed({ connectWebSocket() }, 2000)
handleDisconnect()
} }
override fun onClosed(ws: WebSocket, code: Int, reason: String) {
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
AppLogger.log(tag, "سوکت بسته شد ($code). تلاش مجدد...") AppLogger.log(tag, "سوکت بسته شد ($code). تلاش مجدد...")
handleDisconnect()
}
private fun handleDisconnect() {
isConnected.set(false) isConnected.set(false)
webSocket = null
isConnecting.set(false)
activeWebSocket = null
mainHandler.post { onConnectionStateChanged(false) } mainHandler.post { onConnectionStateChanged(false) }
mainHandler.postDelayed({ connectWebSocket() }, 2000)
// Fast failover between LAN and WAN
currentHostIndex++
retryAttempt++
val delayMs = if (retryAttempt <= 2) 350L else minOf(500L * (1L shl minOf(retryAttempt, 2)), 2000L)
mainHandler.postDelayed({ connectWebSocket() }, delayMs)
} }
}) })
} }
/** /**
* Broadcasts phone's updated text & cursor to Mac in sub-15ms
* Broadcasts phone's live text to Mac for immediate real-time mirroring
*/
fun sendPhoneEdit(text: String, cursor: Int, selection: Int = 0) {
if (!isConnected.get() || activeWebSocket == null) {
connectWebSocket()
}
val payload = JSONObject().apply {
put("type", "phone_input_edit")
put("action", "phone_input_edit")
put("source", "android")
put("text", text)
put("cursor", cursor)
put("selection", selection)
put("timestamp", System.currentTimeMillis() / 1000.0)
}.toString()
activeWebSocket?.send(payload)
}
/**
* Broadcasts phone's full updated text & cursor to Mac for replace (Cmd+A + Cmd+V)
*/ */
fun sendLocalSyncState(text: String, cursor: Int, selection: Int = 0) {
if (!isConnected.get() || webSocket == null) {
fun sendForceReplace(text: String, cursor: Int, selection: Int = 0) {
if (!isConnected.get() || activeWebSocket == null) {
connectWebSocket() connectWebSocket()
} }
val payload = JSONObject().apply { val payload = JSONObject().apply {
put("type", "sync_state")
put("type", "update_input")
put("action", "update_input")
put("source", "android") put("source", "android")
put("text", text) put("text", text)
put("cursor", cursor) put("cursor", cursor)
put("selection", selection) put("selection", selection)
put("timestamp", System.currentTimeMillis() / 1000.0) put("timestamp", System.currentTimeMillis() / 1000.0)
}.toString() }.toString()
webSocket?.send(payload)
activeWebSocket?.send(payload)
} }
@SuppressLint("MissingPermission") @SuppressLint("MissingPermission")
@ -165,7 +209,7 @@ class StreamDictationClient(
isRecording.set(true) isRecording.set(true)
isSessionActive.set(true) isSessionActive.set(true)
if (!isConnected.get() || webSocket == null) {
if (!isConnected.get() || activeWebSocket == null) {
connectWebSocket() connectWebSocket()
} }
@ -174,7 +218,7 @@ class StreamDictationClient(
put("session_id", currentSessionId) put("session_id", currentSessionId)
put("cursor_pos", cursorPos) put("cursor_pos", cursorPos)
}.toString() }.toString()
webSocket?.send(startFrame)
activeWebSocket?.send(startFrame)
AppLogger.log(tag, "🎙️ شروع استریم گفتار در موقعیت نشانگر $cursorPos...") AppLogger.log(tag, "🎙️ شروع استریم گفتار در موقعیت نشانگر $cursorPos...")
val minBufferSize = AudioRecord.getMinBufferSize(sampleRate, channelConfig, audioFormat) val minBufferSize = AudioRecord.getMinBufferSize(sampleRate, channelConfig, audioFormat)
@ -207,7 +251,7 @@ class StreamDictationClient(
val bytesRead = audioRecord?.read(chunk, 0, chunk.size) ?: -1 val bytesRead = audioRecord?.read(chunk, 0, chunk.size) ?: -1
if (bytesRead > 0) { if (bytesRead > 0) {
val slice = if (bytesRead == chunk.size) chunk.clone() else chunk.copyOf(bytesRead) val slice = if (bytesRead == chunk.size) chunk.clone() else chunk.copyOf(bytesRead)
webSocket?.send(slice.toByteString())
activeWebSocket?.send(slice.toByteString())
var sum = 0.0 var sum = 0.0
val samplesCount = bytesRead / 2 val samplesCount = bytesRead / 2
@ -255,7 +299,7 @@ class StreamDictationClient(
put("session_id", currentSessionId) put("session_id", currentSessionId)
put("cursor_pos", cursorPos) put("cursor_pos", cursorPos)
}.toString() }.toString()
webSocket?.send(stopFrame)
activeWebSocket?.send(stopFrame)
AppLogger.log(tag, "⏹️ پایان ضبط. دریافت متن نهایی...") AppLogger.log(tag, "⏹️ پایان ضبط. دریافت متن نهایی...")
} }
@ -263,7 +307,7 @@ class StreamDictationClient(
isRecording.set(false) isRecording.set(false)
try { try {
audioRecord?.release() audioRecord?.release()
webSocket?.close(1000, "Client Shutdown")
activeWebSocket?.close(1000, "Client Shutdown")
} catch (e: Exception) {} } catch (e: Exception) {}
} }
} }

14
android/app/src/main/res/drawable/bg_glow.xml

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="utf-8"?>
<shape xmlns:android="http://schemas.android.com/apk/res/android"
android:shape="oval">
<gradient
android:angle="0"
android:endColor="#003B82F6"
android:centerColor="#263B82F6"
android:startColor="#663B82F6"
android:type="radial"
android:gradientRadius="70dp" />
<size
android:width="140dp"
android:height="140dp" />
</shape>

10
android/app/src/main/res/drawable/bg_mic_button.xml

@ -1,13 +1,17 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<ripple xmlns:android="http://schemas.android.com/apk/res/android" <ripple xmlns:android="http://schemas.android.com/apk/res/android"
android:color="#40FFFFFF">
android:color="#33000000">
<item> <item>
<shape android:shape="oval"> <shape android:shape="oval">
<!-- Modern Apple-Style Frost / Titanium Gradient Backdrop -->
<gradient <gradient
android:angle="45" android:angle="45"
android:endColor="#2563EB"
android:startColor="#3B82F6"
android:endColor="#CBD5E1"
android:startColor="#F1F5F9"
android:type="linear" /> android:type="linear" />
<stroke
android:width="2.5dp"
android:color="#94A3B8" />
<size <size
android:width="140dp" android:width="140dp"
android:height="140dp" /> android:height="140dp" />

3
android/app/src/main/res/drawable/bg_mic_button_active.xml

@ -6,6 +6,9 @@
android:endColor="#DC2626" android:endColor="#DC2626"
android:startColor="#F43F5E" android:startColor="#F43F5E"
android:type="linear" /> android:type="linear" />
<stroke
android:width="3dp"
android:color="#FECDD3" />
<size <size
android:width="140dp" android:width="140dp"
android:height="140dp" /> android:height="140dp" />

42
android/app/src/main/res/drawable/ic_mic_mac.xml

@ -0,0 +1,42 @@
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="64dp"
android:height="64dp"
android:viewportWidth="64"
android:viewportHeight="64">
<!-- Microphone Capsule Base & Stand (Deep Sleek Black / Slate) -->
<!-- Mic Top Head / Capsule -->
<path
android:fillColor="#0F172A"
android:pathData="M32,6 C26.48,6 22,10.48 22,16 L22,30 C22,35.52 26.48,40 32,40 C37.52,40 42,35.52 42,30 L42,16 C42,10.48 37.52,6 32,6 Z" />
<!-- Mic Mesh Highlight / Inner Grille Lines -->
<path
android:fillColor="#334155"
android:pathData="M24,20 L40,20 L40,22 L24,22 Z M24,15 L40,15 L40,17 L24,17 Z M26,10 L38,10 L38,12 L26,12 Z" />
<!-- Outer Mic Cradle / U-Bracket -->
<path
android:fillColor="#0F172A"
android:pathData="M48,27 C48,36.2 40.7,43.7 34.5,44.8 L34.5,52 L40,52 C41.1,52 42,52.9 42,54 C42,55.1 41.1,56 40,56 L24,56 C22.9,56 22,55.1 22,54 C22,52.9 22.9,52 24,52 L29.5,52 L29.5,44.8 C23.3,43.7 16,36.2 16,27 L20,27 C20,34 25.4,39.5 32,39.5 C38.6,39.5 44,34 44,27 L48,27 Z" />
<!-- Mac Desktop / Display Badge Accent on Mic Body -->
<!-- Mini Mac Display / Apple Screen Symbol in Sleek Silver Accent -->
<path
android:fillColor="#38BDF8"
android:pathData="M28,26 L36,26 C37.1,26 38,26.9 38,28 L38,33 C38,34.1 37.1,35 36,35 L28,35 C26.9,35 26,34.1 26,33 L26,28 C26,26.9 26.9,26 28,26 Z" />
<path
android:fillColor="#0F172A"
android:pathData="M29,28 L35,28 L35,32 L29,32 Z" />
<path
android:fillColor="#38BDF8"
android:pathData="M30,35 L34,35 L35,37 L29,37 Z" />
<!-- Apple Leaf Accent at Top Corner -->
<path
android:fillColor="#10B981"
android:pathData="M48,10 C48,7.2 46.2,5.2 44,5 C43.8,7.4 45.4,9.8 48,10 Z" />
<path
android:fillColor="#0F172A"
android:pathData="M49.5,10.2 C48.3,10.2 47.3,10.9 46.5,10.9 C45.7,10.9 44.8,10.3 43.6,10.3 C41.6,10.3 39.8,12 39.8,15.2 C39.8,19.2 43.2,24 46.5,24 C47.5,24 48.2,23.3 49.3,23.3 C50.4,23.3 51,24 52.1,24 C54.3,24 56.5,20.8 57.1,19.5 C53.4,17.7 53,12.5 56.6,10.8 C55.3,8.9 53.3,8.7 52.4,8.7 C51.2,8.7 50.3,9.5 49.5,10.2 Z" />
</vector>

16
android/app/src/main/res/drawable/ic_mic_minimal.xml

@ -0,0 +1,16 @@
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="48dp"
android:height="48dp"
android:viewportWidth="24"
android:viewportHeight="24">
<!-- Minimalist Deep Matte Black Microphone Body -->
<path
android:fillColor="#0F172A"
android:pathData="M12,14 C13.66,14 15,12.66 15,11 L15,5 C15,3.34 13.66,2 12,2 C10.34,2 9,3.34 9,5 L9,11 C9,12.66 10.34,14 12,14 Z" />
<!-- Minimalist Outer Base Stand & Cradle -->
<path
android:fillColor="#0F172A"
android:pathData="M17.3,11 C17.3,13.93 14.93,16.3 12,16.3 C9.07,16.3 6.7,13.93 6.7,11 L5,11 C5,14.49 7.72,17.36 11,17.85 L11,21 L13,21 L13,17.85 C16.28,17.36 19,14.49 19,11 L17.3,11 Z" />
</vector>

47
android/app/src/main/res/layout/activity_main.xml

@ -6,6 +6,8 @@
android:layout_width="match_parent" android:layout_width="match_parent"
android:layout_height="match_parent" android:layout_height="match_parent"
android:background="@color/bg_dark" android:background="@color/bg_dark"
android:clipChildren="false"
android:clipToPadding="false"
android:padding="12dp"> android:padding="12dp">
<!-- Header Section --> <!-- Header Section -->
@ -79,7 +81,7 @@
android:background="@drawable/bg_card" android:background="@drawable/bg_card"
android:contentDescription="لاگ‌ها" android:contentDescription="لاگ‌ها"
android:padding="8dp" android:padding="8dp"
android:src="@drawable/ic_mic"
android:src="@drawable/ic_mic_minimal"
app:tint="@color/accent_blue" /> app:tint="@color/accent_blue" />
</LinearLayout> </LinearLayout>
@ -89,7 +91,7 @@
android:layout_width="match_parent" android:layout_width="match_parent"
android:layout_height="0dp" android:layout_height="0dp"
android:layout_marginTop="8dp" android:layout_marginTop="8dp"
android:layout_marginBottom="8dp"
android:layout_marginBottom="16dp"
android:background="@drawable/bg_card" android:background="@drawable/bg_card"
android:orientation="vertical" android:orientation="vertical"
android:padding="12dp" android:padding="12dp"
@ -133,7 +135,7 @@
android:textSize="11sp" /> android:textSize="11sp" />
</LinearLayout> </LinearLayout>
<!-- Fully Editable Multi-line EditText (Expands to fill entire space) -->
<!-- Fully Editable Multi-line EditText -->
<EditText <EditText
android:id="@+id/etTranscript" android:id="@+id/etTranscript"
android:layout_width="match_parent" android:layout_width="match_parent"
@ -194,11 +196,14 @@
</LinearLayout> </LinearLayout>
</LinearLayout> </LinearLayout>
<!-- Bottom Voice Section (Hidden when soft keyboard opens) -->
<!-- Bottom Voice Section (Centered with generous breathing space) -->
<LinearLayout <LinearLayout
android:id="@+id/bottomVoiceSection" android:id="@+id/bottomVoiceSection"
android:layout_width="match_parent" android:layout_width="match_parent"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginBottom="8dp"
android:clipChildren="false"
android:clipToPadding="false"
android:gravity="center_horizontal" android:gravity="center_horizontal"
android:orientation="vertical" android:orientation="vertical"
app:layout_constraintBottom_toBottomOf="parent"> app:layout_constraintBottom_toBottomOf="parent">
@ -206,35 +211,39 @@
<!-- Center Glowing Mic Button Container --> <!-- Center Glowing Mic Button Container -->
<FrameLayout <FrameLayout
android:id="@+id/micContainer" android:id="@+id/micContainer"
android:layout_width="120dp"
android:layout_height="120dp"
android:layout_marginBottom="6dp">
android:layout_width="140dp"
android:layout_height="140dp"
android:layout_marginBottom="8dp"
android:clipChildren="false"
android:clipToPadding="false">
<!-- Outer Pulsing Glow -->
<!-- Outer Pulsing Glow Ambient Ring -->
<View <View
android:id="@+id/viewGlow" android:id="@+id/viewGlow"
android:layout_width="120dp"
android:layout_height="120dp"
android:layout_width="140dp"
android:layout_height="140dp"
android:layout_gravity="center" android:layout_gravity="center"
android:alpha="0.3"
android:background="@drawable/bg_mic_button" />
android:alpha="0.0"
android:background="@drawable/bg_glow" />
<!-- Main Touch Button -->
<!-- Main Touch Button (Matte Frost Titanium Circle) -->
<FrameLayout <FrameLayout
android:id="@+id/btnMic" android:id="@+id/btnMic"
android:layout_width="95dp"
android:layout_height="95dp"
android:layout_width="96dp"
android:layout_height="96dp"
android:layout_gravity="center" android:layout_gravity="center"
android:background="@drawable/bg_mic_button" android:background="@drawable/bg_mic_button"
android:clickable="true" android:clickable="true"
android:elevation="4dp"
android:focusable="true"> android:focusable="true">
<!-- Minimalist Sleek Matte Black Mic Icon -->
<ImageView <ImageView
android:id="@+id/ivMicIcon" android:id="@+id/ivMicIcon"
android:layout_width="40dp"
android:layout_height="40dp"
android:layout_width="48dp"
android:layout_height="48dp"
android:layout_gravity="center" android:layout_gravity="center"
android:src="@drawable/ic_mic" />
android:src="@drawable/ic_mic_minimal" />
</FrameLayout> </FrameLayout>
</FrameLayout> </FrameLayout>
@ -243,7 +252,7 @@
android:id="@+id/tvInstruction" android:id="@+id/tvInstruction"
android:layout_width="wrap_content" android:layout_width="wrap_content"
android:layout_height="wrap_content" android:layout_height="wrap_content"
android:layout_marginBottom="4dp"
android:layout_marginBottom="6dp"
android:text="@string/hold_to_speak" android:text="@string/hold_to_speak"
android:textColor="@color/text_secondary" android:textColor="@color/text_secondary"
android:textSize="13sp" android:textSize="13sp"

5
mac/package.sh

@ -25,8 +25,8 @@ cp "$BUILD_DIR/SonioxVoice" "$APP_BUNDLE/Contents/MacOS/SonioxVoice"
cp "$PROJECT_DIR/Info.plist" "$APP_BUNDLE/Contents/Info.plist" cp "$PROJECT_DIR/Info.plist" "$APP_BUNDLE/Contents/Info.plist"
cp "$PROJECT_DIR/resources/AppIcon.icns" "$APP_BUNDLE/Contents/Resources/AppIcon.icns" cp "$PROJECT_DIR/resources/AppIcon.icns" "$APP_BUNDLE/Contents/Resources/AppIcon.icns"
# 3. Ad-hoc Codesign
codesign --force --deep --sign - "$APP_BUNDLE"
# 3. Stable Codesign with explicit identifier
codesign --force --deep --sign - --identifier "com.soniox.voice" "$APP_BUNDLE"
echo "✅ App bundle assembled at $APP_BUNDLE" echo "✅ App bundle assembled at $APP_BUNDLE"
@ -52,4 +52,3 @@ rm -rf "/Applications/$APP_NAME.app"
cp -R "$APP_BUNDLE" "/Applications/$APP_NAME.app" cp -R "$APP_BUNDLE" "/Applications/$APP_NAME.app"
echo "🎉 All Done Successfully!" echo "🎉 All Done Successfully!"
ls -lh "$BUILD_DIR"

50
mac/src/AppDelegate.swift

@ -10,6 +10,7 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
private var isBusyFinalizing = false private var isBusyFinalizing = false
private var currentAudioLevel: Float = 0.0 private var currentAudioLevel: Float = 0.0
private var latestPartialText: String? = nil private var latestPartialText: String? = nil
private var permissionPollTimer: Timer?
// Remote Android Phone Local Receiver (Port 8999) // Remote Android Phone Local Receiver (Port 8999)
private var remoteListener: NWListener? private var remoteListener: NWListener?
@ -60,14 +61,20 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
startRemotePasteServer() startRemotePasteServer()
// Connect to Linux Persistent Gateway (via local tunnel 18999 -> 8999) // Connect to Linux Persistent Gateway (via local tunnel 18999 -> 8999)
RelayClient.shared.onRemoteUpdateReceived = { [weak self] text, cursor, isFullReplace in
guard let self = self else { return }
print("AppDelegate: 📥 Clean Remote Input Update (Silent): '\(text.prefix(30))...' (replace: \(isFullReplace))")
HUDOverlayController.shared.show(state: .success(text: text))
RelayClient.shared.onRemoteUpdateReceived = { text, cursor, isFullReplace in
print("AppDelegate: 📥 Clean Remote Input Update: '\(text.prefix(30))...' (replace: \(isFullReplace))")
// 1. FIRST apply the remote update directly into the active input while window is in pristine focus!
FocusedInputSync.shared.applyRemoteUpdate(text: text, cursor: cursor, isFullReplace: isFullReplace) FocusedInputSync.shared.applyRemoteUpdate(text: text, cursor: cursor, isFullReplace: isFullReplace)
// 2. THEN show the floating success HUD notification
DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) {
HUDOverlayController.shared.show(state: .success(text: text))
}
} }
RelayClient.shared.start() RelayClient.shared.start()
HotkeyManager.shared.registerHotkeys()
checkInitialPermissions() checkInitialPermissions()
} }
@ -91,8 +98,8 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
} }
private func handleRemoteConnection(_ connection: NWConnection) { private func handleRemoteConnection(_ connection: NWConnection) {
connection.receive(minimumIncompleteLength: 1, maximumLength: 65536) { [weak self] data, _, _, _ in
guard let self = self, let data = data, let reqStr = String(data: data, encoding: .utf8) else {
connection.receive(minimumIncompleteLength: 1, maximumLength: 65536) { data, _, _, _ in
guard let data = data, let reqStr = String(data: data, encoding: .utf8) else {
connection.cancel() connection.cancel()
return return
} }
@ -130,6 +137,37 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
} }
private func checkInitialPermissions() { private func checkInitialPermissions() {
// 1. Force macOS to trigger the system Accessibility prompt dialog!
let axOptions = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary
let axTrusted = AXIsProcessTrustedWithOptions(axOptions)
print("AppDelegate: 🔒 Accessibility Permission Status (trusted: \(axTrusted))")
if !axTrusted {
print("AppDelegate: ⚠️ Accessibility not yet granted! Prompting user and monitoring...")
// Open Accessibility pane directly on both modern and legacy macOS
DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) {
if let url1 = URL(string: "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?Privacy_Accessibility") {
NSWorkspace.shared.open(url1)
}
if let url2 = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility") {
NSWorkspace.shared.open(url2)
}
}
// Poll every 1.0 seconds until user toggles the switch
permissionPollTimer = Timer.scheduledTimer(withTimeInterval: 1.0, repeats: true) { [weak self] timer in
let nowTrusted = AXIsProcessTrusted()
if nowTrusted {
print("AppDelegate: 🟢 Accessibility Permission GRANTED by user!")
timer.invalidate()
self?.permissionPollTimer = nil
HotkeyManager.shared.registerHotkeys()
HUDOverlayController.shared.show(state: .success(text: "دسترسی Accessibility تایید شد ✅"))
}
}
}
// 2. Microphone permission
audioRecorder.requestMicrophonePermission { granted in audioRecorder.requestMicrophonePermission { granted in
if !granted { if !granted {
print("Warning: Microphone permission not granted.") print("Warning: Microphone permission not granted.")

246
mac/src/FocusedInputSync.swift

@ -1,5 +1,6 @@
import Cocoa import Cocoa
import ApplicationServices import ApplicationServices
import CoreGraphics
public struct MacInputState: Codable { public struct MacInputState: Codable {
public let source: String public let source: String
@ -42,7 +43,44 @@ public final class FocusedInputSync {
AXUIElementSetAttributeValue(systemWideElement, "AXManualAccessibility" as CFString, kCFBooleanTrue) AXUIElementSetAttributeValue(systemWideElement, "AXManualAccessibility" as CFString, kCFBooleanTrue)
} }
private func findFocusedDescendant(_ elem: AXUIElement) -> AXUIElement? {
/// Resolves the actual user-facing application, bypassing system overlays
public func getRealFrontmostApp() -> NSRunningApplication? {
if let front = NSWorkspace.shared.frontmostApplication,
front.activationPolicy == .regular,
let bundleId = front.bundleIdentifier,
!bundleId.contains("notificationcenter"),
!bundleId.contains("controlcenter"),
!bundleId.contains("WindowManager"),
!bundleId.contains("Soniox") {
return front
}
let windowList = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? []
for win in windowList {
let layer = win[kCGWindowLayer as String] as? Int ?? -1
let pid = win[kCGWindowOwnerPID as String] as? pid_t ?? 0
let bounds = win[kCGWindowBounds as String] as? [String: Any] ?? [:]
let width = bounds["Width"] as? CGFloat ?? 0
let height = bounds["Height"] as? CGFloat ?? 0
if layer == 0 && width > 100 && height > 100 {
if let app = NSRunningApplication(processIdentifier: pid),
app.activationPolicy == .regular,
let bundleId = app.bundleIdentifier,
!bundleId.contains("notificationcenter"),
!bundleId.contains("controlcenter"),
!bundleId.contains("WindowManager"),
!bundleId.contains("Soniox") {
return app
}
}
}
return NSWorkspace.shared.frontmostApplication
}
private func findFocusedDescendant(_ elem: AXUIElement, depth: Int = 0) -> AXUIElement? {
if depth > 12 { return nil }
var isFocusedObj: CFTypeRef? var isFocusedObj: CFTypeRef?
if AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &isFocusedObj) == .success, if AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &isFocusedObj) == .success,
let isFocused = isFocusedObj as? Bool, isFocused { let isFocused = isFocusedObj as? Bool, isFocused {
@ -58,7 +96,7 @@ public final class FocusedInputSync {
if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success,
let children = childrenObj as? [AXUIElement] { let children = childrenObj as? [AXUIElement] {
for child in children { for child in children {
if let found = findFocusedDescendant(child) {
if let found = findFocusedDescendant(child, depth: depth + 1) {
return found return found
} }
} }
@ -66,8 +104,8 @@ public final class FocusedInputSync {
return nil return nil
} }
public func getFocusedElement() -> (AXUIElement, String)? {
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil }
public func getFocusedElement() -> (AXUIElement?, String) {
guard let frontApp = getRealFrontmostApp() else { return (nil, "App") }
let appName = frontApp.localizedName ?? "App" let appName = frontApp.localizedName ?? "App"
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) let appElem = AXUIElementCreateApplication(frontApp.processIdentifier)
@ -76,7 +114,7 @@ public final class FocusedInputSync {
var targetElem: AXUIElement? var targetElem: AXUIElement?
// 1. Try system wide focused element
// 1. System-wide focused element
var focusedObj: CFTypeRef? var focusedObj: CFTypeRef?
if AXUIElementCopyAttributeValue(systemWideElement, kAXFocusedUIElementAttribute as CFString, &focusedObj) == .success, if AXUIElementCopyAttributeValue(systemWideElement, kAXFocusedUIElementAttribute as CFString, &focusedObj) == .success,
let obj = focusedObj { let obj = focusedObj {
@ -89,7 +127,7 @@ public final class FocusedInputSync {
} }
} }
// 2. Try App focused element
// 2. App focused element
if targetElem == nil { if targetElem == nil {
if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedObj) == .success, if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedObj) == .success,
let obj = focusedObj { let obj = focusedObj {
@ -103,27 +141,53 @@ public final class FocusedInputSync {
} }
} }
// 3. Recursive search in tree
// 3. App focused window element
if targetElem == nil { if targetElem == nil {
targetElem = findFocusedDescendant(appElem)
var focusedWinObj: CFTypeRef?
if AXUIElementCopyAttributeValue(appElem, kAXFocusedWindowAttribute as CFString, &focusedWinObj) == .success,
let win = focusedWinObj {
var winFocObj: CFTypeRef?
if AXUIElementCopyAttributeValue((win as! AXUIElement), kAXFocusedUIElementAttribute as CFString, &winFocObj) == .success,
let obj = winFocObj {
targetElem = (obj as! AXUIElement)
}
}
} }
if let elem = targetElem {
return (elem, appName)
// 4. Recursive search in tree
if targetElem == nil {
targetElem = findFocusedDescendant(appElem)
} }
return nil
return (targetElem, appName)
} }
/// Inspects the current focused element and returns a state snapshot if changed
/// Inspects current focused element on Mac and returns state snapshot if user typed on Mac
public func inspectCurrentState() -> MacInputState? { public func inspectCurrentState() -> MacInputState? {
let now = Date().timeIntervalSince1970 let now = Date().timeIntervalSince1970
if isApplyingRemoteChange || now < remoteChangeExpiryTime { if isApplyingRemoteChange || now < remoteChangeExpiryTime {
return nil return nil
} }
guard let (elem, appName) = getFocusedElement() else { return nil }
let (elemOpt, appName) = getFocusedElement()
let appChanged = (appName != lastObservedApp)
if appChanged {
lastObservedApp = appName
}
guard let elem = elemOpt else {
if appChanged && !lastObservedText.isEmpty {
localRevision += 1
return MacInputState(app: appName, text: lastObservedText, cursor: lastObservedCursor, selection: 0, revision: localRevision)
}
return nil
}
var roleObj: CFTypeRef?
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj)
let role = roleObj as? String ?? ""
let isTextRole = (role == "AXTextField" || role == "AXTextArea" || role == "AXComboBox" || role == "AXSearchField")
// Extract text
var text = "" var text = ""
var valObj: CFTypeRef? var valObj: CFTypeRef?
if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) == .success, if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) == .success,
@ -135,7 +199,7 @@ public final class FocusedInputSync {
} }
} }
if text.isEmpty {
if text.isEmpty && isTextRole {
var countObj: CFTypeRef? var countObj: CFTypeRef?
if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countObj) == .success, if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countObj) == .success,
let count = countObj as? Int, count > 0 { let count = countObj as? Int, count > 0 {
@ -150,7 +214,11 @@ public final class FocusedInputSync {
} }
} }
// Extract cursor & selection
// Ghost empty suppression for non-native containers
if text.isEmpty && !lastObservedText.isEmpty && !isTextRole {
return nil
}
var cursor = text.count var cursor = text.count
var selLen = 0 var selLen = 0
var rangeObj: CFTypeRef? var rangeObj: CFTypeRef?
@ -163,8 +231,7 @@ public final class FocusedInputSync {
} }
} }
// Check if text or cursor actually changed on Mac
if text == lastObservedText && cursor == lastObservedCursor && appName == lastObservedApp {
if text == lastObservedText && cursor == lastObservedCursor && !appChanged {
return nil return nil
} }
@ -176,84 +243,141 @@ public final class FocusedInputSync {
return MacInputState(app: appName, text: text, cursor: cursor, selection: selLen, revision: localRevision) return MacInputState(app: appName, text: text, cursor: cursor, selection: selLen, revision: localRevision)
} }
/// Applies updated full text or inserts at cursor position directly into active Mac input
/// Perfectly mirrors the full text to the active Mac input box in real-time
@discardableResult @discardableResult
public func applyRemoteUpdate(text: String, cursor: Int? = nil, isFullReplace: Bool = true) -> Bool { public func applyRemoteUpdate(text: String, cursor: Int? = nil, isFullReplace: Bool = true) -> Bool {
isApplyingRemoteChange = true isApplyingRemoteChange = true
remoteChangeExpiryTime = Date().timeIntervalSince1970 + 0.45 // 450ms quiet window
remoteChangeExpiryTime = Date().timeIntervalSince1970 + 0.25
defer { defer {
DispatchQueue.main.asyncAfter(deadline: .now() + 0.45) {
DispatchQueue.main.asyncAfter(deadline: .now() + 0.25) {
self.isApplyingRemoteChange = false self.isApplyingRemoteChange = false
} }
} }
// 1. Clean and normalize text
var cleanText = text.components(separatedBy: .newlines).joined(separator: " ")
cleanText = cleanText.replacingOccurrences(of: "\t", with: " ")
cleanText = cleanText.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression)
// Normalize line endings and preserve intentional multiline text (\n)
var cleanText = text.replacingOccurrences(of: "\r\n", with: "\n").replacingOccurrences(of: "\r", with: "\n")
cleanText = cleanText.trimmingCharacters(in: .whitespacesAndNewlines) cleanText = cleanText.trimmingCharacters(in: .whitespacesAndNewlines)
let targetCursor = cursor ?? cleanText.count let targetCursor = cursor ?? cleanText.count
lastObservedText = cleanText lastObservedText = cleanText
lastObservedCursor = targetCursor lastObservedCursor = targetCursor
guard let (elem, appName) = getFocusedElement() else {
// Fallback: clipboard paste
lastObservedApp = NSWorkspace.shared.frontmostApplication?.localizedName ?? "App"
return pasteViaKeystroke(cleanText, isFullReplace: isFullReplace)
}
let (_, appName) = getFocusedElement()
lastObservedApp = appName lastObservedApp = appName
// Universal Quartz CGEvent Keystroke Engine (Cmd+A -> Cmd+V / Backspace or pure Cmd+V)
// Works 100% reliably across native, web, and Electron/Chromium apps (e.g. Antigravity, VS Code, Slack, Firefox)
print("FocusedInputSync: 🚀 Injecting text into '\(appName)' (replace: \(isFullReplace), len: \(cleanText.count), lines: \(cleanText.components(separatedBy: "\n").count))")
if isFullReplace { if isFullReplace {
// Try setting AXValue directly
let setErr = AXUIElementSetAttributeValue(elem, kAXValueAttribute as CFString, cleanText as CFTypeRef)
if setErr == .success {
if let cursor = cursor {
var range = CFRange(location: min(cursor, cleanText.count), length: 0)
if let axRange = AXValueCreate(.cfRange, &range) {
AXUIElementSetAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, axRange)
return executeCleanFullReplace(cleanText)
} else {
return pasteOnlyViaCleanKeystroke(cleanText)
} }
} }
print("FocusedInputSync: ✅ Updated AXValue for \(appName)")
return true
/**
* Inserts speech directly at active cursor via clean Cmd+V
*/
private func pasteOnlyViaCleanKeystroke(_ text: String) -> Bool {
guard !text.isEmpty else { return true }
let pasteboard = NSPasteboard.general
pasteboard.clearContents()
pasteboard.setString(text, forType: .string)
let src = CGEventSource(stateID: .hidSystemState)
let kVK_ANSI_V: CGKeyCode = 9
// Paste: Cmd + V
if let vDown = CGEvent(keyboardEventSource: src, virtualKey: kVK_ANSI_V, keyDown: true) {
vDown.flags = .maskCommand
vDown.post(tap: .cghidEventTap)
vDown.post(tap: .cgSessionEventTap)
} }
} else {
// Try setting selected text
let setSelErr = AXUIElementSetAttributeValue(elem, kAXSelectedTextAttribute as CFString, cleanText as CFTypeRef)
if setSelErr == .success {
print("FocusedInputSync: ✅ Inserted text via AXSelectedText for \(appName)")
usleep(12000)
if let vUp = CGEvent(keyboardEventSource: src, virtualKey: kVK_ANSI_V, keyDown: false) {
vUp.flags = .maskCommand
vUp.post(tap: .cghidEventTap)
vUp.post(tap: .cgSessionEventTap)
}
usleep(10000)
return true return true
} }
/**
* Replaces the entire content of active input box cleanly in real-time.
* If text is empty: Cmd+A -> Backspace.
* If text is non-empty: Cmd+A -> Cmd+V.
*/
private func executeCleanFullReplace(_ text: String) -> Bool {
let src = CGEventSource(stateID: .hidSystemState)
let kVK_ANSI_A: CGKeyCode = 0
let kVK_ANSI_V: CGKeyCode = 9
let kVK_Delete: CGKeyCode = 51
if text.isEmpty {
// Select all: Cmd + A
if let aDown = CGEvent(keyboardEventSource: src, virtualKey: kVK_ANSI_A, keyDown: true) {
aDown.flags = .maskCommand
aDown.post(tap: .cghidEventTap)
aDown.post(tap: .cgSessionEventTap)
} }
usleep(8000)
if let aUp = CGEvent(keyboardEventSource: src, virtualKey: kVK_ANSI_A, keyDown: false) {
aUp.flags = .maskCommand
aUp.post(tap: .cghidEventTap)
aUp.post(tap: .cgSessionEventTap)
}
usleep(15000)
return pasteViaKeystroke(cleanText, isFullReplace: isFullReplace)
// Backspace to clear
if let delDown = CGEvent(keyboardEventSource: src, virtualKey: kVK_Delete, keyDown: true) {
delDown.post(tap: .cghidEventTap)
delDown.post(tap: .cgSessionEventTap)
}
usleep(8000)
if let delUp = CGEvent(keyboardEventSource: src, virtualKey: kVK_Delete, keyDown: false) {
delUp.post(tap: .cghidEventTap)
delUp.post(tap: .cgSessionEventTap)
}
return true
} }
private func pasteViaKeystroke(_ text: String, isFullReplace: Bool) -> Bool {
// Set clipboard
let pasteboard = NSPasteboard.general let pasteboard = NSPasteboard.general
pasteboard.clearContents() pasteboard.clearContents()
pasteboard.setString(text, forType: .string) pasteboard.setString(text, forType: .string)
let src = CGEventSource(stateID: .hidSystemState)
// 1. Select all: Cmd + A
if let aDown = CGEvent(keyboardEventSource: src, virtualKey: kVK_ANSI_A, keyDown: true) {
aDown.flags = .maskCommand
aDown.post(tap: .cghidEventTap)
aDown.post(tap: .cgSessionEventTap)
}
usleep(8000)
if isFullReplace {
// Select all: Cmd + A
let aDown = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: true)
aDown?.flags = .maskCommand
let aUp = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: false)
aDown?.post(tap: .cghidEventTap)
aUp?.post(tap: .cghidEventTap)
if let aUp = CGEvent(keyboardEventSource: src, virtualKey: kVK_ANSI_A, keyDown: false) {
aUp.flags = .maskCommand
aUp.post(tap: .cghidEventTap)
aUp.post(tap: .cgSessionEventTap)
}
usleep(18000) // 18ms for selection to settle
usleep(20000)
// 2. Paste: Cmd + V
if let vDown = CGEvent(keyboardEventSource: src, virtualKey: kVK_ANSI_V, keyDown: true) {
vDown.flags = .maskCommand
vDown.post(tap: .cghidEventTap)
vDown.post(tap: .cgSessionEventTap)
} }
usleep(12000)
// Paste: Cmd + V
let vDown = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: true)
vDown?.flags = .maskCommand
let vUp = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: false)
vDown?.post(tap: .cghidEventTap)
vUp?.post(tap: .cghidEventTap)
if let vUp = CGEvent(keyboardEventSource: src, virtualKey: kVK_ANSI_V, keyDown: false) {
vUp.flags = .maskCommand
vUp.post(tap: .cghidEventTap)
vUp.post(tap: .cgSessionEventTap)
}
return true return true
} }

11
mac/src/HUDOverlay.swift

@ -8,10 +8,16 @@ public enum HUDState {
case error(message: String) case error(message: String)
} }
final class NonActivatingFloatingPanel: NSPanel {
override var canBecomeKey: Bool { return false }
override var canBecomeMain: Bool { return false }
override var acceptsFirstResponder: Bool { return false }
}
public final class HUDOverlayController { public final class HUDOverlayController {
public static let shared = HUDOverlayController() public static let shared = HUDOverlayController()
private var window: NSPanel?
private var window: NonActivatingFloatingPanel?
private var visualEffectView: NSVisualEffectView? private var visualEffectView: NSVisualEffectView?
private var iconImageView: NSImageView? private var iconImageView: NSImageView?
private var titleLabel: NSTextField? private var titleLabel: NSTextField?
@ -27,7 +33,7 @@ public final class HUDOverlayController {
let width: CGFloat = 460 let width: CGFloat = 460
let height: CGFloat = 80 let height: CGFloat = 80
let panel = NSPanel(
let panel = NonActivatingFloatingPanel(
contentRect: NSRect(x: 0, y: 0, width: width, height: height), contentRect: NSRect(x: 0, y: 0, width: width, height: height),
styleMask: [.borderless, .nonactivatingPanel], styleMask: [.borderless, .nonactivatingPanel],
backing: .buffered, backing: .buffered,
@ -39,6 +45,7 @@ public final class HUDOverlayController {
panel.backgroundColor = .clear panel.backgroundColor = .clear
panel.hasShadow = true panel.hasShadow = true
panel.ignoresMouseEvents = true panel.ignoresMouseEvents = true
panel.becomesKeyOnlyIfNeeded = false
panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary] panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary]
let visualEffect = NSVisualEffectView(frame: NSRect(x: 0, y: 0, width: width, height: height)) let visualEffect = NSVisualEffectView(frame: NSRect(x: 0, y: 0, width: width, height: height))

20
mac/src/RelayClient.swift

@ -19,6 +19,9 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate {
public var onRemoteUpdateReceived: ((String, Int?, Bool) -> Void)? public var onRemoteUpdateReceived: ((String, Int?, Bool) -> Void)?
private var lastInjectedText: String = ""
private var lastInjectedTime: Double = 0
override private init() { override private init() {
super.init() super.init()
let config = URLSessionConfiguration.default let config = URLSessionConfiguration.default
@ -97,7 +100,7 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate {
return return
} }
let action = json["action"] as? String ?? json["type"] as? String
let action = json["action"] as? String ?? json["type"] as? String ?? ""
let source = json["source"] as? String ?? "" let source = json["source"] as? String ?? ""
// Ignore echo messages originated by Mac itself // Ignore echo messages originated by Mac itself
@ -106,9 +109,18 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate {
let updateText = json["text"] as? String ?? json["insert_text"] as? String ?? "" let updateText = json["text"] as? String ?? json["insert_text"] as? String ?? ""
let cursor = json["cursor"] as? Int ?? json["cursor_pos"] as? Int let cursor = json["cursor"] as? Int ?? json["cursor_pos"] as? Int
if action == "paste" || action == "set_text" || action == "update_input" || action == "phone_input_edit" || action == "sync_state" {
let isFullReplace = (action != "insert_at_cursor")
print("RelayClient: ⚡ Received remote Android update: '\(updateText.prefix(30))...' (replace: \(isFullReplace), cursor: \(cursor ?? -1))")
if action == "sync_state" || action == "phone_input_edit" || action == "update_input" || action == "force_replace" || action == "insert_speech" || action == "paste" {
let now = Date().timeIntervalSince1970
// Drop rapid duplicate transmissions within 150ms (unless force_replace)
if updateText == lastInjectedText && (now - lastInjectedTime) < 0.15 && action != "force_replace" {
return
}
lastInjectedText = updateText
lastInjectedTime = now
let isFullReplace = (action != "insert_speech" && action != "speech_insert")
print("RelayClient: ⚡ Applying phone edit: '\(updateText.prefix(30))...' (replace: \(isFullReplace), len: \(updateText.count))")
DispatchQueue.main.async { DispatchQueue.main.async {
self.onRemoteUpdateReceived?(updateText, cursor, isFullReplace) self.onRemoteUpdateReceived?(updateText, cursor, isFullReplace)
} }

58
server/relay_server.py

@ -1,9 +1,9 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
Soniox Collaborative Bi-Directional Input Synchronization Gateway (v5.0)
- Real-time Google Docs / Figma style collaborative mirroring between Mac and Android.
- Monotonic revision counters, echo-loop suppression, and sub-15ms WebSocket routing.
- Cursor-aware voice insertion and instant text editing.
Soniox Collaborative Bi-Directional Input Synchronization Gateway (v5.4)
- Ultra-low latency streaming speech recognition with pre-warmed Soniox pool.
- Strict artifact & lone-punctuation suppression (e.g. «, », ., quotes).
- Clean single-injection pipeline on speech finalization.
""" """
import asyncio import asyncio
@ -59,7 +59,7 @@ def is_ws_open(ws) -> bool:
class SonioxPool: class SonioxPool:
"""Pre-warms upstream WebSockets to Soniox for 0ms speech start delay.""" """Pre-warms upstream WebSockets to Soniox for 0ms speech start delay."""
def __init__(self, size=3):
def __init__(self, size=4):
self._pool = asyncio.Queue(maxsize=size) self._pool = asyncio.Queue(maxsize=size)
self._refilling = False self._refilling = False
@ -83,7 +83,7 @@ class SonioxPool:
SONIOX_WS_URL, SONIOX_WS_URL,
additional_headers=SONIOX_HEADERS, additional_headers=SONIOX_HEADERS,
open_timeout=3.5, open_timeout=3.5,
ping_interval=20,
ping_interval=15,
) )
except Exception as e: except Exception as e:
logger.error("Failed to connect to upstream Soniox: %s", e) logger.error("Failed to connect to upstream Soniox: %s", e)
@ -105,12 +105,25 @@ class SonioxPool:
soniox_pool = SonioxPool() soniox_pool = SonioxPool()
def sanitize_and_flatten_text(text: str) -> str:
def sanitize_text(text: str, allow_multiline: bool = True) -> str:
if not text: if not text:
return "" return ""
flattened = re.sub(r"[\r\n\t]+", " ", text)
flattened = re.sub(r"\s+", " ", flattened).strip()
return flattened
if allow_multiline:
normalized = text.replace("\r\n", "\n").replace("\r", "\n")
lines = [re.sub(r"[ \t]+", " ", line) for line in normalized.split("\n")]
cleaned = "\n".join(lines).strip("\r\n")
else:
cleaned = re.sub(r"[\r\n\t]+", " ", text)
cleaned = re.sub(r"\s+", " ", cleaned).strip()
# Suppress lone punctuation artifacts (e.g. «, », ., ,, !, ?, etc.)
if re.fullmatch(r"[\s«»\.\,\،\؛\؟\!\?\:\;\-\–—\"\'\(\)\[\]\{\}]+", cleaned):
return ""
return cleaned
def sanitize_and_flatten_text(text: str) -> str:
return sanitize_text(text, allow_multiline=False)
async def broadcast_state(payload_dict: dict, exclude_ws=None): async def broadcast_state(payload_dict: dict, exclude_ws=None):
"""Broadcasts state snapshot to all connected clients (Mac and Phone) except sender.""" """Broadcasts state snapshot to all connected clients (Mac and Phone) except sender."""
@ -124,6 +137,10 @@ async def broadcast_state(payload_dict: dict, exclude_ws=None):
current_room_state.update(payload_dict) current_room_state.update(payload_dict)
payload_str = json.dumps(payload_dict, ensure_ascii=False) payload_str = json.dumps(payload_dict, ensure_ascii=False)
logger.info("📡 Broadcasting %s (len: %d) to %d Macs, %d Phones",
payload_dict.get("type"), len(payload_dict.get("text", "")),
len(connected_mac_websockets), len(connected_phone_websockets))
# 1. Send to Phone clients # 1. Send to Phone clients
dead_phones = set() dead_phones = set()
for ws in list(connected_phone_websockets): for ws in list(connected_phone_websockets):
@ -156,7 +173,7 @@ async def broadcast_state(payload_dict: dict, exclude_ws=None):
async def handle_phone_stream_ws(request): async def handle_phone_stream_ws(request):
"""Duplex WebSocket for Android client (Real-time Live Sync + Audio Dictation).""" """Duplex WebSocket for Android client (Real-time Live Sync + Audio Dictation)."""
ws = web.WebSocketResponse(heartbeat=12.0)
ws = web.WebSocketResponse(heartbeat=10.0, autoping=True)
await ws.prepare(request) await ws.prepare(request)
client_ip = request.remote client_ip = request.remote
logger.info("📱 Android Client Connected: %s", client_ip) logger.info("📱 Android Client Connected: %s", client_ip)
@ -238,10 +255,15 @@ async def handle_phone_stream_ws(request):
msg_type = data.get("type") or data.get("action") msg_type = data.get("type") or data.get("action")
sid = data.get("session_id", f"sess_{int(time.time()*1000)}") sid = data.get("session_id", f"sess_{int(time.time()*1000)}")
if msg_type == "sync_state" or msg_type == "phone_input_edit" or msg_type == "update_input":
# Phone edited text: broadcast to Mac immediately!
# ALL text / sync / insert operations must be broadcast to Mac!
if msg_type in ("sync_state", "insert_speech", "speech_insert", "phone_input_edit", "update_input", "paste"):
is_speech = msg_type in ("insert_speech", "speech_insert")
clean_text = sanitize_text(data.get("text", ""), allow_multiline=(not is_speech))
if not clean_text and is_speech:
continue # Do not broadcast empty speech or lone quotes
data["text"] = clean_text
data["source"] = "android" data["source"] = "android"
data["type"] = "sync_state"
data["type"] = msg_type
await broadcast_state(data, exclude_ws=ws) await broadcast_state(data, exclude_ws=ws)
elif msg_type == "start": elif msg_type == "start":
@ -264,7 +286,7 @@ async def handle_phone_stream_ws(request):
if active_soniox_ws and is_ws_open(active_soniox_ws): if active_soniox_ws and is_ws_open(active_soniox_ws):
await active_soniox_ws.send(json.dumps({"type": "finalize"})) await active_soniox_ws.send(json.dumps({"type": "finalize"}))
try: try:
await asyncio.wait_for(stop_event.wait(), timeout=0.55)
await asyncio.wait_for(stop_event.wait(), timeout=0.35)
except asyncio.TimeoutError: except asyncio.TimeoutError:
pass pass
@ -311,7 +333,7 @@ async def handle_phone_stream_ws(request):
async def handle_mac_ws(request): async def handle_mac_ws(request):
"""Persistent WebSocket for Mac Bridge (receives live Mac typing & pushes edits).""" """Persistent WebSocket for Mac Bridge (receives live Mac typing & pushes edits)."""
ws = web.WebSocketResponse(heartbeat=10.0)
ws = web.WebSocketResponse(heartbeat=10.0, autoping=True)
await ws.prepare(request) await ws.prepare(request)
client_ip = request.remote client_ip = request.remote
logger.info("🖥️ Mac client connected: %s", client_ip) logger.info("🖥️ Mac client connected: %s", client_ip)
@ -348,7 +370,7 @@ async def handle_health(request):
state_copy = dict(current_room_state) state_copy = dict(current_room_state)
return web.json_response({ return web.json_response({
"status": "ok", "status": "ok",
"service": "Soniox Collaborative Sync Gateway v5.0",
"service": "Soniox Collaborative Sync Gateway v5.4",
"connected_macs": len(connected_mac_websockets), "connected_macs": len(connected_mac_websockets),
"connected_phones": len(connected_phone_websockets), "connected_phones": len(connected_phone_websockets),
"current_app": state_copy.get("app", ""), "current_app": state_copy.get("app", ""),
@ -362,7 +384,7 @@ async def handle_paste(request):
text = data.get("text", "") text = data.get("text", "")
cursor = data.get("cursor_pos") or data.get("cursor") cursor = data.get("cursor_pos") or data.get("cursor")
data["source"] = "http_post" data["source"] = "http_post"
data["type"] = "sync_state"
data["type"] = "update_input"
await broadcast_state(data) await broadcast_state(data)
return web.json_response({"status": "synced", "revision": current_room_state["revision"]}) return web.json_response({"status": "synced", "revision": current_room_state["revision"]})
except Exception as e: except Exception as e:

Loading…
Cancel
Save