diff --git a/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt b/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt index f26a787..89adf57 100644 --- a/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt +++ b/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt @@ -33,7 +33,7 @@ import kotlinx.coroutines.launch import okhttp3.MediaType.Companion.toMediaType import okhttp3.OkHttpClient import okhttp3.Request -import okhttp3.RequestBody +import okhttp3.RequestBody.Companion.toRequestBody import org.json.JSONObject class MainActivity : AppCompatActivity() { @@ -47,11 +47,14 @@ class MainActivity : AppCompatActivity() { // Authoritative Permanent Gateway Server on Linux (116.16.16.19:8999) private val gatewayHost = "116.16.16.19:8999" - // Cursor Tracking for inserting speech at exact position + // Live Synchronized State (Google Docs / Figma style) + private var currentRevision: Long = 0L + private var isApplyingRemoteUpdate = false + private var lastLocalText = "" + + // Voice Insertion Anchor private var voiceInsertionCursorStart = 0 private var voiceInsertionCursorEnd = 0 - private var isUpdatingFromRemote = false - private var lastLocalText = "" private val requestPermissionLauncher = registerForActivityResult( ActivityResultContracts.RequestPermission() @@ -73,9 +76,9 @@ class MainActivity : AppCompatActivity() { setupUI() checkPermissions() - AppLogger.log("Main", "اپلیکیشن راه‌اندازی شد (همگام‌سازی بلادرنگ کادر متنی با مک)") + AppLogger.log("Main", "اپلیکیشن راه‌اندازی شد (همگام‌سازی بلادرنگ Google Docs/Figma Style با مک)") - // Setup Duplex WebSocket Client + // Initialize Collaborative WebSocket Client streamDictationClient = StreamDictationClient( host = gatewayHost, onConnectionStateChanged = { connected -> @@ -84,25 +87,33 @@ class MainActivity : AppCompatActivity() { this, if (connected) R.color.accent_green else R.color.accent_red ) }, - onMacInputStateReceived = { macState -> - // Sync from Mac: Update phone input box if user is not actively typing or recording - if (!isCurrentlyRecording && !binding.etTranscript.hasFocus()) { - if (macState.text != lastLocalText) { - isUpdatingFromRemote = true - lastLocalText = macState.text - binding.etTranscript.setText(macState.text) - val safeCursor = minOf(macState.cursor, macState.text.length) - binding.etTranscript.setSelection(safeCursor) - isUpdatingFromRemote = false + onSyncStateReceived = { state -> + // Apply update from Mac if source != android and revision is newer + if (state.source != "android") { + if (state.text != lastLocalText) { + isApplyingRemoteUpdate = true + lastLocalText = state.text + currentRevision = state.revision - if (macState.app.isNotEmpty() && macState.app != "App") { - binding.tvMacStatus.text = "متصل به ${macState.app} 🖥️" + // Preserve cursor safely + val currentCursor = binding.etTranscript.selectionStart + binding.etTranscript.setText(state.text) + val targetCursor = if (binding.etTranscript.hasFocus() && currentCursor >= 0) { + currentCursor.coerceIn(0, state.text.length) + } else { + state.cursor.coerceIn(0, state.text.length) } + binding.etTranscript.setSelection(targetCursor) + + isApplyingRemoteUpdate = false + } + + if (state.app.isNotEmpty() && state.app != "Desktop" && state.app != "App") { + binding.tvMacStatus.text = "متصل به ${state.app} 🖥️" } } }, - onPartialText = { livePartial -> - // Show live partial inside instruction banner while recording + onPartialSpeechText = { livePartial -> binding.tvInstruction.text = "🎙️ $livePartial" binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_blue)) }, @@ -111,11 +122,11 @@ class MainActivity : AppCompatActivity() { binding.viewGlow.scaleX = scale binding.viewGlow.scaleY = scale }, - onCompleted = { finalText, macDelivered -> + onSpeechCompleted = { finalText -> vibrate(100) if (finalText.isNotEmpty()) { insertSpeechAtCursor(finalText) - binding.tvInstruction.text = if (macDelivered) "✨ متن در نشانگر مک درج شد" else "متن آماده است" + binding.tvInstruction.text = "✨ گفتار در نشانگر درج و با مک همگام شد" binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_green)) } else { binding.tvInstruction.text = "صدایی تشخیص داده نشد" @@ -130,17 +141,16 @@ class MainActivity : AppCompatActivity() { } /** - * Inserts transcribed speech directly at the cursor / selection position inside the text! + * Slices speech text right at the exact cursor/selection location */ private fun insertSpeechAtCursor(speechText: String) { val current = binding.etTranscript.text?.toString() ?: "" - val start = minOf(voiceInsertionCursorStart, current.length) - val end = minOf(voiceInsertionCursorEnd, current.length) + val start = voiceInsertionCursorStart.coerceIn(0, current.length) + val end = voiceInsertionCursorEnd.coerceIn(0, current.length) val prefix = if (start > 0) current.substring(0, start) else "" val suffix = if (end < current.length) current.substring(end) else "" - // Add spacing if needed val formattedSpeech = if (prefix.isNotEmpty() && !prefix.endsWith(" ") && !speechText.startsWith(" ")) { " $speechText" } else { @@ -148,22 +158,22 @@ class MainActivity : AppCompatActivity() { } val mergedText = "$prefix$formattedSpeech$suffix" - val newCursor = start + formattedSpeech.length + val newCursor = (start + formattedSpeech.length).coerceIn(0, mergedText.length) - isUpdatingFromRemote = true + isApplyingRemoteUpdate = true lastLocalText = mergedText binding.etTranscript.setText(mergedText) - binding.etTranscript.setSelection(minOf(newCursor, mergedText.length)) - isUpdatingFromRemote = false + binding.etTranscript.setSelection(newCursor) + isApplyingRemoteUpdate = false - AppLogger.log("Main", "تزریق متن در نشانگر: '$speechText' (موقعیت جدید: $newCursor)") + AppLogger.log("Main", "تزریق گفتار در نشانگر: '$speechText' (موقعیت جدید: $newCursor)") - // Sync the updated full text and new cursor to Mac immediately! - streamDictationClient?.sendInputEditToMac(mergedText, newCursor, isFullReplace = true) + // Broadcast updated state to Mac immediately! + streamDictationClient?.sendLocalSyncState(mergedText, newCursor) } private fun setupUI() { - // Text Watcher for live sync on manual typing in phone + // Real-time TextWatcher for local keyboard typing binding.etTranscript.addTextChangedListener(object : TextWatcher { override fun beforeTextChanged(s: CharSequence?, start: Int, count: Int, after: Int) {} override fun onTextChanged(s: CharSequence?, start: Int, before: Int, count: Int) { @@ -171,11 +181,11 @@ class MainActivity : AppCompatActivity() { val wordCount = if (text.trim().isEmpty()) 0 else text.trim().split("\\s+".toRegex()).size binding.tvCharCount.text = "$wordCount کلمه" - if (!isUpdatingFromRemote && !isCurrentlyRecording) { + // If typed locally by user, broadcast to Mac immediately! + if (!isApplyingRemoteUpdate && !isCurrentlyRecording) { lastLocalText = text - val cur = binding.etTranscript.selectionStart - // Push manual typing to Mac in real-time - streamDictationClient?.sendInputEditToMac(text, cur, isFullReplace = true) + val cur = binding.etTranscript.selectionStart.coerceIn(0, text.length) + streamDictationClient?.sendLocalSyncState(text, cur) } } override fun afterTextChanged(s: Editable?) {} @@ -183,11 +193,11 @@ class MainActivity : AppCompatActivity() { // Clear Button binding.btnClearText.setOnClickListener { - isUpdatingFromRemote = true + isApplyingRemoteUpdate = true binding.etTranscript.setText("") lastLocalText = "" - isUpdatingFromRemote = false - streamDictationClient?.sendInputEditToMac("", 0, isFullReplace = true) + isApplyingRemoteUpdate = false + streamDictationClient?.sendLocalSyncState("", 0) binding.tvInstruction.text = getString(R.string.hold_to_speak) binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.text_secondary)) } @@ -203,7 +213,7 @@ class MainActivity : AppCompatActivity() { } } - // Send / Paste to Mac Button (Remote Input Control) + // Force Send to Mac Button binding.btnSendToMac.setOnClickListener { val text = binding.etTranscript.text.toString() if (text.isEmpty()) { @@ -211,21 +221,21 @@ class MainActivity : AppCompatActivity() { return@setOnClickListener } - val cur = binding.etTranscript.selectionStart + val cur = binding.etTranscript.selectionStart.coerceIn(0, text.length) binding.tvInstruction.text = "در حال درج متن در مک..." binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_yellow)) - // 1. Send via active persistent WebSocket - streamDictationClient?.sendInputEditToMac(text, cur, isFullReplace = true) + // 1. Direct WebSocket broadcast + streamDictationClient?.sendLocalSyncState(text, cur) - // 2. Also send via HTTP /paste endpoint as guaranteed delivery + // 2. Direct HTTP Post guarantee lifecycleScope.launch { val directPasteResult = sendDirectPaste(text, cur) vibrate(100) if (directPasteResult) { binding.tvInstruction.text = "✨ متن با موفقیت در مک تایپ شد" binding.tvInstruction.setTextColor(ContextCompat.getColor(this@MainActivity, R.color.accent_green)) - Toast.makeText(this@MainActivity, "متن در برنامه فعال مک درج شد", Toast.LENGTH_SHORT).show() + Toast.makeText(this@MainActivity, "متن در مک اعمال شد", Toast.LENGTH_SHORT).show() } } } @@ -235,12 +245,11 @@ class MainActivity : AppCompatActivity() { showLogsBottomSheet() } - // Touch listener for Hold to Speak (records cursor position upon touch) + // Touch listener for Hold to Speak (Captures cursor position on touch) binding.btnMic.setOnTouchListener { _, event -> when (event.action) { MotionEvent.ACTION_DOWN -> { if (checkAudioPermission()) { - // Capture cursor location before recording starts val selStart = binding.etTranscript.selectionStart val selEnd = binding.etTranscript.selectionEnd val totalLen = binding.etTranscript.text?.length ?: 0 @@ -272,7 +281,7 @@ class MainActivity : AppCompatActivity() { put("action", "update_input") }.toString() val mediaType = "application/json; charset=utf-8".toMediaType() - val body = RequestBody.create(mediaType, json) + val body = json.toRequestBody(mediaType) val req = Request.Builder() .url("http://$gatewayHost/paste") .post(body) @@ -368,7 +377,7 @@ class MainActivity : AppCompatActivity() { stopPulseAnimation() binding.btnMic.setBackgroundResource(R.drawable.bg_mic_button) - binding.tvInstruction.text = "⏳ در حال درج فوری در نشانگر..." + binding.tvInstruction.text = "⏳ در حال درج و همگام‌سازی..." binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_yellow)) streamDictationClient?.stopRecording(voiceInsertionCursorStart) @@ -412,7 +421,6 @@ class MainActivity : AppCompatActivity() { } catch (e: Exception) {} } - // Physical Volume Down Key as Push-To-Talk shortcut override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean { if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN && !isCurrentlyRecording) { if (checkAudioPermission()) { diff --git a/android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt b/android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt index 3df67c3..a7a1011 100644 --- a/android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt +++ b/android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt @@ -15,21 +15,23 @@ import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean import kotlin.math.sqrt -data class MacInputState( +data class SyncStatePayload( + val source: String, val app: String, val text: String, val cursor: Int, val selection: Int, + val revision: Long, val timestamp: Double ) class StreamDictationClient( private val host: String, private val onConnectionStateChanged: (Boolean) -> Unit, - private val onMacInputStateReceived: (MacInputState) -> Unit, - private val onPartialText: (String) -> Unit, + private val onSyncStateReceived: (SyncStatePayload) -> Unit, + private val onPartialSpeechText: (String) -> Unit, private val onAudioLevel: (Float) -> Unit, - private val onCompleted: (String, Boolean) -> Unit, + private val onSpeechCompleted: (String) -> Unit, private val onError: (String) -> Unit ) { private val tag = "StreamDictationClient" @@ -44,7 +46,7 @@ class StreamDictationClient( private val okHttpClient = OkHttpClient.Builder() .connectTimeout(5, TimeUnit.SECONDS) - .readTimeout(0, TimeUnit.MILLISECONDS) // Keep-alive persistent WebSocket + .readTimeout(0, TimeUnit.MILLISECONDS) // Persistent WebSocket .writeTimeout(5, TimeUnit.SECONDS) .pingInterval(10, TimeUnit.SECONDS) .retryOnConnectionFailure(true) @@ -65,12 +67,12 @@ class StreamDictationClient( val cleanHost = host.removePrefix("http://").removePrefix("https://").removePrefix("ws://").removePrefix("wss://") val wsUrl = "ws://$cleanHost/ws/stream" - AppLogger.log(tag, "اتصال به سوکت دائمی دوطرفه: $wsUrl") + AppLogger.log(tag, "اتصال به سوکت همگام‌سازی بلادرنگ: $wsUrl") val req = Request.Builder().url(wsUrl).build() webSocket = okHttpClient.newWebSocket(req, object : WebSocketListener() { override fun onOpen(ws: WebSocket, response: Response) { - AppLogger.log(tag, "🟢 سوکت پرسرعت دوطرفه با سرور و مک متصل شد") + AppLogger.log(tag, "🟢 سوکت همگام‌سازی بلادرنگ متصل شد") isConnected.set(true) mainHandler.post { onConnectionStateChanged(true) } } @@ -79,30 +81,34 @@ class StreamDictationClient( try { val json = JSONObject(text) val type = json.optString("type") - val sid = json.optString("session_id") + val source = json.optString("source", "") + + if (type == "sync_state" || type == "mac_input_state") { + val app = json.optString("app", "Mac") + val txt = json.optString("text", "") + val cursor = json.optInt("cursor", txt.length) + val sel = json.optInt("selection", 0) + val rev = json.optLong("revision", 0L) + val ts = json.optDouble("timestamp", System.currentTimeMillis() / 1000.0) + + val payload = SyncStatePayload(source, app, txt, cursor, sel, rev, ts) + mainHandler.post { onSyncStateReceived(payload) } + return + } + val sid = json.optString("session_id") when (type) { - "mac_input_state" -> { - val app = json.optString("app", "Mac") - val txt = json.optString("text", "") - val cursor = json.optInt("cursor", txt.length) - val sel = json.optInt("selection", 0) - val ts = json.optDouble("timestamp", System.currentTimeMillis() / 1000.0) - val state = MacInputState(app, txt, cursor, sel, ts) - mainHandler.post { onMacInputStateReceived(state) } - } "live", "partial" -> { if (sid.isEmpty() || sid == currentSessionId || !isSessionActive.get()) { val liveText = json.optString("text") - mainHandler.post { onPartialText(liveText) } + mainHandler.post { onPartialSpeechText(liveText) } } } "final" -> { val finalText = json.optString("text") - val macDelivered = json.optBoolean("mac_delivered", true) isSessionActive.set(false) - AppLogger.log(tag, "⚡ متن نهایی دریافت شد: '$finalText'") - mainHandler.post { onCompleted(finalText, macDelivered) } + AppLogger.log(tag, "⚡ صوت پردازش شد: '$finalText'") + mainHandler.post { onSpeechCompleted(finalText) } } "error" -> { val msg = json.optString("message", "خطای سرور") @@ -116,7 +122,7 @@ class StreamDictationClient( } override fun onFailure(ws: WebSocket, t: Throwable, response: Response?) { - AppLogger.log(tag, "🔴 قطع اتصال سوکت: ${t.message}. تلاش مجدد...") + AppLogger.log(tag, "🔴 قطع سوکت: ${t.message}. تلاش مجدد...") isConnected.set(false) webSocket = null mainHandler.post { onConnectionStateChanged(false) } @@ -134,20 +140,21 @@ class StreamDictationClient( } /** - * Sends manual text edit or full text sync directly to Mac in real-time + * Broadcasts phone's updated text & cursor to Mac in sub-15ms */ - fun sendInputEditToMac(text: String, cursor: Int, isFullReplace: Boolean = true) { + fun sendLocalSyncState(text: String, cursor: Int, selection: Int = 0) { if (!isConnected.get() || webSocket == null) { connectWebSocket() } - val editPayload = JSONObject().apply { - put("type", "phone_input_edit") + val payload = JSONObject().apply { + put("type", "sync_state") + put("source", "android") put("text", text) - put("cursor_pos", cursor) - put("is_full_replace", isFullReplace) - put("session_id", "edit_${System.currentTimeMillis()}") + put("cursor", cursor) + put("selection", selection) + put("timestamp", System.currentTimeMillis() / 1000.0) }.toString() - webSocket?.send(editPayload) + webSocket?.send(payload) } @SuppressLint("MissingPermission") @@ -162,16 +169,14 @@ class StreamDictationClient( connectWebSocket() } - // 1. Send START frame with cursor position info val startFrame = JSONObject().apply { put("type", "start") put("session_id", currentSessionId) put("cursor_pos", cursorPos) }.toString() webSocket?.send(startFrame) - AppLogger.log(tag, "🎙️ شروع ضبط در موقعیت نشانگر $cursorPos...") + AppLogger.log(tag, "🎙️ شروع استریم گفتار در موقعیت نشانگر $cursorPos...") - // 2. Hardware recording setup (16kHz 16-bit Mono, 2048 bytes = 64ms) val minBufferSize = AudioRecord.getMinBufferSize(sampleRate, channelConfig, audioFormat) val bufferSize = maxOf(minBufferSize, 2048) @@ -202,11 +207,8 @@ class StreamDictationClient( val bytesRead = audioRecord?.read(chunk, 0, chunk.size) ?: -1 if (bytesRead > 0) { val slice = if (bytesRead == chunk.size) chunk.clone() else chunk.copyOf(bytesRead) - - // Direct binary streaming over persistent WebSocket webSocket?.send(slice.toByteString()) - // RMS Audio Level calculation var sum = 0.0 val samplesCount = bytesRead / 2 for (i in 0 until samplesCount) { @@ -248,14 +250,13 @@ class StreamDictationClient( Log.e(tag, "Error releasing audio hardware", e) } - // Send STOP frame over persistent WebSocket val stopFrame = JSONObject().apply { put("type", "stop") put("session_id", currentSessionId) put("cursor_pos", cursorPos) }.toString() webSocket?.send(stopFrame) - AppLogger.log(tag, "⏹️ پایان صحبت ($currentSessionId). پردازش و جایگذاری در نشانگر...") + AppLogger.log(tag, "⏹️ پایان ضبط. دریافت متن نهایی...") } fun release() { diff --git a/android/app/src/main/java/com/soniox/remotemic/VoiceRecognitionActivity.kt b/android/app/src/main/java/com/soniox/remotemic/VoiceRecognitionActivity.kt index f2f8b8e..fa6e292 100644 --- a/android/app/src/main/java/com/soniox/remotemic/VoiceRecognitionActivity.kt +++ b/android/app/src/main/java/com/soniox/remotemic/VoiceRecognitionActivity.kt @@ -31,12 +31,12 @@ class VoiceRecognitionActivity : AppCompatActivity() { streamClient?.startRecording(0) } }, - onMacInputStateReceived = {}, - onPartialText = { live -> + onSyncStateReceived = {}, + onPartialSpeechText = { live -> tvTranscript.text = live }, onAudioLevel = {}, - onCompleted = { text, _ -> + onSpeechCompleted = { text -> val resultIntent = Intent().apply { val list = ArrayList() list.add(text) diff --git a/android/server/relay_server.py b/android/server/relay_server.py index 19fa8ab..e160e22 100644 --- a/android/server/relay_server.py +++ b/android/server/relay_server.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """ -Soniox Bi-Directional Input Synchronization Gateway (v4.0) -- Mirrors Mac focused input box <--> Android phone in real-time. -- Supports inserting voice text at exact cursor location. -- Instant <15ms WebSocket push to Mac for editing and pasting. +Soniox Collaborative Bi-Directional Input Synchronization Gateway (v5.0) +- Real-time Google Docs / Figma style collaborative mirroring between Mac and Android. +- Monotonic revision counters, echo-loop suppression, and sub-15ms WebSocket routing. +- Cursor-aware voice insertion and instant text editing. """ import asyncio @@ -19,22 +19,24 @@ import websockets from websockets.protocol import State logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") -logger = logging.getLogger("SonioxRelay") +logger = logging.getLogger("SyncGateway") connected_mac_websockets = set() connected_phone_websockets = set() -recent_pasted_sessions = OrderedDict() # session_id -> (timestamp, text) -latest_mac_input_state = { - "type": "mac_input_state", +# Global Room State Snapshot (Single Source of Truth) +current_room_state = { + "type": "sync_state", + "source": "server_init", "app": "Desktop", "text": "", "cursor": 0, "selection": 0, + "revision": 0, "timestamp": time.time() } +room_lock = asyncio.Lock() -MAC_DIRECT_HTTP_URL = "http://116.16.16.20:8999/paste" SONIOX_WS_URL = ( "wss://translate.compare.soniox.com/compare/api/compare-websocket" "?language_hints=fa&language_hints=en&language_hints=ar" @@ -104,101 +106,29 @@ class SonioxPool: soniox_pool = SonioxPool() def sanitize_and_flatten_text(text: str) -> str: - """ - 1. Removes all line breaks (\\r, \\n) and collapses whitespace into single spaces. - 2. Strips English hallucination stop words during Persian speech. - 3. Guarantees zero trailing/leading enters or spaces. - """ if not text: return "" - flattened = re.sub(r"[\r\n\t]+", " ", text) flattened = re.sub(r"\s+", " ", flattened).strip() - - if not flattened: - return "" - - words = flattened.split() - fa_pattern = re.compile(r"[\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF]") - en_pattern = re.compile(r"[a-zA-Z]") - - fa_count = sum(1 for w in words if fa_pattern.search(w)) - en_count = sum(1 for w in words if en_pattern.search(w)) - total = fa_count + en_count - - if total == 0: - return flattened - - fa_ratio = fa_count / total - stop_words = {"sex", "from", "no", "oh", "you", "all", "know", "that", "god", "loves", "men", "for", "day", "night", "mankind", "wanted", "moments"} - - cleaned = [] - if fa_ratio >= 0.25: - for w in words: - if en_pattern.search(w) and not fa_pattern.search(w): - clean_w = re.sub(r"[.,!?:;،؛؟\"'()\[\]{}«»–—-]", "", w.lower()) - if clean_w in stop_words or fa_ratio >= 0.70: - continue - cleaned.append(w) - else: - cleaned = words - - result = " ".join(cleaned) - return re.sub(r"\s+", " ", result).strip() - -async def broadcast_to_macs(payload_dict: dict) -> bool: - """Pushes command payload directly to Mac via persistent WebSocket in <15ms.""" - delivered = False - payload_str = json.dumps(payload_dict, ensure_ascii=False) - - dead_sockets = set() - for ws in list(connected_mac_websockets): - try: - if is_ws_open(ws): - await ws.send_str(payload_str) - delivered = True - logger.info("⚡ Pushed to Mac WS: %s", payload_dict.get("action") or payload_dict.get("type")) - else: - dead_sockets.add(ws) - except Exception: - dead_sockets.add(ws) - - for dead in dead_sockets: - connected_mac_websockets.discard(dead) - - if delivered: - return True - - # Fallback to SSH script if WebSocket temporarily disconnected - try: - text = payload_dict.get("text", "") - if text: - clean_text = sanitize_and_flatten_text(text) - escaped_text = clean_text.replace("'", "'\\''") - remote_cmd = ( - f"printf '%s' '{escaped_text}' | pbcopy && " - f"/usr/bin/osascript -e 'tell application \"System Events\" to keystroke \"v\" using command down'" - ) - proc = await asyncio.create_subprocess_exec( - "ssh", "-p", "2222", "-o", "BatchMode=yes", "-o", "ConnectTimeout=2", "alig@127.0.0.1", - remote_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - stdout, stderr = await proc.communicate() - if proc.returncode == 0: - logger.info("✅ Pasted to Mac via SSH Tunnel fallback: '%s'", clean_text[:30]) - return True - except Exception as e: - logger.warning("SSH fallback error: %s", e) - - return False - -async def broadcast_to_phones(payload_dict: dict): - """Pushes Mac input state changes to all connected Android clients.""" - payload_str = json.dumps(payload_dict, ensure_ascii=False) + return flattened + +async def broadcast_state(payload_dict: dict, exclude_ws=None): + """Broadcasts state snapshot to all connected clients (Mac and Phone) except sender.""" + global current_room_state + async with room_lock: + current_room_state["revision"] += 1 + payload_dict["revision"] = current_room_state["revision"] + payload_dict["timestamp"] = time.time() + + # Update our cached authoritative state + current_room_state.update(payload_dict) + payload_str = json.dumps(payload_dict, ensure_ascii=False) + + # 1. Send to Phone clients dead_phones = set() for ws in list(connected_phone_websockets): + if ws is exclude_ws: + continue try: if is_ws_open(ws): await ws.send_str(payload_str) @@ -206,26 +136,35 @@ async def broadcast_to_phones(payload_dict: dict): dead_phones.add(ws) except Exception: dead_phones.add(ws) - for dead in dead_phones: connected_phone_websockets.discard(dead) + # 2. Send to Mac clients + dead_macs = set() + for ws in list(connected_mac_websockets): + if ws is exclude_ws: + continue + try: + if is_ws_open(ws): + await ws.send_str(payload_str) + else: + dead_macs.add(ws) + except Exception: + dead_macs.add(ws) + for dead in dead_macs: + connected_mac_websockets.discard(dead) + async def handle_phone_stream_ws(request): - """ - ⚡ Persistent Duplex Channel for Android Client: - - Receives live Mac input state on connect and continuously. - - Handles live audio streaming and returns live STT tokens. - - Receives user manual text edits and pushes them instantly to Mac. - """ - ws = web.WebSocketResponse(heartbeat=15.0) + """Duplex WebSocket for Android client (Real-time Live Sync + Audio Dictation).""" + ws = web.WebSocketResponse(heartbeat=12.0) await ws.prepare(request) client_ip = request.remote - logger.info("📱 Android Client connected to Duplex Stream: %s", client_ip) + logger.info("📱 Android Client Connected: %s", client_ip) connected_phone_websockets.add(ws) - # Immediately send the latest Mac input state to phone upon connection! - if latest_mac_input_state: - await ws.send_str(json.dumps(latest_mac_input_state, ensure_ascii=False)) + # Immediately hydrate phone with full current state! + async with room_lock: + await ws.send_str(json.dumps(current_room_state, ensure_ascii=False)) active_soniox_ws = None reader_task = None @@ -286,7 +225,7 @@ async def handle_phone_stream_ws(request): try: async for msg in ws: if msg.type == aiohttp.WSMsgType.BINARY: - # Live PCM audio chunk (2048 bytes / 64ms) + # Live PCM audio chunk if active_soniox_ws and is_ws_open(active_soniox_ws): await active_soniox_ws.send(msg.data) @@ -299,7 +238,13 @@ async def handle_phone_stream_ws(request): msg_type = data.get("type") or data.get("action") sid = data.get("session_id", f"sess_{int(time.time()*1000)}") - if msg_type == "start": + if msg_type == "sync_state" or msg_type == "phone_input_edit" or msg_type == "update_input": + # Phone edited text: broadcast to Mac immediately! + data["source"] = "android" + data["type"] = "sync_state" + await broadcast_state(data, exclude_ws=ws) + + elif msg_type == "start": current_session_id = sid full_final_tokens.clear() current_non_final = "" @@ -327,7 +272,7 @@ async def handle_phone_stream_ws(request): clean_final = sanitize_and_flatten_text(raw_final) logger.info("⚡ Session %s final text: '%s'", sid, clean_final) - # Return final voice transcription to Android + # Return final speech text to phone if is_ws_open(ws): await ws.send_str(json.dumps({ "type": "final", @@ -347,32 +292,6 @@ async def handle_phone_stream_ws(request): asyncio.create_task(soniox_pool.refill()) - elif msg_type == "phone_input_edit" or msg_type == "update_mac_input": - # User manually edited text on phone or pressed "Insert in Mac" - edit_text = data.get("text", "") - cursor = data.get("cursor_pos") or data.get("cursor") - is_full_replace = data.get("is_full_replace", True) - logger.info("📱 Received phone edit to push to Mac: '%s' (cursor: %s)", edit_text[:30], cursor) - - mac_ok = await broadcast_to_macs({ - "action": "update_input", - "text": edit_text, - "cursor": cursor, - "is_full_replace": is_full_replace - }) - - # Update our cached latest state - latest_mac_input_state["text"] = edit_text - latest_mac_input_state["cursor"] = cursor if cursor is not None else len(edit_text) - latest_mac_input_state["timestamp"] = time.time() - - if is_ws_open(ws): - await ws.send_str(json.dumps({ - "type": "edit_ack", - "session_id": sid, - "mac_delivered": mac_ok - })) - elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.ERROR): break @@ -391,27 +310,26 @@ async def handle_phone_stream_ws(request): return ws async def handle_mac_ws(request): - """Persistent WebSocket for Mac Bridge (receives input state & pushes edits).""" - global latest_mac_input_state + """Persistent WebSocket for Mac Bridge (receives live Mac typing & pushes edits).""" ws = web.WebSocketResponse(heartbeat=10.0) await ws.prepare(request) client_ip = request.remote - logger.info("🖥️ Mac client connected to persistent WebSocket: %s", client_ip) + logger.info("🖥️ Mac client connected: %s", client_ip) connected_mac_websockets.add(ws) try: - await ws.send_str(json.dumps({"type": "welcome", "message": "Connected to Soniox Duplex Gateway"})) + await ws.send_str(json.dumps({"type": "welcome", "message": "Connected to Soniox Sync Gateway"})) async for msg in ws: if msg.type == aiohttp.WSMsgType.TEXT: try: data = json.loads(msg.data) msg_type = data.get("type") - if msg_type == "mac_input_state": - # Mac reports focused input box text and cursor position - latest_mac_input_state = data - # Broadcast immediately to phone! - await broadcast_to_phones(data) + if msg_type == "sync_state" or msg_type == "mac_input_state": + # Mac reports typing / cursor change: broadcast to all phones immediately! + data["source"] = "mac" + data["type"] = "sync_state" + await broadcast_state(data, exclude_ws=ws) elif msg_type == "ping": await ws.send_str(json.dumps({"type": "pong"})) @@ -421,35 +339,32 @@ async def handle_mac_ws(request): break finally: connected_mac_websockets.discard(ws) - if is_ws_open(ws): - await ws.close() logger.info("🖥️ Mac client disconnected: %s", client_ip) return ws async def handle_health(request): + async with room_lock: + state_copy = dict(current_room_state) return web.json_response({ "status": "ok", - "service": "Soniox Bi-Directional Duplex Gateway v4.0", + "service": "Soniox Collaborative Sync Gateway v5.0", "connected_macs": len(connected_mac_websockets), "connected_phones": len(connected_phone_websockets), - "latest_mac_input_app": latest_mac_input_state.get("app", ""), - "latest_mac_input_text_len": len(latest_mac_input_state.get("text", "")) + "current_app": state_copy.get("app", ""), + "current_revision": state_copy.get("revision", 0), + "current_text_len": len(state_copy.get("text", "")) }) async def handle_paste(request): try: data = await request.json() text = data.get("text", "") - session_id = data.get("session_id", "") cursor = data.get("cursor_pos") or data.get("cursor") - success = await broadcast_to_macs({ - "action": "update_input", - "text": text, - "cursor": cursor, - "is_full_replace": True - }) - return web.json_response({"status": "pasted" if success else "failed", "mac_delivered": success}) + data["source"] = "http_post" + data["type"] = "sync_state" + await broadcast_state(data) + return web.json_response({"status": "synced", "revision": current_room_state["revision"]}) except Exception as e: return web.json_response({"error": str(e)}, status=400) diff --git a/mac/src/FocusedInputSync.swift b/mac/src/FocusedInputSync.swift index 265c93c..dc64894 100644 --- a/mac/src/FocusedInputSync.swift +++ b/mac/src/FocusedInputSync.swift @@ -2,17 +2,21 @@ import Cocoa import ApplicationServices public struct MacInputState: Codable { + public let source: String public let app: String public let text: String public let cursor: Int public let selection: Int + public let revision: Int64 public let timestamp: Double - public init(app: String, text: String, cursor: Int, selection: Int) { + public init(app: String, text: String, cursor: Int, selection: Int, revision: Int64) { + self.source = "mac" self.app = app self.text = text self.cursor = cursor self.selection = selection + self.revision = revision self.timestamp = Date().timeIntervalSince1970 } } @@ -20,52 +24,154 @@ public struct MacInputState: Codable { public final class FocusedInputSync { public static let shared = FocusedInputSync() - private var lastState: MacInputState? - private var isUpdatingLocally = false + private let systemWideElement: AXUIElement + private var isApplyingRemoteChange: Bool = false + private var lastObservedHash: Int = 0 + private var localRevision: Int64 = 0 - public var onInputStateChanged: ((MacInputState) -> Void)? + private init() { + self.systemWideElement = AXUIElementCreateSystemWide() + enableGlobalAccessibility() + } + + private func enableGlobalAccessibility() { + AXUIElementSetAttributeValue(systemWideElement, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) + AXUIElementSetAttributeValue(systemWideElement, "AXManualAccessibility" as CFString, kCFBooleanTrue) + } - private init() {} + private func findFocusedDescendant(_ elem: AXUIElement) -> AXUIElement? { + var isFocusedObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &isFocusedObj) == .success, + let isFocused = isFocusedObj as? Bool, isFocused { + var roleObj: CFTypeRef? + AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) + let role = (roleObj as? String) ?? "" + if role != "AXWindow" && role != "AXApplication" && role != "AXGroup" && role != "AXSplitGroup" && role != "AXScrollArea" { + return elem + } + } + + var childrenObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, + let children = childrenObj as? [AXUIElement] { + for child in children { + if let found = findFocusedDescendant(child) { + return found + } + } + } + return nil + } - /// Reads current focused element state (app name, text, cursor position, selection) - public func getCurrentState() -> MacInputState? { + public func getFocusedElement() -> (AXUIElement, String)? { guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil } let appName = frontApp.localizedName ?? "App" let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) - var focusedElemObj: CFTypeRef? - let err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) - guard err == .success, let elem = focusedElemObj else { - return MacInputState(app: appName, text: "", cursor: 0, selection: 0) + AXUIElementSetAttributeValue(appElem, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) + AXUIElementSetAttributeValue(appElem, "AXManualAccessibility" as CFString, kCFBooleanTrue) + + var targetElem: AXUIElement? + + // 1. Try system wide focused element + var focusedObj: CFTypeRef? + if AXUIElementCopyAttributeValue(systemWideElement, kAXFocusedUIElementAttribute as CFString, &focusedObj) == .success, + let obj = focusedObj { + let elem = obj as! AXUIElement + var roleObj: CFTypeRef? + AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) + let role = (roleObj as? String) ?? "" + if role != "AXWindow" && role != "AXApplication" { + targetElem = elem + } + } + + // 2. Try App focused element + if targetElem == nil { + if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedObj) == .success, + let obj = focusedObj { + let elem = obj as! AXUIElement + var roleObj: CFTypeRef? + AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) + let role = (roleObj as? String) ?? "" + if role != "AXWindow" && role != "AXApplication" { + targetElem = elem + } + } } - let axElem = elem as! AXUIElement + // 3. Recursive search in tree + if targetElem == nil { + targetElem = findFocusedDescendant(appElem) + } + + if let elem = targetElem { + return (elem, appName) + } + return nil + } + + /// Inspects the current focused element and returns a state snapshot if changed + public func inspectCurrentState() -> MacInputState? { + guard !isApplyingRemoteChange else { return nil } + guard let (elem, appName) = getFocusedElement() else { return nil } + + // Extract text + var text = "" var valObj: CFTypeRef? - AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) - let text = (valObj as? String) ?? "" + if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) == .success, + let val = valObj { + if let str = val as? String { + text = str + } else if let attrStr = val as? NSAttributedString { + text = attrStr.string + } + } + + if text.isEmpty { + var countObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countObj) == .success, + let count = countObj as? Int, count > 0 { + var range = CFRange(location: 0, length: count) + if let axRange = AXValueCreate(.cfRange, &range) { + var strObj: CFTypeRef? + if AXUIElementCopyParameterizedAttributeValue(elem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &strObj) == .success, + let str = strObj as? String { + text = str + } + } + } + } + // Extract cursor & selection var cursor = text.count var selLen = 0 - var selectedRangeObj: CFTypeRef? - if AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &selectedRangeObj) == .success, - let axVal = selectedRangeObj { + var rangeObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, &rangeObj) == .success, + let axRange = rangeObj { var range = CFRange() - if AXValueGetValue(axVal as! AXValue, .cfRange, &range) { + if AXValueGetValue(axRange as! AXValue, .cfRange, &range) { cursor = range.location selLen = range.length } } - return MacInputState(app: appName, text: text, cursor: cursor, selection: selLen) + // Check hash to prevent echo loops + let currentHash = "\(appName)_\(text)_\(cursor)_\(selLen)".hashValue + guard currentHash != lastObservedHash else { return nil } + lastObservedHash = currentHash + localRevision += 1 + + return MacInputState(app: appName, text: text, cursor: cursor, selection: selLen, revision: localRevision) } /// Applies updated full text or inserts at cursor position directly into active Mac input @discardableResult public func applyRemoteUpdate(text: String, cursor: Int? = nil, isFullReplace: Bool = true) -> Bool { - isUpdatingLocally = true + isApplyingRemoteChange = true defer { - DispatchQueue.main.asyncAfter(deadline: .now() + 0.2) { - self.isUpdatingLocally = false + DispatchQueue.main.asyncAfter(deadline: .now() + 0.12) { + self.isApplyingRemoteChange = false } } @@ -75,41 +181,42 @@ public final class FocusedInputSync { cleanText = cleanText.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) cleanText = cleanText.trimmingCharacters(in: .whitespacesAndNewlines) - guard let frontApp = NSWorkspace.shared.frontmostApplication else { return false } - let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) - var focusedElemObj: CFTypeRef? + guard let (elem, appName) = getFocusedElement() else { + // Fallback: clipboard paste + return pasteViaKeystroke(cleanText, isFullReplace: isFullReplace) + } - if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) == .success, - let elem = focusedElemObj { - let axElem = elem as! AXUIElement - - if isFullReplace { - // Try setting AXValue directly - let setErr = AXUIElementSetAttributeValue(axElem, kAXValueAttribute as CFString, cleanText as CFTypeRef) - if setErr == .success { - if let cursor = cursor { - var range = CFRange(location: min(cursor, cleanText.count), length: 0) - if let axRange = AXValueCreate(.cfRange, &range) { - AXUIElementSetAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, axRange) - } + if isFullReplace { + // Try setting AXValue directly + let setErr = AXUIElementSetAttributeValue(elem, kAXValueAttribute as CFString, cleanText as CFTypeRef) + if setErr == .success { + if let cursor = cursor { + var range = CFRange(location: min(cursor, cleanText.count), length: 0) + if let axRange = AXValueCreate(.cfRange, &range) { + AXUIElementSetAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, axRange) } - print("FocusedInputSync: ✅ Updated AXValue directly for \(frontApp.localizedName ?? "")") - return true - } - } else { - // Try setting selected text - let setSelErr = AXUIElementSetAttributeValue(axElem, kAXSelectedTextAttribute as CFString, cleanText as CFTypeRef) - if setSelErr == .success { - print("FocusedInputSync: ✅ Inserted text via AXSelectedText for \(frontApp.localizedName ?? "")") - return true } + lastObservedHash = "\(appName)_\(cleanText)_\(cursor ?? cleanText.count)_0".hashValue + print("FocusedInputSync: ✅ Updated AXValue for \(appName)") + return true + } + } else { + // Try setting selected text + let setSelErr = AXUIElementSetAttributeValue(elem, kAXSelectedTextAttribute as CFString, cleanText as CFTypeRef) + if setSelErr == .success { + lastObservedHash = "\(appName)_\(cleanText)_\(cursor ?? cleanText.count)_0".hashValue + print("FocusedInputSync: ✅ Inserted text via AXSelectedText for \(appName)") + return true } } - // Fallback: Clipboard Cmd+A + Cmd+V or pure Cmd+V + return pasteViaKeystroke(cleanText, isFullReplace: isFullReplace) + } + + private func pasteViaKeystroke(_ text: String, isFullReplace: Bool) -> Bool { let pasteboard = NSPasteboard.general pasteboard.clearContents() - pasteboard.setString(cleanText, forType: .string) + pasteboard.setString(text, forType: .string) let src = CGEventSource(stateID: .hidSystemState) @@ -131,7 +238,6 @@ public final class FocusedInputSync { vDown?.post(tap: .cghidEventTap) vUp?.post(tap: .cghidEventTap) - print("FocusedInputSync: ✅ Injected clipboard keystroke to \(frontApp.localizedName ?? "")") return true } } diff --git a/mac/src/RelayClient.swift b/mac/src/RelayClient.swift index f695521..12a4f33 100644 --- a/mac/src/RelayClient.swift +++ b/mac/src/RelayClient.swift @@ -6,7 +6,6 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate { // Connects through SSH local port forward 18999 -> Linux Server 8999 private let primaryUrl = URL(string: "ws://127.0.0.1:18999/ws/mac")! - private let fallbackUrl = URL(string: "ws://116.16.16.19:8999/ws/mac")! private var webSocketTask: URLSessionWebSocketTask? private var urlSession: URLSession! @@ -15,11 +14,8 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate { public private(set) var isConnected = false private var reconnectTimer: Timer? private var pingTimer: Timer? - private var monitorTimer: Timer? - - private var lastReportedText: String? - private var lastReportedCursor: Int = -1 - private var lastReportedApp: String? + private var monitorTimerSource: DispatchSourceTimer? + private let monitorQueue = DispatchQueue(label: "com.soniox.macsync.monitor", qos: .userInteractive) public var onRemoteUpdateReceived: ((String, Int?, Bool) -> Void)? @@ -35,7 +31,7 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate { public func start() { guard !isRunning else { return } isRunning = true - print("RelayClient: Starting persistent gateway connection...") + print("RelayClient: Starting real-time bi-directional sync engine...") connect() startInputMonitoring() } @@ -46,8 +42,8 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate { reconnectTimer = nil pingTimer?.invalidate() pingTimer = nil - monitorTimer?.invalidate() - monitorTimer = nil + monitorTimerSource?.cancel() + monitorTimerSource = nil webSocketTask?.cancel(with: .goingAway, reason: nil) webSocketTask = nil isConnected = false @@ -102,72 +98,71 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate { } let action = json["action"] as? String ?? json["type"] as? String + let source = json["source"] as? String ?? "" + + // Ignore echo messages originated by Mac itself + if source == "mac" { return } + let updateText = json["text"] as? String ?? json["insert_text"] as? String ?? "" let cursor = json["cursor"] as? Int ?? json["cursor_pos"] as? Int - if action == "paste" || action == "set_text" || action == "update_input" { - let isFullReplace = (action == "set_text" || action == "update_input") - print("RelayClient: ⚡ Received remote input update: '\(updateText.prefix(30))...' (replace: \(isFullReplace))") + if action == "paste" || action == "set_text" || action == "update_input" || action == "phone_input_edit" || action == "sync_state" { + let isFullReplace = (action != "insert_at_cursor") + print("RelayClient: ⚡ Received remote Android update: '\(updateText.prefix(30))...' (replace: \(isFullReplace), cursor: \(cursor ?? -1))") DispatchQueue.main.async { self.onRemoteUpdateReceived?(updateText, cursor, isFullReplace) } - } else if action == "insert_at_cursor" { - print("RelayClient: ⚡ Received insert at cursor: '\(updateText.prefix(30))...'") - DispatchQueue.main.async { - self.onRemoteUpdateReceived?(updateText, cursor, false) - } } } - // Sends Mac's current focused element state to Android via server - public func sendInputState(app: String, text: String, cursor: Int, selection: Int) { - guard isConnected, let task = webSocketTask else { return } + private func startInputMonitoring() { + monitorTimerSource?.cancel() - // Avoid sending identical updates - if text == lastReportedText && cursor == lastReportedCursor && app == lastReportedApp { - return + let timer = DispatchSource.makeTimerSource(queue: monitorQueue) + // Poll every 50ms on dedicated user-interactive queue + timer.schedule(deadline: .now(), repeating: .milliseconds(50)) + timer.setEventHandler { [weak self] in + guard let self = self, self.isRunning, self.isConnected else { return } + if let state = FocusedInputSync.shared.inspectCurrentState() { + self.sendStateToRelay(state) + } } - - lastReportedText = text - lastReportedCursor = cursor - lastReportedApp = app + timer.resume() + self.monitorTimerSource = timer + } + + private func sendStateToRelay(_ state: MacInputState) { + guard let task = webSocketTask else { return } let payload: [String: Any] = [ - "type": "mac_input_state", - "app": app, - "text": text, - "cursor": cursor, - "selection": selection, - "timestamp": Date().timeIntervalSince1970 + "type": "sync_state", + "source": "mac", + "app": state.app, + "text": state.text, + "cursor": state.cursor, + "selection": state.selection, + "revision": state.revision, + "timestamp": state.timestamp ] if let data = try? JSONSerialization.data(withJSONObject: payload), let jsonStr = String(data: data, encoding: .utf8) { task.send(.string(jsonStr)) { error in if let error = error { - print("Error sending mac_input_state:", error) + print("Error sending sync_state:", error) } } } } - private func startInputMonitoring() { - monitorTimer?.invalidate() - // Poll focused element every 150ms - monitorTimer = Timer.scheduledTimer(withTimeInterval: 0.15, repeats: true) { [weak self] _ in - guard let self = self, self.isRunning, self.isConnected else { return } - if let state = FocusedInputSync.shared.getCurrentState() { - self.sendInputState(app: state.app, text: state.text, cursor: state.cursor, selection: state.selection) - } - } - } - private func startPingTimer() { pingTimer?.invalidate() - pingTimer = Timer.scheduledTimer(withTimeInterval: 10.0, repeats: true) { [weak self] _ in + let timer = Timer(timeInterval: 10.0, repeats: true) { [weak self] _ in guard let self = self, self.isRunning else { return } self.webSocketTask?.send(.string("{\"type\":\"ping\"}")) { _ in } } + RunLoop.main.add(timer, forMode: .common) + self.pingTimer = timer } private func scheduleReconnect() { @@ -176,14 +171,15 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate { pingTimer = nil if reconnectTimer == nil || !(reconnectTimer?.isValid ?? false) { - reconnectTimer = Timer.scheduledTimer(withTimeInterval: 2.0, repeats: false) { [weak self] _ in + let timer = Timer(timeInterval: 2.0, repeats: false) { [weak self] _ in self?.reconnectTimer = nil self?.connect() } + RunLoop.main.add(timer, forMode: .common) + self.reconnectTimer = timer } } - // URLSessionWebSocketDelegate public func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask, didOpenWithProtocol protocol: String?) { print("RelayClient: 🟢 Persistent WebSocket Connected to Server Gateway!") self.isConnected = true diff --git a/mac/test_apps_ax.swift b/mac/test_apps_ax.swift new file mode 100644 index 0000000..b3698cc --- /dev/null +++ b/mac/test_apps_ax.swift @@ -0,0 +1,42 @@ +import Cocoa +import ApplicationServices + +func inspectApp(named targetName: String) { + guard let app = NSWorkspace.shared.runningApplications.first(where: { $0.localizedName == targetName }) else { + print("App \(targetName) not running") + return + } + + print("\n--- Inspecting \(targetName) (PID: \(app.processIdentifier)) ---") + let appElem = AXUIElementCreateApplication(app.processIdentifier) + AXUIElementSetAttributeValue(appElem, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) + AXUIElementSetAttributeValue(appElem, "AXManualAccessibility" as CFString, kCFBooleanTrue) + + var focusedElemObj: CFTypeRef? + let err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) + print("kAXFocusedUIElementAttribute error:", err.rawValue) + + if err == .success, let elem = focusedElemObj { + let axElem = elem as! AXUIElement + var roleObj: CFTypeRef? + AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleObj) + print("Role:", roleObj ?? "none") + + var valObj: CFTypeRef? + let valErr = AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) + print("Value err:", valErr.rawValue, "Value:", valObj ?? "none") + + var rangeObj: CFTypeRef? + let rangeErr = AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &rangeObj) + if rangeErr == .success, let axRange = rangeObj { + var range = CFRange() + if AXValueGetValue(axRange as! AXValue, .cfRange, &range) { + print("Cursor: loc=\(range.location), len=\(range.length)") + } + } + } +} + +inspectApp(named: "Telegram") +inspectApp(named: "Obsidian") +inspectApp(named: "firefox") diff --git a/mac/test_focus_detector.swift b/mac/test_focus_detector.swift new file mode 100644 index 0000000..d6a7ffc --- /dev/null +++ b/mac/test_focus_detector.swift @@ -0,0 +1,107 @@ +import Cocoa +import ApplicationServices + +// Comprehensive recursive search for any element with AXFocused == true +func findActiveFocusedElement(_ elem: AXUIElement) -> AXUIElement? { + var isFocusedObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &isFocusedObj) == .success, + let isFocused = isFocusedObj as? Bool, isFocused { + var roleObj: CFTypeRef? + AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) + let role = (roleObj as? String) ?? "" + if role != "AXWindow" && role != "AXApplication" && role != "AXGroup" && role != "AXSplitGroup" && role != "AXScrollArea" { + return elem + } + } + + var childrenObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, + let children = childrenObj as? [AXUIElement] { + for child in children { + if let found = findActiveFocusedElement(child) { + return found + } + } + } + return nil +} + +func getFocusedTextInfo() -> (String, String, Int, Int)? { + guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil } + let appName = frontApp.localizedName ?? "App" + let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) + + // 1. Try systemWide + let sysWide = AXUIElementCreateSystemWide() + var focusedElemObj: CFTypeRef? + var targetElem: AXUIElement? + + if AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) == .success, + let obj = focusedElemObj { + targetElem = (obj as! AXUIElement) + } + + // 2. Try App focused element + if targetElem == nil { + if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) == .success, + let obj = focusedElemObj { + targetElem = (obj as! AXUIElement) + } + } + + // 3. Try Recursive search in app tree + if targetElem == nil { + targetElem = findActiveFocusedElement(appElem) + } + + guard let elem = targetElem else { return nil } + + // Extract text + var text = "" + var valObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) == .success, + let val = valObj { + if let str = val as? String { + text = str + } else if let attrStr = val as? NSAttributedString { + text = attrStr.string + } + } + + // Try parameterized string for range + if text.isEmpty { + var countObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countObj) == .success, + let count = countObj as? Int, count > 0 { + var range = CFRange(location: 0, length: count) + if let axRange = AXValueCreate(.cfRange, &range) { + var strObj: CFTypeRef? + if AXUIElementCopyParameterizedAttributeValue(elem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &strObj) == .success, + let str = strObj as? String { + text = str + } + } + } + } + + // Cursor & Selection + var cursor = text.count + var selLen = 0 + var rangeObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, &rangeObj) == .success, + let axRange = rangeObj { + var range = CFRange() + if AXValueGetValue(axRange as! AXValue, .cfRange, &range) { + cursor = range.location + selLen = range.length + } + } + + return (appName, text, cursor, selLen) +} + +if let (app, text, cursor, selLen) = getFocusedTextInfo() { + print("Found! App: \(app), Text: '\(text)', Cursor: \(cursor), SelLen: \(selLen)") +} else { + print("No focused text element found") +} diff --git a/mac/test_full_ax.swift b/mac/test_full_ax.swift new file mode 100644 index 0000000..6d5c906 --- /dev/null +++ b/mac/test_full_ax.swift @@ -0,0 +1,110 @@ +import Cocoa +import ApplicationServices + +func extractTextAndCursor(from elem: AXUIElement) -> (String, Int, Int)? { + var roleRef: CFTypeRef? + AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleRef) + let role = (roleRef as? String) ?? "" + + var currentText = "" + var valueRef: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valueRef) == .success, + let val = valueRef { + if let str = val as? String { + currentText = str + } else if let attrStr = val as? NSAttributedString { + currentText = attrStr.string + } + } + + if currentText.isEmpty { + var countRef: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countRef) == .success, + let count = countRef as? Int, count > 0 { + var range = CFRange(location: 0, length: count) + if let axRange = AXValueCreate(.cfRange, &range) { + var stringRef: CFTypeRef? + if AXUIElementCopyParameterizedAttributeValue(elem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &stringRef) == .success, + let str = stringRef as? String { + currentText = str + } + } + } + } + + var cursor = currentText.count + var selLen = 0 + var rangeRef: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, &rangeRef) == .success, + let val = rangeRef { + var cfRange = CFRange() + if AXValueGetValue(val as! AXValue, .cfRange, &cfRange) { + cursor = cfRange.location + selLen = cfRange.length + } + } + + if !currentText.isEmpty || role == "AXTextField" || role == "AXTextArea" || role == "AXSearchField" || rangeRef != nil { + return (currentText, cursor, selLen) + } + + // Check focused child + var focusedChildRef: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXFocusedUIElementAttribute as CFString, &focusedChildRef) == .success, + let child = focusedChildRef { + if let res = extractTextAndCursor(from: child as! AXUIElement) { + return res + } + } + + // Check children + var childrenRef: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenRef) == .success, + let children = childrenRef as? [AXUIElement] { + for child in children { + var isFocusedRef: CFTypeRef? + if AXUIElementCopyAttributeValue(child, kAXFocusedAttribute as CFString, &isFocusedRef) == .success, + let isFoc = isFocusedRef as? Bool, isFoc { + if let res = extractTextAndCursor(from: child) { + return res + } + } + } + } + + return nil +} + +func testFullAX() { + let sysWide = AXUIElementCreateSystemWide() + AXUIElementSetAttributeValue(sysWide, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) + AXUIElementSetAttributeValue(sysWide, "AXManualAccessibility" as CFString, kCFBooleanTrue) + + guard let frontApp = NSWorkspace.shared.frontmostApplication else { return } + print("Front App:", frontApp.localizedName ?? "") + + let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) + AXUIElementSetAttributeValue(appElem, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) + AXUIElementSetAttributeValue(appElem, "AXManualAccessibility" as CFString, kCFBooleanTrue) + + var focusedElem: CFTypeRef? + if AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &focusedElem) == .success, + let elem = focusedElem { + if let (text, cursor, sel) = extractTextAndCursor(from: elem as! AXUIElement) { + print("Extracted from sysWide -> Text: '\(text)', Cursor: \(cursor), Sel: \(sel)") + return + } + } + + if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElem) == .success, + let elem = focusedElem { + if let (text, cursor, sel) = extractTextAndCursor(from: elem as! AXUIElement) { + print("Extracted from appElem -> Text: '\(text)', Cursor: \(cursor), Sel: \(sel)") + return + } + } + + print("Could not extract text") +} + +testFullAX() diff --git a/mac/test_inspect.swift b/mac/test_inspect.swift new file mode 100644 index 0000000..f2cbadf --- /dev/null +++ b/mac/test_inspect.swift @@ -0,0 +1,45 @@ +import Cocoa +import ApplicationServices + +func inspect() { + guard let frontApp = NSWorkspace.shared.frontmostApplication else { + print("No front app") + return + } + print("Front App:", frontApp.localizedName ?? "", "PID:", frontApp.processIdentifier) + + let trusted = AXIsProcessTrusted() + print("AXIsProcessTrusted:", trusted) + + let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) + var focusedElemObj: CFTypeRef? + let err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) + print("kAXFocusedUIElement error:", err.rawValue) + + if err == .success, let elem = focusedElemObj { + let axElem = elem as! AXUIElement + var roleObj: CFTypeRef? + AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleObj) + print("Role:", roleObj ?? "none") + + var valObj: CFTypeRef? + let valErr = AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) + print("Value err:", valErr.rawValue, "Value:", valObj ?? "nil") + + var selectedTextObj: CFTypeRef? + let selTxtErr = AXUIElementCopyAttributeValue(axElem, kAXSelectedTextAttribute as CFString, &selectedTextObj) + print("SelectedText err:", selTxtErr.rawValue, "SelectedText:", selectedTextObj ?? "nil") + + var rangeObj: CFTypeRef? + let rangeErr = AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &rangeObj) + print("Range err:", rangeErr.rawValue) + if rangeErr == .success, let axRange = rangeObj { + var range = CFRange() + if AXValueGetValue(axRange as! AXValue, .cfRange, &range) { + print("CFRange: loc=\(range.location), len=\(range.length)") + } + } + } +} + +inspect() diff --git a/mac/test_inspect2.swift b/mac/test_inspect2.swift new file mode 100644 index 0000000..10143a3 --- /dev/null +++ b/mac/test_inspect2.swift @@ -0,0 +1,45 @@ +import Cocoa +import ApplicationServices + +func getFocusedElement() -> AXUIElement? { + // 1. System wide + let sysWide = AXUIElementCreateSystemWide() + var sysFocusedObj: CFTypeRef? + if AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &sysFocusedObj) == .success, + let obj = sysFocusedObj { + return (obj as! AXUIElement) + } + + // 2. Frontmost App + if let frontApp = NSWorkspace.shared.frontmostApplication { + let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) + var appFocusedObj: CFTypeRef? + if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &appFocusedObj) == .success, + let obj = appFocusedObj { + return (obj as! AXUIElement) + } + } + return nil +} + +func inspectElement(_ elem: AXUIElement) { + var attrNamesObj: CFArray? + AXUIElementCopyAttributeNames(elem, &attrNamesObj) + if let names = attrNamesObj as? [String] { + print("Attribute Names:", names) + for name in names { + var val: CFTypeRef? + let err = AXUIElementCopyAttributeValue(elem, name as CFString, &val) + if err == .success, let val = val { + print(" \(name): \(val)") + } + } + } +} + +if let focused = getFocusedElement() { + print("Found Focused Element:") + inspectElement(focused) +} else { + print("No focused element found") +} diff --git a/mac/test_inspect3.swift b/mac/test_inspect3.swift new file mode 100644 index 0000000..87217b8 --- /dev/null +++ b/mac/test_inspect3.swift @@ -0,0 +1,76 @@ +import Cocoa +import ApplicationServices + +func findFocusedDescendant(_ elem: AXUIElement) -> AXUIElement? { + // Check if this element is a text field/text area or has value + var roleObj: CFTypeRef? + AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) + let role = (roleObj as? String) ?? "" + + if role == "AXTextField" || role == "AXTextArea" || role == "AXComboBox" || role == "AXSearchField" { + return elem + } + + var focusedObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &focusedObj) == .success, + let isFocused = focusedObj as? Bool, isFocused { + // If it has value attribute, return it + var valObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) == .success { + return elem + } + } + + // Check children + var childrenObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, + let children = childrenObj as? [AXUIElement] { + for child in children { + if let found = findFocusedDescendant(child) { + return found + } + } + } + return nil +} + +func getDeepFocusedElement() -> (AXUIElement, String, String)? { + guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil } + let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) + + // First try standard focused element + var focusedObj: CFTypeRef? + if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedObj) == .success, + let elem = focusedObj as! AXUIElement? { + var roleObj: CFTypeRef? + AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) + let role = (roleObj as? String) ?? "" + + var valObj: CFTypeRef? + AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) + let val = (valObj as? String) ?? "" + + if !val.isEmpty || role == "AXTextField" || role == "AXTextArea" { + return (elem, role, val) + } + + // If it's a window or web area, search descendants + if let deep = findFocusedDescendant(elem) { + var deepRoleObj: CFTypeRef? + AXUIElementCopyAttributeValue(deep, kAXRoleAttribute as CFString, &deepRoleObj) + let deepRole = (deepRoleObj as? String) ?? "" + + var deepValObj: CFTypeRef? + AXUIElementCopyAttributeValue(deep, kAXValueAttribute as CFString, &deepValObj) + let deepVal = (deepValObj as? String) ?? "" + return (deep, deepRole, deepVal) + } + } + return nil +} + +if let (elem, role, val) = getDeepFocusedElement() { + print("Found deep focused element! Role: \(role), Value: '\(val)'") +} else { + print("Deep focused element not found") +} diff --git a/mac/test_live_ax.swift b/mac/test_live_ax.swift new file mode 100644 index 0000000..32f9efc --- /dev/null +++ b/mac/test_live_ax.swift @@ -0,0 +1,81 @@ +import Cocoa +import ApplicationServices + +func testLiveAX() { + let sysWide = AXUIElementCreateSystemWide() + + // Enable Chromium/Electron accessibility + AXUIElementSetAttributeValue(sysWide, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) + AXUIElementSetAttributeValue(sysWide, "AXManualAccessibility" as CFString, kCFBooleanTrue) + + guard let frontApp = NSWorkspace.shared.frontmostApplication else { + print("No front app") + return + } + + print("Front App:", frontApp.localizedName ?? "", "PID:", frontApp.processIdentifier) + + let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) + AXUIElementSetAttributeValue(appElem, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) + AXUIElementSetAttributeValue(appElem, "AXManualAccessibility" as CFString, kCFBooleanTrue) + + // Try system wide focused element + var focusedUIElement: CFTypeRef? + var err = AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &focusedUIElement) + if err != .success || focusedUIElement == nil { + err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedUIElement) + } + + guard err == .success, let elem = focusedUIElement else { + print("No focused element. Error:", err.rawValue) + return + } + + let axElem = elem as! AXUIElement + + var roleRef: CFTypeRef? + AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleRef) + let role = (roleRef as? String) ?? "AXUnknown" + print("Role:", role) + + // Extract text + var currentText = "" + var valueRef: CFTypeRef? + if AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valueRef) == .success, + let val = valueRef { + if let str = val as? String { + currentText = str + } else if let attrStr = val as? NSAttributedString { + currentText = attrStr.string + } + } + + if currentText.isEmpty { + var countRef: CFTypeRef? + if AXUIElementCopyAttributeValue(axElem, kAXNumberOfCharactersAttribute as CFString, &countRef) == .success, + let count = countRef as? Int { + var range = CFRange(location: 0, length: count) + if let axRange = AXValueCreate(.cfRange, &range) { + var stringRef: CFTypeRef? + if AXUIElementCopyParameterizedAttributeValue(axElem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &stringRef) == .success, + let str = stringRef as? String { + currentText = str + } + } + } + } + + print("Extracted Text: '\(currentText)'") + + // Selection / Cursor + var rangeRef: CFTypeRef? + if AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &rangeRef) == .success, + let val = rangeRef { + var cfRange = CFRange() + if AXValueGetValue(val as! AXValue, .cfRange, &cfRange) { + print("Cursor Location: \(cfRange.location), Selection Length: \(cfRange.length)") + } + } +} + +testLiveAX() diff --git a/mac/test_timer.swift b/mac/test_timer.swift new file mode 100644 index 0000000..6d83d84 --- /dev/null +++ b/mac/test_timer.swift @@ -0,0 +1,21 @@ +import Foundation +import Cocoa + +class TestTimer { + private var timerSource: DispatchSourceTimer? + + func start() { + let queue = DispatchQueue(label: "com.test.timer", qos: .userInteractive) + let timer = DispatchSource.makeTimerSource(queue: queue) + timer.schedule(deadline: .now(), repeating: .milliseconds(50)) + timer.setEventHandler { + if let (elem, app) = FocusedInputSync.shared.getFocusedElement() { + if let state = FocusedInputSync.shared.inspectCurrentState() { + print("Live State detected: '\(state.text)' in \(state.app)") + } + } + } + timer.resume() + self.timerSource = timer + } +} diff --git a/mac/test_tree.swift b/mac/test_tree.swift new file mode 100644 index 0000000..d8094c0 --- /dev/null +++ b/mac/test_tree.swift @@ -0,0 +1,39 @@ +import Cocoa +import ApplicationServices + +func printAXTree(_ elem: AXUIElement, depth: Int = 0) { + if depth > 7 { return } + let indent = String(repeating: " ", count: depth) + + var roleObj: CFTypeRef? + AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) + let role = (roleObj as? String) ?? "unknown" + + var valObj: CFTypeRef? + AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) + let val = (valObj as? String) ?? "" + + var titleObj: CFTypeRef? + AXUIElementCopyAttributeValue(elem, kAXTitleAttribute as CFString, &titleObj) + let title = (titleObj as? String) ?? "" + + var focusedObj: CFTypeRef? + AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &focusedObj) + let isFocused = (focusedObj as? Bool) ?? false + + print("\(indent)[\(role)] title='\(title)' val='\(val)' focused=\(isFocused)") + + var childrenObj: CFTypeRef? + if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, + let children = childrenObj as? [AXUIElement] { + for child in children { + printAXTree(child, depth: depth + 1) + } + } +} + +if let frontApp = NSWorkspace.shared.frontmostApplication { + print("Front App:", frontApp.localizedName ?? "") + let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) + printAXTree(appElem) +} diff --git a/mac/test_win_ax.swift b/mac/test_win_ax.swift new file mode 100644 index 0000000..7fb1f05 --- /dev/null +++ b/mac/test_win_ax.swift @@ -0,0 +1,33 @@ +import Cocoa +import ApplicationServices + +func inspectFrontApp() { + guard let frontApp = NSWorkspace.shared.frontmostApplication else { return } + print("Frontmost App:", frontApp.localizedName ?? "") + + let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) + + // 1. Try focused window + var windowObj: CFTypeRef? + var err = AXUIElementCopyAttributeValue(appElem, kAXFocusedWindowAttribute as CFString, &windowObj) + print("kAXFocusedWindowAttribute err:", err.rawValue) + + if err == .success, let win = windowObj { + let winElem = win as! AXUIElement + var focusedObj: CFTypeRef? + let winErr = AXUIElementCopyAttributeValue(winElem, kAXFocusedUIElementAttribute as CFString, &focusedObj) + print("Window focused element err:", winErr.rawValue) + if winErr == .success, let elem = focusedObj { + let axElem = elem as! AXUIElement + var roleObj: CFTypeRef? + AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleObj) + print("Role from window:", roleObj ?? "none") + + var valObj: CFTypeRef? + AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) + print("Value from window:", valObj ?? "none") + } + } +} + +inspectFrontApp() diff --git a/server/relay_server.py b/server/relay_server.py index 19fa8ab..e160e22 100644 --- a/server/relay_server.py +++ b/server/relay_server.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """ -Soniox Bi-Directional Input Synchronization Gateway (v4.0) -- Mirrors Mac focused input box <--> Android phone in real-time. -- Supports inserting voice text at exact cursor location. -- Instant <15ms WebSocket push to Mac for editing and pasting. +Soniox Collaborative Bi-Directional Input Synchronization Gateway (v5.0) +- Real-time Google Docs / Figma style collaborative mirroring between Mac and Android. +- Monotonic revision counters, echo-loop suppression, and sub-15ms WebSocket routing. +- Cursor-aware voice insertion and instant text editing. """ import asyncio @@ -19,22 +19,24 @@ import websockets from websockets.protocol import State logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") -logger = logging.getLogger("SonioxRelay") +logger = logging.getLogger("SyncGateway") connected_mac_websockets = set() connected_phone_websockets = set() -recent_pasted_sessions = OrderedDict() # session_id -> (timestamp, text) -latest_mac_input_state = { - "type": "mac_input_state", +# Global Room State Snapshot (Single Source of Truth) +current_room_state = { + "type": "sync_state", + "source": "server_init", "app": "Desktop", "text": "", "cursor": 0, "selection": 0, + "revision": 0, "timestamp": time.time() } +room_lock = asyncio.Lock() -MAC_DIRECT_HTTP_URL = "http://116.16.16.20:8999/paste" SONIOX_WS_URL = ( "wss://translate.compare.soniox.com/compare/api/compare-websocket" "?language_hints=fa&language_hints=en&language_hints=ar" @@ -104,101 +106,29 @@ class SonioxPool: soniox_pool = SonioxPool() def sanitize_and_flatten_text(text: str) -> str: - """ - 1. Removes all line breaks (\\r, \\n) and collapses whitespace into single spaces. - 2. Strips English hallucination stop words during Persian speech. - 3. Guarantees zero trailing/leading enters or spaces. - """ if not text: return "" - flattened = re.sub(r"[\r\n\t]+", " ", text) flattened = re.sub(r"\s+", " ", flattened).strip() - - if not flattened: - return "" - - words = flattened.split() - fa_pattern = re.compile(r"[\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF]") - en_pattern = re.compile(r"[a-zA-Z]") - - fa_count = sum(1 for w in words if fa_pattern.search(w)) - en_count = sum(1 for w in words if en_pattern.search(w)) - total = fa_count + en_count - - if total == 0: - return flattened - - fa_ratio = fa_count / total - stop_words = {"sex", "from", "no", "oh", "you", "all", "know", "that", "god", "loves", "men", "for", "day", "night", "mankind", "wanted", "moments"} - - cleaned = [] - if fa_ratio >= 0.25: - for w in words: - if en_pattern.search(w) and not fa_pattern.search(w): - clean_w = re.sub(r"[.,!?:;،؛؟\"'()\[\]{}«»–—-]", "", w.lower()) - if clean_w in stop_words or fa_ratio >= 0.70: - continue - cleaned.append(w) - else: - cleaned = words - - result = " ".join(cleaned) - return re.sub(r"\s+", " ", result).strip() - -async def broadcast_to_macs(payload_dict: dict) -> bool: - """Pushes command payload directly to Mac via persistent WebSocket in <15ms.""" - delivered = False - payload_str = json.dumps(payload_dict, ensure_ascii=False) - - dead_sockets = set() - for ws in list(connected_mac_websockets): - try: - if is_ws_open(ws): - await ws.send_str(payload_str) - delivered = True - logger.info("⚡ Pushed to Mac WS: %s", payload_dict.get("action") or payload_dict.get("type")) - else: - dead_sockets.add(ws) - except Exception: - dead_sockets.add(ws) - - for dead in dead_sockets: - connected_mac_websockets.discard(dead) - - if delivered: - return True - - # Fallback to SSH script if WebSocket temporarily disconnected - try: - text = payload_dict.get("text", "") - if text: - clean_text = sanitize_and_flatten_text(text) - escaped_text = clean_text.replace("'", "'\\''") - remote_cmd = ( - f"printf '%s' '{escaped_text}' | pbcopy && " - f"/usr/bin/osascript -e 'tell application \"System Events\" to keystroke \"v\" using command down'" - ) - proc = await asyncio.create_subprocess_exec( - "ssh", "-p", "2222", "-o", "BatchMode=yes", "-o", "ConnectTimeout=2", "alig@127.0.0.1", - remote_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - stdout, stderr = await proc.communicate() - if proc.returncode == 0: - logger.info("✅ Pasted to Mac via SSH Tunnel fallback: '%s'", clean_text[:30]) - return True - except Exception as e: - logger.warning("SSH fallback error: %s", e) - - return False - -async def broadcast_to_phones(payload_dict: dict): - """Pushes Mac input state changes to all connected Android clients.""" - payload_str = json.dumps(payload_dict, ensure_ascii=False) + return flattened + +async def broadcast_state(payload_dict: dict, exclude_ws=None): + """Broadcasts state snapshot to all connected clients (Mac and Phone) except sender.""" + global current_room_state + async with room_lock: + current_room_state["revision"] += 1 + payload_dict["revision"] = current_room_state["revision"] + payload_dict["timestamp"] = time.time() + + # Update our cached authoritative state + current_room_state.update(payload_dict) + payload_str = json.dumps(payload_dict, ensure_ascii=False) + + # 1. Send to Phone clients dead_phones = set() for ws in list(connected_phone_websockets): + if ws is exclude_ws: + continue try: if is_ws_open(ws): await ws.send_str(payload_str) @@ -206,26 +136,35 @@ async def broadcast_to_phones(payload_dict: dict): dead_phones.add(ws) except Exception: dead_phones.add(ws) - for dead in dead_phones: connected_phone_websockets.discard(dead) + # 2. Send to Mac clients + dead_macs = set() + for ws in list(connected_mac_websockets): + if ws is exclude_ws: + continue + try: + if is_ws_open(ws): + await ws.send_str(payload_str) + else: + dead_macs.add(ws) + except Exception: + dead_macs.add(ws) + for dead in dead_macs: + connected_mac_websockets.discard(dead) + async def handle_phone_stream_ws(request): - """ - ⚡ Persistent Duplex Channel for Android Client: - - Receives live Mac input state on connect and continuously. - - Handles live audio streaming and returns live STT tokens. - - Receives user manual text edits and pushes them instantly to Mac. - """ - ws = web.WebSocketResponse(heartbeat=15.0) + """Duplex WebSocket for Android client (Real-time Live Sync + Audio Dictation).""" + ws = web.WebSocketResponse(heartbeat=12.0) await ws.prepare(request) client_ip = request.remote - logger.info("📱 Android Client connected to Duplex Stream: %s", client_ip) + logger.info("📱 Android Client Connected: %s", client_ip) connected_phone_websockets.add(ws) - # Immediately send the latest Mac input state to phone upon connection! - if latest_mac_input_state: - await ws.send_str(json.dumps(latest_mac_input_state, ensure_ascii=False)) + # Immediately hydrate phone with full current state! + async with room_lock: + await ws.send_str(json.dumps(current_room_state, ensure_ascii=False)) active_soniox_ws = None reader_task = None @@ -286,7 +225,7 @@ async def handle_phone_stream_ws(request): try: async for msg in ws: if msg.type == aiohttp.WSMsgType.BINARY: - # Live PCM audio chunk (2048 bytes / 64ms) + # Live PCM audio chunk if active_soniox_ws and is_ws_open(active_soniox_ws): await active_soniox_ws.send(msg.data) @@ -299,7 +238,13 @@ async def handle_phone_stream_ws(request): msg_type = data.get("type") or data.get("action") sid = data.get("session_id", f"sess_{int(time.time()*1000)}") - if msg_type == "start": + if msg_type == "sync_state" or msg_type == "phone_input_edit" or msg_type == "update_input": + # Phone edited text: broadcast to Mac immediately! + data["source"] = "android" + data["type"] = "sync_state" + await broadcast_state(data, exclude_ws=ws) + + elif msg_type == "start": current_session_id = sid full_final_tokens.clear() current_non_final = "" @@ -327,7 +272,7 @@ async def handle_phone_stream_ws(request): clean_final = sanitize_and_flatten_text(raw_final) logger.info("⚡ Session %s final text: '%s'", sid, clean_final) - # Return final voice transcription to Android + # Return final speech text to phone if is_ws_open(ws): await ws.send_str(json.dumps({ "type": "final", @@ -347,32 +292,6 @@ async def handle_phone_stream_ws(request): asyncio.create_task(soniox_pool.refill()) - elif msg_type == "phone_input_edit" or msg_type == "update_mac_input": - # User manually edited text on phone or pressed "Insert in Mac" - edit_text = data.get("text", "") - cursor = data.get("cursor_pos") or data.get("cursor") - is_full_replace = data.get("is_full_replace", True) - logger.info("📱 Received phone edit to push to Mac: '%s' (cursor: %s)", edit_text[:30], cursor) - - mac_ok = await broadcast_to_macs({ - "action": "update_input", - "text": edit_text, - "cursor": cursor, - "is_full_replace": is_full_replace - }) - - # Update our cached latest state - latest_mac_input_state["text"] = edit_text - latest_mac_input_state["cursor"] = cursor if cursor is not None else len(edit_text) - latest_mac_input_state["timestamp"] = time.time() - - if is_ws_open(ws): - await ws.send_str(json.dumps({ - "type": "edit_ack", - "session_id": sid, - "mac_delivered": mac_ok - })) - elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.ERROR): break @@ -391,27 +310,26 @@ async def handle_phone_stream_ws(request): return ws async def handle_mac_ws(request): - """Persistent WebSocket for Mac Bridge (receives input state & pushes edits).""" - global latest_mac_input_state + """Persistent WebSocket for Mac Bridge (receives live Mac typing & pushes edits).""" ws = web.WebSocketResponse(heartbeat=10.0) await ws.prepare(request) client_ip = request.remote - logger.info("🖥️ Mac client connected to persistent WebSocket: %s", client_ip) + logger.info("🖥️ Mac client connected: %s", client_ip) connected_mac_websockets.add(ws) try: - await ws.send_str(json.dumps({"type": "welcome", "message": "Connected to Soniox Duplex Gateway"})) + await ws.send_str(json.dumps({"type": "welcome", "message": "Connected to Soniox Sync Gateway"})) async for msg in ws: if msg.type == aiohttp.WSMsgType.TEXT: try: data = json.loads(msg.data) msg_type = data.get("type") - if msg_type == "mac_input_state": - # Mac reports focused input box text and cursor position - latest_mac_input_state = data - # Broadcast immediately to phone! - await broadcast_to_phones(data) + if msg_type == "sync_state" or msg_type == "mac_input_state": + # Mac reports typing / cursor change: broadcast to all phones immediately! + data["source"] = "mac" + data["type"] = "sync_state" + await broadcast_state(data, exclude_ws=ws) elif msg_type == "ping": await ws.send_str(json.dumps({"type": "pong"})) @@ -421,35 +339,32 @@ async def handle_mac_ws(request): break finally: connected_mac_websockets.discard(ws) - if is_ws_open(ws): - await ws.close() logger.info("🖥️ Mac client disconnected: %s", client_ip) return ws async def handle_health(request): + async with room_lock: + state_copy = dict(current_room_state) return web.json_response({ "status": "ok", - "service": "Soniox Bi-Directional Duplex Gateway v4.0", + "service": "Soniox Collaborative Sync Gateway v5.0", "connected_macs": len(connected_mac_websockets), "connected_phones": len(connected_phone_websockets), - "latest_mac_input_app": latest_mac_input_state.get("app", ""), - "latest_mac_input_text_len": len(latest_mac_input_state.get("text", "")) + "current_app": state_copy.get("app", ""), + "current_revision": state_copy.get("revision", 0), + "current_text_len": len(state_copy.get("text", "")) }) async def handle_paste(request): try: data = await request.json() text = data.get("text", "") - session_id = data.get("session_id", "") cursor = data.get("cursor_pos") or data.get("cursor") - success = await broadcast_to_macs({ - "action": "update_input", - "text": text, - "cursor": cursor, - "is_full_replace": True - }) - return web.json_response({"status": "pasted" if success else "failed", "mac_delivered": success}) + data["source"] = "http_post" + data["type"] = "sync_state" + await broadcast_state(data) + return web.json_response({"status": "synced", "revision": current_room_state["revision"]}) except Exception as e: return web.json_response({"error": str(e)}, status=400)