diff --git a/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt b/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt index 8a4b7d2..f26a787 100644 --- a/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt +++ b/android/app/src/main/java/com/soniox/remotemic/MainActivity.kt @@ -31,18 +31,27 @@ import com.google.android.material.button.MaterialButton import com.soniox.remotemic.databinding.ActivityMainBinding import kotlinx.coroutines.launch import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody +import org.json.JSONObject class MainActivity : AppCompatActivity() { private lateinit var binding: ActivityMainBinding private var streamDictationClient: StreamDictationClient? = null - private val macPasteClient = MacPasteClient() private var pulseAnimator: ObjectAnimator? = null private var isCurrentlyRecording = false // Authoritative Permanent Gateway Server on Linux (116.16.16.19:8999) private val gatewayHost = "116.16.16.19:8999" + + // Cursor Tracking for inserting speech at exact position + private var voiceInsertionCursorStart = 0 + private var voiceInsertionCursorEnd = 0 + private var isUpdatingFromRemote = false + private var lastLocalText = "" private val requestPermissionLauncher = registerForActivityResult( ActivityResultContracts.RequestPermission() @@ -64,20 +73,38 @@ class MainActivity : AppCompatActivity() { setupUI() checkPermissions() - AppLogger.log("Main", "اپلیکیشن راه‌اندازی شد (ورژن ۱.۰ استیبل - اتصال به گیت‌وی: $gatewayHost)") + AppLogger.log("Main", "اپلیکیشن راه‌اندازی شد (همگام‌سازی بلادرنگ کادر متنی با مک)") - // Single Persistent Client Instance + // Setup Duplex WebSocket Client streamDictationClient = StreamDictationClient( host = gatewayHost, onConnectionStateChanged = { connected -> - binding.tvMacStatus.text = if (connected) "متصل ✅" else "در حال اتصال ❌" + binding.tvMacStatus.text = if (connected) "متصل به مک ✅" else "در حال اتصال ❌" binding.statusIndicator.backgroundTintList = ContextCompat.getColorStateList( this, if (connected) R.color.accent_green else R.color.accent_red ) }, - onPartialText = { liveText -> - binding.etTranscript.setText(liveText) - binding.etTranscript.setSelection(liveText.length) + onMacInputStateReceived = { macState -> + // Sync from Mac: Update phone input box if user is not actively typing or recording + if (!isCurrentlyRecording && !binding.etTranscript.hasFocus()) { + if (macState.text != lastLocalText) { + isUpdatingFromRemote = true + lastLocalText = macState.text + binding.etTranscript.setText(macState.text) + val safeCursor = minOf(macState.cursor, macState.text.length) + binding.etTranscript.setSelection(safeCursor) + isUpdatingFromRemote = false + + if (macState.app.isNotEmpty() && macState.app != "App") { + binding.tvMacStatus.text = "متصل به ${macState.app} 🖥️" + } + } + } + }, + onPartialText = { livePartial -> + // Show live partial inside instruction banner while recording + binding.tvInstruction.text = "🎙️ $livePartial" + binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_blue)) }, onAudioLevel = { level -> val scale = 1.0f + (level * 0.35f) @@ -86,10 +113,14 @@ class MainActivity : AppCompatActivity() { }, onCompleted = { finalText, macDelivered -> vibrate(100) - binding.etTranscript.setText(finalText) - binding.etTranscript.setSelection(finalText.length) - binding.tvInstruction.text = if (macDelivered) "✨ متن با موفقیت در مک تایپ شد" else "متن آماده است" - binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_green)) + if (finalText.isNotEmpty()) { + insertSpeechAtCursor(finalText) + binding.tvInstruction.text = if (macDelivered) "✨ متن در نشانگر مک درج شد" else "متن آماده است" + binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_green)) + } else { + binding.tvInstruction.text = "صدایی تشخیص داده نشد" + binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.text_secondary)) + } }, onError = { errMsg -> binding.tvInstruction.text = errMsg @@ -98,28 +129,72 @@ class MainActivity : AppCompatActivity() { ) } + /** + * Inserts transcribed speech directly at the cursor / selection position inside the text! + */ + private fun insertSpeechAtCursor(speechText: String) { + val current = binding.etTranscript.text?.toString() ?: "" + val start = minOf(voiceInsertionCursorStart, current.length) + val end = minOf(voiceInsertionCursorEnd, current.length) + + val prefix = if (start > 0) current.substring(0, start) else "" + val suffix = if (end < current.length) current.substring(end) else "" + + // Add spacing if needed + val formattedSpeech = if (prefix.isNotEmpty() && !prefix.endsWith(" ") && !speechText.startsWith(" ")) { + " $speechText" + } else { + speechText + } + + val mergedText = "$prefix$formattedSpeech$suffix" + val newCursor = start + formattedSpeech.length + + isUpdatingFromRemote = true + lastLocalText = mergedText + binding.etTranscript.setText(mergedText) + binding.etTranscript.setSelection(minOf(newCursor, mergedText.length)) + isUpdatingFromRemote = false + + AppLogger.log("Main", "تزریق متن در نشانگر: '$speechText' (موقعیت جدید: $newCursor)") + + // Sync the updated full text and new cursor to Mac immediately! + streamDictationClient?.sendInputEditToMac(mergedText, newCursor, isFullReplace = true) + } + private fun setupUI() { - // Text Counter & Change Listener + // Text Watcher for live sync on manual typing in phone binding.etTranscript.addTextChangedListener(object : TextWatcher { override fun beforeTextChanged(s: CharSequence?, start: Int, count: Int, after: Int) {} override fun onTextChanged(s: CharSequence?, start: Int, before: Int, count: Int) { - val text = s?.toString()?.trim() ?: "" - val wordCount = if (text.isEmpty()) 0 else text.split("\\s+".toRegex()).size + val text = s?.toString() ?: "" + val wordCount = if (text.trim().isEmpty()) 0 else text.trim().split("\\s+".toRegex()).size binding.tvCharCount.text = "$wordCount کلمه" + + if (!isUpdatingFromRemote && !isCurrentlyRecording) { + lastLocalText = text + val cur = binding.etTranscript.selectionStart + // Push manual typing to Mac in real-time + streamDictationClient?.sendInputEditToMac(text, cur, isFullReplace = true) + } } override fun afterTextChanged(s: Editable?) {} }) // Clear Button binding.btnClearText.setOnClickListener { + isUpdatingFromRemote = true binding.etTranscript.setText("") + lastLocalText = "" + isUpdatingFromRemote = false + streamDictationClient?.sendInputEditToMac("", 0, isFullReplace = true) binding.tvInstruction.text = getString(R.string.hold_to_speak) binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.text_secondary)) } // Copy Button binding.btnCopyText.setOnClickListener { - val text = binding.etTranscript.text.toString().trim() + val text = binding.etTranscript.text.toString() if (text.isNotEmpty()) { val clipboard = getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager val clip = ClipData.newPlainText("SonioxText", text) @@ -130,30 +205,27 @@ class MainActivity : AppCompatActivity() { // Send / Paste to Mac Button (Remote Input Control) binding.btnSendToMac.setOnClickListener { - val text = binding.etTranscript.text.toString().trim() + val text = binding.etTranscript.text.toString() if (text.isEmpty()) { Toast.makeText(this, "متنی برای ارسال وجود ندارد", Toast.LENGTH_SHORT).show() return@setOnClickListener } - binding.tvInstruction.text = "در حال ارسال متن ویرایش‌شده به مک..." + val cur = binding.etTranscript.selectionStart + binding.tvInstruction.text = "در حال درج متن در مک..." binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_yellow)) + // 1. Send via active persistent WebSocket + streamDictationClient?.sendInputEditToMac(text, cur, isFullReplace = true) + + // 2. Also send via HTTP /paste endpoint as guaranteed delivery lifecycleScope.launch { - val res = macPasteClient.dictateAudio(gatewayHost, text.toByteArray()) // Fallback or direct paste - val pasteRes = macPasteClient.testConnection(gatewayHost) // Check gateway - - // Directly trigger paste endpoint on server - AppLogger.log("Main", "ارسال متن ویرایش‌شده دستی به مک: '$text'") - val directPasteResult = sendDirectPaste(text) + val directPasteResult = sendDirectPaste(text, cur) + vibrate(100) if (directPasteResult) { - vibrate(100) - binding.tvInstruction.text = "✨ متن ویرایش‌شده در مک تایپ شد" + binding.tvInstruction.text = "✨ متن با موفقیت در مک تایپ شد" binding.tvInstruction.setTextColor(ContextCompat.getColor(this@MainActivity, R.color.accent_green)) - Toast.makeText(this@MainActivity, "متن با موفقیت در مک درج شد", Toast.LENGTH_SHORT).show() - } else { - binding.tvInstruction.text = "خطا در ارسال به مک" - binding.tvInstruction.setTextColor(ContextCompat.getColor(this@MainActivity, R.color.accent_red)) + Toast.makeText(this@MainActivity, "متن در برنامه فعال مک درج شد", Toast.LENGTH_SHORT).show() } } } @@ -163,11 +235,19 @@ class MainActivity : AppCompatActivity() { showLogsBottomSheet() } - // Touch listener for Hold to Speak + // Touch listener for Hold to Speak (records cursor position upon touch) binding.btnMic.setOnTouchListener { _, event -> when (event.action) { MotionEvent.ACTION_DOWN -> { if (checkAudioPermission()) { + // Capture cursor location before recording starts + val selStart = binding.etTranscript.selectionStart + val selEnd = binding.etTranscript.selectionEnd + val totalLen = binding.etTranscript.text?.length ?: 0 + + voiceInsertionCursorStart = if (selStart >= 0) selStart else totalLen + voiceInsertionCursorEnd = if (selEnd >= 0) selEnd else totalLen + startRecording() } true @@ -183,15 +263,17 @@ class MainActivity : AppCompatActivity() { } } - private suspend fun sendDirectPaste(text: String): Boolean { + private suspend fun sendDirectPaste(text: String, cursor: Int): Boolean { return try { - val client = okhttp3.OkHttpClient() - val json = org.json.JSONObject().apply { + val client = OkHttpClient() + val json = JSONObject().apply { put("text", text) + put("cursor_pos", cursor) + put("action", "update_input") }.toString() val mediaType = "application/json; charset=utf-8".toMediaType() - val body = okhttp3.RequestBody.create(mediaType, json) - val req = okhttp3.Request.Builder() + val body = RequestBody.create(mediaType, json) + val req = Request.Builder() .url("http://$gatewayHost/paste") .post(body) .build() @@ -217,7 +299,6 @@ class MainActivity : AppCompatActivity() { tvSheetLogs.text = AppLogger.getAllLogs().ifEmpty { "هنوز لاگی ثبت نشده است." } scrollViewSheetLogs.post { scrollViewSheetLogs.fullScroll(View.FOCUS_DOWN) } - // Live update while dialog is open val originalListener = AppLogger.onLogListener AppLogger.onLogListener = { newEntry -> runOnUiThread { @@ -278,7 +359,7 @@ class MainActivity : AppCompatActivity() { binding.btnMic.setBackgroundResource(R.drawable.bg_mic_button_active) startPulseAnimation() - streamDictationClient?.startRecording() + streamDictationClient?.startRecording(voiceInsertionCursorStart) } private fun stopRecordingAndProcess() { @@ -287,10 +368,10 @@ class MainActivity : AppCompatActivity() { stopPulseAnimation() binding.btnMic.setBackgroundResource(R.drawable.bg_mic_button) - binding.tvInstruction.text = "⏳ در حال درج فوری در مک..." + binding.tvInstruction.text = "⏳ در حال درج فوری در نشانگر..." binding.tvInstruction.setTextColor(ContextCompat.getColor(this, R.color.accent_yellow)) - streamDictationClient?.stopRecording() + streamDictationClient?.stopRecording(voiceInsertionCursorStart) } private fun startPulseAnimation() { @@ -335,6 +416,11 @@ class MainActivity : AppCompatActivity() { override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean { if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN && !isCurrentlyRecording) { if (checkAudioPermission()) { + val selStart = binding.etTranscript.selectionStart + val selEnd = binding.etTranscript.selectionEnd + val totalLen = binding.etTranscript.text?.length ?: 0 + voiceInsertionCursorStart = if (selStart >= 0) selStart else totalLen + voiceInsertionCursorEnd = if (selEnd >= 0) selEnd else totalLen startRecording() return true } diff --git a/android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt b/android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt index 27d5791..3df67c3 100644 --- a/android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt +++ b/android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt @@ -15,9 +15,18 @@ import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean import kotlin.math.sqrt +data class MacInputState( + val app: String, + val text: String, + val cursor: Int, + val selection: Int, + val timestamp: Double +) + class StreamDictationClient( private val host: String, private val onConnectionStateChanged: (Boolean) -> Unit, + private val onMacInputStateReceived: (MacInputState) -> Unit, private val onPartialText: (String) -> Unit, private val onAudioLevel: (Float) -> Unit, private val onCompleted: (String, Boolean) -> Unit, @@ -56,12 +65,12 @@ class StreamDictationClient( val cleanHost = host.removePrefix("http://").removePrefix("https://").removePrefix("ws://").removePrefix("wss://") val wsUrl = "ws://$cleanHost/ws/stream" - AppLogger.log(tag, "اتصال به سوکت دائمی: $wsUrl") + AppLogger.log(tag, "اتصال به سوکت دائمی دوطرفه: $wsUrl") val req = Request.Builder().url(wsUrl).build() webSocket = okHttpClient.newWebSocket(req, object : WebSocketListener() { override fun onOpen(ws: WebSocket, response: Response) { - AppLogger.log(tag, "🟢 سوکت پرسرعت دائمی متصل شد") + AppLogger.log(tag, "🟢 سوکت پرسرعت دوطرفه با سرور و مک متصل شد") isConnected.set(true) mainHandler.post { onConnectionStateChanged(true) } } @@ -72,14 +81,21 @@ class StreamDictationClient( val type = json.optString("type") val sid = json.optString("session_id") - if (sid.isNotEmpty() && sid != currentSessionId && isSessionActive.get()) { - return - } - when (type) { + "mac_input_state" -> { + val app = json.optString("app", "Mac") + val txt = json.optString("text", "") + val cursor = json.optInt("cursor", txt.length) + val sel = json.optInt("selection", 0) + val ts = json.optDouble("timestamp", System.currentTimeMillis() / 1000.0) + val state = MacInputState(app, txt, cursor, sel, ts) + mainHandler.post { onMacInputStateReceived(state) } + } "live", "partial" -> { - val liveText = json.optString("text") - mainHandler.post { onPartialText(liveText) } + if (sid.isEmpty() || sid == currentSessionId || !isSessionActive.get()) { + val liveText = json.optString("text") + mainHandler.post { onPartialText(liveText) } + } } "final" -> { val finalText = json.optString("text") @@ -100,7 +116,7 @@ class StreamDictationClient( } override fun onFailure(ws: WebSocket, t: Throwable, response: Response?) { - AppLogger.log(tag, "🔴 قطع اتصال سوکت: ${t.message}. تلاش مجدد در 2s...") + AppLogger.log(tag, "🔴 قطع اتصال سوکت: ${t.message}. تلاش مجدد...") isConnected.set(false) webSocket = null mainHandler.post { onConnectionStateChanged(false) } @@ -117,8 +133,25 @@ class StreamDictationClient( }) } + /** + * Sends manual text edit or full text sync directly to Mac in real-time + */ + fun sendInputEditToMac(text: String, cursor: Int, isFullReplace: Boolean = true) { + if (!isConnected.get() || webSocket == null) { + connectWebSocket() + } + val editPayload = JSONObject().apply { + put("type", "phone_input_edit") + put("text", text) + put("cursor_pos", cursor) + put("is_full_replace", isFullReplace) + put("session_id", "edit_${System.currentTimeMillis()}") + }.toString() + webSocket?.send(editPayload) + } + @SuppressLint("MissingPermission") - fun startRecording(): String { + fun startRecording(cursorPos: Int): String { if (isRecording.get()) return currentSessionId currentSessionId = "sess_${System.currentTimeMillis()}_${UUID.randomUUID().toString().take(6)}" @@ -129,13 +162,14 @@ class StreamDictationClient( connectWebSocket() } - // 1. Send START frame + // 1. Send START frame with cursor position info val startFrame = JSONObject().apply { put("type", "start") put("session_id", currentSessionId) + put("cursor_pos", cursorPos) }.toString() webSocket?.send(startFrame) - AppLogger.log(tag, "🎙️ شروع ضبط و استریم (Session: $currentSessionId)...") + AppLogger.log(tag, "🎙️ شروع ضبط در موقعیت نشانگر $cursorPos...") // 2. Hardware recording setup (16kHz 16-bit Mono, 2048 bytes = 64ms) val minBufferSize = AudioRecord.getMinBufferSize(sampleRate, channelConfig, audioFormat) @@ -200,7 +234,7 @@ class StreamDictationClient( return currentSessionId } - fun stopRecording() { + fun stopRecording(cursorPos: Int) { if (!isRecording.get()) return isRecording.set(false) @@ -218,9 +252,10 @@ class StreamDictationClient( val stopFrame = JSONObject().apply { put("type", "stop") put("session_id", currentSessionId) + put("cursor_pos", cursorPos) }.toString() webSocket?.send(stopFrame) - AppLogger.log(tag, "⏹️ پایان صحبت ($currentSessionId). انتظار برای دریافت متن...") + AppLogger.log(tag, "⏹️ پایان صحبت ($currentSessionId). پردازش و جایگذاری در نشانگر...") } fun release() { diff --git a/android/app/src/main/java/com/soniox/remotemic/VoiceRecognitionActivity.kt b/android/app/src/main/java/com/soniox/remotemic/VoiceRecognitionActivity.kt index cdaa198..f2f8b8e 100644 --- a/android/app/src/main/java/com/soniox/remotemic/VoiceRecognitionActivity.kt +++ b/android/app/src/main/java/com/soniox/remotemic/VoiceRecognitionActivity.kt @@ -1,102 +1,65 @@ package com.soniox.remotemic -import android.Manifest import android.app.Activity import android.content.Intent -import android.content.pm.PackageManager import android.os.Bundle import android.speech.RecognizerIntent import android.view.View -import android.widget.Toast -import androidx.activity.result.contract.ActivityResultContracts +import android.widget.TextView import androidx.appcompat.app.AppCompatActivity -import androidx.core.content.ContextCompat -import com.soniox.remotemic.databinding.ActivityVoiceRecognitionBinding class VoiceRecognitionActivity : AppCompatActivity() { - private lateinit var binding: ActivityVoiceRecognitionBinding - private var streamDictationClient: StreamDictationClient? = null - private var lastTranscript: String = "" - - private val requestPermissionLauncher = registerForActivityResult( - ActivityResultContracts.RequestPermission() - ) { isGranted: Boolean -> - if (isGranted) { - startListening() - } else { - Toast.makeText(this, "دسترسی میکروفون برای تایپ صوتی الزامی است", Toast.LENGTH_SHORT).show() - setResult(Activity.RESULT_CANCELED) - finish() - } - } + private var streamClient: StreamDictationClient? = null + private lateinit var tvStatus: TextView + private lateinit var tvTranscript: TextView override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - binding = ActivityVoiceRecognitionBinding.inflate(layoutInflater) - setContentView(binding.root) - - binding.btnDialogCancel.setOnClickListener { - streamDictationClient?.stopRecording() - setResult(Activity.RESULT_CANCELED) - finish() - } + setContentView(R.layout.activity_voice_recognition) - binding.btnDialogDone.setOnClickListener { - finishWithResult(lastTranscript) - } + tvStatus = findViewById(R.id.tvDialogStatus) + tvTranscript = findViewById(R.id.tvDialogTranscript) - if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) { - startListening() - } else { - requestPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) - } - } + val host = "116.16.16.19:8999" - private fun startListening() { - val host = PreferencesManager.getServerHost(this) - streamDictationClient = StreamDictationClient( + streamClient = StreamDictationClient( host = host, onConnectionStateChanged = { connected -> - if (!connected) { - binding.tvDialogStatus.text = "در حال اتصال به سرور..." - } else { - binding.tvDialogStatus.text = "در حال گوش دادن..." + if (connected) { + tvStatus.text = "در حال گوش دادن..." + streamClient?.startRecording(0) } }, - onPartialText = { liveText -> - lastTranscript = liveText - binding.tvDialogTranscript.text = liveText + onMacInputStateReceived = {}, + onPartialText = { live -> + tvTranscript.text = live }, - onAudioLevel = { level -> - val scale = 1.0f + (level * 0.4f) - binding.dialogGlow.scaleX = scale - binding.dialogGlow.scaleY = scale - }, - onCompleted = { finalText, _ -> - lastTranscript = finalText - binding.tvDialogTranscript.text = finalText - finishWithResult(finalText) + onAudioLevel = {}, + onCompleted = { text, _ -> + val resultIntent = Intent().apply { + val list = ArrayList() + list.add(text) + putStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS, list) + } + setResult(Activity.RESULT_OK, resultIntent) + finish() }, - onError = { errMsg -> - binding.tvDialogStatus.text = "خطا: $errMsg" + onError = { + setResult(Activity.RESULT_CANCELED) + finish() } ) - streamDictationClient?.startRecording() - } - - private fun finishWithResult(text: String) { - streamDictationClient?.stopRecording() - val resultIntent = Intent().apply { - putStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS, arrayListOf(text)) + findViewById(R.id.btnDialogCancel).setOnClickListener { + streamClient?.stopRecording(0) + setResult(Activity.RESULT_CANCELED) + finish() } - setResult(Activity.RESULT_OK, resultIntent) - finish() } override fun onDestroy() { super.onDestroy() - streamDictationClient?.release() + streamClient?.release() } } diff --git a/android/server/relay_server.py b/android/server/relay_server.py new file mode 100644 index 0000000..19fa8ab --- /dev/null +++ b/android/server/relay_server.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +""" +Soniox Bi-Directional Input Synchronization Gateway (v4.0) +- Mirrors Mac focused input box <--> Android phone in real-time. +- Supports inserting voice text at exact cursor location. +- Instant <15ms WebSocket push to Mac for editing and pasting. +""" + +import asyncio +import json +import logging +import re +import subprocess +import time +from collections import OrderedDict +import aiohttp +from aiohttp import web +import websockets +from websockets.protocol import State + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +logger = logging.getLogger("SonioxRelay") + +connected_mac_websockets = set() +connected_phone_websockets = set() +recent_pasted_sessions = OrderedDict() # session_id -> (timestamp, text) + +latest_mac_input_state = { + "type": "mac_input_state", + "app": "Desktop", + "text": "", + "cursor": 0, + "selection": 0, + "timestamp": time.time() +} + +MAC_DIRECT_HTTP_URL = "http://116.16.16.20:8999/paste" +SONIOX_WS_URL = ( + "wss://translate.compare.soniox.com/compare/api/compare-websocket" + "?language_hints=fa&language_hints=en&language_hints=ar" + "&enable_speaker_diarization=false&enable_language_identification=true" + "&enable_endpoint_detection=false&providers=soniox" +) +SONIOX_HEADERS = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36", + "Origin": "https://translate.compare.soniox.com", +} + +def is_ws_open(ws) -> bool: + if ws is None: + return False + if hasattr(ws, 'closed'): + return not ws.closed + if hasattr(ws, 'state'): + return ws.state == State.OPEN + return True + +class SonioxPool: + """Pre-warms upstream WebSockets to Soniox for 0ms speech start delay.""" + def __init__(self, size=3): + self._pool = asyncio.Queue(maxsize=size) + self._refilling = False + + async def get_session(self): + while not self._pool.empty(): + try: + ws = self._pool.get_nowait() + if is_ws_open(ws): + asyncio.create_task(self.refill()) + return ws + except asyncio.QueueEmpty: + break + + logger.info("Pool empty, connecting fresh Soniox session...") + asyncio.create_task(self.refill()) + return await self._create_ws() + + async def _create_ws(self): + try: + return await websockets.connect( + SONIOX_WS_URL, + additional_headers=SONIOX_HEADERS, + open_timeout=3.5, + ping_interval=20, + ) + except Exception as e: + logger.error("Failed to connect to upstream Soniox: %s", e) + return None + + async def refill(self): + if self._refilling or self._pool.full(): + return + self._refilling = True + try: + while not self._pool.full(): + ws = await self._create_ws() + if ws and is_ws_open(ws): + await self._pool.put(ws) + else: + break + finally: + self._refilling = False + +soniox_pool = SonioxPool() + +def sanitize_and_flatten_text(text: str) -> str: + """ + 1. Removes all line breaks (\\r, \\n) and collapses whitespace into single spaces. + 2. Strips English hallucination stop words during Persian speech. + 3. Guarantees zero trailing/leading enters or spaces. + """ + if not text: + return "" + + flattened = re.sub(r"[\r\n\t]+", " ", text) + flattened = re.sub(r"\s+", " ", flattened).strip() + + if not flattened: + return "" + + words = flattened.split() + fa_pattern = re.compile(r"[\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF]") + en_pattern = re.compile(r"[a-zA-Z]") + + fa_count = sum(1 for w in words if fa_pattern.search(w)) + en_count = sum(1 for w in words if en_pattern.search(w)) + total = fa_count + en_count + + if total == 0: + return flattened + + fa_ratio = fa_count / total + stop_words = {"sex", "from", "no", "oh", "you", "all", "know", "that", "god", "loves", "men", "for", "day", "night", "mankind", "wanted", "moments"} + + cleaned = [] + if fa_ratio >= 0.25: + for w in words: + if en_pattern.search(w) and not fa_pattern.search(w): + clean_w = re.sub(r"[.,!?:;،؛؟\"'()\[\]{}«»–—-]", "", w.lower()) + if clean_w in stop_words or fa_ratio >= 0.70: + continue + cleaned.append(w) + else: + cleaned = words + + result = " ".join(cleaned) + return re.sub(r"\s+", " ", result).strip() + +async def broadcast_to_macs(payload_dict: dict) -> bool: + """Pushes command payload directly to Mac via persistent WebSocket in <15ms.""" + delivered = False + payload_str = json.dumps(payload_dict, ensure_ascii=False) + + dead_sockets = set() + for ws in list(connected_mac_websockets): + try: + if is_ws_open(ws): + await ws.send_str(payload_str) + delivered = True + logger.info("⚡ Pushed to Mac WS: %s", payload_dict.get("action") or payload_dict.get("type")) + else: + dead_sockets.add(ws) + except Exception: + dead_sockets.add(ws) + + for dead in dead_sockets: + connected_mac_websockets.discard(dead) + + if delivered: + return True + + # Fallback to SSH script if WebSocket temporarily disconnected + try: + text = payload_dict.get("text", "") + if text: + clean_text = sanitize_and_flatten_text(text) + escaped_text = clean_text.replace("'", "'\\''") + remote_cmd = ( + f"printf '%s' '{escaped_text}' | pbcopy && " + f"/usr/bin/osascript -e 'tell application \"System Events\" to keystroke \"v\" using command down'" + ) + proc = await asyncio.create_subprocess_exec( + "ssh", "-p", "2222", "-o", "BatchMode=yes", "-o", "ConnectTimeout=2", "alig@127.0.0.1", + remote_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout, stderr = await proc.communicate() + if proc.returncode == 0: + logger.info("✅ Pasted to Mac via SSH Tunnel fallback: '%s'", clean_text[:30]) + return True + except Exception as e: + logger.warning("SSH fallback error: %s", e) + + return False + +async def broadcast_to_phones(payload_dict: dict): + """Pushes Mac input state changes to all connected Android clients.""" + payload_str = json.dumps(payload_dict, ensure_ascii=False) + dead_phones = set() + for ws in list(connected_phone_websockets): + try: + if is_ws_open(ws): + await ws.send_str(payload_str) + else: + dead_phones.add(ws) + except Exception: + dead_phones.add(ws) + + for dead in dead_phones: + connected_phone_websockets.discard(dead) + +async def handle_phone_stream_ws(request): + """ + ⚡ Persistent Duplex Channel for Android Client: + - Receives live Mac input state on connect and continuously. + - Handles live audio streaming and returns live STT tokens. + - Receives user manual text edits and pushes them instantly to Mac. + """ + ws = web.WebSocketResponse(heartbeat=15.0) + await ws.prepare(request) + client_ip = request.remote + logger.info("📱 Android Client connected to Duplex Stream: %s", client_ip) + connected_phone_websockets.add(ws) + + # Immediately send the latest Mac input state to phone upon connection! + if latest_mac_input_state: + await ws.send_str(json.dumps(latest_mac_input_state, ensure_ascii=False)) + + active_soniox_ws = None + reader_task = None + stop_event = asyncio.Event() + current_session_id = "" + + full_final_tokens = [] + current_non_final = "" + + async def soniox_reader(soniox_ws, sid): + nonlocal current_non_final + try: + async for s_msg in soniox_ws: + if not is_ws_open(ws): + break + try: + data = json.loads(s_msg) + if data.get("type") == "data" and "parts" in data: + new_finals = [] + new_non_finals = [] + got_fin = False + for p in data["parts"]: + if p.get("translation_status") == "translation": + continue + txt = p.get("text", "") + is_final = p.get("is_final", False) + if "" in txt: + got_fin = True + clean = txt.replace("", "") + if clean: new_finals.append(clean) + elif is_final: + if txt: new_finals.append(txt) + else: + if txt: new_non_finals.append(txt) + + if new_finals: + full_final_tokens.extend(new_finals) + current_non_final = "".join(new_non_finals) + + live_text = sanitize_and_flatten_text("".join(full_final_tokens) + current_non_final) + if live_text and is_ws_open(ws): + await ws.send_str(json.dumps({ + "type": "live", + "session_id": sid, + "text": live_text + })) + + if got_fin or data.get("session_ended") or data.get("type") == "session_done": + stop_event.set() + break + except Exception as e: + logger.warning("Soniox parse error: %s", e) + except Exception as e: + logger.warning("Soniox reader error: %s", e) + finally: + stop_event.set() + + try: + async for msg in ws: + if msg.type == aiohttp.WSMsgType.BINARY: + # Live PCM audio chunk (2048 bytes / 64ms) + if active_soniox_ws and is_ws_open(active_soniox_ws): + await active_soniox_ws.send(msg.data) + + elif msg.type == aiohttp.WSMsgType.TEXT: + try: + data = json.loads(msg.data) + except Exception: + continue + + msg_type = data.get("type") or data.get("action") + sid = data.get("session_id", f"sess_{int(time.time()*1000)}") + + if msg_type == "start": + current_session_id = sid + full_final_tokens.clear() + current_non_final = "" + stop_event.clear() + + active_soniox_ws = await soniox_pool.get_session() + if not active_soniox_ws or not is_ws_open(active_soniox_ws): + await ws.send_str(json.dumps({"type": "error", "message": "Upstream Soniox unavailable"})) + continue + + if reader_task and not reader_task.done(): + reader_task.cancel() + reader_task = asyncio.create_task(soniox_reader(active_soniox_ws, current_session_id)) + logger.info("🎙️ Started Live Speech Session: %s", current_session_id) + + elif msg_type == "stop": + if active_soniox_ws and is_ws_open(active_soniox_ws): + await active_soniox_ws.send(json.dumps({"type": "finalize"})) + try: + await asyncio.wait_for(stop_event.wait(), timeout=0.55) + except asyncio.TimeoutError: + pass + + raw_final = "".join(full_final_tokens) + current_non_final + clean_final = sanitize_and_flatten_text(raw_final) + logger.info("⚡ Session %s final text: '%s'", sid, clean_final) + + # Return final voice transcription to Android + if is_ws_open(ws): + await ws.send_str(json.dumps({ + "type": "final", + "session_id": sid, + "text": clean_final, + "cursor_pos": data.get("cursor_pos", -1) + })) + + if reader_task and not reader_task.done(): + reader_task.cancel() + if active_soniox_ws: + try: + await active_soniox_ws.close() + except Exception: + pass + active_soniox_ws = None + + asyncio.create_task(soniox_pool.refill()) + + elif msg_type == "phone_input_edit" or msg_type == "update_mac_input": + # User manually edited text on phone or pressed "Insert in Mac" + edit_text = data.get("text", "") + cursor = data.get("cursor_pos") or data.get("cursor") + is_full_replace = data.get("is_full_replace", True) + logger.info("📱 Received phone edit to push to Mac: '%s' (cursor: %s)", edit_text[:30], cursor) + + mac_ok = await broadcast_to_macs({ + "action": "update_input", + "text": edit_text, + "cursor": cursor, + "is_full_replace": is_full_replace + }) + + # Update our cached latest state + latest_mac_input_state["text"] = edit_text + latest_mac_input_state["cursor"] = cursor if cursor is not None else len(edit_text) + latest_mac_input_state["timestamp"] = time.time() + + if is_ws_open(ws): + await ws.send_str(json.dumps({ + "type": "edit_ack", + "session_id": sid, + "mac_delivered": mac_ok + })) + + elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.ERROR): + break + + finally: + connected_phone_websockets.discard(ws) + if reader_task and not reader_task.done(): + reader_task.cancel() + if active_soniox_ws: + try: + await active_soniox_ws.close() + except Exception: + pass + asyncio.create_task(soniox_pool.refill()) + logger.info("📱 Android Client disconnected: %s", client_ip) + + return ws + +async def handle_mac_ws(request): + """Persistent WebSocket for Mac Bridge (receives input state & pushes edits).""" + global latest_mac_input_state + ws = web.WebSocketResponse(heartbeat=10.0) + await ws.prepare(request) + client_ip = request.remote + logger.info("🖥️ Mac client connected to persistent WebSocket: %s", client_ip) + connected_mac_websockets.add(ws) + + try: + await ws.send_str(json.dumps({"type": "welcome", "message": "Connected to Soniox Duplex Gateway"})) + async for msg in ws: + if msg.type == aiohttp.WSMsgType.TEXT: + try: + data = json.loads(msg.data) + msg_type = data.get("type") + + if msg_type == "mac_input_state": + # Mac reports focused input box text and cursor position + latest_mac_input_state = data + # Broadcast immediately to phone! + await broadcast_to_phones(data) + + elif msg_type == "ping": + await ws.send_str(json.dumps({"type": "pong"})) + except Exception as e: + logger.warning("Mac message error: %s", e) + elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.ERROR): + break + finally: + connected_mac_websockets.discard(ws) + if is_ws_open(ws): + await ws.close() + logger.info("🖥️ Mac client disconnected: %s", client_ip) + + return ws + +async def handle_health(request): + return web.json_response({ + "status": "ok", + "service": "Soniox Bi-Directional Duplex Gateway v4.0", + "connected_macs": len(connected_mac_websockets), + "connected_phones": len(connected_phone_websockets), + "latest_mac_input_app": latest_mac_input_state.get("app", ""), + "latest_mac_input_text_len": len(latest_mac_input_state.get("text", "")) + }) + +async def handle_paste(request): + try: + data = await request.json() + text = data.get("text", "") + session_id = data.get("session_id", "") + cursor = data.get("cursor_pos") or data.get("cursor") + success = await broadcast_to_macs({ + "action": "update_input", + "text": text, + "cursor": cursor, + "is_full_replace": True + }) + return web.json_response({"status": "pasted" if success else "failed", "mac_delivered": success}) + except Exception as e: + return web.json_response({"error": str(e)}, status=400) + +async def start_background_tasks(app): + asyncio.create_task(soniox_pool.refill()) + +def create_app(): + app = web.Application(client_max_size=25 * 1024 * 1024) + app.on_startup.append(start_background_tasks) + app.router.add_get("/health", handle_health) + app.router.add_get("/status", handle_health) + app.router.add_post("/paste", handle_paste) + app.router.add_get("/ws/stream", handle_phone_stream_ws) + app.router.add_get("/ws/mac", handle_mac_ws) + return app + +if __name__ == "__main__": + app = create_app() + web.run_app(app, host="0.0.0.0", port=8999) diff --git a/mac/src/AppDelegate.swift b/mac/src/AppDelegate.swift index 2cfa54d..ab9bd98 100644 --- a/mac/src/AppDelegate.swift +++ b/mac/src/AppDelegate.swift @@ -60,12 +60,13 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { SonioxSessionPool.shared.prewarmNextSession() startRemotePasteServer() - // Connect to Linux Persistent Gateway (116.16.16.19:8999/ws/mac) - RelayClient.shared.onPasteReceived = { [weak self] text in + // Connect to Linux Persistent Gateway (via local tunnel 18999 -> 8999) + RelayClient.shared.onRemoteUpdateReceived = { [weak self] text, cursor, isFullReplace in guard let self = self else { return } - print("AppDelegate: 📥 Clean Remote Dictation Received: '\(text)'") + print("AppDelegate: 📥 Clean Remote Input Update: '\(text.prefix(30))...' (replace: \(isFullReplace))") self.playSystemSound(name: "Tink") - self.pasteTextToFrontmostApp(text: text) + HUDOverlayController.shared.show(state: .success(text: text)) + FocusedInputSync.shared.applyRemoteUpdate(text: text, cursor: cursor, isFullReplace: isFullReplace) } RelayClient.shared.start() @@ -115,7 +116,7 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { DispatchQueue.main.async { self.playSystemSound(name: "Tink") - self.pasteTextToFrontmostApp(text: text) + FocusedInputSync.shared.applyRemoteUpdate(text: text, cursor: nil, isFullReplace: true) } } } @@ -171,7 +172,7 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { switch result { case .success(let text): self.playSystemSound(name: "Hero") - self.pasteTextToFrontmostApp(text: text) + FocusedInputSync.shared.applyRemoteUpdate(text: text, cursor: nil, isFullReplace: false) case .failure(let error): print("Soniox error:", error) HUDOverlayController.shared.show(state: .error(message: error.localizedDescription)) @@ -206,39 +207,6 @@ public final class AppDelegate: NSObject, NSApplicationDelegate { session.finalizeStream() } - /// Inserts text into whatever text field/app is currently focused without adding any newlines or enters - public func pasteTextToFrontmostApp(text: String) { - // 1. Strict single-line flattening (replaces \r\n, \n, \r, \t with single spaces) - var cleaned = text.components(separatedBy: .newlines).joined(separator: " ") - cleaned = cleaned.replacingOccurrences(of: "\t", with: " ") - cleaned = cleaned.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) - cleaned = cleaned.trimmingCharacters(in: .whitespacesAndNewlines) - - guard !cleaned.isEmpty else { return } - - HUDOverlayController.shared.show(state: .success(text: cleaned)) - - // 2. Put clean single line on system clipboard - let pasteboard = NSPasteboard.general - pasteboard.clearContents() - pasteboard.setString(cleaned, forType: .string) - - // 3. Synthesize pure Cmd+V event (KeyCode 9 = 'v') without any Enter/Return - DispatchQueue.main.asyncAfter(deadline: .now() + 0.035) { - let src = CGEventSource(stateID: .hidSystemState) - let vKeyCode: CGKeyCode = 9 // ANSI 'v' - - let keyDown = CGEvent(keyboardEventSource: src, virtualKey: vKeyCode, keyDown: true) - keyDown?.flags = .maskCommand - - let keyUp = CGEvent(keyboardEventSource: src, virtualKey: vKeyCode, keyDown: false) - keyUp?.flags = [] - - keyDown?.post(tap: .cghidEventTap) - keyUp?.post(tap: .cghidEventTap) - } - } - private func playSystemSound(name: String) { guard UserDefaults.standard.bool(forKey: "SonioxPlaySounds") else { return } NSSound(named: name)?.play() diff --git a/mac/src/FocusedInputSync.swift b/mac/src/FocusedInputSync.swift new file mode 100644 index 0000000..265c93c --- /dev/null +++ b/mac/src/FocusedInputSync.swift @@ -0,0 +1,137 @@ +import Cocoa +import ApplicationServices + +public struct MacInputState: Codable { + public let app: String + public let text: String + public let cursor: Int + public let selection: Int + public let timestamp: Double + + public init(app: String, text: String, cursor: Int, selection: Int) { + self.app = app + self.text = text + self.cursor = cursor + self.selection = selection + self.timestamp = Date().timeIntervalSince1970 + } +} + +public final class FocusedInputSync { + public static let shared = FocusedInputSync() + + private var lastState: MacInputState? + private var isUpdatingLocally = false + + public var onInputStateChanged: ((MacInputState) -> Void)? + + private init() {} + + /// Reads current focused element state (app name, text, cursor position, selection) + public func getCurrentState() -> MacInputState? { + guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil } + let appName = frontApp.localizedName ?? "App" + let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) + + var focusedElemObj: CFTypeRef? + let err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) + guard err == .success, let elem = focusedElemObj else { + return MacInputState(app: appName, text: "", cursor: 0, selection: 0) + } + + let axElem = elem as! AXUIElement + var valObj: CFTypeRef? + AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) + let text = (valObj as? String) ?? "" + + var cursor = text.count + var selLen = 0 + var selectedRangeObj: CFTypeRef? + if AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &selectedRangeObj) == .success, + let axVal = selectedRangeObj { + var range = CFRange() + if AXValueGetValue(axVal as! AXValue, .cfRange, &range) { + cursor = range.location + selLen = range.length + } + } + + return MacInputState(app: appName, text: text, cursor: cursor, selection: selLen) + } + + /// Applies updated full text or inserts at cursor position directly into active Mac input + @discardableResult + public func applyRemoteUpdate(text: String, cursor: Int? = nil, isFullReplace: Bool = true) -> Bool { + isUpdatingLocally = true + defer { + DispatchQueue.main.asyncAfter(deadline: .now() + 0.2) { + self.isUpdatingLocally = false + } + } + + // 1. Flatten all newlines and carriage returns + var cleanText = text.components(separatedBy: .newlines).joined(separator: " ") + cleanText = cleanText.replacingOccurrences(of: "\t", with: " ") + cleanText = cleanText.replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) + cleanText = cleanText.trimmingCharacters(in: .whitespacesAndNewlines) + + guard let frontApp = NSWorkspace.shared.frontmostApplication else { return false } + let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) + var focusedElemObj: CFTypeRef? + + if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) == .success, + let elem = focusedElemObj { + let axElem = elem as! AXUIElement + + if isFullReplace { + // Try setting AXValue directly + let setErr = AXUIElementSetAttributeValue(axElem, kAXValueAttribute as CFString, cleanText as CFTypeRef) + if setErr == .success { + if let cursor = cursor { + var range = CFRange(location: min(cursor, cleanText.count), length: 0) + if let axRange = AXValueCreate(.cfRange, &range) { + AXUIElementSetAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, axRange) + } + } + print("FocusedInputSync: ✅ Updated AXValue directly for \(frontApp.localizedName ?? "")") + return true + } + } else { + // Try setting selected text + let setSelErr = AXUIElementSetAttributeValue(axElem, kAXSelectedTextAttribute as CFString, cleanText as CFTypeRef) + if setSelErr == .success { + print("FocusedInputSync: ✅ Inserted text via AXSelectedText for \(frontApp.localizedName ?? "")") + return true + } + } + } + + // Fallback: Clipboard Cmd+A + Cmd+V or pure Cmd+V + let pasteboard = NSPasteboard.general + pasteboard.clearContents() + pasteboard.setString(cleanText, forType: .string) + + let src = CGEventSource(stateID: .hidSystemState) + + if isFullReplace { + // Select all: Cmd + A + let aDown = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: true) + aDown?.flags = .maskCommand + let aUp = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: false) + aDown?.post(tap: .cghidEventTap) + aUp?.post(tap: .cghidEventTap) + + usleep(25000) + } + + // Paste: Cmd + V + let vDown = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: true) + vDown?.flags = .maskCommand + let vUp = CGEvent(keyboardEventSource: src, virtualKey: 9, keyDown: false) + vDown?.post(tap: .cghidEventTap) + vUp?.post(tap: .cghidEventTap) + + print("FocusedInputSync: ✅ Injected clipboard keystroke to \(frontApp.localizedName ?? "")") + return true + } +} diff --git a/mac/src/RelayClient.swift b/mac/src/RelayClient.swift index 2c749be..f695521 100644 --- a/mac/src/RelayClient.swift +++ b/mac/src/RelayClient.swift @@ -4,16 +4,24 @@ import Cocoa public final class RelayClient: NSObject, URLSessionWebSocketDelegate { public static let shared = RelayClient() - private let gatewayUrl = URL(string: "ws://116.16.16.19:8999/ws/mac")! + // Connects through SSH local port forward 18999 -> Linux Server 8999 + private let primaryUrl = URL(string: "ws://127.0.0.1:18999/ws/mac")! + private let fallbackUrl = URL(string: "ws://116.16.16.19:8999/ws/mac")! + private var webSocketTask: URLSessionWebSocketTask? private var urlSession: URLSession! private var isRunning = false - private var isConnected = false + public private(set) var isConnected = false private var reconnectTimer: Timer? private var pingTimer: Timer? + private var monitorTimer: Timer? + + private var lastReportedText: String? + private var lastReportedCursor: Int = -1 + private var lastReportedApp: String? - public var onPasteReceived: ((String) -> Void)? + public var onRemoteUpdateReceived: ((String, Int?, Bool) -> Void)? override private init() { super.init() @@ -27,8 +35,9 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate { public func start() { guard !isRunning else { return } isRunning = true - print("RelayClient: Starting persistent gateway connection to \(gatewayUrl)...") + print("RelayClient: Starting persistent gateway connection...") connect() + startInputMonitoring() } public func stop() { @@ -37,6 +46,8 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate { reconnectTimer = nil pingTimer?.invalidate() pingTimer = nil + monitorTimer?.invalidate() + monitorTimer = nil webSocketTask?.cancel(with: .goingAway, reason: nil) webSocketTask = nil isConnected = false @@ -46,8 +57,8 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate { guard isRunning else { return } webSocketTask?.cancel() - var request = URLRequest(url: gatewayUrl) - request.timeoutInterval = 10 + var request = URLRequest(url: primaryUrl) + request.timeoutInterval = 6 request.setValue("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) SonioxVoice/1.0", forHTTPHeaderField: "User-Agent") webSocketTask = urlSession.webSocketTask(with: request) @@ -74,7 +85,6 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate { @unknown default: break } - // Continue listening self.listenForMessages() case .failure(let error): @@ -91,21 +101,70 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate { return } - let action = json["action"] as? String - if action == "paste", let pasteText = json["text"] as? String { - let trimmed = pasteText.trimmingCharacters(in: .whitespacesAndNewlines) - if !trimmed.isEmpty { - print("RelayClient: ⚡ Received remote paste text from Gateway: '\(trimmed.prefix(30))...'") - DispatchQueue.main.async { - self.onPasteReceived?(trimmed) + let action = json["action"] as? String ?? json["type"] as? String + let updateText = json["text"] as? String ?? json["insert_text"] as? String ?? "" + let cursor = json["cursor"] as? Int ?? json["cursor_pos"] as? Int + + if action == "paste" || action == "set_text" || action == "update_input" { + let isFullReplace = (action == "set_text" || action == "update_input") + print("RelayClient: ⚡ Received remote input update: '\(updateText.prefix(30))...' (replace: \(isFullReplace))") + DispatchQueue.main.async { + self.onRemoteUpdateReceived?(updateText, cursor, isFullReplace) + } + } else if action == "insert_at_cursor" { + print("RelayClient: ⚡ Received insert at cursor: '\(updateText.prefix(30))...'") + DispatchQueue.main.async { + self.onRemoteUpdateReceived?(updateText, cursor, false) + } + } + } + + // Sends Mac's current focused element state to Android via server + public func sendInputState(app: String, text: String, cursor: Int, selection: Int) { + guard isConnected, let task = webSocketTask else { return } + + // Avoid sending identical updates + if text == lastReportedText && cursor == lastReportedCursor && app == lastReportedApp { + return + } + + lastReportedText = text + lastReportedCursor = cursor + lastReportedApp = app + + let payload: [String: Any] = [ + "type": "mac_input_state", + "app": app, + "text": text, + "cursor": cursor, + "selection": selection, + "timestamp": Date().timeIntervalSince1970 + ] + + if let data = try? JSONSerialization.data(withJSONObject: payload), + let jsonStr = String(data: data, encoding: .utf8) { + task.send(.string(jsonStr)) { error in + if let error = error { + print("Error sending mac_input_state:", error) } } } } + private func startInputMonitoring() { + monitorTimer?.invalidate() + // Poll focused element every 150ms + monitorTimer = Timer.scheduledTimer(withTimeInterval: 0.15, repeats: true) { [weak self] _ in + guard let self = self, self.isRunning, self.isConnected else { return } + if let state = FocusedInputSync.shared.getCurrentState() { + self.sendInputState(app: state.app, text: state.text, cursor: state.cursor, selection: state.selection) + } + } + } + private func startPingTimer() { pingTimer?.invalidate() - pingTimer = Timer.scheduledTimer(withTimeInterval: 15.0, repeats: true) { [weak self] _ in + pingTimer = Timer.scheduledTimer(withTimeInterval: 10.0, repeats: true) { [weak self] _ in guard let self = self, self.isRunning else { return } self.webSocketTask?.send(.string("{\"type\":\"ping\"}")) { _ in } } @@ -117,7 +176,7 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate { pingTimer = nil if reconnectTimer == nil || !(reconnectTimer?.isValid ?? false) { - reconnectTimer = Timer.scheduledTimer(withTimeInterval: 3.0, repeats: false) { [weak self] _ in + reconnectTimer = Timer.scheduledTimer(withTimeInterval: 2.0, repeats: false) { [weak self] _ in self?.reconnectTimer = nil self?.connect() } diff --git a/server/relay_server.py b/server/relay_server.py index 4bd25f3..19fa8ab 100644 --- a/server/relay_server.py +++ b/server/relay_server.py @@ -1,10 +1,9 @@ #!/usr/bin/env python3 """ -Soniox Ultra-Low Latency Duplex Gateway (v3.0) -- Single persistent duplex WebSocket to Android. -- Pre-warmed upstream Soniox pool. -- Strict newline stripping & hallucination cleanup. -- Sub-50ms Cmd+V Mac paste dispatch. +Soniox Bi-Directional Input Synchronization Gateway (v4.0) +- Mirrors Mac focused input box <--> Android phone in real-time. +- Supports inserting voice text at exact cursor location. +- Instant <15ms WebSocket push to Mac for editing and pasting. """ import asyncio @@ -23,8 +22,18 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(me logger = logging.getLogger("SonioxRelay") connected_mac_websockets = set() +connected_phone_websockets = set() recent_pasted_sessions = OrderedDict() # session_id -> (timestamp, text) +latest_mac_input_state = { + "type": "mac_input_state", + "app": "Desktop", + "text": "", + "cursor": 0, + "selection": 0, + "timestamp": time.time() +} + MAC_DIRECT_HTTP_URL = "http://116.16.16.20:8999/paste" SONIOX_WS_URL = ( "wss://translate.compare.soniox.com/compare/api/compare-websocket" @@ -103,7 +112,6 @@ def sanitize_and_flatten_text(text: str) -> str: if not text: return "" - # Flatten all newlines, carriage returns, tabs into a single line flattened = re.sub(r"[\r\n\t]+", " ", text) flattened = re.sub(r"\s+", " ", flattened).strip() @@ -138,37 +146,18 @@ def sanitize_and_flatten_text(text: str) -> str: result = " ".join(cleaned) return re.sub(r"\s+", " ", result).strip() -async def broadcast_paste_to_macs(text: str, session_id: str = "") -> bool: - """Pushes clean single-line paste payload to Mac in <50ms with strict deduplication.""" - clean_text = sanitize_and_flatten_text(text) - if not clean_text: - return False - - now = time.time() - if session_id: - if session_id in recent_pasted_sessions: - logger.info("🚫 Suppressed duplicate paste for session_id: %s", session_id) - return True - recent_pasted_sessions[session_id] = (now, clean_text) - while recent_pasted_sessions and (now - next(iter(recent_pasted_sessions.values()))[0] > 60): - recent_pasted_sessions.popitem(last=False) - else: - for prev_sid, (prev_time, prev_txt) in list(recent_pasted_sessions.items())[-5:]: - if prev_txt == clean_text and (now - prev_time) < 2.0: - logger.info("🚫 Suppressed rapid identical text paste: '%s'", clean_text[:25]) - return True - +async def broadcast_to_macs(payload_dict: dict) -> bool: + """Pushes command payload directly to Mac via persistent WebSocket in <15ms.""" delivered = False - payload = json.dumps({"action": "paste", "text": clean_text}, ensure_ascii=False) + payload_str = json.dumps(payload_dict, ensure_ascii=False) - # 1. Primary: Direct Push via Active Persistent WebSocket (<15ms) dead_sockets = set() for ws in list(connected_mac_websockets): try: if is_ws_open(ws): - await ws.send_str(payload) + await ws.send_str(payload_str) delivered = True - logger.info("⚡ Pushed paste via Mac Persistent WS: '%s'", clean_text[:30]) + logger.info("⚡ Pushed to Mac WS: %s", payload_dict.get("action") or payload_dict.get("type")) else: dead_sockets.add(ws) except Exception: @@ -180,47 +169,63 @@ async def broadcast_paste_to_macs(text: str, session_id: str = "") -> bool: if delivered: return True - # 2. Secondary: Direct LAN HTTP (/paste) (<35ms) - try: - async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=0.35)) as session: - async with session.post(MAC_DIRECT_HTTP_URL, json={"text": clean_text}) as resp: - if resp.status == 200: - logger.info("✅ Pasted to Mac via Direct LAN HTTP: '%s'", clean_text[:30]) - return True - except Exception: - pass - - # 3. Tertiary: SSH Tunnel (Port 2222) - FIXED: Zero-Newline printf '%s' pipe + # Fallback to SSH script if WebSocket temporarily disconnected try: - escaped_text = clean_text.replace("'", "'\\''") - remote_cmd = ( - f"printf '%s' '{escaped_text}' | pbcopy && " - f"/usr/bin/osascript -e 'tell application \"System Events\" to keystroke \"v\" using command down'" - ) - proc = await asyncio.create_subprocess_exec( - "ssh", "-p", "2222", "-o", "BatchMode=yes", "-o", "ConnectTimeout=2", "alig@127.0.0.1", - remote_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - stdout, stderr = await proc.communicate() - if proc.returncode == 0: - logger.info("✅ Pasted to Mac via SSH Tunnel (Zero-Newline printf): '%s'", clean_text[:30]) - return True + text = payload_dict.get("text", "") + if text: + clean_text = sanitize_and_flatten_text(text) + escaped_text = clean_text.replace("'", "'\\''") + remote_cmd = ( + f"printf '%s' '{escaped_text}' | pbcopy && " + f"/usr/bin/osascript -e 'tell application \"System Events\" to keystroke \"v\" using command down'" + ) + proc = await asyncio.create_subprocess_exec( + "ssh", "-p", "2222", "-o", "BatchMode=yes", "-o", "ConnectTimeout=2", "alig@127.0.0.1", + remote_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout, stderr = await proc.communicate() + if proc.returncode == 0: + logger.info("✅ Pasted to Mac via SSH Tunnel fallback: '%s'", clean_text[:30]) + return True except Exception as e: - logger.warning("SSH tunnel fallback error: %s", e) + logger.warning("SSH fallback error: %s", e) return False +async def broadcast_to_phones(payload_dict: dict): + """Pushes Mac input state changes to all connected Android clients.""" + payload_str = json.dumps(payload_dict, ensure_ascii=False) + dead_phones = set() + for ws in list(connected_phone_websockets): + try: + if is_ws_open(ws): + await ws.send_str(payload_str) + else: + dead_phones.add(ws) + except Exception: + dead_phones.add(ws) + + for dead in dead_phones: + connected_phone_websockets.discard(dead) + async def handle_phone_stream_ws(request): """ - ⚡ Single Persistent Duplex WebSocket Handler for Android. - Supports multiple sequential sessions without tearing down the connection. + ⚡ Persistent Duplex Channel for Android Client: + - Receives live Mac input state on connect and continuously. + - Handles live audio streaming and returns live STT tokens. + - Receives user manual text edits and pushes them instantly to Mac. """ ws = web.WebSocketResponse(heartbeat=15.0) await ws.prepare(request) client_ip = request.remote - logger.info("📱 Android Client connected to Persistent Stream: %s", client_ip) + logger.info("📱 Android Client connected to Duplex Stream: %s", client_ip) + connected_phone_websockets.add(ws) + + # Immediately send the latest Mac input state to phone upon connection! + if latest_mac_input_state: + await ws.send_str(json.dumps(latest_mac_input_state, ensure_ascii=False)) active_soniox_ws = None reader_task = None @@ -300,7 +305,6 @@ async def handle_phone_stream_ws(request): current_non_final = "" stop_event.clear() - # Acquire pre-warmed Soniox session active_soniox_ws = await soniox_pool.get_session() if not active_soniox_ws or not is_ws_open(active_soniox_ws): await ws.send_str(json.dumps({"type": "error", "message": "Upstream Soniox unavailable"})) @@ -309,7 +313,7 @@ async def handle_phone_stream_ws(request): if reader_task and not reader_task.done(): reader_task.cancel() reader_task = asyncio.create_task(soniox_reader(active_soniox_ws, current_session_id)) - logger.info("🎙️ Started Live Session: %s", current_session_id) + logger.info("🎙️ Started Live Speech Session: %s", current_session_id) elif msg_type == "stop": if active_soniox_ws and is_ws_open(active_soniox_ws): @@ -323,20 +327,15 @@ async def handle_phone_stream_ws(request): clean_final = sanitize_and_flatten_text(raw_final) logger.info("⚡ Session %s final text: '%s'", sid, clean_final) - # Instant <50ms broadcast to Mac - mac_ok = False - if clean_final: - mac_ok = await broadcast_paste_to_macs(clean_final, session_id=sid) - + # Return final voice transcription to Android if is_ws_open(ws): await ws.send_str(json.dumps({ "type": "final", "session_id": sid, "text": clean_final, - "mac_delivered": mac_ok + "cursor_pos": data.get("cursor_pos", -1) })) - # Cleanup Soniox session for this turn if reader_task and not reader_task.done(): reader_task.cancel() if active_soniox_ws: @@ -348,10 +347,37 @@ async def handle_phone_stream_ws(request): asyncio.create_task(soniox_pool.refill()) + elif msg_type == "phone_input_edit" or msg_type == "update_mac_input": + # User manually edited text on phone or pressed "Insert in Mac" + edit_text = data.get("text", "") + cursor = data.get("cursor_pos") or data.get("cursor") + is_full_replace = data.get("is_full_replace", True) + logger.info("📱 Received phone edit to push to Mac: '%s' (cursor: %s)", edit_text[:30], cursor) + + mac_ok = await broadcast_to_macs({ + "action": "update_input", + "text": edit_text, + "cursor": cursor, + "is_full_replace": is_full_replace + }) + + # Update our cached latest state + latest_mac_input_state["text"] = edit_text + latest_mac_input_state["cursor"] = cursor if cursor is not None else len(edit_text) + latest_mac_input_state["timestamp"] = time.time() + + if is_ws_open(ws): + await ws.send_str(json.dumps({ + "type": "edit_ack", + "session_id": sid, + "mac_delivered": mac_ok + })) + elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.ERROR): break finally: + connected_phone_websockets.discard(ws) if reader_task and not reader_task.done(): reader_task.cancel() if active_soniox_ws: @@ -365,7 +391,8 @@ async def handle_phone_stream_ws(request): return ws async def handle_mac_ws(request): - """Persistent WebSocket for Mac Bridge.""" + """Persistent WebSocket for Mac Bridge (receives input state & pushes edits).""" + global latest_mac_input_state ws = web.WebSocketResponse(heartbeat=10.0) await ws.prepare(request) client_ip = request.remote @@ -373,15 +400,23 @@ async def handle_mac_ws(request): connected_mac_websockets.add(ws) try: - await ws.send_str(json.dumps({"type": "welcome", "message": "Connected to Soniox Ultra Gateway"})) + await ws.send_str(json.dumps({"type": "welcome", "message": "Connected to Soniox Duplex Gateway"})) async for msg in ws: if msg.type == aiohttp.WSMsgType.TEXT: try: data = json.loads(msg.data) - if data.get("type") == "ping": + msg_type = data.get("type") + + if msg_type == "mac_input_state": + # Mac reports focused input box text and cursor position + latest_mac_input_state = data + # Broadcast immediately to phone! + await broadcast_to_phones(data) + + elif msg_type == "ping": await ws.send_str(json.dumps({"type": "pong"})) - except Exception: - pass + except Exception as e: + logger.warning("Mac message error: %s", e) elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.ERROR): break finally: @@ -395,9 +430,11 @@ async def handle_mac_ws(request): async def handle_health(request): return web.json_response({ "status": "ok", - "service": "Soniox Pre-Warmed Duplex Gateway v3.0", + "service": "Soniox Bi-Directional Duplex Gateway v4.0", "connected_macs": len(connected_mac_websockets), - "recent_sessions": len(recent_pasted_sessions) + "connected_phones": len(connected_phone_websockets), + "latest_mac_input_app": latest_mac_input_state.get("app", ""), + "latest_mac_input_text_len": len(latest_mac_input_state.get("text", "")) }) async def handle_paste(request): @@ -405,7 +442,13 @@ async def handle_paste(request): data = await request.json() text = data.get("text", "") session_id = data.get("session_id", "") - success = await broadcast_paste_to_macs(text, session_id=session_id) + cursor = data.get("cursor_pos") or data.get("cursor") + success = await broadcast_to_macs({ + "action": "update_input", + "text": text, + "cursor": cursor, + "is_full_replace": True + }) return web.json_response({"status": "pasted" if success else "failed", "mac_delivered": success}) except Exception as e: return web.json_response({"error": str(e)}, status=400)