Browse Source
feat: Google Docs & Figma style real-time bi-directional input synchronization (v5.0)
main
feat: Google Docs & Figma style real-time bi-directional input synchronization (v5.0)
main
17 changed files with 1053 additions and 513 deletions
-
112android/app/src/main/java/com/soniox/remotemic/MainActivity.kt
-
77android/app/src/main/java/com/soniox/remotemic/StreamDictationClient.kt
-
6android/app/src/main/java/com/soniox/remotemic/VoiceRecognitionActivity.kt
-
237android/server/relay_server.py
-
206mac/src/FocusedInputSync.swift
-
92mac/src/RelayClient.swift
-
42mac/test_apps_ax.swift
-
107mac/test_focus_detector.swift
-
110mac/test_full_ax.swift
-
45mac/test_inspect.swift
-
45mac/test_inspect2.swift
-
76mac/test_inspect3.swift
-
81mac/test_live_ax.swift
-
21mac/test_timer.swift
-
39mac/test_tree.swift
-
33mac/test_win_ax.swift
-
237server/relay_server.py
@ -1,9 +1,9 @@ |
|||
#!/usr/bin/env python3 |
|||
""" |
|||
Soniox Bi-Directional Input Synchronization Gateway (v4.0) |
|||
- Mirrors Mac focused input box <--> Android phone in real-time. |
|||
- Supports inserting voice text at exact cursor location. |
|||
- Instant <15ms WebSocket push to Mac for editing and pasting. |
|||
Soniox Collaborative Bi-Directional Input Synchronization Gateway (v5.0) |
|||
- Real-time Google Docs / Figma style collaborative mirroring between Mac and Android. |
|||
- Monotonic revision counters, echo-loop suppression, and sub-15ms WebSocket routing. |
|||
- Cursor-aware voice insertion and instant text editing. |
|||
""" |
|||
|
|||
import asyncio |
|||
@ -19,22 +19,24 @@ import websockets |
|||
from websockets.protocol import State |
|||
|
|||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") |
|||
logger = logging.getLogger("SonioxRelay") |
|||
logger = logging.getLogger("SyncGateway") |
|||
|
|||
connected_mac_websockets = set() |
|||
connected_phone_websockets = set() |
|||
recent_pasted_sessions = OrderedDict() # session_id -> (timestamp, text) |
|||
|
|||
latest_mac_input_state = { |
|||
"type": "mac_input_state", |
|||
# Global Room State Snapshot (Single Source of Truth) |
|||
current_room_state = { |
|||
"type": "sync_state", |
|||
"source": "server_init", |
|||
"app": "Desktop", |
|||
"text": "", |
|||
"cursor": 0, |
|||
"selection": 0, |
|||
"revision": 0, |
|||
"timestamp": time.time() |
|||
} |
|||
room_lock = asyncio.Lock() |
|||
|
|||
MAC_DIRECT_HTTP_URL = "http://116.16.16.20:8999/paste" |
|||
SONIOX_WS_URL = ( |
|||
"wss://translate.compare.soniox.com/compare/api/compare-websocket" |
|||
"?language_hints=fa&language_hints=en&language_hints=ar" |
|||
@ -104,101 +106,29 @@ class SonioxPool: |
|||
soniox_pool = SonioxPool() |
|||
|
|||
def sanitize_and_flatten_text(text: str) -> str: |
|||
""" |
|||
1. Removes all line breaks (\\r, \\n) and collapses whitespace into single spaces. |
|||
2. Strips English hallucination stop words during Persian speech. |
|||
3. Guarantees zero trailing/leading enters or spaces. |
|||
""" |
|||
if not text: |
|||
return "" |
|||
|
|||
flattened = re.sub(r"[\r\n\t]+", " ", text) |
|||
flattened = re.sub(r"\s+", " ", flattened).strip() |
|||
|
|||
if not flattened: |
|||
return "" |
|||
|
|||
words = flattened.split() |
|||
fa_pattern = re.compile(r"[\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF]") |
|||
en_pattern = re.compile(r"[a-zA-Z]") |
|||
|
|||
fa_count = sum(1 for w in words if fa_pattern.search(w)) |
|||
en_count = sum(1 for w in words if en_pattern.search(w)) |
|||
total = fa_count + en_count |
|||
|
|||
if total == 0: |
|||
return flattened |
|||
|
|||
fa_ratio = fa_count / total |
|||
stop_words = {"sex", "from", "no", "oh", "you", "all", "know", "that", "god", "loves", "men", "for", "day", "night", "mankind", "wanted", "moments"} |
|||
|
|||
cleaned = [] |
|||
if fa_ratio >= 0.25: |
|||
for w in words: |
|||
if en_pattern.search(w) and not fa_pattern.search(w): |
|||
clean_w = re.sub(r"[.,!?:;،؛؟\"'()\[\]{}«»–—-]", "", w.lower()) |
|||
if clean_w in stop_words or fa_ratio >= 0.70: |
|||
continue |
|||
cleaned.append(w) |
|||
else: |
|||
cleaned = words |
|||
|
|||
result = " ".join(cleaned) |
|||
return re.sub(r"\s+", " ", result).strip() |
|||
|
|||
async def broadcast_to_macs(payload_dict: dict) -> bool: |
|||
"""Pushes command payload directly to Mac via persistent WebSocket in <15ms.""" |
|||
delivered = False |
|||
payload_str = json.dumps(payload_dict, ensure_ascii=False) |
|||
|
|||
dead_sockets = set() |
|||
for ws in list(connected_mac_websockets): |
|||
try: |
|||
if is_ws_open(ws): |
|||
await ws.send_str(payload_str) |
|||
delivered = True |
|||
logger.info("⚡ Pushed to Mac WS: %s", payload_dict.get("action") or payload_dict.get("type")) |
|||
else: |
|||
dead_sockets.add(ws) |
|||
except Exception: |
|||
dead_sockets.add(ws) |
|||
|
|||
for dead in dead_sockets: |
|||
connected_mac_websockets.discard(dead) |
|||
|
|||
if delivered: |
|||
return True |
|||
|
|||
# Fallback to SSH script if WebSocket temporarily disconnected |
|||
try: |
|||
text = payload_dict.get("text", "") |
|||
if text: |
|||
clean_text = sanitize_and_flatten_text(text) |
|||
escaped_text = clean_text.replace("'", "'\\''") |
|||
remote_cmd = ( |
|||
f"printf '%s' '{escaped_text}' | pbcopy && " |
|||
f"/usr/bin/osascript -e 'tell application \"System Events\" to keystroke \"v\" using command down'" |
|||
) |
|||
proc = await asyncio.create_subprocess_exec( |
|||
"ssh", "-p", "2222", "-o", "BatchMode=yes", "-o", "ConnectTimeout=2", "[email protected]", |
|||
remote_cmd, |
|||
stdout=subprocess.PIPE, |
|||
stderr=subprocess.PIPE |
|||
) |
|||
stdout, stderr = await proc.communicate() |
|||
if proc.returncode == 0: |
|||
logger.info("✅ Pasted to Mac via SSH Tunnel fallback: '%s'", clean_text[:30]) |
|||
return True |
|||
except Exception as e: |
|||
logger.warning("SSH fallback error: %s", e) |
|||
|
|||
return False |
|||
|
|||
async def broadcast_to_phones(payload_dict: dict): |
|||
"""Pushes Mac input state changes to all connected Android clients.""" |
|||
payload_str = json.dumps(payload_dict, ensure_ascii=False) |
|||
return flattened |
|||
|
|||
async def broadcast_state(payload_dict: dict, exclude_ws=None): |
|||
"""Broadcasts state snapshot to all connected clients (Mac and Phone) except sender.""" |
|||
global current_room_state |
|||
async with room_lock: |
|||
current_room_state["revision"] += 1 |
|||
payload_dict["revision"] = current_room_state["revision"] |
|||
payload_dict["timestamp"] = time.time() |
|||
|
|||
# Update our cached authoritative state |
|||
current_room_state.update(payload_dict) |
|||
payload_str = json.dumps(payload_dict, ensure_ascii=False) |
|||
|
|||
# 1. Send to Phone clients |
|||
dead_phones = set() |
|||
for ws in list(connected_phone_websockets): |
|||
if ws is exclude_ws: |
|||
continue |
|||
try: |
|||
if is_ws_open(ws): |
|||
await ws.send_str(payload_str) |
|||
@ -206,26 +136,35 @@ async def broadcast_to_phones(payload_dict: dict): |
|||
dead_phones.add(ws) |
|||
except Exception: |
|||
dead_phones.add(ws) |
|||
|
|||
for dead in dead_phones: |
|||
connected_phone_websockets.discard(dead) |
|||
|
|||
# 2. Send to Mac clients |
|||
dead_macs = set() |
|||
for ws in list(connected_mac_websockets): |
|||
if ws is exclude_ws: |
|||
continue |
|||
try: |
|||
if is_ws_open(ws): |
|||
await ws.send_str(payload_str) |
|||
else: |
|||
dead_macs.add(ws) |
|||
except Exception: |
|||
dead_macs.add(ws) |
|||
for dead in dead_macs: |
|||
connected_mac_websockets.discard(dead) |
|||
|
|||
async def handle_phone_stream_ws(request): |
|||
""" |
|||
⚡ Persistent Duplex Channel for Android Client: |
|||
- Receives live Mac input state on connect and continuously. |
|||
- Handles live audio streaming and returns live STT tokens. |
|||
- Receives user manual text edits and pushes them instantly to Mac. |
|||
""" |
|||
ws = web.WebSocketResponse(heartbeat=15.0) |
|||
"""Duplex WebSocket for Android client (Real-time Live Sync + Audio Dictation).""" |
|||
ws = web.WebSocketResponse(heartbeat=12.0) |
|||
await ws.prepare(request) |
|||
client_ip = request.remote |
|||
logger.info("📱 Android Client connected to Duplex Stream: %s", client_ip) |
|||
logger.info("📱 Android Client Connected: %s", client_ip) |
|||
connected_phone_websockets.add(ws) |
|||
|
|||
# Immediately send the latest Mac input state to phone upon connection! |
|||
if latest_mac_input_state: |
|||
await ws.send_str(json.dumps(latest_mac_input_state, ensure_ascii=False)) |
|||
# Immediately hydrate phone with full current state! |
|||
async with room_lock: |
|||
await ws.send_str(json.dumps(current_room_state, ensure_ascii=False)) |
|||
|
|||
active_soniox_ws = None |
|||
reader_task = None |
|||
@ -286,7 +225,7 @@ async def handle_phone_stream_ws(request): |
|||
try: |
|||
async for msg in ws: |
|||
if msg.type == aiohttp.WSMsgType.BINARY: |
|||
# Live PCM audio chunk (2048 bytes / 64ms) |
|||
# Live PCM audio chunk |
|||
if active_soniox_ws and is_ws_open(active_soniox_ws): |
|||
await active_soniox_ws.send(msg.data) |
|||
|
|||
@ -299,7 +238,13 @@ async def handle_phone_stream_ws(request): |
|||
msg_type = data.get("type") or data.get("action") |
|||
sid = data.get("session_id", f"sess_{int(time.time()*1000)}") |
|||
|
|||
if msg_type == "start": |
|||
if msg_type == "sync_state" or msg_type == "phone_input_edit" or msg_type == "update_input": |
|||
# Phone edited text: broadcast to Mac immediately! |
|||
data["source"] = "android" |
|||
data["type"] = "sync_state" |
|||
await broadcast_state(data, exclude_ws=ws) |
|||
|
|||
elif msg_type == "start": |
|||
current_session_id = sid |
|||
full_final_tokens.clear() |
|||
current_non_final = "" |
|||
@ -327,7 +272,7 @@ async def handle_phone_stream_ws(request): |
|||
clean_final = sanitize_and_flatten_text(raw_final) |
|||
logger.info("⚡ Session %s final text: '%s'", sid, clean_final) |
|||
|
|||
# Return final voice transcription to Android |
|||
# Return final speech text to phone |
|||
if is_ws_open(ws): |
|||
await ws.send_str(json.dumps({ |
|||
"type": "final", |
|||
@ -347,32 +292,6 @@ async def handle_phone_stream_ws(request): |
|||
|
|||
asyncio.create_task(soniox_pool.refill()) |
|||
|
|||
elif msg_type == "phone_input_edit" or msg_type == "update_mac_input": |
|||
# User manually edited text on phone or pressed "Insert in Mac" |
|||
edit_text = data.get("text", "") |
|||
cursor = data.get("cursor_pos") or data.get("cursor") |
|||
is_full_replace = data.get("is_full_replace", True) |
|||
logger.info("📱 Received phone edit to push to Mac: '%s' (cursor: %s)", edit_text[:30], cursor) |
|||
|
|||
mac_ok = await broadcast_to_macs({ |
|||
"action": "update_input", |
|||
"text": edit_text, |
|||
"cursor": cursor, |
|||
"is_full_replace": is_full_replace |
|||
}) |
|||
|
|||
# Update our cached latest state |
|||
latest_mac_input_state["text"] = edit_text |
|||
latest_mac_input_state["cursor"] = cursor if cursor is not None else len(edit_text) |
|||
latest_mac_input_state["timestamp"] = time.time() |
|||
|
|||
if is_ws_open(ws): |
|||
await ws.send_str(json.dumps({ |
|||
"type": "edit_ack", |
|||
"session_id": sid, |
|||
"mac_delivered": mac_ok |
|||
})) |
|||
|
|||
elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.ERROR): |
|||
break |
|||
|
|||
@ -391,27 +310,26 @@ async def handle_phone_stream_ws(request): |
|||
return ws |
|||
|
|||
async def handle_mac_ws(request): |
|||
"""Persistent WebSocket for Mac Bridge (receives input state & pushes edits).""" |
|||
global latest_mac_input_state |
|||
"""Persistent WebSocket for Mac Bridge (receives live Mac typing & pushes edits).""" |
|||
ws = web.WebSocketResponse(heartbeat=10.0) |
|||
await ws.prepare(request) |
|||
client_ip = request.remote |
|||
logger.info("🖥️ Mac client connected to persistent WebSocket: %s", client_ip) |
|||
logger.info("🖥️ Mac client connected: %s", client_ip) |
|||
connected_mac_websockets.add(ws) |
|||
|
|||
try: |
|||
await ws.send_str(json.dumps({"type": "welcome", "message": "Connected to Soniox Duplex Gateway"})) |
|||
await ws.send_str(json.dumps({"type": "welcome", "message": "Connected to Soniox Sync Gateway"})) |
|||
async for msg in ws: |
|||
if msg.type == aiohttp.WSMsgType.TEXT: |
|||
try: |
|||
data = json.loads(msg.data) |
|||
msg_type = data.get("type") |
|||
|
|||
if msg_type == "mac_input_state": |
|||
# Mac reports focused input box text and cursor position |
|||
latest_mac_input_state = data |
|||
# Broadcast immediately to phone! |
|||
await broadcast_to_phones(data) |
|||
if msg_type == "sync_state" or msg_type == "mac_input_state": |
|||
# Mac reports typing / cursor change: broadcast to all phones immediately! |
|||
data["source"] = "mac" |
|||
data["type"] = "sync_state" |
|||
await broadcast_state(data, exclude_ws=ws) |
|||
|
|||
elif msg_type == "ping": |
|||
await ws.send_str(json.dumps({"type": "pong"})) |
|||
@ -421,35 +339,32 @@ async def handle_mac_ws(request): |
|||
break |
|||
finally: |
|||
connected_mac_websockets.discard(ws) |
|||
if is_ws_open(ws): |
|||
await ws.close() |
|||
logger.info("🖥️ Mac client disconnected: %s", client_ip) |
|||
|
|||
return ws |
|||
|
|||
async def handle_health(request): |
|||
async with room_lock: |
|||
state_copy = dict(current_room_state) |
|||
return web.json_response({ |
|||
"status": "ok", |
|||
"service": "Soniox Bi-Directional Duplex Gateway v4.0", |
|||
"service": "Soniox Collaborative Sync Gateway v5.0", |
|||
"connected_macs": len(connected_mac_websockets), |
|||
"connected_phones": len(connected_phone_websockets), |
|||
"latest_mac_input_app": latest_mac_input_state.get("app", ""), |
|||
"latest_mac_input_text_len": len(latest_mac_input_state.get("text", "")) |
|||
"current_app": state_copy.get("app", ""), |
|||
"current_revision": state_copy.get("revision", 0), |
|||
"current_text_len": len(state_copy.get("text", "")) |
|||
}) |
|||
|
|||
async def handle_paste(request): |
|||
try: |
|||
data = await request.json() |
|||
text = data.get("text", "") |
|||
session_id = data.get("session_id", "") |
|||
cursor = data.get("cursor_pos") or data.get("cursor") |
|||
success = await broadcast_to_macs({ |
|||
"action": "update_input", |
|||
"text": text, |
|||
"cursor": cursor, |
|||
"is_full_replace": True |
|||
}) |
|||
return web.json_response({"status": "pasted" if success else "failed", "mac_delivered": success}) |
|||
data["source"] = "http_post" |
|||
data["type"] = "sync_state" |
|||
await broadcast_state(data) |
|||
return web.json_response({"status": "synced", "revision": current_room_state["revision"]}) |
|||
except Exception as e: |
|||
return web.json_response({"error": str(e)}, status=400) |
|||
|
|||
|
|||
@ -0,0 +1,42 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func inspectApp(named targetName: String) { |
|||
guard let app = NSWorkspace.shared.runningApplications.first(where: { $0.localizedName == targetName }) else { |
|||
print("App \(targetName) not running") |
|||
return |
|||
} |
|||
|
|||
print("\n--- Inspecting \(targetName) (PID: \(app.processIdentifier)) ---") |
|||
let appElem = AXUIElementCreateApplication(app.processIdentifier) |
|||
AXUIElementSetAttributeValue(appElem, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
|||
AXUIElementSetAttributeValue(appElem, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
|||
|
|||
var focusedElemObj: CFTypeRef? |
|||
let err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) |
|||
print("kAXFocusedUIElementAttribute error:", err.rawValue) |
|||
|
|||
if err == .success, let elem = focusedElemObj { |
|||
let axElem = elem as! AXUIElement |
|||
var roleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleObj) |
|||
print("Role:", roleObj ?? "none") |
|||
|
|||
var valObj: CFTypeRef? |
|||
let valErr = AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) |
|||
print("Value err:", valErr.rawValue, "Value:", valObj ?? "none") |
|||
|
|||
var rangeObj: CFTypeRef? |
|||
let rangeErr = AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &rangeObj) |
|||
if rangeErr == .success, let axRange = rangeObj { |
|||
var range = CFRange() |
|||
if AXValueGetValue(axRange as! AXValue, .cfRange, &range) { |
|||
print("Cursor: loc=\(range.location), len=\(range.length)") |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
inspectApp(named: "Telegram") |
|||
inspectApp(named: "Obsidian") |
|||
inspectApp(named: "firefox") |
|||
@ -0,0 +1,107 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
// Comprehensive recursive search for any element with AXFocused == true |
|||
func findActiveFocusedElement(_ elem: AXUIElement) -> AXUIElement? { |
|||
var isFocusedObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &isFocusedObj) == .success, |
|||
let isFocused = isFocusedObj as? Bool, isFocused { |
|||
var roleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) |
|||
let role = (roleObj as? String) ?? "" |
|||
if role != "AXWindow" && role != "AXApplication" && role != "AXGroup" && role != "AXSplitGroup" && role != "AXScrollArea" { |
|||
return elem |
|||
} |
|||
} |
|||
|
|||
var childrenObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, |
|||
let children = childrenObj as? [AXUIElement] { |
|||
for child in children { |
|||
if let found = findActiveFocusedElement(child) { |
|||
return found |
|||
} |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func getFocusedTextInfo() -> (String, String, Int, Int)? { |
|||
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil } |
|||
let appName = frontApp.localizedName ?? "App" |
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
|
|||
// 1. Try systemWide |
|||
let sysWide = AXUIElementCreateSystemWide() |
|||
var focusedElemObj: CFTypeRef? |
|||
var targetElem: AXUIElement? |
|||
|
|||
if AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) == .success, |
|||
let obj = focusedElemObj { |
|||
targetElem = (obj as! AXUIElement) |
|||
} |
|||
|
|||
// 2. Try App focused element |
|||
if targetElem == nil { |
|||
if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) == .success, |
|||
let obj = focusedElemObj { |
|||
targetElem = (obj as! AXUIElement) |
|||
} |
|||
} |
|||
|
|||
// 3. Try Recursive search in app tree |
|||
if targetElem == nil { |
|||
targetElem = findActiveFocusedElement(appElem) |
|||
} |
|||
|
|||
guard let elem = targetElem else { return nil } |
|||
|
|||
// Extract text |
|||
var text = "" |
|||
var valObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) == .success, |
|||
let val = valObj { |
|||
if let str = val as? String { |
|||
text = str |
|||
} else if let attrStr = val as? NSAttributedString { |
|||
text = attrStr.string |
|||
} |
|||
} |
|||
|
|||
// Try parameterized string for range |
|||
if text.isEmpty { |
|||
var countObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countObj) == .success, |
|||
let count = countObj as? Int, count > 0 { |
|||
var range = CFRange(location: 0, length: count) |
|||
if let axRange = AXValueCreate(.cfRange, &range) { |
|||
var strObj: CFTypeRef? |
|||
if AXUIElementCopyParameterizedAttributeValue(elem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &strObj) == .success, |
|||
let str = strObj as? String { |
|||
text = str |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
// Cursor & Selection |
|||
var cursor = text.count |
|||
var selLen = 0 |
|||
var rangeObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, &rangeObj) == .success, |
|||
let axRange = rangeObj { |
|||
var range = CFRange() |
|||
if AXValueGetValue(axRange as! AXValue, .cfRange, &range) { |
|||
cursor = range.location |
|||
selLen = range.length |
|||
} |
|||
} |
|||
|
|||
return (appName, text, cursor, selLen) |
|||
} |
|||
|
|||
if let (app, text, cursor, selLen) = getFocusedTextInfo() { |
|||
print("Found! App: \(app), Text: '\(text)', Cursor: \(cursor), SelLen: \(selLen)") |
|||
} else { |
|||
print("No focused text element found") |
|||
} |
|||
@ -0,0 +1,110 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func extractTextAndCursor(from elem: AXUIElement) -> (String, Int, Int)? { |
|||
var roleRef: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleRef) |
|||
let role = (roleRef as? String) ?? "" |
|||
|
|||
var currentText = "" |
|||
var valueRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valueRef) == .success, |
|||
let val = valueRef { |
|||
if let str = val as? String { |
|||
currentText = str |
|||
} else if let attrStr = val as? NSAttributedString { |
|||
currentText = attrStr.string |
|||
} |
|||
} |
|||
|
|||
if currentText.isEmpty { |
|||
var countRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countRef) == .success, |
|||
let count = countRef as? Int, count > 0 { |
|||
var range = CFRange(location: 0, length: count) |
|||
if let axRange = AXValueCreate(.cfRange, &range) { |
|||
var stringRef: CFTypeRef? |
|||
if AXUIElementCopyParameterizedAttributeValue(elem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &stringRef) == .success, |
|||
let str = stringRef as? String { |
|||
currentText = str |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
var cursor = currentText.count |
|||
var selLen = 0 |
|||
var rangeRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXSelectedTextRangeAttribute as CFString, &rangeRef) == .success, |
|||
let val = rangeRef { |
|||
var cfRange = CFRange() |
|||
if AXValueGetValue(val as! AXValue, .cfRange, &cfRange) { |
|||
cursor = cfRange.location |
|||
selLen = cfRange.length |
|||
} |
|||
} |
|||
|
|||
if !currentText.isEmpty || role == "AXTextField" || role == "AXTextArea" || role == "AXSearchField" || rangeRef != nil { |
|||
return (currentText, cursor, selLen) |
|||
} |
|||
|
|||
// Check focused child |
|||
var focusedChildRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXFocusedUIElementAttribute as CFString, &focusedChildRef) == .success, |
|||
let child = focusedChildRef { |
|||
if let res = extractTextAndCursor(from: child as! AXUIElement) { |
|||
return res |
|||
} |
|||
} |
|||
|
|||
// Check children |
|||
var childrenRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenRef) == .success, |
|||
let children = childrenRef as? [AXUIElement] { |
|||
for child in children { |
|||
var isFocusedRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(child, kAXFocusedAttribute as CFString, &isFocusedRef) == .success, |
|||
let isFoc = isFocusedRef as? Bool, isFoc { |
|||
if let res = extractTextAndCursor(from: child) { |
|||
return res |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func testFullAX() { |
|||
let sysWide = AXUIElementCreateSystemWide() |
|||
AXUIElementSetAttributeValue(sysWide, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
|||
AXUIElementSetAttributeValue(sysWide, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
|||
|
|||
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return } |
|||
print("Front App:", frontApp.localizedName ?? "") |
|||
|
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
AXUIElementSetAttributeValue(appElem, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
|||
AXUIElementSetAttributeValue(appElem, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
|||
|
|||
var focusedElem: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &focusedElem) == .success, |
|||
let elem = focusedElem { |
|||
if let (text, cursor, sel) = extractTextAndCursor(from: elem as! AXUIElement) { |
|||
print("Extracted from sysWide -> Text: '\(text)', Cursor: \(cursor), Sel: \(sel)") |
|||
return |
|||
} |
|||
} |
|||
|
|||
if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElem) == .success, |
|||
let elem = focusedElem { |
|||
if let (text, cursor, sel) = extractTextAndCursor(from: elem as! AXUIElement) { |
|||
print("Extracted from appElem -> Text: '\(text)', Cursor: \(cursor), Sel: \(sel)") |
|||
return |
|||
} |
|||
} |
|||
|
|||
print("Could not extract text") |
|||
} |
|||
|
|||
testFullAX() |
|||
@ -0,0 +1,45 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func inspect() { |
|||
guard let frontApp = NSWorkspace.shared.frontmostApplication else { |
|||
print("No front app") |
|||
return |
|||
} |
|||
print("Front App:", frontApp.localizedName ?? "", "PID:", frontApp.processIdentifier) |
|||
|
|||
let trusted = AXIsProcessTrusted() |
|||
print("AXIsProcessTrusted:", trusted) |
|||
|
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
var focusedElemObj: CFTypeRef? |
|||
let err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedElemObj) |
|||
print("kAXFocusedUIElement error:", err.rawValue) |
|||
|
|||
if err == .success, let elem = focusedElemObj { |
|||
let axElem = elem as! AXUIElement |
|||
var roleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleObj) |
|||
print("Role:", roleObj ?? "none") |
|||
|
|||
var valObj: CFTypeRef? |
|||
let valErr = AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) |
|||
print("Value err:", valErr.rawValue, "Value:", valObj ?? "nil") |
|||
|
|||
var selectedTextObj: CFTypeRef? |
|||
let selTxtErr = AXUIElementCopyAttributeValue(axElem, kAXSelectedTextAttribute as CFString, &selectedTextObj) |
|||
print("SelectedText err:", selTxtErr.rawValue, "SelectedText:", selectedTextObj ?? "nil") |
|||
|
|||
var rangeObj: CFTypeRef? |
|||
let rangeErr = AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &rangeObj) |
|||
print("Range err:", rangeErr.rawValue) |
|||
if rangeErr == .success, let axRange = rangeObj { |
|||
var range = CFRange() |
|||
if AXValueGetValue(axRange as! AXValue, .cfRange, &range) { |
|||
print("CFRange: loc=\(range.location), len=\(range.length)") |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
inspect() |
|||
@ -0,0 +1,45 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func getFocusedElement() -> AXUIElement? { |
|||
// 1. System wide |
|||
let sysWide = AXUIElementCreateSystemWide() |
|||
var sysFocusedObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &sysFocusedObj) == .success, |
|||
let obj = sysFocusedObj { |
|||
return (obj as! AXUIElement) |
|||
} |
|||
|
|||
// 2. Frontmost App |
|||
if let frontApp = NSWorkspace.shared.frontmostApplication { |
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
var appFocusedObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &appFocusedObj) == .success, |
|||
let obj = appFocusedObj { |
|||
return (obj as! AXUIElement) |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func inspectElement(_ elem: AXUIElement) { |
|||
var attrNamesObj: CFArray? |
|||
AXUIElementCopyAttributeNames(elem, &attrNamesObj) |
|||
if let names = attrNamesObj as? [String] { |
|||
print("Attribute Names:", names) |
|||
for name in names { |
|||
var val: CFTypeRef? |
|||
let err = AXUIElementCopyAttributeValue(elem, name as CFString, &val) |
|||
if err == .success, let val = val { |
|||
print(" \(name): \(val)") |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
if let focused = getFocusedElement() { |
|||
print("Found Focused Element:") |
|||
inspectElement(focused) |
|||
} else { |
|||
print("No focused element found") |
|||
} |
|||
@ -0,0 +1,76 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func findFocusedDescendant(_ elem: AXUIElement) -> AXUIElement? { |
|||
// Check if this element is a text field/text area or has value |
|||
var roleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) |
|||
let role = (roleObj as? String) ?? "" |
|||
|
|||
if role == "AXTextField" || role == "AXTextArea" || role == "AXComboBox" || role == "AXSearchField" { |
|||
return elem |
|||
} |
|||
|
|||
var focusedObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &focusedObj) == .success, |
|||
let isFocused = focusedObj as? Bool, isFocused { |
|||
// If it has value attribute, return it |
|||
var valObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) == .success { |
|||
return elem |
|||
} |
|||
} |
|||
|
|||
// Check children |
|||
var childrenObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, |
|||
let children = childrenObj as? [AXUIElement] { |
|||
for child in children { |
|||
if let found = findFocusedDescendant(child) { |
|||
return found |
|||
} |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func getDeepFocusedElement() -> (AXUIElement, String, String)? { |
|||
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return nil } |
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
|
|||
// First try standard focused element |
|||
var focusedObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedObj) == .success, |
|||
let elem = focusedObj as! AXUIElement? { |
|||
var roleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) |
|||
let role = (roleObj as? String) ?? "" |
|||
|
|||
var valObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) |
|||
let val = (valObj as? String) ?? "" |
|||
|
|||
if !val.isEmpty || role == "AXTextField" || role == "AXTextArea" { |
|||
return (elem, role, val) |
|||
} |
|||
|
|||
// If it's a window or web area, search descendants |
|||
if let deep = findFocusedDescendant(elem) { |
|||
var deepRoleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(deep, kAXRoleAttribute as CFString, &deepRoleObj) |
|||
let deepRole = (deepRoleObj as? String) ?? "" |
|||
|
|||
var deepValObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(deep, kAXValueAttribute as CFString, &deepValObj) |
|||
let deepVal = (deepValObj as? String) ?? "" |
|||
return (deep, deepRole, deepVal) |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
if let (elem, role, val) = getDeepFocusedElement() { |
|||
print("Found deep focused element! Role: \(role), Value: '\(val)'") |
|||
} else { |
|||
print("Deep focused element not found") |
|||
} |
|||
@ -0,0 +1,81 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func testLiveAX() { |
|||
let sysWide = AXUIElementCreateSystemWide() |
|||
|
|||
// Enable Chromium/Electron accessibility |
|||
AXUIElementSetAttributeValue(sysWide, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
|||
AXUIElementSetAttributeValue(sysWide, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
|||
|
|||
guard let frontApp = NSWorkspace.shared.frontmostApplication else { |
|||
print("No front app") |
|||
return |
|||
} |
|||
|
|||
print("Front App:", frontApp.localizedName ?? "", "PID:", frontApp.processIdentifier) |
|||
|
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
AXUIElementSetAttributeValue(appElem, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) |
|||
AXUIElementSetAttributeValue(appElem, "AXManualAccessibility" as CFString, kCFBooleanTrue) |
|||
|
|||
// Try system wide focused element |
|||
var focusedUIElement: CFTypeRef? |
|||
var err = AXUIElementCopyAttributeValue(sysWide, kAXFocusedUIElementAttribute as CFString, &focusedUIElement) |
|||
if err != .success || focusedUIElement == nil { |
|||
err = AXUIElementCopyAttributeValue(appElem, kAXFocusedUIElementAttribute as CFString, &focusedUIElement) |
|||
} |
|||
|
|||
guard err == .success, let elem = focusedUIElement else { |
|||
print("No focused element. Error:", err.rawValue) |
|||
return |
|||
} |
|||
|
|||
let axElem = elem as! AXUIElement |
|||
|
|||
var roleRef: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleRef) |
|||
let role = (roleRef as? String) ?? "AXUnknown" |
|||
print("Role:", role) |
|||
|
|||
// Extract text |
|||
var currentText = "" |
|||
var valueRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valueRef) == .success, |
|||
let val = valueRef { |
|||
if let str = val as? String { |
|||
currentText = str |
|||
} else if let attrStr = val as? NSAttributedString { |
|||
currentText = attrStr.string |
|||
} |
|||
} |
|||
|
|||
if currentText.isEmpty { |
|||
var countRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(axElem, kAXNumberOfCharactersAttribute as CFString, &countRef) == .success, |
|||
let count = countRef as? Int { |
|||
var range = CFRange(location: 0, length: count) |
|||
if let axRange = AXValueCreate(.cfRange, &range) { |
|||
var stringRef: CFTypeRef? |
|||
if AXUIElementCopyParameterizedAttributeValue(axElem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &stringRef) == .success, |
|||
let str = stringRef as? String { |
|||
currentText = str |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
print("Extracted Text: '\(currentText)'") |
|||
|
|||
// Selection / Cursor |
|||
var rangeRef: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(axElem, kAXSelectedTextRangeAttribute as CFString, &rangeRef) == .success, |
|||
let val = rangeRef { |
|||
var cfRange = CFRange() |
|||
if AXValueGetValue(val as! AXValue, .cfRange, &cfRange) { |
|||
print("Cursor Location: \(cfRange.location), Selection Length: \(cfRange.length)") |
|||
} |
|||
} |
|||
} |
|||
|
|||
testLiveAX() |
|||
@ -0,0 +1,21 @@ |
|||
import Foundation |
|||
import Cocoa |
|||
|
|||
class TestTimer { |
|||
private var timerSource: DispatchSourceTimer? |
|||
|
|||
func start() { |
|||
let queue = DispatchQueue(label: "com.test.timer", qos: .userInteractive) |
|||
let timer = DispatchSource.makeTimerSource(queue: queue) |
|||
timer.schedule(deadline: .now(), repeating: .milliseconds(50)) |
|||
timer.setEventHandler { |
|||
if let (elem, app) = FocusedInputSync.shared.getFocusedElement() { |
|||
if let state = FocusedInputSync.shared.inspectCurrentState() { |
|||
print("Live State detected: '\(state.text)' in \(state.app)") |
|||
} |
|||
} |
|||
} |
|||
timer.resume() |
|||
self.timerSource = timer |
|||
} |
|||
} |
|||
@ -0,0 +1,39 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func printAXTree(_ elem: AXUIElement, depth: Int = 0) { |
|||
if depth > 7 { return } |
|||
let indent = String(repeating: " ", count: depth) |
|||
|
|||
var roleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj) |
|||
let role = (roleObj as? String) ?? "unknown" |
|||
|
|||
var valObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) |
|||
let val = (valObj as? String) ?? "" |
|||
|
|||
var titleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXTitleAttribute as CFString, &titleObj) |
|||
let title = (titleObj as? String) ?? "" |
|||
|
|||
var focusedObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(elem, kAXFocusedAttribute as CFString, &focusedObj) |
|||
let isFocused = (focusedObj as? Bool) ?? false |
|||
|
|||
print("\(indent)[\(role)] title='\(title)' val='\(val)' focused=\(isFocused)") |
|||
|
|||
var childrenObj: CFTypeRef? |
|||
if AXUIElementCopyAttributeValue(elem, kAXChildrenAttribute as CFString, &childrenObj) == .success, |
|||
let children = childrenObj as? [AXUIElement] { |
|||
for child in children { |
|||
printAXTree(child, depth: depth + 1) |
|||
} |
|||
} |
|||
} |
|||
|
|||
if let frontApp = NSWorkspace.shared.frontmostApplication { |
|||
print("Front App:", frontApp.localizedName ?? "") |
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
printAXTree(appElem) |
|||
} |
|||
@ -0,0 +1,33 @@ |
|||
import Cocoa |
|||
import ApplicationServices |
|||
|
|||
func inspectFrontApp() { |
|||
guard let frontApp = NSWorkspace.shared.frontmostApplication else { return } |
|||
print("Frontmost App:", frontApp.localizedName ?? "") |
|||
|
|||
let appElem = AXUIElementCreateApplication(frontApp.processIdentifier) |
|||
|
|||
// 1. Try focused window |
|||
var windowObj: CFTypeRef? |
|||
var err = AXUIElementCopyAttributeValue(appElem, kAXFocusedWindowAttribute as CFString, &windowObj) |
|||
print("kAXFocusedWindowAttribute err:", err.rawValue) |
|||
|
|||
if err == .success, let win = windowObj { |
|||
let winElem = win as! AXUIElement |
|||
var focusedObj: CFTypeRef? |
|||
let winErr = AXUIElementCopyAttributeValue(winElem, kAXFocusedUIElementAttribute as CFString, &focusedObj) |
|||
print("Window focused element err:", winErr.rawValue) |
|||
if winErr == .success, let elem = focusedObj { |
|||
let axElem = elem as! AXUIElement |
|||
var roleObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(axElem, kAXRoleAttribute as CFString, &roleObj) |
|||
print("Role from window:", roleObj ?? "none") |
|||
|
|||
var valObj: CFTypeRef? |
|||
AXUIElementCopyAttributeValue(axElem, kAXValueAttribute as CFString, &valObj) |
|||
print("Value from window:", valObj ?? "none") |
|||
} |
|||
} |
|||
} |
|||
|
|||
inspectFrontApp() |
|||
@ -1,9 +1,9 @@ |
|||
#!/usr/bin/env python3 |
|||
""" |
|||
Soniox Bi-Directional Input Synchronization Gateway (v4.0) |
|||
- Mirrors Mac focused input box <--> Android phone in real-time. |
|||
- Supports inserting voice text at exact cursor location. |
|||
- Instant <15ms WebSocket push to Mac for editing and pasting. |
|||
Soniox Collaborative Bi-Directional Input Synchronization Gateway (v5.0) |
|||
- Real-time Google Docs / Figma style collaborative mirroring between Mac and Android. |
|||
- Monotonic revision counters, echo-loop suppression, and sub-15ms WebSocket routing. |
|||
- Cursor-aware voice insertion and instant text editing. |
|||
""" |
|||
|
|||
import asyncio |
|||
@ -19,22 +19,24 @@ import websockets |
|||
from websockets.protocol import State |
|||
|
|||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") |
|||
logger = logging.getLogger("SonioxRelay") |
|||
logger = logging.getLogger("SyncGateway") |
|||
|
|||
connected_mac_websockets = set() |
|||
connected_phone_websockets = set() |
|||
recent_pasted_sessions = OrderedDict() # session_id -> (timestamp, text) |
|||
|
|||
latest_mac_input_state = { |
|||
"type": "mac_input_state", |
|||
# Global Room State Snapshot (Single Source of Truth) |
|||
current_room_state = { |
|||
"type": "sync_state", |
|||
"source": "server_init", |
|||
"app": "Desktop", |
|||
"text": "", |
|||
"cursor": 0, |
|||
"selection": 0, |
|||
"revision": 0, |
|||
"timestamp": time.time() |
|||
} |
|||
room_lock = asyncio.Lock() |
|||
|
|||
MAC_DIRECT_HTTP_URL = "http://116.16.16.20:8999/paste" |
|||
SONIOX_WS_URL = ( |
|||
"wss://translate.compare.soniox.com/compare/api/compare-websocket" |
|||
"?language_hints=fa&language_hints=en&language_hints=ar" |
|||
@ -104,101 +106,29 @@ class SonioxPool: |
|||
soniox_pool = SonioxPool() |
|||
|
|||
def sanitize_and_flatten_text(text: str) -> str: |
|||
""" |
|||
1. Removes all line breaks (\\r, \\n) and collapses whitespace into single spaces. |
|||
2. Strips English hallucination stop words during Persian speech. |
|||
3. Guarantees zero trailing/leading enters or spaces. |
|||
""" |
|||
if not text: |
|||
return "" |
|||
|
|||
flattened = re.sub(r"[\r\n\t]+", " ", text) |
|||
flattened = re.sub(r"\s+", " ", flattened).strip() |
|||
|
|||
if not flattened: |
|||
return "" |
|||
|
|||
words = flattened.split() |
|||
fa_pattern = re.compile(r"[\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF]") |
|||
en_pattern = re.compile(r"[a-zA-Z]") |
|||
|
|||
fa_count = sum(1 for w in words if fa_pattern.search(w)) |
|||
en_count = sum(1 for w in words if en_pattern.search(w)) |
|||
total = fa_count + en_count |
|||
|
|||
if total == 0: |
|||
return flattened |
|||
|
|||
fa_ratio = fa_count / total |
|||
stop_words = {"sex", "from", "no", "oh", "you", "all", "know", "that", "god", "loves", "men", "for", "day", "night", "mankind", "wanted", "moments"} |
|||
|
|||
cleaned = [] |
|||
if fa_ratio >= 0.25: |
|||
for w in words: |
|||
if en_pattern.search(w) and not fa_pattern.search(w): |
|||
clean_w = re.sub(r"[.,!?:;،؛؟\"'()\[\]{}«»–—-]", "", w.lower()) |
|||
if clean_w in stop_words or fa_ratio >= 0.70: |
|||
continue |
|||
cleaned.append(w) |
|||
else: |
|||
cleaned = words |
|||
|
|||
result = " ".join(cleaned) |
|||
return re.sub(r"\s+", " ", result).strip() |
|||
|
|||
async def broadcast_to_macs(payload_dict: dict) -> bool: |
|||
"""Pushes command payload directly to Mac via persistent WebSocket in <15ms.""" |
|||
delivered = False |
|||
payload_str = json.dumps(payload_dict, ensure_ascii=False) |
|||
|
|||
dead_sockets = set() |
|||
for ws in list(connected_mac_websockets): |
|||
try: |
|||
if is_ws_open(ws): |
|||
await ws.send_str(payload_str) |
|||
delivered = True |
|||
logger.info("⚡ Pushed to Mac WS: %s", payload_dict.get("action") or payload_dict.get("type")) |
|||
else: |
|||
dead_sockets.add(ws) |
|||
except Exception: |
|||
dead_sockets.add(ws) |
|||
|
|||
for dead in dead_sockets: |
|||
connected_mac_websockets.discard(dead) |
|||
|
|||
if delivered: |
|||
return True |
|||
|
|||
# Fallback to SSH script if WebSocket temporarily disconnected |
|||
try: |
|||
text = payload_dict.get("text", "") |
|||
if text: |
|||
clean_text = sanitize_and_flatten_text(text) |
|||
escaped_text = clean_text.replace("'", "'\\''") |
|||
remote_cmd = ( |
|||
f"printf '%s' '{escaped_text}' | pbcopy && " |
|||
f"/usr/bin/osascript -e 'tell application \"System Events\" to keystroke \"v\" using command down'" |
|||
) |
|||
proc = await asyncio.create_subprocess_exec( |
|||
"ssh", "-p", "2222", "-o", "BatchMode=yes", "-o", "ConnectTimeout=2", "[email protected]", |
|||
remote_cmd, |
|||
stdout=subprocess.PIPE, |
|||
stderr=subprocess.PIPE |
|||
) |
|||
stdout, stderr = await proc.communicate() |
|||
if proc.returncode == 0: |
|||
logger.info("✅ Pasted to Mac via SSH Tunnel fallback: '%s'", clean_text[:30]) |
|||
return True |
|||
except Exception as e: |
|||
logger.warning("SSH fallback error: %s", e) |
|||
|
|||
return False |
|||
|
|||
async def broadcast_to_phones(payload_dict: dict): |
|||
"""Pushes Mac input state changes to all connected Android clients.""" |
|||
payload_str = json.dumps(payload_dict, ensure_ascii=False) |
|||
return flattened |
|||
|
|||
async def broadcast_state(payload_dict: dict, exclude_ws=None): |
|||
"""Broadcasts state snapshot to all connected clients (Mac and Phone) except sender.""" |
|||
global current_room_state |
|||
async with room_lock: |
|||
current_room_state["revision"] += 1 |
|||
payload_dict["revision"] = current_room_state["revision"] |
|||
payload_dict["timestamp"] = time.time() |
|||
|
|||
# Update our cached authoritative state |
|||
current_room_state.update(payload_dict) |
|||
payload_str = json.dumps(payload_dict, ensure_ascii=False) |
|||
|
|||
# 1. Send to Phone clients |
|||
dead_phones = set() |
|||
for ws in list(connected_phone_websockets): |
|||
if ws is exclude_ws: |
|||
continue |
|||
try: |
|||
if is_ws_open(ws): |
|||
await ws.send_str(payload_str) |
|||
@ -206,26 +136,35 @@ async def broadcast_to_phones(payload_dict: dict): |
|||
dead_phones.add(ws) |
|||
except Exception: |
|||
dead_phones.add(ws) |
|||
|
|||
for dead in dead_phones: |
|||
connected_phone_websockets.discard(dead) |
|||
|
|||
# 2. Send to Mac clients |
|||
dead_macs = set() |
|||
for ws in list(connected_mac_websockets): |
|||
if ws is exclude_ws: |
|||
continue |
|||
try: |
|||
if is_ws_open(ws): |
|||
await ws.send_str(payload_str) |
|||
else: |
|||
dead_macs.add(ws) |
|||
except Exception: |
|||
dead_macs.add(ws) |
|||
for dead in dead_macs: |
|||
connected_mac_websockets.discard(dead) |
|||
|
|||
async def handle_phone_stream_ws(request): |
|||
""" |
|||
⚡ Persistent Duplex Channel for Android Client: |
|||
- Receives live Mac input state on connect and continuously. |
|||
- Handles live audio streaming and returns live STT tokens. |
|||
- Receives user manual text edits and pushes them instantly to Mac. |
|||
""" |
|||
ws = web.WebSocketResponse(heartbeat=15.0) |
|||
"""Duplex WebSocket for Android client (Real-time Live Sync + Audio Dictation).""" |
|||
ws = web.WebSocketResponse(heartbeat=12.0) |
|||
await ws.prepare(request) |
|||
client_ip = request.remote |
|||
logger.info("📱 Android Client connected to Duplex Stream: %s", client_ip) |
|||
logger.info("📱 Android Client Connected: %s", client_ip) |
|||
connected_phone_websockets.add(ws) |
|||
|
|||
# Immediately send the latest Mac input state to phone upon connection! |
|||
if latest_mac_input_state: |
|||
await ws.send_str(json.dumps(latest_mac_input_state, ensure_ascii=False)) |
|||
# Immediately hydrate phone with full current state! |
|||
async with room_lock: |
|||
await ws.send_str(json.dumps(current_room_state, ensure_ascii=False)) |
|||
|
|||
active_soniox_ws = None |
|||
reader_task = None |
|||
@ -286,7 +225,7 @@ async def handle_phone_stream_ws(request): |
|||
try: |
|||
async for msg in ws: |
|||
if msg.type == aiohttp.WSMsgType.BINARY: |
|||
# Live PCM audio chunk (2048 bytes / 64ms) |
|||
# Live PCM audio chunk |
|||
if active_soniox_ws and is_ws_open(active_soniox_ws): |
|||
await active_soniox_ws.send(msg.data) |
|||
|
|||
@ -299,7 +238,13 @@ async def handle_phone_stream_ws(request): |
|||
msg_type = data.get("type") or data.get("action") |
|||
sid = data.get("session_id", f"sess_{int(time.time()*1000)}") |
|||
|
|||
if msg_type == "start": |
|||
if msg_type == "sync_state" or msg_type == "phone_input_edit" or msg_type == "update_input": |
|||
# Phone edited text: broadcast to Mac immediately! |
|||
data["source"] = "android" |
|||
data["type"] = "sync_state" |
|||
await broadcast_state(data, exclude_ws=ws) |
|||
|
|||
elif msg_type == "start": |
|||
current_session_id = sid |
|||
full_final_tokens.clear() |
|||
current_non_final = "" |
|||
@ -327,7 +272,7 @@ async def handle_phone_stream_ws(request): |
|||
clean_final = sanitize_and_flatten_text(raw_final) |
|||
logger.info("⚡ Session %s final text: '%s'", sid, clean_final) |
|||
|
|||
# Return final voice transcription to Android |
|||
# Return final speech text to phone |
|||
if is_ws_open(ws): |
|||
await ws.send_str(json.dumps({ |
|||
"type": "final", |
|||
@ -347,32 +292,6 @@ async def handle_phone_stream_ws(request): |
|||
|
|||
asyncio.create_task(soniox_pool.refill()) |
|||
|
|||
elif msg_type == "phone_input_edit" or msg_type == "update_mac_input": |
|||
# User manually edited text on phone or pressed "Insert in Mac" |
|||
edit_text = data.get("text", "") |
|||
cursor = data.get("cursor_pos") or data.get("cursor") |
|||
is_full_replace = data.get("is_full_replace", True) |
|||
logger.info("📱 Received phone edit to push to Mac: '%s' (cursor: %s)", edit_text[:30], cursor) |
|||
|
|||
mac_ok = await broadcast_to_macs({ |
|||
"action": "update_input", |
|||
"text": edit_text, |
|||
"cursor": cursor, |
|||
"is_full_replace": is_full_replace |
|||
}) |
|||
|
|||
# Update our cached latest state |
|||
latest_mac_input_state["text"] = edit_text |
|||
latest_mac_input_state["cursor"] = cursor if cursor is not None else len(edit_text) |
|||
latest_mac_input_state["timestamp"] = time.time() |
|||
|
|||
if is_ws_open(ws): |
|||
await ws.send_str(json.dumps({ |
|||
"type": "edit_ack", |
|||
"session_id": sid, |
|||
"mac_delivered": mac_ok |
|||
})) |
|||
|
|||
elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.ERROR): |
|||
break |
|||
|
|||
@ -391,27 +310,26 @@ async def handle_phone_stream_ws(request): |
|||
return ws |
|||
|
|||
async def handle_mac_ws(request): |
|||
"""Persistent WebSocket for Mac Bridge (receives input state & pushes edits).""" |
|||
global latest_mac_input_state |
|||
"""Persistent WebSocket for Mac Bridge (receives live Mac typing & pushes edits).""" |
|||
ws = web.WebSocketResponse(heartbeat=10.0) |
|||
await ws.prepare(request) |
|||
client_ip = request.remote |
|||
logger.info("🖥️ Mac client connected to persistent WebSocket: %s", client_ip) |
|||
logger.info("🖥️ Mac client connected: %s", client_ip) |
|||
connected_mac_websockets.add(ws) |
|||
|
|||
try: |
|||
await ws.send_str(json.dumps({"type": "welcome", "message": "Connected to Soniox Duplex Gateway"})) |
|||
await ws.send_str(json.dumps({"type": "welcome", "message": "Connected to Soniox Sync Gateway"})) |
|||
async for msg in ws: |
|||
if msg.type == aiohttp.WSMsgType.TEXT: |
|||
try: |
|||
data = json.loads(msg.data) |
|||
msg_type = data.get("type") |
|||
|
|||
if msg_type == "mac_input_state": |
|||
# Mac reports focused input box text and cursor position |
|||
latest_mac_input_state = data |
|||
# Broadcast immediately to phone! |
|||
await broadcast_to_phones(data) |
|||
if msg_type == "sync_state" or msg_type == "mac_input_state": |
|||
# Mac reports typing / cursor change: broadcast to all phones immediately! |
|||
data["source"] = "mac" |
|||
data["type"] = "sync_state" |
|||
await broadcast_state(data, exclude_ws=ws) |
|||
|
|||
elif msg_type == "ping": |
|||
await ws.send_str(json.dumps({"type": "pong"})) |
|||
@ -421,35 +339,32 @@ async def handle_mac_ws(request): |
|||
break |
|||
finally: |
|||
connected_mac_websockets.discard(ws) |
|||
if is_ws_open(ws): |
|||
await ws.close() |
|||
logger.info("🖥️ Mac client disconnected: %s", client_ip) |
|||
|
|||
return ws |
|||
|
|||
async def handle_health(request): |
|||
async with room_lock: |
|||
state_copy = dict(current_room_state) |
|||
return web.json_response({ |
|||
"status": "ok", |
|||
"service": "Soniox Bi-Directional Duplex Gateway v4.0", |
|||
"service": "Soniox Collaborative Sync Gateway v5.0", |
|||
"connected_macs": len(connected_mac_websockets), |
|||
"connected_phones": len(connected_phone_websockets), |
|||
"latest_mac_input_app": latest_mac_input_state.get("app", ""), |
|||
"latest_mac_input_text_len": len(latest_mac_input_state.get("text", "")) |
|||
"current_app": state_copy.get("app", ""), |
|||
"current_revision": state_copy.get("revision", 0), |
|||
"current_text_len": len(state_copy.get("text", "")) |
|||
}) |
|||
|
|||
async def handle_paste(request): |
|||
try: |
|||
data = await request.json() |
|||
text = data.get("text", "") |
|||
session_id = data.get("session_id", "") |
|||
cursor = data.get("cursor_pos") or data.get("cursor") |
|||
success = await broadcast_to_macs({ |
|||
"action": "update_input", |
|||
"text": text, |
|||
"cursor": cursor, |
|||
"is_full_replace": True |
|||
}) |
|||
return web.json_response({"status": "pasted" if success else "failed", "mac_delivered": success}) |
|||
data["source"] = "http_post" |
|||
data["type"] = "sync_state" |
|||
await broadcast_state(data) |
|||
return web.json_response({"status": "synced", "revision": current_room_state["revision"]}) |
|||
except Exception as e: |
|||
return web.json_response({"error": str(e)}, status=400) |
|||
|
|||
|
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue