Browse Source

fix(sync): resolve Electron AX empty overwrite conflict and completely remove HUD toasts on Mac

main
Ali Alavi 18 hours ago
parent
commit
05b4e4648e
  1. 10
      android/app/src/main/java/com/soniox/remotemic/MainActivity.kt
  2. 21
      mac/src/AppDelegate.swift
  3. 25
      mac/src/FocusedInputSync.swift
  4. 172
      mac/src/HUDOverlay.swift
  5. 31
      mac/src/RelayClient.swift

10
android/app/src/main/java/com/soniox/remotemic/MainActivity.kt

@ -106,7 +106,7 @@ class MainActivity : AppCompatActivity() {
}, },
onSyncStateReceived = { state -> onSyncStateReceived = { state ->
// Drop echoes originated from phone itself // Drop echoes originated from phone itself
if (state.source != "android") {
if (state.source != "android" && state.source != "http_post") {
val timeSinceLocalEdit = System.currentTimeMillis() - lastLocalUserEditTime val timeSinceLocalEdit = System.currentTimeMillis() - lastLocalUserEditTime
// If user recently edited/cleared on phone within 1200ms, block remote echo resurrecting old text! // If user recently edited/cleared on phone within 1200ms, block remote echo resurrecting old text!
@ -123,12 +123,18 @@ class MainActivity : AppCompatActivity() {
binding.tvMacStatus.text = "متصل به ${state.app} 🖥️" binding.tvMacStatus.text = "متصل به ${state.app} 🖥️"
} }
// If state.text is empty and was sent by server_init, do not wipe local text if we already have content
if (state.text.isEmpty() && state.source == "server_init" && lastLocalText.isNotEmpty()) {
return@StreamDictationClient
}
if (state.text != lastLocalText) { if (state.text != lastLocalText) {
isApplyingRemoteUpdate = true isApplyingRemoteUpdate = true
lastLocalText = state.text lastLocalText = state.text
currentRevision = state.revision currentRevision = state.revision
binding.etTranscript.setText(state.text) binding.etTranscript.setText(state.text)
binding.etTranscript.setTextColor(Color.WHITE)
val targetCursor = state.cursor.coerceIn(0, state.text.length) val targetCursor = state.cursor.coerceIn(0, state.text.length)
binding.etTranscript.setSelection(targetCursor) binding.etTranscript.setSelection(targetCursor)
@ -446,7 +452,7 @@ class MainActivity : AppCompatActivity() {
val mediaType = "application/json; charset=utf-8".toMediaType() val mediaType = "application/json; charset=utf-8".toMediaType()
val body = json.toRequestBody(mediaType) val body = json.toRequestBody(mediaType)
val hosts = listOf(gatewayHost, "116.16.16.19:8999", "2.180.16.250:8999").distinct()
val hosts = listOf(gatewayHost, "2.180.16.250:8089", "2.180.16.250:8999", "116.16.16.19:8999").distinct()
var success = false var success = false
for (h in hosts) { for (h in hosts) {
try { try {

21
mac/src/AppDelegate.swift

@ -46,11 +46,7 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
} }
audioRecorder.onAudioLevelUpdate = { [weak self] level in audioRecorder.onAudioLevelUpdate = { [weak self] level in
guard let self = self else { return }
self.currentAudioLevel = level
if self.audioRecorder.isRecording {
HUDOverlayController.shared.show(state: .recording(level: level, liveText: self.latestPartialText))
}
self?.currentAudioLevel = level
} }
audioRecorder.onAudioChunkAvailable = { [weak self] chunk in audioRecorder.onAudioChunkAvailable = { [weak self] chunk in
@ -64,13 +60,8 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
RelayClient.shared.onRemoteUpdateReceived = { text, cursor, isFullReplace in RelayClient.shared.onRemoteUpdateReceived = { text, cursor, isFullReplace in
print("AppDelegate: 📥 Clean Remote Input Update: '\(text.prefix(30))...' (replace: \(isFullReplace))") print("AppDelegate: 📥 Clean Remote Input Update: '\(text.prefix(30))...' (replace: \(isFullReplace))")
// 1. FIRST apply the remote update directly into the active input while window is in pristine focus!
// Apply the remote update directly into the active input on Mac
FocusedInputSync.shared.applyRemoteUpdate(text: text, cursor: cursor, isFullReplace: isFullReplace) FocusedInputSync.shared.applyRemoteUpdate(text: text, cursor: cursor, isFullReplace: isFullReplace)
// 2. THEN show the floating success HUD notification
DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) {
HUDOverlayController.shared.show(state: .success(text: text))
}
} }
RelayClient.shared.start() RelayClient.shared.start()
@ -162,7 +153,6 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
timer.invalidate() timer.invalidate()
self?.permissionPollTimer = nil self?.permissionPollTimer = nil
HotkeyManager.shared.registerHotkeys() HotkeyManager.shared.registerHotkeys()
HUDOverlayController.shared.show(state: .success(text: "دسترسی Accessibility تایید شد ✅"))
} }
} }
} }
@ -194,7 +184,6 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
session.onPartialText = { [weak self] liveText in session.onPartialText = { [weak self] liveText in
guard let self = self, self.audioRecorder.isRecording else { return } guard let self = self, self.audioRecorder.isRecording else { return }
self.latestPartialText = liveText self.latestPartialText = liveText
HUDOverlayController.shared.show(state: .recording(level: self.currentAudioLevel, liveText: liveText))
} }
session.onFinalResult = { [weak self] result in session.onFinalResult = { [weak self] result in
@ -208,7 +197,6 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
FocusedInputSync.shared.applyRemoteUpdate(text: text, cursor: nil, isFullReplace: false) FocusedInputSync.shared.applyRemoteUpdate(text: text, cursor: nil, isFullReplace: false)
case .failure(let error): case .failure(let error):
print("Soniox error:", error) print("Soniox error:", error)
HUDOverlayController.shared.show(state: .error(message: error.localizedDescription))
} }
} }
@ -216,9 +204,8 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
try audioRecorder.startRecording() try audioRecorder.startRecording()
statusBarController.updateIcon(state: .recording) statusBarController.updateIcon(state: .recording)
statusBarController.buildMenu(isRecording: true) statusBarController.buildMenu(isRecording: true)
HUDOverlayController.shared.show(state: .recording(level: 0.0, liveText: nil))
} catch { } catch {
HUDOverlayController.shared.show(state: .error(message: error.localizedDescription))
print("Audio recording start error:", error)
} }
} }
@ -229,11 +216,9 @@ public final class AppDelegate: NSObject, NSApplicationDelegate {
_ = audioRecorder.stopRecording() _ = audioRecorder.stopRecording()
statusBarController.updateIcon(state: .transcribing) statusBarController.updateIcon(state: .transcribing)
statusBarController.buildMenu(isRecording: false) statusBarController.buildMenu(isRecording: false)
HUDOverlayController.shared.show(state: .transcribing)
guard let session = self.activeSession else { guard let session = self.activeSession else {
self.isBusyFinalizing = false self.isBusyFinalizing = false
HUDOverlayController.shared.hide(animated: true)
return return
} }

25
mac/src/FocusedInputSync.swift

@ -173,35 +173,31 @@ public final class FocusedInputSync {
let appChanged = (appName != lastObservedApp) let appChanged = (appName != lastObservedApp)
if appChanged { if appChanged {
lastObservedApp = appName lastObservedApp = appName
lastObservedText = ""
lastObservedCursor = 0
} }
guard let elem = elemOpt else { guard let elem = elemOpt else {
if appChanged { if appChanged {
localRevision += 1 localRevision += 1
return MacInputState(app: appName, text: "", cursor: 0, selection: 0, revision: localRevision)
return MacInputState(app: appName, text: lastObservedText, cursor: lastObservedCursor, selection: 0, revision: localRevision)
} }
return nil return nil
} }
var roleObj: CFTypeRef?
AXUIElementCopyAttributeValue(elem, kAXRoleAttribute as CFString, &roleObj)
let role = roleObj as? String ?? ""
let isTextRole = (role == "AXTextField" || role == "AXTextArea" || role == "AXComboBox" || role == "AXSearchField")
var canReadValue = false
var text = "" var text = ""
var valObj: CFTypeRef? var valObj: CFTypeRef?
if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) == .success, if AXUIElementCopyAttributeValue(elem, kAXValueAttribute as CFString, &valObj) == .success,
let val = valObj { let val = valObj {
if let str = val as? String { if let str = val as? String {
text = str text = str
canReadValue = true
} else if let attrStr = val as? NSAttributedString { } else if let attrStr = val as? NSAttributedString {
text = attrStr.string text = attrStr.string
canReadValue = true
} }
} }
if text.isEmpty && isTextRole {
if !canReadValue {
var countObj: CFTypeRef? var countObj: CFTypeRef?
if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countObj) == .success, if AXUIElementCopyAttributeValue(elem, kAXNumberOfCharactersAttribute as CFString, &countObj) == .success,
let count = countObj as? Int, count > 0 { let count = countObj as? Int, count > 0 {
@ -211,11 +207,22 @@ public final class FocusedInputSync {
if AXUIElementCopyParameterizedAttributeValue(elem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &strObj) == .success, if AXUIElementCopyParameterizedAttributeValue(elem, kAXStringForRangeParameterizedAttribute as CFString, axRange, &strObj) == .success,
let str = strObj as? String { let str = strObj as? String {
text = str text = str
canReadValue = true
} }
} }
} }
} }
// If element is in an Electron / Monaco / Custom Canvas app (e.g. Antigravity IDE) where AX value is not readable,
// NEVER overwrite the collaborative text with empty string!
if !canReadValue {
if appChanged {
localRevision += 1
return MacInputState(app: appName, text: lastObservedText, cursor: lastObservedCursor, selection: 0, revision: localRevision)
}
return nil
}
var cursor = text.count var cursor = text.count
var selLen = 0 var selLen = 0
var rangeObj: CFTypeRef? var rangeObj: CFTypeRef?

172
mac/src/HUDOverlay.swift

@ -8,180 +8,14 @@ public enum HUDState {
case error(message: String) case error(message: String)
} }
final class NonActivatingFloatingPanel: NSPanel {
override var canBecomeKey: Bool { return false }
override var canBecomeMain: Bool { return false }
override var acceptsFirstResponder: Bool { return false }
}
public final class HUDOverlayController { public final class HUDOverlayController {
public static let shared = HUDOverlayController() public static let shared = HUDOverlayController()
private var window: NonActivatingFloatingPanel?
private var visualEffectView: NSVisualEffectView?
private var iconImageView: NSImageView?
private var titleLabel: NSTextField?
private var subtitleLabel: NSTextField?
private var hideTimer: Timer?
private init() {
setupWindow()
}
private func setupWindow() {
let width: CGFloat = 460
let height: CGFloat = 80
let panel = NonActivatingFloatingPanel(
contentRect: NSRect(x: 0, y: 0, width: width, height: height),
styleMask: [.borderless, .nonactivatingPanel],
backing: .buffered,
defer: false
)
panel.level = .floating
panel.isOpaque = false
panel.backgroundColor = .clear
panel.hasShadow = true
panel.ignoresMouseEvents = true
panel.becomesKeyOnlyIfNeeded = false
panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary]
let visualEffect = NSVisualEffectView(frame: NSRect(x: 0, y: 0, width: width, height: height))
visualEffect.material = .hudWindow
visualEffect.blendingMode = .behindWindow
visualEffect.state = .active
visualEffect.wantsLayer = true
visualEffect.layer?.cornerRadius = 24
visualEffect.layer?.masksToBounds = true
visualEffect.layer?.borderWidth = 1.2
visualEffect.layer?.borderColor = NSColor.white.withAlphaComponent(0.25).cgColor
// Icon Image View
let iconView = NSImageView(frame: NSRect(x: 18, y: (height - 42) / 2, width: 42, height: 42))
iconView.imageScaling = .scaleProportionallyUpOrDown
// Title Label
let tLabel = NSTextField(frame: NSRect(x: 72, y: 40, width: width - 90, height: 24))
tLabel.isBezeled = false
tLabel.drawsBackground = false
tLabel.isEditable = false
tLabel.isSelectable = false
tLabel.font = NSFont.systemFont(ofSize: 14, weight: .bold)
tLabel.textColor = .white
tLabel.alignment = .left
// Subtitle / Preview Label
let sLabel = NSTextField(frame: NSRect(x: 72, y: 14, width: width - 90, height: 22))
sLabel.isBezeled = false
sLabel.drawsBackground = false
sLabel.isEditable = false
sLabel.isSelectable = false
sLabel.font = NSFont.systemFont(ofSize: 13, weight: .medium)
sLabel.textColor = NSColor.white.withAlphaComponent(0.9)
sLabel.alignment = .left
visualEffect.addSubview(iconView)
visualEffect.addSubview(tLabel)
visualEffect.addSubview(sLabel)
panel.contentView = visualEffect
self.window = panel
self.visualEffectView = visualEffect
self.iconImageView = iconView
self.titleLabel = tLabel
self.subtitleLabel = sLabel
}
private init() {}
public func show(state: HUDState) { public func show(state: HUDState) {
hideTimer?.invalidate()
hideTimer = nil
guard let panel = self.window else { return }
// Position at bottom center of current active screen
if let screen = NSScreen.main {
let screenRect = screen.visibleFrame
let x = screenRect.origin.x + (screenRect.width - panel.frame.width) / 2
let y = screenRect.origin.y + 60
panel.setFrameOrigin(NSPoint(x: x, y: y))
// Completely disabled per user direction: 100% silent background operation with NO on-screen toasts or overlays
} }
switch state {
case .hidden:
hide(animated: true)
return
case .recording(let level, let liveText):
let micImage = NSImage(systemSymbolName: "mic.fill", accessibilityDescription: "Recording")
let scale: CGFloat = 22.0 + CGFloat(level) * 6.0
iconImageView?.image = micImage?.withSymbolConfiguration(NSImage.SymbolConfiguration(pointSize: scale, weight: .bold))
iconImageView?.contentTintColor = NSColor.systemRed
titleLabel?.stringValue = "🎙️ در حال تبدیل زنده صدا..."
if let live = liveText, !live.isEmpty {
let preview = live.count > 46 ? "..." + String(live.suffix(46)) : live
subtitleLabel?.stringValue = preview
subtitleLabel?.textColor = NSColor.systemGreen.withAlphaComponent(0.95)
} else {
let mode = HotkeyManager.shared.currentMode == .toggle ? "پایان: کلیک مجدد" : "رها کردن کلید ⌥ جهت درج متن"
subtitleLabel?.stringValue = mode
subtitleLabel?.textColor = NSColor.systemRed.withAlphaComponent(0.9)
}
case .transcribing:
let waveImage = NSImage(systemSymbolName: "waveform.badge.magnifyingglass", accessibilityDescription: "Transcribing")
iconImageView?.image = waveImage?.withSymbolConfiguration(NSImage.SymbolConfiguration(pointSize: 22, weight: .bold))
iconImageView?.contentTintColor = NSColor.systemOrange
titleLabel?.stringValue = "⚡ درج آنی متن..."
subtitleLabel?.stringValue = "در حال تایپ در مکان‌نما"
subtitleLabel?.textColor = NSColor.systemOrange.withAlphaComponent(0.9)
case .success(let text):
let checkImage = NSImage(systemSymbolName: "checkmark.circle.fill", accessibilityDescription: "Done")
iconImageView?.image = checkImage?.withSymbolConfiguration(NSImage.SymbolConfiguration(pointSize: 22, weight: .bold))
iconImageView?.contentTintColor = NSColor.systemGreen
titleLabel?.stringValue = "✨ متن درج شد"
let preview = text.count > 46 ? String(text.prefix(46)) + "..." : text
subtitleLabel?.stringValue = preview.isEmpty ? "کلیپ‌بورد به‌روز شد" : preview
subtitleLabel?.textColor = NSColor.white.withAlphaComponent(0.95)
hideTimer = Timer.scheduledTimer(withTimeInterval: 1.2, repeats: false) { [weak self] _ in
self?.hide(animated: true)
}
case .error(let msg):
let errImage = NSImage(systemSymbolName: "exclamationmark.triangle.fill", accessibilityDescription: "Error")
iconImageView?.image = errImage?.withSymbolConfiguration(NSImage.SymbolConfiguration(pointSize: 22, weight: .bold))
iconImageView?.contentTintColor = NSColor.systemYellow
titleLabel?.stringValue = "⚠️ خطا در تبدیل صوت"
subtitleLabel?.stringValue = msg
subtitleLabel?.textColor = NSColor.systemYellow.withAlphaComponent(0.9)
hideTimer = Timer.scheduledTimer(withTimeInterval: 2.0, repeats: false) { [weak self] _ in
self?.hide(animated: true)
}
}
panel.alphaValue = 1.0
panel.orderFrontRegardless()
}
public func hide(animated: Bool) {
guard let panel = self.window, panel.isVisible else { return }
if animated {
NSAnimationContext.runAnimationGroup({ context in
context.duration = 0.15
panel.animator().alphaValue = 0.0
}, completionHandler: {
panel.orderOut(nil)
})
} else {
panel.alphaValue = 0.0
panel.orderOut(nil)
}
}
public func hide(animated: Bool = true) {}
} }

31
mac/src/RelayClient.swift

@ -4,8 +4,13 @@ import Cocoa
public final class RelayClient: NSObject, URLSessionWebSocketDelegate { public final class RelayClient: NSObject, URLSessionWebSocketDelegate {
public static let shared = RelayClient() public static let shared = RelayClient()
// Connects through SSH local port forward 18999 -> Linux Server 8999
private let primaryUrl = URL(string: "ws://127.0.0.1:18999/ws/mac")!
// Multi-endpoint candidate list with automatic failover
private let candidateUrls = [
URL(string: "ws://127.0.0.1:18999/ws/mac")!,
URL(string: "ws://2.180.16.250:8089/ws/mac")!,
URL(string: "ws://2.180.16.250:8999/ws/mac")!
]
private var currentUrlIndex = 0
private var webSocketTask: URLSessionWebSocketTask? private var webSocketTask: URLSessionWebSocketTask?
private var urlSession: URLSession! private var urlSession: URLSession!
@ -14,6 +19,7 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate {
public private(set) var isConnected = false public private(set) var isConnected = false
private var reconnectTimer: Timer? private var reconnectTimer: Timer?
private var pingTimer: Timer? private var pingTimer: Timer?
private var retryCount = 0
private var monitorTimerSource: DispatchSourceTimer? private var monitorTimerSource: DispatchSourceTimer?
private let monitorQueue = DispatchQueue(label: "com.soniox.macsync.monitor", qos: .userInteractive) private let monitorQueue = DispatchQueue(label: "com.soniox.macsync.monitor", qos: .userInteractive)
@ -26,7 +32,7 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate {
super.init() super.init()
let config = URLSessionConfiguration.default let config = URLSessionConfiguration.default
config.waitsForConnectivity = true config.waitsForConnectivity = true
config.timeoutIntervalForRequest = 30
config.timeoutIntervalForRequest = 10
config.timeoutIntervalForResource = 300 config.timeoutIntervalForResource = 300
self.urlSession = URLSession(configuration: config, delegate: self, delegateQueue: .main) self.urlSession = URLSession(configuration: config, delegate: self, delegateQueue: .main)
} }
@ -34,7 +40,7 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate {
public func start() { public func start() {
guard !isRunning else { return } guard !isRunning else { return }
isRunning = true isRunning = true
print("RelayClient: Starting real-time bi-directional sync engine...")
print("RelayClient: Starting resilient real-time bi-directional sync engine...")
connect() connect()
startInputMonitoring() startInputMonitoring()
} }
@ -56,8 +62,11 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate {
guard isRunning else { return } guard isRunning else { return }
webSocketTask?.cancel() webSocketTask?.cancel()
var request = URLRequest(url: primaryUrl)
request.timeoutInterval = 6
let targetUrl = candidateUrls[currentUrlIndex % candidateUrls.count]
print("RelayClient: 🔌 Connecting to Gateway (\(targetUrl.absoluteString))...")
var request = URLRequest(url: targetUrl)
request.timeoutInterval = 4
request.setValue("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) SonioxVoice/1.0", forHTTPHeaderField: "User-Agent") request.setValue("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) SonioxVoice/1.0", forHTTPHeaderField: "User-Agent")
webSocketTask = urlSession.webSocketTask(with: request) webSocketTask = urlSession.webSocketTask(with: request)
@ -183,7 +192,11 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate {
pingTimer = nil pingTimer = nil
if reconnectTimer == nil || !(reconnectTimer?.isValid ?? false) { if reconnectTimer == nil || !(reconnectTimer?.isValid ?? false) {
let timer = Timer(timeInterval: 2.0, repeats: false) { [weak self] _ in
currentUrlIndex += 1
retryCount += 1
let delay = (retryCount <= 2) ? 0.7 : min(Double(1 << min(retryCount, 3)) * 0.8, 4.0)
let timer = Timer(timeInterval: delay, repeats: false) { [weak self] _ in
self?.reconnectTimer = nil self?.reconnectTimer = nil
self?.connect() self?.connect()
} }
@ -193,8 +206,10 @@ public final class RelayClient: NSObject, URLSessionWebSocketDelegate {
} }
public func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask, didOpenWithProtocol protocol: String?) { public func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask, didOpenWithProtocol protocol: String?) {
print("RelayClient: 🟢 Persistent WebSocket Connected to Server Gateway!")
let connectedUrl = candidateUrls[currentUrlIndex % candidateUrls.count]
print("RelayClient: 🟢 Persistent WebSocket Connected to Gateway (\(connectedUrl.absoluteString))!")
self.isConnected = true self.isConnected = true
self.retryCount = 0
} }
public func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask, didCloseWith closeCode: URLSessionWebSocketTask.CloseCode, reason: Data?) { public func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask, didCloseWith closeCode: URLSessionWebSocketTask.CloseCode, reason: Data?) {

Loading…
Cancel
Save