fix: multi-window, click-move (#7844)

Signed-off-by: fufesou <shuanglongchen@yeah.net>

flutter/lib/common/widgets/remote_input.dart

@@ -203,7 +203,7 @@ class _RawTouchGestureDetectorRegionState
       return;
     }
     if (!handleTouch) {
-      ffi.cursorModel.updatePan(d.delta.dx, d.delta.dy, handleTouch);
+      ffi.cursorModel.updatePan(d.delta, d.localPosition, handleTouch);
     }
   }
 
@@ -222,6 +222,9 @@ class _RawTouchGestureDetectorRegionState
       return;
     }
     if (handleTouch) {
+      if (isDesktop) {
+        ffi.cursorModel.trySetRemoteWindowCoords();
+      }
       inputModel.sendMouse('down', MouseButtons.left);
       ffi.cursorModel.move(d.localPosition.dx, d.localPosition.dy);
     } else {
@@ -241,13 +244,16 @@ class _RawTouchGestureDetectorRegionState
     if (lastDeviceKind != PointerDeviceKind.touch) {
       return;
     }
-    ffi.cursorModel.updatePan(d.delta.dx, d.delta.dy, handleTouch);
+    ffi.cursorModel.updatePan(d.delta, d.localPosition, handleTouch);
   }
 
   onOneFingerPanEnd(DragEndDetails d) {
     if (lastDeviceKind != PointerDeviceKind.touch) {
       return;
     }
+    if (isDesktop) {
+      ffi.cursorModel.clearRemoteWindowCoords();
+    }
     inputModel.sendMouse('up', MouseButtons.left);
   }
 

flutter/lib/consts.dart

@@ -60,6 +60,7 @@ const String kWindowEventActiveSession = "active_session";
 const String kWindowEventActiveDisplaySession = "active_display_session";
 const String kWindowEventGetRemoteList = "get_remote_list";
 const String kWindowEventGetSessionIdList = "get_session_id_list";
+const String kWindowEventRemoteWindowCoords = "remote_window_coords";
 
 const String kWindowEventMoveTabToNewWindow = "move_tab_to_new_window";
 const String kWindowEventGetCachedSessionData = "get_cached_session_data";

flutter/lib/desktop/pages/desktop_home_page.dart

@@ -774,6 +774,12 @@ class _DesktopHomePageState extends State<DesktopHomePage>
         final screenRect = parseParamScreenRect(args);
         await rustDeskWinManager.openMonitorSession(
             windowId, peerId, display, displayCount, screenRect);
+      } else if (call.method == kWindowEventRemoteWindowCoords) {
+        final windowId = int.tryParse(call.arguments);
+        if (windowId != null) {
+          return jsonEncode(
+              await rustDeskWinManager.getOtherRemoteWindowCoords(windowId));
+        }
       }
     });
     _uniLinksSubscription = listenUniLinks();

flutter/lib/desktop/pages/remote_tab_page.dart

@@ -7,6 +7,7 @@ import 'package:flutter/material.dart';
 import 'package:flutter_hbb/common.dart';
 import 'package:flutter_hbb/common/shared_state.dart';
 import 'package:flutter_hbb/consts.dart';
+import 'package:flutter_hbb/models/input_model.dart';
 import 'package:flutter_hbb/models/state_model.dart';
 import 'package:flutter_hbb/desktop/pages/remote_page.dart';
 import 'package:flutter_hbb/desktop/widgets/remote_toolbar.dart';
@@ -107,107 +108,7 @@ class _ConnectionTabPageState extends State<ConnectionTabPage> {
 
     tabController.onRemoved = (_, id) => onRemoveId(id);
 
-    rustDeskWinManager.setMethodHandler((call, fromWindowId) async {
+    rustDeskWinManager.setMethodHandler(_remoteMethodHandler);
-      print(
-          "[Remote Page] call ${call.method} with args ${call.arguments} from window $fromWindowId");
-
-      dynamic returnValue;
-      // for simplify, just replace connectionId
-      if (call.method == kWindowEventNewRemoteDesktop) {
-        final args = jsonDecode(call.arguments);
-        final id = args['id'];
-        final switchUuid = args['switch_uuid'];
-        final sessionId = args['session_id'];
-        final tabWindowId = args['tab_window_id'];
-        final display = args['display'];
-        final displays = args['displays'];
-        final screenRect = parseParamScreenRect(args);
-        windowOnTop(windowId());
-        tryMoveToScreenAndSetFullscreen(screenRect);
-        if (tabController.length == 0) {
-          // Show the hidden window.
-          if (isMacOS && stateGlobal.closeOnFullscreen == true) {
-            stateGlobal.setFullscreen(true);
-          }
-          // Reset the state
-          stateGlobal.closeOnFullscreen = null;
-        }
-        ConnectionTypeState.init(id);
-        _toolbarState.setShow(
-            bind.mainGetUserDefaultOption(key: 'collapse_toolbar') != 'Y');
-        tabController.add(TabInfo(
-          key: id,
-          label: id,
-          selectedIcon: selectedIcon,
-          unselectedIcon: unselectedIcon,
-          onTabCloseButton: () => tabController.closeBy(id),
-          page: RemotePage(
-            key: ValueKey(id),
-            id: id,
-            sessionId: sessionId == null ? null : SessionID(sessionId),
-            tabWindowId: tabWindowId,
-            display: display,
-            displays: displays?.cast<int>(),
-            password: args['password'],
-            toolbarState: _toolbarState,
-            tabController: tabController,
-            switchUuid: switchUuid,
-            forceRelay: args['forceRelay'],
-            isSharedPassword: args['isSharedPassword'],
-          ),
-        ));
-      } else if (call.method == kWindowDisableGrabKeyboard) {
-        // ???
-      } else if (call.method == "onDestroy") {
-        tabController.clear();
-      } else if (call.method == kWindowActionRebuild) {
-        reloadCurrentWindow();
-      } else if (call.method == kWindowEventActiveSession) {
-        final jumpOk = tabController.jumpToByKey(call.arguments);
-        if (jumpOk) {
-          windowOnTop(windowId());
-        }
-        return jumpOk;
-      } else if (call.method == kWindowEventActiveDisplaySession) {
-        final args = jsonDecode(call.arguments);
-        final id = args['id'];
-        final display = args['display'];
-        final jumpOk = tabController.jumpToByKeyAndDisplay(id, display);
-        if (jumpOk) {
-          windowOnTop(windowId());
-        }
-        return jumpOk;
-      } else if (call.method == kWindowEventGetRemoteList) {
-        return tabController.state.value.tabs
-            .map((e) => e.key)
-            .toList()
-            .join(',');
-      } else if (call.method == kWindowEventGetSessionIdList) {
-        return tabController.state.value.tabs
-            .map((e) => '${e.key},${(e.page as RemotePage).ffi.sessionId}')
-            .toList()
-            .join(';');
-      } else if (call.method == kWindowEventGetCachedSessionData) {
-        // Ready to show new window and close old tab.
-        final args = jsonDecode(call.arguments);
-        final id = args['id'];
-        final close = args['close'];
-        try {
-          final remotePage = tabController.state.value.tabs
-              .firstWhere((tab) => tab.key == id)
-              .page as RemotePage;
-          returnValue = remotePage.ffi.ffiModel.cachedPeerData.toString();
-        } catch (e) {
-          debugPrint('Failed to get cached session data: $e');
-        }
-        if (close && returnValue != null) {
-          closeSessionOnDispose[id] = false;
-          tabController.closeBy(id);
-        }
-      }
-      _update_remote_count();
-      return returnValue;
-    });
     if (!_isScreenRectSet) {
       Future.delayed(Duration.zero, () {
         restoreWindowPosition(
@@ -499,4 +400,130 @@ class _ConnectionTabPageState extends State<ConnectionTabPage> {
 
   _update_remote_count() =>
       RemoteCountState.find().value = tabController.length;
+
+  Future<dynamic> _remoteMethodHandler(call, fromWindowId) async {
+    print(
+        "[Remote Page] call ${call.method} with args ${call.arguments} from window $fromWindowId");
+
+    dynamic returnValue;
+    // for simplify, just replace connectionId
+    if (call.method == kWindowEventNewRemoteDesktop) {
+      final args = jsonDecode(call.arguments);
+      final id = args['id'];
+      final switchUuid = args['switch_uuid'];
+      final sessionId = args['session_id'];
+      final tabWindowId = args['tab_window_id'];
+      final display = args['display'];
+      final displays = args['displays'];
+      final screenRect = parseParamScreenRect(args);
+      windowOnTop(windowId());
+      tryMoveToScreenAndSetFullscreen(screenRect);
+      if (tabController.length == 0) {
+        // Show the hidden window.
+        if (isMacOS && stateGlobal.closeOnFullscreen == true) {
+          stateGlobal.setFullscreen(true);
+        }
+        // Reset the state
+        stateGlobal.closeOnFullscreen = null;
+      }
+      ConnectionTypeState.init(id);
+      _toolbarState.setShow(
+          bind.mainGetUserDefaultOption(key: 'collapse_toolbar') != 'Y');
+      tabController.add(TabInfo(
+        key: id,
+        label: id,
+        selectedIcon: selectedIcon,
+        unselectedIcon: unselectedIcon,
+        onTabCloseButton: () => tabController.closeBy(id),
+        page: RemotePage(
+          key: ValueKey(id),
+          id: id,
+          sessionId: sessionId == null ? null : SessionID(sessionId),
+          tabWindowId: tabWindowId,
+          display: display,
+          displays: displays?.cast<int>(),
+          password: args['password'],
+          toolbarState: _toolbarState,
+          tabController: tabController,
+          switchUuid: switchUuid,
+          forceRelay: args['forceRelay'],
+          isSharedPassword: args['isSharedPassword'],
+        ),
+      ));
+    } else if (call.method == kWindowDisableGrabKeyboard) {
+      // ???
+    } else if (call.method == "onDestroy") {
+      tabController.clear();
+    } else if (call.method == kWindowActionRebuild) {
+      reloadCurrentWindow();
+    } else if (call.method == kWindowEventActiveSession) {
+      final jumpOk = tabController.jumpToByKey(call.arguments);
+      if (jumpOk) {
+        windowOnTop(windowId());
+      }
+      return jumpOk;
+    } else if (call.method == kWindowEventActiveDisplaySession) {
+      final args = jsonDecode(call.arguments);
+      final id = args['id'];
+      final display = args['display'];
+      final jumpOk = tabController.jumpToByKeyAndDisplay(id, display);
+      if (jumpOk) {
+        windowOnTop(windowId());
+      }
+      return jumpOk;
+    } else if (call.method == kWindowEventGetRemoteList) {
+      return tabController.state.value.tabs
+          .map((e) => e.key)
+          .toList()
+          .join(',');
+    } else if (call.method == kWindowEventGetSessionIdList) {
+      return tabController.state.value.tabs
+          .map((e) => '${e.key},${(e.page as RemotePage).ffi.sessionId}')
+          .toList()
+          .join(';');
+    } else if (call.method == kWindowEventGetCachedSessionData) {
+      // Ready to show new window and close old tab.
+      final args = jsonDecode(call.arguments);
+      final id = args['id'];
+      final close = args['close'];
+      try {
+        final remotePage = tabController.state.value.tabs
+            .firstWhere((tab) => tab.key == id)
+            .page as RemotePage;
+        returnValue = remotePage.ffi.ffiModel.cachedPeerData.toString();
+      } catch (e) {
+        debugPrint('Failed to get cached session data: $e');
+      }
+      if (close && returnValue != null) {
+        closeSessionOnDispose[id] = false;
+        tabController.closeBy(id);
+      }
+    } else if (call.method == kWindowEventRemoteWindowCoords) {
+      final remotePage =
+          tabController.state.value.selectedTabInfo.page as RemotePage;
+      final ffi = remotePage.ffi;
+      final displayRect = ffi.ffiModel.displaysRect();
+      if (displayRect != null) {
+        final wc = WindowController.fromWindowId(windowId());
+        Rect? frame;
+        try {
+          frame = await wc.getFrame();
+        } catch (e) {
+          debugPrint(
+              "Failed to get frame of window $windowId, it may be hidden");
+        }
+        if (frame != null) {
+          ffi.cursorModel.moveLocal(0, 0);
+          final coords = RemoteWindowCoords(
+              frame,
+              CanvasCoords.fromCanvasModel(ffi.canvasModel),
+              CursorCoords.fromCursorModel(ffi.cursorModel),
+              displayRect);
+          returnValue = jsonEncode(coords.toJson());
+        }
+      }
+    }
+    _update_remote_count();
+    return returnValue;
+  }
 }

flutter/lib/models/input_model.dart

@@ -4,9 +4,12 @@ import 'dart:io';
 import 'dart:math';
 import 'dart:ui' as ui;
 
+import 'package:desktop_multi_window/desktop_multi_window.dart';
 import 'package:flutter/gestures.dart';
 import 'package:flutter/services.dart';
 import 'package:flutter/widgets.dart';
+import 'package:flutter_hbb/main.dart';
+import 'package:flutter_hbb/utils/multi_window_manager.dart';
 import 'package:get/get.dart';
 
 import '../../models/model.dart';
@@ -21,6 +24,128 @@ const _kMouseEventDown = 'mousedown';
 const _kMouseEventUp = 'mouseup';
 const _kMouseEventMove = 'mousemove';
 
+class CanvasCoords {
+  double x = 0;
+  double y = 0;
+  double scale = 1.0;
+  double scrollX = 0;
+  double scrollY = 0;
+  ScrollStyle scrollStyle = ScrollStyle.scrollauto;
+  Size size = Size.zero;
+
+  CanvasCoords();
+
+  Map<String, dynamic> toJson() {
+    return {
+      'x': x,
+      'y': y,
+      'scale': scale,
+      'scrollX': scrollX,
+      'scrollY': scrollY,
+      'scrollStyle':
+          scrollStyle == ScrollStyle.scrollauto ? 'scrollauto' : 'scrollbar',
+      'size': {
+        'w': size.width,
+        'h': size.height,
+      }
+    };
+  }
+
+  static CanvasCoords fromJson(Map<String, dynamic> json) {
+    final model = CanvasCoords();
+    model.x = json['x'];
+    model.y = json['y'];
+    model.scale = json['scale'];
+    model.scrollX = json['scrollX'];
+    model.scrollY = json['scrollY'];
+    model.scrollStyle = json['scrollStyle'] == 'scrollauto'
+        ? ScrollStyle.scrollauto
+        : ScrollStyle.scrollbar;
+    model.size = Size(json['size']['w'], json['size']['h']);
+    return model;
+  }
+
+  static CanvasCoords fromCanvasModel(CanvasModel model) {
+    final coords = CanvasCoords();
+    coords.x = model.x;
+    coords.y = model.y;
+    coords.scale = model.scale;
+    coords.scrollX = model.scrollX;
+    coords.scrollY = model.scrollY;
+    coords.scrollStyle = model.scrollStyle;
+    coords.size = model.size;
+    return coords;
+  }
+}
+
+class CursorCoords {
+  Offset offset = Offset.zero;
+
+  CursorCoords();
+
+  Map<String, dynamic> toJson() {
+    return {
+      'offset_x': offset.dx,
+      'offset_y': offset.dy,
+    };
+  }
+
+  static CursorCoords fromJson(Map<String, dynamic> json) {
+    final model = CursorCoords();
+    model.offset = Offset(json['offset_x'], json['offset_y']);
+    return model;
+  }
+
+  static CursorCoords fromCursorModel(CursorModel model) {
+    final coords = CursorCoords();
+    coords.offset = model.offset;
+    return coords;
+  }
+}
+
+class RemoteWindowCoords {
+  RemoteWindowCoords(
+      this.windowRect, this.canvas, this.cursor, this.remoteRect);
+  Rect windowRect;
+  CanvasCoords canvas;
+  CursorCoords cursor;
+  Rect remoteRect;
+  Offset relativeOffset = Offset.zero;
+
+  Map<String, dynamic> toJson() {
+    return {
+      'canvas': canvas.toJson(),
+      'cursor': cursor.toJson(),
+      'windowRect': rectToJson(windowRect),
+      'remoteRect': rectToJson(remoteRect),
+    };
+  }
+
+  static Map<String, dynamic> rectToJson(Rect r) {
+    return {
+      'l': r.left,
+      't': r.top,
+      'w': r.width,
+      'h': r.height,
+    };
+  }
+
+  static Rect rectFromJson(Map<String, dynamic> json) {
+    return Rect.fromLTWH(
+      json['l'],
+      json['t'],
+      json['w'],
+      json['h'],
+    );
+  }
+
+  RemoteWindowCoords.fromJson(Map<String, dynamic> json)
+      : windowRect = rectFromJson(json['windowRect']),
+        canvas = CanvasCoords.fromJson(json['canvas']),
+        cursor = CursorCoords.fromJson(json['cursor']),
+        remoteRect = rectFromJson(json['remoteRect']);
+}
+
 extension ToString on MouseButtons {
   String get value {
     switch (this) {
@@ -188,12 +313,17 @@ class InputModel {
   int _lastButtons = 0;
   Offset lastMousePos = Offset.zero;
 
+  bool _queryOtherWindowCoords = false;
+  Rect? _windowRect;
+  List<RemoteWindowCoords> _remoteWindowCoords = [];
+
   late final SessionID sessionId;
 
   bool get keyboardPerm => parent.target!.ffiModel.keyboard;
   String get id => parent.target?.id ?? '';
   String? get peerPlatform => parent.target?.ffiModel.pi.platform;
   bool get isViewOnly => parent.target!.ffiModel.viewOnly;
+  double get devicePixelRatio => parent.target!.canvasModel.devicePixelRatio;
 
   InputModel(this.parent) {
     sessionId = parent.target!.sessionId;
@@ -616,6 +746,9 @@ class InputModel {
   void onPointDownImage(PointerDownEvent e) {
     debugPrint("onPointDownImage ${e.kind}");
     _stopFling = true;
+    if (isDesktop) _queryOtherWindowCoords = true;
+    _remoteWindowCoords = [];
+    _windowRect = null;
     if (isViewOnly) return;
     if (e.kind != ui.PointerDeviceKind.mouse) {
       if (isPhysicalMouse.value) {
@@ -628,6 +761,7 @@ class InputModel {
   }
 
   void onPointUpImage(PointerUpEvent e) {
+    if (isDesktop) _queryOtherWindowCoords = false;
     if (isViewOnly) return;
     if (e.kind != ui.PointerDeviceKind.mouse) return;
     if (isPhysicalMouse.value) {
@@ -638,11 +772,37 @@ class InputModel {
   void onPointMoveImage(PointerMoveEvent e) {
     if (isViewOnly) return;
     if (e.kind != ui.PointerDeviceKind.mouse) return;
+    if (_queryOtherWindowCoords) {
+      Future.delayed(Duration.zero, () async {
+        _windowRect = await fillRemoteCoordsAndGetCurFrame(_remoteWindowCoords);
+      });
+      _queryOtherWindowCoords = false;
+    }
     if (isPhysicalMouse.value) {
       handleMouse(_getMouseEvent(e, _kMouseEventMove), e.position);
     }
   }
 
+  static Future<Rect?> fillRemoteCoordsAndGetCurFrame(
+      List<RemoteWindowCoords> remoteWindowCoords) async {
+    final coords =
+        await rustDeskWinManager.getOtherRemoteWindowCoordsFromMain();
+    final wc = WindowController.fromWindowId(kWindowId!);
+    try {
+      final frame = await wc.getFrame();
+      for (final c in coords) {
+        c.relativeOffset = Offset(
+            c.windowRect.left - frame.left, c.windowRect.top - frame.top);
+        remoteWindowCoords.add(c);
+      }
+      return frame;
+    } catch (e) {
+      // Unreachable code
+      debugPrint("Failed to get frame of window $kWindowId, it may be hidden");
+    }
+    return null;
+  }
+
   void onPointerSignalImage(PointerSignalEvent e) {
     if (isViewOnly) return;
     if (e is PointerScrollEvent) {
@@ -843,43 +1003,107 @@ class InputModel {
     bool onExit = false,
     int buttons = kPrimaryMouseButton,
   }) {
-    y -= CanvasModel.topToEdge;
-    x -= CanvasModel.leftToEdge;
-    final canvasModel = parent.target!.canvasModel;
     final ffiModel = parent.target!.ffiModel;
+    CanvasCoords canvas =
+        CanvasCoords.fromCanvasModel(parent.target!.canvasModel);
+    Rect? rect = ffiModel.rect;
+
     if (isMove) {
-      canvasModel.moveDesktopMouse(x, y);
+      if (_remoteWindowCoords.isNotEmpty &&
+          _windowRect != null &&
+          !_isInCurrentWindow(x, y)) {
+        final coords =
+            findRemoteCoords(x, y, _remoteWindowCoords, devicePixelRatio);
+        if (coords != null) {
+          isMove = false;
+          canvas = coords.canvas;
+          rect = coords.remoteRect;
+          x -= coords.relativeOffset.dx / devicePixelRatio;
+          y -= coords.relativeOffset.dy / devicePixelRatio;
+        }
+      }
     }
 
-    final nearThr = 3;
+    y -= CanvasModel.topToEdge;
-    var nearRight = (canvasModel.size.width - x) < nearThr;
+    x -= CanvasModel.leftToEdge;
-    var nearBottom = (canvasModel.size.height - y) < nearThr;
+    if (isMove) {
-    final rect = ffiModel.rect;
+      parent.target!.canvasModel.moveDesktopMouse(x, y);
+    }
+
+    return _handlePointerDevicePos(
+      kind,
+      x,
+      y,
+      isMove,
+      canvas,
+      rect,
+      evtType,
+      onExit: onExit,
+      buttons: buttons,
+    );
+  }
+
+  bool _isInCurrentWindow(double x, double y) {
+    final w = _windowRect!.width / devicePixelRatio;
+    final h = _windowRect!.width / devicePixelRatio;
+    return x >= 0 && y >= 0 && x <= w && y <= h;
+  }
+
+  static RemoteWindowCoords? findRemoteCoords(double x, double y,
+      List<RemoteWindowCoords> remoteWindowCoords, double devicePixelRatio) {
+    x *= devicePixelRatio;
+    y *= devicePixelRatio;
+    for (final c in remoteWindowCoords) {
+      if (x >= c.relativeOffset.dx &&
+          y >= c.relativeOffset.dy &&
+          x <= c.relativeOffset.dx + c.windowRect.width &&
+          y <= c.relativeOffset.dy + c.windowRect.height) {
+        return c;
+      }
+    }
+    return null;
+  }
+
+  Point? _handlePointerDevicePos(
+    String kind,
+    double x,
+    double y,
+    bool moveInCanvas,
+    CanvasCoords canvas,
+    Rect? rect,
+    String evtType, {
+    bool onExit = false,
+    int buttons = kPrimaryMouseButton,
+  }) {
     if (rect == null) {
       return null;
     }
-    final imageWidth = rect.width * canvasModel.scale;
+
-    final imageHeight = rect.height * canvasModel.scale;
+    final nearThr = 3;
-    if (canvasModel.scrollStyle == ScrollStyle.scrollbar) {
+    var nearRight = (canvas.size.width - x) < nearThr;
-      x += imageWidth * canvasModel.scrollX;
+    var nearBottom = (canvas.size.height - y) < nearThr;
-      y += imageHeight * canvasModel.scrollY;
+    final imageWidth = rect.width * canvas.scale;
+    final imageHeight = rect.height * canvas.scale;
+    if (canvas.scrollStyle == ScrollStyle.scrollbar) {
+      x += imageWidth * canvas.scrollX;
+      y += imageHeight * canvas.scrollY;
 
       // boxed size is a center widget
-      if (canvasModel.size.width > imageWidth) {
+      if (canvas.size.width > imageWidth) {
-        x -= ((canvasModel.size.width - imageWidth) / 2);
+        x -= ((canvas.size.width - imageWidth) / 2);
       }
-      if (canvasModel.size.height > imageHeight) {
+      if (canvas.size.height > imageHeight) {
-        y -= ((canvasModel.size.height - imageHeight) / 2);
+        y -= ((canvas.size.height - imageHeight) / 2);
       }
     } else {
-      x -= canvasModel.x;
+      x -= canvas.x;
-      y -= canvasModel.y;
+      y -= canvas.y;
     }
 
-    x /= canvasModel.scale;
+    x /= canvas.scale;
-    y /= canvasModel.scale;
+    y /= canvas.scale;
-    if (canvasModel.scale > 0 && canvasModel.scale < 1) {
+    if (canvas.scale > 0 && canvas.scale < 1) {
-      final step = 1.0 / canvasModel.scale - 1;
+      final step = 1.0 / canvas.scale - 1;
       if (nearRight) {
         x += step;
       }
@@ -902,8 +1126,7 @@ class InputModel {
       evtX = x.round();
       evtY = y.round();
     } catch (e) {
-      debugPrintStack(
+      debugPrintStack(label: 'canvas.scale value ${canvas.scale}, $e');
-          label: 'canvasModel.scale value ${canvasModel.scale}, $e');
       return null;
     }
 

flutter/lib/models/model.dart

@@ -1729,6 +1729,8 @@ class CursorModel with ChangeNotifier {
   double _displayOriginX = 0;
   double _displayOriginY = 0;
   DateTime? _firstUpdateMouseTime;
+  Rect? _windowRect;
+  List<RemoteWindowCoords> _remoteWindowCoords = [];
   bool gotMouseControl = true;
   DateTime _lastPeerMouse = DateTime.now()
       .subtract(Duration(milliseconds: 3000 * kMouseControlTimeoutMSec));
@@ -1741,6 +1743,8 @@ class CursorModel with ChangeNotifier {
   double get x => _x - _displayOriginX;
   double get y => _y - _displayOriginY;
 
+  double get devicePixelRatio => parent.target!.canvasModel.devicePixelRatio;
+
   Offset get offset => Offset(_x, _y);
 
   double get hotx => _hotx;
@@ -1810,15 +1814,13 @@ class CursorModel with ChangeNotifier {
     notifyListeners();
   }
 
-  updatePan(double dx, double dy, bool touchMode) {
+  updatePan(Offset delta, Offset localPosition, bool touchMode) {
     if (touchMode) {
-      final scale = parent.target?.canvasModel.scale ?? 1.0;
+      _handleTouchMode(delta, localPosition);
-      _x += dx / scale;
-      _y += dy / scale;
-      parent.target?.inputModel.moveMouse(_x, _y);
-      notifyListeners();
       return;
     }
+    double dx = delta.dx;
+    double dy = delta.dy;
     if (parent.target?.imageModel.image == null) return;
     final scale = parent.target?.canvasModel.scale ?? 1.0;
     dx /= scale;
@@ -1885,6 +1887,41 @@ class CursorModel with ChangeNotifier {
     notifyListeners();
   }
 
+  bool _isInCurrentWindow(double x, double y) {
+    final w = _windowRect!.width / devicePixelRatio;
+    final h = _windowRect!.width / devicePixelRatio;
+    return x >= 0 && y >= 0 && x <= w && y <= h;
+  }
+
+  _handleTouchMode(Offset delta, Offset localPosition) {
+    bool isMoved = false;
+    if (_remoteWindowCoords.isNotEmpty &&
+        _windowRect != null &&
+        !_isInCurrentWindow(localPosition.dx, localPosition.dy)) {
+      final coords = InputModel.findRemoteCoords(localPosition.dx,
+          localPosition.dy, _remoteWindowCoords, devicePixelRatio);
+      if (coords != null) {
+        double x2 =
+            (localPosition.dx - coords.relativeOffset.dx / devicePixelRatio) /
+                coords.canvas.scale;
+        double y2 =
+            (localPosition.dy - coords.relativeOffset.dy / devicePixelRatio) /
+                coords.canvas.scale;
+        x2 += coords.cursor.offset.dx;
+        y2 += coords.cursor.offset.dy;
+        parent.target?.inputModel.moveMouse(x2, y2);
+        isMoved = true;
+      }
+    }
+    if (!isMoved) {
+      final scale = parent.target?.canvasModel.scale ?? 1.0;
+      _x += delta.dx / scale;
+      _y += delta.dy / scale;
+      parent.target?.inputModel.moveMouse(_x, _y);
+    }
+    notifyListeners();
+  }
+
   updateCursorData(Map<String, dynamic> evt) async {
     final id = int.parse(evt['id']);
     final hotx = double.parse(evt['hotx']);
@@ -2024,6 +2061,18 @@ class CursorModel with ChangeNotifier {
       deleteCustomCursor(k);
     }
   }
+
+  trySetRemoteWindowCoords() {
+    Future.delayed(Duration.zero, () async {
+      _windowRect =
+          await InputModel.fillRemoteCoordsAndGetCurFrame(_remoteWindowCoords);
+    });
+  }
+
+  clearRemoteWindowCoords() {
+    _windowRect = null;
+    _remoteWindowCoords.clear();
+  }
 }
 
 class QualityMonitorData {

flutter/lib/utils/multi_window_manager.dart

@@ -6,6 +6,8 @@ import 'package:flutter/material.dart';
 import 'package:flutter/services.dart';
 import 'package:flutter_hbb/consts.dart';
 import 'package:flutter_hbb/common.dart';
+import 'package:flutter_hbb/main.dart';
+import 'package:flutter_hbb/models/input_model.dart';
 
 /// must keep the order
 // ignore: constant_identifier_names
@@ -431,6 +433,39 @@ class RustDeskMultiWindowManager {
   void unregisterActiveWindowListener(AsyncCallback callback) {
     _windowActiveCallbacks.remove(callback);
   }
+
+  // This function is called from the main window.
+  // It will query the active remote windows to get their coords.
+  Future<List<String>> getOtherRemoteWindowCoords(int wId) async {
+    List<String> coords = [];
+    for (final windowId in _remoteDesktopWindows) {
+      if (windowId != wId) {
+        if (_activeWindows.contains(windowId)) {
+          final res = await DesktopMultiWindow.invokeMethod(
+              windowId, kWindowEventRemoteWindowCoords, '');
+          if (res != null) {
+            coords.add(res);
+          }
+        }
+      }
+    }
+    return coords;
+  }
+
+  // This function is called from one remote window.
+  // Only the main window knows `_remoteDesktopWindows` and `_activeWindows`.
+  // So we need to call the main window to get the other remote windows' coords.
+  Future<List<RemoteWindowCoords>> getOtherRemoteWindowCoordsFromMain() async {
+    List<RemoteWindowCoords> coords = [];
+    // Call the main window to get the coords of other remote windows.
+    String res = await DesktopMultiWindow.invokeMethod(
+        kMainWindowId, kWindowEventRemoteWindowCoords, kWindowId.toString());
+    List<dynamic> list = jsonDecode(res);
+    for (var item in list) {
+      coords.add(RemoteWindowCoords.fromJson(jsonDecode(item)));
+    }
+    return coords;
+  }
 }
 
 final rustDeskWinManager = RustDeskMultiWindowManager.instance;