enable retina scale factor (#7269)

* enable retina scale factor

* enabled only when there are only one video service running
* scale mouse event
* scale cursor position
* scale remote menu display button
* adjust resolution

Signed-off-by: 21pages <pages21@163.com>

* Update server.rs

---------

Signed-off-by: 21pages <pages21@163.com>
Co-authored-by: RustDesk <71636191+rustdesk@users.noreply.github.com>

flutter/lib/desktop/widgets/remote_toolbar.dart

@@ -759,15 +759,18 @@ class _MonitorMenu extends StatelessWidget {
       final children = <Widget>[];
       for (var i = 0; i < pi.displays.length; i++) {
         final d = pi.displays[i];
-        final fontSize = (d.width * scale < d.height * scale
-                ? d.width * scale
-                : d.height * scale) *
+        double s = d.scale;
+        int dWidth = d.width.toDouble() ~/ s;
+        int dHeight = d.height.toDouble() ~/ s;
+        final fontSize = (dWidth * scale < dHeight * scale
+                ? dWidth * scale
+                : dHeight * scale) *
             0.65;
         children.add(Positioned(
           left: (d.x - rect.left) * scale + startX,
           top: (d.y - rect.top) * scale + startY,
-          width: d.width * scale,
-          height: d.height * scale,
+          width: dWidth * scale,
+          height: dHeight * scale,
           child: Container(
             decoration: BoxDecoration(
               border: Border.all(
@@ -1287,7 +1290,9 @@ class _ResolutionsMenuState extends State<_ResolutionsMenu> {
     if (lastGroupValue == _kCustomResolutionValue) {
       _groupValue = _kCustomResolutionValue;
     } else {
-      _groupValue = '${rect?.width.toInt()}x${rect?.height.toInt()}';
+      var scale = pi.scaleOfDisplay(pi.currentDisplay);
+      _groupValue =
+          '${(rect?.width ?? 0) ~/ scale}x${(rect?.height ?? 0) ~/ scale}';
     }
   }
 
@@ -1386,6 +1391,11 @@ class _ResolutionsMenuState extends State<_ResolutionsMenu> {
     if (display == null) {
       return Offstage();
     }
+    if (!resolutions.any((e) =>
+        e.width == display.originalWidth &&
+        e.height == display.originalHeight)) {
+      return Offstage();
+    }
     return Offstage(
       offstage: !showOriginalBtn,
       child: MenuButton(

flutter/lib/models/model.dart

@@ -138,21 +138,29 @@ class FfiModel with ChangeNotifier {
     sessionId = parent.target!.sessionId;
   }
 
-  Rect? globalDisplaysRect() => _getDisplaysRect(_pi.displays);
-  Rect? displaysRect() => _getDisplaysRect(_pi.getCurDisplays());
-  Rect? _getDisplaysRect(List<Display> displays) {
+  Rect? globalDisplaysRect() => _getDisplaysRect(_pi.displays, true);
+  Rect? displaysRect() => _getDisplaysRect(_pi.getCurDisplays(), false);
+  Rect? _getDisplaysRect(List<Display> displays, bool useDisplayScale) {
     if (displays.isEmpty) {
       return null;
     }
+    int scale(int len, double s) {
+      if (useDisplayScale) {
+        return len.toDouble() ~/ s;
+      } else {
+        return len;
+      }
+    }
+
     double l = displays[0].x;
     double t = displays[0].y;
-    double r = displays[0].x + displays[0].width;
-    double b = displays[0].y + displays[0].height;
+    double r = displays[0].x + scale(displays[0].width, displays[0].scale);
+    double b = displays[0].y + scale(displays[0].height, displays[0].scale);
     for (var display in displays.sublist(1)) {
       l = min(l, display.x);
       t = min(t, display.y);
-      r = max(r, display.x + display.width);
-      b = max(b, display.y + display.height);
+      r = max(r, display.x + scale(display.width, display.scale));
+      b = max(b, display.y + scale(display.height, display.scale));
     }
     return Rect.fromLTRB(l, t, r, b);
   }
@@ -476,6 +484,7 @@ class FfiModel with ChangeNotifier {
         int.tryParse(evt['original_width']) ?? kInvalidResolutionValue;
     newDisplay.originalHeight =
         int.tryParse(evt['original_height']) ?? kInvalidResolutionValue;
+    newDisplay._scale = _pi.scaleOfDisplay(display);
     _pi.displays[display] = newDisplay;
 
     if (!_pi.isSupportMultiUiSession || _pi.currentDisplay == display) {
@@ -890,6 +899,8 @@ class FfiModel with ChangeNotifier {
     d.cursorEmbedded = evt['cursor_embedded'] == 1;
     d.originalWidth = evt['original_width'] ?? kInvalidResolutionValue;
     d.originalHeight = evt['original_height'] ?? kInvalidResolutionValue;
+    double v = (evt['scale']?.toDouble() ?? 100.0) / 100;
+    d._scale = v > 1.0 ? v : 1.0;
     return d;
   }
 
@@ -2384,6 +2395,8 @@ class Display {
   bool cursorEmbedded = false;
   int originalWidth = kInvalidResolutionValue;
   int originalHeight = kInvalidResolutionValue;
+  double _scale = 1.0;
+  double get scale => _scale > 1.0 ? _scale : 1.0;
 
   Display() {
     width = (isDesktop || isWebDesktop)
@@ -2503,6 +2516,13 @@ class PeerInfo with ChangeNotifier {
       }
     }
   }
+
+  double scaleOfDisplay(int display) {
+    if (display >= 0 && display < displays.length) {
+      return displays[display].scale;
+    }
+    return 1.0;
+  }
 }
 
 const canvasKey = 'canvas';

libs/hbb_common/protos/message.proto

@@ -49,6 +49,7 @@ message DisplayInfo {
   bool online = 6;
   bool cursor_embedded = 7;
   Resolution original_resolution = 8;
+  double scale = 9;
 }
 
 message PortForward {

libs/scrap/src/common/quartz.rs

@@ -128,6 +128,10 @@ impl Display {
         self.0.height()
     }
 
+    pub fn scale(&self) -> f64 {
+        self.0.scale()
+    }
+
     pub fn name(&self) -> String {
         self.0.id().to_string()
     }

libs/scrap/src/quartz/display.rs

@@ -38,7 +38,7 @@ impl Display {
         let w = unsafe { CGDisplayPixelsWide(self.0) };
         let s = self.scale();
         if s > 1.0 {
-           ((w as f64) * s).round() as usize
+            ((w as f64) * s).round() as usize
         } else {
             w
         }
@@ -48,7 +48,7 @@ impl Display {
         let h = unsafe { CGDisplayPixelsHigh(self.0) };
         let s = self.scale();
         if s > 1.0 {
-           ((h as f64) * s).round() as usize
+            ((h as f64) * s).round() as usize
         } else {
             h
         }
@@ -71,7 +71,13 @@ impl Display {
     }
 
     pub fn scale(self) -> f64 {
-        // unsafe { BackingScaleFactor() as _ }
+        let s = unsafe { BackingScaleFactor() as _ };
+        if s > 1. {
+            let enable_retina = super::ENABLE_RETINA.lock().unwrap().clone();
+            if enable_retina {
+                return s;
+            }
+        }
         1.
     }
 

libs/scrap/src/quartz/mod.rs

@@ -9,3 +9,9 @@ mod config;
 mod display;
 pub mod ffi;
 mod frame;
+
+use std::sync::{Arc, Mutex};
+
+lazy_static::lazy_static! {
+    pub static ref ENABLE_RETINA: Arc<Mutex<bool>> = Arc::new(Mutex::new(true));
+}

src/flutter.rs

@@ -512,6 +512,7 @@ impl FlutterHandler {
                 h.insert("original_width", original_resolution.width);
                 h.insert("original_height", original_resolution.height);
             }
+            h.insert("scale", (d.scale * 100.0f64) as i32);
             msg_vec.push(h);
         }
         serde_json::ser::to_string(&msg_vec).unwrap_or("".to_owned())

src/server.rs

@@ -283,6 +283,8 @@ impl Server {
                 s.on_subscribe(conn.clone());
             }
         }
+        #[cfg(target_os = "macos")]
+        self.update_enable_retina();
         self.connections.insert(conn.id(), conn);
         *CONN_COUNT.lock().unwrap() = self.connections.len();
     }
@@ -293,6 +295,8 @@ impl Server {
         }
         self.connections.remove(&conn.id());
         *CONN_COUNT.lock().unwrap() = self.connections.len();
+        #[cfg(target_os = "macos")]
+        self.update_enable_retina();
     }
 
     pub fn close_connections(&mut self) {
@@ -325,6 +329,8 @@ impl Server {
             } else {
                 s.on_unsubscribe(conn.id());
             }
+            #[cfg(target_os = "macos")]
+            self.update_enable_retina();
         }
     }
 
@@ -374,6 +380,17 @@ impl Server {
             }
         }
     }
+
+    #[cfg(target_os = "macos")]
+    fn update_enable_retina(&self) {
+        let mut video_service_count = 0;
+        for (name, service) in self.services.iter() {
+            if Self::is_video_service_name(&name) && service.ok() {
+                video_service_count += 1;
+            }
+        }
+        *scrap::quartz::ENABLE_RETINA.lock().unwrap() = video_service_count < 2;
+    }
 }
 
 impl Drop for Server {

src/server/connection.rs

@@ -239,6 +239,8 @@ pub struct Connection {
     supported_encoding_flag: (bool, Option<bool>),
     services_subed: bool,
     delayed_read_dir: Option<(String, bool)>,
+    #[cfg(target_os = "macos")]
+    retina: Retina,
 }
 
 impl ConnInner {
@@ -388,6 +390,8 @@ impl Connection {
             supported_encoding_flag: (false, None),
             services_subed: false,
             delayed_read_dir: None,
+            #[cfg(target_os = "macos")]
+            retina: Retina::default(),
         };
         let addr = hbb_common::try_into_v4(addr);
         if !conn.on_open(addr).await {
@@ -629,7 +633,8 @@ impl Connection {
                 },
                 Some((instant, value)) = rx.recv() => {
                     let latency = instant.elapsed().as_millis() as i64;
-                    let msg: &Message = &value;
+                    #[allow(unused_mut)]
+                    let mut msg = value;
 
                     if latency > 1000 {
                         match &msg.union {
@@ -651,11 +656,20 @@ impl Connection {
                                 _ => {},
                             }
                         }
-                        Some(message::Union::PeerInfo(..)) => {
+                        Some(message::Union::PeerInfo(_pi)) => {
                             conn.refresh_video_display(None);
+                            #[cfg(target_os = "macos")]
+                            conn.retina.set_displays(&_pi.displays);
+                        }
+                        #[cfg(target_os = "macos")]
+                        Some(message::Union::CursorPosition(pos)) => {
+                            if let Some(new_msg) = conn.retina.on_cursor_pos(&pos, conn.display_idx) {
+                                msg = Arc::new(new_msg);
+                            }
                         }
                         _ => {}
                     }
+                    let msg: &Message = &msg;
                     if let Err(err) = conn.stream.send(msg).await {
                         conn.on_close(&err.to_string(), false).await;
                         break;
@@ -1229,6 +1243,10 @@ impl Connection {
                 Ok(displays) => {
                     // For compatibility with old versions, we need to send the displays to the peer.
                     // But the displays may be updated later, before creating the video capturer.
+                    #[cfg(target_os = "macos")]
+                    {
+                        self.retina.set_displays(&displays);
+                    }
                     pi.displays = displays;
                     pi.current_display = self.display_idx as _;
                     res.set_peer_info(pi);
@@ -1811,7 +1829,8 @@ impl Connection {
             }
         } else if self.authorized {
             match msg.union {
-                Some(message::Union::MouseEvent(me)) => {
+                #[allow(unused_mut)]
+                Some(message::Union::MouseEvent(mut me)) => {
                     #[cfg(any(target_os = "android", target_os = "ios"))]
                     if let Err(e) = call_main_service_pointer_input("mouse", me.mask, me.x, me.y) {
                         log::debug!("call_main_service_pointer_input fail:{}", e);
@@ -1823,6 +1842,8 @@ impl Connection {
                         } else {
                             MOUSE_MOVE_TIME.store(get_time(), Ordering::SeqCst);
                         }
+                        #[cfg(target_os = "macos")]
+                        self.retina.on_mouse_event(&mut me, self.display_idx);
                         self.input_mouse(me, self.inner.id());
                     }
                     self.update_auto_disconnect_timer();
@@ -3488,6 +3509,54 @@ extern "C" fn connection_shutdown_hook() {
     }
 }
 
+#[cfg(target_os = "macos")]
+#[derive(Debug, Default)]
+struct Retina {
+    displays: Vec<DisplayInfo>,
+}
+
+#[cfg(target_os = "macos")]
+impl Retina {
+    #[inline]
+    fn set_displays(&mut self, displays: &Vec<DisplayInfo>) {
+        self.displays = displays.clone();
+    }
+
+    #[inline]
+    fn on_mouse_event(&mut self, e: &mut MouseEvent, current: usize) {
+        let Some(d) = self.displays.get(current) else {
+            return;
+        };
+        let s = d.scale;
+        if s > 1.0 && e.x >= d.x && e.y >= d.y && e.x < d.x + d.width && e.y < d.y + d.height {
+            e.x = d.x + ((e.x - d.x) as f64 / s) as i32;
+            e.y = d.y + ((e.y - d.y) as f64 / s) as i32;
+        }
+    }
+
+    #[inline]
+    fn on_cursor_pos(&mut self, pos: &CursorPosition, current: usize) -> Option<Message> {
+        let Some(d) = self.displays.get(current) else {
+            return None;
+        };
+        let s = d.scale;
+        if s > 1.0
+            && pos.x >= d.x
+            && pos.y >= d.y
+            && (pos.x - d.x) as f64 * s < d.width as f64
+            && (pos.y - d.y) as f64 * s < d.height as f64
+        {
+            let mut pos = pos.clone();
+            pos.x = d.x + ((pos.x - d.x) as f64 * s) as i32;
+            pos.y = d.y + ((pos.y - d.y) as f64 * s) as i32;
+            let mut msg = Message::new();
+            msg.set_cursor_position(pos);
+            return Some(msg);
+        }
+        None
+    }
+}
+
 mod raii {
     // CONN_COUNT: remote connection count in fact
     // ALIVE_CONNS: all connections, including unauthorized connections
@@ -3578,3 +3647,40 @@ mod raii {
         }
     }
 }
+
+mod test {
+    #[allow(unused)]
+    use super::*;
+
+    #[cfg(target_os = "macos")]
+    #[test]
+    fn retina() {
+        let mut retina = Retina {
+            displays: vec![DisplayInfo {
+                x: 10,
+                y: 10,
+                width: 1000,
+                height: 1000,
+                scale: 2.0,
+                ..Default::default()
+            }],
+        };
+        let mut mouse: MouseEvent = MouseEvent {
+            x: 510,
+            y: 510,
+            ..Default::default()
+        };
+        retina.on_mouse_event(&mut mouse, 0);
+        assert_eq!(mouse.x, 260);
+        assert_eq!(mouse.y, 260);
+        let pos = CursorPosition {
+            x: 260,
+            y: 260,
+            ..Default::default()
+        };
+        let msg = retina.on_cursor_pos(&pos, 0).unwrap();
+        let pos = msg.cursor_position();
+        assert_eq!(pos.x, 510);
+        assert_eq!(pos.y, 510);
+    }
+}

src/server/display_service.rs

@@ -273,7 +273,18 @@ pub(super) fn check_update_displays(all: &Vec<Display>) {
         .iter()
         .map(|d| {
             let display_name = d.name();
-            let original_resolution = get_original_resolution(&display_name, d.width(), d.height());
+            #[allow(unused_assignments)]
+            #[allow(unused_mut)]
+            let mut scale = 1.0;
+            #[cfg(target_os = "macos")]
+            {
+                scale = d.scale();
+            }
+            let original_resolution = get_original_resolution(
+                &display_name,
+                ((d.width() as f64) / scale).round() as usize,
+                (d.height() as f64 / scale).round() as usize,
+            );
             DisplayInfo {
                 x: d.origin().0 as _,
                 y: d.origin().1 as _,
@@ -283,6 +294,7 @@ pub(super) fn check_update_displays(all: &Vec<Display>) {
                 online: d.is_online(),
                 cursor_embedded: false,
                 original_resolution,
+                scale,
                 ..Default::default()
             }
         })

src/server/service.rs

@@ -14,6 +14,7 @@ pub trait Service: Send + Sync {
     fn join(&self);
     fn get_option(&self, opt: &str) -> Option<String>;
     fn set_option(&self, opt: &str, val: &str) -> Option<String>;
+    fn ok(&self) -> bool;
 }
 
 pub trait Subscriber: Default + Send + Sync + 'static {
@@ -142,6 +143,12 @@ impl<T: Subscriber + From<ConnInner>> Service for ServiceTmpl<T> {
             .options
             .insert(opt.to_string(), val.to_string())
     }
+
+    #[inline]
+    fn ok(&self) -> bool {
+        let lock = self.0.read().unwrap();
+        lock.active && lock.has_subscribes()
+    }
 }
 
 impl<T: Subscriber + From<ConnInner>> Clone for ServiceTmpl<T> {
@@ -180,12 +187,6 @@ impl<T: Subscriber + From<ConnInner>> ServiceTmpl<T> {
         self.0.read().unwrap().has_subscribes()
     }
 
-    #[inline]
-    pub fn ok(&self) -> bool {
-        let lock = self.0.read().unwrap();
-        lock.active && lock.has_subscribes()
-    }
-
     pub fn snapshot<F>(&self, callback: F) -> ResultType<()>
     where
         F: FnMut(ServiceSwap<T>) -> ResultType<()>,