Fix wayland input after Lock and Wake

Signed-off-by: fufesou <shuanglongchen@yeah.net>

libs/enigo/src/lib.rs

@@ -118,6 +118,13 @@ pub enum MouseButton {
 /// Representing an interface and a set of mouse functions every
 /// operating system implementation _should_ implement.
 pub trait MouseControllable {
+    // https://stackoverflow.com/a/33687996
+    /// Offer the ability to confer concrete type.
+    fn as_any(&self) -> &dyn std::any::Any;
+
+    /// Offer the ability to confer concrete type.
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any;
+
     /// Lets the mouse cursor move to the specified x and y coordinates.
     ///
     /// The topleft corner of your monitor screen is x=0 y=0. Move
@@ -425,6 +432,13 @@ pub enum Key {
 /// Representing an interface and a set of keyboard functions every
 /// operating system implementation _should_ implement.
 pub trait KeyboardControllable {
+    // https://stackoverflow.com/a/33687996
+    /// Offer the ability to confer concrete type.
+    fn as_any(&self) -> &dyn std::any::Any;
+
+    /// Offer the ability to confer concrete type.
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any;
+
     /// Types the string parsed with DSL.
     ///
     /// Typing {+SHIFT}hello{-SHIFT} becomes HELLO.

libs/enigo/src/linux/nix_impl.rs

@@ -3,14 +3,17 @@ use crate::{Key, KeyboardControllable, MouseButton, MouseControllable};
 use std::io::Read;
 use tfc::{traits::*, Context as TFC_Context, Key as TFC_Key};
 
+pub type CustomKeyboard = Box<dyn KeyboardControllable + Send>;
+pub type CustomMouce = Box<dyn MouseControllable + Send>;
+
 /// The main struct for handling the event emitting
 // #[derive(Default)]
 pub struct Enigo {
     xdo: EnigoXdo,
     is_x11: bool,
     tfc: Option<TFC_Context>,
-    uinput_keyboard: Option<Box<dyn KeyboardControllable + Send>>,
-    uinput_mouse: Option<Box<dyn MouseControllable + Send>>,
+    custom_keyboard: Option<CustomKeyboard>,
+    cutsom_mouse: Option<CustomMouce>,
 }
 
 impl Enigo {
@@ -22,16 +25,21 @@ impl Enigo {
     pub fn set_delay(&mut self, delay: u64) {
         self.xdo.set_delay(delay)
     }
-    /// Set uinput keyboard.
-    pub fn set_uinput_keyboard(
-        &mut self,
-        uinput_keyboard: Option<Box<dyn KeyboardControllable + Send>>,
-    ) {
-        self.uinput_keyboard = uinput_keyboard
+    /// Set custom keyboard.
+    pub fn set_custom_keyboard(&mut self, custom_keyboard: CustomKeyboard) {
+        self.custom_keyboard = Some(custom_keyboard)
     }
-    /// Set uinput mouse.
-    pub fn set_uinput_mouse(&mut self, uinput_mouse: Option<Box<dyn MouseControllable + Send>>) {
-        self.uinput_mouse = uinput_mouse
+    /// Set custom mouse.
+    pub fn set_custom_mouse(&mut self, custom_mouse: CustomMouce) {
+        self.cutsom_mouse = Some(custom_mouse)
+    }
+    /// Get custom keyboard.
+    pub fn get_custom_keyboard(&mut self) -> &mut Option<CustomKeyboard> {
+        &mut self.custom_keyboard
+    }
+    /// Get custom mouse.
+    pub fn get_custom_mouse(&mut self) -> &mut Option<CustomMouce> {
+        &mut self.cutsom_mouse
     }
 
     fn tfc_key_down_or_up(&mut self, key: Key, down: bool, up: bool) -> bool {
@@ -84,19 +92,27 @@ impl Default for Enigo {
             } else {
                 None
             },
-            uinput_keyboard: None,
-            uinput_mouse: None,
+            custom_keyboard: None,
+            cutsom_mouse: None,
             xdo: EnigoXdo::default(),
         }
     }
 }
 
 impl MouseControllable for Enigo {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+
     fn mouse_move_to(&mut self, x: i32, y: i32) {
         if self.is_x11 {
             self.xdo.mouse_move_to(x, y);
         } else {
-            if let Some(mouse) = &mut self.uinput_mouse {
+            if let Some(mouse) = &mut self.cutsom_mouse {
                 mouse.mouse_move_to(x, y)
             }
         }
@@ -105,7 +121,7 @@ impl MouseControllable for Enigo {
         if self.is_x11 {
             self.xdo.mouse_move_relative(x, y);
         } else {
-            if let Some(mouse) = &mut self.uinput_mouse {
+            if let Some(mouse) = &mut self.cutsom_mouse {
                 mouse.mouse_move_relative(x, y)
             }
         }
@@ -114,7 +130,7 @@ impl MouseControllable for Enigo {
         if self.is_x11 {
             self.xdo.mouse_down(button)
         } else {
-            if let Some(mouse) = &mut self.uinput_mouse {
+            if let Some(mouse) = &mut self.cutsom_mouse {
                 mouse.mouse_down(button)
             } else {
                 Ok(())
@@ -125,7 +141,7 @@ impl MouseControllable for Enigo {
         if self.is_x11 {
             self.xdo.mouse_up(button)
         } else {
-            if let Some(mouse) = &mut self.uinput_mouse {
+            if let Some(mouse) = &mut self.cutsom_mouse {
                 mouse.mouse_up(button)
             }
         }
@@ -134,7 +150,7 @@ impl MouseControllable for Enigo {
         if self.is_x11 {
             self.xdo.mouse_click(button)
         } else {
-            if let Some(mouse) = &mut self.uinput_mouse {
+            if let Some(mouse) = &mut self.cutsom_mouse {
                 mouse.mouse_click(button)
             }
         }
@@ -143,7 +159,7 @@ impl MouseControllable for Enigo {
         if self.is_x11 {
             self.xdo.mouse_scroll_x(length)
         } else {
-            if let Some(mouse) = &mut self.uinput_mouse {
+            if let Some(mouse) = &mut self.cutsom_mouse {
                 mouse.mouse_scroll_x(length)
             }
         }
@@ -152,7 +168,7 @@ impl MouseControllable for Enigo {
         if self.is_x11 {
             self.xdo.mouse_scroll_y(length)
         } else {
-            if let Some(mouse) = &mut self.uinput_mouse {
+            if let Some(mouse) = &mut self.cutsom_mouse {
                 mouse.mouse_scroll_y(length)
             }
         }
@@ -180,11 +196,19 @@ fn get_led_state(key: Key) -> bool {
 }
 
 impl KeyboardControllable for Enigo {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+
     fn get_key_state(&mut self, key: Key) -> bool {
         if self.is_x11 {
             self.xdo.get_key_state(key)
         } else {
-            if let Some(keyboard) = &mut self.uinput_keyboard {
+            if let Some(keyboard) = &mut self.custom_keyboard {
                 keyboard.get_key_state(key)
             } else {
                 get_led_state(key)
@@ -196,7 +220,7 @@ impl KeyboardControllable for Enigo {
         if self.is_x11 {
             self.xdo.key_sequence(sequence)
         } else {
-            if let Some(keyboard) = &mut self.uinput_keyboard {
+            if let Some(keyboard) = &mut self.custom_keyboard {
                 keyboard.key_sequence(sequence)
             }
         }
@@ -211,7 +235,7 @@ impl KeyboardControllable for Enigo {
                 Ok(())
             }
         } else {
-            if let Some(keyboard) = &mut self.uinput_keyboard {
+            if let Some(keyboard) = &mut self.custom_keyboard {
                 keyboard.key_down(key)
             } else {
                 Ok(())
@@ -225,7 +249,7 @@ impl KeyboardControllable for Enigo {
                 self.xdo.key_up(key)
             }
         } else {
-            if let Some(keyboard) = &mut self.uinput_keyboard {
+            if let Some(keyboard) = &mut self.custom_keyboard {
                 keyboard.key_up(key)
             }
         }

libs/enigo/src/linux/xdo.rs

@@ -102,6 +102,14 @@ impl Drop for EnigoXdo {
     }
 }
 impl MouseControllable for EnigoXdo {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+
     fn mouse_move_to(&mut self, x: i32, y: i32) {
         if self.xdo.is_null() {
             return;
@@ -277,6 +285,14 @@ fn keysequence<'a>(key: Key) -> Cow<'a, str> {
     })
 }
 impl KeyboardControllable for EnigoXdo {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+
     fn get_key_state(&mut self, key: Key) -> bool {
         if self.xdo.is_null() {
             return false;

src/ipc.rs

@@ -117,6 +117,7 @@ pub enum DataMouse {
     Click(enigo::MouseButton),
     ScrollX(i32),
     ScrollY(i32),
+    Refresh,
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone)]

src/server.rs

@@ -1,6 +1,8 @@
 use crate::ipc::Data;
 use bytes::Bytes;
 pub use connection::*;
+#[cfg(not(any(target_os = "android", target_os = "ios")))]
+use hbb_common::config::Config2;
 use hbb_common::{
     allow_err,
     anyhow::{anyhow, Context},
@@ -15,8 +17,6 @@ use hbb_common::{
     timeout, tokio, ResultType, Stream,
 };
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
-use hbb_common::config::Config2;
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
 use service::ServiceTmpl;
 use service::{GenericService, Service, Subscriber};
 use std::{
@@ -377,6 +377,7 @@ pub async fn start_server(is_server: bool) {
         #[cfg(windows)]
         crate::platform::windows::bootstrap();
         input_service::fix_key_down_timeout_loop();
+        allow_err!(video_service::check_init().await);
         #[cfg(target_os = "macos")]
         tokio::spawn(async { sync_and_watch_config_dir().await });
         crate::RendezvousMediator::start_all().await;

src/server/input_service.rs

@@ -219,19 +219,43 @@ lazy_static::lazy_static! {
     static ref IS_SERVER: bool =  std::env::args().nth(1) == Some("--server".to_owned());
 }
 
+// First call set_uinput() will create keyboard and mouse clients.
+// The clients are ipc connections that must live shorter than tokio runtime.
+// Thus this funtion must not be called in a temporary runtime.
 #[cfg(target_os = "linux")]
 pub async fn set_uinput() -> ResultType<()> {
     // Keyboard and mouse both open /dev/uinput
     // TODO: Make sure there's no race
+
+    if ENIGO.lock().unwrap().get_custom_keyboard().is_none() {
         let keyboard = super::uinput::client::UInputKeyboard::new().await?;
         log::info!("UInput keyboard created");
+        ENIGO
+            .lock()
+            .unwrap()
+            .set_custom_keyboard(Box::new(keyboard));
+    }
+
+    let mouse_created = ENIGO.lock().unwrap().get_custom_mouse().is_some();
+    if mouse_created {
+        std::thread::spawn(|| {
+            if let Some(mouse) = ENIGO.lock().unwrap().get_custom_mouse() {
+                if let Some(mouse) = mouse
+                    .as_mut_any()
+                    .downcast_mut::<super::uinput::client::UInputMouse>()
+                {
+                    allow_err!(mouse.send_refresh());
+                } else {
+                    log::error!("failed downcast uinput mouse");
+                }
+            }
+        });
+    } else {
         let mouse = super::uinput::client::UInputMouse::new().await?;
         log::info!("UInput mouse created");
+        ENIGO.lock().unwrap().set_custom_mouse(Box::new(mouse));
+    }
 
-    let xxx = ENIGO.lock();
-    let mut en = xxx.unwrap();
-    en.set_uinput_keyboard(Some(Box::new(keyboard)));
-    en.set_uinput_mouse(Some(Box::new(mouse)));
     Ok(())
 }
 

src/server/uinput.rs

@@ -63,6 +63,14 @@ pub mod client {
     }
 
     impl KeyboardControllable for UInputKeyboard {
+        fn as_any(&self) -> &dyn std::any::Any {
+            self
+        }
+
+        fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+            self
+        }
+
         fn get_key_state(&mut self, key: Key) -> bool {
             match self.send_get_key_state(Data::Keyboard(DataKeyboard::GetKeyState(key))) {
                 Ok(state) => state,
@@ -105,9 +113,21 @@ pub mod client {
         async fn send(&mut self, data: Data) -> ResultType<()> {
             self.conn.send(&data).await
         }
+
+        pub fn send_refresh(&mut self) -> ResultType<()> {
+            self.send(Data::Mouse(DataMouse::Refresh))
+        }
     }
 
     impl MouseControllable for UInputMouse {
+        fn as_any(&self) -> &dyn std::any::Any {
+            self
+        }
+
+        fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+            self
+        }
+
         fn mouse_move_to(&mut self, x: i32, y: i32) {
             allow_err!(self.send(Data::Mouse(DataMouse::MoveTo(x, y))));
         }
@@ -492,6 +512,9 @@ pub mod service {
                     allow_err!(mouse.scroll_wheel(&scroll))
                 }
             }
+            DataMouse::Refresh => {
+                // unreachable!()
+            }
         }
     }
 
@@ -562,8 +585,18 @@ pub mod service {
                             Ok(Some(data)) => {
                                 match data {
                                     Data::Mouse(data) => {
+                                        if let DataMouse::Refresh = data {
+                                            mouse = match mouce::Mouse::new_uinput(rng_x, rng_y) {
+                                                Ok(mouse) => mouse,
+                                                Err(e) => {
+                                                    log::error!("Failed to create mouse, {}", e);
+                                                    return;
+                                                }
+                                            }
+                                        } else {
                                             handle_mouse(&mut mouse, &data);
                                         }
+                                    }
                                     _ => {
                                     }
                                 }

src/server/video_service.rs

@@ -759,6 +759,16 @@ fn get_display_num() -> usize {
     }
 }
 
+pub async fn check_init() -> ResultType<()> {
+    #[cfg(target_os = "linux")]
+    {
+        if !scrap::is_x11() {
+            return super::wayland::check_init().await;
+        }
+    }
+    Ok(())
+}
+
 pub(super) fn get_displays_2(all: &Vec<Display>) -> (usize, Vec<DisplayInfo>) {
     let mut displays = Vec::new();
     let mut primary = 0;

src/server/wayland.rs

@@ -115,7 +115,7 @@ pub(super) fn is_inited() -> Option<Message> {
     }
 }
 
-async fn check_init() -> ResultType<()> {
+pub(super) async fn check_init() -> ResultType<()> {
     if !scrap::is_x11() {
         let mut minx = 0;
         let mut maxx = 0;